From 3d7fd070c7ae423358cc82133e0beb05d5ba5a07 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 3 Mar 2021 11:45:33 +0800 Subject: [PATCH] Refeactor replication policy APIs Refeactor replication policy APIs Signed-off-by: Wenkai Yin --- api/v2.0/legacy_swagger.yaml | 233 ---------- api/v2.0/swagger.yaml | 221 +++++++++ .../handler/webhook/artifact/replication.go | 2 +- .../webhook/artifact/replication_test.go | 58 +-- .../{controller.go => execution.go} | 34 +- .../{controller_test.go => execution_test.go} | 29 +- src/controller/replication/flow/controller.go | 5 +- src/controller/replication/flow/copy.go | 5 +- src/controller/replication/flow/copy_test.go | 3 +- src/controller/replication/flow/deletion.go | 5 +- .../replication/flow/deletion_test.go | 3 +- src/controller/replication/flow/stage.go | 9 +- src/controller/replication/flow/stage_test.go | 9 +- .../replication/mock_flow_controller_test.go | 6 +- src/controller/replication/policy.go | 188 ++++++++ src/controller/replication/policy_test.go | 133 ++++++ src/core/api/harborapi_test.go | 3 - src/core/api/registry.go | 27 +- src/core/api/replication_policy.go | 289 ------------ src/core/api/replication_policy_test.go | 438 ------------------ src/pkg/reg/dao/dao.go | 128 +++++ src/pkg/reg/dao/dao_test.go | 162 +++++++ src/pkg/reg/manager.go | 104 +++++ src/pkg/reg/manager_test.go | 94 ++++ src/pkg/replication/dao/dao.go | 127 +++++ src/pkg/replication/dao/dao_test.go | 157 +++++++ src/pkg/replication/dao/model.go | 48 ++ src/pkg/replication/manager.go | 106 +++++ src/pkg/replication/manager_test.go | 93 ++++ src/pkg/replication/model.go | 350 ++++++++++++++ src/pkg/replication/model_test.go | 252 ++++++++++ src/pkg/task/execution.go | 31 ++ src/replication/dao/models/base.go | 3 +- src/replication/dao/models/policy.go | 26 -- src/replication/dao/policy.go | 115 ----- src/replication/dao/policy_test.go | 284 ------------ src/replication/event/handler.go | 70 +-- src/replication/event/handler_test.go | 305 ------------ src/replication/model/policy.go | 122 ----- src/replication/model/policy_test.go | 225 --------- src/replication/policy/controller.go | 35 -- .../policy/controller/controller.go | 138 ------ .../policy/controller/controller_test.go | 342 -------------- src/replication/policy/manager/manager.go | 332 ------------- .../policy/manager/manager_test.go | 266 ----------- src/replication/registry/manager.go | 18 +- src/replication/replication.go | 41 +- src/replication/replication_test.go | 1 - src/server/v2.0/handler/base.go | 36 +- src/server/v2.0/handler/replication.go | 240 +++++++++- src/server/v2.0/route/legacy.go | 2 - .../controller/replication/controller.go | 167 ++++++- src/testing/pkg/pkg.go | 4 + src/testing/pkg/reg/dao/dao.go | 141 ++++++ src/testing/pkg/reg/manager.go | 140 ++++++ src/testing/pkg/replication/dao/dao.go | 140 ++++++ src/testing/pkg/replication/manager.go | 140 ++++++ src/testing/pkg/task/execution_manager.go | 14 + tests/apitests/python/library/replication.py | 54 ++- .../apitests/python/library/replication_v2.py | 33 -- .../python/test_replication_from_dockerhub.py | 9 +- tests/robot-cases/Group0-BAT/API_DB.robot | 2 +- 62 files changed, 3322 insertions(+), 3475 deletions(-) rename src/controller/replication/{controller.go => execution.go} (85%) rename src/controller/replication/{controller_test.go => execution_test.go} (87%) create mode 100644 src/controller/replication/policy.go create mode 100644 src/controller/replication/policy_test.go delete mode 100644 src/core/api/replication_policy.go delete mode 100644 src/core/api/replication_policy_test.go create mode 100644 src/pkg/reg/dao/dao.go create mode 100644 src/pkg/reg/dao/dao_test.go create mode 100644 src/pkg/reg/manager.go create mode 100644 src/pkg/reg/manager_test.go create mode 100644 src/pkg/replication/dao/dao.go create mode 100644 src/pkg/replication/dao/dao_test.go create mode 100644 src/pkg/replication/dao/model.go create mode 100644 src/pkg/replication/manager.go create mode 100644 src/pkg/replication/manager_test.go create mode 100644 src/pkg/replication/model.go create mode 100644 src/pkg/replication/model_test.go delete mode 100644 src/replication/dao/models/policy.go delete mode 100644 src/replication/dao/policy.go delete mode 100644 src/replication/dao/policy_test.go delete mode 100644 src/replication/event/handler_test.go delete mode 100644 src/replication/model/policy_test.go delete mode 100644 src/replication/policy/controller.go delete mode 100644 src/replication/policy/controller/controller.go delete mode 100644 src/replication/policy/controller/controller_test.go delete mode 100644 src/replication/policy/manager/manager.go delete mode 100644 src/replication/policy/manager/manager_test.go create mode 100644 src/testing/pkg/reg/dao/dao.go create mode 100644 src/testing/pkg/reg/manager.go create mode 100644 src/testing/pkg/replication/dao/dao.go create mode 100644 src/testing/pkg/replication/manager.go delete mode 100644 tests/apitests/python/library/replication_v2.py diff --git a/api/v2.0/legacy_swagger.yaml b/api/v2.0/legacy_swagger.yaml index 19edea0c1..830936d30 100644 --- a/api/v2.0/legacy_swagger.yaml +++ b/api/v2.0/legacy_swagger.yaml @@ -738,172 +738,6 @@ paths: description: The auth mode of the system is not "oidc_auth", or the user is not onboarded via OIDC AuthN. '500': description: Unexpected internal errors. - /replication/policies: - get: - summary: List replication policies - description: | - This endpoint let user list replication policies - parameters: - - name: name - in: query - type: string - required: false - description: The replication policy name. - - name: page - in: query - type: integer - format: int32 - required: false - description: The page number. - - name: page_size - in: query - type: integer - format: int32 - required: false - description: The size of per page. - tags: - - Products - responses: - '200': - description: Get policy successfully. - schema: - type: array - items: - $ref: '#/definitions/ReplicationPolicy' - headers: - X-Total-Count: - description: The total count of available items - type: integer - Link: - description: Link to previous page and next page - type: string - '400': - $ref: '#/responses/BadRequest' - '401': - $ref: '#/responses/Unauthorized' - '403': - $ref: '#/responses/Forbidden' - '500': - $ref: '#/responses/InternalServerError' - post: - summary: Create a replication policy - description: | - This endpoint let user create a replication policy - parameters: - - name: policy - in: body - description: The policy model. - required: true - schema: - $ref: '#/definitions/ReplicationPolicy' - tags: - - Products - responses: - '201': - $ref: '#/responses/Created' - '400': - $ref: '#/responses/BadRequest' - '401': - $ref: '#/responses/Unauthorized' - '403': - $ref: '#/responses/Forbidden' - '409': - $ref: '#/responses/Conflict' - '415': - $ref: '#/responses/UnsupportedMediaType' - '500': - $ref: '#/responses/InternalServerError' - '/replication/policies/{id}': - get: - summary: Get replication policy. - description: | - This endpoint let user get replication policy by specific ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: policy ID - tags: - - Products - responses: - '200': - description: Get the replication policy successfully. - schema: - $ref: '#/definitions/ReplicationPolicy' - '400': - $ref: '#/responses/BadRequest' - '401': - $ref: '#/responses/Unauthorized' - '403': - $ref: '#/responses/Forbidden' - '404': - $ref: '#/responses/NotFound' - '500': - $ref: '#/responses/InternalServerError' - put: - summary: Update the replication policy - description: | - This endpoint let user update policy. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: policy ID - - name: policy - in: body - description: The replication policy model. - required: true - schema: - $ref: '#/definitions/ReplicationPolicy' - tags: - - Products - responses: - '200': - $ref: '#/responses/OK' - '400': - $ref: '#/responses/BadRequest' - '401': - $ref: '#/responses/Unauthorized' - '403': - $ref: '#/responses/Forbidden' - '404': - $ref: '#/responses/NotFound' - '409': - $ref: '#/responses/Conflict' - '500': - $ref: '#/responses/InternalServerError' - delete: - summary: Delete the replication policy specified by ID. - description: | - Delete the replication policy specified by ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: Replication policy ID - tags: - - Products - responses: - '200': - $ref: '#/responses/OK' - '400': - $ref: '#/responses/BadRequest' - '401': - $ref: '#/responses/Unauthorized' - '403': - $ref: '#/responses/Forbidden' - '404': - $ref: '#/responses/NotFound' - '412': - $ref: '#/responses/PreconditionFailed' - '500': - $ref: '#/responses/InternalServerError' /labels: get: summary: List labels according to the query strings. @@ -2205,73 +2039,6 @@ definitions: type: integer format: int32 description: 'The count of the total repositories, only be seen when the user is admin.' - ReplicationPolicy: - type: object - properties: - id: - type: integer - format: int64 - description: The policy ID. - name: - type: string - description: The policy name. - description: - type: string - description: The description of the policy. - src_registry: - description: The source registry. - $ref: '#/definitions/Registry' - dest_registry: - description: The destination registry. - $ref: '#/definitions/Registry' - dest_namespace: - type: string - description: The destination namespace. - trigger: - $ref: '#/definitions/ReplicationTrigger' - filters: - type: array - description: The replication policy filter array. - items: - $ref: '#/definitions/ReplicationFilter' - deletion: - type: boolean - description: Whether to replicate the deletion operation. - override: - type: boolean - description: Whether to override the resources on the destination registry. - enabled: - type: boolean - description: Whether the policy is enabled or not. - creation_time: - type: string - description: The create time of the policy. - update_time: - type: string - description: The update time of the policy. - ReplicationTrigger: - type: object - properties: - type: - type: string - description: 'The replication policy trigger type. The valid values are manual, event_based and scheduled.' - trigger_settings: - $ref: '#/definitions/TriggerSettings' - TriggerSettings: - type: object - properties: - cron: - type: string - description: The cron string for scheduled trigger - ReplicationFilter: - type: object - properties: - type: - type: string - description: 'The replication policy filter type.' - value: - type: string - description: 'The value of replication policy filter.' RegistryCredential: type: object properties: diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index b8449661b..911dc03d2 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -2255,6 +2255,152 @@ paths: $ref: '#/responses/404' '500': $ref: '#/responses/500' + /replication/policies: + get: + summary: List replication policies + description: List replication policies + tags: + - replication + operationId: listReplicationPolicies + parameters: + - $ref: '#/parameters/query' + - $ref: '#/parameters/sort' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - name: name + in: query + type: string + required: false + description: Deprecated, use "query" instead. The policy name. + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of the resources + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/ReplicationPolicy' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '500': + $ref: '#/responses/500' + post: + summary: Create a replication policy + description: Create a replication policy + tags: + - replication + operationId: createReplicationPolicy + parameters: + - name: policy + in: body + description: The replication policy + required: true + schema: + $ref: '#/definitions/ReplicationPolicy' + responses: + '201': + $ref: '#/responses/201' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '409': + $ref: '#/responses/409' + '500': + $ref: '#/responses/500' + /replication/policies/{id}: + get: + summary: Get the specific replication policy + description: Get the specific replication policy + tags: + - replication + operationId: getReplicationPolicy + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Policy ID + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ReplicationPolicy' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '500': + $ref: '#/responses/500' + delete: + summary: Delete the specific replication policy + description: Delete the specific replication policy + tags: + - replication + operationId: deleteReplicationPolicy + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Replication policy ID + responses: + '200': + $ref: '#/responses/200' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '412': + $ref: '#/responses/412' + '500': + $ref: '#/responses/500' + put: + summary: Update the replication policy + description: Update the replication policy + tags: + - replication + operationId: updateReplicationPolicy + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: The policy ID + - name: policy + in: body + description: The replication policy + required: true + schema: + $ref: '#/definitions/ReplicationPolicy' + responses: + '200': + $ref: '#/responses/200' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '409': + $ref: '#/responses/409' + '500': + $ref: '#/responses/500' /replication/executions: get: summary: List replication executions @@ -4407,6 +4553,78 @@ definitions: cve_id: type: string description: The ID of the CVE, such as "CVE-2019-10164" + ReplicationPolicy: + type: object + properties: + id: + type: integer + format: int64 + description: The policy ID. + name: + type: string + description: The policy name. + description: + type: string + description: The description of the policy. + src_registry: + description: The source registry. + $ref: '#/definitions/Registry' + dest_registry: + description: The destination registry. + $ref: '#/definitions/Registry' + dest_namespace: + type: string + description: The destination namespace. + trigger: + $ref: '#/definitions/ReplicationTrigger' + filters: + type: array + description: The replication policy filter array. + items: + $ref: '#/definitions/ReplicationFilter' + replicate_deletion: + type: boolean + description: Whether to replicate the deletion operation. + deletion: + type: boolean + description: Deprecated, use "replicate_deletion" instead. Whether to replicate the deletion operation. + override: + type: boolean + description: Whether to override the resources on the destination registry. + enabled: + type: boolean + description: Whether the policy is enabled or not. + creation_time: + type: string + format: date-time + description: The create time of the policy. + update_time: + type: string + format: date-time + description: The update time of the policy. + ReplicationTrigger: + type: object + properties: + type: + type: string + description: 'The replication policy trigger type. The valid values are manual, event_based and scheduled.' + trigger_settings: + $ref: '#/definitions/ReplicationTriggerSettings' + ReplicationTriggerSettings: + type: object + properties: + cron: + type: string + description: The cron string for scheduled trigger + ReplicationFilter: + type: object + properties: + type: + type: string + description: 'The replication policy filter type.' + value: + type: object + description: 'The value of replication policy filter.' RegistryCredential: type: object properties: @@ -4426,6 +4644,7 @@ definitions: type: integer format: int64 description: The registry ID. + x-omitempty: false url: type: string description: The registry URL string. @@ -4448,9 +4667,11 @@ definitions: description: Health status of the registry. creation_time: type: string + format: date-time description: The create time of the policy. update_time: type: string + format: date-time description: The update time of the policy. ResourceList: type: object diff --git a/src/controller/event/handler/webhook/artifact/replication.go b/src/controller/event/handler/webhook/artifact/replication.go index 5ec595fa0..1405c47ff 100644 --- a/src/controller/event/handler/webhook/artifact/replication.go +++ b/src/controller/event/handler/webhook/artifact/replication.go @@ -85,7 +85,7 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, return nil, nil, err } - rpPolicy, err := rep.PolicyCtl.Get(execution.PolicyID) + rpPolicy, err := replication.Ctl.GetPolicy(ctx, execution.PolicyID) if err != nil { log.Errorf("failed to get replication policy %d: error: %v", execution.PolicyID, err) return nil, nil, err diff --git a/src/controller/event/handler/webhook/artifact/replication_test.go b/src/controller/event/handler/webhook/artifact/replication_test.go index 6f882da61..85cad0399 100644 --- a/src/controller/event/handler/webhook/artifact/replication_test.go +++ b/src/controller/event/handler/webhook/artifact/replication_test.go @@ -22,10 +22,11 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/project" - rep "github.com/goharbor/harbor/src/controller/replication" + repctl "github.com/goharbor/harbor/src/controller/replication" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/notification" + reppkg "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/model" projecttesting "github.com/goharbor/harbor/src/testing/controller/project" @@ -38,9 +39,6 @@ import ( type fakedNotificationPolicyMgr struct { } -type fakedReplicationPolicyMgr struct { -} - type fakedReplicationRegistryMgr struct { } @@ -87,44 +85,6 @@ func (f *fakedNotificationPolicyMgr) GetRelatedPolices(int64, string) ([]*models }, nil } -// Create new policy -func (f *fakedReplicationPolicyMgr) Create(*model.Policy) (int64, error) { - return 0, nil -} - -// List the policies, returns the total count, policy list and error -func (f *fakedReplicationPolicyMgr) List(...*model.PolicyQuery) (int64, []*model.Policy, error) { - return 0, nil, nil -} - -// Get policy with specified ID -func (f *fakedReplicationPolicyMgr) Get(int64) (*model.Policy, error) { - return &model.Policy{ - ID: 1, - SrcRegistry: &model.Registry{ - ID: 0, - }, - DestRegistry: &model.Registry{ - ID: 0, - }, - }, nil -} - -// Get policy by the name -func (f *fakedReplicationPolicyMgr) GetByName(string) (*model.Policy, error) { - return nil, nil -} - -// Update the specified policy -func (f *fakedReplicationPolicyMgr) Update(policy *model.Policy) error { - return nil -} - -// Remove the specified policy -func (f *fakedReplicationPolicyMgr) Remove(int64) error { - return nil -} - // Add new registry func (f *fakedReplicationRegistryMgr) Add(*model.Registry) (int64, error) { return 0, nil @@ -171,27 +131,25 @@ func TestReplicationHandler_Handle(t *testing.T) { config.Init() PolicyMgr := notification.PolicyMgr - rpPolicy := replication.PolicyCtl rpRegistry := replication.RegistryMgr prj := project.Ctl - repCtl := rep.Ctl + repCtl := repctl.Ctl defer func() { notification.PolicyMgr = PolicyMgr - replication.PolicyCtl = rpPolicy replication.RegistryMgr = rpRegistry project.Ctl = prj - rep.Ctl = repCtl + repctl.Ctl = repCtl }() notification.PolicyMgr = &fakedNotificationPolicyMgr{} - replication.PolicyCtl = &fakedReplicationPolicyMgr{} replication.RegistryMgr = &fakedReplicationRegistryMgr{} projectCtl := &projecttesting.Controller{} project.Ctl = projectCtl mockRepCtl := &replicationtesting.Controller{} - rep.Ctl = mockRepCtl - mockRepCtl.On("GetTask", mock.Anything, mock.Anything).Return(&rep.Task{}, nil) - mockRepCtl.On("GetExecution", mock.Anything, mock.Anything).Return(&rep.Execution{}, nil) + repctl.Ctl = mockRepCtl + mockRepCtl.On("GetPolicy", mock.Anything, mock.Anything).Return(&reppkg.Policy{ID: 1}, nil) + mockRepCtl.On("GetTask", mock.Anything, mock.Anything).Return(&repctl.Task{}, nil) + mockRepCtl.On("GetExecution", mock.Anything, mock.Anything).Return(&repctl.Execution{}, nil) mock.OnAnything(projectCtl, "GetByName").Return(&models.Project{ProjectID: 1}, nil) diff --git a/src/controller/replication/controller.go b/src/controller/replication/execution.go similarity index 85% rename from src/controller/replication/controller.go rename to src/controller/replication/execution.go index fac89d4b6..458e7d77e 100644 --- a/src/controller/replication/controller.go +++ b/src/controller/replication/execution.go @@ -26,6 +26,9 @@ import ( "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/reg" + "github.com/goharbor/harbor/src/pkg/replication" + "github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/replication/model" ) @@ -35,10 +38,25 @@ func init() { task.SetExecutionSweeperCount(job.Replication, 50) } +// Ctl is a global replication controller instance +var Ctl = NewController() + // Controller defines the operations related with replication type Controller interface { + // PolicyCount returns the total count of policies according to the query + PolicyCount(ctx context.Context, query *q.Query) (count int64, err error) + // ListPolicies lists the policies according to the query + ListPolicies(ctx context.Context, query *q.Query) (policies []*replication.Policy, err error) + // GetPolicy gets the specific policy + GetPolicy(ctx context.Context, id int64) (policy *replication.Policy, err error) + // CreatePolicy creates a policy + CreatePolicy(ctx context.Context, policy *replication.Policy) (id int64, err error) + // UpdatePolicy updates the specific policy + UpdatePolicy(ctx context.Context, policy *replication.Policy, props ...string) (err error) + // DeletePolicy deletes the specific policy + DeletePolicy(ctx context.Context, id int64) (err error) // Start the replication according to the policy - Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (executionID int64, err error) + Start(ctx context.Context, policy *replication.Policy, resource *model.Resource, trigger string) (executionID int64, err error) // Stop the replication specified by the execution ID Stop(ctx context.Context, executionID int64) (err error) // ExecutionCount returns the total count of executions according to the query @@ -57,17 +75,14 @@ type Controller interface { GetTaskLog(ctx context.Context, taskID int64) (log []byte, err error) } -var ( - // Ctl is a global replication controller instance - Ctl = NewController() - _ Controller = &controller{} -) - // NewController creates a new instance of the replication controller func NewController() Controller { return &controller{ + repMgr: replication.Mgr, execMgr: task.ExecMgr, taskMgr: task.Mgr, + regMgr: reg.Mgr, + scheduler: scheduler.Sched, flowCtl: flow.NewController(), ormCreator: orm.Crt, wp: lib.NewWorkerPool(1024), @@ -75,14 +90,17 @@ func NewController() Controller { } type controller struct { + repMgr replication.Manager execMgr task.ExecutionManager taskMgr task.Manager + regMgr reg.Manager + scheduler scheduler.Scheduler flowCtl flow.Controller ormCreator orm.Creator wp *lib.WorkerPool } -func (c *controller) Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (int64, error) { +func (c *controller) Start(ctx context.Context, policy *replication.Policy, resource *model.Resource, trigger string) (int64, error) { logger := log.GetLogger(ctx) if !policy.Enabled { return 0, errors.New(nil).WithCode(errors.PreconditionCode). diff --git a/src/controller/replication/controller_test.go b/src/controller/replication/execution_test.go similarity index 87% rename from src/controller/replication/controller_test.go rename to src/controller/replication/execution_test.go index 1079c2b98..29d86988c 100644 --- a/src/controller/replication/controller_test.go +++ b/src/controller/replication/execution_test.go @@ -22,11 +22,14 @@ import ( "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib" + "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/pkg/task/dao" - "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/testing/lib/orm" "github.com/goharbor/harbor/src/testing/mock" + testingreg "github.com/goharbor/harbor/src/testing/pkg/reg" + testingrep "github.com/goharbor/harbor/src/testing/pkg/replication" + testingscheduler "github.com/goharbor/harbor/src/testing/pkg/scheduler" testingTask "github.com/goharbor/harbor/src/testing/pkg/task" "github.com/stretchr/testify/suite" ) @@ -34,18 +37,27 @@ import ( type replicationTestSuite struct { suite.Suite ctl *controller + repMgr *testingrep.Manager + regMgr *testingreg.Manager execMgr *testingTask.ExecutionManager taskMgr *testingTask.Manager + scheduler *testingscheduler.Scheduler flowCtl *flowController ormCreator *orm.Creator } -func (r *replicationTestSuite) SetupSuite() { +func (r *replicationTestSuite) SetupTest() { + r.repMgr = &testingrep.Manager{} + r.regMgr = &testingreg.Manager{} r.execMgr = &testingTask.ExecutionManager{} r.taskMgr = &testingTask.Manager{} + r.scheduler = &testingscheduler.Scheduler{} r.flowCtl = &flowController{} r.ormCreator = &orm.Creator{} r.ctl = &controller{ + repMgr: r.repMgr, + regMgr: r.regMgr, + scheduler: r.scheduler, execMgr: r.execMgr, taskMgr: r.taskMgr, flowCtl: r.flowCtl, @@ -56,7 +68,7 @@ func (r *replicationTestSuite) SetupSuite() { func (r *replicationTestSuite) TestStart() { // policy is disabled - id, err := r.ctl.Start(context.Background(), &model.Policy{Enabled: false}, nil, task.ExecutionTriggerManual) + id, err := r.ctl.Start(context.Background(), &replication.Policy{Enabled: false}, nil, task.ExecutionTriggerManual) r.Require().NotNil(err) // got error when running the replication flow @@ -66,7 +78,7 @@ func (r *replicationTestSuite) TestStart() { r.execMgr.On("MarkError", mock.Anything, mock.Anything, mock.Anything).Return(nil) r.flowCtl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("error")) r.ormCreator.On("Create").Return(nil) - id, err = r.ctl.Start(context.Background(), &model.Policy{Enabled: true}, nil, task.ExecutionTriggerManual) + id, err = r.ctl.Start(context.Background(), &replication.Policy{Enabled: true}, nil, task.ExecutionTriggerManual) r.Require().Nil(err) r.Equal(int64(1), id) time.Sleep(1 * time.Second) // wait the functions called in the goroutine @@ -75,14 +87,14 @@ func (r *replicationTestSuite) TestStart() { r.ormCreator.AssertExpectations(r.T()) // reset the mocks - r.SetupSuite() + r.SetupTest() // got no error when running the replication flow r.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) r.execMgr.On("Get", mock.Anything, mock.Anything).Return(&task.Execution{}, nil) r.flowCtl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) r.ormCreator.On("Create").Return(nil) - id, err = r.ctl.Start(context.Background(), &model.Policy{Enabled: true}, nil, task.ExecutionTriggerManual) + id, err = r.ctl.Start(context.Background(), &replication.Policy{Enabled: true}, nil, task.ExecutionTriggerManual) r.Require().Nil(err) r.Equal(int64(1), id) time.Sleep(1 * time.Second) // wait the functions called in the goroutine @@ -213,6 +225,11 @@ func (r *replicationTestSuite) TestGetTask() { } func (r *replicationTestSuite) TestGetTaskLog() { + r.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{ + { + ID: 1, + }, + }, nil) r.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte{'a'}, nil) data, err := r.ctl.GetTaskLog(nil, 1) r.Require().Nil(err) diff --git a/src/controller/replication/flow/controller.go b/src/controller/replication/flow/controller.go index 55d53cf2f..4be7b2e73 100644 --- a/src/controller/replication/flow/controller.go +++ b/src/controller/replication/flow/controller.go @@ -16,6 +16,7 @@ package flow import ( "context" + "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/replication/model" ) @@ -27,7 +28,7 @@ type Flow interface { // Controller controls the replication flow type Controller interface { - Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) (err error) + Start(ctx context.Context, executionID int64, policy *replication.Policy, resource *model.Resource) (err error) } // NewController returns an instance of the default flow controller @@ -37,7 +38,7 @@ func NewController() Controller { type controller struct{} -func (c *controller) Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) error { +func (c *controller) Start(ctx context.Context, executionID int64, policy *replication.Policy, resource *model.Resource) error { // deletion flow if resource != nil && resource.Deleted { return NewDeletionFlow(executionID, policy, resource).Run(ctx) diff --git a/src/controller/replication/flow/copy.go b/src/controller/replication/flow/copy.go index 58b4e3050..902fec4ff 100644 --- a/src/controller/replication/flow/copy.go +++ b/src/controller/replication/flow/copy.go @@ -17,6 +17,7 @@ package flow import ( "context" "encoding/json" + "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/log" @@ -27,7 +28,7 @@ import ( type copyFlow struct { executionID int64 resources []*model.Resource - policy *model.Policy + policy *replication.Policy executionMgr task.ExecutionManager taskMgr task.Manager } @@ -35,7 +36,7 @@ type copyFlow struct { // NewCopyFlow returns an instance of the copy flow which replicates the resources from // the source registry to the destination registry. If the parameter "resources" isn't provided, // will fetch the resources first -func NewCopyFlow(executionID int64, policy *model.Policy, resources ...*model.Resource) Flow { +func NewCopyFlow(executionID int64, policy *replication.Policy, resources ...*model.Resource) Flow { return ©Flow{ executionMgr: task.ExecMgr, taskMgr: task.Mgr, diff --git a/src/controller/replication/flow/copy_test.go b/src/controller/replication/flow/copy_test.go index 705febbde..712092aca 100644 --- a/src/controller/replication/flow/copy_test.go +++ b/src/controller/replication/flow/copy_test.go @@ -13,6 +13,7 @@ package flow import ( "context" "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/replication/adapter" "github.com/stretchr/testify/mock" "testing" @@ -60,7 +61,7 @@ func (c *copyFlowTestSuite) TestRun() { taskMgr := &testingTask.Manager{} taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) - policy := &model.Policy{ + policy := &replication.Policy{ SrcRegistry: &model.Registry{ Type: "TEST_FOR_COPY_FLOW", }, diff --git a/src/controller/replication/flow/deletion.go b/src/controller/replication/flow/deletion.go index 23c1361c4..ccbcb1626 100644 --- a/src/controller/replication/flow/deletion.go +++ b/src/controller/replication/flow/deletion.go @@ -17,6 +17,7 @@ package flow import ( "context" "encoding/json" + "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/task" @@ -25,7 +26,7 @@ import ( type deletionFlow struct { executionID int64 - policy *model.Policy + policy *replication.Policy executionMgr task.ExecutionManager taskMgr task.Manager resources []*model.Resource @@ -33,7 +34,7 @@ type deletionFlow struct { // NewDeletionFlow returns an instance of the delete flow which deletes the resources // on the destination registry -func NewDeletionFlow(executionID int64, policy *model.Policy, resources ...*model.Resource) Flow { +func NewDeletionFlow(executionID int64, policy *replication.Policy, resources ...*model.Resource) Flow { return &deletionFlow{ executionMgr: task.ExecMgr, taskMgr: task.Mgr, diff --git a/src/controller/replication/flow/deletion_test.go b/src/controller/replication/flow/deletion_test.go index 2640bfc1d..845f7e4a1 100644 --- a/src/controller/replication/flow/deletion_test.go +++ b/src/controller/replication/flow/deletion_test.go @@ -16,6 +16,7 @@ package flow import ( "context" + "github.com/goharbor/harbor/src/pkg/replication" "testing" "github.com/goharbor/harbor/src/replication/model" @@ -32,7 +33,7 @@ func (d *deletionFlowTestSuite) TestRun() { taskMgr := &task.Manager{} taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) - policy := &model.Policy{ + policy := &replication.Policy{ SrcRegistry: &model.Registry{ Type: model.RegistryTypeHarbor, }, diff --git a/src/controller/replication/flow/stage.go b/src/controller/replication/flow/stage.go index 83e282ce5..4365d711c 100644 --- a/src/controller/replication/flow/stage.go +++ b/src/controller/replication/flow/stage.go @@ -16,6 +16,7 @@ package flow import ( "fmt" + "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/lib/log" adp "github.com/goharbor/harbor/src/replication/adapter" @@ -24,7 +25,7 @@ import ( ) // get/create the source registry, destination registry, source adapter and destination adapter -func initialize(policy *model.Policy) (adp.Adapter, adp.Adapter, error) { +func initialize(policy *replication.Policy) (adp.Adapter, adp.Adapter, error) { var srcAdapter, dstAdapter adp.Adapter var err error @@ -52,7 +53,7 @@ func initialize(policy *model.Policy) (adp.Adapter, adp.Adapter, error) { } // fetch resources from the source registry -func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resource, error) { +func fetchResources(adapter adp.Adapter, policy *replication.Policy) ([]*model.Resource, error) { var resTypes []model.ResourceType for _, filter := range policy.Filters { if filter.Type == model.FilterTypeResource { @@ -111,7 +112,7 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc // assemble the source resources by filling the registry information func assembleSourceResources(resources []*model.Resource, - policy *model.Policy) []*model.Resource { + policy *replication.Policy) []*model.Resource { for _, resource := range resources { resource.Registry = policy.SrcRegistry } @@ -121,7 +122,7 @@ func assembleSourceResources(resources []*model.Resource, // assemble the destination resources by filling the metadata, registry and override properties func assembleDestinationResources(resources []*model.Resource, - policy *model.Policy) []*model.Resource { + policy *replication.Policy) []*model.Resource { var result []*model.Resource for _, resource := range resources { res := &model.Resource{ diff --git a/src/controller/replication/flow/stage_test.go b/src/controller/replication/flow/stage_test.go index 4378759d7..4195b8732 100644 --- a/src/controller/replication/flow/stage_test.go +++ b/src/controller/replication/flow/stage_test.go @@ -15,6 +15,7 @@ package flow import ( + "github.com/goharbor/harbor/src/pkg/replication" "testing" "github.com/goharbor/harbor/src/replication/adapter" @@ -35,7 +36,7 @@ func (s *stageTestSuite) TestInitialize() { factory.On("AdapterPattern").Return(nil) adapter.RegisterFactory(model.RegistryTypeHarbor, factory) - policy := &model.Policy{ + policy := &replication.Policy{ SrcRegistry: &model.Registry{ Type: model.RegistryTypeHarbor, }, @@ -60,7 +61,7 @@ func (s *stageTestSuite) TestFetchResources() { {}, {}, }, nil) - policy := &model.Policy{} + policy := &replication.Policy{} resources, err := fetchResources(adapter, policy) s.Require().Nil(err) s.Len(resources, 2) @@ -80,7 +81,7 @@ func (s *stageTestSuite) TestAssembleSourceResources() { Override: false, }, } - policy := &model.Policy{ + policy := &replication.Policy{ SrcRegistry: &model.Registry{ ID: 1, }, @@ -103,7 +104,7 @@ func (s *stageTestSuite) TestAssembleDestinationResources() { Override: false, }, } - policy := &model.Policy{ + policy := &replication.Policy{ DestRegistry: &model.Registry{}, DestNamespace: "test", Override: true, diff --git a/src/controller/replication/mock_flow_controller_test.go b/src/controller/replication/mock_flow_controller_test.go index 01c23beb6..e53128d94 100644 --- a/src/controller/replication/mock_flow_controller_test.go +++ b/src/controller/replication/mock_flow_controller_test.go @@ -8,6 +8,8 @@ import ( mock "github.com/stretchr/testify/mock" model "github.com/goharbor/harbor/src/replication/model" + + replication "github.com/goharbor/harbor/src/pkg/replication" ) // flowController is an autogenerated mock type for the Controller type @@ -16,11 +18,11 @@ type flowController struct { } // Start provides a mock function with given fields: ctx, executionID, policy, resource -func (_m *flowController) Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) error { +func (_m *flowController) Start(ctx context.Context, executionID int64, policy *replication.Policy, resource *model.Resource) error { ret := _m.Called(ctx, executionID, policy, resource) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int64, *model.Policy, *model.Resource) error); ok { + if rf, ok := ret.Get(0).(func(context.Context, int64, *replication.Policy, *model.Resource) error); ok { r0 = rf(ctx, executionID, policy, resource) } else { r0 = ret.Error(0) diff --git a/src/controller/replication/policy.go b/src/controller/replication/policy.go new file mode 100644 index 000000000..5d002833e --- /dev/null +++ b/src/controller/replication/policy.go @@ -0,0 +1,188 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "context" + "strconv" + + commonthttp "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/replication" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/pkg/task" + "github.com/goharbor/harbor/src/replication/config" + "github.com/goharbor/harbor/src/replication/model" +) + +const callbackFuncName = "REPLICATION_CALLBACK" + +func init() { + callbackFunc := func(ctx context.Context, param string) error { + policyID, err := strconv.ParseInt(param, 10, 64) + if err != nil { + return err + } + policy, err := Ctl.GetPolicy(ctx, policyID) + if err != nil { + return err + } + _, err = Ctl.Start(ctx, policy, nil, task.ExecutionTriggerSchedule) + return err + } + err := scheduler.RegisterCallbackFunc(callbackFuncName, callbackFunc) + if err != nil { + log.Errorf("failed to register the callback function for replication: %v", err) + } +} + +func (c *controller) PolicyCount(ctx context.Context, query *q.Query) (int64, error) { + return c.repMgr.Count(ctx, query) +} + +func (c *controller) ListPolicies(ctx context.Context, query *q.Query) ([]*replication.Policy, error) { + policies, err := c.repMgr.List(ctx, query) + if err != nil { + return nil, err + } + for _, policy := range policies { + if err := c.populateRegistry(ctx, policy); err != nil { + return nil, err + } + } + return policies, nil +} + +func (c *controller) populateRegistry(ctx context.Context, policy *replication.Policy) error { + if policy.SrcRegistry != nil && policy.SrcRegistry.ID > 0 { + registry, err := c.regMgr.Get(ctx, policy.SrcRegistry.ID) + if err != nil { + return err + } + policy.SrcRegistry = registry + policy.DestRegistry = GetLocalRegistry() + return nil + } + + registry, err := c.regMgr.Get(ctx, policy.DestRegistry.ID) + if err != nil { + return err + } + policy.DestRegistry = registry + policy.SrcRegistry = GetLocalRegistry() + return nil +} + +func (c *controller) GetPolicy(ctx context.Context, id int64) (*replication.Policy, error) { + policy, err := c.repMgr.Get(ctx, id) + if err != nil { + return nil, err + } + if err = c.populateRegistry(ctx, policy); err != nil { + return nil, err + } + return policy, nil +} + +func (c *controller) CreatePolicy(ctx context.Context, policy *replication.Policy) (int64, error) { + if err := c.validatePolicy(ctx, policy); err != nil { + return 0, err + } + // create policy + id, err := c.repMgr.Create(ctx, policy) + if err != nil { + return 0, err + } + // create schedule if needed + if policy.IsScheduledTrigger() { + if _, err = c.scheduler.Schedule(ctx, job.Replication, id, "", policy.Trigger.Settings.Cron, + callbackFuncName, policy.ID, map[string]interface{}{}); err != nil { + return 0, err + } + } + return id, nil +} + +func (c *controller) UpdatePolicy(ctx context.Context, policy *replication.Policy, props ...string) error { + if err := c.validatePolicy(ctx, policy); err != nil { + return err + } + // delete the schedule + if err := c.scheduler.UnScheduleByVendor(ctx, job.Replication, policy.ID); err != nil { + return err + } + // update the policy + if err := c.repMgr.Update(ctx, policy); err != nil { + return err + } + // create schedule if needed + if policy.IsScheduledTrigger() { + if _, err := c.scheduler.Schedule(ctx, job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, + callbackFuncName, policy.ID, map[string]interface{}{}); err != nil { + return err + } + } + return nil +} + +func (c *controller) validatePolicy(ctx context.Context, policy *replication.Policy) error { + if err := policy.Validate(); err != nil { + return err + } + if policy.SrcRegistry != nil { + if _, err := c.regMgr.Get(ctx, policy.SrcRegistry.ID); err != nil { + return err + } + } + if policy.DestRegistry != nil { + if _, err := c.regMgr.Get(ctx, policy.DestRegistry.ID); err != nil { + return err + } + } + return nil +} + +func (c *controller) DeletePolicy(ctx context.Context, id int64) error { + // delete the executions + if err := c.execMgr.DeleteByVendor(ctx, job.Replication, id); err != nil { + return err + } + // delete the schedule + if err := c.scheduler.UnScheduleByVendor(ctx, job.Replication, id); err != nil { + return err + } + // delete the policy + return c.repMgr.Delete(ctx, id) +} + +// GetLocalRegistry returns the info of the local Harbor registry +// TODO move it into the registry package +func GetLocalRegistry() *model.Registry { + return &model.Registry{ + Type: model.RegistryTypeHarbor, + Name: "Local", + URL: config.Config.CoreURL, + TokenServiceURL: config.Config.TokenServiceURL, + Status: "healthy", + Credential: &model.Credential{ + Type: model.CredentialTypeSecret, + // use secret to do the auth for the local Harbor + AccessSecret: config.Config.JobserviceSecret, + }, + Insecure: !commonthttp.InternalTLSEnabled(), + } +} diff --git a/src/controller/replication/policy_test.go b/src/controller/replication/policy_test.go new file mode 100644 index 000000000..a6209ea24 --- /dev/null +++ b/src/controller/replication/policy_test.go @@ -0,0 +1,133 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "github.com/goharbor/harbor/src/pkg/replication" + "github.com/goharbor/harbor/src/replication/config" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/testing/mock" +) + +func (r *replicationTestSuite) TestPolicyCount() { + mock.OnAnything(r.repMgr, "Count").Return(int64(1), nil) + count, err := r.ctl.PolicyCount(nil, nil) + r.Require().Nil(err) + r.Equal(int64(1), count) + r.repMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestListPolicies() { + mock.OnAnything(r.repMgr, "List").Return([]*replication.Policy{ + { + ID: 1, + SrcRegistry: &model.Registry{ + ID: 1, + }, + }, + }, nil) + mock.OnAnything(r.regMgr, "Get").Return(&model.Registry{ + ID: 1, + }, nil) + config.Config = &config.Configuration{} + policies, err := r.ctl.ListPolicies(nil, nil) + r.Require().Nil(err) + r.Require().Len(policies, 1) + r.Equal(int64(1), policies[0].ID) + r.repMgr.AssertExpectations(r.T()) + r.regMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestGetPolicy() { + mock.OnAnything(r.repMgr, "Get").Return(&replication.Policy{ + ID: 1, + SrcRegistry: &model.Registry{ + ID: 1, + }, + }, nil) + mock.OnAnything(r.regMgr, "Get").Return(&model.Registry{ + ID: 1, + }, nil) + config.Config = &config.Configuration{} + policy, err := r.ctl.GetPolicy(nil, 1) + r.Require().Nil(err) + r.Equal(int64(1), policy.ID) + r.repMgr.AssertExpectations(r.T()) + r.regMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestCreatePolicy() { + mock.OnAnything(r.repMgr, "Create").Return(int64(1), nil) + mock.OnAnything(r.regMgr, "Get").Return(&model.Registry{ + ID: 1, + }, nil) + mock.OnAnything(r.scheduler, "Schedule").Return(int64(1), nil) + id, err := r.ctl.CreatePolicy(nil, &replication.Policy{ + Name: "rule", + SrcRegistry: &model.Registry{ + ID: 1, + }, + Trigger: &model.Trigger{ + Type: model.TriggerTypeScheduled, + Settings: &model.TriggerSettings{ + Cron: "0 * * * * *", + }, + }, + Enabled: true, + }) + r.Require().Nil(err) + r.Equal(int64(1), id) + r.repMgr.AssertExpectations(r.T()) + r.regMgr.AssertExpectations(r.T()) + r.scheduler.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestUpdatePolicy() { + mock.OnAnything(r.regMgr, "Get").Return(&model.Registry{ + ID: 1, + }, nil) + mock.OnAnything(r.scheduler, "UnScheduleByVendor").Return(nil) + mock.OnAnything(r.scheduler, "Schedule").Return(int64(1), nil) + mock.OnAnything(r.repMgr, "Update").Return(nil) + err := r.ctl.UpdatePolicy(nil, &replication.Policy{ + ID: 1, + Name: "rule", + SrcRegistry: &model.Registry{ + ID: 1, + }, + Trigger: &model.Trigger{ + Type: model.TriggerTypeScheduled, + Settings: &model.TriggerSettings{ + Cron: "0 * * * * *", + }, + }, + Enabled: true, + }) + r.Require().Nil(err) + r.repMgr.AssertExpectations(r.T()) + r.regMgr.AssertExpectations(r.T()) + r.scheduler.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestDeletePolicy() { + mock.OnAnything(r.execMgr, "DeleteByVendor").Return(nil) + mock.OnAnything(r.scheduler, "UnScheduleByVendor").Return(nil) + mock.OnAnything(r.repMgr, "Delete").Return(nil) + err := r.ctl.DeletePolicy(nil, 1) + r.Require().Nil(err) + r.repMgr.AssertExpectations(r.T()) + r.execMgr.AssertExpectations(r.T()) + r.scheduler.AssertExpectations(r.T()) +} diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 61043b7ed..a4f70d0f6 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -120,9 +120,6 @@ func init() { beego.Router("/api/replication/adapters", &ReplicationAdapterAPI{}, "get:List") - beego.Router("/api/replication/policies", &ReplicationPolicyAPI{}, "get:List;post:Create") - beego.Router("/api/replication/policies/:id([0-9]+)", &ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete") - beego.Router("/api/projects/:pid([0-9]+)/webhook/policies", &NotificationPolicyAPI{}, "get:List;post:Post") beego.Router("/api/projects/:pid([0-9]+)/webhook/policies/:id([0-9]+)", &NotificationPolicyAPI{}) beego.Router("/api/projects/:pid([0-9]+)/webhook/policies/test", &NotificationPolicyAPI{}, "post:Test") diff --git a/src/core/api/registry.go b/src/core/api/registry.go index 0bf3a1cab..5e0c8ad60 100644 --- a/src/core/api/registry.go +++ b/src/core/api/registry.go @@ -11,23 +11,21 @@ import ( "strconv" "github.com/goharbor/harbor/src/common/utils" + rep "github.com/goharbor/harbor/src/controller/replication" "github.com/goharbor/harbor/src/core/api/models" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/event" "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/policy" "github.com/goharbor/harbor/src/replication/registry" ) // RegistryAPI handles requests to /api/registries/{}. It manages registries integrated to Harbor. type RegistryAPI struct { BaseController - manager registry.Manager - policyCtl policy.Controller - resource types.Resource + manager registry.Manager + resource types.Resource } // Prepare validates the user @@ -40,7 +38,6 @@ func (t *RegistryAPI) Prepare() { t.resource = system.NewNamespace().Resource(rbac.ResourceRegistry) t.manager = replication.RegistryMgr - t.policyCtl = replication.PolicyCtl } // Ping checks health status of a registry @@ -368,11 +365,11 @@ func (t *RegistryAPI) Delete() { } // Check whether there are replication policies that use this registry as source registry. - total, _, err := t.policyCtl.List([]*model.PolicyQuery{ - { - SrcRegistry: id, + total, err := rep.Ctl.PolicyCount(orm.Context(), &q.Query{ + Keywords: map[string]interface{}{ + "SrcRegistryID": id, }, - }...) + }) if err != nil { t.SendInternalServerError(fmt.Errorf("List replication policies with source registry %d error: %v", id, err)) return @@ -385,11 +382,11 @@ func (t *RegistryAPI) Delete() { } // Check whether there are replication policies that use this registry as destination registry. - total, _, err = t.policyCtl.List([]*model.PolicyQuery{ - { - DestRegistry: id, + total, err = rep.Ctl.PolicyCount(orm.Context(), &q.Query{ + Keywords: map[string]interface{}{ + "DestRegistryID": id, }, - }...) + }) if err != nil { t.SendInternalServerError(fmt.Errorf("List replication policies with destination registry %d error: %v", id, err)) return @@ -434,7 +431,7 @@ func (t *RegistryAPI) GetInfo() { } var registry *model.Registry if id == 0 { - registry = event.GetLocalRegistry() + registry = rep.GetLocalRegistry() } else { registry, err = t.manager.Get(id) if err != nil { diff --git a/src/core/api/replication_policy.go b/src/core/api/replication_policy.go deleted file mode 100644 index 917e78ece..000000000 --- a/src/core/api/replication_policy.go +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "errors" - "fmt" - "github.com/goharbor/harbor/src/common/rbac" - "github.com/goharbor/harbor/src/common/rbac/system" - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/lib/orm" - "github.com/goharbor/harbor/src/lib/q" - "github.com/goharbor/harbor/src/pkg/permission/types" - "net/http" - "strconv" - - replica "github.com/goharbor/harbor/src/controller/replication" - "github.com/goharbor/harbor/src/pkg/task" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/event" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/registry" -) - -// TODO rename the file to "replication.go" - -// ReplicationPolicyAPI handles the replication policy requests -type ReplicationPolicyAPI struct { - BaseController - resource types.Resource -} - -// Prepare ... -func (r *ReplicationPolicyAPI) Prepare() { - r.BaseController.Prepare() - if !r.SecurityCtx.IsAuthenticated() { - r.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - r.resource = system.NewNamespace().Resource(rbac.ResourceReplicationPolicy) -} - -// List the replication policies -func (r *ReplicationPolicyAPI) List() { - if !r.SecurityCtx.Can(r.Context(), rbac.ActionList, r.resource) { - r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername())) - return - } - page, size, err := r.GetPaginationParams() - if err != nil { - r.SendInternalServerError(err) - return - } - // TODO: support more query - query := &model.PolicyQuery{ - Name: r.GetString("name"), - Page: page, - Size: size, - } - - total, policies, err := replication.PolicyCtl.List(query) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to list policies: %v", err)) - return - } - for _, policy := range policies { - if err = populateRegistries(replication.RegistryMgr, policy); err != nil { - r.SendInternalServerError(fmt.Errorf("failed to populate registries for policy %d: %v", policy.ID, err)) - return - } - } - r.SetPaginationHeader(total, query.Page, query.Size) - r.WriteJSONData(policies) -} - -// Create the replication policy -func (r *ReplicationPolicyAPI) Create() { - if !r.SecurityCtx.Can(r.Context(), rbac.ActionCreate, r.resource) { - r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername())) - return - } - policy := &model.Policy{} - isValid, err := r.DecodeJSONReqAndValidate(policy) - if !isValid { - r.SendBadRequestError(err) - return - } - - if !r.validateName(policy) { - return - } - if !r.validateRegistry(policy) { - return - } - - policy.Creator = r.SecurityCtx.GetUsername() - id, err := replication.PolicyCtl.Create(policy) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to create the policy: %v", err)) - return - } - r.Redirect(http.StatusCreated, strconv.FormatInt(id, 10)) -} - -// make sure the policy name doesn't exist -func (r *ReplicationPolicyAPI) validateName(policy *model.Policy) bool { - p, err := replication.PolicyCtl.GetByName(policy.Name) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to get policy %s: %v", policy.Name, err)) - return false - } - if p != nil { - r.SendConflictError(fmt.Errorf("policy %s already exists", policy.Name)) - return false - } - return true -} - -// make sure the registry referred exists -func (r *ReplicationPolicyAPI) validateRegistry(policy *model.Policy) bool { - var registryID int64 - if policy.SrcRegistry != nil && policy.SrcRegistry.ID > 0 { - registryID = policy.SrcRegistry.ID - } else { - registryID = policy.DestRegistry.ID - } - registry, err := replication.RegistryMgr.Get(registryID) - if err != nil { - r.SendConflictError(fmt.Errorf("failed to get registry %d: %v", registryID, err)) - return false - } - if registry == nil { - r.SendBadRequestError(fmt.Errorf("registry %d not found", registryID)) - return false - } - return true -} - -// Get the specified replication policy -func (r *ReplicationPolicyAPI) Get() { - if !r.SecurityCtx.Can(r.Context(), rbac.ActionRead, r.resource) { - r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername())) - return - } - id, err := r.GetInt64FromPath(":id") - if id <= 0 || err != nil { - r.SendBadRequestError(errors.New("invalid policy ID")) - return - } - - policy, err := replication.PolicyCtl.Get(id) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to get the policy %d: %v", id, err)) - return - } - if policy == nil { - r.SendNotFoundError(fmt.Errorf("policy %d not found", id)) - return - } - if err = populateRegistries(replication.RegistryMgr, policy); err != nil { - r.SendInternalServerError(fmt.Errorf("failed to populate registries for policy %d: %v", policy.ID, err)) - return - } - - r.WriteJSONData(policy) -} - -// Update the replication policy -func (r *ReplicationPolicyAPI) Update() { - if !r.SecurityCtx.Can(r.Context(), rbac.ActionUpdate, r.resource) { - r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername())) - return - } - id, err := r.GetInt64FromPath(":id") - if id <= 0 || err != nil { - r.SendBadRequestError(errors.New("invalid policy ID")) - return - } - - originalPolicy, err := replication.PolicyCtl.Get(id) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to get the policy %d: %v", id, err)) - return - } - if originalPolicy == nil { - r.SendNotFoundError(fmt.Errorf("policy %d not found", id)) - return - } - - policy := &model.Policy{} - isValid, err := r.DecodeJSONReqAndValidate(policy) - if !isValid { - r.SendBadRequestError(err) - return - } - - if policy.Name != originalPolicy.Name && - !r.validateName(policy) { - return - } - - if !r.validateRegistry(policy) { - return - } - - policy.ID = id - if err := replication.PolicyCtl.Update(policy); err != nil { - r.SendInternalServerError(fmt.Errorf("failed to update the policy %d: %v", id, err)) - return - } -} - -// Delete the replication policy -func (r *ReplicationPolicyAPI) Delete() { - if !r.SecurityCtx.Can(r.Context(), rbac.ActionDelete, r.resource) { - r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername())) - return - } - id, err := r.GetInt64FromPath(":id") - if id <= 0 || err != nil { - r.SendBadRequestError(errors.New("invalid policy ID")) - return - } - - policy, err := replication.PolicyCtl.Get(id) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to get the policy %d: %v", id, err)) - return - } - if policy == nil { - r.SendNotFoundError(fmt.Errorf("policy %d not found", id)) - return - } - - ctx := orm.Context() - executions, err := replica.Ctl.ListExecutions(ctx, &q.Query{ - Keywords: map[string]interface{}{ - "PolicyID": id, - }, - }) - if err != nil { - r.SendInternalServerError(err) - return - } - for _, execution := range executions { - if execution.Status != job.RunningStatus.String() { - continue - } - r.SendPreconditionFailedError(fmt.Errorf("the policy %d has running executions, can not be deleted", id)) - return - } - for _, execution := range executions { - if err = task.ExecMgr.Delete(ctx, execution.ID); err != nil { - r.SendInternalServerError(err) - return - } - } - - if err := replication.PolicyCtl.Remove(id); err != nil { - r.SendInternalServerError(fmt.Errorf("failed to delete the policy %d: %v", id, err)) - return - } -} - -// ignore the credential for the registries -func populateRegistries(registryMgr registry.Manager, policy *model.Policy) error { - if err := event.PopulateRegistries(registryMgr, policy); err != nil { - return err - } - if policy.SrcRegistry != nil { - hideAccessSecret(policy.SrcRegistry.Credential) - } - if policy.DestRegistry != nil { - hideAccessSecret(policy.DestRegistry.Credential) - } - return nil -} diff --git a/src/core/api/replication_policy_test.go b/src/core/api/replication_policy_test.go deleted file mode 100644 index 8ce75d678..000000000 --- a/src/core/api/replication_policy_test.go +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "github.com/goharbor/harbor/src/lib/q" - "net/http" - "testing" - - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/model" -) - -// TODO rename the file to "replication.go" - -type fakedRegistryManager struct{} - -func (f *fakedRegistryManager) Add(*model.Registry) (int64, error) { - return 0, nil -} -func (f *fakedRegistryManager) List(query *q.Query) (int64, []*model.Registry, error) { - return 0, nil, nil -} -func (f *fakedRegistryManager) Get(id int64) (*model.Registry, error) { - if id == 1 { - return &model.Registry{ - Type: "faked_registry", - }, nil - } - return nil, nil -} -func (f *fakedRegistryManager) GetByName(string) (*model.Registry, error) { - return nil, nil -} -func (f *fakedRegistryManager) Update(*model.Registry, ...string) error { - return nil -} -func (f *fakedRegistryManager) Remove(int64) error { - return nil -} -func (f *fakedRegistryManager) HealthCheck() error { - return nil -} - -type fakedPolicyManager struct{} - -func (f *fakedPolicyManager) Create(*model.Policy) (int64, error) { - return 0, nil -} -func (f *fakedPolicyManager) List(...*model.PolicyQuery) (int64, []*model.Policy, error) { - return 0, nil, nil -} -func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) { - if id == 1 { - return &model.Policy{ - ID: 1, - Enabled: true, - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, nil - } - if id == 2 { - return &model.Policy{ - ID: 2, - Enabled: false, - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, nil - } - return nil, nil -} -func (f *fakedPolicyManager) GetByName(name string) (*model.Policy, error) { - if name == "duplicate_name" { - return &model.Policy{ - Name: "duplicate_name", - }, nil - } - return nil, nil -} -func (f *fakedPolicyManager) Update(*model.Policy) error { - return nil -} -func (f *fakedPolicyManager) Remove(int64) error { - return nil -} - -func TestReplicationPolicyAPIList(t *testing.T) { - policyMgr := replication.PolicyCtl - defer func() { - replication.PolicyCtl = policyMgr - }() - replication.PolicyCtl = &fakedPolicyManager{} - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/policies", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/policies", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/policies", - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestReplicationPolicyAPICreate(t *testing.T) { - policyMgr := replication.PolicyCtl - registryMgr := replication.RegistryMgr - defer func() { - replication.PolicyCtl = policyMgr - replication.RegistryMgr = registryMgr - }() - replication.PolicyCtl = &fakedPolicyManager{} - replication.RegistryMgr = &fakedRegistryManager{} - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/policies", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/policies", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 400 empty policy name - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/policies", - credential: sysAdmin, - bodyJSON: &model.Policy{ - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, - }, - code: http.StatusBadRequest, - }, - // 400 empty registry - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/policies", - credential: sysAdmin, - bodyJSON: &model.Policy{ - Name: "policy01", - }, - }, - code: http.StatusBadRequest, - }, - // 409, duplicate policy name - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/policies", - credential: sysAdmin, - bodyJSON: &model.Policy{ - Name: "duplicate_name", - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, - }, - code: http.StatusConflict, - }, - // 400, registry not found - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/policies", - credential: sysAdmin, - bodyJSON: &model.Policy{ - Name: "policy01", - SrcRegistry: &model.Registry{ - ID: 2, - }, - }, - }, - code: http.StatusBadRequest, - }, - // 201 - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/policies", - credential: sysAdmin, - bodyJSON: &model.Policy{ - Name: "policy01", - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, - }, - code: http.StatusCreated, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestReplicationPolicyAPIGet(t *testing.T) { - policyMgr := replication.PolicyCtl - registryMgr := replication.RegistryMgr - defer func() { - replication.PolicyCtl = policyMgr - replication.RegistryMgr = registryMgr - }() - replication.PolicyCtl = &fakedPolicyManager{} - replication.RegistryMgr = &fakedRegistryManager{} - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/policies/1", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/policies/1", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 404, policy not found - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/policies/3", - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/policies/1", - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestReplicationPolicyAPIUpdate(t *testing.T) { - policyMgr := replication.PolicyCtl - registryMgr := replication.RegistryMgr - defer func() { - replication.PolicyCtl = policyMgr - replication.RegistryMgr = registryMgr - }() - replication.PolicyCtl = &fakedPolicyManager{} - replication.RegistryMgr = &fakedRegistryManager{} - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/policies/1", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/policies/1", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 404 policy not found - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/policies/3", - credential: sysAdmin, - bodyJSON: &model.Policy{}, - }, - code: http.StatusNotFound, - }, - // 400 empty policy name - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/policies/1", - credential: sysAdmin, - bodyJSON: &model.Policy{ - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, - }, - code: http.StatusBadRequest, - }, - // 409, duplicate policy name - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/policies/1", - credential: sysAdmin, - bodyJSON: &model.Policy{ - Name: "duplicate_name", - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, - }, - code: http.StatusConflict, - }, - // 400, registry not found - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/policies/1", - credential: sysAdmin, - bodyJSON: &model.Policy{ - Name: "policy01", - SrcRegistry: &model.Registry{ - ID: 2, - }, - }, - }, - code: http.StatusBadRequest, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/policies/1", - credential: sysAdmin, - bodyJSON: &model.Policy{ - Name: "policy01", - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestReplicationPolicyAPIDelete(t *testing.T) { - policyMgr := replication.PolicyCtl - defer func() { - replication.PolicyCtl = policyMgr - }() - replication.PolicyCtl = &fakedPolicyManager{} - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodDelete, - url: "/api/replication/policies/1", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodDelete, - url: "/api/replication/policies/1", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 404, policy not found - { - request: &testingRequest{ - method: http.MethodDelete, - url: "/api/replication/policies/3", - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodDelete, - url: "/api/replication/policies/1", - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} diff --git a/src/pkg/reg/dao/dao.go b/src/pkg/reg/dao/dao.go new file mode 100644 index 000000000..162367bb4 --- /dev/null +++ b/src/pkg/reg/dao/dao.go @@ -0,0 +1,128 @@ +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "context" + + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/replication/dao/models" +) + +// DAO defines the DAO operations of registry +type DAO interface { + // Create the registry + Create(ctx context.Context, registry *models.Registry) (id int64, err error) + // Count returns the count of registries according to the query + Count(ctx context.Context, query *q.Query) (count int64, err error) + // List the registries according to the query + List(ctx context.Context, query *q.Query) (registries []*models.Registry, err error) + // Get the registry specified by ID + Get(ctx context.Context, id int64) (registry *models.Registry, err error) + // Update the specified registry + Update(ctx context.Context, registry *models.Registry, props ...string) (err error) + // Delete the registry specified by ID + Delete(ctx context.Context, id int64) (err error) +} + +// NewDAO creates an instance of DAO +func NewDAO() DAO { + return &dao{} +} + +type dao struct{} + +func (d *dao) Create(ctx context.Context, registry *models.Registry) (int64, error) { + ormer, err := orm.FromContext(ctx) + if err != nil { + return 0, err + } + id, err := ormer.Insert(registry) + if e := orm.AsConflictError(err, "registry %s already exists", registry.Name); e != nil { + err = e + } + return id, err +} + +func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) { + qs, err := orm.QuerySetterForCount(ctx, &models.Registry{}, query) + if err != nil { + return 0, err + } + return qs.Count() +} + +func (d *dao) List(ctx context.Context, query *q.Query) ([]*models.Registry, error) { + registries := []*models.Registry{} + qs, err := orm.QuerySetter(ctx, &models.Registry{}, query) + if err != nil { + return nil, err + } + if _, err = qs.All(®istries); err != nil { + return nil, err + } + return registries, nil +} + +func (d *dao) Get(ctx context.Context, id int64) (*models.Registry, error) { + registry := &models.Registry{ + ID: id, + } + ormer, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + if err := ormer.Read(registry); err != nil { + if e := orm.AsNotFoundError(err, "registry %d not found", id); e != nil { + err = e + } + return nil, err + } + return registry, nil +} + +func (d *dao) Update(ctx context.Context, registry *models.Registry, props ...string) error { + ormer, err := orm.FromContext(ctx) + if err != nil { + return err + } + n, err := ormer.Update(registry, props...) + if err != nil { + if e := orm.AsConflictError(err, "registry %s already exists", registry.Name); e != nil { + err = e + } + return err + } + if n == 0 { + return errors.NotFoundError(nil).WithMessage("registry %d not found", registry.ID) + } + return nil +} + +func (d *dao) Delete(ctx context.Context, id int64) error { + ormer, err := orm.FromContext(ctx) + if err != nil { + return err + } + n, err := ormer.Delete(&models.Registry{ + ID: id, + }) + if err != nil { + return err + } + if n == 0 { + return errors.NotFoundError(nil).WithMessage("registry %d not found", id) + } + return nil +} diff --git a/src/pkg/reg/dao/dao_test.go b/src/pkg/reg/dao/dao_test.go new file mode 100644 index 000000000..835a54e09 --- /dev/null +++ b/src/pkg/reg/dao/dao_test.go @@ -0,0 +1,162 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "context" + "testing" + + beegoorm "github.com/astaxie/beego/orm" + common_dao "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/replication/dao/models" + "github.com/stretchr/testify/suite" +) + +type daoTestSuite struct { + suite.Suite + dao DAO + ctx context.Context + id int64 +} + +func (d *daoTestSuite) SetupSuite() { + d.dao = NewDAO() + common_dao.PrepareTestForPostgresSQL() + d.ctx = orm.NewContext(nil, beegoorm.NewOrm()) +} + +func (d *daoTestSuite) SetupTest() { + registry := &models.Registry{ + URL: "http://harbor.local", + Name: "harbor", + Type: "harbor", + Insecure: false, + Health: "health", + } + id, err := d.dao.Create(d.ctx, registry) + d.Require().Nil(err) + d.id = id +} + +func (d *daoTestSuite) TearDownTest() { + err := d.dao.Delete(d.ctx, d.id) + d.Require().Nil(err) +} + +func (d *daoTestSuite) TestCount() { + // nil query + total, err := d.dao.Count(d.ctx, nil) + d.Require().Nil(err) + d.True(total > 0) + + // query by name + total, err = d.dao.Count(d.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "Name": "harbor", + }, + }) + d.Require().Nil(err) + d.Equal(int64(1), total) +} + +func (d *daoTestSuite) TestList() { + // nil query + registries, err := d.dao.List(d.ctx, nil) + d.Require().Nil(err) + found := false + for _, registry := range registries { + if registry.ID == d.id { + found = true + break + } + } + d.True(found) + + // query by name + registries, err = d.dao.List(d.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "Name": "harbor", + }, + }) + d.Require().Nil(err) + d.Require().Equal(1, len(registries)) + d.Equal(d.id, registries[0].ID) +} + +func (d *daoTestSuite) TestGet() { + // get the non-exist registry + _, err := d.dao.Get(d.ctx, 10000) + d.Require().NotNil(err) + d.True(errors.IsErr(err, errors.NotFoundCode)) + + // get the exist registry + registry, err := d.dao.Get(d.ctx, d.id) + d.Require().Nil(err) + d.Require().NotNil(registry) + d.Equal(d.id, registry.ID) +} + +func (d *daoTestSuite) TestCreate() { + // the happy pass case is covered in Setup + + // conflict + registry := &models.Registry{ + Name: "harbor", + } + _, err := d.dao.Create(d.ctx, registry) + d.Require().NotNil(err) + d.True(errors.IsErr(err, errors.ConflictCode)) +} + +func (d *daoTestSuite) TestDelete() { + // the happy pass case is covered in TearDown + + // not exist + err := d.dao.Delete(d.ctx, 100021) + d.Require().NotNil(err) + var e *errors.Error + d.Require().True(errors.As(err, &e)) + d.Equal(errors.NotFoundCode, e.Code) +} + +func (d *daoTestSuite) TestUpdate() { + // pass + err := d.dao.Update(d.ctx, &models.Registry{ + ID: d.id, + Description: "description", + }, "Description") + d.Require().Nil(err) + + registry, err := d.dao.Get(d.ctx, d.id) + d.Require().Nil(err) + d.Require().NotNil(registry) + d.Equal("description", registry.Description) + + // not exist + err = d.dao.Update(d.ctx, &models.Registry{ + ID: 10000, + }) + d.Require().NotNil(err) + var e *errors.Error + d.Require().True(errors.As(err, &e)) + d.Equal(errors.NotFoundCode, e.Code) +} + +func TestDaoTestSuite(t *testing.T) { + suite.Run(t, &daoTestSuite{}) +} diff --git a/src/pkg/reg/manager.go b/src/pkg/reg/manager.go new file mode 100644 index 000000000..b0ad20cff --- /dev/null +++ b/src/pkg/reg/manager.go @@ -0,0 +1,104 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reg + +import ( + "context" + + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/reg/dao" + "github.com/goharbor/harbor/src/replication/model" + reg "github.com/goharbor/harbor/src/replication/registry" +) + +var ( + // Mgr is the global registry manager instance + Mgr = NewManager() +) + +// Manager defines the registry related operations +type Manager interface { + // Create the registry + Create(ctx context.Context, registry *model.Registry) (id int64, err error) + // Count returns the count of registries according to the query + Count(ctx context.Context, query *q.Query) (count int64, err error) + // List registries according to the query + List(ctx context.Context, query *q.Query) (registries []*model.Registry, err error) + // Get the registry specified by ID + Get(ctx context.Context, id int64) (registry *model.Registry, err error) + // Update the specified registry + Update(ctx context.Context, registry *model.Registry, props ...string) (err error) + // Delete the registry specified by ID + Delete(ctx context.Context, id int64) (err error) +} + +// NewManager creates an instance of registry manager +func NewManager() Manager { + return &manager{ + dao: dao.NewDAO(), + } +} + +type manager struct { + dao dao.DAO +} + +func (m *manager) Create(ctx context.Context, registry *model.Registry) (int64, error) { + reg, err := reg.ToDaoModel(registry) + if err != nil { + return 0, err + } + return m.dao.Create(ctx, reg) +} + +func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) { + return m.dao.Count(ctx, query) +} + +func (m *manager) List(ctx context.Context, query *q.Query) ([]*model.Registry, error) { + registries, err := m.dao.List(ctx, query) + if err != nil { + return nil, err + } + var regs []*model.Registry + for _, registry := range registries { + r, err := reg.FromDaoModel(registry) + if err != nil { + return nil, err + } + regs = append(regs, r) + } + return regs, nil +} + +func (m *manager) Get(ctx context.Context, id int64) (*model.Registry, error) { + registry, err := m.dao.Get(ctx, id) + if err != nil { + return nil, err + } + return reg.FromDaoModel(registry) +} + +func (m *manager) Update(ctx context.Context, registry *model.Registry, props ...string) error { + reg, err := reg.ToDaoModel(registry) + if err != nil { + return err + } + return m.dao.Update(ctx, reg, props...) +} + +func (m *manager) Delete(ctx context.Context, id int64) error { + return m.dao.Delete(ctx, id) +} diff --git a/src/pkg/reg/manager_test.go b/src/pkg/reg/manager_test.go new file mode 100644 index 000000000..3cd7b8b53 --- /dev/null +++ b/src/pkg/reg/manager_test.go @@ -0,0 +1,94 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reg + +import ( + "testing" + + "github.com/goharbor/harbor/src/replication/dao/models" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/testing/mock" + "github.com/goharbor/harbor/src/testing/pkg/reg/dao" + "github.com/stretchr/testify/suite" +) + +type managerTestSuite struct { + suite.Suite + mgr *manager + dao *dao.DAO +} + +func (m *managerTestSuite) SetupTest() { + m.dao = &dao.DAO{} + m.mgr = &manager{ + dao: m.dao, + } +} + +func (m *managerTestSuite) TestCount() { + mock.OnAnything(m.dao, "Count").Return(int64(1), nil) + n, err := m.mgr.Count(nil, nil) + m.Require().Nil(err) + m.Equal(int64(1), n) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestList() { + mock.OnAnything(m.dao, "List").Return([]*models.Registry{ + { + ID: 1, + }, + }, nil) + registries, err := m.mgr.List(nil, nil) + m.Require().Nil(err) + m.Require().Equal(1, len(registries)) + m.Equal(int64(1), registries[0].ID) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestGet() { + mock.OnAnything(m.dao, "Get").Return(&models.Registry{ + ID: 1, + }, nil) + registry, err := m.mgr.Get(nil, 1) + m.Require().Nil(err) + m.Equal(int64(1), registry.ID) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestCreate() { + mock.OnAnything(m.dao, "Create").Return(int64(1), nil) + _, err := m.mgr.Create(nil, &model.Registry{}) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestDelete() { + mock.OnAnything(m.dao, "Delete").Return(nil) + err := m.mgr.Delete(nil, 1) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestUpdate() { + mock.OnAnything(m.dao, "Update").Return(nil) + err := m.mgr.Update(nil, &model.Registry{}) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) +} + +func TestManager(t *testing.T) { + suite.Run(t, &managerTestSuite{}) +} diff --git a/src/pkg/replication/dao/dao.go b/src/pkg/replication/dao/dao.go new file mode 100644 index 000000000..6868b7841 --- /dev/null +++ b/src/pkg/replication/dao/dao.go @@ -0,0 +1,127 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "context" + + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" +) + +// DAO defines the DAO operations of replication policy +type DAO interface { + // Count returns the count of replication policies according to the query + Count(ctx context.Context, query *q.Query) (count int64, err error) + // List the replication policies according to the query + List(ctx context.Context, query *q.Query) (policies []*Policy, err error) + // Get the replication policy specified by ID + Get(ctx context.Context, id int64) (policy *Policy, err error) + // Create the replication policy + Create(ctx context.Context, policy *Policy) (id int64, err error) + // Update the specified replication policy + Update(ctx context.Context, policy *Policy, props ...string) (err error) + // Delete the replication policy specified by ID + Delete(ctx context.Context, id int64) (err error) +} + +// NewDAO creates an instance of DAO +func NewDAO() DAO { + return &dao{} +} + +type dao struct{} + +func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) { + qs, err := orm.QuerySetterForCount(ctx, &Policy{}, query) + if err != nil { + return 0, err + } + return qs.Count() +} + +func (d *dao) List(ctx context.Context, query *q.Query) ([]*Policy, error) { + policies := []*Policy{} + qs, err := orm.QuerySetter(ctx, &Policy{}, query) + if err != nil { + return nil, err + } + if _, err = qs.All(&policies); err != nil { + return nil, err + } + return policies, nil +} + +func (d *dao) Get(ctx context.Context, id int64) (*Policy, error) { + policy := &Policy{ + ID: id, + } + ormer, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + if err := ormer.Read(policy); err != nil { + if e := orm.AsNotFoundError(err, "replication policy %d not found", id); e != nil { + err = e + } + return nil, err + } + return policy, nil +} + +func (d *dao) Create(ctx context.Context, policy *Policy) (int64, error) { + ormer, err := orm.FromContext(ctx) + if err != nil { + return 0, err + } + id, err := ormer.Insert(policy) + if e := orm.AsConflictError(err, "replication policy %s already exists", policy.Name); e != nil { + err = e + } + return id, err +} + +func (d *dao) Update(ctx context.Context, policy *Policy, props ...string) error { + ormer, err := orm.FromContext(ctx) + if err != nil { + return err + } + n, err := ormer.Update(policy, props...) + if e := orm.AsConflictError(err, "replication policy %s already exists", policy.Name); e != nil { + err = e + } + if n == 0 { + return errors.NotFoundError(nil).WithMessage("replication policy %d not found", policy.ID) + } + return nil +} + +func (d *dao) Delete(ctx context.Context, id int64) error { + ormer, err := orm.FromContext(ctx) + if err != nil { + return err + } + n, err := ormer.Delete(&Policy{ + ID: id, + }) + if err != nil { + return err + } + if n == 0 { + return errors.NotFoundError(nil).WithMessage("replication policy %d not found", id) + } + return nil +} diff --git a/src/pkg/replication/dao/dao_test.go b/src/pkg/replication/dao/dao_test.go new file mode 100644 index 000000000..8baf6c2be --- /dev/null +++ b/src/pkg/replication/dao/dao_test.go @@ -0,0 +1,157 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "context" + "testing" + + beegoorm "github.com/astaxie/beego/orm" + common_dao "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/stretchr/testify/suite" +) + +type daoTestSuite struct { + suite.Suite + dao DAO + ctx context.Context + id int64 +} + +func (d *daoTestSuite) SetupSuite() { + d.dao = NewDAO() + common_dao.PrepareTestForPostgresSQL() + d.ctx = orm.NewContext(nil, beegoorm.NewOrm()) +} + +func (d *daoTestSuite) SetupTest() { + registry := &Policy{ + Name: "test-rule", + } + id, err := d.dao.Create(d.ctx, registry) + d.Require().Nil(err) + d.id = id +} + +func (d *daoTestSuite) TearDownTest() { + err := d.dao.Delete(d.ctx, d.id) + d.Require().Nil(err) +} + +func (d *daoTestSuite) TestCount() { + // nil query + total, err := d.dao.Count(d.ctx, nil) + d.Require().Nil(err) + d.True(total > 0) + + // query by name + total, err = d.dao.Count(d.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "Name": "test-rule", + }, + }) + d.Require().Nil(err) + d.Equal(int64(1), total) +} + +func (d *daoTestSuite) TestList() { + // nil query + policies, err := d.dao.List(d.ctx, nil) + d.Require().Nil(err) + found := false + for _, policy := range policies { + if policy.ID == d.id { + found = true + break + } + } + d.True(found) + + // query by name + policies, err = d.dao.List(d.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "Name": "test-rule", + }, + }) + d.Require().Nil(err) + d.Require().Equal(1, len(policies)) + d.Equal(d.id, policies[0].ID) +} + +func (d *daoTestSuite) TestGet() { + // get the non-exist policy + _, err := d.dao.Get(d.ctx, 10000) + d.Require().NotNil(err) + d.True(errors.IsErr(err, errors.NotFoundCode)) + + // get the exist policy + policy, err := d.dao.Get(d.ctx, d.id) + d.Require().Nil(err) + d.Require().NotNil(policy) + d.Equal(d.id, policy.ID) +} + +func (d *daoTestSuite) TestCreate() { + // the happy pass case is covered in Setup + + // conflict + policy := &Policy{ + Name: "test-rule", + } + _, err := d.dao.Create(d.ctx, policy) + d.Require().NotNil(err) + d.True(errors.IsErr(err, errors.ConflictCode)) +} + +func (d *daoTestSuite) TestDelete() { + // the happy pass case is covered in TearDown + + // not exist + err := d.dao.Delete(d.ctx, 100021) + d.Require().NotNil(err) + var e *errors.Error + d.Require().True(errors.As(err, &e)) + d.Equal(errors.NotFoundCode, e.Code) +} + +func (d *daoTestSuite) TestUpdate() { + // pass + err := d.dao.Update(d.ctx, &Policy{ + ID: d.id, + Description: "description", + }, "Description") + d.Require().Nil(err) + + policy, err := d.dao.Get(d.ctx, d.id) + d.Require().Nil(err) + d.Require().NotNil(policy) + d.Equal("description", policy.Description) + + // not exist + err = d.dao.Update(d.ctx, &Policy{ + ID: 10000, + }) + d.Require().NotNil(err) + var e *errors.Error + d.Require().True(errors.As(err, &e)) + d.Equal(errors.NotFoundCode, e.Code) +} + +func TestDaoTestSuite(t *testing.T) { + suite.Run(t, &daoTestSuite{}) +} diff --git a/src/pkg/replication/dao/model.go b/src/pkg/replication/dao/model.go new file mode 100644 index 000000000..c6fe1a205 --- /dev/null +++ b/src/pkg/replication/dao/model.go @@ -0,0 +1,48 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "time" + + "github.com/astaxie/beego/orm" +) + +func init() { + orm.RegisterModel(new(Policy)) +} + +// Policy is the model for replication policy +type Policy struct { + ID int64 `orm:"pk;auto;column(id)"` + Name string `orm:"column(name)"` + Description string `orm:"column(description)"` + Creator string `orm:"column(creator)"` + SrcRegistryID int64 `orm:"column(src_registry_id)"` + DestRegistryID int64 `orm:"column(dest_registry_id)"` + DestNamespace string `orm:"column(dest_namespace)"` + Override bool `orm:"column(override)"` + Enabled bool `orm:"column(enabled)"` + Trigger string `orm:"column(trigger)"` + Filters string `orm:"column(filters)"` + ReplicateDeletion bool `orm:"column(replicate_deletion)"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" sort:"default:desc"` + UpdateTime time.Time `orm:"column(update_time);auto_now"` +} + +// TableName set table name for ORM +func (p *Policy) TableName() string { + return "replication_policy" +} diff --git a/src/pkg/replication/manager.go b/src/pkg/replication/manager.go new file mode 100644 index 000000000..73ce4aa5a --- /dev/null +++ b/src/pkg/replication/manager.go @@ -0,0 +1,106 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "context" + + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/replication/dao" +) + +var ( + // Mgr is the global replication policy manager instance + Mgr = NewManager() +) + +// Manager defines the replication policy related operations +type Manager interface { + // Count returns the count of replication policies according to the query + Count(ctx context.Context, query *q.Query) (count int64, err error) + // List replication policies according to the query + List(ctx context.Context, query *q.Query) (policies []*Policy, err error) + // Get the replication policy specified by ID + Get(ctx context.Context, id int64) (policy *Policy, err error) + // Create the replication policy + Create(ctx context.Context, policy *Policy) (id int64, err error) + // Update the specified replication policy + Update(ctx context.Context, policy *Policy, props ...string) (err error) + // Delete the replication policy specified by ID + Delete(ctx context.Context, id int64) (err error) +} + +// NewManager creates an instance of replication policy manager +func NewManager() Manager { + return &manager{ + dao: dao.NewDAO(), + } +} + +type manager struct { + dao dao.DAO +} + +func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) { + return m.dao.Count(ctx, query) +} + +func (m *manager) List(ctx context.Context, query *q.Query) ([]*Policy, error) { + policies, err := m.dao.List(ctx, query) + if err != nil { + return nil, err + } + var result []*Policy + for _, policy := range policies { + p := &Policy{} + if err = p.From(policy); err != nil { + return nil, err + } + result = append(result, p) + } + return result, nil +} + +func (m *manager) Get(ctx context.Context, id int64) (*Policy, error) { + policy, err := m.dao.Get(ctx, id) + if err != nil { + return nil, err + } + p := &Policy{} + if err = p.From(policy); err != nil { + return nil, err + } + return p, nil +} + +func (m *manager) Create(ctx context.Context, policy *Policy) (int64, error) { + p, err := policy.To() + if err != nil { + return 0, err + } + return m.dao.Create(ctx, p) +} + +func (m *manager) Update(ctx context.Context, policy *Policy, props ...string) error { + p, err := policy.To() + if err != nil { + return err + } + return m.dao.Update(ctx, p, props...) +} + +func (m *manager) Delete(ctx context.Context, id int64) error { + return m.dao.Delete(ctx, id) +} diff --git a/src/pkg/replication/manager_test.go b/src/pkg/replication/manager_test.go new file mode 100644 index 000000000..1ad116ba6 --- /dev/null +++ b/src/pkg/replication/manager_test.go @@ -0,0 +1,93 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "testing" + + "github.com/goharbor/harbor/src/pkg/replication/dao" + "github.com/goharbor/harbor/src/testing/mock" + testingdao "github.com/goharbor/harbor/src/testing/pkg/replication/dao" + "github.com/stretchr/testify/suite" +) + +type managerTestSuite struct { + suite.Suite + mgr *manager + dao *testingdao.DAO +} + +func (m *managerTestSuite) SetupTest() { + m.dao = &testingdao.DAO{} + m.mgr = &manager{ + dao: m.dao, + } +} + +func (m *managerTestSuite) TestCount() { + mock.OnAnything(m.dao, "Count").Return(int64(1), nil) + n, err := m.mgr.Count(nil, nil) + m.Require().Nil(err) + m.Equal(int64(1), n) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestList() { + mock.OnAnything(m.dao, "List").Return([]*dao.Policy{ + { + ID: 1, + }, + }, nil) + policies, err := m.mgr.List(nil, nil) + m.Require().Nil(err) + m.Require().Equal(1, len(policies)) + m.Equal(int64(1), policies[0].ID) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestGet() { + mock.OnAnything(m.dao, "Get").Return(&dao.Policy{ + ID: 1, + }, nil) + policy, err := m.mgr.Get(nil, 1) + m.Require().Nil(err) + m.Equal(int64(1), policy.ID) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestCreate() { + mock.OnAnything(m.dao, "Create").Return(int64(1), nil) + _, err := m.mgr.Create(nil, &Policy{}) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestDelete() { + mock.OnAnything(m.dao, "Delete").Return(nil) + err := m.mgr.Delete(nil, 1) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) +} + +func (m *managerTestSuite) TestUpdate() { + mock.OnAnything(m.dao, "Update").Return(nil) + err := m.mgr.Update(nil, &Policy{}) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) +} + +func TestManager(t *testing.T) { + suite.Run(t, &managerTestSuite{}) +} diff --git a/src/pkg/replication/model.go b/src/pkg/replication/model.go new file mode 100644 index 000000000..cc6bbb135 --- /dev/null +++ b/src/pkg/replication/model.go @@ -0,0 +1,350 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "encoding/json" + "fmt" + "github.com/robfig/cron" + "strings" + "time" + + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/replication/dao" + "github.com/goharbor/harbor/src/replication/model" +) + +// Policy defines the structure of a replication policy +type Policy struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Creator string `json:"creator"` + SrcRegistry *model.Registry `json:"src_registry"` + DestRegistry *model.Registry `json:"dest_registry"` + DestNamespace string `json:"dest_namespace"` + Filters []*model.Filter `json:"filters"` + Trigger *model.Trigger `json:"trigger"` + ReplicateDeletion bool `json:"deletion"` + Override bool `json:"override"` + Enabled bool `json:"enabled"` + CreationTime time.Time `json:"creation_time"` + UpdateTime time.Time `json:"update_time"` +} + +// IsScheduledTrigger returns true when the policy is scheduled trigger and enabled +func (p *Policy) IsScheduledTrigger() bool { + if !p.Enabled { + return false + } + if p.Trigger == nil { + return false + } + return p.Trigger.Type == model.TriggerTypeScheduled +} + +// Validate the policy +func (p *Policy) Validate() error { + if len(p.Name) == 0 { + return errors.New(nil).WithCode(errors.BadRequestCode).WithMessage("empty name") + } + var srcRegistryID, dstRegistryID int64 + if p.SrcRegistry != nil { + srcRegistryID = p.SrcRegistry.ID + } + if p.DestRegistry != nil { + dstRegistryID = p.DestRegistry.ID + } + + // one of the source registry and destination registry must be Harbor itself + if srcRegistryID != 0 && dstRegistryID != 0 || + srcRegistryID == 0 && dstRegistryID == 0 { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("either src_registry or dest_registry should be empty and the other one shouldn't be empty") + } + + // valid the filters + for _, filter := range p.Filters { + switch filter.Type { + case model.FilterTypeResource, model.FilterTypeName, model.FilterTypeTag: + value, ok := filter.Value.(string) + if !ok { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("the type of filter value isn't string") + } + if filter.Type == model.FilterTypeResource { + rt := model.ResourceType(value) + if !(rt == model.ResourceTypeArtifact || rt == model.ResourceTypeImage || rt == model.ResourceTypeChart) { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("invalid resource filter: %s", value) + } + } + case model.FilterTypeLabel: + labels, ok := filter.Value.([]interface{}) + if !ok { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("the type of label filter value isn't string slice") + } + for _, label := range labels { + _, ok := label.(string) + if !ok { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("the type of label filter value isn't string slice") + } + } + default: + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("invalid filter type") + } + } + + // valid trigger + if p.Trigger != nil { + switch p.Trigger.Type { + case model.TriggerTypeManual, model.TriggerTypeEventBased: + case model.TriggerTypeScheduled: + if p.Trigger.Settings == nil || len(p.Trigger.Settings.Cron) == 0 { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("the cron string cannot be empty when the trigger type is %s", model.TriggerTypeScheduled) + } + if _, err := cron.Parse(p.Trigger.Settings.Cron); err != nil { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("invalid cron string for scheduled trigger: %s", p.Trigger.Settings.Cron) + } + default: + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("invalid trigger type") + } + } + return nil +} + +// From converts the DAO model into the Policy +func (p *Policy) From(policy *dao.Policy) error { + if policy == nil { + return nil + } + p.ID = policy.ID + p.Name = policy.Name + p.Description = policy.Description + p.Creator = policy.Creator + p.DestNamespace = policy.DestNamespace + p.ReplicateDeletion = policy.ReplicateDeletion + p.Override = policy.Override + p.Enabled = policy.Enabled + p.CreationTime = policy.CreationTime + p.UpdateTime = policy.UpdateTime + + if policy.SrcRegistryID > 0 { + p.SrcRegistry = &model.Registry{ + ID: policy.SrcRegistryID, + } + } + if policy.DestRegistryID > 0 { + p.DestRegistry = &model.Registry{ + ID: policy.DestRegistryID, + } + } + + // parse Filters + filters, err := parseFilters(policy.Filters) + if err != nil { + return err + } + p.Filters = filters + + // parse Trigger + trigger, err := parseTrigger(policy.Trigger) + if err != nil { + return err + } + p.Trigger = trigger + + return nil +} + +// To converts to DAO model +func (p *Policy) To() (*dao.Policy, error) { + policy := &dao.Policy{ + ID: p.ID, + Name: p.Name, + Description: p.Description, + Creator: p.Creator, + DestNamespace: p.DestNamespace, + Override: p.Override, + Enabled: p.Enabled, + ReplicateDeletion: p.ReplicateDeletion, + CreationTime: p.CreationTime, + UpdateTime: p.UpdateTime, + } + if p.SrcRegistry != nil { + policy.SrcRegistryID = p.SrcRegistry.ID + } + if p.DestRegistry != nil { + policy.DestRegistryID = p.DestRegistry.ID + } + + if p.Trigger != nil { + trigger, err := json.Marshal(p.Trigger) + if err != nil { + return nil, err + } + policy.Trigger = string(trigger) + } + + if len(p.Filters) > 0 { + filters, err := json.Marshal(p.Filters) + if err != nil { + return nil, err + } + policy.Filters = string(filters) + } + + return policy, nil +} + +type filter struct { + Type model.FilterType `json:"type"` + Value interface{} `json:"value"` + Kind string `json:"kind"` + Pattern string `json:"pattern"` +} + +type trigger struct { + Type model.TriggerType `json:"type"` + Settings *model.TriggerSettings `json:"trigger_settings"` + Kind string `json:"kind"` + ScheduleParam *scheduleParam `json:"schedule_param"` +} + +type scheduleParam struct { + Type string `json:"type"` + Weekday int8 `json:"weekday"` + Offtime int64 `json:"offtime"` +} + +func parseFilters(str string) ([]*model.Filter, error) { + if len(str) == 0 { + return nil, nil + } + items := []*filter{} + if err := json.Unmarshal([]byte(str), &items); err != nil { + return nil, err + } + + filters := []*model.Filter{} + for _, item := range items { + filter := &model.Filter{ + Type: item.Type, + Value: item.Value, + } + // keep backwards compatibility + if len(filter.Type) == 0 { + if filter.Value == nil { + filter.Value = item.Pattern + } + switch item.Kind { + case "repository": + // a name filter "project_name/**" must exist after running upgrade + // if there is any repository filter, merge it into the name filter + repository, ok := filter.Value.(string) + if ok && len(repository) > 0 { + for _, item := range items { + if item.Type == model.FilterTypeName { + name, ok := item.Value.(string) + if ok && len(name) > 0 { + item.Value = strings.Replace(name, "**", repository, 1) + } + break + } + } + } + continue + case "tag": + filter.Type = model.FilterTypeTag + case "label": + // drop all legend label filters + continue + default: + log.Warningf("unknown filter type: %s", filter.Type) + continue + } + } + + // convert the type of value from string to model.ResourceType if the filter + // is a resource type filter + if filter.Type == model.FilterTypeResource { + filter.Value = (model.ResourceType)(filter.Value.(string)) + } + if filter.Type == model.FilterTypeLabel { + labels := []string{} + for _, label := range filter.Value.([]interface{}) { + labels = append(labels, label.(string)) + } + filter.Value = labels + } + filters = append(filters, filter) + } + return filters, nil +} + +func parseTrigger(str string) (*model.Trigger, error) { + if len(str) == 0 { + return nil, nil + } + item := &trigger{} + if err := json.Unmarshal([]byte(str), item); err != nil { + return nil, err + } + trigger := &model.Trigger{ + Type: item.Type, + Settings: item.Settings, + } + // keep backwards compatibility + if len(trigger.Type) == 0 { + switch item.Kind { + case "Manual": + trigger.Type = model.TriggerTypeManual + case "Immediate": + trigger.Type = model.TriggerTypeEventBased + case "Scheduled": + trigger.Type = model.TriggerTypeScheduled + trigger.Settings = &model.TriggerSettings{ + Cron: parseScheduleParamToCron(item.ScheduleParam), + } + default: + log.Warningf("unknown trigger type: %s", item.Kind) + return nil, nil + } + } + return trigger, nil +} + +func parseScheduleParamToCron(param *scheduleParam) string { + if param == nil { + return "" + } + offtime := param.Offtime + offtime = offtime % (3600 * 24) + hour := int(offtime / 3600) + offtime = offtime % 3600 + minute := int(offtime / 60) + second := int(offtime % 60) + if param.Type == "Weekly" { + return fmt.Sprintf("%d %d %d * * %d", second, minute, hour, param.Weekday%7) + } + return fmt.Sprintf("%d %d %d * * *", second, minute, hour) +} diff --git a/src/pkg/replication/model_test.go b/src/pkg/replication/model_test.go new file mode 100644 index 000000000..c4b942d5e --- /dev/null +++ b/src/pkg/replication/model_test.go @@ -0,0 +1,252 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "testing" + + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" +) + +func TestIsScheduledTrigger(t *testing.T) { + assert := assert.New(t) + + // policy is disabled + policy := &Policy{ + Enabled: false, + } + b := policy.IsScheduledTrigger() + assert.False(b) + + // no trigger + policy = &Policy{ + Enabled: true, + } + b = policy.IsScheduledTrigger() + assert.False(b) + + // isn't scheduled trigger + policy = &Policy{ + Trigger: &model.Trigger{ + Type: model.TriggerTypeEventBased, + }, + Enabled: true, + } + b = policy.IsScheduledTrigger() + assert.False(b) + + // scheduled trigger + policy = &Policy{ + Trigger: &model.Trigger{ + Type: model.TriggerTypeScheduled, + }, + Enabled: true, + } + b = policy.IsScheduledTrigger() + assert.True(b) +} + +func TestValidate(t *testing.T) { + assert := assert.New(t) + + // empty name + policy := &Policy{} + err := policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // empty source registry and destination registry + policy = &Policy{ + Name: "policy01", + } + err = policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // source registry and destination registry both not empty + policy = &Policy{ + Name: "policy01", + SrcRegistry: &model.Registry{ + ID: 1, + }, + DestRegistry: &model.Registry{ + ID: 2, + }, + } + err = policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // invalid filter + policy = &Policy{ + Name: "policy01", + SrcRegistry: &model.Registry{ + ID: 0, + }, + DestRegistry: &model.Registry{ + ID: 1, + }, + Filters: []*model.Filter{ + { + Type: "invalid_type", + }, + }, + } + err = policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // invalid filter + policy = &Policy{ + Name: "policy01", + SrcRegistry: &model.Registry{ + ID: 0, + }, + DestRegistry: &model.Registry{ + ID: 1, + }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeResource, + Value: "invalid_resource_type", + }, + }, + } + err = policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // invalid filter + policy = &Policy{ + Name: "policy01", + SrcRegistry: &model.Registry{ + ID: 0, + }, + DestRegistry: &model.Registry{ + ID: 1, + }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeResource, + Value: model.ResourceTypeImage, + }, + { + Type: model.FilterTypeTag, + Value: "", + }, + }, + } + err = policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // invalid trigger + policy = &Policy{ + Name: "policy01", + SrcRegistry: &model.Registry{ + ID: 0, + }, + DestRegistry: &model.Registry{ + ID: 1, + }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "library", + }, + }, + Trigger: &model.Trigger{ + Type: "invalid_type", + }, + } + err = policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // invalid trigger + policy = &Policy{ + Name: "policy01", + SrcRegistry: &model.Registry{ + ID: 0, + }, + DestRegistry: &model.Registry{ + ID: 1, + }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "library", + }, + }, + Trigger: &model.Trigger{ + Type: model.TriggerTypeScheduled, + }, + } + err = policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // invalid cron + policy = &Policy{ + Name: "policy01", + SrcRegistry: &model.Registry{ + ID: 0, + }, + DestRegistry: &model.Registry{ + ID: 1, + }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeResource, + Value: "image", + }, + { + Type: model.FilterTypeName, + Value: "library/**", + }, + }, + Trigger: &model.Trigger{ + Type: model.TriggerTypeScheduled, + Settings: &model.TriggerSettings{ + Cron: "* * *", + }, + }, + } + err = policy.Validate() + assert.True(errors.IsErr(err, errors.BadRequestCode)) + + // pass + policy = &Policy{ + Name: "policy01", + SrcRegistry: &model.Registry{ + ID: 0, + }, + DestRegistry: &model.Registry{ + ID: 1, + }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeResource, + Value: "image", + }, + { + Type: model.FilterTypeName, + Value: "library/**", + }, + }, + Trigger: &model.Trigger{ + Type: model.TriggerTypeScheduled, + Settings: &model.TriggerSettings{ + Cron: "* * * * * *", + }, + }, + } + err = policy.Validate() + assert.Nil(err) +} diff --git a/src/pkg/task/execution.go b/src/pkg/task/execution.go index 24a6a7df7..8d3508717 100644 --- a/src/pkg/task/execution.go +++ b/src/pkg/task/execution.go @@ -63,6 +63,9 @@ type ExecutionManager interface { StopAndWait(ctx context.Context, id int64, timeout time.Duration) (err error) // Delete the specified execution and its tasks Delete(ctx context.Context, id int64) (err error) + // Delete all executions and tasks of the specific vendor. They can be deleted only when all the executions/tasks + // of the vendor are in final status + DeleteByVendor(ctx context.Context, vendorType string, vendorID int64) (err error) // Get the specified execution Get(ctx context.Context, id int64) (execution *Execution, err error) // List executions according to the query @@ -334,6 +337,34 @@ func (e *executionManager) Delete(ctx context.Context, id int64) error { return e.executionDAO.Delete(ctx, id) } +func (e *executionManager) DeleteByVendor(ctx context.Context, vendorType string, vendorID int64) error { + executions, err := e.executionDAO.List(ctx, &q.Query{ + Keywords: map[string]interface{}{ + "VendorType": vendorType, + "VendorID": vendorID, + }}) + if err != nil { + return err + } + // check the status + for _, execution := range executions { + if !job.Status(execution.Status).Final() { + return errors.New(nil).WithCode(errors.PreconditionCode). + WithMessage("contains executions that aren't in final status, stop the execution first") + } + } + // delete the executions + for _, execution := range executions { + if err = e.Delete(ctx, execution.ID); err != nil { + if errors.IsNotFoundErr(err) { + continue + } + return err + } + } + return nil +} + func (e *executionManager) Get(ctx context.Context, id int64) (*Execution, error) { execution, err := e.executionDAO.Get(ctx, id) if err != nil { diff --git a/src/replication/dao/models/base.go b/src/replication/dao/models/base.go index 6d34967a4..93777ea5d 100644 --- a/src/replication/dao/models/base.go +++ b/src/replication/dao/models/base.go @@ -6,6 +6,5 @@ import ( func init() { orm.RegisterModel( - new(Registry), - new(RepPolicy)) + new(Registry)) } diff --git a/src/replication/dao/models/policy.go b/src/replication/dao/models/policy.go deleted file mode 100644 index 6f96e1c17..000000000 --- a/src/replication/dao/models/policy.go +++ /dev/null @@ -1,26 +0,0 @@ -package models - -import "time" - -// RepPolicy is the model for a ng replication policy. -type RepPolicy struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - Name string `orm:"column(name)" json:"name"` - Description string `orm:"column(description)" json:"description"` - Creator string `orm:"column(creator)" json:"creator"` - SrcRegistryID int64 `orm:"column(src_registry_id)" json:"src_registry_id"` - DestRegistryID int64 `orm:"column(dest_registry_id)" json:"dest_registry_id"` - DestNamespace string `orm:"column(dest_namespace)" json:"dest_namespace"` - Override bool `orm:"column(override)" json:"override"` - Enabled bool `orm:"column(enabled)" json:"enabled"` - Trigger string `orm:"column(trigger)" json:"trigger"` - Filters string `orm:"column(filters)" json:"filters"` - ReplicateDeletion bool `orm:"column(replicate_deletion)" json:"replicate_deletion"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` -} - -// TableName set table name for ORM. -func (r *RepPolicy) TableName() string { - return "replication_policy" -} diff --git a/src/replication/dao/policy.go b/src/replication/dao/policy.go deleted file mode 100644 index 995d9298e..000000000 --- a/src/replication/dao/policy.go +++ /dev/null @@ -1,115 +0,0 @@ -package dao - -import ( - "errors" - "time" - - "github.com/astaxie/beego/orm" - common_dao "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" -) - -// AddRepPolicy insert new policy to DB. -func AddRepPolicy(policy *models.RepPolicy) (int64, error) { - o := common_dao.GetOrmer() - now := time.Now() - policy.CreationTime = now - policy.UpdateTime = now - - return o.Insert(policy) -} - -// GetPolicies list polices with given query parameters. -func GetPolicies(queries ...*model.PolicyQuery) (int64, []*models.RepPolicy, error) { - var qs = common_dao.GetOrmer().QueryTable(new(models.RepPolicy)) - var policies []*models.RepPolicy - - if len(queries) == 0 { - total, err := qs.Count() - if err != nil { - return -1, nil, err - } - - _, err = qs.All(&policies) - if err != nil { - return total, nil, err - } - - return total, policies, nil - } - - query := queries[0] - if len(query.Name) != 0 { - qs = qs.Filter("Name__icontains", common_dao.Escape(query.Name)) - } - if len(query.Namespace) != 0 { - // TODO: Namespace filter not implemented yet - } - if query.SrcRegistry > 0 { - qs = qs.Filter("SrcRegistryID__exact", query.SrcRegistry) - } - if query.DestRegistry > 0 { - qs = qs.Filter("DestRegistryID__exact", query.DestRegistry) - } - - total, err := qs.Count() - if err != nil { - return -1, nil, err - } - - if query.Page > 0 && query.Size > 0 { - qs = qs.Limit(query.Size, (query.Page-1)*query.Size) - } - _, err = qs.OrderBy("-CreationTime").All(&policies) - if err != nil { - return total, nil, err - } - - return total, policies, nil -} - -// GetRepPolicy return special policy by id. -func GetRepPolicy(id int64) (policy *models.RepPolicy, err error) { - policy = new(models.RepPolicy) - err = common_dao.GetOrmer().QueryTable(policy). - Filter("id", id).One(policy) - if err == orm.ErrNoRows { - return nil, nil - } - - return -} - -// GetRepPolicyByName return special policy by name. -func GetRepPolicyByName(name string) (policy *models.RepPolicy, err error) { - policy = new(models.RepPolicy) - err = common_dao.GetOrmer().QueryTable(policy). - Filter("name", name).One(policy) - if err == orm.ErrNoRows { - return nil, nil - } - - return -} - -// UpdateRepPolicy update fields by props -func UpdateRepPolicy(policy *models.RepPolicy, props ...string) (err error) { - var o = common_dao.GetOrmer() - - if policy != nil { - _, err = o.Update(policy, props...) - } else { - err = errors.New("Nil policy") - } - - return -} - -// DeleteRepPolicy will hard delete database item -func DeleteRepPolicy(id int64) error { - o := common_dao.GetOrmer() - - _, err := o.Delete(&models.RepPolicy{ID: id}) - return err -} diff --git a/src/replication/dao/policy_test.go b/src/replication/dao/policy_test.go deleted file mode 100644 index a9c01b377..000000000 --- a/src/replication/dao/policy_test.go +++ /dev/null @@ -1,284 +0,0 @@ -package dao - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var ( - testPolic1 = &models.RepPolicy{ - // ID: 999, - Name: "Policy Test 1", - Description: "Policy Description", - Creator: "someone", - SrcRegistryID: 123, - DestRegistryID: 456, - DestNamespace: "target_ns", - ReplicateDeletion: true, - Override: true, - Enabled: true, - Trigger: "{\"type\":\"\",\"trigger_settings\":null}", - Filters: "[{\"type\":\"registry\",\"value\":\"abc\"}]", - } - - testPolic2 = &models.RepPolicy{ - // ID: 999, - Name: "Policy Test 2", - Description: "Policy Description", - Creator: "someone", - SrcRegistryID: 123, - DestRegistryID: 456, - DestNamespace: "target_ns", - ReplicateDeletion: true, - Override: true, - Enabled: true, - Trigger: "{\"type\":\"\",\"trigger_settings\":null}", - Filters: "[{\"type\":\"registry\",\"value\":\"abc\"}]", - } - - testPolic3 = &models.RepPolicy{ - // ID: 999, - Name: "Policy Test 3", - Description: "Policy Description", - Creator: "someone", - SrcRegistryID: 123, - DestRegistryID: 456, - DestNamespace: "target_ns", - ReplicateDeletion: true, - Override: true, - Enabled: true, - Trigger: "{\"type\":\"\",\"trigger_settings\":null}", - Filters: "[{\"type\":\"registry\",\"value\":\"abc\"}]", - } -) - -func TestAddRepPolicy(t *testing.T) { - tests := []struct { - name string - policy *models.RepPolicy - want int64 - wantErr bool - }{ - {name: "AddRepPolicy 1", policy: testPolic1, want: 1}, - {name: "AddRepPolicy 2", policy: testPolic2, want: 2}, - {name: "AddRepPolicy 3", policy: testPolic3, want: 3}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := AddRepPolicy(tt.policy) - - if tt.wantErr { - require.NotNil(t, err, "wantErr: %s", err) - return - } - - require.Nil(t, err) - assert.Equal(t, tt.want, got) - }) - } -} - -func TestGetPolicies(t *testing.T) { - type args struct { - name string - namespace string - page int64 - pageSize int64 - } - tests := []struct { - name string - args args - wantPolicies []*models.RepPolicy - wantErr bool - }{ - {name: "GetTotalOfRepPolicies nil", args: args{name: "Test 0"}, wantPolicies: []*models.RepPolicy{}}, - {name: "GetTotalOfRepPolicies 1", args: args{name: "Test 1"}, wantPolicies: []*models.RepPolicy{testPolic1}}, - {name: "GetTotalOfRepPolicies 2", args: args{name: "Test", page: 1, pageSize: 2}, wantPolicies: []*models.RepPolicy{testPolic3, testPolic2}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, gotPolicies, err := GetPolicies([]*model.PolicyQuery{ - { - Name: tt.args.name, - Namespace: tt.args.namespace, - Page: tt.args.page, - Size: tt.args.pageSize, - }, - }...) - if tt.wantErr { - require.NotNil(t, err, "wantErr: %s", err) - return - } - - require.Nil(t, err) - for i, gotPolicy := range gotPolicies { - assert.Equal(t, tt.wantPolicies[i].Name, gotPolicy.Name) - assert.Equal(t, tt.wantPolicies[i].Description, gotPolicy.Description) - assert.Equal(t, tt.wantPolicies[i].Creator, gotPolicy.Creator) - assert.Equal(t, tt.wantPolicies[i].SrcRegistryID, gotPolicy.SrcRegistryID) - assert.Equal(t, tt.wantPolicies[i].DestRegistryID, gotPolicy.DestRegistryID) - assert.Equal(t, tt.wantPolicies[i].DestNamespace, gotPolicy.DestNamespace) - assert.Equal(t, tt.wantPolicies[i].ReplicateDeletion, gotPolicy.ReplicateDeletion) - assert.Equal(t, tt.wantPolicies[i].Override, gotPolicy.Override) - assert.Equal(t, tt.wantPolicies[i].Enabled, gotPolicy.Enabled) - assert.Equal(t, tt.wantPolicies[i].Trigger, gotPolicy.Trigger) - assert.Equal(t, tt.wantPolicies[i].Filters, gotPolicy.Filters) - } - }) - } -} - -func TestGetRepPolicy(t *testing.T) { - tests := []struct { - name string - id int64 - wantPolicy *models.RepPolicy - wantErr bool - }{ - {name: "GetRepPolicy 1", id: 1, wantPolicy: testPolic1}, - {name: "GetRepPolicy 2", id: 2, wantPolicy: testPolic2}, - {name: "GetRepPolicy 3", id: 3, wantPolicy: testPolic3}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotPolicy, err := GetRepPolicy(tt.id) - if tt.wantErr { - require.NotNil(t, err, "wantErr: %s", err) - return - } - - require.Nil(t, err) - assert.Equal(t, tt.wantPolicy.Name, gotPolicy.Name) - assert.Equal(t, tt.wantPolicy.Description, gotPolicy.Description) - assert.Equal(t, tt.wantPolicy.Creator, gotPolicy.Creator) - assert.Equal(t, tt.wantPolicy.SrcRegistryID, gotPolicy.SrcRegistryID) - assert.Equal(t, tt.wantPolicy.DestRegistryID, gotPolicy.DestRegistryID) - assert.Equal(t, tt.wantPolicy.DestNamespace, gotPolicy.DestNamespace) - assert.Equal(t, tt.wantPolicy.ReplicateDeletion, gotPolicy.ReplicateDeletion) - assert.Equal(t, tt.wantPolicy.Override, gotPolicy.Override) - assert.Equal(t, tt.wantPolicy.Enabled, gotPolicy.Enabled) - assert.Equal(t, tt.wantPolicy.Trigger, gotPolicy.Trigger) - assert.Equal(t, tt.wantPolicy.Filters, gotPolicy.Filters) - }) - } -} - -func TestGetRepPolicyByName(t *testing.T) { - type args struct { - name string - } - tests := []struct { - name string - args args - wantPolicy *models.RepPolicy - wantErr bool - }{ - {name: "GetRepPolicyByName 1", args: args{name: testPolic1.Name}, wantPolicy: testPolic1}, - {name: "GetRepPolicyByName 2", args: args{name: testPolic2.Name}, wantPolicy: testPolic2}, - {name: "GetRepPolicyByName 3", args: args{name: testPolic3.Name}, wantPolicy: testPolic3}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotPolicy, err := GetRepPolicyByName(tt.args.name) - if tt.wantErr { - require.NotNil(t, err, "wantErr: %s", err) - return - } - - require.Nil(t, err) - assert.Equal(t, tt.wantPolicy.Name, gotPolicy.Name) - assert.Equal(t, tt.wantPolicy.Description, gotPolicy.Description) - assert.Equal(t, tt.wantPolicy.Creator, gotPolicy.Creator) - assert.Equal(t, tt.wantPolicy.SrcRegistryID, gotPolicy.SrcRegistryID) - assert.Equal(t, tt.wantPolicy.DestRegistryID, gotPolicy.DestRegistryID) - assert.Equal(t, tt.wantPolicy.DestNamespace, gotPolicy.DestNamespace) - assert.Equal(t, tt.wantPolicy.ReplicateDeletion, gotPolicy.ReplicateDeletion) - assert.Equal(t, tt.wantPolicy.Override, gotPolicy.Override) - assert.Equal(t, tt.wantPolicy.Enabled, gotPolicy.Enabled) - assert.Equal(t, tt.wantPolicy.Trigger, gotPolicy.Trigger) - assert.Equal(t, tt.wantPolicy.Filters, gotPolicy.Filters) - }) - } -} - -func TestUpdateRepPolicy(t *testing.T) { - type args struct { - policy *models.RepPolicy - props []string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {name: "UpdateRepPolicy Want Error", args: args{policy: nil}, wantErr: true}, - { - name: "UpdateRepPolicy 1", - args: args{ - policy: &models.RepPolicy{ID: 1, Description: "Policy Description 1", Creator: "Someone 1"}, - props: []string{"description", "creator"}, - }, - }, - { - name: "UpdateRepPolicy 2", - args: args{ - policy: &models.RepPolicy{ID: 2, Description: "Policy Description 2", Creator: "Someone 2"}, - props: []string{"description", "creator"}, - }, - }, - { - name: "UpdateRepPolicy 3", - args: args{ - policy: &models.RepPolicy{ID: 3, Description: "Policy Description 3", Creator: "Someone 3"}, - props: []string{"description", "creator"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := UpdateRepPolicy(tt.args.policy, tt.args.props...) - - if tt.wantErr { - require.NotNil(t, err, "Error: %s", err) - return - } - - require.Nil(t, err) - gotPolicy, err := GetRepPolicy(tt.args.policy.ID) - - require.Nil(t, err) - assert.Equal(t, tt.args.policy.Description, gotPolicy.Description) - assert.Equal(t, tt.args.policy.Creator, gotPolicy.Creator) - }) - } -} - -func TestDeleteRepPolicy(t *testing.T) { - tests := []struct { - name string - id int64 - wantErr bool - }{ - {name: "DeleteRepPolicy 1", id: 1, wantErr: false}, - {name: "DeleteRepPolicy 2", id: 2, wantErr: false}, - {name: "DeleteRepPolicy 3", id: 3, wantErr: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := DeleteRepPolicy(tt.id) - if tt.wantErr { - require.NotNil(t, err, "wantErr: %s", err) - return - } - - require.Nil(t, err) - policy, err := GetRepPolicy(tt.id) - require.Nil(t, err) - assert.Nil(t, policy) - }) - } -} diff --git a/src/replication/event/handler.go b/src/replication/event/handler.go index f90c6ddb5..2b1accc02 100644 --- a/src/replication/event/handler.go +++ b/src/replication/event/handler.go @@ -18,15 +18,13 @@ import ( "errors" "fmt" - commonthttp "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/controller/replication" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/orm" + rep "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/pkg/task" - "github.com/goharbor/harbor/src/replication/config" "github.com/goharbor/harbor/src/replication/filter" "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/policy" "github.com/goharbor/harbor/src/replication/registry" ) @@ -36,16 +34,14 @@ type Handler interface { } // NewHandler ... -func NewHandler(policyCtl policy.Controller, registryMgr registry.Manager) Handler { +func NewHandler(registryMgr registry.Manager) Handler { return &handler{ - policyCtl: policyCtl, registryMgr: registryMgr, ctl: replication.Ctl, } } type handler struct { - policyCtl policy.Controller registryMgr registry.Manager ctl replication.Controller } @@ -56,7 +52,7 @@ func (h *handler) Handle(event *Event) error { len(event.Resource.Metadata.Artifacts) == 0 { return errors.New("invalid event") } - var policies []*model.Policy + var policies []*rep.Policy var err error switch event.Type { case EventTypeArtifactPush, EventTypeChartUpload, EventTypeTagDelete, @@ -75,9 +71,6 @@ func (h *handler) Handle(event *Event) error { } for _, policy := range policies { - if err := PopulateRegistries(h.registryMgr, policy); err != nil { - return err - } id, err := h.ctl.Start(orm.Context(), policy, event.Resource, task.ExecutionTriggerEvent) if err != nil { return err @@ -87,12 +80,12 @@ func (h *handler) Handle(event *Event) error { return nil } -func (h *handler) getRelatedPolicies(resource *model.Resource) ([]*model.Policy, error) { - _, policies, err := h.policyCtl.List() +func (h *handler) getRelatedPolicies(resource *model.Resource) ([]*rep.Policy, error) { + policies, err := replication.Ctl.ListPolicies(orm.Context(), nil) if err != nil { return nil, err } - result := []*model.Policy{} + result := []*rep.Policy{} for _, policy := range policies { // disabled if !policy.Enabled { @@ -112,7 +105,7 @@ func (h *handler) getRelatedPolicies(resource *model.Resource) ([]*model.Policy, continue } // doesn't replicate deletion - if resource.Deleted && !policy.Deletion { + if resource.Deleted && !policy.ReplicateDeletion { continue } @@ -129,52 +122,3 @@ func (h *handler) getRelatedPolicies(resource *model.Resource) ([]*model.Policy, } return result, nil } - -// PopulateRegistries populates the source registry and destination registry properties for policy -func PopulateRegistries(registryMgr registry.Manager, policy *model.Policy) error { - if policy == nil { - return nil - } - registry, err := getRegistry(registryMgr, policy.SrcRegistry) - if err != nil { - return err - } - policy.SrcRegistry = registry - registry, err = getRegistry(registryMgr, policy.DestRegistry) - if err != nil { - return err - } - policy.DestRegistry = registry - return nil -} - -func getRegistry(registryMgr registry.Manager, registry *model.Registry) (*model.Registry, error) { - if registry == nil || registry.ID == 0 { - return GetLocalRegistry(), nil - } - reg, err := registryMgr.Get(registry.ID) - if err != nil { - return nil, err - } - if reg == nil { - return nil, fmt.Errorf("registry %d not found", registry.ID) - } - return reg, nil -} - -// GetLocalRegistry returns the info of the local Harbor registry -func GetLocalRegistry() *model.Registry { - return &model.Registry{ - Type: model.RegistryTypeHarbor, - Name: "Local", - URL: config.Config.CoreURL, - TokenServiceURL: config.Config.TokenServiceURL, - Status: "healthy", - Credential: &model.Credential{ - Type: model.CredentialTypeSecret, - // use secret to do the auth for the local Harbor - AccessSecret: config.Config.JobserviceSecret, - }, - Insecure: !commonthttp.InternalTLSEnabled(), - } -} diff --git a/src/replication/event/handler_test.go b/src/replication/event/handler_test.go deleted file mode 100644 index da8353c31..000000000 --- a/src/replication/event/handler_test.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package event - -import ( - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/testing/controller/replication" - "github.com/goharbor/harbor/src/testing/mock" - "testing" - - "github.com/goharbor/harbor/src/lib/q" - "github.com/goharbor/harbor/src/replication/config" - "github.com/goharbor/harbor/src/replication/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type fakedPolicyController struct{} - -func (f *fakedPolicyController) Create(*model.Policy) (int64, error) { - return 0, nil -} -func (f *fakedPolicyController) List(...*model.PolicyQuery) (int64, []*model.Policy, error) { - polices := []*model.Policy{ - { - ID: 1, - Enabled: true, - Deletion: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeEventBased, - }, - Filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "test/*", - }, - }, - DestRegistry: &model.Registry{ - ID: 1, - }, - }, - // nil trigger - { - ID: 2, - Enabled: true, - Deletion: true, - Trigger: nil, - Filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "library/*", - }, - }, - DestRegistry: &model.Registry{ - ID: 1, - }, - }, - // doesn't replicate deletion - { - ID: 3, - Enabled: true, - Deletion: false, - Trigger: &model.Trigger{ - Type: model.TriggerTypeEventBased, - }, - Filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "library/*", - }, - }, - DestRegistry: &model.Registry{ - ID: 1, - }, - }, - // replicate deletion - { - ID: 4, - Enabled: true, - Deletion: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeEventBased, - }, - Filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "library/*", - }, - }, - DestRegistry: &model.Registry{ - ID: 1, - }, - }, - // disabled - { - ID: 5, - Enabled: false, - Deletion: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeEventBased, - }, - Filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "library/*", - }, - }, - DestRegistry: &model.Registry{ - ID: 1, - }, - }, - // the source registry is not local Harbor - { - ID: 6, - Enabled: true, - Deletion: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeEventBased, - }, - Filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "library/*", - }, - }, - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, - } - return int64(len(polices)), polices, nil -} -func (f *fakedPolicyController) Get(id int64) (*model.Policy, error) { - return nil, nil -} -func (f *fakedPolicyController) GetByName(name string) (*model.Policy, error) { - return nil, nil -} -func (f *fakedPolicyController) Update(*model.Policy) error { - return nil -} -func (f *fakedPolicyController) Remove(int64) error { - return nil -} - -type fakedRegistryManager struct{} - -func (f *fakedRegistryManager) Add(*model.Registry) (int64, error) { - return 0, nil -} -func (f *fakedRegistryManager) List(query *q.Query) (int64, []*model.Registry, error) { - return 0, nil, nil -} -func (f *fakedRegistryManager) Get(id int64) (*model.Registry, error) { - return &model.Registry{ - ID: 1, - Type: model.RegistryTypeHarbor, - }, nil -} -func (f *fakedRegistryManager) GetByName(name string) (*model.Registry, error) { - return nil, nil -} -func (f *fakedRegistryManager) Update(*model.Registry, ...string) error { - return nil -} -func (f *fakedRegistryManager) Remove(int64) error { - return nil -} -func (f *fakedRegistryManager) HealthCheck() error { - return nil -} -func TestGetRelatedPolicies(t *testing.T) { - handler := &handler{ - policyCtl: &fakedPolicyController{}, - } - policies, err := handler.getRelatedPolicies(&model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Artifacts: []*model.Artifact{ - { - Type: "image", - Digest: "sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042", - Tags: []string{"latest"}, - }, - }, - }, - }) - require.Nil(t, err) - assert.Equal(t, 2, len(policies)) - assert.Equal(t, int64(3), policies[0].ID) - assert.Equal(t, int64(4), policies[1].ID) - - policies, err = handler.getRelatedPolicies(&model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Artifacts: []*model.Artifact{ - { - Type: "image", - Digest: "sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042", - Tags: []string{"latest"}, - }, - }, - }, - Deleted: true, - }) - require.Nil(t, err) - assert.Equal(t, 1, len(policies)) - assert.Equal(t, int64(4), policies[0].ID) -} - -func TestHandle(t *testing.T) { - dao.PrepareTestForPostgresSQL() - config.Config = &config.Configuration{} - ctl := &replication.Controller{} - ctl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) - handler := &handler{ - policyCtl: &fakedPolicyController{}, - registryMgr: &fakedRegistryManager{}, - ctl: ctl, - } - - // nil event - err := handler.Handle(nil) - require.NotNil(t, err) - - // nil vtags - err = handler.Handle(&Event{ - Resource: &model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{}, - }, - }, - Type: EventTypeArtifactPush, - }) - require.NotNil(t, err) - - // unsupported event type - err = handler.Handle(&Event{ - Resource: &model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"latest"}, - }, - }, - Type: "unsupported", - }) - require.NotNil(t, err) - - // push image - err = handler.Handle(&Event{ - Resource: &model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Artifacts: []*model.Artifact{ - { - Tags: []string{"latest"}, - }, - }, - }, - }, - Type: EventTypeArtifactPush, - }) - require.Nil(t, err) - - // delete image - err = handler.Handle(&Event{ - Resource: &model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Artifacts: []*model.Artifact{ - { - Tags: []string{"latest"}, - }, - }, - }, - }, - Type: EventTypeArtifactDelete, - }) - require.Nil(t, err) -} diff --git a/src/replication/model/policy.go b/src/replication/model/policy.go index f27ea0b65..ef3614bdc 100644 --- a/src/replication/model/policy.go +++ b/src/replication/model/policy.go @@ -14,13 +14,6 @@ package model -import ( - "fmt" - "github.com/astaxie/beego/validation" - "github.com/robfig/cron" - "time" -) - // const definition const ( FilterTypeResource FilterType = "resource" @@ -33,109 +26,6 @@ const ( TriggerTypeEventBased TriggerType = "event_based" ) -// Policy defines the structure of a replication policy -type Policy struct { - ID int64 `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - Creator string `json:"creator"` - // source - SrcRegistry *Registry `json:"src_registry"` - // destination - DestRegistry *Registry `json:"dest_registry"` - // Only support two dest namespace modes: - // Put all the src resources to the one single dest namespace - // or keep namespaces same with the source ones (under this case, - // the DestNamespace should be set to empty) - DestNamespace string `json:"dest_namespace"` - // Filters - Filters []*Filter `json:"filters"` - // Trigger - Trigger *Trigger `json:"trigger"` - // Settings - // TODO: rename the property name - Deletion bool `json:"deletion"` - // If override the image tag - Override bool `json:"override"` - // Operations - Enabled bool `json:"enabled"` - CreationTime time.Time `json:"creation_time"` - UpdateTime time.Time `json:"update_time"` -} - -// Valid the policy -func (p *Policy) Valid(v *validation.Validation) { - if len(p.Name) == 0 { - v.SetError("name", "cannot be empty") - } - var srcRegistryID, dstRegistryID int64 - if p.SrcRegistry != nil { - srcRegistryID = p.SrcRegistry.ID - } - if p.DestRegistry != nil { - dstRegistryID = p.DestRegistry.ID - } - - // one of the source registry and destination registry must be Harbor itself - if srcRegistryID != 0 && dstRegistryID != 0 || - srcRegistryID == 0 && dstRegistryID == 0 { - v.SetError("src_registry, dest_registry", "one of them should be empty and the other one shouldn't be empty") - } - - // valid the filters - for _, filter := range p.Filters { - switch filter.Type { - case FilterTypeResource, FilterTypeName, FilterTypeTag: - value, ok := filter.Value.(string) - if !ok { - v.SetError("filters", "the type of filter value isn't string") - break - } - if filter.Type == FilterTypeResource { - rt := ResourceType(value) - if !(rt == ResourceTypeArtifact || rt == ResourceTypeImage || rt == ResourceTypeChart) { - v.SetError("filters", fmt.Sprintf("invalid resource filter: %s", value)) - break - } - } - case FilterTypeLabel: - labels, ok := filter.Value.([]interface{}) - if !ok { - v.SetError("filters", "the type of label filter value isn't string slice") - break - } - for _, label := range labels { - _, ok := label.(string) - if !ok { - v.SetError("filters", "the type of label filter value isn't string slice") - break - } - } - default: - v.SetError("filters", "invalid filter type") - break - } - } - - // valid trigger - if p.Trigger != nil { - switch p.Trigger.Type { - case TriggerTypeManual, TriggerTypeEventBased: - case TriggerTypeScheduled: - if p.Trigger.Settings == nil || len(p.Trigger.Settings.Cron) == 0 { - v.SetError("trigger", fmt.Sprintf("the cron string cannot be empty when the trigger type is %s", TriggerTypeScheduled)) - } else { - _, err := cron.Parse(p.Trigger.Settings.Cron) - if err != nil { - v.SetError("trigger", fmt.Sprintf("invalid cron string for scheduled trigger: %s", p.Trigger.Settings.Cron)) - } - } - default: - v.SetError("trigger", "invalid trigger type") - } - } -} - // FilterType represents the type info of the filter. type FilterType string @@ -158,15 +48,3 @@ type Trigger struct { type TriggerSettings struct { Cron string `json:"cron"` } - -// PolicyQuery defines the query conditions for listing policies -type PolicyQuery struct { - Name string - // TODO: need to consider how to support listing the policies - // of one namespace in both pull and push modes - Namespace string - SrcRegistry int64 - DestRegistry int64 - Page int64 - Size int64 -} diff --git a/src/replication/model/policy_test.go b/src/replication/model/policy_test.go deleted file mode 100644 index 262c0b23f..000000000 --- a/src/replication/model/policy_test.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "fmt" - "testing" - - "github.com/astaxie/beego/validation" - "github.com/stretchr/testify/assert" -) - -func TestValidOfPolicy(t *testing.T) { - cases := []struct { - policy *Policy - pass bool - }{ - // empty name - { - policy: &Policy{}, - pass: false, - }, - // empty source registry and destination registry - { - policy: &Policy{ - Name: "policy01", - }, - pass: false, - }, - // source registry and destination registry both not empty - { - policy: &Policy{ - Name: "policy01", - SrcRegistry: &Registry{ - ID: 1, - }, - DestRegistry: &Registry{ - ID: 2, - }, - }, - pass: false, - }, - // invalid filter - { - policy: &Policy{ - Name: "policy01", - SrcRegistry: &Registry{ - ID: 0, - }, - DestRegistry: &Registry{ - ID: 1, - }, - Filters: []*Filter{ - { - Type: "invalid_type", - }, - }, - }, - pass: false, - }, - // invalid filter - { - policy: &Policy{ - Name: "policy01", - SrcRegistry: &Registry{ - ID: 0, - }, - DestRegistry: &Registry{ - ID: 1, - }, - Filters: []*Filter{ - { - Type: FilterTypeResource, - Value: "invalid_resource_type", - }, - }, - }, - pass: false, - }, - // invalid filter - { - policy: &Policy{ - Name: "policy01", - SrcRegistry: &Registry{ - ID: 0, - }, - DestRegistry: &Registry{ - ID: 1, - }, - Filters: []*Filter{ - { - Type: FilterTypeResource, - Value: ResourceTypeImage, - }, - { - Type: FilterTypeTag, - Value: "", - }, - }, - }, - pass: false, - }, - // invalid trigger - { - policy: &Policy{ - Name: "policy01", - SrcRegistry: &Registry{ - ID: 0, - }, - DestRegistry: &Registry{ - ID: 1, - }, - Filters: []*Filter{ - { - Type: FilterTypeName, - Value: "library", - }, - }, - Trigger: &Trigger{ - Type: "invalid_type", - }, - }, - pass: false, - }, - // invalid trigger - { - policy: &Policy{ - Name: "policy01", - SrcRegistry: &Registry{ - ID: 0, - }, - DestRegistry: &Registry{ - ID: 1, - }, - Filters: []*Filter{ - { - Type: FilterTypeName, - Value: "library", - }, - }, - Trigger: &Trigger{ - Type: TriggerTypeScheduled, - }, - }, - pass: false, - }, - // invalid cron - { - policy: &Policy{ - Name: "policy01", - SrcRegistry: &Registry{ - ID: 0, - }, - DestRegistry: &Registry{ - ID: 1, - }, - Filters: []*Filter{ - { - Type: FilterTypeResource, - Value: "image", - }, - { - Type: FilterTypeName, - Value: "library/**", - }, - }, - Trigger: &Trigger{ - Type: TriggerTypeScheduled, - Settings: &TriggerSettings{ - Cron: "* * *", - }, - }, - }, - pass: false, - }, - // pass - { - policy: &Policy{ - Name: "policy01", - SrcRegistry: &Registry{ - ID: 0, - }, - DestRegistry: &Registry{ - ID: 1, - }, - Filters: []*Filter{ - { - Type: FilterTypeResource, - Value: "image", - }, - { - Type: FilterTypeName, - Value: "library/**", - }, - }, - Trigger: &Trigger{ - Type: TriggerTypeScheduled, - Settings: &TriggerSettings{ - Cron: "* * * * * *", - }, - }, - }, - pass: true, - }, - } - - for i, c := range cases { - fmt.Printf("running case %d ...\n", i) - v := &validation.Validation{} - c.policy.Valid(v) - assert.Equal(t, c.pass, len(v.Errors) == 0) - } -} diff --git a/src/replication/policy/controller.go b/src/replication/policy/controller.go deleted file mode 100644 index db9eb929b..000000000 --- a/src/replication/policy/controller.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package policy - -import ( - "github.com/goharbor/harbor/src/replication/model" -) - -// Controller controls the replication policies -type Controller interface { - // Create new policy - Create(*model.Policy) (int64, error) - // List the policies, returns the total count, policy list and error - List(...*model.PolicyQuery) (int64, []*model.Policy, error) - // Get policy with specified ID - Get(int64) (*model.Policy, error) - // Get policy by the name - GetByName(string) (*model.Policy, error) - // Update the specified policy - Update(policy *model.Policy) error - // Remove the specified policy - Remove(int64) error -} diff --git a/src/replication/policy/controller/controller.go b/src/replication/policy/controller/controller.go deleted file mode 100644 index 33aaaf813..000000000 --- a/src/replication/policy/controller/controller.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller - -import ( - "fmt" - - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/lib/orm" - "github.com/goharbor/harbor/src/pkg/scheduler" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/policy" - "github.com/goharbor/harbor/src/replication/policy/manager" -) - -// const definitions -const ( - CallbackFuncName = "REPLICATION_CALLBACK" -) - -// NewController returns a policy controller which can CURD and schedule policies -func NewController() policy.Controller { - mgr := manager.NewDefaultManager() - ctl := &controller{ - scheduler: scheduler.Sched, - } - ctl.Controller = mgr - return ctl -} - -type controller struct { - policy.Controller - scheduler scheduler.Scheduler -} - -func (c *controller) Create(policy *model.Policy) (int64, error) { - id, err := c.Controller.Create(policy) - if err != nil { - return 0, err - } - if isScheduledTrigger(policy) { - extras := make(map[string]interface{}) - if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id, extras); err != nil { - log.Errorf("failed to schedule the policy %d: %v", id, err) - } - } - return id, nil -} - -func (c *controller) Update(policy *model.Policy) error { - origin, err := c.Controller.Get(policy.ID) - if err != nil { - return err - } - if origin == nil { - return fmt.Errorf("policy %d not found", policy.ID) - } - // if no need to reschedule the policy, just update it - if !isScheduleTriggerChanged(origin, policy) { - return c.Controller.Update(policy) - } - // need to reschedule the policy - // unschedule first if needed - if isScheduledTrigger(origin) { - if err = c.scheduler.UnScheduleByVendor(orm.Context(), job.Replication, origin.ID); err != nil { - return fmt.Errorf("failed to unschedule the policy %d: %v", origin.ID, err) - } - } - // update the policy - if err = c.Controller.Update(policy); err != nil { - return err - } - // schedule again if needed - if isScheduledTrigger(policy) { - extras := make(map[string]interface{}) - if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID, extras); err != nil { - return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err) - } - } - return nil -} - -func (c *controller) Remove(policyID int64) error { - policy, err := c.Controller.Get(policyID) - if err != nil { - return err - } - if policy == nil { - return fmt.Errorf("policy %d not found", policyID) - } - if isScheduledTrigger(policy) { - if err = c.scheduler.UnScheduleByVendor(orm.Context(), job.Replication, policyID); err != nil { - return err - } - } - return c.Controller.Remove(policyID) -} - -func isScheduledTrigger(policy *model.Policy) bool { - if policy == nil { - return false - } - if !policy.Enabled { - return false - } - if policy.Trigger == nil { - return false - } - return policy.Trigger.Type == model.TriggerTypeScheduled -} - -func isScheduleTriggerChanged(origin, current *model.Policy) bool { - o := isScheduledTrigger(origin) - c := isScheduledTrigger(current) - // both triggers are not scheduled - if !o && !c { - return false - } - // both triggers are scheduled - if o && c { - return origin.Trigger.Settings.Cron != current.Trigger.Settings.Cron - } - // one is scheduled but the other one isn't - return true -} diff --git a/src/replication/policy/controller/controller_test.go b/src/replication/policy/controller/controller_test.go deleted file mode 100644 index 0bcc1a5a5..000000000 --- a/src/replication/policy/controller/controller_test.go +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/testing/mock" - "github.com/goharbor/harbor/src/testing/pkg/scheduler" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type fakedPolicyController struct { - policy *model.Policy -} - -func (f *fakedPolicyController) Create(*model.Policy) (int64, error) { - return 0, nil -} -func (f *fakedPolicyController) List(...*model.PolicyQuery) (int64, []*model.Policy, error) { - return 0, nil, nil -} -func (f *fakedPolicyController) Get(id int64) (*model.Policy, error) { - return f.policy, nil -} -func (f *fakedPolicyController) GetByName(name string) (*model.Policy, error) { - return nil, nil -} -func (f *fakedPolicyController) Update(*model.Policy) error { - return nil -} -func (f *fakedPolicyController) Remove(int64) error { - return nil -} - -func TestIsScheduledTrigger(t *testing.T) { - cases := []struct { - policy *model.Policy - expected bool - }{ - // policy is nil - { - policy: nil, - expected: false, - }, - // policy is disabled - { - policy: &model.Policy{ - Enabled: false, - }, - expected: false, - }, - // trigger is nil - { - policy: &model.Policy{ - Enabled: true, - }, - expected: false, - }, - // trigger type isn't scheduled - { - policy: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeManual, - }, - }, - expected: false, - }, - // trigger type is scheduled - { - policy: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - }, - }, - expected: true, - }, - } - for _, c := range cases { - assert.Equal(t, c.expected, isScheduledTrigger(c.policy)) - } -} - -func TestIsScheduleTriggerChanged(t *testing.T) { - cases := []struct { - origin *model.Policy - current *model.Policy - expected bool - }{ - // both triggers are not scheduled - { - origin: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeManual, - }, - }, - current: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeManual, - }, - }, - expected: false, - }, - // both triggers are scheduled and the crons are not same - { - origin: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - }, - current: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 * * * *", - }, - }, - }, - expected: true, - }, - // both triggers are scheduled and the crons are same - { - origin: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - }, - current: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - }, - expected: false, - }, - // one trigger is scheduled but the other one isn't - { - origin: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - }, - current: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeManual, - }, - }, - expected: true, - }, - // one trigger is scheduled but disabled and - // the other one is scheduled but enabled - { - origin: &model.Policy{ - Enabled: false, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - }, - current: &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - }, - expected: true, - }, - } - for _, c := range cases { - assert.Equal(t, c.expected, isScheduleTriggerChanged(c.origin, c.current)) - } -} - -func TestCreate(t *testing.T) { - dao.PrepareTestForPostgresSQL() - scheduler := &scheduler.Scheduler{} - ctl := &controller{ - scheduler: scheduler, - } - ctl.Controller = &fakedPolicyController{} - - // not scheduled trigger - _, err := ctl.Create(&model.Policy{}) - require.Nil(t, err) - - // scheduled trigger - scheduler.On("Schedule", mock.Anything, mock.Anything, - mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) - _, err = ctl.Create(&model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - }) - require.Nil(t, err) - scheduler.AssertExpectations(t) -} - -func TestUpdate(t *testing.T) { - scheduler := &scheduler.Scheduler{} - c := &fakedPolicyController{} - ctl := &controller{ - scheduler: scheduler, - } - ctl.Controller = c - - var origin, current *model.Policy - // origin policy is nil - current = &model.Policy{ - ID: 1, - Enabled: true, - } - err := ctl.Update(current) - assert.NotNil(t, err) - - // the trigger doesn't change - origin = &model.Policy{ - ID: 1, - Enabled: true, - } - c.policy = origin - current = origin - err = ctl.Update(current) - require.Nil(t, err) - - // the trigger changed - scheduler.On("Schedule", mock.Anything, mock.Anything, - mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) - scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, - mock.Anything).Return(nil) - - origin = &model.Policy{ - ID: 1, - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - } - c.policy = origin - current = &model.Policy{ - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 * * * *", - }, - }, - } - err = ctl.Update(current) - require.Nil(t, err) - scheduler.AssertExpectations(t) -} - -func TestRemove(t *testing.T) { - scheduler := &scheduler.Scheduler{} - c := &fakedPolicyController{} - ctl := &controller{ - scheduler: scheduler, - } - ctl.Controller = c - - // policy is nil - err := ctl.Remove(1) - assert.NotNil(t, err) - - // the trigger type isn't scheduled - policy := &model.Policy{ - ID: 1, - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeManual, - }, - } - c.policy = policy - err = ctl.Remove(1) - require.Nil(t, err) - - // the trigger type is scheduled - scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, - mock.Anything).Return(nil) - policy = &model.Policy{ - ID: 1, - Enabled: true, - Trigger: &model.Trigger{ - Type: model.TriggerTypeScheduled, - Settings: &model.TriggerSettings{ - Cron: "03 05 * * *", - }, - }, - } - c.policy = policy - err = ctl.Remove(1) - require.Nil(t, err) - scheduler.AssertExpectations(t) -} diff --git a/src/replication/policy/manager/manager.go b/src/replication/policy/manager/manager.go deleted file mode 100644 index 4c37a7151..000000000 --- a/src/replication/policy/manager/manager.go +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package manager - -import ( - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/replication/dao" - persist_models "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/policy" -) - -var errNilPolicyModel = errors.New("nil policy model") - -func convertFromPersistModel(policy *persist_models.RepPolicy) (*model.Policy, error) { - if policy == nil { - return nil, nil - } - - ply := model.Policy{ - ID: policy.ID, - Name: policy.Name, - Description: policy.Description, - Creator: policy.Creator, - DestNamespace: policy.DestNamespace, - Deletion: policy.ReplicateDeletion, - Override: policy.Override, - Enabled: policy.Enabled, - CreationTime: policy.CreationTime, - UpdateTime: policy.UpdateTime, - } - if policy.SrcRegistryID > 0 { - ply.SrcRegistry = &model.Registry{ - ID: policy.SrcRegistryID, - } - } - if policy.DestRegistryID > 0 { - ply.DestRegistry = &model.Registry{ - ID: policy.DestRegistryID, - } - } - - // parse Filters - filters, err := parseFilters(policy.Filters) - if err != nil { - return nil, err - } - ply.Filters = filters - - // parse Trigger - trigger, err := parseTrigger(policy.Trigger) - if err != nil { - return nil, err - } - ply.Trigger = trigger - - return &ply, nil -} - -func convertToPersistModel(policy *model.Policy) (*persist_models.RepPolicy, error) { - if policy == nil { - return nil, errNilPolicyModel - } - - ply := &persist_models.RepPolicy{ - ID: policy.ID, - Name: policy.Name, - Description: policy.Description, - Creator: policy.Creator, - DestNamespace: policy.DestNamespace, - Override: policy.Override, - Enabled: policy.Enabled, - ReplicateDeletion: policy.Deletion, - CreationTime: policy.CreationTime, - UpdateTime: time.Now(), - } - if policy.SrcRegistry != nil { - ply.SrcRegistryID = policy.SrcRegistry.ID - } - if policy.DestRegistry != nil { - ply.DestRegistryID = policy.DestRegistry.ID - } - - if policy.Trigger != nil { - trigger, err := json.Marshal(policy.Trigger) - if err != nil { - return nil, err - } - ply.Trigger = string(trigger) - } - - if len(policy.Filters) > 0 { - filters, err := json.Marshal(policy.Filters) - if err != nil { - return nil, err - } - ply.Filters = string(filters) - } - - return ply, nil -} - -// DefaultManager provides replication policy CURD capabilities. -type DefaultManager struct{} - -var _ policy.Controller = &DefaultManager{} - -// NewDefaultManager is the constructor of DefaultManager. -func NewDefaultManager() *DefaultManager { - return &DefaultManager{} -} - -// Create creates a new policy with the provided data; -// If creating failed, error will be returned; -// If creating succeed, ID of the new created policy will be returned. -func (m *DefaultManager) Create(policy *model.Policy) (int64, error) { - ply, err := convertToPersistModel(policy) - if err != nil { - return 0, err - } - return dao.AddRepPolicy(ply) -} - -// List returns all the policies -func (m *DefaultManager) List(queries ...*model.PolicyQuery) (total int64, policies []*model.Policy, err error) { - var persistPolicies []*persist_models.RepPolicy - total, persistPolicies, err = dao.GetPolicies(queries...) - if err != nil { - return - } - - for _, policy := range persistPolicies { - ply, err := convertFromPersistModel(policy) - if err != nil { - return 0, nil, err - } - - policies = append(policies, ply) - } - - if policies == nil { - policies = []*model.Policy{} - } - - return -} - -// Get returns the policy with the specified ID -func (m *DefaultManager) Get(policyID int64) (*model.Policy, error) { - policy, err := dao.GetRepPolicy(policyID) - if err != nil { - return nil, err - } - - return convertFromPersistModel(policy) -} - -// GetByName returns the policy with the specified name -func (m *DefaultManager) GetByName(name string) (*model.Policy, error) { - policy, err := dao.GetRepPolicyByName(name) - if err != nil { - return nil, err - } - - return convertFromPersistModel(policy) -} - -// Update Update the specified policy -func (m *DefaultManager) Update(policy *model.Policy) error { - updatePolicy, err := convertToPersistModel(policy) - if err != nil { - return err - } - - return dao.UpdateRepPolicy(updatePolicy) -} - -// Remove Remove the specified policy -func (m *DefaultManager) Remove(policyID int64) error { - return dao.DeleteRepPolicy(policyID) -} - -type filter struct { - Type model.FilterType `json:"type"` - Value interface{} `json:"value"` - Kind string `json:"kind"` - Pattern string `json:"pattern"` -} - -type trigger struct { - Type model.TriggerType `json:"type"` - Settings *model.TriggerSettings `json:"trigger_settings"` - Kind string `json:"kind"` - ScheduleParam *scheduleParam `json:"schedule_param"` -} - -type scheduleParam struct { - Type string `json:"type"` - Weekday int8 `json:"weekday"` - Offtime int64 `json:"offtime"` -} - -func parseFilters(str string) ([]*model.Filter, error) { - if len(str) == 0 { - return nil, nil - } - items := []*filter{} - if err := json.Unmarshal([]byte(str), &items); err != nil { - return nil, err - } - - filters := []*model.Filter{} - for _, item := range items { - filter := &model.Filter{ - Type: item.Type, - Value: item.Value, - } - // keep backwards compatibility - if len(filter.Type) == 0 { - if filter.Value == nil { - filter.Value = item.Pattern - } - switch item.Kind { - case "repository": - // a name filter "project_name/**" must exist after running upgrade - // if there is any repository filter, merge it into the name filter - repository, ok := filter.Value.(string) - if ok && len(repository) > 0 { - for _, item := range items { - if item.Type == model.FilterTypeName { - name, ok := item.Value.(string) - if ok && len(name) > 0 { - item.Value = strings.Replace(name, "**", repository, 1) - } - break - } - } - } - continue - case "tag": - filter.Type = model.FilterTypeTag - case "label": - // drop all legend label filters - continue - default: - log.Warningf("unknown filter type: %s", filter.Type) - continue - } - } - - // convert the type of value from string to model.ResourceType if the filter - // is a resource type filter - if filter.Type == model.FilterTypeResource { - filter.Value = (model.ResourceType)(filter.Value.(string)) - } - if filter.Type == model.FilterTypeLabel { - labels := []string{} - for _, label := range filter.Value.([]interface{}) { - labels = append(labels, label.(string)) - } - filter.Value = labels - } - filters = append(filters, filter) - } - return filters, nil -} - -func parseTrigger(str string) (*model.Trigger, error) { - if len(str) == 0 { - return nil, nil - } - item := &trigger{} - if err := json.Unmarshal([]byte(str), item); err != nil { - return nil, err - } - trigger := &model.Trigger{ - Type: item.Type, - Settings: item.Settings, - } - // keep backwards compatibility - if len(trigger.Type) == 0 { - switch item.Kind { - case "Manual": - trigger.Type = model.TriggerTypeManual - case "Immediate": - trigger.Type = model.TriggerTypeEventBased - case "Scheduled": - trigger.Type = model.TriggerTypeScheduled - trigger.Settings = &model.TriggerSettings{ - Cron: parseScheduleParamToCron(item.ScheduleParam), - } - default: - log.Warningf("unknown trigger type: %s", item.Kind) - return nil, nil - } - } - return trigger, nil -} - -func parseScheduleParamToCron(param *scheduleParam) string { - if param == nil { - return "" - } - offtime := param.Offtime - offtime = offtime % (3600 * 24) - hour := int(offtime / 3600) - offtime = offtime % 3600 - minute := int(offtime / 60) - second := int(offtime % 60) - if param.Type == "Weekly" { - return fmt.Sprintf("%d %d %d * * %d", second, minute, hour, param.Weekday%7) - } - return fmt.Sprintf("%d %d %d * * *", second, minute, hour) -} diff --git a/src/replication/policy/manager/manager_test.go b/src/replication/policy/manager/manager_test.go deleted file mode 100644 index 7cdc5f4c3..000000000 --- a/src/replication/policy/manager/manager_test.go +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package manager - -import ( - "reflect" - "testing" - - persist_models "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_convertFromPersistModel(t *testing.T) { - tests := []struct { - name string - from *persist_models.RepPolicy - want *model.Policy - wantErr bool - }{ - { - name: "Nil Persist Model", - from: nil, - want: nil, - wantErr: false, - }, - { - name: "parse Filters Error", - from: &persist_models.RepPolicy{Filters: "abc"}, - want: nil, wantErr: true, - }, - { - name: "parse Trigger Error", - from: &persist_models.RepPolicy{Trigger: "abc"}, - want: nil, wantErr: true, - }, - { - name: "Persist Model", from: &persist_models.RepPolicy{ - ID: 999, - Name: "Policy Test", - Description: "Policy Description", - Creator: "someone", - SrcRegistryID: 123, - DestRegistryID: 456, - DestNamespace: "target_ns", - ReplicateDeletion: true, - Override: true, - Enabled: true, - Trigger: "", - Filters: "[]", - }, want: &model.Policy{ - ID: 999, - Name: "Policy Test", - Description: "Policy Description", - Creator: "someone", - SrcRegistry: &model.Registry{ - ID: 123, - }, - DestRegistry: &model.Registry{ - ID: 456, - }, - DestNamespace: "target_ns", - Deletion: true, - Override: true, - Enabled: true, - Trigger: nil, - Filters: []*model.Filter{}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := convertFromPersistModel(tt.from) - - if tt.wantErr { - require.NotNil(t, err) - return - } - - if tt.want == nil { - require.Nil(t, got) - return - } - - require.Nil(t, err, tt.name) - assert.Equal(t, tt.want.ID, got.ID) - assert.Equal(t, tt.want.Name, got.Name) - assert.Equal(t, tt.want.Description, got.Description) - assert.Equal(t, tt.want.Creator, got.Creator) - assert.Equal(t, tt.want.SrcRegistry.ID, got.SrcRegistry.ID) - assert.Equal(t, tt.want.DestRegistry.ID, got.DestRegistry.ID) - assert.Equal(t, tt.want.DestNamespace, got.DestNamespace) - assert.Equal(t, tt.want.Deletion, got.Deletion) - assert.Equal(t, tt.want.Override, got.Override) - assert.Equal(t, tt.want.Enabled, got.Enabled) - assert.Equal(t, tt.want.Trigger, got.Trigger) - assert.Equal(t, tt.want.Filters, got.Filters) - - }) - } -} - -func Test_convertToPersistModel(t *testing.T) { - tests := []struct { - name string - from *model.Policy - want *persist_models.RepPolicy - wantErr bool - }{ - {name: "Nil Model", from: nil, want: nil, wantErr: true}, - { - name: "Persist Model", from: &model.Policy{ - ID: 999, - Name: "Policy Test", - Description: "Policy Description", - Creator: "someone", - SrcRegistry: &model.Registry{ - ID: 123, - }, - DestRegistry: &model.Registry{ - ID: 456, - }, - DestNamespace: "target_ns", - Deletion: true, - Override: true, - Enabled: true, - Trigger: &model.Trigger{}, - Filters: []*model.Filter{{Type: "registry", Value: "abc"}}, - }, want: &persist_models.RepPolicy{ - ID: 999, - Name: "Policy Test", - Description: "Policy Description", - Creator: "someone", - SrcRegistryID: 123, - DestRegistryID: 456, - DestNamespace: "target_ns", - ReplicateDeletion: true, - Override: true, - Enabled: true, - Trigger: "{\"type\":\"\",\"trigger_settings\":null}", - Filters: "[{\"type\":\"registry\",\"value\":\"abc\"}]", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := convertToPersistModel(tt.from) - - if tt.wantErr { - assert.Equal(t, err, errNilPolicyModel) - return - } - - require.Nil(t, err, tt.name) - assert.Equal(t, tt.want.ID, got.ID) - assert.Equal(t, tt.want.Name, got.Name) - assert.Equal(t, tt.want.Description, got.Description) - assert.Equal(t, tt.want.Creator, got.Creator) - assert.Equal(t, tt.want.SrcRegistryID, got.SrcRegistryID) - assert.Equal(t, tt.want.DestRegistryID, got.DestRegistryID) - assert.Equal(t, tt.want.DestNamespace, got.DestNamespace) - assert.Equal(t, tt.want.ReplicateDeletion, got.ReplicateDeletion) - assert.Equal(t, tt.want.Override, got.Override) - assert.Equal(t, tt.want.Enabled, got.Enabled) - assert.Equal(t, tt.want.Trigger, got.Trigger) - assert.Equal(t, tt.want.Filters, got.Filters) - - }) - } -} - -func TestNewDefaultManager(t *testing.T) { - tests := []struct { - name string - want *DefaultManager - }{ - {want: &DefaultManager{}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := NewDefaultManager(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("NewDefaultManager() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseFilters(t *testing.T) { - // nil filter string - str := "" - filters, err := parseFilters(str) - require.Nil(t, err) - assert.Nil(t, filters) - // only contains the fields that introduced in the latest version - str = `[{"type":"name","value":"library/hello-world"}]` - filters, err = parseFilters(str) - require.Nil(t, err) - require.Equal(t, 1, len(filters)) - assert.Equal(t, model.FilterTypeName, filters[0].Type) - assert.Equal(t, "library/hello-world", filters[0].Value.(string)) - // contains "kind" from previous versions - str = `[{"kind":"repository","value":"hello-world"},{"type":"name","value":"library/**"}]` - filters, err = parseFilters(str) - require.Nil(t, err) - require.Equal(t, 1, len(filters)) - assert.Equal(t, model.FilterTypeName, filters[0].Type) - assert.Equal(t, "library/hello-world", filters[0].Value.(string)) - // contains "pattern" from previous versions - str = `[{"kind":"repository","pattern":"hello-world"},{"type":"name","value":"library/**"}]` - filters, err = parseFilters(str) - require.Nil(t, err) - require.Equal(t, 1, len(filters)) - assert.Equal(t, model.FilterTypeName, filters[0].Type) - assert.Equal(t, "library/hello-world", filters[0].Value.(string)) -} - -func TestParseTrigger(t *testing.T) { - // nil trigger string - str := "" - trigger, err := parseTrigger(str) - require.Nil(t, err) - assert.Nil(t, trigger) - // only contains the fields that introduced in the latest version - str = `{"type":"scheduled", "trigger_settings":{"cron":"1 * * * * *"}}` - trigger, err = parseTrigger(str) - require.Nil(t, err) - assert.Equal(t, model.TriggerTypeScheduled, trigger.Type) - assert.Equal(t, "1 * * * * *", trigger.Settings.Cron) - // contains "kind" from previous versions - str = `{"kind":"Manual"}` - trigger, err = parseTrigger(str) - require.Nil(t, err) - assert.Equal(t, model.TriggerTypeManual, trigger.Type) - assert.Nil(t, trigger.Settings) - // contains "kind" from previous versions - str = `{"kind":"Immediate"}` - trigger, err = parseTrigger(str) - require.Nil(t, err) - assert.Equal(t, model.TriggerTypeEventBased, trigger.Type) - assert.Nil(t, trigger.Settings) - // contains "schedule_param" from previous versions - str = `{"kind":"Scheduled","schedule_param":{"type":"Weekly","weekday":1,"offtime":0}}` - trigger, err = parseTrigger(str) - require.Nil(t, err) - assert.Equal(t, model.TriggerTypeScheduled, trigger.Type) - assert.Equal(t, "0 0 0 * * 1", trigger.Settings.Cron) - // contains "schedule_param" from previous versions - str = `{"kind":"Scheduled","schedule_param":{"type":"Daily","offtime":0}}` - trigger, err = parseTrigger(str) - require.Nil(t, err) - assert.Equal(t, model.TriggerTypeScheduled, trigger.Type) - assert.Equal(t, "0 0 0 * * *", trigger.Settings.Cron) -} diff --git a/src/replication/registry/manager.go b/src/replication/registry/manager.go index abce101d2..d0c52ebf4 100644 --- a/src/replication/registry/manager.go +++ b/src/replication/registry/manager.go @@ -69,7 +69,7 @@ func (m *DefaultManager) Get(id int64) (*model.Registry, error) { return nil, nil } - return fromDaoModel(registry) + return FromDaoModel(registry) } // GetByName gets a registry by its name @@ -83,7 +83,7 @@ func (m *DefaultManager) GetByName(name string) (*model.Registry, error) { return nil, nil } - return fromDaoModel(registry) + return FromDaoModel(registry) } // List lists registries according to query provided. @@ -95,7 +95,7 @@ func (m *DefaultManager) List(query *q.Query) (int64, []*model.Registry, error) var results []*model.Registry for _, r := range registries { - registry, err := fromDaoModel(r) + registry, err := FromDaoModel(r) if err != nil { return 0, nil, err } @@ -107,7 +107,7 @@ func (m *DefaultManager) List(query *q.Query) (int64, []*model.Registry, error) // Add adds a new registry func (m *DefaultManager) Add(registry *model.Registry) (int64, error) { - r, err := toDaoModel(registry) + r, err := ToDaoModel(registry) if err != nil { log.Errorf("Convert registry model to dao layer model error: %v", err) return -1, err @@ -124,7 +124,7 @@ func (m *DefaultManager) Add(registry *model.Registry) (int64, error) { // Update updates a registry func (m *DefaultManager) Update(registry *model.Registry, props ...string) error { - r, err := toDaoModel(registry) + r, err := ToDaoModel(registry) if err != nil { log.Errorf("Convert registry model to dao layer model error: %v", err) return err @@ -219,9 +219,9 @@ func encrypt(secret string) (string, error) { return encrypted, nil } -// fromDaoModel converts DAO layer registry model to replication model. +// FromDaoModel converts DAO layer registry model to replication model. // Also, if access secret is provided, decrypt it. -func fromDaoModel(registry *models.Registry) (*model.Registry, error) { +func FromDaoModel(registry *models.Registry) (*model.Registry, error) { r := &model.Registry{ ID: registry.ID, Name: registry.Name, @@ -254,9 +254,9 @@ func fromDaoModel(registry *models.Registry) (*model.Registry, error) { return r, nil } -// toDaoModel converts registry model from replication to DAO layer model. +// ToDaoModel converts registry model from replication to DAO layer model. // Also, if access secret is provided, encrypt it. -func toDaoModel(registry *model.Registry) (*models.Registry, error) { +func ToDaoModel(registry *model.Registry) (*models.Registry, error) { m := &models.Registry{ ID: registry.ID, URL: registry.URL, diff --git a/src/replication/replication.go b/src/replication/replication.go index c9d493b42..04b0e4e6d 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -15,20 +15,12 @@ package replication import ( - "context" - "fmt" - "strconv" "time" - "github.com/goharbor/harbor/src/controller/replication" cfg "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/pkg/scheduler" - "github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/replication/config" "github.com/goharbor/harbor/src/replication/event" - "github.com/goharbor/harbor/src/replication/policy" - "github.com/goharbor/harbor/src/replication/policy/controller" "github.com/goharbor/harbor/src/replication/registry" // register the Harbor adapter @@ -66,41 +58,12 @@ import ( ) var ( - // PolicyCtl is a global policy controller - PolicyCtl policy.Controller // RegistryMgr is a global registry manager RegistryMgr registry.Manager // EventHandler handles images/chart pull/push events EventHandler event.Handler ) -func init() { - callbackFunc := func(ctx context.Context, param string) error { - policyID, err := strconv.ParseInt(param, 10, 64) - if err != nil { - return err - } - - policy, err := PolicyCtl.Get(policyID) - if err != nil { - return err - } - if policy == nil { - return fmt.Errorf("policy %d not found", policyID) - } - if err = event.PopulateRegistries(RegistryMgr, policy); err != nil { - return err - } - - _, err = replication.Ctl.Start(ctx, policy, nil, task.ExecutionTriggerSchedule) - return err - } - err := scheduler.RegisterCallbackFunc(controller.CallbackFuncName, callbackFunc) - if err != nil { - log.Errorf("failed to register the callback function for replication: %v", err) - } -} - // Init the global variables and configurations func Init(closing, done chan struct{}) error { // init config @@ -116,10 +79,8 @@ func Init(closing, done chan struct{}) error { } // init registry manager RegistryMgr = registry.NewDefaultManager() - // init policy controller - PolicyCtl = controller.NewController() // init event handler - EventHandler = event.NewHandler(PolicyCtl, RegistryMgr) + EventHandler = event.NewHandler(RegistryMgr) log.Debug("the replication initialization completed") // Start health checker for registries diff --git a/src/replication/replication_test.go b/src/replication/replication_test.go index 0f018333b..6cb53a16f 100644 --- a/src/replication/replication_test.go +++ b/src/replication/replication_test.go @@ -36,7 +36,6 @@ func TestInit(t *testing.T) { config.InitWithSettings(nil) err = Init(make(chan struct{}), make(chan struct{})) require.Nil(t, err) - assert.NotNil(t, PolicyCtl) assert.NotNil(t, RegistryMgr) assert.NotNil(t, EventHandler) } diff --git a/src/server/v2.0/handler/base.go b/src/server/v2.0/handler/base.go index 574a0ca5c..3aa7d78f0 100644 --- a/src/server/v2.0/handler/base.go +++ b/src/server/v2.0/handler/base.go @@ -56,14 +56,22 @@ func (*BaseAPI) SendError(ctx context.Context, err error) middleware.Responder { return NewErrResponder(err) } -// HasPermission returns true when the request has action permission on resource -func (*BaseAPI) HasPermission(ctx context.Context, action rbac.Action, resource rbac.Resource) bool { - s, ok := security.FromContext(ctx) +// GetSecurityContext from the provided context +func (*BaseAPI) GetSecurityContext(ctx context.Context) (security.Context, error) { + sc, ok := security.FromContext(ctx) if !ok { - log.Warningf("security not found in the context") + return nil, errors.UnauthorizedError(errors.New("security context not found")) + } + return sc, nil +} + +// HasPermission returns true when the request has action permission on resource +func (b *BaseAPI) HasPermission(ctx context.Context, action rbac.Action, resource rbac.Resource) bool { + s, err := b.GetSecurityContext(ctx) + if err != nil { + log.Warningf("security context not found") return false } - return s.Can(ctx, action, resource) } @@ -98,9 +106,9 @@ func (b *BaseAPI) RequireProjectAccess(ctx context.Context, projectIDOrName inte if b.HasProjectPermission(ctx, projectIDOrName, action, subresource...) { return nil } - secCtx, ok := security.FromContext(ctx) - if !ok { - return errors.UnauthorizedError(errors.New("security context not found")) + secCtx, err := b.GetSecurityContext(ctx) + if err != nil { + return err } if !secCtx.IsAuthenticated() { return errors.UnauthorizedError(nil) @@ -110,9 +118,9 @@ func (b *BaseAPI) RequireProjectAccess(ctx context.Context, projectIDOrName inte // RequireSystemAccess checks the system admin permission according to the security context func (b *BaseAPI) RequireSystemAccess(ctx context.Context, action rbac.Action, subresource ...rbac.Resource) error { - secCtx, ok := security.FromContext(ctx) - if !ok { - return errors.UnauthorizedError(errors.New("security context not found")) + secCtx, err := b.GetSecurityContext(ctx) + if err != nil { + return err } if !secCtx.IsAuthenticated() { return errors.UnauthorizedError(nil) @@ -126,9 +134,9 @@ func (b *BaseAPI) RequireSystemAccess(ctx context.Context, action rbac.Action, s // RequireAuthenticated checks it's authenticated according to the security context func (b *BaseAPI) RequireAuthenticated(ctx context.Context) error { - secCtx, ok := security.FromContext(ctx) - if !ok { - return errors.UnauthorizedError(errors.New("security context not found")) + secCtx, err := b.GetSecurityContext(ctx) + if err != nil { + return err } if !secCtx.IsAuthenticated() { return errors.UnauthorizedError(nil) diff --git a/src/server/v2.0/handler/replication.go b/src/server/v2.0/handler/replication.go index fdffd3dc1..f62ae2fdb 100644 --- a/src/server/v2.0/handler/replication.go +++ b/src/server/v2.0/handler/replication.go @@ -16,59 +16,198 @@ package handler import ( "context" - "github.com/goharbor/harbor/src/common/rbac" + "fmt" "strconv" "strings" "github.com/go-openapi/runtime/middleware" "github.com/go-openapi/strfmt" + "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/controller/replication" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/q" + rep "github.com/goharbor/harbor/src/pkg/replication" "github.com/goharbor/harbor/src/pkg/task" - replica "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/event" - "github.com/goharbor/harbor/src/replication/policy" - "github.com/goharbor/harbor/src/replication/policy/manager" + "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/server/v2.0/models" operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/replication" ) func newReplicationAPI() *replicationAPI { return &replicationAPI{ - ctl: replication.Ctl, - policyMgr: manager.NewDefaultManager(), + ctl: replication.Ctl, } } type replicationAPI struct { BaseAPI - ctl replication.Controller - policyMgr policy.Controller + ctl replication.Controller } func (r *replicationAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder { return nil } -func (r *replicationAPI) StartReplication(ctx context.Context, params operation.StartReplicationParams) middleware.Responder { - // TODO move the following logic to the replication controller after refactoring the policy management part with the new programming model - if err := r.RequireSystemAccess(ctx, rbac.ActionCreate, rbac.ResourceReplication); err != nil { +func (r *replicationAPI) CreateReplicationPolicy(ctx context.Context, params operation.CreateReplicationPolicyParams) middleware.Responder { + if err := r.RequireSystemAccess(ctx, rbac.ActionCreate, rbac.ResourceReplicationPolicy); err != nil { return r.SendError(ctx, err) } - policy, err := r.policyMgr.Get(params.Execution.PolicyID) + sc, err := r.GetSecurityContext(ctx) if err != nil { return r.SendError(ctx, err) } - if policy == nil { - return r.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode). - WithMessage("the replication policy %d not found", params.Execution.PolicyID)) + policy := &rep.Policy{ + Name: params.Policy.Name, + Description: params.Policy.Description, + Creator: sc.GetUsername(), + DestNamespace: params.Policy.DestNamespace, + ReplicateDeletion: params.Policy.Deletion, + Override: params.Policy.Override, + Enabled: params.Policy.Enabled, } - if err = event.PopulateRegistries(replica.RegistryMgr, policy); err != nil { + if params.Policy.SrcRegistry != nil { + policy.SrcRegistry = &model.Registry{ + ID: params.Policy.SrcRegistry.ID, + } + } + if params.Policy.DestRegistry != nil { + policy.DestRegistry = &model.Registry{ + ID: params.Policy.DestRegistry.ID, + } + } + if len(params.Policy.Filters) > 0 { + for _, filter := range params.Policy.Filters { + policy.Filters = append(policy.Filters, &model.Filter{ + Type: model.FilterType(filter.Type), + Value: filter.Value, + }) + } + } + if params.Policy.Trigger != nil { + policy.Trigger = &model.Trigger{ + Type: model.TriggerType(params.Policy.Trigger.Type), + } + if params.Policy.Trigger.TriggerSettings != nil { + policy.Trigger.Settings = &model.TriggerSettings{ + Cron: params.Policy.Trigger.TriggerSettings.Cron, + } + } + } + id, err := r.ctl.CreatePolicy(ctx, policy) + if err != nil { return r.SendError(ctx, err) } + location := fmt.Sprintf("%s/%d", strings.TrimSuffix(params.HTTPRequest.URL.Path, "/"), id) + return operation.NewCreateReplicationPolicyCreated().WithLocation(location) +} +func (r *replicationAPI) UpdateReplicationPolicy(ctx context.Context, params operation.UpdateReplicationPolicyParams) middleware.Responder { + if err := r.RequireSystemAccess(ctx, rbac.ActionUpdate, rbac.ResourceReplicationPolicy); err != nil { + return r.SendError(ctx, err) + } + policy := &rep.Policy{ + ID: params.ID, + Name: params.Policy.Name, + Description: params.Policy.Description, + ReplicateDeletion: params.Policy.Deletion, + Override: params.Policy.Override, + Enabled: params.Policy.Enabled, + } + if params.Policy.SrcRegistry != nil { + policy.SrcRegistry = &model.Registry{ + ID: params.Policy.SrcRegistry.ID, + } + } + if params.Policy.DestRegistry != nil { + policy.DestRegistry = &model.Registry{ + ID: params.Policy.DestRegistry.ID, + } + } + if len(params.Policy.Filters) > 0 { + for _, filter := range params.Policy.Filters { + policy.Filters = append(policy.Filters, &model.Filter{ + Type: model.FilterType(filter.Type), + Value: filter.Value, + }) + } + } + if params.Policy.Trigger != nil { + policy.Trigger = &model.Trigger{ + Type: model.TriggerType(params.Policy.Trigger.Type), + } + if params.Policy.Trigger.TriggerSettings != nil { + policy.Trigger.Settings = &model.TriggerSettings{ + Cron: params.Policy.Trigger.TriggerSettings.Cron, + } + } + } + if err := r.ctl.UpdatePolicy(ctx, policy); err != nil { + return r.SendError(ctx, err) + } + return operation.NewUpdateReplicationPolicyOK() +} + +func (r *replicationAPI) ListReplicationPolicies(ctx context.Context, params operation.ListReplicationPoliciesParams) middleware.Responder { + if err := r.RequireSystemAccess(ctx, rbac.ActionList, rbac.ResourceReplicationPolicy); err != nil { + return r.SendError(ctx, err) + } + query, err := r.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize) + if err != nil { + return r.SendError(ctx, err) + } + if params.Name != nil { + query.Keywords["Name"] = &q.FuzzyMatchValue{ + Value: *params.Name, + } + } + total, err := r.ctl.PolicyCount(ctx, query) + if err != nil { + return r.SendError(ctx, err) + } + policies, err := r.ctl.ListPolicies(ctx, query) + if err != nil { + return r.SendError(ctx, err) + } + var result []*models.ReplicationPolicy + for _, policy := range policies { + result = append(result, convertReplicationPolicy(policy)) + } + return operation.NewListReplicationPoliciesOK(). + WithXTotalCount(total). + WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). + WithPayload(result) +} + +func (r *replicationAPI) GetReplicationPolicy(ctx context.Context, params operation.GetReplicationPolicyParams) middleware.Responder { + if err := r.RequireSystemAccess(ctx, rbac.ActionRead, rbac.ResourceReplicationPolicy); err != nil { + return r.SendError(ctx, err) + } + policy, err := r.ctl.GetPolicy(ctx, params.ID) + if err != nil { + return r.SendError(ctx, err) + } + return operation.NewGetReplicationPolicyOK().WithPayload(convertReplicationPolicy(policy)) +} + +func (r *replicationAPI) DeleteReplicationPolicy(ctx context.Context, params operation.DeleteReplicationPolicyParams) middleware.Responder { + if err := r.RequireSystemAccess(ctx, rbac.ActionDelete, rbac.ResourceReplicationPolicy); err != nil { + return r.SendError(ctx, err) + } + if err := r.ctl.DeletePolicy(ctx, params.ID); err != nil { + return r.SendError(ctx, err) + } + return operation.NewDeleteReplicationPolicyOK() +} + +func (r *replicationAPI) StartReplication(ctx context.Context, params operation.StartReplicationParams) middleware.Responder { + if err := r.RequireSystemAccess(ctx, rbac.ActionCreate, rbac.ResourceReplication); err != nil { + return r.SendError(ctx, err) + } + policy, err := r.ctl.GetPolicy(ctx, params.Execution.PolicyID) + if err != nil { + return r.SendError(ctx, err) + } // the legacy replication scheduler job("src/jobservice/job/impl/replication/scheduler.go") calls the start replication API // to trigger the scheduled replication, a query string "trigger" is added when sending the request // here is the logic to cover this part @@ -245,6 +384,73 @@ func (r *replicationAPI) GetReplicationLog(ctx context.Context, params operation } return operation.NewGetReplicationLogOK().WithContentType("text/plain").WithPayload(string(log)) } + +func convertReplicationPolicy(policy *rep.Policy) *models.ReplicationPolicy { + p := &models.ReplicationPolicy{ + CreationTime: strfmt.DateTime(policy.CreationTime), + Deletion: policy.ReplicateDeletion, + Description: policy.Description, + DestNamespace: policy.DestNamespace, + Enabled: policy.Enabled, + ID: policy.ID, + Name: policy.Name, + Override: policy.Override, + ReplicateDeletion: policy.ReplicateDeletion, + UpdateTime: strfmt.DateTime(policy.UpdateTime), + } + if policy.SrcRegistry != nil { + p.SrcRegistry = convertRegistry(policy.SrcRegistry) + } + if policy.DestRegistry != nil { + p.DestRegistry = convertRegistry(policy.DestRegistry) + } + if len(policy.Filters) > 0 { + for _, filter := range policy.Filters { + p.Filters = append(p.Filters, &models.ReplicationFilter{ + Type: string(filter.Type), + Value: filter.Value, + }) + } + } + if policy.Trigger != nil { + trigger := &models.ReplicationTrigger{ + Type: string(policy.Trigger.Type), + } + if policy.Trigger.Settings != nil { + trigger.TriggerSettings = &models.ReplicationTriggerSettings{ + Cron: policy.Trigger.Settings.Cron, + } + } + p.Trigger = trigger + } + return p +} + +func convertRegistry(registry *model.Registry) *models.Registry { + r := &models.Registry{ + CreationTime: strfmt.DateTime(registry.CreationTime), + Description: registry.Description, + ID: registry.ID, + Insecure: registry.Insecure, + Name: registry.Name, + Status: registry.Status, + Type: string(registry.Type), + UpdateTime: strfmt.DateTime(registry.UpdateTime), + URL: registry.URL, + } + if registry.Credential != nil { + credential := &models.RegistryCredential{ + AccessKey: registry.Credential.AccessKey, + Type: string(registry.Credential.Type), + } + if len(registry.Credential.AccessSecret) > 0 { + credential.AccessSecret = "*****" + } + r.Credential = credential + } + return r +} + func convertExecution(execution *replication.Execution) *models.ReplicationExecution { exec := &models.ReplicationExecution{ ID: execution.ID, diff --git a/src/server/v2.0/route/legacy.go b/src/server/v2.0/route/legacy.go index 9208aa9ec..2f5858f61 100755 --- a/src/server/v2.0/route/legacy.go +++ b/src/server/v2.0/route/legacy.go @@ -40,8 +40,6 @@ func registerLegacyRoutes() { beego.Router("/api/"+version+"/replication/adapters", &api.ReplicationAdapterAPI{}, "get:List") beego.Router("/api/"+version+"/replication/adapterinfos", &api.ReplicationAdapterAPI{}, "get:ListAdapterInfos") - beego.Router("/api/"+version+"/replication/policies", &api.ReplicationPolicyAPI{}, "get:List;post:Create") - beego.Router("/api/"+version+"/replication/policies/:id([0-9]+)", &api.ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete") beego.Router("/api/"+version+"/projects/:pid([0-9]+)/webhook/policies", &api.NotificationPolicyAPI{}, "get:List;post:Post") beego.Router("/api/"+version+"/projects/:pid([0-9]+)/webhook/policies/:id([0-9]+)", &api.NotificationPolicyAPI{}) diff --git a/src/testing/controller/replication/controller.go b/src/testing/controller/replication/controller.go index 296e99867..645897b4a 100644 --- a/src/testing/controller/replication/controller.go +++ b/src/testing/controller/replication/controller.go @@ -5,12 +5,14 @@ package replication import ( context "context" - model "github.com/goharbor/harbor/src/replication/model" + controllerreplication "github.com/goharbor/harbor/src/controller/replication" mock "github.com/stretchr/testify/mock" + model "github.com/goharbor/harbor/src/replication/model" + q "github.com/goharbor/harbor/src/lib/q" - replication "github.com/goharbor/harbor/src/controller/replication" + replication "github.com/goharbor/harbor/src/pkg/replication" ) // Controller is an autogenerated mock type for the Controller type @@ -18,6 +20,41 @@ type Controller struct { mock.Mock } +// CreatePolicy provides a mock function with given fields: ctx, policy +func (_m *Controller) CreatePolicy(ctx context.Context, policy *replication.Policy) (int64, error) { + ret := _m.Called(ctx, policy) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *replication.Policy) int64); ok { + r0 = rf(ctx, policy) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *replication.Policy) error); ok { + r1 = rf(ctx, policy) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DeletePolicy provides a mock function with given fields: ctx, id +func (_m *Controller) DeletePolicy(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // ExecutionCount provides a mock function with given fields: ctx, query func (_m *Controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) { ret := _m.Called(ctx, query) @@ -40,15 +77,15 @@ func (_m *Controller) ExecutionCount(ctx context.Context, query *q.Query) (int64 } // GetExecution provides a mock function with given fields: ctx, executionID -func (_m *Controller) GetExecution(ctx context.Context, executionID int64) (*replication.Execution, error) { +func (_m *Controller) GetExecution(ctx context.Context, executionID int64) (*controllerreplication.Execution, error) { ret := _m.Called(ctx, executionID) - var r0 *replication.Execution - if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Execution); ok { + var r0 *controllerreplication.Execution + if rf, ok := ret.Get(0).(func(context.Context, int64) *controllerreplication.Execution); ok { r0 = rf(ctx, executionID) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*replication.Execution) + r0 = ret.Get(0).(*controllerreplication.Execution) } } @@ -62,16 +99,39 @@ func (_m *Controller) GetExecution(ctx context.Context, executionID int64) (*rep return r0, r1 } +// GetPolicy provides a mock function with given fields: ctx, id +func (_m *Controller) GetPolicy(ctx context.Context, id int64) (*replication.Policy, error) { + ret := _m.Called(ctx, id) + + var r0 *replication.Policy + if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Policy); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*replication.Policy) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetTask provides a mock function with given fields: ctx, taskID -func (_m *Controller) GetTask(ctx context.Context, taskID int64) (*replication.Task, error) { +func (_m *Controller) GetTask(ctx context.Context, taskID int64) (*controllerreplication.Task, error) { ret := _m.Called(ctx, taskID) - var r0 *replication.Task - if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Task); ok { + var r0 *controllerreplication.Task + if rf, ok := ret.Get(0).(func(context.Context, int64) *controllerreplication.Task); ok { r0 = rf(ctx, taskID) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*replication.Task) + r0 = ret.Get(0).(*controllerreplication.Task) } } @@ -109,15 +169,38 @@ func (_m *Controller) GetTaskLog(ctx context.Context, taskID int64) ([]byte, err } // ListExecutions provides a mock function with given fields: ctx, query -func (_m *Controller) ListExecutions(ctx context.Context, query *q.Query) ([]*replication.Execution, error) { +func (_m *Controller) ListExecutions(ctx context.Context, query *q.Query) ([]*controllerreplication.Execution, error) { ret := _m.Called(ctx, query) - var r0 []*replication.Execution - if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Execution); ok { + var r0 []*controllerreplication.Execution + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*controllerreplication.Execution); ok { r0 = rf(ctx, query) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]*replication.Execution) + r0 = ret.Get(0).([]*controllerreplication.Execution) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListPolicies provides a mock function with given fields: ctx, query +func (_m *Controller) ListPolicies(ctx context.Context, query *q.Query) ([]*replication.Policy, error) { + ret := _m.Called(ctx, query) + + var r0 []*replication.Policy + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Policy); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*replication.Policy) } } @@ -132,15 +215,15 @@ func (_m *Controller) ListExecutions(ctx context.Context, query *q.Query) ([]*re } // ListTasks provides a mock function with given fields: ctx, query -func (_m *Controller) ListTasks(ctx context.Context, query *q.Query) ([]*replication.Task, error) { +func (_m *Controller) ListTasks(ctx context.Context, query *q.Query) ([]*controllerreplication.Task, error) { ret := _m.Called(ctx, query) - var r0 []*replication.Task - if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Task); ok { + var r0 []*controllerreplication.Task + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*controllerreplication.Task); ok { r0 = rf(ctx, query) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]*replication.Task) + r0 = ret.Get(0).([]*controllerreplication.Task) } } @@ -154,19 +237,40 @@ func (_m *Controller) ListTasks(ctx context.Context, query *q.Query) ([]*replica return r0, r1 } +// PolicyCount provides a mock function with given fields: ctx, query +func (_m *Controller) PolicyCount(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Start provides a mock function with given fields: ctx, policy, resource, trigger -func (_m *Controller) Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (int64, error) { +func (_m *Controller) Start(ctx context.Context, policy *replication.Policy, resource *model.Resource, trigger string) (int64, error) { ret := _m.Called(ctx, policy, resource, trigger) var r0 int64 - if rf, ok := ret.Get(0).(func(context.Context, *model.Policy, *model.Resource, string) int64); ok { + if rf, ok := ret.Get(0).(func(context.Context, *replication.Policy, *model.Resource, string) int64); ok { r0 = rf(ctx, policy, resource, trigger) } else { r0 = ret.Get(0).(int64) } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *model.Policy, *model.Resource, string) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *replication.Policy, *model.Resource, string) error); ok { r1 = rf(ctx, policy, resource, trigger) } else { r1 = ret.Error(1) @@ -209,3 +313,24 @@ func (_m *Controller) TaskCount(ctx context.Context, query *q.Query) (int64, err return r0, r1 } + +// UpdatePolicy provides a mock function with given fields: ctx, policy, props +func (_m *Controller) UpdatePolicy(ctx context.Context, policy *replication.Policy, props ...string) error { + _va := make([]interface{}, len(props)) + for _i := range props { + _va[_i] = props[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, policy) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *replication.Policy, ...string) error); ok { + r0 = rf(ctx, policy, props...) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/testing/pkg/pkg.go b/src/testing/pkg/pkg.go index 6fc9f1cc9..f307cc8c2 100644 --- a/src/testing/pkg/pkg.go +++ b/src/testing/pkg/pkg.go @@ -37,3 +37,7 @@ package pkg //go:generate mockery --case snake --dir ../../pkg/ldap --name Manager --output ./ldap --outpkg ldap //go:generate mockery --case snake --dir ../../pkg/allowlist --name Manager --output ./allowlist --outpkg robot //go:generate mockery --case snake --dir ../../pkg/allowlist/dao --name DAO --output ./allowlist/dao --outpkg dao +//go:generate mockery --case snake --dir ../../pkg/reg/dao --name DAO --output ./reg/dao --outpkg dao +//go:generate mockery --case snake --dir ../../pkg/reg --name Manager --output ./reg --outpkg manager +//go:generate mockery --case snake --dir ../../pkg/replication/dao --name DAO --output ./replication/dao --outpkg dao +//go:generate mockery --case snake --dir ../../pkg/replication --name Manager --output ./replication --outpkg manager diff --git a/src/testing/pkg/reg/dao/dao.go b/src/testing/pkg/reg/dao/dao.go new file mode 100644 index 000000000..9f73e1957 --- /dev/null +++ b/src/testing/pkg/reg/dao/dao.go @@ -0,0 +1,141 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package dao + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + models "github.com/goharbor/harbor/src/replication/dao/models" + + q "github.com/goharbor/harbor/src/lib/q" +) + +// DAO is an autogenerated mock type for the DAO type +type DAO struct { + mock.Mock +} + +// Count provides a mock function with given fields: ctx, query +func (_m *DAO) Count(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Create provides a mock function with given fields: ctx, registry +func (_m *DAO) Create(ctx context.Context, registry *models.Registry) (int64, error) { + ret := _m.Called(ctx, registry) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *models.Registry) int64); ok { + r0 = rf(ctx, registry) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *models.Registry) error); ok { + r1 = rf(ctx, registry) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: ctx, id +func (_m *DAO) Delete(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Get provides a mock function with given fields: ctx, id +func (_m *DAO) Get(ctx context.Context, id int64) (*models.Registry, error) { + ret := _m.Called(ctx, id) + + var r0 *models.Registry + if rf, ok := ret.Get(0).(func(context.Context, int64) *models.Registry); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*models.Registry) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// List provides a mock function with given fields: ctx, query +func (_m *DAO) List(ctx context.Context, query *q.Query) ([]*models.Registry, error) { + ret := _m.Called(ctx, query) + + var r0 []*models.Registry + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*models.Registry); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Registry) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Update provides a mock function with given fields: ctx, registry, props +func (_m *DAO) Update(ctx context.Context, registry *models.Registry, props ...string) error { + _va := make([]interface{}, len(props)) + for _i := range props { + _va[_i] = props[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, registry) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *models.Registry, ...string) error); ok { + r0 = rf(ctx, registry, props...) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/testing/pkg/reg/manager.go b/src/testing/pkg/reg/manager.go new file mode 100644 index 000000000..4058b0b66 --- /dev/null +++ b/src/testing/pkg/reg/manager.go @@ -0,0 +1,140 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package manager + +import ( + context "context" + + model "github.com/goharbor/harbor/src/replication/model" + mock "github.com/stretchr/testify/mock" + + q "github.com/goharbor/harbor/src/lib/q" +) + +// Manager is an autogenerated mock type for the Manager type +type Manager struct { + mock.Mock +} + +// Count provides a mock function with given fields: ctx, query +func (_m *Manager) Count(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Create provides a mock function with given fields: ctx, registry +func (_m *Manager) Create(ctx context.Context, registry *model.Registry) (int64, error) { + ret := _m.Called(ctx, registry) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *model.Registry) int64); ok { + r0 = rf(ctx, registry) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *model.Registry) error); ok { + r1 = rf(ctx, registry) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: ctx, id +func (_m *Manager) Delete(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Get provides a mock function with given fields: ctx, id +func (_m *Manager) Get(ctx context.Context, id int64) (*model.Registry, error) { + ret := _m.Called(ctx, id) + + var r0 *model.Registry + if rf, ok := ret.Get(0).(func(context.Context, int64) *model.Registry); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Registry) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// List provides a mock function with given fields: ctx, query +func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*model.Registry, error) { + ret := _m.Called(ctx, query) + + var r0 []*model.Registry + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*model.Registry); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.Registry) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Update provides a mock function with given fields: ctx, registry, props +func (_m *Manager) Update(ctx context.Context, registry *model.Registry, props ...string) error { + _va := make([]interface{}, len(props)) + for _i := range props { + _va[_i] = props[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, registry) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *model.Registry, ...string) error); ok { + r0 = rf(ctx, registry, props...) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/testing/pkg/replication/dao/dao.go b/src/testing/pkg/replication/dao/dao.go new file mode 100644 index 000000000..12603a25d --- /dev/null +++ b/src/testing/pkg/replication/dao/dao.go @@ -0,0 +1,140 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package dao + +import ( + context "context" + + dao "github.com/goharbor/harbor/src/pkg/replication/dao" + mock "github.com/stretchr/testify/mock" + + q "github.com/goharbor/harbor/src/lib/q" +) + +// DAO is an autogenerated mock type for the DAO type +type DAO struct { + mock.Mock +} + +// Count provides a mock function with given fields: ctx, query +func (_m *DAO) Count(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Create provides a mock function with given fields: ctx, policy +func (_m *DAO) Create(ctx context.Context, policy *dao.Policy) (int64, error) { + ret := _m.Called(ctx, policy) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *dao.Policy) int64); ok { + r0 = rf(ctx, policy) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *dao.Policy) error); ok { + r1 = rf(ctx, policy) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: ctx, id +func (_m *DAO) Delete(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Get provides a mock function with given fields: ctx, id +func (_m *DAO) Get(ctx context.Context, id int64) (*dao.Policy, error) { + ret := _m.Called(ctx, id) + + var r0 *dao.Policy + if rf, ok := ret.Get(0).(func(context.Context, int64) *dao.Policy); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*dao.Policy) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// List provides a mock function with given fields: ctx, query +func (_m *DAO) List(ctx context.Context, query *q.Query) ([]*dao.Policy, error) { + ret := _m.Called(ctx, query) + + var r0 []*dao.Policy + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*dao.Policy); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*dao.Policy) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Update provides a mock function with given fields: ctx, policy, props +func (_m *DAO) Update(ctx context.Context, policy *dao.Policy, props ...string) error { + _va := make([]interface{}, len(props)) + for _i := range props { + _va[_i] = props[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, policy) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *dao.Policy, ...string) error); ok { + r0 = rf(ctx, policy, props...) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/testing/pkg/replication/manager.go b/src/testing/pkg/replication/manager.go new file mode 100644 index 000000000..b1995f727 --- /dev/null +++ b/src/testing/pkg/replication/manager.go @@ -0,0 +1,140 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package manager + +import ( + context "context" + + q "github.com/goharbor/harbor/src/lib/q" + mock "github.com/stretchr/testify/mock" + + replication "github.com/goharbor/harbor/src/pkg/replication" +) + +// Manager is an autogenerated mock type for the Manager type +type Manager struct { + mock.Mock +} + +// Count provides a mock function with given fields: ctx, query +func (_m *Manager) Count(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Create provides a mock function with given fields: ctx, policy +func (_m *Manager) Create(ctx context.Context, policy *replication.Policy) (int64, error) { + ret := _m.Called(ctx, policy) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *replication.Policy) int64); ok { + r0 = rf(ctx, policy) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *replication.Policy) error); ok { + r1 = rf(ctx, policy) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: ctx, id +func (_m *Manager) Delete(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Get provides a mock function with given fields: ctx, id +func (_m *Manager) Get(ctx context.Context, id int64) (*replication.Policy, error) { + ret := _m.Called(ctx, id) + + var r0 *replication.Policy + if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Policy); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*replication.Policy) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// List provides a mock function with given fields: ctx, query +func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*replication.Policy, error) { + ret := _m.Called(ctx, query) + + var r0 []*replication.Policy + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Policy); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*replication.Policy) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Update provides a mock function with given fields: ctx, policy, props +func (_m *Manager) Update(ctx context.Context, policy *replication.Policy, props ...string) error { + _va := make([]interface{}, len(props)) + for _i := range props { + _va[_i] = props[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, policy) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *replication.Policy, ...string) error); ok { + r0 = rf(ctx, policy, props...) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/testing/pkg/task/execution_manager.go b/src/testing/pkg/task/execution_manager.go index 0467f9779..17ffac959 100644 --- a/src/testing/pkg/task/execution_manager.go +++ b/src/testing/pkg/task/execution_manager.go @@ -81,6 +81,20 @@ func (_m *ExecutionManager) Delete(ctx context.Context, id int64) error { return r0 } +// DeleteByVendor provides a mock function with given fields: ctx, vendorType, vendorID +func (_m *ExecutionManager) DeleteByVendor(ctx context.Context, vendorType string, vendorID int64) error { + ret := _m.Called(ctx, vendorType, vendorID) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, int64) error); ok { + r0 = rf(ctx, vendorType, vendorID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Get provides a mock function with given fields: ctx, id func (_m *ExecutionManager) Get(ctx context.Context, id int64) (*task.Execution, error) { ret := _m.Called(ctx, id) diff --git a/tests/apitests/python/library/replication.py b/tests/apitests/python/library/replication.py index fd80e513e..3f173e26d 100644 --- a/tests/apitests/python/library/replication.py +++ b/tests/apitests/python/library/replication.py @@ -1,35 +1,57 @@ # -*- coding: utf-8 -*- -import sys import time import base -import swagger_client +import v2_swagger_client + +class Replication(base.Base, object): + def __init__(self): + super(Replication,self).__init__(api_type = "replication") + + def wait_until_jobs_finish(self, rule_id, retry=10, interval=5, **kwargs): + Succeed = False + for i in range(retry): + Succeed = False + jobs = self.get_replication_executions(rule_id, **kwargs) + for job in jobs: + if job.status == "Succeed": + return + if not Succeed: + time.sleep(interval) + if not Succeed: + raise Exception("The jobs not Succeed") + + def trigger_replication_executions(self, rule_id, expect_status_code = 201, **kwargs): + _, status_code, _ = self._get_client(**kwargs).start_replication_with_http_info({"policy_id":rule_id}) + base._assert_status_code(expect_status_code, status_code) + + def get_replication_executions(self, rule_id, expect_status_code = 200, **kwargs): + data, status_code, _ = self._get_client(**kwargs).list_replication_executions_with_http_info(policy_id=rule_id) + base._assert_status_code(expect_status_code, status_code) + return data -class Replication(base.Base): def create_replication_policy(self, dest_registry=None, src_registry=None, name=None, description="", - dest_namespace = "", filters=None, trigger=swagger_client.ReplicationTrigger(type="manual",trigger_settings=swagger_client.TriggerSettings(cron="")), - deletion=False, override=True, enabled=True, expect_status_code = 201, **kwargs): + dest_namespace = "", filters=None, trigger=v2_swagger_client.ReplicationTrigger(type="manual",trigger_settings=v2_swagger_client.ReplicationTriggerSettings(cron="")), + deletion=False, override=True, enabled=True, expect_status_code = 201, **kwargs): if name is None: name = base._random_name("rule") if filters is None: filters = [] - client = self._get_client(**kwargs) - policy = swagger_client.ReplicationPolicy(name=name, description=description,dest_namespace=dest_namespace, - dest_registry=dest_registry, src_registry=src_registry,filters=filters, - trigger=trigger, deletion=deletion, override=override, enabled=enabled) - _, status_code, header = client.replication_policies_post_with_http_info(policy) + policy = v2_swagger_client.ReplicationPolicy(name=name, description=description,dest_namespace=dest_namespace, + dest_registry=dest_registry, src_registry=src_registry,filters=filters, + trigger=trigger, deletion=deletion, override=override, enabled=enabled) + _, status_code, header = self._get_client(**kwargs).create_replication_policy_with_http_info(policy) base._assert_status_code(expect_status_code, status_code) return base._get_id_from_header(header), name def get_replication_rule(self, param = None, rule_id = None, expect_status_code = 200, **kwargs): - client = self._get_client(**kwargs) if rule_id is None: if param is None: param = dict() - data, status_code, _ = client.replication_policies_id_get_with_http_info(param) + data, status_code, _ = self._get_client(**kwargs).get_replication_policy_with_http_info(param) else: - data, status_code, _ = client.replication_policies_id_get_with_http_info(rule_id) + data, status_code, _ = self._get_client(**kwargs).get_replication_policy_with_http_info(rule_id) base._assert_status_code(expect_status_code, status_code) return data @@ -46,6 +68,6 @@ class Replication(base.Base): # raise Exception(r"Check replication rule trigger failed, expect <{}> actual <{}>.".format(expect_trigger, get_trigger)) def delete_replication_rule(self, rule_id, expect_status_code = 200, **kwargs): - client = self._get_client(**kwargs) - _, status_code, _ = client.replication_policies_id_delete_with_http_info(rule_id) - base._assert_status_code(expect_status_code, status_code) \ No newline at end of file + _, status_code, _ = self._get_client(**kwargs).delete_replication_policy_with_http_info(rule_id) + base._assert_status_code(expect_status_code, status_code) + diff --git a/tests/apitests/python/library/replication_v2.py b/tests/apitests/python/library/replication_v2.py deleted file mode 100644 index 721b0b49d..000000000 --- a/tests/apitests/python/library/replication_v2.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- - -import time -import base -import v2_swagger_client -from v2_swagger_client.rest import ApiException - -class ReplicationV2(base.Base, object): - def __init__(self): - super(ReplicationV2,self).__init__(api_type = "replication") - - def wait_until_jobs_finish(self, rule_id, retry=10, interval=5, **kwargs): - Succeed = False - for i in range(retry): - Succeed = False - jobs = self.get_replication_executions(rule_id, **kwargs) - for job in jobs: - if job.status == "Succeed": - return - if not Succeed: - time.sleep(interval) - if not Succeed: - raise Exception("The jobs not Succeed") - - def trigger_replication_executions(self, rule_id, expect_status_code = 201, **kwargs): - _, status_code, _ = self._get_client(**kwargs).start_replication_with_http_info({"policy_id":rule_id}) - base._assert_status_code(expect_status_code, status_code) - - def get_replication_executions(self, rule_id, expect_status_code = 200, **kwargs): - data, status_code, _ = self._get_client(**kwargs).list_replication_executions_with_http_info(policy_id=rule_id) - base._assert_status_code(expect_status_code, status_code) - return data - diff --git a/tests/apitests/python/test_replication_from_dockerhub.py b/tests/apitests/python/test_replication_from_dockerhub.py index b59d5d7ae..406df3959 100644 --- a/tests/apitests/python/test_replication_from_dockerhub.py +++ b/tests/apitests/python/test_replication_from_dockerhub.py @@ -9,8 +9,8 @@ from library.replication import Replication from library.registry import Registry from library.artifact import Artifact from library.repository import Repository -from library.replication_v2 import ReplicationV2 import swagger_client +import v2_swagger_client from testutils import DOCKER_USER, DOCKER_PWD class TestProjects(unittest.TestCase): @@ -19,7 +19,6 @@ class TestProjects(unittest.TestCase): self.project = Project() self.user = User() self.replication = Replication() - self.replication_v2 = ReplicationV2() self.registry = Registry() self.artifact = Artifact() self.repo = Repository() @@ -83,17 +82,17 @@ class TestProjects(unittest.TestCase): #4. Create a pull-based rule for this registry; TestProjects.rule_id, rule_name = self.replication.create_replication_policy(src_registry=swagger_client.Registry(id=int(TestProjects.registry_id)), dest_namespace=TestProjects.project_name, - filters=[swagger_client.ReplicationFilter(type="name",value="library/"+self.image),swagger_client.ReplicationFilter(type="tag",value=self.tag)], + filters=[v2_swagger_client.ReplicationFilter(type="name",value="library/"+self.image),v2_swagger_client.ReplicationFilter(type="tag",value=self.tag)], **ADMIN_CLIENT) #5. Check rule should be exist; self.replication.check_replication_rule_should_exist(TestProjects.rule_id, rule_name, **ADMIN_CLIENT) #6. Trigger the rule; - self.replication_v2.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT) + self.replication.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT) #7. Wait for completion of this replication job; - self.replication_v2.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT) + self.replication.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT) #8. Check image is replicated into target project successfully. artifact = self.artifact.get_reference_info(TestProjects.project_name, self.image, self.tag, **ADMIN_CLIENT) diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index e1ad73d94..591c0cf05 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -159,4 +159,4 @@ Test Case - Metrics Test Case - Project Level Policy Content Trust [Tags] content_trust - Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py + Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py \ No newline at end of file