From 294385c34dec9800fb2a35207a31bb21b1677b99 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 30 Sep 2020 15:14:00 +0800 Subject: [PATCH] Refactor the replication execution 1. Use the task manager to manage the underlying execution/task 2. Use the pkg/scheduler to schedule the periodical job 3. Apply the new program model 4. Migration the old data into the new data model Signed-off-by: Wenkai Yin --- api/v2.0/legacy_swagger.yaml | 296 ------------ api/v2.0/swagger.yaml | 295 ++++++++++++ .../postgresql/0050_2.2.0_schema.up.sql | 194 ++++++++ src/controller/event/handler/init.go | 13 + .../handler/webhook/artifact/replication.go | 22 +- .../webhook/artifact/replication_test.go | 55 +-- src/controller/replication/controller.go | 242 ++++++++++ src/controller/replication/controller_test.go | 223 +++++++++ src/controller/replication/flow/controller.go | 51 ++ src/controller/replication/flow/copy.go | 130 +++++ src/controller/replication/flow/copy_test.go | 83 ++++ src/controller/replication/flow/deletion.go | 103 ++++ .../replication}/flow/deletion_test.go | 33 +- src/controller/replication/flow/mock.go | 28 ++ .../flow/mock_adapter_factory_test.go | 54 +++ .../replication/flow/mock_adapter_test.go | 278 +++++++++++ .../replication}/flow/stage.go | 109 +---- src/controller/replication/flow/stage_test.go | 221 +++++++++ .../replication/mock.go} | 17 +- .../replication/mock_flow_controller_test.go | 30 ++ src/controller/replication/model.go | 51 ++ src/core/api/base.go | 20 +- src/core/api/harborapi_test.go | 4 - src/core/api/replication_execution.go | 248 ---------- src/core/api/replication_execution_test.go | 443 ------------------ src/core/api/replication_policy.go | 43 +- src/core/api/replication_policy_test.go | 44 ++ .../service/notifications/jobs/handler.go | 41 -- .../notifications/scheduler/handler.go | 51 -- src/lib/log/context.go | 5 +- .../controller_test.go => lib/orm/creator.go} | 33 +- src/lib/q/query.go | 15 +- .../flow/controller.go => lib/worker_pool.go} | 30 +- src/pkg/scheduler/callback.go | 7 +- src/pkg/scheduler/callback_test.go | 7 +- src/pkg/scheduler/scheduler.go | 35 -- src/pkg/scheduler/scheduler_test.go | 6 +- src/pkg/task/checkin.go | 38 -- src/pkg/task/dao/execution.go | 70 ++- src/pkg/task/dao/execution_test.go | 61 ++- src/pkg/task/dao/model.go | 3 +- src/pkg/task/dao/task.go | 31 +- src/pkg/task/dao/task_test.go | 32 +- src/pkg/task/execution.go | 11 +- src/pkg/task/execution_test.go | 3 + src/pkg/task/hook.go | 69 ++- src/pkg/task/hook_test.go | 33 +- src/pkg/task/mock_execution_dao_test.go | 24 +- src/pkg/task/model.go | 33 +- src/pkg/task/registry.go | 62 +++ src/pkg/task/registry_test.go | 48 ++ src/pkg/task/task.go | 30 +- src/pkg/task/task_test.go | 13 +- src/replication/config/config.go | 2 - src/replication/dao/base.go | 13 - src/replication/dao/execution.go | 355 -------------- src/replication/dao/execution_test.go | 294 ------------ src/replication/dao/models/base.go | 11 +- src/replication/dao/models/execution.go | 156 ------ src/replication/dao/models/schedule_job.go | 40 -- src/replication/dao/schedule_job.go | 92 ---- src/replication/dao/schedule_job_test.go | 81 ---- src/replication/event/handler.go | 12 +- src/replication/event/handler_test.go | 45 +- src/replication/operation/controller.go | 243 ---------- src/replication/operation/controller_test.go | 390 --------------- .../operation/execution/execution.go | 182 ------- .../operation/execution/execution_test.go | 135 ------ src/replication/operation/flow/copy.go | 109 ----- src/replication/operation/flow/copy_test.go | 36 -- src/replication/operation/flow/deletion.go | 68 --- src/replication/operation/flow/stage_test.go | 427 ----------------- src/replication/operation/hook/task.go | 47 -- src/replication/operation/hook/task_test.go | 94 ---- .../operation/scheduler/scheduler.go | 141 ------ .../operation/scheduler/scheduler_test.go | 79 ---- .../policy/controller/controller.go | 26 +- .../policy/controller/controller_test.go | 44 +- src/replication/policy/scheduler/scheduler.go | 121 ----- .../policy/scheduler/scheduler_test.go | 172 ------- src/replication/replication.go | 47 +- src/replication/replication_test.go | 1 - src/server/handler/job_status_hook.go | 17 +- src/server/route.go | 9 +- src/server/v2.0/handler/handler.go | 15 +- src/server/v2.0/handler/preheat.go | 2 +- src/server/v2.0/handler/replication.go | 297 ++++++++++++ src/server/v2.0/route/legacy.go | 4 - src/testing/controller/controller.go | 1 + .../controller/replication/controller.go | 211 +++++++++ .../status.go => testing/lib/lib.go} | 15 +- src/testing/lib/orm/creator.go | 29 ++ tests/apitests/python/library/base.py | 3 +- tests/apitests/python/library/replication.py | 36 +- .../apitests/python/library/replication_v2.py | 35 ++ .../python/test_replication_from_dockerhub.py | 6 +- 96 files changed, 3290 insertions(+), 4974 deletions(-) create mode 100644 src/controller/replication/controller.go create mode 100644 src/controller/replication/controller_test.go create mode 100644 src/controller/replication/flow/controller.go create mode 100644 src/controller/replication/flow/copy.go create mode 100644 src/controller/replication/flow/copy_test.go create mode 100644 src/controller/replication/flow/deletion.go rename src/{replication/operation => controller/replication}/flow/deletion_test.go (63%) create mode 100644 src/controller/replication/flow/mock.go create mode 100644 src/controller/replication/flow/mock_adapter_factory_test.go create mode 100644 src/controller/replication/flow/mock_adapter_test.go rename src/{replication/operation => controller/replication}/flow/stage.go (64%) create mode 100644 src/controller/replication/flow/stage_test.go rename src/{pkg/task/checkin_test.go => controller/replication/mock.go} (68%) create mode 100644 src/controller/replication/mock_flow_controller_test.go create mode 100644 src/controller/replication/model.go delete mode 100644 src/core/api/replication_execution.go delete mode 100644 src/core/api/replication_execution_test.go delete mode 100644 src/core/service/notifications/scheduler/handler.go rename src/{replication/operation/flow/controller_test.go => lib/orm/creator.go} (58%) rename src/{replication/operation/flow/controller.go => lib/worker_pool.go} (53%) delete mode 100644 src/pkg/task/checkin.go create mode 100644 src/pkg/task/registry.go create mode 100644 src/pkg/task/registry_test.go delete mode 100644 src/replication/dao/base.go delete mode 100644 src/replication/dao/execution.go delete mode 100644 src/replication/dao/execution_test.go delete mode 100644 src/replication/dao/models/execution.go delete mode 100644 src/replication/dao/models/schedule_job.go delete mode 100644 src/replication/dao/schedule_job.go delete mode 100644 src/replication/dao/schedule_job_test.go delete mode 100644 src/replication/operation/controller.go delete mode 100644 src/replication/operation/controller_test.go delete mode 100644 src/replication/operation/execution/execution.go delete mode 100644 src/replication/operation/execution/execution_test.go delete mode 100644 src/replication/operation/flow/copy.go delete mode 100644 src/replication/operation/flow/copy_test.go delete mode 100644 src/replication/operation/flow/deletion.go delete mode 100644 src/replication/operation/flow/stage_test.go delete mode 100644 src/replication/operation/hook/task.go delete mode 100644 src/replication/operation/hook/task_test.go delete mode 100644 src/replication/operation/scheduler/scheduler.go delete mode 100644 src/replication/operation/scheduler/scheduler_test.go delete mode 100644 src/replication/policy/scheduler/scheduler.go delete mode 100644 src/replication/policy/scheduler/scheduler_test.go create mode 100644 src/server/v2.0/handler/replication.go create mode 100644 src/testing/controller/replication/controller.go rename src/{replication/policy/scheduler/status.go => testing/lib/lib.go} (63%) create mode 100644 src/testing/lib/orm/creator.go create mode 100644 tests/apitests/python/library/replication_v2.py diff --git a/api/v2.0/legacy_swagger.yaml b/api/v2.0/legacy_swagger.yaml index 30468f6f8..4c33e565c 100644 --- a/api/v2.0/legacy_swagger.yaml +++ b/api/v2.0/legacy_swagger.yaml @@ -738,231 +738,6 @@ paths: description: The auth mode of the system is not "oidc_auth", or the user is not onboarded via OIDC AuthN. '500': description: Unexpected internal errors. - /replication/executions: - get: - summary: List replication executions. - description: | - This endpoint let user list replication executions. - parameters: - - name: policy_id - in: query - type: integer - required: false - description: The policy ID. - - name: status - in: query - type: string - required: false - description: The execution status. - - name: trigger - in: query - type: string - required: false - description: The trigger mode. - - name: page - in: query - type: integer - required: false - description: The page. - - name: page_size - in: query - type: integer - required: false - description: The page size. - tags: - - Products - responses: - '200': - description: Success - schema: - type: array - items: - $ref: '#/definitions/ReplicationExecution' - headers: - X-Total-Count: - description: The total count of available items - type: integer - Link: - description: Link to previous page and next page - type: string - '401': - description: User need to login first. - '403': - description: User has no privilege for the operation. - '500': - description: Unexpected internal errors. - post: - summary: Start one execution of the replication. - description: | - This endpoint is for user to start one execution of the replication. - parameters: - - name: execution - in: body - description: The execution that needs to be started, only the property "policy_id" is needed. - required: true - schema: - $ref: '#/definitions/ReplicationExecution' - tags: - - Products - responses: - '201': - description: Success. - headers: - Location: - type: string - description: The URL of the created resource - '400': - description: Bad request. - '401': - description: User need to login first. - '403': - description: User has no privilege for the operation. - '415': - $ref: '#/responses/UnsupportedMediaType' - '500': - description: Unexpected internal errors. - /replication/executions/{id}: - get: - summary: Get the execution of the replication. - description: | - This endpoint is for user to get one execution of the replication. - parameters: - - name: id - in: path - type: integer - format: int64 - description: The execution ID. - required: true - tags: - - Products - responses: - '200': - description: Success. - schema: - $ref: '#/definitions/ReplicationExecution' - '400': - description: Bad request. - '401': - description: User need to login first. - '403': - description: User has no privilege for the operation. - '404': - description: Resource requested does not exist. - '415': - $ref: '#/responses/UnsupportedMediaType' - '500': - description: Unexpected internal errors. - put: - summary: Stop the execution of the replication. - description: | - This endpoint is for user to stop one execution of the replication. - parameters: - - name: id - in: path - type: integer - format: int64 - description: The execution ID. - required: true - tags: - - Products - responses: - '200': - description: Success. - '400': - description: Bad request. - '401': - description: User need to login first. - '403': - description: User has no privilege for the operation. - '404': - description: Resource requested does not exist. - '415': - $ref: '#/responses/UnsupportedMediaType' - '500': - description: Unexpected internal errors. - /replication/executions/{id}/tasks: - get: - summary: Get the task list of one execution. - description: | - This endpoint is for user to get the task list of one execution. - parameters: - - name: id - in: path - type: integer - format: int64 - description: The execution ID. - required: true - - name: page - in: query - type: integer - format: int32 - required: false - description: The page number. - - name: page_size - in: query - type: integer - format: int32 - required: false - description: The size of per page. - tags: - - Products - responses: - '200': - description: Success. - schema: - type: array - items: - $ref: '#/definitions/ReplicationTask' - headers: - X-Total-Count: - description: The total count of available items - type: integer - Link: - description: Link to previous page and next page - type: string - '400': - description: Bad request. - '401': - description: User need to login first. - '403': - description: User has no privilege for the operation. - '404': - description: Resource requested does not exist. - '500': - description: Unexpected internal errors. - /replication/executions/{id}/tasks/{task_id}/log: - get: - summary: Get the log of one task. - description: | - This endpoint is for user to get the log of one task. - parameters: - - name: id - in: path - type: integer - format: int64 - description: The execution ID. - required: true - - name: task_id - in: path - type: integer - format: int64 - description: The task ID. - required: true - tags: - - Products - responses: - '200': - description: Success. - '400': - description: Bad request. - '401': - description: User need to login first. - '403': - description: User has no privilege for the operation. - '404': - description: Resource requested does not exist. - '500': - description: Unexpected internal errors. /replication/policies: get: summary: List replication policies @@ -4972,77 +4747,6 @@ definitions: description: The filter values items: type: string - ReplicationExecution: - type: object - description: The replication execution - properties: - id: - type: integer - description: The ID - policy_id: - type: integer - description: The policy ID - status: - type: string - description: The status - status_text: - type: string - description: The status text - trigger: - type: string - description: The trigger mode - total: - type: integer - description: The total count of all tasks - failed: - type: integer - description: The count of failed tasks - succeed: - type: integer - description: The count of succeed tasks - in_progress: - type: integer - description: The count of in_progress tasks - stopped: - type: integer - description: The count of stopped tasks - start_time: - type: string - description: The start time - end_time: - type: string - description: The end time - ReplicationTask: - type: object - description: The replication task - properties: - id: - type: integer - description: The ID - execution_id: - type: integer - description: The execution ID - resource_type: - type: string - description: The resource type - src_resource: - type: string - description: The source resource - dst_resource: - type: string - description: The destination resource - job_id: - type: string - description: The job ID - status: - type: string - description: The status - start_time: - type: string - description: The start time - end_time: - type: string - description: The end time Namespace: type: object description: The namespace of registry diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 35c7cea3c..bf1c65b30 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -1431,6 +1431,209 @@ paths: $ref: '#/responses/404' '500': $ref: '#/responses/500' + /replication/executions: + get: + summary: List replication executions + description: List replication executions + tags: + - replication + operationId: listReplicationExecutions + parameters: + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - name: policy_id + in: query + type: integer + required: false + description: The ID of the policy that the executions belong to. + - name: status + in: query + type: string + required: false + description: The execution status. + - name: trigger + in: query + type: string + required: false + description: The trigger mode. + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of the resources + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/ReplicationExecution' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '500': + $ref: '#/responses/500' + post: + summary: Start one replication execution + description: Start one replication execution according to the policy + tags: + - replication + operationId: startReplication + parameters: + - name: execution + in: body + description: The ID of policy that the execution belongs to + required: true + schema: + $ref: '#/definitions/StartReplicationExecution' + responses: + '201': + $ref: '#/responses/201' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '500': + $ref: '#/responses/500' + /replication/executions/{id}: + get: + summary: Get the specific replication execution + description: Get the replication execution specified by ID + tags: + - replication + operationId: getReplicationExecution + parameters: + - name: id + in: path + type: integer + format: int64 + description: The ID of the execution. + required: true + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ReplicationExecution' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + put: + summary: Stop the specific replication execution + description: Stop the replication execution specified by ID + tags: + - replication + operationId: stopReplication + parameters: + - name: id + in: path + type: integer + format: int64 + description: The ID of the execution. + required: true + responses: + '200': + $ref: '#/responses/200' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + /replication/executions/{id}/tasks: + get: + summary: List replication tasks for a specific execution + description: List replication tasks for a specific execution + tags: + - replication + operationId: listReplicationTasks + parameters: + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - name: id + in: path + type: integer + format: int64 + description: The ID of the execution that the tasks belongs to. + required: true + - name: status + in: query + type: string + required: false + description: The task status. + - name: resource_type + in: query + type: string + required: false + description: The resource type. + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of the resources + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/ReplicationTask' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '500': + $ref: '#/responses/500' + /replication/executions/{id}/tasks/{task_id}/log: + get: + summary: Get the log of the specific replication task + description: Get the log of the specific replication task + tags: + - replication + operationId: getReplicationLog + parameters: + - name: id + in: path + type: integer + format: int64 + description: The ID of the execution that the tasks belongs to. + required: true + - name: task_id + in: path + type: integer + format: int64 + description: The ID of the task. + required: true + responses: + '200': + description: Success + headers: + Content-Type: + description: The content type of response body + type: string + schema: + type: string + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' parameters: query: name: q @@ -2123,6 +2326,7 @@ definitions: description: The status message of task run_count: type: integer + format: int32 description: The count of task run extra_attrs: $ref: '#/definitions/ExtraAttrs' @@ -2398,3 +2602,94 @@ definitions: additionalProperties: type: integer format: int64 + ReplicationExecution: + type: object + description: The replication execution + properties: + id: + type: integer + description: The ID of the execution + policy_id: + type: integer + description: The ID if the policy that the execution belongs to + status: + type: string + description: The status of the execution + trigger: + type: string + description: The trigger mode + start_time: + type: string + format: date-time + description: The start time + end_time: + type: string + format: date-time + description: The end time + status_text: + type: string + x-omitempty: false + description: The status text + total: + type: integer + x-omitempty: false + description: The total count of all executions + failed: + type: integer + x-omitempty: false + description: The count of failed executions + succeed: + type: integer + x-omitempty: false + description: The count of succeed executions + in_progress: + type: integer + x-omitempty: false + description: The count of in_progress executions + stopped: + type: integer + x-omitempty: false + description: The count of stopped executions + StartReplicationExecution: + type: object + properties: + policy_id: + type: integer + format: int64 + description: The ID of policy that the execution belongs to. + ReplicationTask: + type: object + description: The replication task + properties: + id: + type: integer + description: The ID of the task + execution_id: + type: integer + description: The ID of the execution that the task belongs to + status: + type: string + description: The status of the task + job_id: + type: string + description: The ID of the underlying job that the task related to + operation: + type: string + description: The operation of the task + resource_type: + type: string + description: The type of the resource that the task operates + src_resource: + type: string + description: The source resource that the task operates + dst_resource: + type: string + description: The destination resource that the task operates + start_time: + type: string + format: date-time + description: The start time of the task + end_time: + type: string + format: date-time + description: The end time of the task diff --git a/make/migrations/postgresql/0050_2.2.0_schema.up.sql b/make/migrations/postgresql/0050_2.2.0_schema.up.sql index dc54298dc..4deeb9ce1 100644 --- a/make/migrations/postgresql/0050_2.2.0_schema.up.sql +++ b/make/migrations/postgresql/0050_2.2.0_schema.up.sql @@ -7,6 +7,10 @@ UPDATE role SET role_id=5 WHERE name='limitedGuest' AND role_id!=5; ALTER TABLE schedule ADD COLUMN IF NOT EXISTS cron_type varchar(64); +ALTER TABLE task ADD COLUMN IF NOT EXISTS vendor_type varchar(16); +UPDATE task SET vendor_type = execution.vendor_type FROM execution WHERE task.execution_id = execution.id; +ALTER TABLE task ALTER COLUMN vendor_type SET NOT NULL; + DO $$ DECLARE art RECORD; @@ -45,3 +49,193 @@ CREATE TABLE IF NOT EXISTS permission_policy ( creation_time timestamp default CURRENT_TIMESTAMP, CONSTRAINT unique_rbac_policy UNIQUE (scope, resource, action, effect) ); + +/*delete the replication execution records whose policy doesn't exist*/ +DELETE FROM replication_execution + WHERE id IN (SELECT re.id FROM replication_execution re + LEFT JOIN replication_policy rp ON re.policy_id=rp.id + WHERE rp.id IS NULL); + +/*delete the replication task records whose execution doesn't exist*/ +DELETE FROM replication_task + WHERE id IN (SELECT rt.id FROM replication_task rt + LEFT JOIN replication_execution re ON rt.execution_id=re.id + WHERE re.id IS NULL); + +/*fill the task count, status and end_time of execution based on the tasks*/ +DO $$ +DECLARE + rep_exec RECORD; + status_count RECORD; + rep_status varchar(32); +BEGIN + FOR rep_exec IN SELECT * FROM replication_execution + LOOP + /*the replication status is set directly in some cases, so skip if the status is a final one*/ + IF rep_exec.status='Stopped' OR rep_exec.status='Failed' OR rep_exec.status='Succeed' THEN + CONTINUE; + END IF; + /*fulfill the status count*/ + FOR status_count IN SELECT status, COUNT(*) as c FROM replication_task WHERE execution_id=rep_exec.id GROUP BY status + LOOP + IF status_count.status = 'Stopped' THEN + UPDATE replication_execution SET stopped=status_count.c WHERE id=rep_exec.id; + ELSIF status_count.status = 'Failed' THEN + UPDATE replication_execution SET failed=status_count.c WHERE id=rep_exec.id; + ELSIF status_count.status = 'Succeed' THEN + UPDATE replication_execution SET succeed=status_count.c WHERE id=rep_exec.id; + ELSE + UPDATE replication_execution SET in_progress=status_count.c WHERE id=rep_exec.id; + END IF; + END LOOP; + + /*reload the execution record*/ + SELECT * INTO rep_exec FROM replication_execution where id=rep_exec.id; + + /*calculate the status*/ + IF rep_exec.in_progress>0 THEN + rep_status = 'InProgress'; + ELSIF rep_exec.failed>0 THEN + rep_status = 'Failed'; + ELSIF rep_exec.stopped>0 THEN + rep_status = 'Stopped'; + ELSE + rep_status = 'Succeed'; + END IF; + UPDATE replication_execution SET status=rep_status WHERE id=rep_exec.id; + + /*update the end time if the status is a final one*/ + IF rep_status='Failed' OR rep_status='Stopped' OR rep_status='Succeed' THEN + UPDATE replication_execution + SET end_time=(SELECT MAX (end_time) FROM replication_task WHERE execution_id=rep_exec.id) + WHERE id=rep_exec.id; + END IF; + END LOOP; +END $$; + +/*move the replication execution records into the new execution table*/ +ALTER TABLE replication_execution ADD COLUMN IF NOT EXISTS new_execution_id int; +DO $$ +DECLARE + rep_exec RECORD; + trigger varchar(64); + status varchar(32); + new_exec_id integer; +BEGIN + FOR rep_exec IN SELECT * FROM replication_execution + LOOP + IF rep_exec.trigger = 'scheduled' THEN + trigger = 'SCHEDULE'; + ELSIF rep_exec.trigger = 'event_based' THEN + trigger = 'EVENT'; + ELSE + trigger = 'MANUAL'; + END IF; + + IF rep_exec.status = 'InProgress' THEN + status = 'Running'; + ELSIF rep_exec.status = 'Stopped' THEN + status = 'Stopped'; + ELSIF rep_exec.status = 'Failed' THEN + status = 'Error'; + ELSIF rep_exec.status = 'Succeed' THEN + status = 'Success'; + END IF; + + INSERT INTO execution (vendor_type, vendor_id, status, status_message, revision, trigger, start_time, end_time) + VALUES ('REPLICATION', rep_exec.policy_id, status, rep_exec.status_text, 0, trigger, rep_exec.start_time, rep_exec.end_time) RETURNING id INTO new_exec_id; + UPDATE replication_execution SET new_execution_id=new_exec_id WHERE id=rep_exec.id; + END LOOP; +END $$; + +/*move the replication task records into the new task table*/ +DO $$ +DECLARE + rep_task RECORD; + status varchar(32); + status_code integer; +BEGIN + FOR rep_task IN SELECT * FROM replication_task + LOOP + IF rep_task.status = 'InProgress' THEN + status = 'Running'; + status_code = 2; + ELSIF rep_task.status = 'Stopped' THEN + status = 'Stopped'; + status_code = 3; + ELSIF rep_task.status = 'Failed' THEN + status = 'Error'; + status_code = 3; + ELSIF rep_task.status = 'Succeed' THEN + status = 'Success'; + status_code = 3; + ELSE + status = 'Pending'; + status_code = 0; + END IF; + INSERT INTO task (vendor_type, execution_id, job_id, status, status_code, status_revision, + run_count, extra_attrs, creation_time, start_time, update_time, end_time) + VALUES ('REPLICATION', (SELECT new_execution_id FROM replication_execution WHERE id=rep_task.execution_id), + rep_task.job_id, status, status_code, rep_task.status_revision, + 1, CONCAT('{"resource_type":"', rep_task.resource_type,'","source_resource":"', rep_task.src_resource, '","destination_resource":"', rep_task.dst_resource, '","operation":"', rep_task.operation,'"}')::json, + rep_task.start_time, rep_task.start_time, rep_task.end_time, rep_task.end_time); + END LOOP; +END $$; + +DROP TABLE IF EXISTS replication_task; +DROP TABLE IF EXISTS replication_execution; + +/*move the replication schedule job records into the new schedule table*/ +DO $$ +DECLARE + schd RECORD; + new_schd_id integer; + exec_id integer; + exec_status varchar(32); + task_status varchar(32); + task_status_code integer; +BEGIN + FOR schd IN SELECT * FROM replication_schedule_job + LOOP + INSERT INTO schedule (vendor_type, vendor_id, cron, callback_func_name, + callback_func_param, creation_time, update_time) + VALUES ('REPLICATION', schd.policy_id, + (SELECT trigger::json->'trigger_settings'->>'cron' FROM replication_policy WHERE id=schd.policy_id), + 'REPLICATION_CALLBACK', schd.policy_id, schd.creation_time, schd.update_time) RETURNING id INTO new_schd_id; + IF schd.status = 'stopped' THEN + exec_status = 'Stopped'; + task_status = 'Stopped'; + task_status_code = 3; + ELSIF schd.status = 'error' THEN + exec_status = 'Error'; + task_status = 'Error'; + task_status_code = 3; + ELSIF schd.status = 'finished' THEN + exec_status = 'Success'; + task_status = 'Success'; + task_status_code = 3; + ELSIF schd.status = 'running' THEN + exec_status = 'Running'; + task_status = 'Running'; + task_status_code = 2; + ELSEIF schd.status = 'pending' THEN + exec_status = 'Running'; + task_status = 'Pending'; + task_status_code = 0; + ELSEIF schd.status = 'scheduled' THEN + exec_status = 'Running'; + task_status = 'Scheduled'; + task_status_code = 1; + ELSE + exec_status = 'Running'; + task_status = 'Pending'; + task_status_code = 0; + END IF; + INSERT INTO execution (vendor_type, vendor_id, status, revision, trigger, start_time, end_time) + VALUES ('SCHEDULER', new_schd_id, exec_status, 0, 'MANUAL', schd.creation_time, schd.update_time) RETURNING id INTO exec_id; + INSERT INTO task (vendor_type, execution_id, job_id, status, status_code, status_revision, run_count, creation_time, start_time, update_time, end_time) + VALUES ('SCHEDULER',exec_id, schd.job_id, task_status, task_status_code, 0, 1, schd.creation_time, schd.creation_time, schd.update_time, schd.update_time); + END LOOP; +END $$; + +DROP TABLE IF EXISTS replication_schedule_job; diff --git a/src/controller/event/handler/init.go b/src/controller/event/handler/init.go index 125ac45d9..c463e4f31 100644 --- a/src/controller/event/handler/init.go +++ b/src/controller/event/handler/init.go @@ -1,6 +1,7 @@ package handler import ( + "context" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/event/handler/auditlog" "github.com/goharbor/harbor/src/controller/event/handler/internal" @@ -10,8 +11,12 @@ import ( "github.com/goharbor/harbor/src/controller/event/handler/webhook/chart" "github.com/goharbor/harbor/src/controller/event/handler/webhook/quota" "github.com/goharbor/harbor/src/controller/event/handler/webhook/scan" + "github.com/goharbor/harbor/src/controller/event/metadata" + "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notifier" + "github.com/goharbor/harbor/src/pkg/task" ) func init() { @@ -54,4 +59,12 @@ func init() { // internal notifier.Subscribe(event.TopicPullArtifact, &internal.Handler{Context: orm.Context}) notifier.Subscribe(event.TopicPushArtifact, &internal.Handler{Context: orm.Context}) + + task.RegisterTaskStatusChangePostFunc(job.Replication, func(ctx context.Context, taskID int64, status string) error { + notification.AddEvent(ctx, &metadata.ReplicationMetaData{ + ReplicationTaskID: taskID, + Status: status, + }) + return nil + }) } diff --git a/src/controller/event/handler/webhook/artifact/replication.go b/src/controller/event/handler/webhook/artifact/replication.go index 8b62b415e..5ec595fa0 100644 --- a/src/controller/event/handler/webhook/artifact/replication.go +++ b/src/controller/event/handler/webhook/artifact/replication.go @@ -10,13 +10,14 @@ import ( "github.com/goharbor/harbor/src/controller/event/handler/util" ctlModel "github.com/goharbor/harbor/src/controller/event/model" "github.com/goharbor/harbor/src/controller/project" + "github.com/goharbor/harbor/src/controller/replication" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notifier/model" - "github.com/goharbor/harbor/src/replication" + rep "github.com/goharbor/harbor/src/replication" rpModel "github.com/goharbor/harbor/src/replication/model" ) @@ -71,25 +72,20 @@ func (r *ReplicationHandler) IsStateful() bool { } func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, *commonModels.Project, error) { - task, err := replication.OperationCtl.GetTask(event.ReplicationTaskID) + ctx := orm.Context() + task, err := replication.Ctl.GetTask(ctx, event.ReplicationTaskID) if err != nil { log.Errorf("failed to get replication task %d: error: %v", event.ReplicationTaskID, err) return nil, nil, err } - if task == nil { - return nil, nil, fmt.Errorf("task %d not found with replication event", event.ReplicationTaskID) - } - execution, err := replication.OperationCtl.GetExecution(task.ExecutionID) + execution, err := replication.Ctl.GetExecution(ctx, task.ExecutionID) if err != nil { log.Errorf("failed to get replication execution %d: error: %v", task.ExecutionID, err) return nil, nil, err } - if execution == nil { - return nil, nil, fmt.Errorf("execution %d not found with replication event", task.ExecutionID) - } - rpPolicy, err := replication.PolicyCtl.Get(execution.PolicyID) + rpPolicy, err := rep.PolicyCtl.Get(execution.PolicyID) if err != nil { log.Errorf("failed to get replication policy %d: error: %v", execution.PolicyID, err) return nil, nil, err @@ -107,7 +103,7 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, remoteRegID = rpPolicy.DestRegistry.ID } - remoteRegistry, err := replication.RegistryMgr.Get(remoteRegID) + remoteRegistry, err := rep.RegistryMgr.Get(remoteRegID) if err != nil { log.Errorf("failed to get replication remoteRegistry registry %d: error: %v", remoteRegID, err) return nil, nil, err @@ -116,8 +112,8 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, return nil, nil, fmt.Errorf("registry %d not found with replication event", remoteRegID) } - srcNamespace, srcNameAndTag := getMetadataFromResource(task.SrcResource) - destNamespace, destNameAndTag := getMetadataFromResource(task.DstResource) + srcNamespace, srcNameAndTag := getMetadataFromResource(task.SourceResource) + destNamespace, destNameAndTag := getMetadataFromResource(task.DestinationResource) extURL, err := config.ExtURL() if err != nil { diff --git a/src/controller/event/handler/webhook/artifact/replication_test.go b/src/controller/event/handler/webhook/artifact/replication_test.go index a89254911..6f882da61 100644 --- a/src/controller/event/handler/webhook/artifact/replication_test.go +++ b/src/controller/event/handler/webhook/artifact/replication_test.go @@ -22,13 +22,14 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/project" + rep "github.com/goharbor/harbor/src/controller/replication" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/replication" - daoModels "github.com/goharbor/harbor/src/replication/dao/models" "github.com/goharbor/harbor/src/replication/model" projecttesting "github.com/goharbor/harbor/src/testing/controller/project" + replicationtesting "github.com/goharbor/harbor/src/testing/controller/replication" "github.com/goharbor/harbor/src/testing/mock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,9 +41,6 @@ type fakedNotificationPolicyMgr struct { type fakedReplicationPolicyMgr struct { } -type fakedReplicationMgr struct { -} - type fakedReplicationRegistryMgr struct { } @@ -89,46 +87,6 @@ func (f *fakedNotificationPolicyMgr) GetRelatedPolices(int64, string) ([]*models }, nil } -func (f *fakedReplicationMgr) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) { - return 0, nil -} -func (f *fakedReplicationMgr) StopReplication(int64) error { - return nil -} -func (f *fakedReplicationMgr) ListExecutions(...*daoModels.ExecutionQuery) (int64, []*daoModels.Execution, error) { - return 0, nil, nil -} -func (f *fakedReplicationMgr) GetExecution(int64) (*daoModels.Execution, error) { - return &daoModels.Execution{ - PolicyID: 1, - Trigger: "manual", - }, nil -} -func (f *fakedReplicationMgr) ListTasks(...*daoModels.TaskQuery) (int64, []*daoModels.Task, error) { - return 0, nil, nil -} -func (f *fakedReplicationMgr) GetTask(id int64) (*daoModels.Task, error) { - if id == 1 { - return &daoModels.Task{ - ExecutionID: 1, - // project info not included when replicating with docker registry - SrcResource: "alpine:[v1]", - DstResource: "gxt/alpine:[v1] ", - }, nil - } - return &daoModels.Task{ - ExecutionID: 1, - SrcResource: "library/alpine:[v1]", - DstResource: "gxt/alpine:[v1] ", - }, nil -} -func (f *fakedReplicationMgr) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error { - return nil -} -func (f *fakedReplicationMgr) GetTaskLog(int64) ([]byte, error) { - return nil, nil -} - // Create new policy func (f *fakedReplicationPolicyMgr) Create(*model.Policy) (int64, error) { return 0, nil @@ -213,24 +171,27 @@ func TestReplicationHandler_Handle(t *testing.T) { config.Init() PolicyMgr := notification.PolicyMgr - execution := replication.OperationCtl rpPolicy := replication.PolicyCtl rpRegistry := replication.RegistryMgr prj := project.Ctl + repCtl := rep.Ctl defer func() { notification.PolicyMgr = PolicyMgr - replication.OperationCtl = execution replication.PolicyCtl = rpPolicy replication.RegistryMgr = rpRegistry project.Ctl = prj + rep.Ctl = repCtl }() notification.PolicyMgr = &fakedNotificationPolicyMgr{} - replication.OperationCtl = &fakedReplicationMgr{} replication.PolicyCtl = &fakedReplicationPolicyMgr{} replication.RegistryMgr = &fakedReplicationRegistryMgr{} projectCtl := &projecttesting.Controller{} project.Ctl = projectCtl + mockRepCtl := &replicationtesting.Controller{} + rep.Ctl = mockRepCtl + mockRepCtl.On("GetTask", mock.Anything, mock.Anything).Return(&rep.Task{}, nil) + mockRepCtl.On("GetExecution", mock.Anything, mock.Anything).Return(&rep.Execution{}, nil) mock.OnAnything(projectCtl, "GetByName").Return(&models.Project{ProjectID: 1}, nil) diff --git a/src/controller/replication/controller.go b/src/controller/replication/controller.go new file mode 100644 index 000000000..44ecca11e --- /dev/null +++ b/src/controller/replication/controller.go @@ -0,0 +1,242 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "context" + "time" + + "github.com/goharbor/harbor/src/controller/replication/flow" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/task" + "github.com/goharbor/harbor/src/replication/model" +) + +// Controller defines the operations related with replication +type Controller interface { + // Start the replication according to the policy + Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (executionID int64, err error) + // Stop the replication specified by the execution ID + Stop(ctx context.Context, executionID int64) (err error) + // ExecutionCount returns the total count of executions according to the query + ExecutionCount(ctx context.Context, query *q.Query) (count int64, err error) + // ListExecutions lists the executions according to the query + ListExecutions(ctx context.Context, query *q.Query) (executions []*Execution, err error) + // GetExecution gets the specific execution + GetExecution(ctx context.Context, executionID int64) (execution *Execution, err error) + // TaskCount returns the total count of tasks according to the query + TaskCount(ctx context.Context, query *q.Query) (count int64, err error) + // ListTasks lists the tasks according to the query + ListTasks(ctx context.Context, query *q.Query) (tasks []*Task, err error) + // GetTask gets the specific task + GetTask(ctx context.Context, taskID int64) (task *Task, err error) + // GetTaskLog gets the log of the specific task + GetTaskLog(ctx context.Context, taskID int64) (log []byte, err error) +} + +var ( + // Ctl is a global replication controller instance + Ctl = NewController() + _ Controller = &controller{} +) + +// NewController creates a new instance of the replication controller +func NewController() Controller { + return &controller{ + execMgr: task.ExecMgr, + taskMgr: task.Mgr, + flowCtl: flow.NewController(), + ormCreator: orm.Crt, + wp: lib.NewWorkerPool(1024), + } +} + +type controller struct { + execMgr task.ExecutionManager + taskMgr task.Manager + flowCtl flow.Controller + ormCreator orm.Creator + wp *lib.WorkerPool +} + +func (c *controller) Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (int64, error) { + logger := log.GetLogger(ctx) + if !policy.Enabled { + return 0, errors.New(nil).WithCode(errors.PreconditionCode). + WithMessage("the policy %d is disabled", policy.ID) + } + // create an execution record + id, err := c.execMgr.Create(ctx, job.Replication, policy.ID, trigger) + if err != nil { + return 0, err + } + c.wp.GetWorker() + // start the replication flow in background + go func() { + defer c.wp.ReleaseWorker() + // as the process runs inside a goroutine, the transaction in the outer ctx + // may be submitted already when the process starts, so create a new context + // with orm populated + ctxx := orm.NewContext(context.Background(), c.ormCreator.Create()) + err := c.flowCtl.Start(ctxx, id, policy, resource) + if err == nil { + // no err, return directly + return + } + // got error, try to stop the execution first in case that some tasks are already created + if err := c.execMgr.StopAndWait(ctxx, id, 10*time.Second); err != nil { + logger.Errorf("failed to stop the execution %d: %v", id, err) + } + if err := c.execMgr.MarkError(ctxx, id, err.Error()); err != nil { + logger.Errorf("failed to mark error for the execution %d: %v", id, err) + } + }() + return id, nil +} + +func (c *controller) Stop(ctx context.Context, id int64) error { + return c.execMgr.Stop(ctx, id) +} + +func (c *controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) { + query = q.MustClone(query) + query.Keywords["VendorType"] = job.Replication + return c.execMgr.Count(ctx, query) +} + +func (c *controller) ListExecutions(ctx context.Context, query *q.Query) ([]*Execution, error) { + // as the following logic may change the content of the query, clone it first + query = q.MustClone(query) + query.Keywords["VendorType"] = job.Replication + // convert the query keyword "PolicyID" or "policy_id" to the "VendorID" + if value, exist := query.Keywords["PolicyID"]; exist { + query.Keywords["VendorID"] = value + delete(query.Keywords, "PolicyID") + } + if value, exist := query.Keywords["policy_id"]; exist { + query.Keywords["VendorID"] = value + delete(query.Keywords, "policy_id") + } + + execs, err := c.execMgr.List(ctx, query) + if err != nil { + return nil, err + } + var executions []*Execution + for _, exec := range execs { + executions = append(executions, convertExecution(exec)) + } + return executions, nil +} + +func (c *controller) GetExecution(ctx context.Context, id int64) (*Execution, error) { + execs, err := c.execMgr.List(ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ID": id, + "VendorType": job.Replication, + }, + }) + if err != nil { + return nil, err + } + if len(execs) == 0 { + return nil, errors.New(nil).WithCode(errors.NotFoundCode). + WithMessage("replication execution %d not found", id) + } + return convertExecution(execs[0]), nil +} + +func (c *controller) TaskCount(ctx context.Context, query *q.Query) (int64, error) { + query = q.MustClone(query) + query.Keywords["VendorType"] = job.Replication + return c.taskMgr.Count(ctx, query) +} + +func (c *controller) ListTasks(ctx context.Context, query *q.Query) ([]*Task, error) { + query = q.MustClone(query) + query.Keywords["VendorType"] = job.Replication + tks, err := c.taskMgr.List(ctx, query) + if err != nil { + return nil, err + } + var tasks []*Task + for _, tk := range tks { + tasks = append(tasks, convertTask(tk)) + } + return tasks, nil +} + +func (c *controller) GetTask(ctx context.Context, id int64) (*Task, error) { + tasks, err := c.taskMgr.List(ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ID": id, + "VendorType": job.Replication, + }, + }) + if err != nil { + return nil, err + } + if len(tasks) == 0 { + return nil, errors.New(nil).WithCode(errors.NotFoundCode). + WithMessage("replication task %d not found", id) + } + return convertTask(tasks[0]), nil +} + +func (c *controller) GetTaskLog(ctx context.Context, id int64) ([]byte, error) { + // make sure the task specified by ID is replication task + _, err := c.GetTask(ctx, id) + if err != nil { + return nil, err + } + return c.taskMgr.GetLog(ctx, id) +} + +func convertExecution(exec *task.Execution) *Execution { + return &Execution{ + ID: exec.ID, + PolicyID: exec.VendorID, + Status: exec.Status, + StatusMessage: exec.StatusMessage, + Metrics: exec.Metrics, + Trigger: exec.Trigger, + StartTime: exec.StartTime, + EndTime: exec.EndTime, + } +} + +func convertTask(task *task.Task) *Task { + return &Task{ + ID: task.ID, + ExecutionID: task.ExecutionID, + Status: task.Status, + StatusMessage: task.StatusMessage, + RunCount: task.RunCount, + ResourceType: task.GetStringFromExtraAttrs("resource_type"), + SourceResource: task.GetStringFromExtraAttrs("source_resource"), + DestinationResource: task.GetStringFromExtraAttrs("destination_resource"), + Operation: task.GetStringFromExtraAttrs("operation"), + JobID: task.JobID, + CreationTime: task.CreationTime, + StartTime: task.StartTime, + UpdateTime: task.UpdateTime, + EndTime: task.EndTime, + } +} diff --git a/src/controller/replication/controller_test.go b/src/controller/replication/controller_test.go new file mode 100644 index 000000000..2ddaf207b --- /dev/null +++ b/src/controller/replication/controller_test.go @@ -0,0 +1,223 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib" + "github.com/goharbor/harbor/src/pkg/task" + "github.com/goharbor/harbor/src/pkg/task/dao" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/testing/lib/orm" + "github.com/goharbor/harbor/src/testing/mock" + testingTask "github.com/goharbor/harbor/src/testing/pkg/task" + "github.com/stretchr/testify/suite" +) + +type replicationTestSuite struct { + suite.Suite + ctl *controller + execMgr *testingTask.ExecutionManager + taskMgr *testingTask.Manager + flowCtl *flowController + ormCreator *orm.Creator +} + +func (r *replicationTestSuite) SetupSuite() { + r.execMgr = &testingTask.ExecutionManager{} + r.taskMgr = &testingTask.Manager{} + r.flowCtl = &flowController{} + r.ormCreator = &orm.Creator{} + r.ctl = &controller{ + execMgr: r.execMgr, + taskMgr: r.taskMgr, + flowCtl: r.flowCtl, + ormCreator: r.ormCreator, + wp: lib.NewWorkerPool(1024), + } +} + +func (r *replicationTestSuite) TestStart() { + // policy is disabled + id, err := r.ctl.Start(context.Background(), &model.Policy{Enabled: false}, nil, task.ExecutionTriggerManual) + r.Require().NotNil(err) + + // got error when running the replication flow + r.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + r.execMgr.On("StopAndWait", mock.Anything, mock.Anything, mock.Anything).Return(nil) + r.execMgr.On("MarkError", mock.Anything, mock.Anything, mock.Anything).Return(nil) + r.flowCtl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("error")) + r.ormCreator.On("Create").Return(nil) + id, err = r.ctl.Start(context.Background(), &model.Policy{Enabled: true}, nil, task.ExecutionTriggerManual) + r.Require().Nil(err) + r.Equal(int64(1), id) + time.Sleep(1 * time.Second) // wait the functions called in the goroutine + r.execMgr.AssertExpectations(r.T()) + r.flowCtl.AssertExpectations(r.T()) + r.ormCreator.AssertExpectations(r.T()) + + // reset the mocks + r.SetupSuite() + + // got no error when running the replication flow + r.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + r.flowCtl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + r.ormCreator.On("Create").Return(nil) + id, err = r.ctl.Start(context.Background(), &model.Policy{Enabled: true}, nil, task.ExecutionTriggerManual) + r.Require().Nil(err) + r.Equal(int64(1), id) + time.Sleep(1 * time.Second) // wait the functions called in the goroutine + r.execMgr.AssertExpectations(r.T()) + r.flowCtl.AssertExpectations(r.T()) + r.ormCreator.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestStop() { + r.execMgr.On("Stop", mock.Anything, mock.Anything).Return(nil) + err := r.ctl.Stop(nil, 1) + r.Require().Nil(err) + r.execMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestExecutionCount() { + r.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil) + total, err := r.ctl.ExecutionCount(nil, nil) + r.Require().Nil(err) + r.Equal(int64(1), total) + r.execMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestListExecutions() { + r.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{ + { + ID: 1, + VendorType: job.Replication, + VendorID: 1, + Status: job.RunningStatus.String(), + Metrics: &dao.Metrics{ + TaskCount: 1, + RunningTaskCount: 1, + }, + Trigger: task.ExecutionTriggerManual, + StartTime: time.Time{}, + EndTime: time.Time{}, + }, + }, nil) + executions, err := r.ctl.ListExecutions(nil, nil) + r.Require().Nil(err) + r.Require().Len(executions, 1) + r.Equal(int64(1), executions[0].ID) + r.Equal(int64(1), executions[0].PolicyID) + r.execMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestGetExecution() { + r.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{ + { + ID: 1, + VendorType: job.Replication, + VendorID: 1, + Status: job.RunningStatus.String(), + Metrics: &dao.Metrics{ + TaskCount: 1, + RunningTaskCount: 1, + }, + Trigger: task.ExecutionTriggerManual, + StartTime: time.Time{}, + EndTime: time.Time{}, + }, + }, nil) + execution, err := r.ctl.GetExecution(nil, 1) + r.Require().Nil(err) + r.Equal(int64(1), execution.ID) + r.Equal(int64(1), execution.PolicyID) + r.execMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestTaskCount() { + r.taskMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil) + total, err := r.ctl.TaskCount(nil, nil) + r.Require().Nil(err) + r.Equal(int64(1), total) + r.taskMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestListTasks() { + r.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{ + { + ID: 1, + ExecutionID: 1, + Status: job.RunningStatus.String(), + ExtraAttrs: map[string]interface{}{ + "resource_type": "artifact", + "source_resource": "library/hello-world", + "destination_resource": "library/hello-world", + "operation": "copy", + }, + }, + }, nil) + tasks, err := r.ctl.ListTasks(nil, nil) + r.Require().Nil(err) + r.Require().Len(tasks, 1) + r.Equal(int64(1), tasks[0].ID) + r.Equal(int64(1), tasks[0].ExecutionID) + r.Equal("artifact", tasks[0].ResourceType) + r.Equal("library/hello-world", tasks[0].SourceResource) + r.Equal("library/hello-world", tasks[0].DestinationResource) + r.Equal("copy", tasks[0].Operation) + r.taskMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestGetTask() { + r.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{ + { + ID: 1, + ExecutionID: 1, + Status: job.RunningStatus.String(), + ExtraAttrs: map[string]interface{}{ + "resource_type": "artifact", + "source_resource": "library/hello-world", + "destination_resource": "library/hello-world", + "operation": "copy", + }, + }, + }, nil) + task, err := r.ctl.GetTask(nil, 1) + r.Require().Nil(err) + r.Equal(int64(1), task.ID) + r.Equal(int64(1), task.ExecutionID) + r.Equal("artifact", task.ResourceType) + r.Equal("library/hello-world", task.SourceResource) + r.Equal("library/hello-world", task.DestinationResource) + r.Equal("copy", task.Operation) + r.taskMgr.AssertExpectations(r.T()) +} + +func (r *replicationTestSuite) TestGetTaskLog() { + r.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte{'a'}, nil) + data, err := r.ctl.GetTaskLog(nil, 1) + r.Require().Nil(err) + r.Equal([]byte{'a'}, data) + r.taskMgr.AssertExpectations(r.T()) +} + +func TestReplicationTestSuite(t *testing.T) { + suite.Run(t, &replicationTestSuite{}) +} diff --git a/src/controller/replication/flow/controller.go b/src/controller/replication/flow/controller.go new file mode 100644 index 000000000..55d53cf2f --- /dev/null +++ b/src/controller/replication/flow/controller.go @@ -0,0 +1,51 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flow + +import ( + "context" + + "github.com/goharbor/harbor/src/replication/model" +) + +// Flow defines a specific replication flow +type Flow interface { + Run(ctx context.Context) (err error) +} + +// Controller controls the replication flow +type Controller interface { + Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) (err error) +} + +// NewController returns an instance of the default flow controller +func NewController() Controller { + return &controller{} +} + +type controller struct{} + +func (c *controller) Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) error { + // deletion flow + if resource != nil && resource.Deleted { + return NewDeletionFlow(executionID, policy, resource).Run(ctx) + } + // copy flow + resources := []*model.Resource{} + if resource != nil { + resources = append(resources, resource) + } + return NewCopyFlow(executionID, policy, resources...).Run(ctx) +} diff --git a/src/controller/replication/flow/copy.go b/src/controller/replication/flow/copy.go new file mode 100644 index 000000000..c1cd55845 --- /dev/null +++ b/src/controller/replication/flow/copy.go @@ -0,0 +1,130 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flow + +import ( + "context" + "encoding/json" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/task" + "github.com/goharbor/harbor/src/replication/model" +) + +type copyFlow struct { + executionID int64 + resources []*model.Resource + policy *model.Policy + executionMgr task.ExecutionManager + taskMgr task.Manager +} + +// NewCopyFlow returns an instance of the copy flow which replicates the resources from +// the source registry to the destination registry. If the parameter "resources" isn't provided, +// will fetch the resources first +func NewCopyFlow(executionID int64, policy *model.Policy, resources ...*model.Resource) Flow { + return ©Flow{ + executionMgr: task.ExecMgr, + taskMgr: task.Mgr, + executionID: executionID, + policy: policy, + resources: resources, + } +} + +func (c *copyFlow) Run(ctx context.Context) error { + logger := log.GetLogger(ctx) + srcAdapter, dstAdapter, err := initialize(c.policy) + if err != nil { + return err + } + var srcResources []*model.Resource + if len(c.resources) > 0 { + srcResources, err = filterResources(c.resources, c.policy.Filters) + } else { + srcResources, err = fetchResources(srcAdapter, c.policy) + } + if err != nil { + return err + } + + isStopped, err := c.isExecutionStopped(ctx) + if err != nil { + return err + } + if isStopped { + logger.Debugf("the execution %d is stopped, stop the flow", c.executionID) + return nil + } + + if len(srcResources) == 0 { + // no candidates, mark the execution as done directly + if err := c.executionMgr.MarkDone(ctx, c.executionID, "no resources need to be replicated"); err != nil { + logger.Errorf("failed to mark done for the execution %d: %v", c.executionID, err) + } + return nil + } + + srcResources = assembleSourceResources(srcResources, c.policy) + dstResources := assembleDestinationResources(srcResources, c.policy) + + if err = prepareForPush(dstAdapter, dstResources); err != nil { + return err + } + + return c.createTasks(ctx, srcResources, dstResources) +} + +func (c *copyFlow) isExecutionStopped(ctx context.Context) (bool, error) { + execution, err := c.executionMgr.Get(ctx, c.executionID) + if err != nil { + return false, err + } + return execution.Status == job.StoppedStatus.String(), nil +} + +func (c *copyFlow) createTasks(ctx context.Context, srcResources, dstResources []*model.Resource) error { + for i, resource := range srcResources { + src, err := json.Marshal(resource) + if err != nil { + return err + } + dest, err := json.Marshal(dstResources[i]) + if err != nil { + return err + } + + job := &task.Job{ + Name: job.Replication, + Metadata: &job.Metadata{ + JobKind: job.KindGeneric, + }, + Parameters: map[string]interface{}{ + "src_resource": string(src), + "dst_resource": string(dest), + }, + } + + if _, err = c.taskMgr.Create(ctx, c.executionID, job, map[string]interface{}{ + "operation": "copy", + "resource_type": string(resource.Type), + "source_resource": getResourceName(resource), + "destination_resource": getResourceName(dstResources[i])}); err != nil { + return err + } + } + return nil +} diff --git a/src/controller/replication/flow/copy_test.go b/src/controller/replication/flow/copy_test.go new file mode 100644 index 000000000..705febbde --- /dev/null +++ b/src/controller/replication/flow/copy_test.go @@ -0,0 +1,83 @@ +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flow + +import ( + "context" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/replication/adapter" + "github.com/stretchr/testify/mock" + "testing" + + "github.com/goharbor/harbor/src/pkg/task" + "github.com/goharbor/harbor/src/replication/model" + testingTask "github.com/goharbor/harbor/src/testing/pkg/task" + "github.com/stretchr/testify/suite" +) + +type copyFlowTestSuite struct { + suite.Suite +} + +func (c *copyFlowTestSuite) TestRun() { + adp := &mockAdapter{} + factory := &mockFactory{} + factory.On("AdapterPattern").Return(nil) + factory.On("Create", mock.Anything).Return(adp, nil) + adapter.RegisterFactory("TEST_FOR_COPY_FLOW", factory) + + adp.On("Info").Return(&model.RegistryInfo{ + SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeArtifact, + }, + }, nil) + adp.On("FetchArtifacts", mock.Anything).Return([]*model.Resource{ + { + Type: model.ResourceTypeChart, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/hello-world", + }, + Vtags: []string{"latest"}, + }, + Override: false, + }, + }, nil) + adp.On("PrepareForPush", mock.Anything).Return(nil) + + execMgr := &testingTask.ExecutionManager{} + execMgr.On("Get", mock.Anything, mock.Anything).Return(&task.Execution{ + Status: job.RunningStatus.String(), + }, nil) + + taskMgr := &testingTask.Manager{} + taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + policy := &model.Policy{ + SrcRegistry: &model.Registry{ + Type: "TEST_FOR_COPY_FLOW", + }, + DestRegistry: &model.Registry{ + Type: "TEST_FOR_COPY_FLOW", + }, + } + flow := ©Flow{ + executionID: 1, + policy: policy, + executionMgr: execMgr, + taskMgr: taskMgr, + } + err := flow.Run(context.Background()) + c.Require().Nil(err) +} + +func TestCopyFlowTestSuite(t *testing.T) { + suite.Run(t, ©FlowTestSuite{}) +} diff --git a/src/controller/replication/flow/deletion.go b/src/controller/replication/flow/deletion.go new file mode 100644 index 000000000..c2e2172b4 --- /dev/null +++ b/src/controller/replication/flow/deletion.go @@ -0,0 +1,103 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flow + +import ( + "context" + "encoding/json" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/task" + "github.com/goharbor/harbor/src/replication/model" +) + +type deletionFlow struct { + executionID int64 + policy *model.Policy + executionMgr task.ExecutionManager + taskMgr task.Manager + resources []*model.Resource +} + +// NewDeletionFlow returns an instance of the delete flow which deletes the resources +// on the destination registry +func NewDeletionFlow(executionID int64, policy *model.Policy, resources ...*model.Resource) Flow { + return &deletionFlow{ + executionMgr: task.ExecMgr, + taskMgr: task.Mgr, + executionID: executionID, + policy: policy, + resources: resources, + } +} + +func (d *deletionFlow) Run(ctx context.Context) error { + logger := log.GetLogger(ctx) + srcResources, err := filterResources(d.resources, d.policy.Filters) + if err != nil { + return err + } + if len(srcResources) == 0 { + // no candidates, mark the execution as done directly + if err := d.executionMgr.MarkDone(ctx, d.executionID, "no resources need to be replicated"); err != nil { + logger.Errorf("failed to mark done for the execution %d: %v", d.executionID, err) + } + return nil + } + + srcResources = assembleSourceResources(srcResources, d.policy) + dstResources := assembleDestinationResources(srcResources, d.policy) + + return d.createTasks(ctx, srcResources, dstResources) +} + +func (d *deletionFlow) createTasks(ctx context.Context, srcResources, dstResources []*model.Resource) error { + for i, resource := range srcResources { + src, err := json.Marshal(resource) + if err != nil { + return err + } + dest, err := json.Marshal(dstResources[i]) + if err != nil { + return err + } + + job := &task.Job{ + Name: job.Replication, + Metadata: &job.Metadata{ + JobKind: job.KindGeneric, + }, + Parameters: map[string]interface{}{ + "src_resource": string(src), + "dst_resource": string(dest), + }, + } + + operation := "deletion" + if dstResources[i].IsDeleteTag { + operation = "tag deletion" + } + + if _, err = d.taskMgr.Create(ctx, d.executionID, job, map[string]interface{}{ + "operation": operation, + "resource_type": string(resource.Type), + "source_resource": getResourceName(resource), + "destination_resource": getResourceName(dstResources[i])}); err != nil { + return err + } + } + return nil +} diff --git a/src/replication/operation/flow/deletion_test.go b/src/controller/replication/flow/deletion_test.go similarity index 63% rename from src/replication/operation/flow/deletion_test.go rename to src/controller/replication/flow/deletion_test.go index 60afac3c2..2640bfc1d 100644 --- a/src/replication/operation/flow/deletion_test.go +++ b/src/controller/replication/flow/deletion_test.go @@ -15,16 +15,23 @@ package flow import ( + "context" "testing" "github.com/goharbor/harbor/src/replication/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "github.com/goharbor/harbor/src/testing/pkg/task" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" ) -func TestRunOfDeletionFlow(t *testing.T) { - scheduler := &fakedScheduler{} - executionMgr := &fakedExecutionManager{} +type deletionFlowTestSuite struct { + suite.Suite +} + +func (d *deletionFlowTestSuite) TestRun() { + taskMgr := &task.Manager{} + taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + policy := &model.Policy{ SrcRegistry: &model.Registry{ Type: model.RegistryTypeHarbor, @@ -47,8 +54,16 @@ func TestRunOfDeletionFlow(t *testing.T) { }, }, } - flow := NewDeletionFlow(executionMgr, scheduler, 1, policy, resources...) - n, err := flow.Run(nil) - require.Nil(t, err) - assert.Equal(t, 1, n) + flow := &deletionFlow{ + executionID: 1, + policy: policy, + taskMgr: taskMgr, + resources: resources, + } + err := flow.Run(context.Background()) + d.Require().Nil(err) +} + +func TestDeletionFlowTestSuite(t *testing.T) { + suite.Run(t, &deletionFlowTestSuite{}) } diff --git a/src/controller/replication/flow/mock.go b/src/controller/replication/flow/mock.go new file mode 100644 index 000000000..0d422263b --- /dev/null +++ b/src/controller/replication/flow/mock.go @@ -0,0 +1,28 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flow + +import ( + "github.com/goharbor/harbor/src/replication/adapter" +) + +// define a new interface to combine the two interfaces of adapter for mockery to generate the mocks +type registryAdapter interface { + adapter.Adapter + adapter.ArtifactRegistry +} + +//go:generate mockery --dir . --name registryAdapter --output . --outpkg flow --filename mock_adapter_test.go --structname mockAdapter +//go:generate mockery --dir ../../../replication/adapter --name Factory --output . --outpkg flow --filename mock_adapter_factory_test.go --structname mockFactory diff --git a/src/controller/replication/flow/mock_adapter_factory_test.go b/src/controller/replication/flow/mock_adapter_factory_test.go new file mode 100644 index 000000000..d9e14d4c8 --- /dev/null +++ b/src/controller/replication/flow/mock_adapter_factory_test.go @@ -0,0 +1,54 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package flow + +import ( + adapter "github.com/goharbor/harbor/src/replication/adapter" + mock "github.com/stretchr/testify/mock" + + model "github.com/goharbor/harbor/src/replication/model" +) + +// mockFactory is an autogenerated mock type for the Factory type +type mockFactory struct { + mock.Mock +} + +// AdapterPattern provides a mock function with given fields: +func (_m *mockFactory) AdapterPattern() *model.AdapterPattern { + ret := _m.Called() + + var r0 *model.AdapterPattern + if rf, ok := ret.Get(0).(func() *model.AdapterPattern); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AdapterPattern) + } + } + + return r0 +} + +// Create provides a mock function with given fields: _a0 +func (_m *mockFactory) Create(_a0 *model.Registry) (adapter.Adapter, error) { + ret := _m.Called(_a0) + + var r0 adapter.Adapter + if rf, ok := ret.Get(0).(func(*model.Registry) adapter.Adapter); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(adapter.Adapter) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*model.Registry) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/src/controller/replication/flow/mock_adapter_test.go b/src/controller/replication/flow/mock_adapter_test.go new file mode 100644 index 000000000..8a3413ae6 --- /dev/null +++ b/src/controller/replication/flow/mock_adapter_test.go @@ -0,0 +1,278 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package flow + +import ( + distribution "github.com/docker/distribution" + + io "io" + + mock "github.com/stretchr/testify/mock" + + model "github.com/goharbor/harbor/src/replication/model" +) + +// mockAdapter is an autogenerated mock type for the registryAdapter type +type mockAdapter struct { + mock.Mock +} + +// BlobExist provides a mock function with given fields: repository, digest +func (_m *mockAdapter) BlobExist(repository string, digest string) (bool, error) { + ret := _m.Called(repository, digest) + + var r0 bool + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(repository, digest) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(repository, digest) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DeleteManifest provides a mock function with given fields: repository, reference +func (_m *mockAdapter) DeleteManifest(repository string, reference string) error { + ret := _m.Called(repository, reference) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string) error); ok { + r0 = rf(repository, reference) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteTag provides a mock function with given fields: repository, tag +func (_m *mockAdapter) DeleteTag(repository string, tag string) error { + ret := _m.Called(repository, tag) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string) error); ok { + r0 = rf(repository, tag) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// FetchArtifacts provides a mock function with given fields: filters +func (_m *mockAdapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) { + ret := _m.Called(filters) + + var r0 []*model.Resource + if rf, ok := ret.Get(0).(func([]*model.Filter) []*model.Resource); ok { + r0 = rf(filters) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.Resource) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func([]*model.Filter) error); ok { + r1 = rf(filters) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// HealthCheck provides a mock function with given fields: +func (_m *mockAdapter) HealthCheck() (model.HealthStatus, error) { + ret := _m.Called() + + var r0 model.HealthStatus + if rf, ok := ret.Get(0).(func() model.HealthStatus); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(model.HealthStatus) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Info provides a mock function with given fields: +func (_m *mockAdapter) Info() (*model.RegistryInfo, error) { + ret := _m.Called() + + var r0 *model.RegistryInfo + if rf, ok := ret.Get(0).(func() *model.RegistryInfo); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.RegistryInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ManifestExist provides a mock function with given fields: repository, reference +func (_m *mockAdapter) ManifestExist(repository string, reference string) (bool, string, error) { + ret := _m.Called(repository, reference) + + var r0 bool + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(repository, reference) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 string + if rf, ok := ret.Get(1).(func(string, string) string); ok { + r1 = rf(repository, reference) + } else { + r1 = ret.Get(1).(string) + } + + var r2 error + if rf, ok := ret.Get(2).(func(string, string) error); ok { + r2 = rf(repository, reference) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// PrepareForPush provides a mock function with given fields: _a0 +func (_m *mockAdapter) PrepareForPush(_a0 []*model.Resource) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func([]*model.Resource) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// PullBlob provides a mock function with given fields: repository, digest +func (_m *mockAdapter) PullBlob(repository string, digest string) (int64, io.ReadCloser, error) { + ret := _m.Called(repository, digest) + + var r0 int64 + if rf, ok := ret.Get(0).(func(string, string) int64); ok { + r0 = rf(repository, digest) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 io.ReadCloser + if rf, ok := ret.Get(1).(func(string, string) io.ReadCloser); ok { + r1 = rf(repository, digest) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(io.ReadCloser) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func(string, string) error); ok { + r2 = rf(repository, digest) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// PullManifest provides a mock function with given fields: repository, reference, accepttedMediaTypes +func (_m *mockAdapter) PullManifest(repository string, reference string, accepttedMediaTypes ...string) (distribution.Manifest, string, error) { + _va := make([]interface{}, len(accepttedMediaTypes)) + for _i := range accepttedMediaTypes { + _va[_i] = accepttedMediaTypes[_i] + } + var _ca []interface{} + _ca = append(_ca, repository, reference) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 distribution.Manifest + if rf, ok := ret.Get(0).(func(string, string, ...string) distribution.Manifest); ok { + r0 = rf(repository, reference, accepttedMediaTypes...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(distribution.Manifest) + } + } + + var r1 string + if rf, ok := ret.Get(1).(func(string, string, ...string) string); ok { + r1 = rf(repository, reference, accepttedMediaTypes...) + } else { + r1 = ret.Get(1).(string) + } + + var r2 error + if rf, ok := ret.Get(2).(func(string, string, ...string) error); ok { + r2 = rf(repository, reference, accepttedMediaTypes...) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// PushBlob provides a mock function with given fields: repository, digest, size, blob +func (_m *mockAdapter) PushBlob(repository string, digest string, size int64, blob io.Reader) error { + ret := _m.Called(repository, digest, size, blob) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, int64, io.Reader) error); ok { + r0 = rf(repository, digest, size, blob) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// PushManifest provides a mock function with given fields: repository, reference, mediaType, payload +func (_m *mockAdapter) PushManifest(repository string, reference string, mediaType string, payload []byte) (string, error) { + ret := _m.Called(repository, reference, mediaType, payload) + + var r0 string + if rf, ok := ret.Get(0).(func(string, string, string, []byte) string); ok { + r0 = rf(repository, reference, mediaType, payload) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, string, string, []byte) error); ok { + r1 = rf(repository, reference, mediaType, payload) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/src/replication/operation/flow/stage.go b/src/controller/replication/flow/stage.go similarity index 64% rename from src/replication/operation/flow/stage.go rename to src/controller/replication/flow/stage.go index d0356eaeb..ce2bd2a7b 100644 --- a/src/replication/operation/flow/stage.go +++ b/src/controller/replication/flow/stage.go @@ -15,17 +15,12 @@ package flow import ( - "errors" "fmt" - "github.com/goharbor/harbor/src/replication/filter" - "time" "github.com/goharbor/harbor/src/lib/log" adp "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/dao/models" + "github.com/goharbor/harbor/src/replication/filter" "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/operation/execution" - "github.com/goharbor/harbor/src/replication/operation/scheduler" "github.com/goharbor/harbor/src/replication/util" ) @@ -171,108 +166,6 @@ func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error { return nil } -// preprocess -func preprocess(scheduler scheduler.Scheduler, srcResources, dstResources []*model.Resource) ([]*scheduler.ScheduleItem, error) { - items, err := scheduler.Preprocess(srcResources, dstResources) - if err != nil { - return nil, fmt.Errorf("failed to preprocess the resources: %v", err) - } - log.Debug("preprocess the resources completed") - return items, nil -} - -// create task records in database -func createTasks(mgr execution.Manager, executionID int64, items []*scheduler.ScheduleItem) error { - for _, item := range items { - operation := "copy" - if item.DstResource.Deleted { - operation = "deletion" - if item.DstResource.IsDeleteTag { - operation = "tag deletion" - } - } - - task := &models.Task{ - ExecutionID: executionID, - Status: models.TaskStatusInitialized, - ResourceType: string(item.SrcResource.Type), - SrcResource: getResourceName(item.SrcResource), - DstResource: getResourceName(item.DstResource), - Operation: operation, - } - - id, err := mgr.CreateTask(task) - if err != nil { - // if failed to create the task for one of the items, - // the whole execution is marked as failure and all - // the items will not be submitted - return fmt.Errorf("failed to create task records for the execution %d: %v", executionID, err) - } - - item.TaskID = id - log.Debugf("task record %d for the execution %d created", id, executionID) - } - return nil -} - -// schedule the replication tasks and update the task's status -// returns the count of tasks which have been scheduled and the error -func schedule(scheduler scheduler.Scheduler, executionMgr execution.Manager, items []*scheduler.ScheduleItem) (int, error) { - results, err := scheduler.Schedule(items) - if err != nil { - return 0, fmt.Errorf("failed to schedule the tasks: %v", err) - } - - allFailed := true - n := len(results) - for _, result := range results { - // if the task is failed to be submitted, update the status of the - // task as failure - now := time.Now() - if result.Error != nil { - log.Errorf("failed to schedule the task %d: %v", result.TaskID, result.Error) - if err = executionMgr.UpdateTask(&models.Task{ - ID: result.TaskID, - Status: models.TaskStatusFailed, - EndTime: now, - }, "Status", "EndTime"); err != nil { - log.Errorf("failed to update the task status %d: %v", result.TaskID, err) - } - continue - } - allFailed = false - // if the task is submitted successfully, update the status, job ID and start time - if err = executionMgr.UpdateTaskStatus(result.TaskID, models.TaskStatusPending, 0, models.TaskStatusInitialized); err != nil { - log.Errorf("failed to update the task status %d: %v", result.TaskID, err) - } - if err = executionMgr.UpdateTask(&models.Task{ - ID: result.TaskID, - JobID: result.JobID, - StartTime: now, - }, "JobID", "StartTime"); err != nil { - log.Errorf("failed to update the task %d: %v", result.TaskID, err) - } - log.Debugf("the task %d scheduled", result.TaskID) - } - // if all the tasks are failed, return err - if allFailed { - return n, errors.New("all tasks are failed") - } - return n, nil -} - -// check whether the execution is stopped -func isExecutionStopped(mgr execution.Manager, id int64) (bool, error) { - execution, err := mgr.Get(id) - if err != nil { - return false, err - } - if execution == nil { - return false, fmt.Errorf("execution %d not found", id) - } - return execution.Status == models.ExecutionStatusStopped, nil -} - // return the name with format "res_name" or "res_name:[vtag1,vtag2,vtag3]" // if the resource has vtags func getResourceName(res *model.Resource) string { diff --git a/src/controller/replication/flow/stage_test.go b/src/controller/replication/flow/stage_test.go new file mode 100644 index 000000000..f36d27089 --- /dev/null +++ b/src/controller/replication/flow/stage_test.go @@ -0,0 +1,221 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flow + +import ( + "testing" + + "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/testing/mock" + "github.com/stretchr/testify/suite" +) + +type stageTestSuite struct { + suite.Suite +} + +func (s *stageTestSuite) SetupTest() { +} + +func (s *stageTestSuite) TestInitialize() { + factory := &mockFactory{} + factory.On("AdapterPattern").Return(nil) + adapter.RegisterFactory(model.RegistryTypeHarbor, factory) + + policy := &model.Policy{ + SrcRegistry: &model.Registry{ + Type: model.RegistryTypeHarbor, + }, + DestRegistry: &model.Registry{ + Type: model.RegistryTypeHarbor, + }, + } + factory.On("Create", mock.Anything).Return(&mockAdapter{}, nil) + _, _, err := initialize(policy) + s.Nil(err) + factory.AssertExpectations(s.T()) +} + +func (s *stageTestSuite) TestFetchResources() { + adapter := &mockAdapter{} + adapter.On("Info").Return(&model.RegistryInfo{ + SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeArtifact, + }, + }, nil) + adapter.On("FetchArtifacts", mock.Anything).Return([]*model.Resource{ + {}, + {}, + }, nil) + policy := &model.Policy{} + resources, err := fetchResources(adapter, policy) + s.Require().Nil(err) + s.Len(resources, 2) + adapter.AssertExpectations(s.T()) +} + +func (s *stageTestSuite) TestFilterResources() { + resources := []*model.Resource{ + { + Type: model.ResourceTypeImage, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/hello-world", + }, + Artifacts: []*model.Artifact{ + { + Tags: []string{"latest"}, + }, + }, + }, + Deleted: true, + Override: true, + }, + { + Type: model.ResourceTypeChart, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/harbor", + }, + Artifacts: []*model.Artifact{ + { + Tags: []string{"0.2.0"}, + }, + { + Tags: []string{"0.3.0"}, + }, + }, + }, + Deleted: true, + Override: true, + }, + { + Type: model.ResourceTypeChart, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/mysql", + }, + Artifacts: []*model.Artifact{ + { + Tags: []string{"1.0"}, + }, + }, + }, + Deleted: true, + Override: true, + }, + } + filters := []*model.Filter{ + { + Type: model.FilterTypeResource, + Value: model.ResourceTypeChart, + }, + { + Type: model.FilterTypeName, + Value: "library/*", + }, + { + Type: model.FilterTypeName, + Value: "library/harbor", + }, + { + Type: model.FilterTypeTag, + Value: "0.2.?", + }, + } + res, err := filterResources(resources, filters) + s.Require().Nil(err) + s.Len(res, 1) + s.Equal("library/harbor", res[0].Metadata.Repository.Name) + s.Equal(1, len(res[0].Metadata.Artifacts)) + s.Equal("0.2.0", res[0].Metadata.Artifacts[0].Tags[0]) +} + +func (s *stageTestSuite) TestAssembleSourceResources() { + resources := []*model.Resource{ + { + Type: model.ResourceTypeChart, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/hello-world", + }, + Vtags: []string{"latest"}, + }, + Override: false, + }, + } + policy := &model.Policy{ + SrcRegistry: &model.Registry{ + ID: 1, + }, + } + res := assembleSourceResources(resources, policy) + s.Len(res, 1) + s.Equal(int64(1), res[0].Registry.ID) +} + +func (s *stageTestSuite) TestAssembleDestinationResources() { + resources := []*model.Resource{ + { + Type: model.ResourceTypeChart, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/hello-world", + }, + Vtags: []string{"latest"}, + }, + Override: false, + }, + } + policy := &model.Policy{ + DestRegistry: &model.Registry{}, + DestNamespace: "test", + Override: true, + } + res := assembleDestinationResources(resources, policy) + s.Len(res, 1) + s.Equal(model.ResourceTypeChart, res[0].Type) + s.Equal("test/hello-world", res[0].Metadata.Repository.Name) + s.Equal(1, len(res[0].Metadata.Vtags)) + s.Equal("latest", res[0].Metadata.Vtags[0]) +} + +func (s *stageTestSuite) TestReplaceNamespace() { + // empty namespace + repository := "c" + namespace := "" + result := replaceNamespace(repository, namespace) + s.Equal("c", result) + // repository contains no "/" + repository = "c" + namespace = "n" + result = replaceNamespace(repository, namespace) + s.Equal("n/c", result) + // repository contains only one "/" + repository = "b/c" + namespace = "n" + result = replaceNamespace(repository, namespace) + s.Equal("n/c", result) + // repository contains more than one "/" + repository = "a/b/c" + namespace = "n" + result = replaceNamespace(repository, namespace) + s.Equal("n/c", result) +} + +func TestStage(t *testing.T) { + suite.Run(t, &stageTestSuite{}) +} diff --git a/src/pkg/task/checkin_test.go b/src/controller/replication/mock.go similarity index 68% rename from src/pkg/task/checkin_test.go rename to src/controller/replication/mock.go index 3c12f20fe..e434d2586 100644 --- a/src/pkg/task/checkin_test.go +++ b/src/controller/replication/mock.go @@ -12,19 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -package task +package replication -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestRegisterCheckInProcessor(t *testing.T) { - err := RegisterCheckInProcessor("test", nil) - assert.Nil(t, err) - - // already exist - err = RegisterCheckInProcessor("test", nil) - assert.NotNil(t, err) -} +//go:generate mockery --dir ./flow --name Controller --output . --outpkg replication --filename mock_flow_controller_test.go --structname flowController diff --git a/src/controller/replication/mock_flow_controller_test.go b/src/controller/replication/mock_flow_controller_test.go new file mode 100644 index 000000000..01c23beb6 --- /dev/null +++ b/src/controller/replication/mock_flow_controller_test.go @@ -0,0 +1,30 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package replication + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + model "github.com/goharbor/harbor/src/replication/model" +) + +// flowController is an autogenerated mock type for the Controller type +type flowController struct { + mock.Mock +} + +// Start provides a mock function with given fields: ctx, executionID, policy, resource +func (_m *flowController) Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) error { + ret := _m.Called(ctx, executionID, policy, resource) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64, *model.Policy, *model.Resource) error); ok { + r0 = rf(ctx, executionID, policy, resource) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/controller/replication/model.go b/src/controller/replication/model.go new file mode 100644 index 000000000..3c4cd08a5 --- /dev/null +++ b/src/controller/replication/model.go @@ -0,0 +1,51 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replication + +import ( + "time" + + "github.com/goharbor/harbor/src/pkg/task/dao" +) + +// Execution model for replication +type Execution struct { + ID int64 + PolicyID int64 + Status string + StatusMessage string + Metrics *dao.Metrics + Trigger string + StartTime time.Time + EndTime time.Time +} + +// Task model for replication +type Task struct { + ID int64 + ExecutionID int64 + Status string + StatusMessage string + RunCount int32 + ResourceType string + SourceResource string + DestinationResource string + Operation string + JobID string + CreationTime time.Time + StartTime time.Time + UpdateTime time.Time + EndTime time.Time +} diff --git a/src/core/api/base.go b/src/core/api/base.go index 3e84c0487..7356afb98 100644 --- a/src/core/api/base.go +++ b/src/core/api/base.go @@ -15,6 +15,7 @@ package api import ( + "context" "encoding/json" "fmt" "net/http" @@ -30,7 +31,6 @@ import ( "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/repository" "github.com/goharbor/harbor/src/pkg/retention" @@ -194,13 +194,9 @@ func Init() error { retentionController = retention.NewAPIController(retentionMgr, projectMgr, repository.Mgr, scheduler.Sched, retentionLauncher) - retentionCallbackFun := func(p interface{}) error { - str, ok := p.(string) - if !ok { - return fmt.Errorf("the type of param %v isn't string", p) - } + retentionCallbackFun := func(ctx context.Context, p string) error { param := &retention.TriggerParam{} - if err := json.Unmarshal([]byte(str), param); err != nil { + if err := json.Unmarshal([]byte(p), param); err != nil { return fmt.Errorf("failed to unmarshal the param: %v", err) } _, err := retentionController.TriggerRetentionExec(param.PolicyID, param.Trigger, false) @@ -211,16 +207,12 @@ func Init() error { return err } - p2pPreheatCallbackFun := func(p interface{}) error { - str, ok := p.(string) - if !ok { - return fmt.Errorf("the type of param %v isn't string", p) - } + p2pPreheatCallbackFun := func(ctx context.Context, p string) error { param := &preheat.TriggerParam{} - if err := json.Unmarshal([]byte(str), param); err != nil { + if err := json.Unmarshal([]byte(p), param); err != nil { return fmt.Errorf("failed to unmarshal the param: %v", err) } - _, err := preheat.Enf.EnforcePolicy(orm.Context(), param.PolicyID) + _, err := preheat.Enf.EnforcePolicy(ctx, param.PolicyID) return err } err = scheduler.RegisterCallbackFunc(preheat.SchedulerCallback, p2pPreheatCallbackFun) diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 56b72f5f9..4bd80ce93 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -137,10 +137,6 @@ func init() { beego.Router("/api/projects/:pid([0-9]+)/robots/:id([0-9]+)", &RobotAPI{}, "get:Get;put:Put;delete:Delete") beego.Router("/api/replication/adapters", &ReplicationAdapterAPI{}, "get:List") - beego.Router("/api/replication/executions", &ReplicationOperationAPI{}, "get:ListExecutions;post:CreateExecution") - beego.Router("/api/replication/executions/:id([0-9]+)", &ReplicationOperationAPI{}, "get:GetExecution;put:StopExecution") - beego.Router("/api/replication/executions/:id([0-9]+)/tasks", &ReplicationOperationAPI{}, "get:ListTasks") - beego.Router("/api/replication/executions/:id([0-9]+)/tasks/:tid([0-9]+)/log", &ReplicationOperationAPI{}, "get:GetTaskLog") beego.Router("/api/replication/policies", &ReplicationPolicyAPI{}, "get:List;post:Create") beego.Router("/api/replication/policies/:id([0-9]+)", &ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete") diff --git a/src/core/api/replication_execution.go b/src/core/api/replication_execution.go deleted file mode 100644 index 2719cdafa..000000000 --- a/src/core/api/replication_execution.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "errors" - "fmt" - "net/http" - "strconv" - - common_http "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/event" - "github.com/goharbor/harbor/src/replication/model" -) - -// ReplicationOperationAPI handles the replication operation requests -type ReplicationOperationAPI struct { - BaseController - execution *models.Execution - task *models.Task -} - -// Prepare ... -func (r *ReplicationOperationAPI) Prepare() { - r.BaseController.Prepare() - // As we delegate the jobservice to trigger the scheduled replication, - // we need to allow the jobservice to call the API - if !(r.SecurityCtx.IsSysAdmin() || r.SecurityCtx.IsSolutionUser()) { - if !r.SecurityCtx.IsAuthenticated() { - r.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername())) - return - } - - // check the existence of execution if execution ID is provided in the request path - executionIDStr := r.GetStringFromPath(":id") - if len(executionIDStr) > 0 { - executionID, err := strconv.ParseInt(executionIDStr, 10, 64) - if err != nil || executionID <= 0 { - r.SendBadRequestError(fmt.Errorf("invalid execution ID: %s", executionIDStr)) - return - } - execution, err := replication.OperationCtl.GetExecution(executionID) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to get execution %d: %v", executionID, err)) - return - } - if execution == nil { - r.SendNotFoundError(fmt.Errorf("execution %d not found", executionID)) - return - } - r.execution = execution - - // check the existence of task if task ID is provided in the request path - taskIDStr := r.GetStringFromPath(":tid") - if len(taskIDStr) > 0 { - taskID, err := strconv.ParseInt(taskIDStr, 10, 64) - if err != nil || taskID <= 0 { - r.SendBadRequestError(fmt.Errorf("invalid task ID: %s", taskIDStr)) - return - } - task, err := replication.OperationCtl.GetTask(taskID) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to get task %d: %v", taskID, err)) - return - } - if task == nil || task.ExecutionID != executionID { - r.SendNotFoundError(fmt.Errorf("task %d not found", taskID)) - return - } - r.task = task - } - } -} - -// The API is open only for system admin currently, we can use -// the code commentted below to make the API available to the -// users who have permission for all projects that the policy -// refers -/* -func (r *ReplicationOperationAPI) authorized(policy *model.Policy, resource rbac.Resource, action rbac.Action) bool { - - projects := []string{} - // pull mode - if policy.SrcRegistryID != 0 { - projects = append(projects, policy.DestNamespace) - } else { - // push mode - projects = append(projects, policy.SrcNamespaces...) - } - - for _, project := range projects { - resource := rbac.NewProjectNamespace(project).Resource(resource) - if !r.SecurityCtx.Can(action, resource) { - r.HandleForbidden(r.SecurityCtx.GetUsername()) - return false - } - } - - return true -} -*/ - -// ListExecutions ... -func (r *ReplicationOperationAPI) ListExecutions() { - query := &models.ExecutionQuery{ - Trigger: r.GetString("trigger"), - } - - if len(r.GetString("status")) > 0 { - query.Statuses = []string{r.GetString("status")} - } - if len(r.GetString("policy_id")) > 0 { - policyID, err := r.GetInt64("policy_id") - if err != nil || policyID <= 0 { - r.SendBadRequestError(fmt.Errorf("invalid policy_id %s", r.GetString("policy_id"))) - return - } - query.PolicyID = policyID - } - page, size, err := r.GetPaginationParams() - if err != nil { - r.SendBadRequestError(err) - return - } - query.Page = page - query.Size = size - - total, executions, err := replication.OperationCtl.ListExecutions(query) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to list executions: %v", err)) - return - } - r.SetPaginationHeader(total, query.Page, query.Size) - r.WriteJSONData(executions) -} - -// CreateExecution starts a replication -func (r *ReplicationOperationAPI) CreateExecution() { - execution := &models.Execution{} - if err := r.DecodeJSONReq(execution); err != nil { - r.SendBadRequestError(err) - return - } - - policy, err := replication.PolicyCtl.Get(execution.PolicyID) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", execution.PolicyID, err)) - return - } - if policy == nil { - r.SendNotFoundError(fmt.Errorf("policy %d not found", execution.PolicyID)) - return - } - if !policy.Enabled { - r.SendBadRequestError(fmt.Errorf("the policy %d is disabled", execution.PolicyID)) - return - } - if err = event.PopulateRegistries(replication.RegistryMgr, policy); err != nil { - r.SendInternalServerError(fmt.Errorf("failed to populate registries for policy %d: %v", execution.PolicyID, err)) - return - } - - trigger := r.GetString("trigger", string(model.TriggerTypeManual)) - executionID, err := replication.OperationCtl.StartReplication(policy, nil, model.TriggerType(trigger)) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to start replication for policy %d: %v", execution.PolicyID, err)) - return - } - r.Redirect(http.StatusCreated, strconv.FormatInt(executionID, 10)) -} - -// GetExecution gets one execution of the replication -func (r *ReplicationOperationAPI) GetExecution() { - r.WriteJSONData(r.execution) -} - -// StopExecution stops one execution of the replication -func (r *ReplicationOperationAPI) StopExecution() { - if err := replication.OperationCtl.StopReplication(r.execution.ID); err != nil { - r.SendInternalServerError(fmt.Errorf("failed to stop execution %d: %v", r.execution.ID, err)) - return - } -} - -// ListTasks ... -func (r *ReplicationOperationAPI) ListTasks() { - query := &models.TaskQuery{ - ExecutionID: r.execution.ID, - ResourceType: r.GetString("resource_type"), - } - status := r.GetString("status") - if len(status) > 0 { - query.Statuses = []string{status} - } - page, size, err := r.GetPaginationParams() - if err != nil { - r.SendBadRequestError(err) - return - } - query.Page = page - query.Size = size - total, tasks, err := replication.OperationCtl.ListTasks(query) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to list tasks: %v", err)) - return - } - r.SetPaginationHeader(total, query.Page, query.Size) - r.WriteJSONData(tasks) -} - -// GetTaskLog ... -func (r *ReplicationOperationAPI) GetTaskLog() { - logBytes, err := replication.OperationCtl.GetTaskLog(r.task.ID) - if err != nil { - if httpErr, ok := err.(*common_http.Error); ok { - if ok && httpErr.Code == http.StatusNotFound { - r.SendNotFoundError(fmt.Errorf("the log of task %d not found", r.task.ID)) - return - } - } - r.SendInternalServerError(fmt.Errorf("failed to get log of task %d: %v", r.task.ID, err)) - return - } - r.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes))) - r.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain") - _, err = r.Ctx.ResponseWriter.Write(logBytes) - if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to write log of task %d: %v", r.task.ID, err)) - return - } -} diff --git a/src/core/api/replication_execution_test.go b/src/core/api/replication_execution_test.go deleted file mode 100644 index 2b7687d21..000000000 --- a/src/core/api/replication_execution_test.go +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "net/http" - "testing" - - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" -) - -type fakedOperationController struct{} - -func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) { - return 1, nil -} -func (f *fakedOperationController) StopReplication(int64) error { - return nil -} -func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) { - return 1, []*models.Execution{ - { - ID: 1, - PolicyID: 1, - }, - }, nil -} -func (f *fakedOperationController) GetExecution(id int64) (*models.Execution, error) { - if id == 1 { - return &models.Execution{ - ID: 1, - PolicyID: 1, - }, nil - } - return nil, nil -} -func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) { - return 2, []*models.Task{ - { - ID: 1, - ExecutionID: 1, - }, - }, nil -} -func (f *fakedOperationController) GetTask(id int64) (*models.Task, error) { - if id == 1 { - return &models.Task{ - ID: 1, - ExecutionID: 1, - }, nil - } - if id == 3 { - return &models.Task{ - ID: 3, - ExecutionID: 2, - }, nil - } - return nil, nil -} -func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error { - return nil -} -func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) { - return []byte("success"), nil -} - -type fakedPolicyManager struct{} - -func (f *fakedPolicyManager) Create(*model.Policy) (int64, error) { - return 0, nil -} -func (f *fakedPolicyManager) List(...*model.PolicyQuery) (int64, []*model.Policy, error) { - return 0, nil, nil -} -func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) { - if id == 1 { - return &model.Policy{ - ID: 1, - Enabled: true, - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, nil - } - if id == 2 { - return &model.Policy{ - ID: 2, - Enabled: false, - SrcRegistry: &model.Registry{ - ID: 1, - }, - }, nil - } - return nil, nil -} -func (f *fakedPolicyManager) GetByName(name string) (*model.Policy, error) { - if name == "duplicate_name" { - return &model.Policy{ - Name: "duplicate_name", - }, nil - } - return nil, nil -} -func (f *fakedPolicyManager) Update(*model.Policy) error { - return nil -} -func (f *fakedPolicyManager) Remove(int64) error { - return nil -} - -func TestListExecutions(t *testing.T) { - operationCtl := replication.OperationCtl - defer func() { - replication.OperationCtl = operationCtl - }() - replication.OperationCtl = &fakedOperationController{} - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions", - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestCreateExecution(t *testing.T) { - operationCtl := replication.OperationCtl - policyMgr := replication.PolicyCtl - registryMgr := replication.RegistryMgr - defer func() { - replication.OperationCtl = operationCtl - replication.PolicyCtl = policyMgr - replication.RegistryMgr = registryMgr - }() - replication.OperationCtl = &fakedOperationController{} - replication.PolicyCtl = &fakedPolicyManager{} - replication.RegistryMgr = &fakedRegistryManager{} - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/executions", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/executions", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 404 - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/executions", - bodyJSON: &models.Execution{ - PolicyID: 3, - }, - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 400 - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/executions", - bodyJSON: &models.Execution{ - PolicyID: 2, - }, - credential: sysAdmin, - }, - code: http.StatusBadRequest, - }, - // 201 - { - request: &testingRequest{ - method: http.MethodPost, - url: "/api/replication/executions", - bodyJSON: &models.Execution{ - PolicyID: 1, - }, - credential: sysAdmin, - }, - code: http.StatusCreated, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestGetExecution(t *testing.T) { - operationCtl := replication.OperationCtl - defer func() { - replication.OperationCtl = operationCtl - }() - replication.OperationCtl = &fakedOperationController{} - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 404 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/2", - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1", - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} -func TestStopExecution(t *testing.T) { - operationCtl := replication.OperationCtl - defer func() { - replication.OperationCtl = operationCtl - }() - replication.OperationCtl = &fakedOperationController{} - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/executions/1", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/executions/1", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 404 - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/executions/2", - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodPut, - url: "/api/replication/executions/1", - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestListTasks(t *testing.T) { - operationCtl := replication.OperationCtl - defer func() { - replication.OperationCtl = operationCtl - }() - replication.OperationCtl = &fakedOperationController{} - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1/tasks", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1/tasks", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 404 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/2/tasks", - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1/tasks", - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestGetTaskLog(t *testing.T) { - operationCtl := replication.OperationCtl - defer func() { - replication.OperationCtl = operationCtl - }() - replication.OperationCtl = &fakedOperationController{} - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1/tasks/1/log", - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1/tasks/1/log", - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - // 404, execution not found - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/2/tasks/1/log", - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 404, task not found - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1/tasks/2/log", - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 404, task not found - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1/tasks/3/log", - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodGet, - url: "/api/replication/executions/1/tasks/1/log", - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} diff --git a/src/core/api/replication_policy.go b/src/core/api/replication_policy.go index 02bbda9de..415b931c5 100644 --- a/src/core/api/replication_policy.go +++ b/src/core/api/replication_policy.go @@ -17,11 +17,15 @@ package api import ( "errors" "fmt" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" "net/http" "strconv" + replica "github.com/goharbor/harbor/src/controller/replication" + "github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/dao/models" "github.com/goharbor/harbor/src/replication/event" "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/registry" @@ -219,16 +223,29 @@ func (r *ReplicationPolicyAPI) Delete() { return } - isRunning, err := hasRunningExecutions(id) + ctx := orm.Context() + executions, err := replica.Ctl.ListExecutions(ctx, &q.Query{ + Keywords: map[string]interface{}{ + "PolicyID": id, + }, + }) if err != nil { - r.SendInternalServerError(fmt.Errorf("failed to check the execution status of policy %d: %v", id, err)) + r.SendInternalServerError(err) return } - - if isRunning { + for _, execution := range executions { + if execution.Status != job.RunningStatus.String() { + continue + } r.SendPreconditionFailedError(fmt.Errorf("the policy %d has running executions, can not be deleted", id)) return } + for _, execution := range executions { + if err = task.ExecMgr.Delete(ctx, execution.ID); err != nil { + r.SendInternalServerError(err) + return + } + } if err := replication.PolicyCtl.Remove(id); err != nil { r.SendInternalServerError(fmt.Errorf("failed to delete the policy %d: %v", id, err)) @@ -236,22 +253,6 @@ func (r *ReplicationPolicyAPI) Delete() { } } -func hasRunningExecutions(policyID int64) (bool, error) { - _, executions, err := replication.OperationCtl.ListExecutions(&models.ExecutionQuery{ - PolicyID: policyID, - }) - if err != nil { - return false, err - } - for _, execution := range executions { - if execution.Status != models.ExecutionStatusInProgress { - continue - } - return true, nil - } - return false, nil -} - // ignore the credential for the registries func populateRegistries(registryMgr registry.Manager, policy *model.Policy) error { if err := event.PopulateRegistries(registryMgr, policy); err != nil { diff --git a/src/core/api/replication_policy_test.go b/src/core/api/replication_policy_test.go index 18ece2b9e..8ce75d678 100644 --- a/src/core/api/replication_policy_test.go +++ b/src/core/api/replication_policy_test.go @@ -54,6 +54,50 @@ func (f *fakedRegistryManager) HealthCheck() error { return nil } +type fakedPolicyManager struct{} + +func (f *fakedPolicyManager) Create(*model.Policy) (int64, error) { + return 0, nil +} +func (f *fakedPolicyManager) List(...*model.PolicyQuery) (int64, []*model.Policy, error) { + return 0, nil, nil +} +func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) { + if id == 1 { + return &model.Policy{ + ID: 1, + Enabled: true, + SrcRegistry: &model.Registry{ + ID: 1, + }, + }, nil + } + if id == 2 { + return &model.Policy{ + ID: 2, + Enabled: false, + SrcRegistry: &model.Registry{ + ID: 1, + }, + }, nil + } + return nil, nil +} +func (f *fakedPolicyManager) GetByName(name string) (*model.Policy, error) { + if name == "duplicate_name" { + return &model.Policy{ + Name: "duplicate_name", + }, nil + } + return nil, nil +} +func (f *fakedPolicyManager) Update(*model.Policy) error { + return nil +} +func (f *fakedPolicyManager) Remove(int64) error { + return nil +} + func TestReplicationPolicyAPIList(t *testing.T) { policyMgr := replication.PolicyCtl defer func() { diff --git a/src/core/service/notifications/jobs/handler.go b/src/core/service/notifications/jobs/handler.go index efe354ec9..8882b7c81 100755 --- a/src/core/service/notifications/jobs/handler.go +++ b/src/core/service/notifications/jobs/handler.go @@ -31,9 +31,6 @@ import ( "github.com/goharbor/harbor/src/pkg/notifier/event" "github.com/goharbor/harbor/src/pkg/retention" sc "github.com/goharbor/harbor/src/pkg/scan" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/operation/hook" - "github.com/goharbor/harbor/src/replication/policy/scheduler" ) var statusMap = map[string]string{ @@ -141,44 +138,6 @@ func (h *Handler) HandleScan() { } } -// HandleReplicationScheduleJob handles the webhook of replication schedule job -func (h *Handler) HandleReplicationScheduleJob() { - log.Debugf("received replication schedule job status update event: schedule-job-%d, status-%s", h.id, h.status) - if err := scheduler.UpdateStatus(h.id, h.status); err != nil { - log.Errorf("Failed to update job status, id: %d, status: %s", h.id, h.status) - h.SendInternalServerError(err) - return - } -} - -// HandleReplicationTask handles the webhook of replication task -func (h *Handler) HandleReplicationTask() { - log.Debugf("received replication task status update event: task-%d, status-%s", h.id, h.status) - - if err := hook.UpdateTask(replication.OperationCtl, h.id, h.rawStatus, h.revision); err != nil { - log.Errorf("failed to update the status of the replication task %d: %v", h.id, err) - h.SendInternalServerError(err) - return - } - - // Trigger artifict webhook event only for JobFinished and JobError status - if h.status == models.JobFinished || h.status == models.JobError || h.status == models.JobStopped { - e := &event.Event{} - metaData := &metadata.ReplicationMetaData{ - ReplicationTaskID: h.id, - Status: h.rawStatus, - } - - if err := e.Build(metaData); err == nil { - if err := e.Publish(); err != nil { - log.Error(errors.Wrap(err, "replication job hook handler: event publish")) - } - } else { - log.Error(errors.Wrap(err, "replication job hook handler: event publish")) - } - } -} - // HandleRetentionTask handles the webhook of retention task func (h *Handler) HandleRetentionTask() { taskID := h.id diff --git a/src/core/service/notifications/scheduler/handler.go b/src/core/service/notifications/scheduler/handler.go deleted file mode 100644 index f31800889..000000000 --- a/src/core/service/notifications/scheduler/handler.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scheduler - -import ( - "encoding/json" - - "github.com/goharbor/harbor/src/core/service/notifications" - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/pkg/scheduler" -) - -// Handler handles the scheduler requests -type Handler struct { - notifications.BaseHandler -} - -// Handle ... -func (h *Handler) Handle() { - log.Debugf("received scheduler hook event for schedule %s", h.GetStringFromPath(":id")) - - var data job.StatusChange - if err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data); err != nil { - log.Errorf("failed to decode hook event: %v", err) - return - } - - schedulerID, err := h.GetInt64FromPath(":id") - if err != nil { - log.Errorf("failed to get the schedule ID: %v", err) - return - } - - if err = scheduler.HandleLegacyHook(h.Ctx.Request.Context(), schedulerID, &data); err != nil { - log.Errorf("failed to handle the legacy hook: %v", err) - return - } -} diff --git a/src/lib/log/context.go b/src/lib/log/context.go index 85f7c055d..eb7280709 100644 --- a/src/lib/log/context.go +++ b/src/lib/log/context.go @@ -35,8 +35,11 @@ func WithLogger(ctx context.Context, logger *Logger) context.Context { // GetLogger retrieves the current logger from the context. // If no logger is available, the default logger is returned. func GetLogger(ctx context.Context) *Logger { - logger := ctx.Value(loggerKey{}) + if ctx == nil { + return L + } + logger := ctx.Value(loggerKey{}) if logger == nil { return L } diff --git a/src/replication/operation/flow/controller_test.go b/src/lib/orm/creator.go similarity index 58% rename from src/replication/operation/flow/controller_test.go rename to src/lib/orm/creator.go index 7645e8144..f06bd2b73 100644 --- a/src/replication/operation/flow/controller_test.go +++ b/src/lib/orm/creator.go @@ -12,25 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -package flow +package orm -import ( - "testing" +import "github.com/astaxie/beego/orm" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" +var ( + // Crt is a global instance of ORM creator + Crt = NewCreator() ) -type fakedFlow struct{} - -func (f *fakedFlow) Run(interface{}) (int, error) { - return 1, nil +// NewCreator creates an ORM creator +func NewCreator() Creator { + return &creator{} } -func TestStart(t *testing.T) { - flow := &fakedFlow{} - controller := NewController() - n, err := controller.Start(flow) - require.Nil(t, err) - assert.Equal(t, 1, n) +// Creator creates ORMer +// Introducing the "Creator" interface to eliminate the dependency on database +type Creator interface { + Create() orm.Ormer +} + +type creator struct{} + +func (c *creator) Create() orm.Ormer { + return orm.NewOrm() } diff --git a/src/lib/q/query.go b/src/lib/q/query.go index 267681eda..09a700eeb 100644 --- a/src/lib/q/query.go +++ b/src/lib/q/query.go @@ -37,11 +37,18 @@ func New(kw KeyWords) *Query { // MustClone returns the clone of query when it's not nil // or returns a new Query instance func MustClone(query *Query) *Query { - if query != nil { - clone := *query - return &clone + q := &Query{ + Keywords: map[string]interface{}{}, } - return New(KeyWords{}) + if query != nil { + q.PageNumber = query.PageNumber + q.PageSize = query.PageSize + q.Sorting = query.Sorting + for k, v := range query.Keywords { + q.Keywords[k] = v + } + } + return q } // Range query diff --git a/src/replication/operation/flow/controller.go b/src/lib/worker_pool.go similarity index 53% rename from src/replication/operation/flow/controller.go rename to src/lib/worker_pool.go index 2119cfa3f..81993fd72 100644 --- a/src/replication/operation/flow/controller.go +++ b/src/lib/worker_pool.go @@ -12,26 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package flow +package lib -// Flow defines the replication flow -type Flow interface { - // returns the count of tasks which have been scheduled and the error - Run(interface{}) (int, error) +// NewWorkerPool creates a new worker pool with specified size +func NewWorkerPool(size int32) *WorkerPool { + wp := &WorkerPool{} + wp.queue = make(chan struct{}, size) + return wp } -// Controller is the controller that controls the replication flows -type Controller interface { - Start(Flow) (int, error) +// WorkerPool controls the concurrency limit of task/process +type WorkerPool struct { + queue chan struct{} } -// NewController returns an instance of the default flow controller -func NewController() Controller { - return &controller{} +// GetWorker hangs until a free worker available +func (w *WorkerPool) GetWorker() { + w.queue <- struct{}{} } -type controller struct{} - -func (c *controller) Start(flow Flow) (int, error) { - return flow.Run(nil) +// ReleaseWorker hangs until the worker return back into the pool +func (w *WorkerPool) ReleaseWorker() { + <-w.queue } diff --git a/src/pkg/scheduler/callback.go b/src/pkg/scheduler/callback.go index af2d19ae0..1bbe9d9de 100644 --- a/src/pkg/scheduler/callback.go +++ b/src/pkg/scheduler/callback.go @@ -18,7 +18,6 @@ import ( "context" "fmt" - "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/task" @@ -29,7 +28,7 @@ var ( ) // CallbackFunc defines the function that the scheduler calls when triggered -type CallbackFunc func(interface{}) error +type CallbackFunc func(ctx context.Context, param string) error func init() { if err := task.RegisterCheckInProcessor(JobNameScheduler, triggerCallback); err != nil { @@ -68,7 +67,7 @@ func callbackFuncExist(name string) bool { return exist } -func triggerCallback(ctx context.Context, task *task.Task, change *job.StatusChange) (err error) { +func triggerCallback(ctx context.Context, task *task.Task, data string) (err error) { execution, err := Sched.(*scheduler).execMgr.Get(ctx, task.ExecutionID) if err != nil { return err @@ -85,5 +84,5 @@ func triggerCallback(ctx context.Context, task *task.Task, change *job.StatusCha if err != nil { return err } - return callbackFunc(schedule.CallbackFuncParam) + return callbackFunc(ctx, schedule.CallbackFuncParam) } diff --git a/src/pkg/scheduler/callback_test.go b/src/pkg/scheduler/callback_test.go index 62525fb00..f57f1ef17 100644 --- a/src/pkg/scheduler/callback_test.go +++ b/src/pkg/scheduler/callback_test.go @@ -15,6 +15,7 @@ package scheduler import ( + "context" "testing" "github.com/stretchr/testify/suite" @@ -26,7 +27,7 @@ type callbackTestSuite struct { func (c *callbackTestSuite) SetupTest() { registry = map[string]CallbackFunc{} - err := RegisterCallbackFunc("callback", func(interface{}) error { return nil }) + err := RegisterCallbackFunc("callback", func(context.Context, string) error { return nil }) c.Require().Nil(err) } @@ -40,11 +41,11 @@ func (c *callbackTestSuite) TestRegisterCallbackFunc() { c.NotNil(err) // pass - err = RegisterCallbackFunc("test", func(interface{}) error { return nil }) + err = RegisterCallbackFunc("test", func(context.Context, string) error { return nil }) c.Nil(err) // duplicate name - err = RegisterCallbackFunc("test", func(interface{}) error { return nil }) + err = RegisterCallbackFunc("test", func(context.Context, string) error { return nil }) c.NotNil(err) } diff --git a/src/pkg/scheduler/scheduler.go b/src/pkg/scheduler/scheduler.go index a80d33aa2..dc7501abc 100644 --- a/src/pkg/scheduler/scheduler.go +++ b/src/pkg/scheduler/scheduler.go @@ -289,38 +289,3 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S } return schd, nil } - -// HandleLegacyHook handles the legacy web hook for scheduler -// We rewrite the implementation of scheduler with task manager mechanism in v2.1, -// this method is used to handle the job status hook for the legacy implementation -// We can remove the method and the hook endpoint after several releases -func HandleLegacyHook(ctx context.Context, scheduleID int64, sc *job.StatusChange) error { - scheduler := Sched.(*scheduler) - executions, err := scheduler.execMgr.List(ctx, &q.Query{ - Keywords: map[string]interface{}{ - "VendorType": JobNameScheduler, - "VendorID": scheduleID, - }, - }) - if err != nil { - return err - } - if len(executions) == 0 { - return errors.New(nil).WithCode(errors.NotFoundCode). - WithMessage("no execution found for the schedule %d", scheduleID) - } - - tasks, err := scheduler.taskMgr.List(ctx, &q.Query{ - Keywords: map[string]interface{}{ - "ExecutionID": executions[0].ID, - }, - }) - if err != nil { - return err - } - if len(tasks) == 0 { - return errors.New(nil).WithCode(errors.NotFoundCode). - WithMessage("no task found for the execution %d", executions[0].ID) - } - return task.NewHookHandler().Handle(ctx, tasks[0].ID, sc) -} diff --git a/src/pkg/scheduler/scheduler_test.go b/src/pkg/scheduler/scheduler_test.go index cd3394ead..e8b01b387 100644 --- a/src/pkg/scheduler/scheduler_test.go +++ b/src/pkg/scheduler/scheduler_test.go @@ -15,13 +15,15 @@ package scheduler import ( + "context" "fmt" + "testing" + "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/testing/mock" tasktesting "github.com/goharbor/harbor/src/testing/pkg/task" "github.com/stretchr/testify/suite" - "testing" ) type schedulerTestSuite struct { @@ -34,7 +36,7 @@ type schedulerTestSuite struct { func (s *schedulerTestSuite) SetupTest() { registry = map[string]CallbackFunc{} - err := RegisterCallbackFunc("callback", func(interface{}) error { return nil }) + err := RegisterCallbackFunc("callback", func(context.Context, string) error { return nil }) s.Require().Nil(err) s.dao = &mockDAO{} diff --git a/src/pkg/task/checkin.go b/src/pkg/task/checkin.go deleted file mode 100644 index 2bd503f38..000000000 --- a/src/pkg/task/checkin.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package task - -import ( - "context" - "fmt" - - "github.com/goharbor/harbor/src/jobservice/job" -) - -var ( - registry = map[string]CheckInProcessor{} -) - -// CheckInProcessor is the processor to process the check in data which is sent by jobservice via webhook -type CheckInProcessor func(ctx context.Context, task *Task, change *job.StatusChange) (err error) - -// RegisterCheckInProcessor registers check in processor for the specific vendor type -func RegisterCheckInProcessor(vendorType string, processor CheckInProcessor) error { - if _, exist := registry[vendorType]; exist { - return fmt.Errorf("check in processor for %s already exists", vendorType) - } - registry[vendorType] = processor - return nil -} diff --git a/src/pkg/task/dao/execution.go b/src/pkg/task/dao/execution.go index 7aecc9e8a..cbf7ecf77 100644 --- a/src/pkg/task/dao/execution.go +++ b/src/pkg/task/dao/execution.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/goharbor/harbor/src/lib/log" + "strings" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/errors" @@ -28,8 +29,10 @@ import ( // ExecutionDAO is the data access object interface for execution type ExecutionDAO interface { // Count returns the total count of executions according to the query + // Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"' Count(ctx context.Context, query *q.Query) (count int64, err error) // List the executions according to the query + // Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"' List(ctx context.Context, query *q.Query) (executions []*Execution, err error) // Get the specified execution Get(ctx context.Context, id int64) (execution *Execution, err error) @@ -43,7 +46,9 @@ type ExecutionDAO interface { GetMetrics(ctx context.Context, id int64) (metrics *Metrics, err error) // RefreshStatus refreshes the status of the specified execution according to it's tasks. If it's status // is final, update the end time as well - RefreshStatus(ctx context.Context, id int64) (err error) + // If the status is changed, the returning "statusChanged" is set as "true" and the current status indicates + // the changed status + RefreshStatus(ctx context.Context, id int64) (statusChanged bool, currentStatus string, err error) } // NewExecutionDAO returns an instance of ExecutionDAO @@ -64,7 +69,7 @@ func (e *executionDAO) Count(ctx context.Context, query *q.Query) (int64, error) Keywords: query.Keywords, } } - qs, err := orm.QuerySetter(ctx, &Execution{}, query) + qs, err := e.querySetter(ctx, query) if err != nil { return 0, err } @@ -73,7 +78,7 @@ func (e *executionDAO) Count(ctx context.Context, query *q.Query) (int64, error) func (e *executionDAO) List(ctx context.Context, query *q.Query) ([]*Execution, error) { executions := []*Execution{} - qs, err := orm.QuerySetter(ctx, &Execution{}, query) + qs, err := e.querySetter(ctx, query) if err != nil { return nil, err } @@ -178,33 +183,39 @@ func (e *executionDAO) GetMetrics(ctx context.Context, id int64) (*Metrics, erro metrics.ScheduledTaskCount + metrics.StoppedTaskCount return metrics, nil } -func (e *executionDAO) RefreshStatus(ctx context.Context, id int64) error { + +func (e *executionDAO) RefreshStatus(ctx context.Context, id int64) (bool, string, error) { // as the status of the execution can be refreshed by multiple operators concurrently // we use the optimistic locking to avoid the conflict and retry 5 times at most for i := 0; i < 5; i++ { - retry, err := e.refreshStatus(ctx, id) + statusChanged, currentStatus, retry, err := e.refreshStatus(ctx, id) if err != nil { - return err + return false, "", err } if !retry { - return nil + return statusChanged, currentStatus, nil } } - return fmt.Errorf("failed to refresh the status of the execution %d after %d retries", id, 5) + return false, "", fmt.Errorf("failed to refresh the status of the execution %d after %d retries", id, 5) } -func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error) { +// the returning values: +// 1. bool: is the status changed +// 2. string: the current status if changed +// 3. bool: whether a retry is needed +// 4. error: the error +func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, string, bool, error) { execution, err := e.Get(ctx, id) if err != nil { - return false, err + return false, "", false, err } metrics, err := e.GetMetrics(ctx, id) if err != nil { - return false, err + return false, "", false, err } // no task, return directly if metrics.TaskCount == 0 { - return false, nil + return false, "", false, nil } var status string @@ -220,20 +231,22 @@ func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error ormer, err := orm.FromContext(ctx) if err != nil { - return false, err + return false, "", false, err } + sql := `update execution set status = ?, revision = revision+1 where id = ? and revision = ?` result, err := ormer.Raw(sql, status, id, execution.Revision).Exec() if err != nil { - return false, err + return false, "", false, err } n, err := result.RowsAffected() if err != nil { - return false, err + return false, "", false, err } + // if the count of affected rows is 0, that means the execution is updating by others, retry if n == 0 { - return true, nil + return false, "", true, nil } /* this is another solution to solve the concurrency issue for refreshing the execution status @@ -290,5 +303,28 @@ func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error where id=?` sql = fmt.Sprintf(sql, job.ErrorStatus.String(), job.StoppedStatus.String(), job.SuccessStatus.String()) _, err = ormer.Raw(sql, id, id).Exec() - return false, err + return status != execution.Status, status, false, err +} + +func (e *executionDAO) querySetter(ctx context.Context, query *q.Query) (orm.QuerySeter, error) { + qs, err := orm.QuerySetter(ctx, &Execution{}, query) + if err != nil { + return nil, err + } + + // append the filter for "extra attrs" + if query != nil && len(query.Keywords) > 0 { + for key, value := range query.Keywords { + if strings.HasPrefix(key, "ExtraAttrs.") { + qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from execution where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "ExtraAttrs."), value)) + break + } + if strings.HasPrefix(key, "extra_attrs.") { + qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from execution where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "extra_attrs."), value)) + break + } + } + } + + return qs, nil } diff --git a/src/pkg/task/dao/execution_test.go b/src/pkg/task/dao/execution_test.go index cf1cdbe81..1e54eed02 100644 --- a/src/pkg/task/dao/execution_test.go +++ b/src/pkg/task/dao/execution_test.go @@ -48,7 +48,7 @@ func (e *executionDAOTestSuite) SetupTest() { id, err := e.executionDAO.Create(e.ctx, &Execution{ VendorType: "test", Trigger: "test", - ExtraAttrs: "{}", + ExtraAttrs: `{"key":"value"}`, }) e.Require().Nil(err) e.executionID = id @@ -62,22 +62,42 @@ func (e *executionDAOTestSuite) TearDownTest() { func (e *executionDAOTestSuite) TestCount() { count, err := e.executionDAO.Count(e.ctx, &q.Query{ Keywords: map[string]interface{}{ - "VendorType": "test", + "VendorType": "test", + "ExtraAttrs.key": "value", }, }) e.Require().Nil(err) e.Equal(int64(1), count) + + count, err = e.executionDAO.Count(e.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "VendorType": "test", + "ExtraAttrs.key": "incorrect-value", + }, + }) + e.Require().Nil(err) + e.Equal(int64(0), count) } func (e *executionDAOTestSuite) TestList() { executions, err := e.executionDAO.List(e.ctx, &q.Query{ Keywords: map[string]interface{}{ - "VendorType": "test", + "VendorType": "test", + "ExtraAttrs.key": "value", }, }) e.Require().Nil(err) e.Require().Len(executions, 1) e.Equal(e.executionID, executions[0].ID) + + executions, err = e.executionDAO.List(e.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "VendorType": "test", + "ExtraAttrs.key": "incorrect-value", + }, + }) + e.Require().Nil(err) + e.Require().Len(executions, 0) } func (e *executionDAOTestSuite) TestGet() { @@ -200,8 +220,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() { e.Require().Nil(err) defer e.taskDao.Delete(e.ctx, taskID01) - err = e.executionDAO.RefreshStatus(e.ctx, e.executionID) + statusChanged, currentStatus, err := e.executionDAO.RefreshStatus(e.ctx, e.executionID) e.Require().Nil(err) + e.True(statusChanged) + e.Equal(job.SuccessStatus.String(), currentStatus) execution, err := e.executionDAO.Get(e.ctx, e.executionID) e.Require().Nil(err) e.Equal(job.SuccessStatus.String(), execution.Status) @@ -218,8 +240,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() { e.Require().Nil(err) defer e.taskDao.Delete(e.ctx, taskID02) - err = e.executionDAO.RefreshStatus(e.ctx, e.executionID) + statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID) e.Require().Nil(err) + e.True(statusChanged) + e.Equal(job.StoppedStatus.String(), currentStatus) execution, err = e.executionDAO.Get(e.ctx, e.executionID) e.Require().Nil(err) e.Equal(job.StoppedStatus.String(), execution.Status) @@ -236,8 +260,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() { e.Require().Nil(err) defer e.taskDao.Delete(e.ctx, taskID03) - err = e.executionDAO.RefreshStatus(e.ctx, e.executionID) + statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID) e.Require().Nil(err) + e.True(statusChanged) + e.Equal(job.ErrorStatus.String(), currentStatus) execution, err = e.executionDAO.Get(e.ctx, e.executionID) e.Require().Nil(err) e.Equal(job.ErrorStatus.String(), execution.Status) @@ -271,8 +297,29 @@ func (e *executionDAOTestSuite) TestRefreshStatus() { e.Require().Nil(err) defer e.taskDao.Delete(e.ctx, taskID06) - err = e.executionDAO.RefreshStatus(e.ctx, e.executionID) + statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID) e.Require().Nil(err) + e.True(statusChanged) + e.Equal(job.RunningStatus.String(), currentStatus) + execution, err = e.executionDAO.Get(e.ctx, e.executionID) + e.Require().Nil(err) + e.Equal(job.RunningStatus.String(), execution.Status) + e.Empty(execution.EndTime) + + // add another running task, the status shouldn't be changed + taskID07, err := e.taskDao.Create(e.ctx, &Task{ + ExecutionID: e.executionID, + Status: job.RunningStatus.String(), + StatusCode: job.RunningStatus.Code(), + ExtraAttrs: "{}", + }) + e.Require().Nil(err) + defer e.taskDao.Delete(e.ctx, taskID07) + + statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID) + e.Require().Nil(err) + e.False(statusChanged) + e.Equal(job.RunningStatus.String(), currentStatus) execution, err = e.executionDAO.Get(e.ctx, e.executionID) e.Require().Nil(err) e.Equal(job.RunningStatus.String(), execution.Status) diff --git a/src/pkg/task/dao/model.go b/src/pkg/task/dao/model.go index 4862e8a35..83659a130 100644 --- a/src/pkg/task/dao/model.go +++ b/src/pkg/task/dao/model.go @@ -56,13 +56,14 @@ type Metrics struct { // Task database model type Task struct { ID int64 `orm:"pk;auto;column(id)"` + VendorType string `orm:"column(vendor_type)"` ExecutionID int64 `orm:"column(execution_id)"` JobID string `orm:"column(job_id)"` Status string `orm:"column(status)"` StatusCode int `orm:"column(status_code)"` StatusRevision int64 `orm:"column(status_revision)"` StatusMessage string `orm:"column(status_message)"` - RunCount int `orm:"column(run_count)"` + RunCount int32 `orm:"column(run_count)"` ExtraAttrs string `orm:"column(extra_attrs)"` // json string CreationTime time.Time `orm:"column(creation_time)"` StartTime time.Time `orm:"column(start_time)"` diff --git a/src/pkg/task/dao/task.go b/src/pkg/task/dao/task.go index 1ec3cc169..9515edcdf 100644 --- a/src/pkg/task/dao/task.go +++ b/src/pkg/task/dao/task.go @@ -16,6 +16,8 @@ package dao import ( "context" + "fmt" + "strings" "time" "github.com/goharbor/harbor/src/jobservice/job" @@ -27,8 +29,10 @@ import ( // TaskDAO is the data access object interface for task type TaskDAO interface { // Count returns the total count of tasks according to the query + // Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"' Count(ctx context.Context, query *q.Query) (count int64, err error) // List the tasks according to the query + // Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"' List(ctx context.Context, query *q.Query) (tasks []*Task, err error) // Get the specified task Get(ctx context.Context, id int64) (task *Task, err error) @@ -60,7 +64,7 @@ func (t *taskDAO) Count(ctx context.Context, query *q.Query) (int64, error) { Keywords: query.Keywords, } } - qs, err := orm.QuerySetter(ctx, &Task{}, query) + qs, err := t.querySetter(ctx, query) if err != nil { return 0, err } @@ -69,7 +73,7 @@ func (t *taskDAO) Count(ctx context.Context, query *q.Query) (int64, error) { func (t *taskDAO) List(ctx context.Context, query *q.Query) ([]*Task, error) { tasks := []*Task{} - qs, err := orm.QuerySetter(ctx, &Task{}, query) + qs, err := t.querySetter(ctx, query) if err != nil { return nil, err } @@ -203,3 +207,26 @@ func (t *taskDAO) GetMaxEndTime(ctx context.Context, executionID int64) (time.Ti QueryRow(&endTime) return endTime, nil } + +func (t *taskDAO) querySetter(ctx context.Context, query *q.Query) (orm.QuerySeter, error) { + qs, err := orm.QuerySetter(ctx, &Task{}, query) + if err != nil { + return nil, err + } + + // append the filter for "extra attrs" + if query != nil && len(query.Keywords) > 0 { + for key, value := range query.Keywords { + if strings.HasPrefix(key, "ExtraAttrs.") { + qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from task where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "ExtraAttrs."), value)) + break + } + if strings.HasPrefix(key, "extra_attrs.") { + qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from task where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "extra_attrs."), value)) + break + } + } + } + + return qs, nil +} diff --git a/src/pkg/task/dao/task_test.go b/src/pkg/task/dao/task_test.go index eb59c1054..289c112bd 100644 --- a/src/pkg/task/dao/task_test.go +++ b/src/pkg/task/dao/task_test.go @@ -53,7 +53,7 @@ func (t *taskDAOTestSuite) SetupTest() { ExecutionID: t.executionID, Status: "success", StatusCode: 1, - ExtraAttrs: "{}", + ExtraAttrs: `{"key":"value"}`, }) t.Require().Nil(err) t.taskID = id @@ -70,22 +70,42 @@ func (t *taskDAOTestSuite) TearDownTest() { func (t *taskDAOTestSuite) TestCount() { count, err := t.taskDAO.Count(t.ctx, &q.Query{ Keywords: map[string]interface{}{ - "ExecutionID": t.executionID, + "ExecutionID": t.executionID, + "ExtraAttrs.key": "value", }, }) t.Require().Nil(err) t.Equal(int64(1), count) + + count, err = t.taskDAO.Count(t.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ExecutionID": t.executionID, + "ExtraAttrs.key": "incorrect-value", + }, + }) + t.Require().Nil(err) + t.Equal(int64(0), count) } func (t *taskDAOTestSuite) TestList() { tasks, err := t.taskDAO.List(t.ctx, &q.Query{ Keywords: map[string]interface{}{ - "ExecutionID": t.executionID, + "ExecutionID": t.executionID, + "ExtraAttrs.key": "value", }, }) t.Require().Nil(err) t.Require().Len(tasks, 1) t.Equal(t.taskID, tasks[0].ID) + + tasks, err = t.taskDAO.List(t.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ExecutionID": t.executionID, + "ExtraAttrs.key": "incorrect-value", + }, + }) + t.Require().Nil(err) + t.Require().Len(tasks, 0) } func (t *taskDAOTestSuite) TestGet() { @@ -140,7 +160,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() { task, err := t.taskDAO.Get(t.ctx, t.taskID) t.Require().Nil(err) - t.Equal(1, task.RunCount) + t.Equal(int32(1), task.RunCount) t.True(time.Unix(statusRevision, 0).Equal(task.StartTime)) t.Equal(status, task.Status) t.Equal(job.RunningStatus.Code(), task.StatusCode) @@ -155,7 +175,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() { task, err = t.taskDAO.Get(t.ctx, t.taskID) t.Require().Nil(err) - t.Equal(1, task.RunCount) + t.Equal(int32(1), task.RunCount) t.True(time.Unix(statusRevision, 0).Equal(task.StartTime)) t.Equal(status, task.Status) t.Equal(job.SuccessStatus.Code(), task.StatusCode) @@ -170,7 +190,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() { task, err = t.taskDAO.Get(t.ctx, t.taskID) t.Require().Nil(err) - t.Equal(2, task.RunCount) + t.Equal(int32(2), task.RunCount) t.True(time.Unix(statusRevision, 0).Equal(task.StartTime)) t.Equal(status, task.Status) t.Equal(job.RunningStatus.Code(), task.StatusCode) diff --git a/src/pkg/task/execution.go b/src/pkg/task/execution.go index 69a9d5a76..fff64fa21 100644 --- a/src/pkg/task/execution.go +++ b/src/pkg/task/execution.go @@ -62,8 +62,10 @@ type ExecutionManager interface { // Get the specified execution Get(ctx context.Context, id int64) (execution *Execution, err error) // List executions according to the query + // Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"' List(ctx context.Context, query *q.Query) (executions []*Execution, err error) - // Count counts total. + // Count counts total of executions according to the query. + // Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"' Count(ctx context.Context, query *q.Query) (int64, error) } @@ -100,6 +102,7 @@ func (e *executionManager) Create(ctx context.Context, vendorType string, vendor execution := &dao.Execution{ VendorType: vendorType, VendorID: vendorID, + Status: job.RunningStatus.String(), Trigger: trigger, ExtraAttrs: string(data), StartTime: time.Now(), @@ -156,7 +159,11 @@ func (e *executionManager) Stop(ctx context.Context, id int64) error { continue } } - return nil + + // refresh the status explicitly in case that the execution status + // isn't refreshed by task status change hook + _, _, err = e.executionDAO.RefreshStatus(ctx, id) + return err } func (e *executionManager) StopAndWait(ctx context.Context, id int64, timeout time.Duration) error { diff --git a/src/pkg/task/execution_test.go b/src/pkg/task/execution_test.go index 6d6ff32c8..c5b50993e 100644 --- a/src/pkg/task/execution_test.go +++ b/src/pkg/task/execution_test.go @@ -104,6 +104,7 @@ func (e *executionManagerTestSuite) TestStop() { }, }, nil) e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil) + e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil) err = e.execMgr.Stop(nil, 1) e.Require().Nil(err) e.taskDAO.AssertExpectations(e.T()) @@ -124,6 +125,7 @@ func (e *executionManagerTestSuite) TestStopAndWait() { }, }, nil) e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil) + e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil) err := e.execMgr.StopAndWait(nil, 1, 1*time.Second) e.Require().NotNil(err) e.taskDAO.AssertExpectations(e.T()) @@ -145,6 +147,7 @@ func (e *executionManagerTestSuite) TestStopAndWait() { }, }, nil) e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil) + e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil) err = e.execMgr.StopAndWait(nil, 1, 1*time.Second) e.Require().Nil(err) e.taskDAO.AssertExpectations(e.T()) diff --git a/src/pkg/task/hook.go b/src/pkg/task/hook.go index 320616a69..e050e5f6e 100644 --- a/src/pkg/task/hook.go +++ b/src/pkg/task/hook.go @@ -19,9 +19,17 @@ import ( "fmt" "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/task/dao" ) +var ( + // HkHandler is a global instance of the HookHandler + HkHandler = NewHookHandler() +) + // NewHookHandler creates a hook handler instance func NewHookHandler() *HookHandler { return &HookHandler{ @@ -37,31 +45,68 @@ type HookHandler struct { } // Handle the job status changing webhook -func (h *HookHandler) Handle(ctx context.Context, taskID int64, sc *job.StatusChange) error { - task, err := h.taskDAO.Get(ctx, taskID) +func (h *HookHandler) Handle(ctx context.Context, sc *job.StatusChange) error { + logger := log.GetLogger(ctx) + + jobID := sc.JobID + // when a "KindScheduled" job is scheduled by a periodical job, it's "JobID" field + // is set as "87bbdee19bed5ce09c48a149@1605104520" which contains "@". In this case, + // read the parent periodical job ID from "sc.Metadata.UpstreamJobID" + if sc.Metadata.JobKind == job.KindScheduled { + if len(sc.Metadata.UpstreamJobID) > 0 { + jobID = sc.Metadata.UpstreamJobID + } + } + tasks, err := h.taskDAO.List(ctx, &q.Query{ + Keywords: map[string]interface{}{ + "JobID": jobID, + }, + }) if err != nil { return err } + if len(tasks) == 0 { + return errors.New(nil).WithCode(errors.NotFoundCode). + WithMessage("task with job ID %s not found", sc.JobID) + } + task := tasks[0] + execution, err := h.executionDAO.Get(ctx, task.ExecutionID) + if err != nil { + return err + } + // process check in data if len(sc.CheckIn) > 0 { - execution, err := h.executionDAO.Get(ctx, task.ExecutionID) - if err != nil { - return err - } - - processor, exist := registry[execution.VendorType] + processor, exist := checkInProcessorRegistry[execution.VendorType] if !exist { - return fmt.Errorf("the check in processor for task %d not found", taskID) + return fmt.Errorf("the check in processor for task %d not found", task.ID) } t := &Task{} t.From(task) - return processor(ctx, t, sc) + return processor(ctx, t, sc.CheckIn) } // update task status - if err = h.taskDAO.UpdateStatus(ctx, taskID, sc.Status, sc.Metadata.Revision); err != nil { + if err = h.taskDAO.UpdateStatus(ctx, task.ID, sc.Status, sc.Metadata.Revision); err != nil { return err } + // run the status change post function + if fc, exist := statusChangePostFuncRegistry[execution.VendorType]; exist { + if err = fc(ctx, task.ID, sc.Status); err != nil { + logger.Errorf("failed to run the task status change post function for task %d: %v", task.ID, err) + } + } + // update execution status - return h.executionDAO.RefreshStatus(ctx, task.ExecutionID) + statusChanged, currentStatus, err := h.executionDAO.RefreshStatus(ctx, task.ExecutionID) + if err != nil { + return err + } + // run the status change post function + if fc, exist := executionStatusChangePostFuncRegistry[execution.VendorType]; exist && statusChanged { + if err = fc(ctx, task.ExecutionID, currentStatus); err != nil { + logger.Errorf("failed to run the execution status change post function for execution %d: %v", task.ExecutionID, err) + } + } + return nil } diff --git a/src/pkg/task/hook_test.go b/src/pkg/task/hook_test.go index 87d732d9b..8a975bae7 100644 --- a/src/pkg/task/hook_test.go +++ b/src/pkg/task/hook_test.go @@ -43,19 +43,23 @@ func (h *hookHandlerTestSuite) SetupTest() { func (h *hookHandlerTestSuite) TestHandle() { // handle check in data - registry["test"] = func(ctx context.Context, task *Task, change *job.StatusChange) (err error) { return nil } - h.taskDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Task{ - ID: 1, - ExecutionID: 1, + checkInProcessorRegistry["test"] = func(ctx context.Context, task *Task, data string) (err error) { return nil } + defer delete(checkInProcessorRegistry, "test") + h.taskDAO.On("List", mock.Anything, mock.Anything).Return([]*dao.Task{ + { + ID: 1, + ExecutionID: 1, + }, }, nil) h.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{ ID: 1, VendorType: "test", }, nil) sc := &job.StatusChange{ - CheckIn: "data", + CheckIn: "data", + Metadata: &job.StatsInfo{}, } - err := h.handler.Handle(nil, 1, sc) + err := h.handler.Handle(nil, sc) h.Require().Nil(err) h.taskDAO.AssertExpectations(h.T()) h.execDAO.AssertExpectations(h.T()) @@ -64,21 +68,28 @@ func (h *hookHandlerTestSuite) TestHandle() { h.SetupTest() // handle status changing - h.taskDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Task{ - ID: 1, - ExecutionID: 1, + h.taskDAO.On("List", mock.Anything, mock.Anything).Return([]*dao.Task{ + { + ID: 1, + ExecutionID: 1, + }, }, nil) h.taskDAO.On("UpdateStatus", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - h.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(nil) + h.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{ + ID: 1, + VendorType: "test", + }, nil) + h.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(true, job.RunningStatus.String(), nil) sc = &job.StatusChange{ Status: job.SuccessStatus.String(), Metadata: &job.StatsInfo{ Revision: time.Now().Unix(), }, } - err = h.handler.Handle(nil, 1, sc) + err = h.handler.Handle(nil, sc) h.Require().Nil(err) h.taskDAO.AssertExpectations(h.T()) + h.execDAO.AssertExpectations(h.T()) } func TestHookHandlerTestSuite(t *testing.T) { diff --git a/src/pkg/task/mock_execution_dao_test.go b/src/pkg/task/mock_execution_dao_test.go index 7480412cc..827466340 100644 --- a/src/pkg/task/mock_execution_dao_test.go +++ b/src/pkg/task/mock_execution_dao_test.go @@ -142,17 +142,31 @@ func (_m *mockExecutionDAO) List(ctx context.Context, query *q.Query) ([]*dao.Ex } // RefreshStatus provides a mock function with given fields: ctx, id -func (_m *mockExecutionDAO) RefreshStatus(ctx context.Context, id int64) error { +func (_m *mockExecutionDAO) RefreshStatus(ctx context.Context, id int64) (bool, string, error) { ret := _m.Called(ctx, id) - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + var r0 bool + if rf, ok := ret.Get(0).(func(context.Context, int64) bool); ok { r0 = rf(ctx, id) } else { - r0 = ret.Error(0) + r0 = ret.Get(0).(bool) } - return r0 + var r1 string + if rf, ok := ret.Get(1).(func(context.Context, int64) string); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Get(1).(string) + } + + var r2 error + if rf, ok := ret.Get(2).(func(context.Context, int64) error); ok { + r2 = rf(ctx, id) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 } // Update provides a mock function with given fields: ctx, execution, props diff --git a/src/pkg/task/model.go b/src/pkg/task/model.go index ddc0b8018..d0800a2a1 100644 --- a/src/pkg/task/model.go +++ b/src/pkg/task/model.go @@ -25,13 +25,6 @@ import ( // const definitions const ( - ExecutionVendorTypeReplication = "REPLICATION" - ExecutionVendorTypeGarbageCollection = "GARBAGE_COLLECTION" - ExecutionVendorTypeRetention = "RETENTION" - ExecutionVendorTypeScan = "SCAN" - ExecutionVendorTypeScanAll = "SCAN_ALL" - ExecutionVendorTypeScheduler = "SCHEDULER" - ExecutionTriggerManual = "MANUAL" ExecutionTriggerSchedule = "SCHEDULE" ExecutionTriggerEvent = "EVENT" @@ -61,14 +54,18 @@ type Execution struct { // Task is the unit for running. It stores the jobservice job records and related information type Task struct { - ID int64 `json:"id"` + ID int64 `json:"id"` + // indicate the task type: replication/GC/retention/scan/etc. + VendorType string `json:"vendor_type"` ExecutionID int64 `json:"execution_id"` Status string `json:"status"` // the detail message to explain the status in some cases. e.g. // When the job is failed to submit to jobservice, this field can be used to explain the reason StatusMessage string `json:"status_message"` // the underlying job may retry several times - RunCount int `json:"run_count"` + RunCount int32 `json:"run_count"` + // the ID of jobservice job + JobID string `json:"job_id"` // the customized attributes for different kinds of consumers ExtraAttrs map[string]interface{} `json:"extra_attrs"` // the time that the task record created @@ -82,10 +79,12 @@ type Task struct { // From constructs a task from DAO model func (t *Task) From(task *dao.Task) { t.ID = task.ID + t.VendorType = task.VendorType t.ExecutionID = task.ExecutionID t.Status = task.Status t.StatusMessage = task.StatusMessage t.RunCount = task.RunCount + t.JobID = task.JobID t.CreationTime = task.CreationTime t.StartTime = task.StartTime t.UpdateTime = task.UpdateTime @@ -100,6 +99,22 @@ func (t *Task) From(task *dao.Task) { } } +// GetStringFromExtraAttrs returns the string value specified by key +func (t *Task) GetStringFromExtraAttrs(key string) string { + if len(t.ExtraAttrs) == 0 { + return "" + } + rt, exist := t.ExtraAttrs[key] + if !exist { + return "" + } + str, ok := rt.(string) + if !ok { + return "" + } + return str +} + // Job is the model represents the requested jobservice job type Job struct { Name string diff --git a/src/pkg/task/registry.go b/src/pkg/task/registry.go new file mode 100644 index 000000000..378915aaf --- /dev/null +++ b/src/pkg/task/registry.go @@ -0,0 +1,62 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task + +import ( + "context" + "fmt" +) + +var ( + checkInProcessorRegistry = map[string]CheckInProcessor{} + statusChangePostFuncRegistry = map[string]StatusChangePostFunc{} + executionStatusChangePostFuncRegistry = map[string]ExecutionStatusChangePostFunc{} +) + +// CheckInProcessor is the processor to process the check in data which is sent by jobservice via webhook +type CheckInProcessor func(ctx context.Context, task *Task, data string) (err error) + +// StatusChangePostFunc is the function called after the task status changed +type StatusChangePostFunc func(ctx context.Context, taskID int64, status string) (err error) + +// ExecutionStatusChangePostFunc is the function called after the execution status changed +type ExecutionStatusChangePostFunc func(ctx context.Context, executionID int64, status string) (err error) + +// RegisterCheckInProcessor registers check in processor for the specific vendor type +func RegisterCheckInProcessor(vendorType string, processor CheckInProcessor) error { + if _, exist := checkInProcessorRegistry[vendorType]; exist { + return fmt.Errorf("check in processor for %s already exists", vendorType) + } + checkInProcessorRegistry[vendorType] = processor + return nil +} + +// RegisterTaskStatusChangePostFunc registers a task status change post function for the specific vendor type +func RegisterTaskStatusChangePostFunc(vendorType string, fc StatusChangePostFunc) error { + if _, exist := statusChangePostFuncRegistry[vendorType]; exist { + return fmt.Errorf("the task status change post function for %s already exists", vendorType) + } + statusChangePostFuncRegistry[vendorType] = fc + return nil +} + +// RegisterExecutionStatusChangePostFunc registers an execution status change post function for the specific vendor type +func RegisterExecutionStatusChangePostFunc(vendorType string, fc ExecutionStatusChangePostFunc) error { + if _, exist := executionStatusChangePostFuncRegistry[vendorType]; exist { + return fmt.Errorf("the execution status change post function for %s already exists", vendorType) + } + executionStatusChangePostFuncRegistry[vendorType] = fc + return nil +} diff --git a/src/pkg/task/registry_test.go b/src/pkg/task/registry_test.go new file mode 100644 index 000000000..cbf9c4247 --- /dev/null +++ b/src/pkg/task/registry_test.go @@ -0,0 +1,48 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRegisterCheckInProcessor(t *testing.T) { + err := RegisterCheckInProcessor("test", nil) + assert.Nil(t, err) + + // already exist + err = RegisterCheckInProcessor("test", nil) + assert.NotNil(t, err) +} + +func TestRegisterTaskStatusChangePostFunc(t *testing.T) { + err := RegisterTaskStatusChangePostFunc("test", nil) + assert.Nil(t, err) + + // already exist + err = RegisterTaskStatusChangePostFunc("test", nil) + assert.NotNil(t, err) +} + +func TestRegisterExecutionStatusChangePostFunc(t *testing.T) { + err := RegisterExecutionStatusChangePostFunc("test", nil) + assert.Nil(t, err) + + // already exist + err = RegisterExecutionStatusChangePostFunc("test", nil) + assert.NotNil(t, err) +} diff --git a/src/pkg/task/task.go b/src/pkg/task/task.go index eb7723285..f67773e2b 100644 --- a/src/pkg/task/task.go +++ b/src/pkg/task/task.go @@ -47,10 +47,12 @@ type Manager interface { // Get the specified task Get(ctx context.Context, id int64) (task *Task, err error) // List the tasks according to the query + // Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"' List(ctx context.Context, query *q.Query) (tasks []*Task, err error) // Get the log of the specified task GetLog(ctx context.Context, id int64) (log []byte, err error) - // Count counts total. + // Count counts total of tasks according to the query. + // Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"' Count(ctx context.Context, query *q.Query) (int64, error) } @@ -58,6 +60,7 @@ type Manager interface { func NewManager() Manager { return &manager{ dao: dao.NewTaskDAO(), + execDAO: dao.NewExecutionDAO(), jsClient: cjob.GlobalClient, coreURL: config.GetCoreURL(), } @@ -65,6 +68,7 @@ func NewManager() Manager { type manager struct { dao dao.TaskDAO + execDAO dao.ExecutionDAO jsClient cjob.Client coreURL string } @@ -84,22 +88,11 @@ func (m *manager) Create(ctx context.Context, executionID int64, jb *Job, extraA // submit job to jobservice jobID, err := m.submitJob(ctx, id, jb) if err != nil { - // failed to submit job to jobservice, update the status of task to error - err = fmt.Errorf("failed to submit job to jobservice: %v", err) - log.Error(err) - now := time.Now() - err = m.dao.Update(ctx, &dao.Task{ - ID: id, - Status: job.ErrorStatus.String(), - StatusCode: job.ErrorStatus.Code(), - StatusMessage: err.Error(), - UpdateTime: now, - EndTime: now, - }, "Status", "StatusCode", "StatusMessage", "UpdateTime", "EndTime") - if err != nil { - log.Errorf("failed to update task %d: %v", id, err) + // failed to submit job to jobservice, delete the task record + if err := m.dao.Delete(ctx, id); err != nil { + log.Errorf("failed to delete the task %d: %v", id, err) } - return id, nil + return 0, err } log.Debugf("the task %d is submitted to jobservice, the job ID is %s", id, jobID) @@ -116,6 +109,10 @@ func (m *manager) Create(ctx context.Context, executionID int64, jb *Job, extraA } func (m *manager) createTaskRecord(ctx context.Context, executionID int64, extraAttrs ...map[string]interface{}) (int64, error) { + exec, err := m.execDAO.Get(ctx, executionID) + if err != nil { + return 0, err + } extras := map[string]interface{}{} if len(extraAttrs) > 0 && extraAttrs[0] != nil { extras = extraAttrs[0] @@ -127,6 +124,7 @@ func (m *manager) createTaskRecord(ctx context.Context, executionID int64, extra now := time.Now() return m.dao.Create(ctx, &dao.Task{ + VendorType: exec.VendorType, ExecutionID: executionID, Status: job.PendingStatus.String(), StatusCode: job.PendingStatus.Code(), diff --git a/src/pkg/task/task_test.go b/src/pkg/task/task_test.go index 3b289cbe5..f9f1e2b6e 100644 --- a/src/pkg/task/task_test.go +++ b/src/pkg/task/task_test.go @@ -31,14 +31,17 @@ type taskManagerTestSuite struct { suite.Suite mgr *manager dao *mockTaskDAO + execDAO *mockExecutionDAO jsClient *mockJobserviceClient } func (t *taskManagerTestSuite) SetupTest() { t.dao = &mockTaskDAO{} + t.execDAO = &mockExecutionDAO{} t.jsClient = &mockJobserviceClient{} t.mgr = &manager{ dao: t.dao, + execDAO: t.execDAO, jsClient: t.jsClient, } } @@ -53,6 +56,7 @@ func (t *taskManagerTestSuite) TestCount() { func (t *taskManagerTestSuite) TestCreate() { // success to submit job to jobservice + t.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{}, nil) t.dao.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil) t.jsClient.On("SubmitJob", mock.Anything).Return("1", nil) t.dao.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(nil) @@ -61,21 +65,22 @@ func (t *taskManagerTestSuite) TestCreate() { t.Require().Nil(err) t.Equal(int64(1), id) t.dao.AssertExpectations(t.T()) + t.execDAO.AssertExpectations(t.T()) t.jsClient.AssertExpectations(t.T()) // reset mock t.SetupTest() // failed to submit job to jobservice + t.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{}, nil) t.dao.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil) t.jsClient.On("SubmitJob", mock.Anything).Return("", errors.New("error")) - t.dao.On("Update", mock.Anything, mock.Anything, mock.Anything, - mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + t.dao.On("Delete", mock.Anything, mock.Anything).Return(nil) id, err = t.mgr.Create(nil, 1, &Job{}, map[string]interface{}{"a": "b"}) - t.Require().Nil(err) - t.Equal(int64(1), id) + t.Require().NotNil(err) t.dao.AssertExpectations(t.T()) + t.execDAO.AssertExpectations(t.T()) t.jsClient.AssertExpectations(t.T()) } diff --git a/src/replication/config/config.go b/src/replication/config/config.go index 4423a68e9..14f466973 100644 --- a/src/replication/config/config.go +++ b/src/replication/config/config.go @@ -23,9 +23,7 @@ var ( type Configuration struct { CoreURL string TokenServiceURL string - JobserviceURL string SecretKey string // TODO consider to use a specified secret for replication - CoreSecret string JobserviceSecret string } diff --git a/src/replication/dao/base.go b/src/replication/dao/base.go deleted file mode 100644 index 29a567d8b..000000000 --- a/src/replication/dao/base.go +++ /dev/null @@ -1,13 +0,0 @@ -package dao - -import "github.com/astaxie/beego/orm" - -func paginateForQuerySetter(qs orm.QuerySeter, page, size int64) orm.QuerySeter { - if size > 0 { - qs = qs.Limit(size) - if page > 0 { - qs = qs.Offset((page - 1) * size) - } - } - return qs -} diff --git a/src/replication/dao/execution.go b/src/replication/dao/execution.go deleted file mode 100644 index bddc30194..000000000 --- a/src/replication/dao/execution.go +++ /dev/null @@ -1,355 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dao - -import ( - "fmt" - "time" - - "github.com/astaxie/beego/orm" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/replication/dao/models" -) - -// AddExecution ... -func AddExecution(execution *models.Execution) (int64, error) { - o := dao.GetOrmer() - now := time.Now() - execution.StartTime = now - - return o.Insert(execution) -} - -// GetTotalOfExecutions returns the total count of replication execution -func GetTotalOfExecutions(query ...*models.ExecutionQuery) (int64, error) { - qs := executionQueryConditions(query...) - return qs.Count() -} - -// GetExecutions ... -func GetExecutions(query ...*models.ExecutionQuery) ([]*models.Execution, error) { - executions := []*models.Execution{} - - qs := executionQueryConditions(query...) - if len(query) > 0 && query[0] != nil { - qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size) - } - - qs = qs.OrderBy("-StartTime") - - _, err := qs.All(&executions) - if err != nil || len(executions) == 0 { - return executions, err - } - for _, e := range executions { - fillExecution(e) - } - return executions, err -} - -func executionQueryConditions(query ...*models.ExecutionQuery) orm.QuerySeter { - qs := dao.GetOrmer().QueryTable(new(models.Execution)) - if len(query) == 0 || query[0] == nil { - return qs - } - - q := query[0] - if q.PolicyID != 0 { - qs = qs.Filter("PolicyID", q.PolicyID) - } - if len(q.Trigger) > 0 { - qs = qs.Filter("Trigger", q.Trigger) - } - if len(q.Statuses) > 0 { - qs = qs.Filter("Status__in", q.Statuses) - } - return qs -} - -// GetExecution ... -func GetExecution(id int64) (*models.Execution, error) { - o := dao.GetOrmer() - t := models.Execution{ID: id} - err := o.Read(&t) - if err == orm.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, err - } - fillExecution(&t) - return &t, err -} - -// fillExecution will fill the statistics data and status by tasks data -func fillExecution(execution *models.Execution) error { - if executionFinished(execution.Status) { - return nil - } - - o := dao.GetOrmer() - sql := `select status, count(*) as c from replication_task where execution_id = ? group by status` - queryParam := make([]interface{}, 1) - queryParam = append(queryParam, execution.ID) - - dt := []*models.TaskStat{} - count, err := o.Raw(sql, queryParam).QueryRows(&dt) - - if err != nil { - log.Errorf("Query tasks error execution %d: %v", execution.ID, err) - return err - } - - if count == 0 { - return nil - } - - total := 0 - for _, d := range dt { - status, _ := getStatus(d.Status) - updateStatusCount(execution, status, d.C) - total += d.C - } - - if execution.Total != total { - log.Debugf("execution task count inconsistent and fixed, executionID=%d, execution.total=%d, tasks.count=%d", - execution.ID, execution.Total, total) - execution.Total = total - } - resetExecutionStatus(execution) - - return nil -} - -func getStatus(status string) (string, error) { - switch status { - case models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress: - return models.ExecutionStatusInProgress, nil - case models.TaskStatusSucceed: - return models.ExecutionStatusSucceed, nil - case models.TaskStatusStopped: - return models.ExecutionStatusStopped, nil - case models.TaskStatusFailed: - return models.ExecutionStatusFailed, nil - } - return "", fmt.Errorf("Not support task status ") -} - -func updateStatusCount(execution *models.Execution, status string, delta int) error { - switch status { - case models.ExecutionStatusInProgress: - execution.InProgress += delta - case models.ExecutionStatusSucceed: - execution.Succeed += delta - case models.ExecutionStatusStopped: - execution.Stopped += delta - case models.ExecutionStatusFailed: - execution.Failed += delta - } - return nil -} - -func resetExecutionStatus(execution *models.Execution) error { - execution.Status = generateStatus(execution) - if executionFinished(execution.Status) { - o := dao.GetOrmer() - sql := `select max(end_time) from replication_task where execution_id = ?` - queryParam := make([]interface{}, 1) - queryParam = append(queryParam, execution.ID) - - var et time.Time - err := o.Raw(sql, queryParam).QueryRow(&et) - if err != nil { - log.Errorf("Query end_time from tasks error execution %d: %v", execution.ID, err) - et = time.Now() - } - execution.EndTime = et - } - return nil -} - -func generateStatus(execution *models.Execution) string { - if execution.InProgress > 0 { - return models.ExecutionStatusInProgress - } else if execution.Failed > 0 { - return models.ExecutionStatusFailed - } else if execution.Stopped > 0 { - return models.ExecutionStatusStopped - } - return models.ExecutionStatusSucceed -} - -func executionFinished(status string) bool { - if status == models.ExecutionStatusStopped || - status == models.ExecutionStatusSucceed || - status == models.ExecutionStatusFailed { - return true - } - return false -} - -// DeleteExecution ... -func DeleteExecution(id int64) error { - o := dao.GetOrmer() - _, err := o.Delete(&models.Execution{ID: id}) - return err -} - -// DeleteAllExecutions ... -func DeleteAllExecutions(policyID int64) error { - o := dao.GetOrmer() - _, err := o.Delete(&models.Execution{PolicyID: policyID}, "PolicyID") - return err -} - -// UpdateExecution ... -func UpdateExecution(execution *models.Execution, props ...string) (int64, error) { - if execution.ID == 0 { - return 0, fmt.Errorf("execution ID is empty") - } - o := dao.GetOrmer() - return o.Update(execution, props...) -} - -// AddTask ... -func AddTask(task *models.Task) (int64, error) { - o := dao.GetOrmer() - now := time.Now() - task.StartTime = now - - return o.Insert(task) -} - -// GetTask ... -func GetTask(id int64) (*models.Task, error) { - o := dao.GetOrmer() - sql := `select * from replication_task where id = ?` - - var task models.Task - - if err := o.Raw(sql, id).QueryRow(&task); err != nil { - if err == orm.ErrNoRows { - return nil, nil - } - return nil, err - } - - return &task, nil -} - -// GetTotalOfTasks ... -func GetTotalOfTasks(query ...*models.TaskQuery) (int64, error) { - qs := taskQueryConditions(query...) - return qs.Count() -} - -// GetTasks ... -func GetTasks(query ...*models.TaskQuery) ([]*models.Task, error) { - tasks := []*models.Task{} - - qs := taskQueryConditions(query...) - if len(query) > 0 && query[0] != nil { - qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size) - } - - qs = qs.OrderBy("-StartTime") - - _, err := qs.All(&tasks) - return tasks, err -} - -func taskQueryConditions(query ...*models.TaskQuery) orm.QuerySeter { - qs := dao.GetOrmer().QueryTable(new(models.Task)) - if len(query) == 0 || query[0] == nil { - return qs - } - - q := query[0] - if q.ExecutionID != 0 { - qs = qs.Filter("ExecutionID", q.ExecutionID) - } - if len(q.JobID) > 0 { - qs = qs.Filter("JobID", q.JobID) - } - if len(q.ResourceType) > 0 { - qs = qs.Filter("ResourceType", q.ResourceType) - } - if len(q.Statuses) > 0 { - qs = qs.Filter("Status__in", q.Statuses) - } - return qs -} - -// DeleteTask ... -func DeleteTask(id int64) error { - o := dao.GetOrmer() - _, err := o.Delete(&models.Task{ID: id}) - return err -} - -// DeleteAllTasks ... -func DeleteAllTasks(executionID int64) error { - o := dao.GetOrmer() - _, err := o.Delete(&models.Task{ExecutionID: executionID}, "ExecutionID") - return err -} - -// UpdateTask ... -func UpdateTask(task *models.Task, props ...string) (int64, error) { - if task.ID == 0 { - return 0, fmt.Errorf("task ID is empty") - } - o := dao.GetOrmer() - return o.Update(task, props...) -} - -// UpdateTaskStatus updates the status of task. -// The implementation uses raw sql rather than QuerySetter.Filter... as QuerySetter -// will generate sql like: -// `UPDATE "replication_task" SET "end_time" = $1, "status" = $2 -// WHERE "id" IN ( SELECT T0."id" FROM "replication_task" T0 WHERE T0."id" = $3 -// AND T0."status" IN ($4, $5, $6))]` -// which is not a "single" sql statement, this will cause issues when running in concurrency -func UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) (int64, error) { - params := []interface{}{} - sql := `update replication_task set status = ?, status_revision = ?, end_time = ? ` - params = append(params, status, statusRevision) - var t time.Time - // when the task is in final status, update the endtime - // when the task re-runs again, the endtime should be cleared - // so set the endtime to null if the task isn't in final status - if taskFinished(status) { - t = time.Now() - } - params = append(params, t) - - sql += fmt.Sprintf(`where id = ? and (status_revision < ? or status_revision = ? and status in (%s)) `, dao.ParamPlaceholderForIn(len(statusCondition))) - params = append(params, id, statusRevision, statusRevision, statusCondition) - - result, err := dao.GetOrmer().Raw(sql, params...).Exec() - if err != nil { - return 0, err - } - n, _ := result.RowsAffected() - if n > 0 { - log.Debugf("update task status %d: -> %s", id, status) - } - return n, err -} - -func taskFinished(status string) bool { - return status == models.TaskStatusFailed || status == models.TaskStatusStopped || status == models.TaskStatusSucceed -} diff --git a/src/replication/dao/execution_test.go b/src/replication/dao/execution_test.go deleted file mode 100644 index b112c4d54..000000000 --- a/src/replication/dao/execution_test.go +++ /dev/null @@ -1,294 +0,0 @@ -package dao - -import ( - "testing" - "time" - - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMethodOfExecution(t *testing.T) { - execution1 := &models.Execution{ - PolicyID: 11209, - Status: "InProgress", - StatusText: "None", - Total: 12, - Failed: 0, - Succeed: 7, - InProgress: 5, - Stopped: 0, - Trigger: "Event", - StartTime: time.Now(), - } - execution2 := &models.Execution{ - PolicyID: 11209, - Status: "Failed", - StatusText: "Network error", - Total: 9, - Failed: 1, - Succeed: 8, - InProgress: 0, - Stopped: 0, - Trigger: "Manual", - StartTime: time.Now(), - } - - // test add - id1, err := AddExecution(execution1) - require.Nil(t, err) - - _, err = AddExecution(execution2) - require.Nil(t, err) - - // test list - query := &models.ExecutionQuery{ - Statuses: []string{"InProgress", "Failed"}, - Pagination: models.Pagination{ - Page: 1, - Size: 10, - }, - } - executions, err := GetExecutions(query) - require.Nil(t, err) - assert.Equal(t, 2, len(executions)) - - total, err := GetTotalOfExecutions(query) - require.Nil(t, err) - assert.Equal(t, int64(2), total) - - // test get - execution, err := GetExecution(id1) - require.Nil(t, err) - assert.Equal(t, execution1.Status, execution.Status) - - // test update - executionNew := &models.Execution{ - ID: id1, - Status: "Succeed", - Succeed: 12, - InProgress: 0, - EndTime: time.Now(), - } - n, err := UpdateExecution(executionNew, models.ExecutionPropsName.Status, models.ExecutionPropsName.Succeed, models.ExecutionPropsName.InProgress, - models.ExecutionPropsName.EndTime) - require.Nil(t, err) - assert.Equal(t, int64(1), n) - - // test delete - require.Nil(t, DeleteExecution(execution1.ID)) - execution, err = GetExecution(execution1.ID) - require.Nil(t, err) - require.Nil(t, execution) - - // test delete all - require.Nil(t, DeleteAllExecutions(execution1.PolicyID)) - query = &models.ExecutionQuery{} - n, err = GetTotalOfExecutions(query) - require.Nil(t, err) - assert.Equal(t, int64(0), n) -} - -func TestMethodOfTask(t *testing.T) { - now := time.Now() - task1 := &models.Task{ - ExecutionID: 112200, - ResourceType: "resourceType1", - SrcResource: "srcResource1", - DstResource: "dstResource1", - JobID: "jobID1", - Status: "Initialized", - StatusRevision: 1, - StartTime: now, - } - task2 := &models.Task{ - ExecutionID: 112200, - ResourceType: "resourceType2", - SrcResource: "srcResource2", - DstResource: "dstResource2", - JobID: "jobID2", - Status: "Stopped", - StatusRevision: 1, - StartTime: now, - EndTime: now, - } - - // test add - id1, err := AddTask(task1) - require.Nil(t, err) - - _, err = AddTask(task2) - require.Nil(t, err) - - // test list - query := &models.TaskQuery{ - ResourceType: "resourceType1", - Pagination: models.Pagination{ - Page: 1, - Size: 10, - }, - } - tasks, err := GetTasks(query) - require.Nil(t, err) - assert.Equal(t, 1, len(tasks)) - - total, err := GetTotalOfTasks(query) - require.Nil(t, err) - assert.Equal(t, int64(1), total) - - // test get - task, err := GetTask(id1) - require.Nil(t, err) - assert.Equal(t, task1.Status, task.Status) - - // test update - taskNew := &models.Task{ - ID: id1, - Status: "Failed", - EndTime: now, - } - n, err := UpdateTask(taskNew, models.TaskPropsName.Status, models.TaskPropsName.EndTime) - require.Nil(t, err) - assert.Equal(t, int64(1), n) - - // test update status - n, err = UpdateTaskStatus(id1, "Succeed", 2, "Initialized") - require.Nil(t, err) - assert.Equal(t, int64(1), n) - task, _ = GetTask(id1) - assert.Equal(t, "Succeed", task.Status) - assert.Equal(t, int64(2), task.StatusRevision) - - // test delete - require.Nil(t, DeleteTask(id1)) - task, err = GetTask(id1) - require.Nil(t, err) - require.Nil(t, task) - - // test delete all - require.Nil(t, DeleteAllTasks(task1.ExecutionID)) - query = &models.TaskQuery{} - n, err = GetTotalOfTasks(query) - require.Nil(t, err) - assert.Equal(t, int64(0), n) -} - -func TestExecutionFill(t *testing.T) { - now := time.Now() - execution := &models.Execution{ - PolicyID: 11209, - Status: "InProgress", - StatusText: "None", - Total: 2, - Trigger: "Event", - StartTime: time.Now(), - } - executionID, _ := AddExecution(execution) - et1, _ := time.Parse("2006-01-02 15:04:05", "2019-03-21 08:01:01") - et2, _ := time.Parse("2006-01-02 15:04:05", "2019-04-01 10:11:53") - task1 := &models.Task{ - ID: 20191, - ExecutionID: executionID, - ResourceType: "resourceType1", - SrcResource: "srcResource1", - DstResource: "dstResource1", - JobID: "jobID1", - Status: "Succeed", - StartTime: now, - EndTime: et1, - } - task2 := &models.Task{ - ID: 20192, - ExecutionID: executionID, - ResourceType: "resourceType2", - SrcResource: "srcResource2", - DstResource: "dstResource2", - JobID: "jobID2", - Status: "Stopped", - StartTime: now, - EndTime: et2, - } - AddTask(task1) - AddTask(task2) - - defer func() { - DeleteAllTasks(executionID) - DeleteAllExecutions(11209) - }() - - // query and fill - exe, err := GetExecution(executionID) - require.Nil(t, err) - assert.Equal(t, "Stopped", exe.Status) - assert.Equal(t, 0, exe.InProgress) - assert.Equal(t, 1, exe.Stopped) - assert.Equal(t, 0, exe.Failed) - assert.Equal(t, 1, exe.Succeed) - assert.Equal(t, et2.Second(), exe.EndTime.Second()) -} - -func TestExecutionFill2(t *testing.T) { - now := time.Now() - execution := &models.Execution{ - PolicyID: 11209, - Status: "InProgress", - StatusText: "None", - Total: 2, - Trigger: "Event", - StartTime: time.Now(), - } - executionID, _ := AddExecution(execution) - task1 := &models.Task{ - ID: 20191, - ExecutionID: executionID, - ResourceType: "resourceType1", - SrcResource: "srcResource1", - DstResource: "dstResource1", - JobID: "jobID1", - Status: models.TaskStatusInProgress, - StatusRevision: 1, - StartTime: now, - } - task2 := &models.Task{ - ID: 20192, - ExecutionID: executionID, - ResourceType: "resourceType2", - SrcResource: "srcResource2", - DstResource: "dstResource2", - JobID: "jobID2", - Status: "Stopped", - StatusRevision: 1, - StartTime: now, - EndTime: now, - } - taskID1, _ := AddTask(task1) - AddTask(task2) - - defer func() { - DeleteAllTasks(executionID) - DeleteAllExecutions(11209) - }() - - // query and fill - exe, err := GetExecution(executionID) - require.Nil(t, err) - assert.Equal(t, models.ExecutionStatusInProgress, exe.Status) - assert.Equal(t, 1, exe.InProgress) - assert.Equal(t, 1, exe.Stopped) - assert.Equal(t, 0, exe.Failed) - assert.Equal(t, 0, exe.Succeed) - - // update task status and query and fill - UpdateTaskStatus(taskID1, models.TaskStatusFailed, 2, models.TaskStatusInProgress) - exes, err := GetExecutions(&models.ExecutionQuery{ - PolicyID: 11209, - }) - require.Nil(t, err) - assert.Equal(t, 1, len(exes)) - assert.Equal(t, models.ExecutionStatusFailed, exes[0].Status) - assert.Equal(t, 0, exes[0].InProgress) - assert.Equal(t, 1, exes[0].Stopped) - assert.Equal(t, 1, exes[0].Failed) - assert.Equal(t, 0, exes[0].Succeed) -} diff --git a/src/replication/dao/models/base.go b/src/replication/dao/models/base.go index ce97d84c8..6d34967a4 100644 --- a/src/replication/dao/models/base.go +++ b/src/replication/dao/models/base.go @@ -7,14 +7,5 @@ import ( func init() { orm.RegisterModel( new(Registry), - new(RepPolicy), - new(Execution), - new(Task), - new(ScheduleJob)) -} - -// Pagination ... -type Pagination struct { - Page int64 - Size int64 + new(RepPolicy)) } diff --git a/src/replication/dao/models/execution.go b/src/replication/dao/models/execution.go deleted file mode 100644 index d0dbce87f..000000000 --- a/src/replication/dao/models/execution.go +++ /dev/null @@ -1,156 +0,0 @@ -package models - -import ( - "time" - - "github.com/goharbor/harbor/src/replication/model" -) - -const ( - // ExecutionTable is the table name for replication executions - ExecutionTable = "replication_execution" - // TaskTable is table name for replication tasks - TaskTable = "replication_task" -) - -// execution/task status/trigger const -const ( - ExecutionStatusFailed string = "Failed" - ExecutionStatusSucceed string = "Succeed" - ExecutionStatusStopped string = "Stopped" - ExecutionStatusInProgress string = "InProgress" - - ExecutionTriggerManual string = "Manual" - ExecutionTriggerEvent string = "Event" - ExecutionTriggerSchedule string = "Schedule" - - // The task has been persisted in db but not submitted to Jobservice - TaskStatusInitialized string = "Initialized" - TaskStatusPending string = "Pending" - TaskStatusInProgress string = "InProgress" - TaskStatusSucceed string = "Succeed" - TaskStatusFailed string = "Failed" - TaskStatusStopped string = "Stopped" -) - -// ExecutionPropsName defines the names of fields of Execution -var ExecutionPropsName = ExecutionFieldsName{ - ID: "ID", - PolicyID: "PolicyID", - Status: "Status", - StatusText: "StatusText", - Total: "Total", - Failed: "Failed", - Succeed: "Succeed", - InProgress: "InProgress", - Stopped: "Stopped", - Trigger: "Trigger", - StartTime: "StartTime", - EndTime: "EndTime", -} - -// ExecutionFieldsName defines the props of Execution -type ExecutionFieldsName struct { - ID string - PolicyID string - Status string - StatusText string - Total string - Failed string - Succeed string - InProgress string - Stopped string - Trigger string - StartTime string - EndTime string -} - -// Execution holds information about once replication execution. -type Execution struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` - Status string `orm:"column(status)" json:"status"` - StatusText string `orm:"column(status_text)" json:"status_text"` - Total int `orm:"column(total)" json:"total"` - Failed int `orm:"column(failed)" json:"failed"` - Succeed int `orm:"column(succeed)" json:"succeed"` - InProgress int `orm:"column(in_progress)" json:"in_progress"` - Stopped int `orm:"column(stopped)" json:"stopped"` - Trigger model.TriggerType `orm:"column(trigger)" json:"trigger"` - StartTime time.Time `orm:"column(start_time)" json:"start_time"` - EndTime time.Time `orm:"column(end_time)" json:"end_time"` -} - -// TaskPropsName defines the names of fields of Task -var TaskPropsName = TaskFieldsName{ - ID: "ID", - ExecutionID: "ExecutionID", - ResourceType: "ResourceType", - SrcResource: "SrcResource", - DstResource: "DstResource", - JobID: "JobID", - Status: "Status", - StartTime: "StartTime", - EndTime: "EndTime", -} - -// TaskFieldsName defines the props of Task -type TaskFieldsName struct { - ID string - ExecutionID string - ResourceType string - SrcResource string - DstResource string - JobID string - Status string - StartTime string - EndTime string -} - -// Task represent the tasks in one execution. -type Task struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - ExecutionID int64 `orm:"column(execution_id)" json:"execution_id"` - ResourceType string `orm:"column(resource_type)" json:"resource_type"` - SrcResource string `orm:"column(src_resource)" json:"src_resource"` - DstResource string `orm:"column(dst_resource)" json:"dst_resource"` - Operation string `orm:"column(operation)" json:"operation"` - JobID string `orm:"column(job_id)" json:"job_id"` - Status string `orm:"column(status)" json:"status"` - StatusRevision int64 `orm:"column(status_revision)"` - StartTime time.Time `orm:"column(start_time)" json:"start_time"` - EndTime time.Time `orm:"column(end_time)" json:"end_time,omitempty"` -} - -// TableName is required by by beego orm to map Execution to table replication_execution -func (r *Execution) TableName() string { - return ExecutionTable -} - -// TableName is required by by beego orm to map Task to table replication_task -func (r *Task) TableName() string { - return TaskTable -} - -// ExecutionQuery holds the query conditions for replication executions -type ExecutionQuery struct { - PolicyID int64 - Statuses []string - Trigger string - Pagination -} - -// TaskQuery holds the query conditions for replication task -type TaskQuery struct { - ExecutionID int64 - JobID string - Statuses []string - ResourceType string - Pagination -} - -// TaskStat holds statistics of task by status -type TaskStat struct { - Status string `orm:"column(status)"` - C int `orm:"column(c)"` -} diff --git a/src/replication/dao/models/schedule_job.go b/src/replication/dao/models/schedule_job.go deleted file mode 100644 index 7375e489a..000000000 --- a/src/replication/dao/models/schedule_job.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO rename the package name to model - -package models - -import "time" - -// ScheduleJob is the persistent model for the schedule job which is -// used as a scheduler -type ScheduleJob struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` - JobID string `orm:"column(job_id)" json:"job_id"` - Status string `orm:"column(status)" json:"status"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` -} - -// TableName is required by by beego orm to map the object to the database table -func (s *ScheduleJob) TableName() string { - return "replication_schedule_job" -} - -// ScheduleJobQuery is the query used to list schedule jobs -type ScheduleJobQuery struct { - PolicyID int64 -} diff --git a/src/replication/dao/schedule_job.go b/src/replication/dao/schedule_job.go deleted file mode 100644 index a0f4eda73..000000000 --- a/src/replication/dao/schedule_job.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dao - -import ( - "time" - - "github.com/astaxie/beego/orm" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/replication/dao/models" -) - -// ScheduleJob is the DAO for schedule job -var ScheduleJob ScheduleJobDAO = &scheduleJobDAO{} - -// ScheduleJobDAO ... -type ScheduleJobDAO interface { - Add(*models.ScheduleJob) (int64, error) - Get(int64) (*models.ScheduleJob, error) - Update(*models.ScheduleJob, ...string) error - Delete(int64) error - List(...*models.ScheduleJobQuery) ([]*models.ScheduleJob, error) -} - -type scheduleJobDAO struct{} - -func (s *scheduleJobDAO) Add(sj *models.ScheduleJob) (int64, error) { - now := time.Now() - sj.CreationTime = now - sj.UpdateTime = now - return dao.GetOrmer().Insert(sj) -} - -func (s *scheduleJobDAO) Get(id int64) (*models.ScheduleJob, error) { - sj := &models.ScheduleJob{ - ID: id, - } - if err := dao.GetOrmer().Read(sj); err != nil { - if err == orm.ErrNoRows { - return nil, nil - } - return nil, err - } - return sj, nil -} - -func (s *scheduleJobDAO) Update(sj *models.ScheduleJob, props ...string) error { - if sj.UpdateTime.IsZero() { - now := time.Now() - sj.UpdateTime = now - if len(props) > 0 { - props = append(props, "UpdateTime") - } - } - - _, err := dao.GetOrmer().Update(sj, props...) - return err -} - -func (s *scheduleJobDAO) Delete(id int64) error { - _, err := dao.GetOrmer().Delete(&models.ScheduleJob{ - ID: id, - }) - return err -} - -func (s *scheduleJobDAO) List(query ...*models.ScheduleJobQuery) ([]*models.ScheduleJob, error) { - qs := dao.GetOrmer().QueryTable(&models.ScheduleJob{}) - if len(query) > 0 && query[0] != nil { - if query[0].PolicyID > 0 { - qs = qs.Filter("PolicyID", query[0].PolicyID) - } - } - sjs := []*models.ScheduleJob{} - _, err := qs.All(&sjs) - if err != nil { - return nil, err - } - return sjs, nil -} diff --git a/src/replication/dao/schedule_job_test.go b/src/replication/dao/schedule_job_test.go deleted file mode 100644 index 6623ff8d6..000000000 --- a/src/replication/dao/schedule_job_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dao - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var sjID int64 - -func TestAddScheduleJob(t *testing.T) { - sj := &models.ScheduleJob{ - PolicyID: 1, - JobID: "uuid", - Status: "running", - } - id, err := ScheduleJob.Add(sj) - require.Nil(t, err) - sjID = id -} - -func TestUpdateScheduleJob(t *testing.T) { - err := ScheduleJob.Update(&models.ScheduleJob{ - ID: sjID, - Status: "success", - }, "Status") - require.Nil(t, err) -} - -func TestGetScheduleJob(t *testing.T) { - sj, err := ScheduleJob.Get(sjID) - require.Nil(t, err) - assert.Equal(t, int64(1), sj.PolicyID) - assert.Equal(t, "success", sj.Status) -} - -func TestListScheduleJobs(t *testing.T) { - // nil query - sjs, err := ScheduleJob.List() - require.Nil(t, err) - assert.Equal(t, 1, len(sjs)) - - // query - sjs, err = ScheduleJob.List(&models.ScheduleJobQuery{ - PolicyID: 1, - }) - require.Nil(t, err) - assert.Equal(t, 1, len(sjs)) - - // query - sjs, err = ScheduleJob.List(&models.ScheduleJobQuery{ - PolicyID: 2, - }) - require.Nil(t, err) - assert.Equal(t, 0, len(sjs)) -} - -func TestDeleteScheduleJob(t *testing.T) { - err := ScheduleJob.Delete(sjID) - require.Nil(t, err) - - sj, err := ScheduleJob.Get(sjID) - require.Nil(t, err) - assert.Nil(t, sj) -} diff --git a/src/replication/event/handler.go b/src/replication/event/handler.go index f326aa766..f6cc7ae12 100644 --- a/src/replication/event/handler.go +++ b/src/replication/event/handler.go @@ -17,12 +17,14 @@ package event import ( "errors" "fmt" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/pkg/task" commonthttp "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/controller/replication" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/replication/config" "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/operation" "github.com/goharbor/harbor/src/replication/policy" "github.com/goharbor/harbor/src/replication/registry" "github.com/goharbor/harbor/src/replication/util" @@ -34,18 +36,18 @@ type Handler interface { } // NewHandler ... -func NewHandler(policyCtl policy.Controller, registryMgr registry.Manager, opCtl operation.Controller) Handler { +func NewHandler(policyCtl policy.Controller, registryMgr registry.Manager) Handler { return &handler{ policyCtl: policyCtl, registryMgr: registryMgr, - opCtl: opCtl, + ctl: replication.Ctl, } } type handler struct { policyCtl policy.Controller registryMgr registry.Manager - opCtl operation.Controller + ctl replication.Controller } func (h *handler) Handle(event *Event) error { @@ -76,7 +78,7 @@ func (h *handler) Handle(event *Event) error { if err := PopulateRegistries(h.registryMgr, policy); err != nil { return err } - id, err := h.opCtl.StartReplication(policy, event.Resource, model.TriggerTypeEventBased) + id, err := h.ctl.Start(orm.Context(), policy, event.Resource, task.ExecutionTriggerEvent) if err != nil { return err } diff --git a/src/replication/event/handler_test.go b/src/replication/event/handler_test.go index 09a64ad07..dfe740bd8 100644 --- a/src/replication/event/handler_test.go +++ b/src/replication/event/handler_test.go @@ -15,43 +15,18 @@ package event import ( - "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/testing/controller/replication" + "github.com/goharbor/harbor/src/testing/mock" "testing" + "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/replication/config" - "github.com/goharbor/harbor/src/replication/dao/models" "github.com/goharbor/harbor/src/replication/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type fakedOperationController struct{} - -func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) { - return 1, nil -} -func (f *fakedOperationController) StopReplication(int64) error { - return nil -} -func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) { - return 0, nil, nil -} -func (f *fakedOperationController) GetExecution(id int64) (*models.Execution, error) { - return nil, nil -} -func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) { - return 0, nil, nil -} -func (f *fakedOperationController) GetTask(id int64) (*models.Task, error) { - return nil, nil -} -func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error { - return nil -} -func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) { - return nil, nil -} - type fakedPolicyController struct{} func (f *fakedPolicyController) Create(*model.Policy) (int64, error) { @@ -236,10 +211,16 @@ func TestGetRelatedPolicies(t *testing.T) { } func TestHandle(t *testing.T) { + dao.PrepareTestForPostgresSQL() config.Config = &config.Configuration{} - handler := NewHandler(&fakedPolicyController{}, - &fakedRegistryManager{}, - &fakedOperationController{}) + ctl := &replication.Controller{} + ctl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + handler := &handler{ + policyCtl: &fakedPolicyController{}, + registryMgr: &fakedRegistryManager{}, + ctl: ctl, + } + // nil event err := handler.Handle(nil) require.NotNil(t, err) diff --git a/src/replication/operation/controller.go b/src/replication/operation/controller.go deleted file mode 100644 index 06a29226d..000000000 --- a/src/replication/operation/controller.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package operation - -import ( - "fmt" - "time" - - "github.com/goharbor/harbor/src/common/job" - hjob "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/operation/execution" - "github.com/goharbor/harbor/src/replication/operation/flow" - "github.com/goharbor/harbor/src/replication/operation/scheduler" -) - -// Controller handles the replication-related operations: start, -// stop, query, etc. -type Controller interface { - // trigger is used to specify what this replication is triggered by - StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) - StopReplication(int64) error - ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) - GetExecution(int64) (*models.Execution, error) - ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) - GetTask(int64) (*models.Task, error) - UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error - GetTaskLog(int64) ([]byte, error) -} - -const ( - maxReplicators = 1024 -) - -// NewController returns a controller implementation -func NewController(js job.Client) Controller { - ctl := &controller{ - replicators: make(chan struct{}, maxReplicators), - executionMgr: execution.NewDefaultManager(), - scheduler: scheduler.NewScheduler(js), - flowCtl: flow.NewController(), - } - for i := 0; i < maxReplicators; i++ { - ctl.replicators <- struct{}{} - } - return ctl -} - -type controller struct { - replicators chan struct{} - flowCtl flow.Controller - executionMgr execution.Manager - scheduler scheduler.Scheduler -} - -func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) { - if !policy.Enabled { - return 0, fmt.Errorf("the policy %d is disabled", policy.ID) - } - if len(trigger) == 0 { - trigger = model.TriggerTypeManual - } - id, err := createExecution(c.executionMgr, policy.ID, trigger) - if err != nil { - return 0, err - } - // control the count of concurrent replication requests - log.Debugf("waiting for the available replicator ...") - <-c.replicators - log.Debugf("got an available replicator, starting the replication ...") - go func() { - defer func() { - c.replicators <- struct{}{} - }() - flow := c.createFlow(id, policy, resource) - if n, err := c.flowCtl.Start(flow); err != nil { - // only update the execution when got error. - // if got no error, it will be updated automatically - // when listing the execution records - if e := c.executionMgr.Update(&models.Execution{ - ID: id, - Status: models.ExecutionStatusFailed, - StatusText: err.Error(), - Total: n, - Failed: n, - }, "Status", "StatusText", "Total", "Failed"); e != nil { - log.Errorf("failed to update the execution %d: %v", id, e) - } - log.Errorf("the execution %d failed: %v", id, err) - } - }() - return id, nil -} - -// create different replication flows according to the input parameters -func (c *controller) createFlow(executionID int64, policy *model.Policy, resource *model.Resource) flow.Flow { - // replicate the deletion operation, so create a deletion flow - if resource != nil && resource.Deleted { - return flow.NewDeletionFlow(c.executionMgr, c.scheduler, executionID, policy, resource) - } - resources := []*model.Resource{} - if resource != nil { - resources = append(resources, resource) - } - return flow.NewCopyFlow(c.executionMgr, c.scheduler, executionID, policy, resources...) -} - -func (c *controller) StopReplication(executionID int64) error { - _, tasks, err := c.ListTasks(&models.TaskQuery{ - ExecutionID: executionID, - }) - if err != nil { - return err - } - // no tasks, just set its status to "stopped" - if len(tasks) == 0 { - execution, err := c.executionMgr.Get(executionID) - if err != nil { - return err - } - if execution == nil { - return fmt.Errorf("the execution %d not found", executionID) - } - if execution.Status != models.ExecutionStatusInProgress { - log.Debugf("the execution %d isn't in progress, no need to stop", executionID) - return nil - } - if err = c.executionMgr.Update(&models.Execution{ - ID: executionID, - Status: models.ExecutionStatusStopped, - EndTime: time.Now(), - }, models.ExecutionPropsName.Status, models.ExecutionPropsName.EndTime); err != nil { - return err - } - log.Debugf("the status of execution %d is set to stopped", executionID) - } - // got tasks, stopping the tasks one by one - for _, task := range tasks { - if isTaskInFinalStatus(task) { - log.Debugf("the task %d(job ID: %s) is in final status, its status is %s, skip", task.ID, task.JobID, task.Status) - continue - } - if err = c.scheduler.Stop(task.JobID); err != nil { - isStatusBehindError, ok := err.(*job.StatusBehindError) - if ok { - status := isStatusBehindError.Status() - switch hjob.Status(status) { - case hjob.ErrorStatus: - status = models.TaskStatusFailed - case hjob.SuccessStatus: - status = models.TaskStatusSucceed - } - e := c.executionMgr.UpdateTask(&models.Task{ - ID: task.ID, - Status: status, - }, "Status") - if e != nil { - log.Errorf("failed to update the status the task %d(job ID: %s): %v", task.ID, task.JobID, e) - } else { - log.Debugf("got status behind error for task %d, update it's status to %s directly", task.ID, status) - } - continue - } - if err == job.ErrJobNotFound { - e := c.executionMgr.UpdateTask(&models.Task{ - ID: task.ID, - Status: models.ExecutionStatusStopped, - }, "Status") - if e != nil { - log.Errorf("failed to update the status the task %d(job ID: %s): %v", task.ID, task.JobID, e) - } else { - log.Debugf("got job not found error for task %d, update it's status to %s directly", task.ID, models.ExecutionStatusStopped) - } - continue - } - log.Errorf("failed to stop the task %d(job ID: %s): %v", task.ID, task.JobID, err) - continue - } - log.Debugf("the stop request for task %d(job ID: %s) sent", task.ID, task.JobID) - } - return nil -} - -func isTaskInFinalStatus(task *models.Task) bool { - if task == nil { - return false - } - switch task.Status { - case models.TaskStatusSucceed, - models.TaskStatusStopped, - models.TaskStatusFailed: - return true - } - return false -} - -func (c *controller) ListExecutions(query ...*models.ExecutionQuery) (int64, []*models.Execution, error) { - return c.executionMgr.List(query...) -} -func (c *controller) GetExecution(executionID int64) (*models.Execution, error) { - return c.executionMgr.Get(executionID) -} -func (c *controller) ListTasks(query ...*models.TaskQuery) (int64, []*models.Task, error) { - return c.executionMgr.ListTasks(query...) -} -func (c *controller) GetTask(id int64) (*models.Task, error) { - return c.executionMgr.GetTask(id) -} -func (c *controller) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error { - return c.executionMgr.UpdateTaskStatus(id, status, statusRevision, statusCondition...) -} -func (c *controller) GetTaskLog(taskID int64) ([]byte, error) { - return c.executionMgr.GetTaskLog(taskID) -} - -// create the execution record in database -func createExecution(mgr execution.Manager, policyID int64, trigger model.TriggerType) (int64, error) { - id, err := mgr.Create(&models.Execution{ - PolicyID: policyID, - Trigger: trigger, - Status: models.ExecutionStatusInProgress, - StartTime: time.Now(), - }) - if err != nil { - return 0, fmt.Errorf("failed to create the execution record for replication based on policy %d: %v", policyID, err) - } - log.Debugf("an execution record for replication based on the policy %d created: %d", policyID, id) - return id, nil -} diff --git a/src/replication/operation/controller_test.go b/src/replication/operation/controller_test.go deleted file mode 100644 index 7c39bbfef..000000000 --- a/src/replication/operation/controller_test.go +++ /dev/null @@ -1,390 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package operation - -import ( - "io" - "os" - "testing" - - "github.com/docker/distribution" - "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/config" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/operation/flow" - "github.com/goharbor/harbor/src/replication/operation/scheduler" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type fakedExecutionManager struct{} - -func (f *fakedExecutionManager) Create(*models.Execution) (int64, error) { - return 1, nil -} -func (f *fakedExecutionManager) List(...*models.ExecutionQuery) (int64, []*models.Execution, error) { - return 1, []*models.Execution{ - { - ID: 1, - }, - }, nil -} -func (f *fakedExecutionManager) Get(int64) (*models.Execution, error) { - return &models.Execution{ - ID: 1, - }, nil -} -func (f *fakedExecutionManager) Update(*models.Execution, ...string) error { - return nil -} -func (f *fakedExecutionManager) Remove(int64) error { - return nil -} -func (f *fakedExecutionManager) RemoveAll(int64) error { - return nil -} -func (f *fakedExecutionManager) CreateTask(*models.Task) (int64, error) { - return 1, nil -} -func (f *fakedExecutionManager) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) { - return 1, []*models.Task{ - { - ID: 1, - }, - }, nil -} -func (f *fakedExecutionManager) GetTask(int64) (*models.Task, error) { - return &models.Task{ - ID: 1, - }, nil -} -func (f *fakedExecutionManager) UpdateTask(*models.Task, ...string) error { - return nil -} -func (f *fakedExecutionManager) UpdateTaskStatus(int64, string, int64, ...string) error { - return nil -} -func (f *fakedExecutionManager) RemoveTask(int64) error { - return nil -} -func (f *fakedExecutionManager) RemoveAllTasks(int64) error { - return nil -} -func (f *fakedExecutionManager) GetTaskLog(int64) ([]byte, error) { - return []byte("message"), nil -} - -type fakedScheduler struct{} - -func (f *fakedScheduler) Preprocess(src []*model.Resource, dst []*model.Resource) ([]*scheduler.ScheduleItem, error) { - items := make([]*scheduler.ScheduleItem, 0) - for i, res := range src { - items = append(items, &scheduler.ScheduleItem{ - SrcResource: res, - DstResource: dst[i], - }) - } - return items, nil -} -func (f *fakedScheduler) Schedule(items []*scheduler.ScheduleItem) ([]*scheduler.ScheduleResult, error) { - results := make([]*scheduler.ScheduleResult, 0) - for _, item := range items { - results = append(results, &scheduler.ScheduleResult{ - TaskID: item.TaskID, - Error: nil, - }) - } - return results, nil -} -func (f *fakedScheduler) Stop(id string) error { - return nil -} - -type fakedFactory struct { -} - -func (fakedFactory) Create(*model.Registry) (adapter.Adapter, error) { - return &fakedAdapter{}, nil -} - -func (fakedFactory) AdapterPattern() *model.AdapterPattern { - return nil -} - -type fakedAdapter struct{} - -func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { - return &model.RegistryInfo{ - Type: model.RegistryTypeHarbor, - SupportedResourceTypes: []model.ResourceType{ - model.ResourceTypeImage, - model.ResourceTypeChart, - }, - SupportedTriggers: []model.TriggerType{model.TriggerTypeManual}, - }, nil -} - -func (f *fakedAdapter) PrepareForPush([]*model.Resource) error { - return nil -} -func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { - return model.Healthy, nil -} -func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) { - return []*model.Resource{ - { - Type: model.ResourceTypeImage, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"latest"}, - }, - Override: false, - }, - }, nil -} - -func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) { - return false, "", nil -} -func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) { - return nil, "", nil -} -func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) error { - return nil -} -func (f *fakedAdapter) DeleteManifest(repository, digest string) error { - return nil -} -func (f *fakedAdapter) BlobExist(repository, digest string) (exist bool, err error) { - return false, nil -} -func (f *fakedAdapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) { - return 0, nil, nil -} -func (f *fakedAdapter) PushBlob(repository, digest string, size int64, blob io.Reader) error { - return nil -} -func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { - return []*model.Resource{ - { - Type: model.ResourceTypeChart, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/harbor", - }, - Vtags: []string{"0.2.0"}, - }, - }, - }, nil -} -func (f *fakedAdapter) ChartExist(name, version string) (bool, error) { - return false, nil -} -func (f *fakedAdapter) DownloadChart(name, version string) (io.ReadCloser, error) { - return nil, nil -} -func (f *fakedAdapter) UploadChart(name, version string, chart io.Reader) error { - return nil -} -func (f *fakedAdapter) DeleteChart(name, version string) error { - return nil -} - -var ctl *controller - -func TestMain(m *testing.M) { - ctl = &controller{ - replicators: make(chan struct{}, 1), - executionMgr: &fakedExecutionManager{}, - scheduler: &fakedScheduler{}, - flowCtl: flow.NewController(), - } - ctl.replicators <- struct{}{} - os.Exit(m.Run()) -} - -func TestStartReplication(t *testing.T) { - err := adapter.RegisterFactory(model.RegistryTypeHarbor, new(fakedFactory)) - require.Nil(t, err) - config.Config = &config.Configuration{} - - // policy is disabled - policy := &model.Policy{ - SrcRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - DestRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - } - resource := &model.Resource{ - Type: model.ResourceTypeImage, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"1.0", "2.0"}, - }, - } - _, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased) - require.NotNil(t, err) - - // replicate resource deletion - policy = &model.Policy{ - SrcRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - DestRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - Enabled: true, - } - resource = &model.Resource{ - Type: model.ResourceTypeImage, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"1.0", "2.0"}, - }, - Deleted: true, - } - id, err := ctl.StartReplication(policy, resource, model.TriggerTypeEventBased) - require.Nil(t, err) - assert.Equal(t, int64(1), id) - - // replicate resource copy - policy = &model.Policy{ - SrcRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - DestRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - Enabled: true, - } - resource = &model.Resource{ - Type: model.ResourceTypeImage, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"1.0", "2.0"}, - }, - Deleted: false, - } - id, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased) - require.Nil(t, err) - assert.Equal(t, int64(1), id) - - // nil resource - policy = &model.Policy{ - SrcRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - DestRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - Enabled: true, - } - id, err = ctl.StartReplication(policy, nil, model.TriggerTypeEventBased) - require.Nil(t, err) - assert.Equal(t, int64(1), id) -} - -func TestStopReplication(t *testing.T) { - err := ctl.StopReplication(1) - require.Nil(t, err) -} - -func TestListExecutions(t *testing.T) { - n, executions, err := ctl.ListExecutions() - require.Nil(t, err) - assert.Equal(t, int64(1), n) - assert.Equal(t, int64(1), executions[0].ID) -} - -func TestGetExecution(t *testing.T) { - execution, err := ctl.GetExecution(1) - require.Nil(t, err) - assert.Equal(t, int64(1), execution.ID) -} - -func TestListTasks(t *testing.T) { - n, tasks, err := ctl.ListTasks() - require.Nil(t, err) - assert.Equal(t, int64(1), n) - assert.Equal(t, int64(1), tasks[0].ID) -} - -func TestGetTask(t *testing.T) { - task, err := ctl.GetTask(1) - require.Nil(t, err) - assert.Equal(t, int64(1), task.ID) -} - -func TestUpdateTaskStatus(t *testing.T) { - err := ctl.UpdateTaskStatus(1, "running", 1) - require.Nil(t, err) -} - -func TestGetTaskLog(t *testing.T) { - log, err := ctl.GetTaskLog(1) - require.Nil(t, err) - assert.Equal(t, "message", string(log)) -} - -func TestIsTaskRunning(t *testing.T) { - cases := []struct { - task *models.Task - isFinalStatus bool - }{ - { - task: nil, - isFinalStatus: false, - }, - { - task: &models.Task{ - Status: models.TaskStatusSucceed, - }, - isFinalStatus: true, - }, - { - task: &models.Task{ - Status: models.TaskStatusFailed, - }, - isFinalStatus: true, - }, - { - task: &models.Task{ - Status: models.TaskStatusStopped, - }, - isFinalStatus: true, - }, - { - task: &models.Task{ - Status: models.TaskStatusInProgress, - }, - isFinalStatus: false, - }, - } - - for _, c := range cases { - assert.Equal(t, c.isFinalStatus, isTaskInFinalStatus(c.task)) - } -} diff --git a/src/replication/operation/execution/execution.go b/src/replication/operation/execution/execution.go deleted file mode 100644 index f91f82c8d..000000000 --- a/src/replication/operation/execution/execution.go +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package execution - -import ( - "fmt" - - "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/replication/dao" - "github.com/goharbor/harbor/src/replication/dao/models" -) - -// Manager manages the executions -type Manager interface { - // Create a new execution - Create(*models.Execution) (int64, error) - // List the summaries of executions - List(...*models.ExecutionQuery) (int64, []*models.Execution, error) - // Get the specified execution - Get(int64) (*models.Execution, error) - // Update the data of the specified execution, the "props" are the - // properties of execution that need to be updated - Update(execution *models.Execution, props ...string) error - // Remove the execution specified by the ID - Remove(int64) error - // Remove all executions of one policy specified by the policy ID - RemoveAll(int64) error - // Create a task - CreateTask(*models.Task) (int64, error) - // List the tasks according to the query - ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) - // Get one specified task - GetTask(int64) (*models.Task, error) - // Update the task, the "props" are the properties of task - // that need to be updated, it cannot include "status". If - // you want to update the status, use "UpdateTaskStatus" instead - UpdateTask(task *models.Task, props ...string) error - // UpdateTaskStatus only updates the task status. If "statusCondition" - // presents, only the tasks whose status equal to "statusCondition" - // will be updated - UpdateTaskStatus(taskID int64, status string, statusRevision int64, statusCondition ...string) error - // Remove one task specified by task ID - RemoveTask(int64) error - // Remove all tasks of one execution specified by the execution ID - RemoveAllTasks(int64) error - // Get the log of one specific task - GetTaskLog(int64) ([]byte, error) -} - -// DefaultManager .. -type DefaultManager struct { -} - -// NewDefaultManager ... -func NewDefaultManager() Manager { - return &DefaultManager{} -} - -// Create a new execution -func (dm *DefaultManager) Create(execution *models.Execution) (int64, error) { - return dao.AddExecution(execution) -} - -// List the summaries of executions -func (dm *DefaultManager) List(queries ...*models.ExecutionQuery) (int64, []*models.Execution, error) { - total, err := dao.GetTotalOfExecutions(queries...) - if err != nil { - return 0, nil, err - } - - executions, err := dao.GetExecutions(queries...) - if err != nil { - return 0, nil, err - } - return total, executions, nil -} - -// Get the specified execution -func (dm *DefaultManager) Get(id int64) (*models.Execution, error) { - return dao.GetExecution(id) -} - -// Update ... -func (dm *DefaultManager) Update(execution *models.Execution, props ...string) error { - n, err := dao.UpdateExecution(execution, props...) - if err != nil { - return err - } - if n == 0 { - return fmt.Errorf("Execution not found error: %d ", execution.ID) - } - return nil -} - -// Remove the execution specified by the ID -func (dm *DefaultManager) Remove(id int64) error { - return dao.DeleteExecution(id) -} - -// RemoveAll executions of one policy specified by the policy ID -func (dm *DefaultManager) RemoveAll(policyID int64) error { - return dao.DeleteAllExecutions(policyID) -} - -// CreateTask used to create a task -func (dm *DefaultManager) CreateTask(task *models.Task) (int64, error) { - return dao.AddTask(task) -} - -// ListTasks list the tasks according to the query -func (dm *DefaultManager) ListTasks(queries ...*models.TaskQuery) (int64, []*models.Task, error) { - total, err := dao.GetTotalOfTasks(queries...) - if err != nil { - return 0, nil, err - } - - tasks, err := dao.GetTasks(queries...) - if err != nil { - return 0, nil, err - } - return total, tasks, nil -} - -// GetTask get one specified task -func (dm *DefaultManager) GetTask(id int64) (*models.Task, error) { - return dao.GetTask(id) -} - -// UpdateTask ... -func (dm *DefaultManager) UpdateTask(task *models.Task, props ...string) error { - n, err := dao.UpdateTask(task, props...) - if err != nil { - return err - } - if n == 0 { - return fmt.Errorf("Task not found error: %d ", task.ID) - } - return nil -} - -// UpdateTaskStatus ... -func (dm *DefaultManager) UpdateTaskStatus(taskID int64, status string, statusRevision int64, statusCondition ...string) error { - if _, err := dao.UpdateTaskStatus(taskID, status, statusRevision, statusCondition...); err != nil { - return err - } - return nil -} - -// RemoveTask remove one task specified by task ID -func (dm *DefaultManager) RemoveTask(id int64) error { - return dao.DeleteTask(id) -} - -// RemoveAllTasks of one execution specified by the execution ID -func (dm *DefaultManager) RemoveAllTasks(executionID int64) error { - return dao.DeleteAllTasks(executionID) -} - -// GetTaskLog get the log of one specific task -func (dm *DefaultManager) GetTaskLog(taskID int64) ([]byte, error) { - task, err := dao.GetTask(taskID) - if err != nil { - return nil, err - } - if task == nil { - return nil, fmt.Errorf("Task not found %d ", taskID) - } - - return utils.GetJobServiceClient().GetJobLog(task.JobID) -} diff --git a/src/replication/operation/execution/execution_test.go b/src/replication/operation/execution/execution_test.go deleted file mode 100644 index ba1d8f1f8..000000000 --- a/src/replication/operation/execution/execution_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package execution - -import ( - "os" - "testing" - "time" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var executionManager = NewDefaultManager() - -func TestMain(m *testing.M) { - dao.PrepareTestForPostgresSQL() - os.Exit(m.Run()) -} - -func TestMethodOfExecutionManager(t *testing.T) { - execution := &models.Execution{ - PolicyID: 11209, - Status: "InProgress", - StatusText: "None", - Total: 12, - Failed: 0, - Succeed: 7, - InProgress: 5, - Stopped: 0, - Trigger: "Event", - StartTime: time.Now(), - } - - defer func() { - executionManager.RemoveAll(execution.PolicyID) - }() - - // Create - id, err := executionManager.Create(execution) - require.Nil(t, err) - - // List - query := &models.ExecutionQuery{ - Statuses: []string{"InProgress", "Failed"}, - Pagination: models.Pagination{ - Page: 1, - Size: 10, - }, - } - count, executions, err := executionManager.List(query) - require.Nil(t, err) - assert.Equal(t, int64(1), count) - assert.Equal(t, 1, len(executions)) - - // Get - _, err = executionManager.Get(id) - require.Nil(t, err) - - // Update - executionNew := &models.Execution{ - ID: id, - Status: "Failed", - Succeed: 12, - InProgress: 0, - EndTime: time.Now(), - } - err = executionManager.Update(executionNew, models.ExecutionPropsName.Status, models.ExecutionPropsName.Succeed, models.ExecutionPropsName.InProgress, - models.ExecutionPropsName.EndTime) - require.Nil(t, err) - - // Remove - require.Nil(t, executionManager.Remove(id)) -} - -func TestMethodOfTaskManager(t *testing.T) { - now := time.Now() - task := &models.Task{ - ExecutionID: 112200, - ResourceType: "resourceType1", - SrcResource: "srcResource1", - DstResource: "dstResource1", - JobID: "jobID1", - Status: "Initialized", - StatusRevision: 1, - StartTime: now, - } - - defer func() { - executionManager.RemoveAllTasks(task.ExecutionID) - }() - - // CreateTask - id, err := executionManager.CreateTask(task) - require.Nil(t, err) - - // ListTasks - query := &models.TaskQuery{ - ResourceType: "resourceType1", - Pagination: models.Pagination{ - Page: 1, - Size: 10, - }, - } - count, tasks, err := executionManager.ListTasks(query) - require.Nil(t, err) - assert.Equal(t, 1, len(tasks)) - assert.Equal(t, int64(1), count) - - // GetTask - _, err = executionManager.GetTask(id) - require.Nil(t, err) - - // UpdateTask - taskNew := &models.Task{ - ID: id, - SrcResource: "srcResourceChanged", - } - err = executionManager.UpdateTask(taskNew, models.TaskPropsName.SrcResource) - require.Nil(t, err) - taskUpdate, _ := executionManager.GetTask(id) - assert.Equal(t, taskNew.SrcResource, taskUpdate.SrcResource) - - // UpdateTaskStatus - err = executionManager.UpdateTaskStatus(id, models.TaskStatusSucceed, 1, models.TaskStatusInitialized) - require.Nil(t, err) - taskUpdate, _ = executionManager.GetTask(id) - assert.Equal(t, models.TaskStatusSucceed, taskUpdate.Status) - - // Remove - require.Nil(t, executionManager.RemoveTask(id)) - - // RemoveAll - require.Nil(t, executionManager.RemoveAll(id)) -} diff --git a/src/replication/operation/flow/copy.go b/src/replication/operation/flow/copy.go deleted file mode 100644 index 993f81c51..000000000 --- a/src/replication/operation/flow/copy.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flow - -import ( - "time" - - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/operation/execution" - "github.com/goharbor/harbor/src/replication/operation/scheduler" -) - -type copyFlow struct { - executionID int64 - resources []*model.Resource - policy *model.Policy - executionMgr execution.Manager - scheduler scheduler.Scheduler -} - -// NewCopyFlow returns an instance of the copy flow which replicates the resources from -// the source registry to the destination registry. If the parameter "resources" isn't provided, -// will fetch the resources first -func NewCopyFlow(executionMgr execution.Manager, scheduler scheduler.Scheduler, - executionID int64, policy *model.Policy, resources ...*model.Resource) Flow { - return ©Flow{ - executionMgr: executionMgr, - scheduler: scheduler, - executionID: executionID, - policy: policy, - resources: resources, - } -} - -func (c *copyFlow) Run(interface{}) (int, error) { - srcAdapter, dstAdapter, err := initialize(c.policy) - if err != nil { - return 0, err - } - var srcResources []*model.Resource - if len(c.resources) > 0 { - srcResources, err = filterResources(c.resources, c.policy.Filters) - } else { - srcResources, err = fetchResources(srcAdapter, c.policy) - } - if err != nil { - return 0, err - } - - isStopped, err := isExecutionStopped(c.executionMgr, c.executionID) - if err != nil { - return 0, err - } - if isStopped { - log.Debugf("the execution %d is stopped, stop the flow", c.executionID) - return 0, nil - } - - if len(srcResources) == 0 { - markExecutionSuccess(c.executionMgr, c.executionID, "no resources need to be replicated") - log.Infof("no resources need to be replicated for the execution %d, skip", c.executionID) - return 0, nil - } - - srcResources = assembleSourceResources(srcResources, c.policy) - dstResources := assembleDestinationResources(srcResources, c.policy) - - if err = prepareForPush(dstAdapter, dstResources); err != nil { - return 0, err - } - items, err := preprocess(c.scheduler, srcResources, dstResources) - if err != nil { - return 0, err - } - if err = createTasks(c.executionMgr, c.executionID, items); err != nil { - return 0, err - } - - return schedule(c.scheduler, c.executionMgr, items) -} - -// mark the execution as success in database -func markExecutionSuccess(mgr execution.Manager, id int64, message string) { - err := mgr.Update( - &models.Execution{ - ID: id, - Status: models.ExecutionStatusSucceed, - StatusText: message, - EndTime: time.Now(), - }, "Status", "StatusText", "EndTime") - if err != nil { - log.Errorf("failed to update the execution %d: %v", id, err) - return - } -} diff --git a/src/replication/operation/flow/copy_test.go b/src/replication/operation/flow/copy_test.go deleted file mode 100644 index 6da4f8a16..000000000 --- a/src/replication/operation/flow/copy_test.go +++ /dev/null @@ -1,36 +0,0 @@ -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flow - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRunOfCopyFlow(t *testing.T) { - scheduler := &fakedScheduler{} - executionMgr := &fakedExecutionManager{} - policy := &model.Policy{ - SrcRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - DestRegistry: &model.Registry{ - Type: model.RegistryTypeHarbor, - }, - } - flow := NewCopyFlow(executionMgr, scheduler, 1, policy) - n, err := flow.Run(nil) - require.Nil(t, err) - assert.Equal(t, 2, n) -} diff --git a/src/replication/operation/flow/deletion.go b/src/replication/operation/flow/deletion.go deleted file mode 100644 index 64058b73c..000000000 --- a/src/replication/operation/flow/deletion.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flow - -import ( - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/operation/execution" - "github.com/goharbor/harbor/src/replication/operation/scheduler" -) - -type deletionFlow struct { - executionID int64 - policy *model.Policy - executionMgr execution.Manager - scheduler scheduler.Scheduler - resources []*model.Resource -} - -// NewDeletionFlow returns an instance of the delete flow which deletes the resources -// on the destination registry -func NewDeletionFlow(executionMgr execution.Manager, scheduler scheduler.Scheduler, - executionID int64, policy *model.Policy, resources ...*model.Resource) Flow { - return &deletionFlow{ - executionMgr: executionMgr, - scheduler: scheduler, - executionID: executionID, - policy: policy, - resources: resources, - } -} - -func (d *deletionFlow) Run(interface{}) (int, error) { - srcResources, err := filterResources(d.resources, d.policy.Filters) - if err != nil { - return 0, err - } - if len(srcResources) == 0 { - markExecutionSuccess(d.executionMgr, d.executionID, "no resources need to be replicated") - log.Infof("no resources need to be replicated for the execution %d, skip", d.executionID) - return 0, nil - } - - srcResources = assembleSourceResources(srcResources, d.policy) - dstResources := assembleDestinationResources(srcResources, d.policy) - - items, err := preprocess(d.scheduler, srcResources, dstResources) - if err != nil { - return 0, err - } - if err = createTasks(d.executionMgr, d.executionID, items); err != nil { - return 0, err - } - - return schedule(d.scheduler, d.executionMgr, items) -} diff --git a/src/replication/operation/flow/stage_test.go b/src/replication/operation/flow/stage_test.go deleted file mode 100644 index 691f857b0..000000000 --- a/src/replication/operation/flow/stage_test.go +++ /dev/null @@ -1,427 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flow - -import ( - "io" - "os" - "testing" - - "github.com/docker/distribution" - "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/config" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/operation/scheduler" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type fakedFactory struct { -} - -func (fakedFactory) Create(*model.Registry) (adapter.Adapter, error) { - return &fakedAdapter{}, nil -} - -func (fakedFactory) AdapterPattern() *model.AdapterPattern { - return nil -} - -type fakedAdapter struct{} - -func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { - return &model.RegistryInfo{ - Type: model.RegistryTypeHarbor, - SupportedResourceTypes: []model.ResourceType{ - model.ResourceTypeImage, - model.ResourceTypeChart, - }, - SupportedTriggers: []model.TriggerType{model.TriggerTypeManual}, - }, nil -} - -func (f *fakedAdapter) PrepareForPush([]*model.Resource) error { - return nil -} -func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { - return model.Healthy, nil -} -func (f *fakedAdapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) { - return []*model.Resource{ - { - Type: model.ResourceTypeImage, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"latest"}, - }, - Override: false, - }, - }, nil -} - -func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) { - return false, "", nil -} -func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error) { - return nil, "", nil -} -func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) { - return "", nil -} -func (f *fakedAdapter) DeleteManifest(repository, digest string) error { - return nil -} -func (f *fakedAdapter) BlobExist(repository, digest string) (exist bool, err error) { - return false, nil -} -func (f *fakedAdapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) { - return 0, nil, nil -} -func (f *fakedAdapter) PushBlob(repository, digest string, size int64, blob io.Reader) error { - return nil -} -func (f *fakedAdapter) DeleteTag(repository, tag string) error { - return nil -} -func (f *fakedAdapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { - return []*model.Resource{ - { - Type: model.ResourceTypeChart, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/harbor", - }, - Vtags: []string{"0.2.0"}, - }, - }, - }, nil -} -func (f *fakedAdapter) ChartExist(name, version string) (bool, error) { - return false, nil -} -func (f *fakedAdapter) DownloadChart(name, version string) (io.ReadCloser, error) { - return nil, nil -} -func (f *fakedAdapter) UploadChart(name, version string, chart io.Reader) error { - return nil -} -func (f *fakedAdapter) DeleteChart(name, version string) error { - return nil -} - -type fakedScheduler struct{} - -func (f *fakedScheduler) Preprocess(src []*model.Resource, dst []*model.Resource) ([]*scheduler.ScheduleItem, error) { - items := []*scheduler.ScheduleItem{} - for i, res := range src { - items = append(items, &scheduler.ScheduleItem{ - SrcResource: res, - DstResource: dst[i], - }) - } - return items, nil -} -func (f *fakedScheduler) Schedule(items []*scheduler.ScheduleItem) ([]*scheduler.ScheduleResult, error) { - results := []*scheduler.ScheduleResult{} - for _, item := range items { - results = append(results, &scheduler.ScheduleResult{ - TaskID: item.TaskID, - Error: nil, - }) - } - return results, nil -} -func (f *fakedScheduler) Stop(id string) error { - return nil -} - -type fakedExecutionManager struct { - taskID int64 -} - -func (f *fakedExecutionManager) Create(*models.Execution) (int64, error) { - return 1, nil -} -func (f *fakedExecutionManager) List(...*models.ExecutionQuery) (int64, []*models.Execution, error) { - return 0, nil, nil -} -func (f *fakedExecutionManager) Get(int64) (*models.Execution, error) { - return &models.Execution{}, nil -} -func (f *fakedExecutionManager) Update(*models.Execution, ...string) error { - return nil -} -func (f *fakedExecutionManager) Remove(int64) error { - return nil -} -func (f *fakedExecutionManager) RemoveAll(int64) error { - return nil -} -func (f *fakedExecutionManager) CreateTask(*models.Task) (int64, error) { - f.taskID++ - id := f.taskID - return id, nil -} -func (f *fakedExecutionManager) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) { - return 0, nil, nil -} -func (f *fakedExecutionManager) GetTask(int64) (*models.Task, error) { - return nil, nil -} -func (f *fakedExecutionManager) UpdateTask(*models.Task, ...string) error { - return nil -} -func (f *fakedExecutionManager) UpdateTaskStatus(int64, string, int64, ...string) error { - return nil -} -func (f *fakedExecutionManager) RemoveTask(int64) error { - return nil -} -func (f *fakedExecutionManager) RemoveAllTasks(int64) error { - return nil -} -func (f *fakedExecutionManager) GetTaskLog(int64) ([]byte, error) { - return nil, nil -} - -func TestMain(m *testing.M) { - url := "https://registry.harbor.local" - config.Config = &config.Configuration{ - CoreURL: url, - } - if err := adapter.RegisterFactory(model.RegistryTypeHarbor, new(fakedFactory)); err != nil { - os.Exit(1) - } - os.Exit(m.Run()) -} - -func TestFetchResources(t *testing.T) { - adapter := &fakedAdapter{} - policy := &model.Policy{} - resources, err := fetchResources(adapter, policy) - require.Nil(t, err) - assert.Equal(t, 2, len(resources)) -} - -func TestFilterResources(t *testing.T) { - resources := []*model.Resource{ - { - Type: model.ResourceTypeImage, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Artifacts: []*model.Artifact{ - { - Tags: []string{"latest"}, - }, - }, - }, - Deleted: true, - Override: true, - }, - { - Type: model.ResourceTypeChart, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/harbor", - }, - Artifacts: []*model.Artifact{ - { - Tags: []string{"0.2.0"}, - }, - { - Tags: []string{"0.3.0"}, - }, - }, - }, - Deleted: true, - Override: true, - }, - { - Type: model.ResourceTypeChart, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/mysql", - }, - Artifacts: []*model.Artifact{ - { - Tags: []string{"1.0"}, - }, - }, - }, - Deleted: true, - Override: true, - }, - } - filters := []*model.Filter{ - { - Type: model.FilterTypeResource, - Value: model.ResourceTypeChart, - }, - { - Type: model.FilterTypeName, - Value: "library/*", - }, - { - Type: model.FilterTypeName, - Value: "library/harbor", - }, - { - Type: model.FilterTypeTag, - Value: "0.2.?", - }, - } - res, err := filterResources(resources, filters) - require.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "library/harbor", res[0].Metadata.Repository.Name) - assert.Equal(t, 1, len(res[0].Metadata.Artifacts)) - assert.Equal(t, "0.2.0", res[0].Metadata.Artifacts[0].Tags[0]) -} - -func TestAssembleSourceResources(t *testing.T) { - resources := []*model.Resource{ - { - Type: model.ResourceTypeChart, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"latest"}, - }, - Override: false, - }, - } - policy := &model.Policy{ - SrcRegistry: &model.Registry{ - ID: 1, - }, - } - res := assembleSourceResources(resources, policy) - assert.Equal(t, 1, len(res)) - assert.Equal(t, int64(1), res[0].Registry.ID) -} - -func TestAssembleDestinationResources(t *testing.T) { - resources := []*model.Resource{ - { - Type: model.ResourceTypeChart, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"latest"}, - }, - Override: false, - }, - } - policy := &model.Policy{ - DestRegistry: &model.Registry{}, - DestNamespace: "test", - Override: true, - } - res := assembleDestinationResources(resources, policy) - assert.Equal(t, 1, len(res)) - assert.Equal(t, model.ResourceTypeChart, res[0].Type) - assert.Equal(t, "test/hello-world", res[0].Metadata.Repository.Name) - assert.Equal(t, 1, len(res[0].Metadata.Vtags)) - assert.Equal(t, "latest", res[0].Metadata.Vtags[0]) -} - -func TestPreprocess(t *testing.T) { - scheduler := &fakedScheduler{} - srcResources := []*model.Resource{ - { - Type: model.ResourceTypeChart, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", - }, - Vtags: []string{"latest"}, - }, - Override: false, - }, - } - dstResources := []*model.Resource{ - { - Type: model.ResourceTypeChart, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "test/hello-world", - }, - Vtags: []string{"latest"}, - }, - Override: false, - }, - } - items, err := preprocess(scheduler, srcResources, dstResources) - require.Nil(t, err) - assert.Equal(t, 1, len(items)) -} - -func TestCreateTasks(t *testing.T) { - mgr := &fakedExecutionManager{} - items := []*scheduler.ScheduleItem{ - { - SrcResource: &model.Resource{}, - DstResource: &model.Resource{}, - }, - } - err := createTasks(mgr, 1, items) - require.Nil(t, err) - assert.Equal(t, int64(1), items[0].TaskID) -} - -func TestSchedule(t *testing.T) { - sched := &fakedScheduler{} - mgr := &fakedExecutionManager{} - items := []*scheduler.ScheduleItem{ - { - SrcResource: &model.Resource{}, - DstResource: &model.Resource{}, - TaskID: 1, - }, - } - n, err := schedule(sched, mgr, items) - require.Nil(t, err) - assert.Equal(t, 1, n) -} - -func TestReplaceNamespace(t *testing.T) { - // empty namespace - repository := "c" - namespace := "" - result := replaceNamespace(repository, namespace) - assert.Equal(t, "c", result) - // repository contains no "/" - repository = "c" - namespace = "n" - result = replaceNamespace(repository, namespace) - assert.Equal(t, "n/c", result) - // repository contains only one "/" - repository = "b/c" - namespace = "n" - result = replaceNamespace(repository, namespace) - assert.Equal(t, "n/c", result) - // repository contains more than one "/" - repository = "a/b/c" - namespace = "n" - result = replaceNamespace(repository, namespace) - assert.Equal(t, "n/c", result) -} diff --git a/src/replication/operation/hook/task.go b/src/replication/operation/hook/task.go deleted file mode 100644 index af512f7f2..000000000 --- a/src/replication/operation/hook/task.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hook - -import ( - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/operation" -) - -// UpdateTask update the status of the task -func UpdateTask(ctl operation.Controller, id int64, status string, statusRevision int64) error { - jobStatus := job.Status(status) - // convert the job status to task status - s := "" - preStatus := []string{} - switch jobStatus { - case job.PendingStatus: - s = models.TaskStatusPending - preStatus = append(preStatus, models.TaskStatusInitialized) - case job.ScheduledStatus, job.RunningStatus: - s = models.TaskStatusInProgress - preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending) - case job.StoppedStatus: - s = models.TaskStatusStopped - preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress) - case job.ErrorStatus: - s = models.TaskStatusFailed - preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress) - case job.SuccessStatus: - s = models.TaskStatusSucceed - preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress) - } - return ctl.UpdateTaskStatus(id, s, statusRevision, preStatus...) -} diff --git a/src/replication/operation/hook/task_test.go b/src/replication/operation/hook/task_test.go deleted file mode 100644 index a06c44961..000000000 --- a/src/replication/operation/hook/task_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hook - -import ( - "testing" - - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/goharbor/harbor/src/replication/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type fakedOperationController struct { - status string -} - -func (f *fakedOperationController) StartReplication(*model.Policy, *model.Resource, model.TriggerType) (int64, error) { - return 0, nil -} -func (f *fakedOperationController) StopReplication(int64) error { - return nil -} -func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) { - return 0, nil, nil -} -func (f *fakedOperationController) GetExecution(int64) (*models.Execution, error) { - return nil, nil -} -func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) { - return 0, nil, nil -} -func (f *fakedOperationController) GetTask(int64) (*models.Task, error) { - return nil, nil -} -func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error { - f.status = status - return nil -} -func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) { - return nil, nil -} - -func TestUpdateTask(t *testing.T) { - mgr := &fakedOperationController{} - cases := []struct { - inputStatus string - expectedStatus string - }{ - { - inputStatus: job.PendingStatus.String(), - expectedStatus: models.TaskStatusPending, - }, - { - inputStatus: job.ScheduledStatus.String(), - expectedStatus: models.TaskStatusInProgress, - }, - { - inputStatus: job.RunningStatus.String(), - expectedStatus: models.TaskStatusInProgress, - }, - { - inputStatus: job.StoppedStatus.String(), - expectedStatus: models.TaskStatusStopped, - }, - { - inputStatus: job.ErrorStatus.String(), - expectedStatus: models.TaskStatusFailed, - }, - { - inputStatus: job.SuccessStatus.String(), - expectedStatus: models.TaskStatusSucceed, - }, - } - - for _, c := range cases { - err := UpdateTask(mgr, 1, c.inputStatus, 1) - require.Nil(t, err) - assert.Equal(t, c.expectedStatus, mgr.status) - } -} diff --git a/src/replication/operation/scheduler/scheduler.go b/src/replication/operation/scheduler/scheduler.go deleted file mode 100644 index 8ea7acd87..000000000 --- a/src/replication/operation/scheduler/scheduler.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scheduler - -import ( - "encoding/json" - "errors" - "fmt" - - cjob "github.com/goharbor/harbor/src/common/job" - "github.com/goharbor/harbor/src/common/job/models" - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/replication/config" - "github.com/goharbor/harbor/src/replication/model" -) - -type defaultScheduler struct { - client cjob.Client -} - -// NewScheduler returns an instance of Scheduler -func NewScheduler(js cjob.Client) Scheduler { - return &defaultScheduler{ - client: js, - } -} - -// ScheduleItem is an item that can be scheduled -type ScheduleItem struct { - TaskID int64 // used as the param in the hook - SrcResource *model.Resource - DstResource *model.Resource -} - -// ScheduleResult is the result of the schedule for one item -type ScheduleResult struct { - TaskID int64 - JobID string - Error error -} - -// Scheduler schedules -type Scheduler interface { - // Preprocess the resources and returns the item list that can be scheduled - Preprocess([]*model.Resource, []*model.Resource) ([]*ScheduleItem, error) - // Schedule the items. If got error when scheduling one of the items, - // the error should be put in the corresponding ScheduleResult and the - // returning error of this function should be nil - Schedule([]*ScheduleItem) ([]*ScheduleResult, error) - // Stop the job specified by ID - Stop(id string) error -} - -// Preprocess the resources and returns the item list that can be scheduled -func (d *defaultScheduler) Preprocess(srcResources []*model.Resource, destResources []*model.Resource) ([]*ScheduleItem, error) { - if len(srcResources) != len(destResources) { - err := errors.New("srcResources has different length with destResources") - return nil, err - } - var items []*ScheduleItem - for index, srcResource := range srcResources { - destResource := destResources[index] - item := &ScheduleItem{ - SrcResource: srcResource, - DstResource: destResource, - } - items = append(items, item) - - } - return items, nil -} - -// Schedule transfer the tasks to jobs,and then submit these jobs to job service. -func (d *defaultScheduler) Schedule(items []*ScheduleItem) ([]*ScheduleResult, error) { - var results []*ScheduleResult - for _, item := range items { - result := &ScheduleResult{ - TaskID: item.TaskID, - } - if item.TaskID == 0 { - result.Error = errors.New("some tasks do not have a ID") - results = append(results, result) - continue - } - j := &models.JobData{ - Metadata: &models.JobMetadata{ - JobKind: job.KindGeneric, - }, - StatusHook: fmt.Sprintf("%s/service/notifications/jobs/replication/task/%d", config.Config.CoreURL, item.TaskID), - } - - j.Name = job.Replication - src, err := json.Marshal(item.SrcResource) - if err != nil { - result.Error = err - results = append(results, result) - continue - } - dest, err := json.Marshal(item.DstResource) - if err != nil { - result.Error = err - results = append(results, result) - continue - } - j.Parameters = map[string]interface{}{ - "src_resource": string(src), - "dst_resource": string(dest), - } - id, joberr := d.client.SubmitJob(j) - if joberr != nil { - result.Error = joberr - results = append(results, result) - continue - } - result.JobID = id - results = append(results, result) - } - return results, nil -} - -// Stop the transfer job -func (d *defaultScheduler) Stop(id string) error { - err := d.client.PostAction(id, string(job.StopCommand)) - if err != nil { - return err - } - return nil - -} diff --git a/src/replication/operation/scheduler/scheduler_test.go b/src/replication/operation/scheduler/scheduler_test.go deleted file mode 100644 index cb85814ac..000000000 --- a/src/replication/operation/scheduler/scheduler_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package scheduler - -import ( - "encoding/json" - "testing" - - "github.com/goharbor/harbor/src/common/job/models" - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/replication/model" -) - -var scheduler = &defaultScheduler{ - client: TestClient{}, -} - -type TestClient struct { -} - -func (client TestClient) SubmitJob(*models.JobData) (string, error) { - return "submited-uuid", nil -} -func (client TestClient) GetJobLog(uuid string) ([]byte, error) { - return []byte("job log"), nil -} -func (client TestClient) PostAction(uuid, action string) error { - return nil -} -func (client TestClient) GetExecutions(uuid string) ([]job.Stats, error) { - return nil, nil -} - -func TestPreprocess(t *testing.T) { - items, err := generateData() - if err != nil { - t.Error(err) - } - for _, item := range items { - content, err := json.Marshal(item) - if err != nil { - t.Error(err) - } - t.Log(string(content)) - } - -} - -func TestStop(t *testing.T) { - err := scheduler.Stop("id") - if err != nil { - t.Error(err) - } -} - -func generateData() ([]*ScheduleItem, error) { - srcResource := &model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "namespace1", - }, - Vtags: []string{"latest"}, - }, - Registry: &model.Registry{ - Credential: &model.Credential{}, - }, - } - destResource := &model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "namespace2", - }, - Vtags: []string{"v1", "v2"}, - }, - Registry: &model.Registry{ - Credential: &model.Credential{}, - }, - } - items, err := scheduler.Preprocess([]*model.Resource{srcResource}, []*model.Resource{destResource}) - return items, err -} diff --git a/src/replication/policy/controller/controller.go b/src/replication/policy/controller/controller.go index c9a2e5a6f..3e8da362f 100644 --- a/src/replication/policy/controller/controller.go +++ b/src/replication/policy/controller/controller.go @@ -17,20 +17,25 @@ package controller import ( "fmt" - "github.com/goharbor/harbor/src/common/job" + "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/policy" "github.com/goharbor/harbor/src/replication/policy/manager" - "github.com/goharbor/harbor/src/replication/policy/scheduler" +) + +// const definitions +const ( + CallbackFuncName = "REPLICATION_CALLBACK" ) // NewController returns a policy controller which can CURD and schedule policies -func NewController(js job.Client) policy.Controller { +func NewController() policy.Controller { mgr := manager.NewDefaultManager() - scheduler := scheduler.NewScheduler(js) ctl := &controller{ - scheduler: scheduler, + scheduler: scheduler.Sched, } ctl.Controller = mgr return ctl @@ -47,10 +52,7 @@ func (c *controller) Create(policy *model.Policy) (int64, error) { return 0, err } if isScheduledTrigger(policy) { - // TODO: need a way to show the schedule status to users - // maybe we can add a property "schedule status" for - // listing policy API - if err = c.scheduler.Schedule(id, policy.Trigger.Settings.Cron); err != nil { + if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id); err != nil { log.Errorf("failed to schedule the policy %d: %v", id, err) } } @@ -72,7 +74,7 @@ func (c *controller) Update(policy *model.Policy) error { // need to reschedule the policy // unschedule first if needed if isScheduledTrigger(origin) { - if err = c.scheduler.Unschedule(origin.ID); err != nil { + if err = c.scheduler.UnScheduleByVendor(orm.Context(), job.Replication, origin.ID); err != nil { return fmt.Errorf("failed to unschedule the policy %d: %v", origin.ID, err) } } @@ -82,7 +84,7 @@ func (c *controller) Update(policy *model.Policy) error { } // schedule again if needed if isScheduledTrigger(policy) { - if err = c.scheduler.Schedule(policy.ID, policy.Trigger.Settings.Cron); err != nil { + if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID); err != nil { return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err) } } @@ -98,7 +100,7 @@ func (c *controller) Remove(policyID int64) error { return fmt.Errorf("policy %d not found", policyID) } if isScheduledTrigger(policy) { - if err = c.scheduler.Unschedule(policyID); err != nil { + if err = c.scheduler.UnScheduleByVendor(orm.Context(), job.Replication, policyID); err != nil { return err } } diff --git a/src/replication/policy/controller/controller_test.go b/src/replication/policy/controller/controller_test.go index 8515d4126..ea28fe969 100644 --- a/src/replication/policy/controller/controller_test.go +++ b/src/replication/policy/controller/controller_test.go @@ -17,7 +17,10 @@ package controller import ( "testing" + "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/testing/mock" + "github.com/goharbor/harbor/src/testing/pkg/scheduler" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,20 +48,6 @@ func (f *fakedPolicyController) Remove(int64) error { return nil } -type fakedScheduler struct { - scheduled bool - unscheduled bool -} - -func (f *fakedScheduler) Schedule(policyID int64, cron string) error { - f.scheduled = true - return nil -} -func (f *fakedScheduler) Unschedule(policyID int64) error { - f.unscheduled = true - return nil -} - func TestIsScheduledTrigger(t *testing.T) { cases := []struct { policy *model.Policy @@ -224,7 +213,8 @@ func TestIsScheduleTriggerChanged(t *testing.T) { } func TestCreate(t *testing.T) { - scheduler := &fakedScheduler{} + dao.PrepareTestForPostgresSQL() + scheduler := &scheduler.Scheduler{} ctl := &controller{ scheduler: scheduler, } @@ -233,9 +223,10 @@ func TestCreate(t *testing.T) { // not scheduled trigger _, err := ctl.Create(&model.Policy{}) require.Nil(t, err) - assert.False(t, scheduler.scheduled) // scheduled trigger + scheduler.On("Schedule", mock.Anything, mock.Anything, + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) _, err = ctl.Create(&model.Policy{ Enabled: true, Trigger: &model.Trigger{ @@ -246,11 +237,11 @@ func TestCreate(t *testing.T) { }, }) require.Nil(t, err) - assert.True(t, scheduler.scheduled) + scheduler.AssertExpectations(t) } func TestUpdate(t *testing.T) { - scheduler := &fakedScheduler{} + scheduler := &scheduler.Scheduler{} c := &fakedPolicyController{} ctl := &controller{ scheduler: scheduler, @@ -275,10 +266,13 @@ func TestUpdate(t *testing.T) { current = origin err = ctl.Update(current) require.Nil(t, err) - assert.False(t, scheduler.scheduled) - assert.False(t, scheduler.unscheduled) // the trigger changed + scheduler.On("Schedule", mock.Anything, mock.Anything, + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, + mock.Anything).Return(nil) + origin = &model.Policy{ ID: 1, Enabled: true, @@ -301,12 +295,11 @@ func TestUpdate(t *testing.T) { } err = ctl.Update(current) require.Nil(t, err) - assert.True(t, scheduler.unscheduled) - assert.True(t, scheduler.scheduled) + scheduler.AssertExpectations(t) } func TestRemove(t *testing.T) { - scheduler := &fakedScheduler{} + scheduler := &scheduler.Scheduler{} c := &fakedPolicyController{} ctl := &controller{ scheduler: scheduler, @@ -328,9 +321,10 @@ func TestRemove(t *testing.T) { c.policy = policy err = ctl.Remove(1) require.Nil(t, err) - assert.False(t, scheduler.unscheduled) // the trigger type is scheduled + scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, + mock.Anything).Return(nil) policy = &model.Policy{ ID: 1, Enabled: true, @@ -344,5 +338,5 @@ func TestRemove(t *testing.T) { c.policy = policy err = ctl.Remove(1) require.Nil(t, err) - assert.True(t, scheduler.unscheduled) + scheduler.AssertExpectations(t) } diff --git a/src/replication/policy/scheduler/scheduler.go b/src/replication/policy/scheduler/scheduler.go deleted file mode 100644 index 2096861ec..000000000 --- a/src/replication/policy/scheduler/scheduler.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scheduler - -import ( - "fmt" - "net/http" - "strings" - "time" - - commonHttp "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/common/job" - jobModels "github.com/goharbor/harbor/src/common/job/models" - jsJob "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/replication/config" - "github.com/goharbor/harbor/src/replication/dao" - "github.com/goharbor/harbor/src/replication/dao/models" -) - -// Scheduler can be used to schedule or unschedule a scheduled policy -// Currently, the default scheduler implements its capabilities by delegating -// the scheduled job of jobservice -type Scheduler interface { - Schedule(policyID int64, cron string) error - Unschedule(policyID int64) error -} - -// NewScheduler returns an instance of scheduler -func NewScheduler(js job.Client) Scheduler { - return &scheduler{ - jobservice: js, - } -} - -type scheduler struct { - jobservice job.Client -} - -func (s *scheduler) Schedule(policyID int64, cron string) error { - now := time.Now() - id, err := dao.ScheduleJob.Add(&models.ScheduleJob{ - PolicyID: policyID, - Status: job.JobServiceStatusPending, - CreationTime: now, - UpdateTime: now, - }) - if err != nil { - return err - } - log.Debugf("the schedule job record %d added", id) - - statusHookURL := fmt.Sprintf("%s/service/notifications/jobs/replication/%d", config.Config.CoreURL, id) - jobID, err := s.jobservice.SubmitJob(&jobModels.JobData{ - Name: jsJob.ReplicationScheduler, - Parameters: map[string]interface{}{ - "url": config.Config.CoreURL, - "policy_id": policyID, - }, - Metadata: &jobModels.JobMetadata{ - JobKind: job.JobKindPeriodic, - Cron: cron, - }, - StatusHook: statusHookURL, - }) - if err != nil { - // clean up the record in database - if e := dao.ScheduleJob.Delete(id); e != nil { - log.Errorf("failed to delete the schedule job %d: %v", id, e) - } else { - log.Debugf("the schedule job record %d deleted", id) - } - return err - } - log.Debugf("the schedule job for policy %d submitted to the jobservice", policyID) - - err = dao.ScheduleJob.Update(&models.ScheduleJob{ - ID: id, - JobID: jobID, - }, "JobID") - log.Debugf("the policy %d scheduled", policyID) - return err -} - -func (s *scheduler) Unschedule(policyID int64) error { - sjs, err := dao.ScheduleJob.List(&models.ScheduleJobQuery{ - PolicyID: policyID, - }) - if err != nil { - return err - } - for _, sj := range sjs { - if err = s.jobservice.PostAction(sj.JobID, job.JobActionStop); err != nil { - // if the job specified by jobID is not found in jobservice, just delete - // the record from database - if e, ok := err.(*commonHttp.Error); !ok || (e.Code != http.StatusNotFound && - !strings.Contains(e.Message, "no valid periodic job policy found")) { - return err - } - log.Debugf("the stop action for schedule job %s submitted to the jobservice", sj.JobID) - } - if err = dao.ScheduleJob.Delete(sj.ID); err != nil { - return err - } - log.Debugf("the schedule job record %d deleted", sj.ID) - } - log.Debugf("the policy %d unscheduled", policyID) - return nil -} diff --git a/src/replication/policy/scheduler/scheduler_test.go b/src/replication/policy/scheduler/scheduler_test.go deleted file mode 100644 index 27f6c9456..000000000 --- a/src/replication/policy/scheduler/scheduler_test.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scheduler - -import ( - "fmt" - "testing" - - "github.com/goharbor/harbor/src/common/job/models" - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/replication/config" - "github.com/goharbor/harbor/src/replication/dao" - rep_models "github.com/goharbor/harbor/src/replication/dao/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TODO share the faked implementation in a separated common package? -// TODO can we use a mock framework? - -var ( - uuid = "uuid" - policyID int64 = 100 -) - -type fakedJobserviceClient struct { - jobData *models.JobData - stopped bool -} - -func (f *fakedJobserviceClient) SubmitJob(jobData *models.JobData) (string, error) { - f.jobData = jobData - return uuid, nil -} -func (f *fakedJobserviceClient) GetJobLog(uuid string) ([]byte, error) { - f.stopped = true - return nil, nil -} -func (f *fakedJobserviceClient) PostAction(uuid, action string) error { - f.stopped = true - return nil -} -func (f *fakedJobserviceClient) GetExecutions(uuid string) ([]job.Stats, error) { - f.stopped = true - return nil, nil -} - -type fakedScheduleJobDAO struct { - idCounter int64 - sjs map[int64]*rep_models.ScheduleJob -} - -func (f *fakedScheduleJobDAO) Add(sj *rep_models.ScheduleJob) (int64, error) { - if f.sjs == nil { - f.sjs = make(map[int64]*rep_models.ScheduleJob) - } - id := f.idCounter + 1 - sj.ID = id - f.sjs[id] = sj - return id, nil -} -func (f *fakedScheduleJobDAO) Get(id int64) (*rep_models.ScheduleJob, error) { - if f.sjs == nil { - return nil, nil - } - return f.sjs[id], nil -} -func (f *fakedScheduleJobDAO) Update(sj *rep_models.ScheduleJob, props ...string) error { - err := fmt.Errorf("schedule job %d not found", sj.ID) - if f.sjs == nil { - return err - } - j, exist := f.sjs[sj.ID] - if !exist { - return err - } - if len(props) == 0 { - f.sjs[sj.ID] = sj - return nil - } - - for _, prop := range props { - switch prop { - case "PolicyID": - j.PolicyID = sj.PolicyID - case "JobID": - j.JobID = sj.JobID - case "Status": - j.Status = sj.Status - case "UpdateTime": - j.UpdateTime = sj.UpdateTime - } - } - return nil -} -func (f *fakedScheduleJobDAO) Delete(id int64) error { - if f.sjs == nil { - return nil - } - delete(f.sjs, id) - return nil -} -func (f *fakedScheduleJobDAO) List(query ...*rep_models.ScheduleJobQuery) ([]*rep_models.ScheduleJob, error) { - var policyID int64 - if len(query) > 0 { - policyID = query[0].PolicyID - } - sjs := []*rep_models.ScheduleJob{} - for _, sj := range f.sjs { - if policyID == 0 { - sjs = append(sjs, sj) - continue - } - if sj.PolicyID == policyID { - sjs = append(sjs, sj) - } - } - return sjs, nil -} - -func TestSchedule(t *testing.T) { - config.Config = &config.Configuration{} - dao.ScheduleJob = &fakedScheduleJobDAO{} - js := &fakedJobserviceClient{} - scheduler := NewScheduler(js) - err := scheduler.Schedule(policyID, "1 * * * *") - require.Nil(t, err) - - sjs, err := dao.ScheduleJob.List(&rep_models.ScheduleJobQuery{ - PolicyID: policyID, - }) - require.Nil(t, err) - require.Equal(t, 1, len(sjs)) - assert.Equal(t, uuid, sjs[0].JobID) - - policyID, ok := js.jobData.Parameters["policy_id"].(int64) - require.True(t, ok) - assert.Equal(t, policyID, policyID) -} - -func TestUnschedule(t *testing.T) { - config.Config = &config.Configuration{} - dao.ScheduleJob = &fakedScheduleJobDAO{} - _, err := dao.ScheduleJob.Add(&rep_models.ScheduleJob{ - PolicyID: policyID, - }) - require.Nil(t, err) - js := &fakedJobserviceClient{} - scheduler := NewScheduler(js) - err = scheduler.Unschedule(policyID) - require.Nil(t, err) - - sjs, err := dao.ScheduleJob.List(&rep_models.ScheduleJobQuery{ - PolicyID: policyID, - }) - require.Nil(t, err) - require.Equal(t, 0, len(sjs)) - - assert.True(t, js.stopped) -} diff --git a/src/replication/replication.go b/src/replication/replication.go index ddcf97246..8642cf81d 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -15,14 +15,18 @@ package replication import ( + "context" + "fmt" + "strconv" "time" - "github.com/goharbor/harbor/src/common/job" + "github.com/goharbor/harbor/src/controller/replication" cfg "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/replication/config" "github.com/goharbor/harbor/src/replication/event" - "github.com/goharbor/harbor/src/replication/operation" "github.com/goharbor/harbor/src/replication/policy" "github.com/goharbor/harbor/src/replication/policy/controller" "github.com/goharbor/harbor/src/replication/registry" @@ -60,12 +64,37 @@ var ( PolicyCtl policy.Controller // RegistryMgr is a global registry manager RegistryMgr registry.Manager - // OperationCtl is a global operation controller - OperationCtl operation.Controller // EventHandler handles images/chart pull/push events EventHandler event.Handler ) +func init() { + callbackFunc := func(ctx context.Context, param string) error { + policyID, err := strconv.ParseInt(param, 10, 64) + if err != nil { + return err + } + + policy, err := PolicyCtl.Get(policyID) + if err != nil { + return err + } + if policy == nil { + return fmt.Errorf("policy %d not found", policyID) + } + if err = event.PopulateRegistries(RegistryMgr, policy); err != nil { + return err + } + + _, err = replication.Ctl.Start(ctx, policy, nil, task.ExecutionTriggerSchedule) + return err + } + err := scheduler.RegisterCallbackFunc(controller.CallbackFuncName, callbackFunc) + if err != nil { + log.Errorf("failed to register the callback function for replication: %v", err) + } +} + // Init the global variables and configurations func Init(closing, done chan struct{}) error { // init config @@ -76,21 +105,15 @@ func Init(closing, done chan struct{}) error { config.Config = &config.Configuration{ CoreURL: cfg.InternalCoreURL(), TokenServiceURL: cfg.InternalTokenServiceEndpoint(), - JobserviceURL: cfg.InternalJobServiceURL(), SecretKey: secretKey, - CoreSecret: cfg.CoreSecret(), JobserviceSecret: cfg.JobserviceSecret(), } - // TODO use a global http transport - js := job.NewDefaultClient(config.Config.JobserviceURL, config.Config.CoreSecret) // init registry manager RegistryMgr = registry.NewDefaultManager() // init policy controller - PolicyCtl = controller.NewController(js) - // init operation controller - OperationCtl = operation.NewController(js) + PolicyCtl = controller.NewController() // init event handler - EventHandler = event.NewHandler(PolicyCtl, RegistryMgr, OperationCtl) + EventHandler = event.NewHandler(PolicyCtl, RegistryMgr) log.Debug("the replication initialization completed") // Start health checker for registries diff --git a/src/replication/replication_test.go b/src/replication/replication_test.go index 58d41ed0c..0f018333b 100644 --- a/src/replication/replication_test.go +++ b/src/replication/replication_test.go @@ -38,6 +38,5 @@ func TestInit(t *testing.T) { require.Nil(t, err) assert.NotNil(t, PolicyCtl) assert.NotNil(t, RegistryMgr) - assert.NotNil(t, OperationCtl) assert.NotNil(t, EventHandler) } diff --git a/src/server/handler/job_status_hook.go b/src/server/handler/job_status_hook.go index 11b92f115..1b259b12d 100644 --- a/src/server/handler/job_status_hook.go +++ b/src/server/handler/job_status_hook.go @@ -17,20 +17,18 @@ package handler import ( "encoding/json" "net/http" - "strconv" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/errors" libhttp "github.com/goharbor/harbor/src/lib/http" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/task" - "github.com/goharbor/harbor/src/server/router" ) // NewJobStatusHandler creates a handler to handle the job status changing func NewJobStatusHandler() http.Handler { return &jobStatusHandler{ - handler: task.NewHookHandler(), + handler: task.HkHandler, } } @@ -40,23 +38,16 @@ type jobStatusHandler struct { func (j *jobStatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() - taskIDParam := router.Param(r.Context(), ":id") - taskID, err := strconv.ParseInt(taskIDParam, 10, 64) - if err != nil { - libhttp.SendError(w, err) - return - } sc := &job.StatusChange{} - if err = json.NewDecoder(r.Body).Decode(sc); err != nil { + if err := json.NewDecoder(r.Body).Decode(sc); err != nil { libhttp.SendError(w, err) return } - - if err = j.handler.Handle(r.Context(), taskID, sc); err != nil { + if err := j.handler.Handle(r.Context(), sc); err != nil { // ignore the not found error to avoid the jobservice re-sending the hook if errors.IsNotFoundErr(err) { - log.Warningf("task %d does not exist, ignore the not found error to avoid subsequent retrying webhooks from jobservice", taskID) + log.Warningf("got not found error: %v, ignore it to avoid subsequent retrying webhooks from jobservice", err) return } libhttp.SendError(w, err) diff --git a/src/server/route.go b/src/server/route.go index a89eae75e..96071f486 100644 --- a/src/server/route.go +++ b/src/server/route.go @@ -15,6 +15,8 @@ package server import ( + "net/http" + "github.com/astaxie/beego" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/core/api" @@ -22,11 +24,9 @@ import ( "github.com/goharbor/harbor/src/core/controllers" "github.com/goharbor/harbor/src/core/service/notifications/admin" "github.com/goharbor/harbor/src/core/service/notifications/jobs" - "github.com/goharbor/harbor/src/core/service/notifications/scheduler" "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/server/handler" "github.com/goharbor/harbor/src/server/router" - "net/http" ) func registerRoutes() { @@ -47,12 +47,11 @@ func registerRoutes() { beego.Router("/api/internal/syncquota", &api.InternalAPI{}, "post:SyncQuota") beego.Router("/service/notifications/jobs/adminjob/:id([0-9]+)", &admin.Handler{}, "post:HandleAdminJob") - beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationScheduleJob") - beego.Router("/service/notifications/jobs/replication/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationTask") beego.Router("/service/notifications/jobs/webhook/:id([0-9]+)", &jobs.Handler{}, "post:HandleNotificationJob") beego.Router("/service/notifications/jobs/retention/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleRetentionTask") - beego.Router("/service/notifications/schedules/:id([0-9]+)", &scheduler.Handler{}, "post:Handle") beego.Router("/service/notifications/jobs/scan/:uuid", &jobs.Handler{}, "post:HandleScan") + router.NewRoute().Method(http.MethodPost).Path("/service/notifications/schedules/:id([0-9]+)").Handler(handler.NewJobStatusHandler()) // legacy job status hook endpoint for scheduler + router.NewRoute().Method(http.MethodPost).Path("/service/notifications/jobs/replication/:id([0-9]+)").Handler(handler.NewJobStatusHandler()) // legacy job status hook endpoint for replication scheduler router.NewRoute().Method(http.MethodPost).Path("/service/notifications/tasks/:id").Handler(handler.NewJobStatusHandler()) beego.Router("/service/token", &token.Handler{}) diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index 1ca8bc8c0..6c3e19957 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -29,13 +29,14 @@ import ( // New returns http handler for API V2.0 func New() http.Handler { h, api, err := restapi.HandlerAPI(restapi.Config{ - ArtifactAPI: newArtifactAPI(), - RepositoryAPI: newRepositoryAPI(), - AuditlogAPI: newAuditLogAPI(), - ScanAPI: newScanAPI(), - ProjectAPI: newProjectAPI(), - PreheatAPI: newPreheatAPI(), - IconAPI: newIconAPI(), + ArtifactAPI: newArtifactAPI(), + RepositoryAPI: newRepositoryAPI(), + AuditlogAPI: newAuditLogAPI(), + ScanAPI: newScanAPI(), + ProjectAPI: newProjectAPI(), + PreheatAPI: newPreheatAPI(), + IconAPI: newIconAPI(), + ReplicationAPI: newReplicationAPI(), }) if err != nil { log.Fatal(err) diff --git a/src/server/v2.0/handler/preheat.go b/src/server/v2.0/handler/preheat.go index f81e0d18d..043c632a9 100644 --- a/src/server/v2.0/handler/preheat.go +++ b/src/server/v2.0/handler/preheat.go @@ -663,7 +663,7 @@ func convertTaskToPayload(model *task.Task) (*models.Task, error) { ExecutionID: model.ExecutionID, ExtraAttrs: model.ExtraAttrs, ID: model.ID, - RunCount: int64(model.RunCount), + RunCount: model.RunCount, StartTime: model.StartTime.String(), Status: model.Status, StatusMessage: model.StatusMessage, diff --git a/src/server/v2.0/handler/replication.go b/src/server/v2.0/handler/replication.go new file mode 100644 index 000000000..31bf5c9ef --- /dev/null +++ b/src/server/v2.0/handler/replication.go @@ -0,0 +1,297 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handler + +import ( + "context" + "strconv" + "strings" + + "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" + "github.com/goharbor/harbor/src/controller/replication" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/task" + replica "github.com/goharbor/harbor/src/replication" + "github.com/goharbor/harbor/src/replication/event" + "github.com/goharbor/harbor/src/replication/policy" + "github.com/goharbor/harbor/src/replication/policy/manager" + "github.com/goharbor/harbor/src/server/v2.0/models" + operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/replication" +) + +func newReplicationAPI() *replicationAPI { + return &replicationAPI{ + ctl: replication.Ctl, + policyMgr: manager.NewDefaultManager(), + } +} + +type replicationAPI struct { + BaseAPI + ctl replication.Controller + policyMgr policy.Controller +} + +func (r *replicationAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder { + if err := r.RequireSysAdmin(ctx); err != nil { + r.SendError(ctx, err) + } + return nil +} + +func (r *replicationAPI) StartReplication(ctx context.Context, params operation.StartReplicationParams) middleware.Responder { + // TODO move the following logic to the replication controller after refactoring the policy management part with the new programming model + policy, err := r.policyMgr.Get(params.Execution.PolicyID) + if err != nil { + return r.SendError(ctx, err) + } + if policy == nil { + return r.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode). + WithMessage("the replication policy %d not found", params.Execution.PolicyID)) + } + if err = event.PopulateRegistries(replica.RegistryMgr, policy); err != nil { + return r.SendError(ctx, err) + } + + // the legacy replication scheduler job("src/jobservice/job/impl/replication/scheduler.go") calls the start replication API + // to trigger the scheduled replication, a query string "trigger" is added when sending the request + // here is the logic to cover this part + trigger := task.ExecutionTriggerManual + if params.HTTPRequest.URL.Query().Get("trigger") == "scheduled" { + trigger = task.ExecutionTriggerSchedule + } + + executionID, err := r.ctl.Start(ctx, policy, nil, trigger) + if err != nil { + return r.SendError(ctx, err) + } + location := strings.TrimSuffix(params.HTTPRequest.URL.Path, "/") + "/" + strconv.FormatInt(executionID, 10) + return operation.NewStartReplicationCreated().WithLocation(location) +} + +func (r *replicationAPI) StopReplication(ctx context.Context, params operation.StopReplicationParams) middleware.Responder { + if err := r.ctl.Stop(ctx, params.ID); err != nil { + return r.SendError(ctx, err) + } + return nil +} + +func (r *replicationAPI) ListReplicationExecutions(ctx context.Context, params operation.ListReplicationExecutionsParams) middleware.Responder { + query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize) + if err != nil { + return r.SendError(ctx, err) + } + if params.PolicyID != nil { + query.Keywords["PolicyID"] = *params.PolicyID + } + if params.Status != nil { + status := *params.Status + // as we convert the status when responding requests to keep the backward compatibility, + // here we need to reverse-convert the status + switch status { + case "InProgress": + status = job.RunningStatus.String() + case "Succeed": + status = job.SuccessStatus.String() + case "Stopped": + status = job.StoppedStatus.String() + case "Failed": + status = job.ErrorStatus.String() + } + query.Keywords["Status"] = status + } + if params.Trigger != nil { + trigger := *params.Trigger + // as we convert the trigger when responding requests to keep the backward compatibility, + // here we need to reverse-convert the trigger + switch trigger { + case "manual": + trigger = task.ExecutionTriggerManual + case "scheduled": + trigger = task.ExecutionTriggerSchedule + case "event_based": + trigger = task.ExecutionTriggerEvent + } + query.Keywords["Trigger"] = trigger + } + + total, err := r.ctl.ExecutionCount(ctx, query) + if err != nil { + return r.SendError(ctx, err) + } + executions, err := r.ctl.ListExecutions(ctx, query) + if err != nil { + return r.SendError(ctx, err) + } + + var execs []*models.ReplicationExecution + for _, execution := range executions { + execs = append(execs, convertExecution(execution)) + } + + return operation.NewListReplicationExecutionsOK(). + WithXTotalCount(total). + WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). + WithPayload(execs) +} + +func (r *replicationAPI) GetReplicationExecution(ctx context.Context, params operation.GetReplicationExecutionParams) middleware.Responder { + execution, err := r.ctl.GetExecution(ctx, params.ID) + if err != nil { + return r.SendError(ctx, err) + } + return operation.NewGetReplicationExecutionOK().WithPayload(convertExecution(execution)) +} + +func (r *replicationAPI) ListReplicationTasks(ctx context.Context, params operation.ListReplicationTasksParams) middleware.Responder { + query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize) + if err != nil { + return r.SendError(ctx, err) + } + query.Keywords["ExecutionID"] = params.ID + if params.Status != nil { + var status interface{} = *params.Status + // as we convert the status when responding requests to keep the backward compatibility, + // here we need to reverse-convert the status + // the status "pending" and "stopped" is same with jobservice, no need to convert + switch status { + case "InProgress": + status = &q.OrList{ + Values: []interface{}{ + job.ScheduledStatus.String(), + job.RunningStatus.String(), + }, + } + case "Succeed": + status = job.SuccessStatus.String() + case "Failed": + status = job.ErrorStatus.String() + } + query.Keywords["Status"] = status + } + if params.ResourceType != nil { + query.Keywords["ExtraAttrs.resource_type"] = *params.ResourceType + } + + total, err := r.ctl.TaskCount(ctx, query) + if err != nil { + return r.SendError(ctx, err) + } + + tasks, err := r.ctl.ListTasks(ctx, query) + if err != nil { + return r.SendError(ctx, err) + } + + var tks []*models.ReplicationTask + for _, task := range tasks { + tks = append(tks, convertTask(task)) + } + + return operation.NewListReplicationTasksOK(). + WithXTotalCount(total). + WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). + WithPayload(tks) +} + +func (r *replicationAPI) GetReplicationLog(ctx context.Context, params operation.GetReplicationLogParams) middleware.Responder { + execution, err := r.ctl.GetExecution(ctx, params.ID) + if err != nil { + return r.SendError(ctx, err) + } + task, err := r.ctl.GetTask(ctx, params.TaskID) + if err != nil { + return r.SendError(ctx, err) + } + if execution.ID != task.ExecutionID { + return r.SendError(ctx, errors.New(nil). + WithCode(errors.NotFoundCode). + WithMessage("execution %d contains no task with ID %d", params.ID, params.TaskID)) + } + log, err := r.ctl.GetTaskLog(ctx, params.TaskID) + if err != nil { + return r.SendError(ctx, err) + } + return operation.NewGetReplicationLogOK().WithContentType("text/plain").WithPayload(string(log)) +} + +func convertExecution(execution *replication.Execution) *models.ReplicationExecution { + exec := &models.ReplicationExecution{ + ID: execution.ID, + PolicyID: execution.PolicyID, + StatusText: execution.StatusMessage, + StartTime: strfmt.DateTime(execution.StartTime), + EndTime: strfmt.DateTime(execution.EndTime), + } + // keep backward compatibility + if execution.Metrics != nil { + exec.Total = execution.Metrics.TaskCount + exec.Succeed = execution.Metrics.SuccessTaskCount + exec.Failed = execution.Metrics.ErrorTaskCount + exec.InProgress = execution.Metrics.PendingTaskCount + + execution.Metrics.ScheduledTaskCount + execution.Metrics.RunningTaskCount + exec.Stopped = execution.Metrics.StoppedTaskCount + } + switch execution.Trigger { + case task.ExecutionTriggerManual: + exec.Trigger = "manual" + case task.ExecutionTriggerSchedule: + exec.Trigger = "scheduled" + case task.ExecutionTriggerEvent: + exec.Trigger = "event_based" + } + switch execution.Status { + case job.RunningStatus.String(): + exec.Status = "InProgress" + case job.SuccessStatus.String(): + exec.Status = "Succeed" + case job.StoppedStatus.String(): + exec.Status = "Stopped" + case job.ErrorStatus.String(): + exec.Status = "Failed" + } + + return exec +} + +func convertTask(task *replication.Task) *models.ReplicationTask { + tk := &models.ReplicationTask{ + ID: task.ID, + ExecutionID: task.ExecutionID, + JobID: task.JobID, + Operation: task.Operation, + ResourceType: task.ResourceType, + SrcResource: task.SourceResource, + DstResource: task.DestinationResource, + StartTime: strfmt.DateTime(task.StartTime), + EndTime: strfmt.DateTime(task.EndTime), + } + // keep backward compatibility + switch task.Status { + case job.ScheduledStatus.String(), job.RunningStatus.String(): + tk.Status = "InProgress" + case job.SuccessStatus.String(): + tk.Status = "Succeed" + case job.ErrorStatus.String(): + tk.Status = "Failed" + // the status "pending" and "stopped" is same with jobservice, no need to convert + default: + tk.Status = task.Status + } + return tk +} diff --git a/src/server/v2.0/route/legacy.go b/src/server/v2.0/route/legacy.go index 9e3b56bb1..5b3ab55d1 100755 --- a/src/server/v2.0/route/legacy.go +++ b/src/server/v2.0/route/legacy.go @@ -58,10 +58,6 @@ func registerLegacyRoutes() { beego.Router("/api/"+version+"/replication/adapters", &api.ReplicationAdapterAPI{}, "get:List") beego.Router("/api/"+version+"/replication/adapterinfos", &api.ReplicationAdapterAPI{}, "get:ListAdapterInfos") - beego.Router("/api/"+version+"/replication/executions", &api.ReplicationOperationAPI{}, "get:ListExecutions;post:CreateExecution") - beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)", &api.ReplicationOperationAPI{}, "get:GetExecution;put:StopExecution") - beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)/tasks", &api.ReplicationOperationAPI{}, "get:ListTasks") - beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)/tasks/:tid([0-9]+)/log", &api.ReplicationOperationAPI{}, "get:GetTaskLog") beego.Router("/api/"+version+"/replication/policies", &api.ReplicationPolicyAPI{}, "get:List;post:Create") beego.Router("/api/"+version+"/replication/policies/:id([0-9]+)", &api.ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete") diff --git a/src/testing/controller/controller.go b/src/testing/controller/controller.go index 01949ca4f..f9d067e70 100644 --- a/src/testing/controller/controller.go +++ b/src/testing/controller/controller.go @@ -21,3 +21,4 @@ package controller //go:generate mockery --case snake --dir ../../controller/scan --name Controller --output ./scan --outpkg scan //go:generate mockery --case snake --dir ../../controller/scan --name Checker --output ./scan --outpkg scan //go:generate mockery --case snake --dir ../../controller/scanner --name Controller --output ./scanner --outpkg scanner +//go:generate mockery --case snake --dir ../../controller/replication --name Controller --output ./replication --outpkg replication diff --git a/src/testing/controller/replication/controller.go b/src/testing/controller/replication/controller.go new file mode 100644 index 000000000..296e99867 --- /dev/null +++ b/src/testing/controller/replication/controller.go @@ -0,0 +1,211 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package replication + +import ( + context "context" + + model "github.com/goharbor/harbor/src/replication/model" + mock "github.com/stretchr/testify/mock" + + q "github.com/goharbor/harbor/src/lib/q" + + replication "github.com/goharbor/harbor/src/controller/replication" +) + +// Controller is an autogenerated mock type for the Controller type +type Controller struct { + mock.Mock +} + +// ExecutionCount provides a mock function with given fields: ctx, query +func (_m *Controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetExecution provides a mock function with given fields: ctx, executionID +func (_m *Controller) GetExecution(ctx context.Context, executionID int64) (*replication.Execution, error) { + ret := _m.Called(ctx, executionID) + + var r0 *replication.Execution + if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Execution); ok { + r0 = rf(ctx, executionID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*replication.Execution) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, executionID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetTask provides a mock function with given fields: ctx, taskID +func (_m *Controller) GetTask(ctx context.Context, taskID int64) (*replication.Task, error) { + ret := _m.Called(ctx, taskID) + + var r0 *replication.Task + if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Task); ok { + r0 = rf(ctx, taskID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*replication.Task) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, taskID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetTaskLog provides a mock function with given fields: ctx, taskID +func (_m *Controller) GetTaskLog(ctx context.Context, taskID int64) ([]byte, error) { + ret := _m.Called(ctx, taskID) + + var r0 []byte + if rf, ok := ret.Get(0).(func(context.Context, int64) []byte); ok { + r0 = rf(ctx, taskID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, taskID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListExecutions provides a mock function with given fields: ctx, query +func (_m *Controller) ListExecutions(ctx context.Context, query *q.Query) ([]*replication.Execution, error) { + ret := _m.Called(ctx, query) + + var r0 []*replication.Execution + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Execution); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*replication.Execution) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListTasks provides a mock function with given fields: ctx, query +func (_m *Controller) ListTasks(ctx context.Context, query *q.Query) ([]*replication.Task, error) { + ret := _m.Called(ctx, query) + + var r0 []*replication.Task + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Task); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*replication.Task) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Start provides a mock function with given fields: ctx, policy, resource, trigger +func (_m *Controller) Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (int64, error) { + ret := _m.Called(ctx, policy, resource, trigger) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *model.Policy, *model.Resource, string) int64); ok { + r0 = rf(ctx, policy, resource, trigger) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *model.Policy, *model.Resource, string) error); ok { + r1 = rf(ctx, policy, resource, trigger) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Stop provides a mock function with given fields: ctx, executionID +func (_m *Controller) Stop(ctx context.Context, executionID int64) error { + ret := _m.Called(ctx, executionID) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, executionID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TaskCount provides a mock function with given fields: ctx, query +func (_m *Controller) TaskCount(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/src/replication/policy/scheduler/status.go b/src/testing/lib/lib.go similarity index 63% rename from src/replication/policy/scheduler/status.go rename to src/testing/lib/lib.go index f3a13b288..2a925942b 100644 --- a/src/replication/policy/scheduler/status.go +++ b/src/testing/lib/lib.go @@ -12,17 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -package scheduler +package lib -import ( - "github.com/goharbor/harbor/src/replication/dao" - "github.com/goharbor/harbor/src/replication/dao/models" -) - -// UpdateStatus updates the schedule job status -func UpdateStatus(id int64, status string) error { - return dao.ScheduleJob.Update(&models.ScheduleJob{ - ID: id, - Status: status, - }, "Status") -} +//go:generate mockery --case snake --dir ../../lib/orm --name Creator --output ./orm --outpkg orm diff --git a/src/testing/lib/orm/creator.go b/src/testing/lib/orm/creator.go new file mode 100644 index 000000000..031276f85 --- /dev/null +++ b/src/testing/lib/orm/creator.go @@ -0,0 +1,29 @@ +// Code generated by mockery v2.1.0. DO NOT EDIT. + +package orm + +import ( + orm "github.com/astaxie/beego/orm" + mock "github.com/stretchr/testify/mock" +) + +// Creator is an autogenerated mock type for the Creator type +type Creator struct { + mock.Mock +} + +// Create provides a mock function with given fields: +func (_m *Creator) Create() orm.Ormer { + ret := _m.Called() + + var r0 orm.Ormer + if rf, ok := ret.Get(0).(func() orm.Ormer); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(orm.Ormer) + } + } + + return r0 +} diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index 89f07cc0e..34ca30126 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -28,7 +28,7 @@ def get_endpoint(): def _create_client(server, credential, debug, api_type="products"): cfg = None - if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat'): + if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat', 'replication'): cfg = v2_swagger_client.Configuration() else: cfg = swagger_client.Configuration() @@ -59,6 +59,7 @@ def _create_client(server, credential, debug, api_type="products"): "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)), "scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)), + "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)), }.get(api_type,'Error: Wrong API type') def _assert_status_code(expect_code, return_code): diff --git a/tests/apitests/python/library/replication.py b/tests/apitests/python/library/replication.py index 055e53843..fd80e513e 100644 --- a/tests/apitests/python/library/replication.py +++ b/tests/apitests/python/library/replication.py @@ -45,41 +45,7 @@ class Replication(base.Base): #else: # raise Exception(r"Check replication rule trigger failed, expect <{}> actual <{}>.".format(expect_trigger, get_trigger)) - - def start_replication(self, rule_id, **kwargs): - client = self._get_client(**kwargs) - return client.replications_post(swagger_client.Replication(int(rule_id))) - - def list_replication_jobs(self, rule_id, **kwargs): - client = self._get_client(**kwargs) - return client.jobs_replication_get(int(rule_id)) - - def wait_until_jobs_finish(self, rule_id, retry=10, interval=5, **kwargs): - Succeed = False - for i in range(retry): - Succeed = False - jobs = self.get_replication_executions(rule_id, **kwargs) - for job in jobs: - if job.status == "Succeed": - return - if not Succeed: - time.sleep(interval) - if not Succeed: - raise Exception("The jobs not Succeed") - def delete_replication_rule(self, rule_id, expect_status_code = 200, **kwargs): client = self._get_client(**kwargs) _, status_code, _ = client.replication_policies_id_delete_with_http_info(rule_id) - base._assert_status_code(expect_status_code, status_code) - - def trigger_replication_executions(self, rule_id, expect_status_code = 201, **kwargs): - client = self._get_client(**kwargs) - _, status_code, _ = client.replication_executions_post_with_http_info({"policy_id":rule_id}) - base._assert_status_code(expect_status_code, status_code) - - - def get_replication_executions(self, rule_id, expect_status_code = 200, **kwargs): - client = self._get_client(**kwargs) - data, status_code, _ = client.replication_executions_get_with_http_info(policy_id=rule_id) - base._assert_status_code(expect_status_code, status_code) - return data + base._assert_status_code(expect_status_code, status_code) \ No newline at end of file diff --git a/tests/apitests/python/library/replication_v2.py b/tests/apitests/python/library/replication_v2.py new file mode 100644 index 000000000..04efab457 --- /dev/null +++ b/tests/apitests/python/library/replication_v2.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +import time +import base +import v2_swagger_client +from v2_swagger_client.rest import ApiException + +class ReplicationV2(base.Base, object): + def __init__(self): + super(ReplicationV2,self).__init__(api_type = "replication") + + def wait_until_jobs_finish(self, rule_id, retry=10, interval=5, **kwargs): + Succeed = False + for i in range(retry): + Succeed = False + jobs = self.get_replication_executions(rule_id, **kwargs) + for job in jobs: + if job.status == "Succeed": + return + if not Succeed: + time.sleep(interval) + if not Succeed: + raise Exception("The jobs not Succeed") + + def trigger_replication_executions(self, rule_id, expect_status_code = 201, **kwargs): + client = self._get_client(**kwargs) + _, status_code, _ = client.start_replication_with_http_info({"policy_id":rule_id}) + base._assert_status_code(expect_status_code, status_code) + + def get_replication_executions(self, rule_id, expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + data, status_code, _ = client.list_replication_executions_with_http_info(policy_id=rule_id) + base._assert_status_code(expect_status_code, status_code) + return data + diff --git a/tests/apitests/python/test_replication_from_dockerhub.py b/tests/apitests/python/test_replication_from_dockerhub.py index 21833230d..0bc1750dd 100644 --- a/tests/apitests/python/test_replication_from_dockerhub.py +++ b/tests/apitests/python/test_replication_from_dockerhub.py @@ -9,6 +9,7 @@ from library.replication import Replication from library.registry import Registry from library.artifact import Artifact from library.repository import Repository +from library.replication_v2 import ReplicationV2 import swagger_client class TestProjects(unittest.TestCase): @@ -17,6 +18,7 @@ class TestProjects(unittest.TestCase): self.project = Project() self.user = User() self.replication = Replication() + self.replication_v2 = ReplicationV2() self.registry = Registry() self.artifact = Artifact() self.repo = Repository() @@ -87,10 +89,10 @@ class TestProjects(unittest.TestCase): self.replication.check_replication_rule_should_exist(TestProjects.rule_id, rule_name, **ADMIN_CLIENT) #6. Trigger the rule; - self.replication.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT) + self.replication_v2.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT) #7. Wait for completion of this replication job; - self.replication.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT) + self.replication_v2.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT) #8. Check image is replicated into target project successfully. artifact = self.artifact.get_reference_info(TestProjects.project_name, self.image, self.tag, **ADMIN_CLIENT)