mirror of https://github.com/goharbor/harbor.git
Refactor the replication execution
1. Use the task manager to manage the underlying execution/task 2. Use the pkg/scheduler to schedule the periodical job 3. Apply the new program model 4. Migration the old data into the new data model Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin
parent
ac8bc94012
commit
294385c34d
|
|
@ -738,231 +738,6 @@ paths:
|
|||
description: The auth mode of the system is not "oidc_auth", or the user is not onboarded via OIDC AuthN.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/replication/executions:
|
||||
get:
|
||||
summary: List replication executions.
|
||||
description: |
|
||||
This endpoint let user list replication executions.
|
||||
parameters:
|
||||
- name: policy_id
|
||||
in: query
|
||||
type: integer
|
||||
required: false
|
||||
description: The policy ID.
|
||||
- name: status
|
||||
in: query
|
||||
type: string
|
||||
required: false
|
||||
description: The execution status.
|
||||
- name: trigger
|
||||
in: query
|
||||
type: string
|
||||
required: false
|
||||
description: The trigger mode.
|
||||
- name: page
|
||||
in: query
|
||||
type: integer
|
||||
required: false
|
||||
description: The page.
|
||||
- name: page_size
|
||||
in: query
|
||||
type: integer
|
||||
required: false
|
||||
description: The page size.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Success
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/ReplicationExecution'
|
||||
headers:
|
||||
X-Total-Count:
|
||||
description: The total count of available items
|
||||
type: integer
|
||||
Link:
|
||||
description: Link to previous page and next page
|
||||
type: string
|
||||
'401':
|
||||
description: User need to login first.
|
||||
'403':
|
||||
description: User has no privilege for the operation.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
post:
|
||||
summary: Start one execution of the replication.
|
||||
description: |
|
||||
This endpoint is for user to start one execution of the replication.
|
||||
parameters:
|
||||
- name: execution
|
||||
in: body
|
||||
description: The execution that needs to be started, only the property "policy_id" is needed.
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/ReplicationExecution'
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'201':
|
||||
description: Success.
|
||||
headers:
|
||||
Location:
|
||||
type: string
|
||||
description: The URL of the created resource
|
||||
'400':
|
||||
description: Bad request.
|
||||
'401':
|
||||
description: User need to login first.
|
||||
'403':
|
||||
description: User has no privilege for the operation.
|
||||
'415':
|
||||
$ref: '#/responses/UnsupportedMediaType'
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/replication/executions/{id}:
|
||||
get:
|
||||
summary: Get the execution of the replication.
|
||||
description: |
|
||||
This endpoint is for user to get one execution of the replication.
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The execution ID.
|
||||
required: true
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Success.
|
||||
schema:
|
||||
$ref: '#/definitions/ReplicationExecution'
|
||||
'400':
|
||||
description: Bad request.
|
||||
'401':
|
||||
description: User need to login first.
|
||||
'403':
|
||||
description: User has no privilege for the operation.
|
||||
'404':
|
||||
description: Resource requested does not exist.
|
||||
'415':
|
||||
$ref: '#/responses/UnsupportedMediaType'
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
put:
|
||||
summary: Stop the execution of the replication.
|
||||
description: |
|
||||
This endpoint is for user to stop one execution of the replication.
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The execution ID.
|
||||
required: true
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Success.
|
||||
'400':
|
||||
description: Bad request.
|
||||
'401':
|
||||
description: User need to login first.
|
||||
'403':
|
||||
description: User has no privilege for the operation.
|
||||
'404':
|
||||
description: Resource requested does not exist.
|
||||
'415':
|
||||
$ref: '#/responses/UnsupportedMediaType'
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/replication/executions/{id}/tasks:
|
||||
get:
|
||||
summary: Get the task list of one execution.
|
||||
description: |
|
||||
This endpoint is for user to get the task list of one execution.
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The execution ID.
|
||||
required: true
|
||||
- name: page
|
||||
in: query
|
||||
type: integer
|
||||
format: int32
|
||||
required: false
|
||||
description: The page number.
|
||||
- name: page_size
|
||||
in: query
|
||||
type: integer
|
||||
format: int32
|
||||
required: false
|
||||
description: The size of per page.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Success.
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/ReplicationTask'
|
||||
headers:
|
||||
X-Total-Count:
|
||||
description: The total count of available items
|
||||
type: integer
|
||||
Link:
|
||||
description: Link to previous page and next page
|
||||
type: string
|
||||
'400':
|
||||
description: Bad request.
|
||||
'401':
|
||||
description: User need to login first.
|
||||
'403':
|
||||
description: User has no privilege for the operation.
|
||||
'404':
|
||||
description: Resource requested does not exist.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/replication/executions/{id}/tasks/{task_id}/log:
|
||||
get:
|
||||
summary: Get the log of one task.
|
||||
description: |
|
||||
This endpoint is for user to get the log of one task.
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The execution ID.
|
||||
required: true
|
||||
- name: task_id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The task ID.
|
||||
required: true
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Success.
|
||||
'400':
|
||||
description: Bad request.
|
||||
'401':
|
||||
description: User need to login first.
|
||||
'403':
|
||||
description: User has no privilege for the operation.
|
||||
'404':
|
||||
description: Resource requested does not exist.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/replication/policies:
|
||||
get:
|
||||
summary: List replication policies
|
||||
|
|
@ -4972,77 +4747,6 @@ definitions:
|
|||
description: The filter values
|
||||
items:
|
||||
type: string
|
||||
ReplicationExecution:
|
||||
type: object
|
||||
description: The replication execution
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
description: The ID
|
||||
policy_id:
|
||||
type: integer
|
||||
description: The policy ID
|
||||
status:
|
||||
type: string
|
||||
description: The status
|
||||
status_text:
|
||||
type: string
|
||||
description: The status text
|
||||
trigger:
|
||||
type: string
|
||||
description: The trigger mode
|
||||
total:
|
||||
type: integer
|
||||
description: The total count of all tasks
|
||||
failed:
|
||||
type: integer
|
||||
description: The count of failed tasks
|
||||
succeed:
|
||||
type: integer
|
||||
description: The count of succeed tasks
|
||||
in_progress:
|
||||
type: integer
|
||||
description: The count of in_progress tasks
|
||||
stopped:
|
||||
type: integer
|
||||
description: The count of stopped tasks
|
||||
start_time:
|
||||
type: string
|
||||
description: The start time
|
||||
end_time:
|
||||
type: string
|
||||
description: The end time
|
||||
ReplicationTask:
|
||||
type: object
|
||||
description: The replication task
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
description: The ID
|
||||
execution_id:
|
||||
type: integer
|
||||
description: The execution ID
|
||||
resource_type:
|
||||
type: string
|
||||
description: The resource type
|
||||
src_resource:
|
||||
type: string
|
||||
description: The source resource
|
||||
dst_resource:
|
||||
type: string
|
||||
description: The destination resource
|
||||
job_id:
|
||||
type: string
|
||||
description: The job ID
|
||||
status:
|
||||
type: string
|
||||
description: The status
|
||||
start_time:
|
||||
type: string
|
||||
description: The start time
|
||||
end_time:
|
||||
type: string
|
||||
description: The end time
|
||||
Namespace:
|
||||
type: object
|
||||
description: The namespace of registry
|
||||
|
|
|
|||
|
|
@ -1431,6 +1431,209 @@ paths:
|
|||
$ref: '#/responses/404'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/replication/executions:
|
||||
get:
|
||||
summary: List replication executions
|
||||
description: List replication executions
|
||||
tags:
|
||||
- replication
|
||||
operationId: listReplicationExecutions
|
||||
parameters:
|
||||
- $ref: '#/parameters/page'
|
||||
- $ref: '#/parameters/pageSize'
|
||||
- name: policy_id
|
||||
in: query
|
||||
type: integer
|
||||
required: false
|
||||
description: The ID of the policy that the executions belong to.
|
||||
- name: status
|
||||
in: query
|
||||
type: string
|
||||
required: false
|
||||
description: The execution status.
|
||||
- name: trigger
|
||||
in: query
|
||||
type: string
|
||||
required: false
|
||||
description: The trigger mode.
|
||||
responses:
|
||||
'200':
|
||||
description: Success
|
||||
headers:
|
||||
X-Total-Count:
|
||||
description: The total count of the resources
|
||||
type: integer
|
||||
Link:
|
||||
description: Link refers to the previous page and next page
|
||||
type: string
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/ReplicationExecution'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
post:
|
||||
summary: Start one replication execution
|
||||
description: Start one replication execution according to the policy
|
||||
tags:
|
||||
- replication
|
||||
operationId: startReplication
|
||||
parameters:
|
||||
- name: execution
|
||||
in: body
|
||||
description: The ID of policy that the execution belongs to
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/StartReplicationExecution'
|
||||
responses:
|
||||
'201':
|
||||
$ref: '#/responses/201'
|
||||
'400':
|
||||
$ref: '#/responses/400'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/replication/executions/{id}:
|
||||
get:
|
||||
summary: Get the specific replication execution
|
||||
description: Get the replication execution specified by ID
|
||||
tags:
|
||||
- replication
|
||||
operationId: getReplicationExecution
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The ID of the execution.
|
||||
required: true
|
||||
responses:
|
||||
'200':
|
||||
description: Success
|
||||
schema:
|
||||
$ref: '#/definitions/ReplicationExecution'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'404':
|
||||
$ref: '#/responses/404'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
put:
|
||||
summary: Stop the specific replication execution
|
||||
description: Stop the replication execution specified by ID
|
||||
tags:
|
||||
- replication
|
||||
operationId: stopReplication
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The ID of the execution.
|
||||
required: true
|
||||
responses:
|
||||
'200':
|
||||
$ref: '#/responses/200'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'404':
|
||||
$ref: '#/responses/404'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/replication/executions/{id}/tasks:
|
||||
get:
|
||||
summary: List replication tasks for a specific execution
|
||||
description: List replication tasks for a specific execution
|
||||
tags:
|
||||
- replication
|
||||
operationId: listReplicationTasks
|
||||
parameters:
|
||||
- $ref: '#/parameters/page'
|
||||
- $ref: '#/parameters/pageSize'
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The ID of the execution that the tasks belongs to.
|
||||
required: true
|
||||
- name: status
|
||||
in: query
|
||||
type: string
|
||||
required: false
|
||||
description: The task status.
|
||||
- name: resource_type
|
||||
in: query
|
||||
type: string
|
||||
required: false
|
||||
description: The resource type.
|
||||
responses:
|
||||
'200':
|
||||
description: Success
|
||||
headers:
|
||||
X-Total-Count:
|
||||
description: The total count of the resources
|
||||
type: integer
|
||||
Link:
|
||||
description: Link refers to the previous page and next page
|
||||
type: string
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/ReplicationTask'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/replication/executions/{id}/tasks/{task_id}/log:
|
||||
get:
|
||||
summary: Get the log of the specific replication task
|
||||
description: Get the log of the specific replication task
|
||||
tags:
|
||||
- replication
|
||||
operationId: getReplicationLog
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The ID of the execution that the tasks belongs to.
|
||||
required: true
|
||||
- name: task_id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
description: The ID of the task.
|
||||
required: true
|
||||
responses:
|
||||
'200':
|
||||
description: Success
|
||||
headers:
|
||||
Content-Type:
|
||||
description: The content type of response body
|
||||
type: string
|
||||
schema:
|
||||
type: string
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'404':
|
||||
$ref: '#/responses/404'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
parameters:
|
||||
query:
|
||||
name: q
|
||||
|
|
@ -2123,6 +2326,7 @@ definitions:
|
|||
description: The status message of task
|
||||
run_count:
|
||||
type: integer
|
||||
format: int32
|
||||
description: The count of task run
|
||||
extra_attrs:
|
||||
$ref: '#/definitions/ExtraAttrs'
|
||||
|
|
@ -2398,3 +2602,94 @@ definitions:
|
|||
additionalProperties:
|
||||
type: integer
|
||||
format: int64
|
||||
ReplicationExecution:
|
||||
type: object
|
||||
description: The replication execution
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
description: The ID of the execution
|
||||
policy_id:
|
||||
type: integer
|
||||
description: The ID if the policy that the execution belongs to
|
||||
status:
|
||||
type: string
|
||||
description: The status of the execution
|
||||
trigger:
|
||||
type: string
|
||||
description: The trigger mode
|
||||
start_time:
|
||||
type: string
|
||||
format: date-time
|
||||
description: The start time
|
||||
end_time:
|
||||
type: string
|
||||
format: date-time
|
||||
description: The end time
|
||||
status_text:
|
||||
type: string
|
||||
x-omitempty: false
|
||||
description: The status text
|
||||
total:
|
||||
type: integer
|
||||
x-omitempty: false
|
||||
description: The total count of all executions
|
||||
failed:
|
||||
type: integer
|
||||
x-omitempty: false
|
||||
description: The count of failed executions
|
||||
succeed:
|
||||
type: integer
|
||||
x-omitempty: false
|
||||
description: The count of succeed executions
|
||||
in_progress:
|
||||
type: integer
|
||||
x-omitempty: false
|
||||
description: The count of in_progress executions
|
||||
stopped:
|
||||
type: integer
|
||||
x-omitempty: false
|
||||
description: The count of stopped executions
|
||||
StartReplicationExecution:
|
||||
type: object
|
||||
properties:
|
||||
policy_id:
|
||||
type: integer
|
||||
format: int64
|
||||
description: The ID of policy that the execution belongs to.
|
||||
ReplicationTask:
|
||||
type: object
|
||||
description: The replication task
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
description: The ID of the task
|
||||
execution_id:
|
||||
type: integer
|
||||
description: The ID of the execution that the task belongs to
|
||||
status:
|
||||
type: string
|
||||
description: The status of the task
|
||||
job_id:
|
||||
type: string
|
||||
description: The ID of the underlying job that the task related to
|
||||
operation:
|
||||
type: string
|
||||
description: The operation of the task
|
||||
resource_type:
|
||||
type: string
|
||||
description: The type of the resource that the task operates
|
||||
src_resource:
|
||||
type: string
|
||||
description: The source resource that the task operates
|
||||
dst_resource:
|
||||
type: string
|
||||
description: The destination resource that the task operates
|
||||
start_time:
|
||||
type: string
|
||||
format: date-time
|
||||
description: The start time of the task
|
||||
end_time:
|
||||
type: string
|
||||
format: date-time
|
||||
description: The end time of the task
|
||||
|
|
|
|||
|
|
@ -7,6 +7,10 @@ UPDATE role SET role_id=5 WHERE name='limitedGuest' AND role_id!=5;
|
|||
|
||||
ALTER TABLE schedule ADD COLUMN IF NOT EXISTS cron_type varchar(64);
|
||||
|
||||
ALTER TABLE task ADD COLUMN IF NOT EXISTS vendor_type varchar(16);
|
||||
UPDATE task SET vendor_type = execution.vendor_type FROM execution WHERE task.execution_id = execution.id;
|
||||
ALTER TABLE task ALTER COLUMN vendor_type SET NOT NULL;
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
art RECORD;
|
||||
|
|
@ -45,3 +49,193 @@ CREATE TABLE IF NOT EXISTS permission_policy (
|
|||
creation_time timestamp default CURRENT_TIMESTAMP,
|
||||
CONSTRAINT unique_rbac_policy UNIQUE (scope, resource, action, effect)
|
||||
);
|
||||
|
||||
/*delete the replication execution records whose policy doesn't exist*/
|
||||
DELETE FROM replication_execution
|
||||
WHERE id IN (SELECT re.id FROM replication_execution re
|
||||
LEFT JOIN replication_policy rp ON re.policy_id=rp.id
|
||||
WHERE rp.id IS NULL);
|
||||
|
||||
/*delete the replication task records whose execution doesn't exist*/
|
||||
DELETE FROM replication_task
|
||||
WHERE id IN (SELECT rt.id FROM replication_task rt
|
||||
LEFT JOIN replication_execution re ON rt.execution_id=re.id
|
||||
WHERE re.id IS NULL);
|
||||
|
||||
/*fill the task count, status and end_time of execution based on the tasks*/
|
||||
DO $$
|
||||
DECLARE
|
||||
rep_exec RECORD;
|
||||
status_count RECORD;
|
||||
rep_status varchar(32);
|
||||
BEGIN
|
||||
FOR rep_exec IN SELECT * FROM replication_execution
|
||||
LOOP
|
||||
/*the replication status is set directly in some cases, so skip if the status is a final one*/
|
||||
IF rep_exec.status='Stopped' OR rep_exec.status='Failed' OR rep_exec.status='Succeed' THEN
|
||||
CONTINUE;
|
||||
END IF;
|
||||
/*fulfill the status count*/
|
||||
FOR status_count IN SELECT status, COUNT(*) as c FROM replication_task WHERE execution_id=rep_exec.id GROUP BY status
|
||||
LOOP
|
||||
IF status_count.status = 'Stopped' THEN
|
||||
UPDATE replication_execution SET stopped=status_count.c WHERE id=rep_exec.id;
|
||||
ELSIF status_count.status = 'Failed' THEN
|
||||
UPDATE replication_execution SET failed=status_count.c WHERE id=rep_exec.id;
|
||||
ELSIF status_count.status = 'Succeed' THEN
|
||||
UPDATE replication_execution SET succeed=status_count.c WHERE id=rep_exec.id;
|
||||
ELSE
|
||||
UPDATE replication_execution SET in_progress=status_count.c WHERE id=rep_exec.id;
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
/*reload the execution record*/
|
||||
SELECT * INTO rep_exec FROM replication_execution where id=rep_exec.id;
|
||||
|
||||
/*calculate the status*/
|
||||
IF rep_exec.in_progress>0 THEN
|
||||
rep_status = 'InProgress';
|
||||
ELSIF rep_exec.failed>0 THEN
|
||||
rep_status = 'Failed';
|
||||
ELSIF rep_exec.stopped>0 THEN
|
||||
rep_status = 'Stopped';
|
||||
ELSE
|
||||
rep_status = 'Succeed';
|
||||
END IF;
|
||||
UPDATE replication_execution SET status=rep_status WHERE id=rep_exec.id;
|
||||
|
||||
/*update the end time if the status is a final one*/
|
||||
IF rep_status='Failed' OR rep_status='Stopped' OR rep_status='Succeed' THEN
|
||||
UPDATE replication_execution
|
||||
SET end_time=(SELECT MAX (end_time) FROM replication_task WHERE execution_id=rep_exec.id)
|
||||
WHERE id=rep_exec.id;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
/*move the replication execution records into the new execution table*/
|
||||
ALTER TABLE replication_execution ADD COLUMN IF NOT EXISTS new_execution_id int;
|
||||
DO $$
|
||||
DECLARE
|
||||
rep_exec RECORD;
|
||||
trigger varchar(64);
|
||||
status varchar(32);
|
||||
new_exec_id integer;
|
||||
BEGIN
|
||||
FOR rep_exec IN SELECT * FROM replication_execution
|
||||
LOOP
|
||||
IF rep_exec.trigger = 'scheduled' THEN
|
||||
trigger = 'SCHEDULE';
|
||||
ELSIF rep_exec.trigger = 'event_based' THEN
|
||||
trigger = 'EVENT';
|
||||
ELSE
|
||||
trigger = 'MANUAL';
|
||||
END IF;
|
||||
|
||||
IF rep_exec.status = 'InProgress' THEN
|
||||
status = 'Running';
|
||||
ELSIF rep_exec.status = 'Stopped' THEN
|
||||
status = 'Stopped';
|
||||
ELSIF rep_exec.status = 'Failed' THEN
|
||||
status = 'Error';
|
||||
ELSIF rep_exec.status = 'Succeed' THEN
|
||||
status = 'Success';
|
||||
END IF;
|
||||
|
||||
INSERT INTO execution (vendor_type, vendor_id, status, status_message, revision, trigger, start_time, end_time)
|
||||
VALUES ('REPLICATION', rep_exec.policy_id, status, rep_exec.status_text, 0, trigger, rep_exec.start_time, rep_exec.end_time) RETURNING id INTO new_exec_id;
|
||||
UPDATE replication_execution SET new_execution_id=new_exec_id WHERE id=rep_exec.id;
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
/*move the replication task records into the new task table*/
|
||||
DO $$
|
||||
DECLARE
|
||||
rep_task RECORD;
|
||||
status varchar(32);
|
||||
status_code integer;
|
||||
BEGIN
|
||||
FOR rep_task IN SELECT * FROM replication_task
|
||||
LOOP
|
||||
IF rep_task.status = 'InProgress' THEN
|
||||
status = 'Running';
|
||||
status_code = 2;
|
||||
ELSIF rep_task.status = 'Stopped' THEN
|
||||
status = 'Stopped';
|
||||
status_code = 3;
|
||||
ELSIF rep_task.status = 'Failed' THEN
|
||||
status = 'Error';
|
||||
status_code = 3;
|
||||
ELSIF rep_task.status = 'Succeed' THEN
|
||||
status = 'Success';
|
||||
status_code = 3;
|
||||
ELSE
|
||||
status = 'Pending';
|
||||
status_code = 0;
|
||||
END IF;
|
||||
INSERT INTO task (vendor_type, execution_id, job_id, status, status_code, status_revision,
|
||||
run_count, extra_attrs, creation_time, start_time, update_time, end_time)
|
||||
VALUES ('REPLICATION', (SELECT new_execution_id FROM replication_execution WHERE id=rep_task.execution_id),
|
||||
rep_task.job_id, status, status_code, rep_task.status_revision,
|
||||
1, CONCAT('{"resource_type":"', rep_task.resource_type,'","source_resource":"', rep_task.src_resource, '","destination_resource":"', rep_task.dst_resource, '","operation":"', rep_task.operation,'"}')::json,
|
||||
rep_task.start_time, rep_task.start_time, rep_task.end_time, rep_task.end_time);
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
DROP TABLE IF EXISTS replication_task;
|
||||
DROP TABLE IF EXISTS replication_execution;
|
||||
|
||||
/*move the replication schedule job records into the new schedule table*/
|
||||
DO $$
|
||||
DECLARE
|
||||
schd RECORD;
|
||||
new_schd_id integer;
|
||||
exec_id integer;
|
||||
exec_status varchar(32);
|
||||
task_status varchar(32);
|
||||
task_status_code integer;
|
||||
BEGIN
|
||||
FOR schd IN SELECT * FROM replication_schedule_job
|
||||
LOOP
|
||||
INSERT INTO schedule (vendor_type, vendor_id, cron, callback_func_name,
|
||||
callback_func_param, creation_time, update_time)
|
||||
VALUES ('REPLICATION', schd.policy_id,
|
||||
(SELECT trigger::json->'trigger_settings'->>'cron' FROM replication_policy WHERE id=schd.policy_id),
|
||||
'REPLICATION_CALLBACK', schd.policy_id, schd.creation_time, schd.update_time) RETURNING id INTO new_schd_id;
|
||||
IF schd.status = 'stopped' THEN
|
||||
exec_status = 'Stopped';
|
||||
task_status = 'Stopped';
|
||||
task_status_code = 3;
|
||||
ELSIF schd.status = 'error' THEN
|
||||
exec_status = 'Error';
|
||||
task_status = 'Error';
|
||||
task_status_code = 3;
|
||||
ELSIF schd.status = 'finished' THEN
|
||||
exec_status = 'Success';
|
||||
task_status = 'Success';
|
||||
task_status_code = 3;
|
||||
ELSIF schd.status = 'running' THEN
|
||||
exec_status = 'Running';
|
||||
task_status = 'Running';
|
||||
task_status_code = 2;
|
||||
ELSEIF schd.status = 'pending' THEN
|
||||
exec_status = 'Running';
|
||||
task_status = 'Pending';
|
||||
task_status_code = 0;
|
||||
ELSEIF schd.status = 'scheduled' THEN
|
||||
exec_status = 'Running';
|
||||
task_status = 'Scheduled';
|
||||
task_status_code = 1;
|
||||
ELSE
|
||||
exec_status = 'Running';
|
||||
task_status = 'Pending';
|
||||
task_status_code = 0;
|
||||
END IF;
|
||||
INSERT INTO execution (vendor_type, vendor_id, status, revision, trigger, start_time, end_time)
|
||||
VALUES ('SCHEDULER', new_schd_id, exec_status, 0, 'MANUAL', schd.creation_time, schd.update_time) RETURNING id INTO exec_id;
|
||||
INSERT INTO task (vendor_type, execution_id, job_id, status, status_code, status_revision, run_count, creation_time, start_time, update_time, end_time)
|
||||
VALUES ('SCHEDULER',exec_id, schd.job_id, task_status, task_status_code, 0, 1, schd.creation_time, schd.creation_time, schd.update_time, schd.update_time);
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
DROP TABLE IF EXISTS replication_schedule_job;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/goharbor/harbor/src/controller/event"
|
||||
"github.com/goharbor/harbor/src/controller/event/handler/auditlog"
|
||||
"github.com/goharbor/harbor/src/controller/event/handler/internal"
|
||||
|
|
@ -10,8 +11,12 @@ import (
|
|||
"github.com/goharbor/harbor/src/controller/event/handler/webhook/chart"
|
||||
"github.com/goharbor/harbor/src/controller/event/handler/webhook/quota"
|
||||
"github.com/goharbor/harbor/src/controller/event/handler/webhook/scan"
|
||||
"github.com/goharbor/harbor/src/controller/event/metadata"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/orm"
|
||||
"github.com/goharbor/harbor/src/pkg/notification"
|
||||
"github.com/goharbor/harbor/src/pkg/notifier"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
@ -54,4 +59,12 @@ func init() {
|
|||
// internal
|
||||
notifier.Subscribe(event.TopicPullArtifact, &internal.Handler{Context: orm.Context})
|
||||
notifier.Subscribe(event.TopicPushArtifact, &internal.Handler{Context: orm.Context})
|
||||
|
||||
task.RegisterTaskStatusChangePostFunc(job.Replication, func(ctx context.Context, taskID int64, status string) error {
|
||||
notification.AddEvent(ctx, &metadata.ReplicationMetaData{
|
||||
ReplicationTaskID: taskID,
|
||||
Status: status,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,13 +10,14 @@ import (
|
|||
"github.com/goharbor/harbor/src/controller/event/handler/util"
|
||||
ctlModel "github.com/goharbor/harbor/src/controller/event/model"
|
||||
"github.com/goharbor/harbor/src/controller/project"
|
||||
"github.com/goharbor/harbor/src/controller/replication"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/lib/orm"
|
||||
"github.com/goharbor/harbor/src/pkg/notification"
|
||||
"github.com/goharbor/harbor/src/pkg/notifier/model"
|
||||
"github.com/goharbor/harbor/src/replication"
|
||||
rep "github.com/goharbor/harbor/src/replication"
|
||||
rpModel "github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
|
|
@ -71,25 +72,20 @@ func (r *ReplicationHandler) IsStateful() bool {
|
|||
}
|
||||
|
||||
func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, *commonModels.Project, error) {
|
||||
task, err := replication.OperationCtl.GetTask(event.ReplicationTaskID)
|
||||
ctx := orm.Context()
|
||||
task, err := replication.Ctl.GetTask(ctx, event.ReplicationTaskID)
|
||||
if err != nil {
|
||||
log.Errorf("failed to get replication task %d: error: %v", event.ReplicationTaskID, err)
|
||||
return nil, nil, err
|
||||
}
|
||||
if task == nil {
|
||||
return nil, nil, fmt.Errorf("task %d not found with replication event", event.ReplicationTaskID)
|
||||
}
|
||||
|
||||
execution, err := replication.OperationCtl.GetExecution(task.ExecutionID)
|
||||
execution, err := replication.Ctl.GetExecution(ctx, task.ExecutionID)
|
||||
if err != nil {
|
||||
log.Errorf("failed to get replication execution %d: error: %v", task.ExecutionID, err)
|
||||
return nil, nil, err
|
||||
}
|
||||
if execution == nil {
|
||||
return nil, nil, fmt.Errorf("execution %d not found with replication event", task.ExecutionID)
|
||||
}
|
||||
|
||||
rpPolicy, err := replication.PolicyCtl.Get(execution.PolicyID)
|
||||
rpPolicy, err := rep.PolicyCtl.Get(execution.PolicyID)
|
||||
if err != nil {
|
||||
log.Errorf("failed to get replication policy %d: error: %v", execution.PolicyID, err)
|
||||
return nil, nil, err
|
||||
|
|
@ -107,7 +103,7 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
|
|||
remoteRegID = rpPolicy.DestRegistry.ID
|
||||
}
|
||||
|
||||
remoteRegistry, err := replication.RegistryMgr.Get(remoteRegID)
|
||||
remoteRegistry, err := rep.RegistryMgr.Get(remoteRegID)
|
||||
if err != nil {
|
||||
log.Errorf("failed to get replication remoteRegistry registry %d: error: %v", remoteRegID, err)
|
||||
return nil, nil, err
|
||||
|
|
@ -116,8 +112,8 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
|
|||
return nil, nil, fmt.Errorf("registry %d not found with replication event", remoteRegID)
|
||||
}
|
||||
|
||||
srcNamespace, srcNameAndTag := getMetadataFromResource(task.SrcResource)
|
||||
destNamespace, destNameAndTag := getMetadataFromResource(task.DstResource)
|
||||
srcNamespace, srcNameAndTag := getMetadataFromResource(task.SourceResource)
|
||||
destNamespace, destNameAndTag := getMetadataFromResource(task.DestinationResource)
|
||||
|
||||
extURL, err := config.ExtURL()
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -22,13 +22,14 @@ import (
|
|||
"github.com/goharbor/harbor/src/common/models"
|
||||
"github.com/goharbor/harbor/src/controller/event"
|
||||
"github.com/goharbor/harbor/src/controller/project"
|
||||
rep "github.com/goharbor/harbor/src/controller/replication"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/notification"
|
||||
"github.com/goharbor/harbor/src/replication"
|
||||
daoModels "github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
projecttesting "github.com/goharbor/harbor/src/testing/controller/project"
|
||||
replicationtesting "github.com/goharbor/harbor/src/testing/controller/replication"
|
||||
"github.com/goharbor/harbor/src/testing/mock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
|
@ -40,9 +41,6 @@ type fakedNotificationPolicyMgr struct {
|
|||
type fakedReplicationPolicyMgr struct {
|
||||
}
|
||||
|
||||
type fakedReplicationMgr struct {
|
||||
}
|
||||
|
||||
type fakedReplicationRegistryMgr struct {
|
||||
}
|
||||
|
||||
|
|
@ -89,46 +87,6 @@ func (f *fakedNotificationPolicyMgr) GetRelatedPolices(int64, string) ([]*models
|
|||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakedReplicationMgr) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (f *fakedReplicationMgr) StopReplication(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedReplicationMgr) ListExecutions(...*daoModels.ExecutionQuery) (int64, []*daoModels.Execution, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedReplicationMgr) GetExecution(int64) (*daoModels.Execution, error) {
|
||||
return &daoModels.Execution{
|
||||
PolicyID: 1,
|
||||
Trigger: "manual",
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedReplicationMgr) ListTasks(...*daoModels.TaskQuery) (int64, []*daoModels.Task, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedReplicationMgr) GetTask(id int64) (*daoModels.Task, error) {
|
||||
if id == 1 {
|
||||
return &daoModels.Task{
|
||||
ExecutionID: 1,
|
||||
// project info not included when replicating with docker registry
|
||||
SrcResource: "alpine:[v1]",
|
||||
DstResource: "gxt/alpine:[v1] ",
|
||||
}, nil
|
||||
}
|
||||
return &daoModels.Task{
|
||||
ExecutionID: 1,
|
||||
SrcResource: "library/alpine:[v1]",
|
||||
DstResource: "gxt/alpine:[v1] ",
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedReplicationMgr) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedReplicationMgr) GetTaskLog(int64) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Create new policy
|
||||
func (f *fakedReplicationPolicyMgr) Create(*model.Policy) (int64, error) {
|
||||
return 0, nil
|
||||
|
|
@ -213,24 +171,27 @@ func TestReplicationHandler_Handle(t *testing.T) {
|
|||
config.Init()
|
||||
|
||||
PolicyMgr := notification.PolicyMgr
|
||||
execution := replication.OperationCtl
|
||||
rpPolicy := replication.PolicyCtl
|
||||
rpRegistry := replication.RegistryMgr
|
||||
prj := project.Ctl
|
||||
repCtl := rep.Ctl
|
||||
|
||||
defer func() {
|
||||
notification.PolicyMgr = PolicyMgr
|
||||
replication.OperationCtl = execution
|
||||
replication.PolicyCtl = rpPolicy
|
||||
replication.RegistryMgr = rpRegistry
|
||||
project.Ctl = prj
|
||||
rep.Ctl = repCtl
|
||||
}()
|
||||
notification.PolicyMgr = &fakedNotificationPolicyMgr{}
|
||||
replication.OperationCtl = &fakedReplicationMgr{}
|
||||
replication.PolicyCtl = &fakedReplicationPolicyMgr{}
|
||||
replication.RegistryMgr = &fakedReplicationRegistryMgr{}
|
||||
projectCtl := &projecttesting.Controller{}
|
||||
project.Ctl = projectCtl
|
||||
mockRepCtl := &replicationtesting.Controller{}
|
||||
rep.Ctl = mockRepCtl
|
||||
mockRepCtl.On("GetTask", mock.Anything, mock.Anything).Return(&rep.Task{}, nil)
|
||||
mockRepCtl.On("GetExecution", mock.Anything, mock.Anything).Return(&rep.Execution{}, nil)
|
||||
|
||||
mock.OnAnything(projectCtl, "GetByName").Return(&models.Project{ProjectID: 1}, nil)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,242 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package replication
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/controller/replication/flow"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/lib/orm"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
// Controller defines the operations related with replication
|
||||
type Controller interface {
|
||||
// Start the replication according to the policy
|
||||
Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (executionID int64, err error)
|
||||
// Stop the replication specified by the execution ID
|
||||
Stop(ctx context.Context, executionID int64) (err error)
|
||||
// ExecutionCount returns the total count of executions according to the query
|
||||
ExecutionCount(ctx context.Context, query *q.Query) (count int64, err error)
|
||||
// ListExecutions lists the executions according to the query
|
||||
ListExecutions(ctx context.Context, query *q.Query) (executions []*Execution, err error)
|
||||
// GetExecution gets the specific execution
|
||||
GetExecution(ctx context.Context, executionID int64) (execution *Execution, err error)
|
||||
// TaskCount returns the total count of tasks according to the query
|
||||
TaskCount(ctx context.Context, query *q.Query) (count int64, err error)
|
||||
// ListTasks lists the tasks according to the query
|
||||
ListTasks(ctx context.Context, query *q.Query) (tasks []*Task, err error)
|
||||
// GetTask gets the specific task
|
||||
GetTask(ctx context.Context, taskID int64) (task *Task, err error)
|
||||
// GetTaskLog gets the log of the specific task
|
||||
GetTaskLog(ctx context.Context, taskID int64) (log []byte, err error)
|
||||
}
|
||||
|
||||
var (
|
||||
// Ctl is a global replication controller instance
|
||||
Ctl = NewController()
|
||||
_ Controller = &controller{}
|
||||
)
|
||||
|
||||
// NewController creates a new instance of the replication controller
|
||||
func NewController() Controller {
|
||||
return &controller{
|
||||
execMgr: task.ExecMgr,
|
||||
taskMgr: task.Mgr,
|
||||
flowCtl: flow.NewController(),
|
||||
ormCreator: orm.Crt,
|
||||
wp: lib.NewWorkerPool(1024),
|
||||
}
|
||||
}
|
||||
|
||||
type controller struct {
|
||||
execMgr task.ExecutionManager
|
||||
taskMgr task.Manager
|
||||
flowCtl flow.Controller
|
||||
ormCreator orm.Creator
|
||||
wp *lib.WorkerPool
|
||||
}
|
||||
|
||||
func (c *controller) Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (int64, error) {
|
||||
logger := log.GetLogger(ctx)
|
||||
if !policy.Enabled {
|
||||
return 0, errors.New(nil).WithCode(errors.PreconditionCode).
|
||||
WithMessage("the policy %d is disabled", policy.ID)
|
||||
}
|
||||
// create an execution record
|
||||
id, err := c.execMgr.Create(ctx, job.Replication, policy.ID, trigger)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
c.wp.GetWorker()
|
||||
// start the replication flow in background
|
||||
go func() {
|
||||
defer c.wp.ReleaseWorker()
|
||||
// as the process runs inside a goroutine, the transaction in the outer ctx
|
||||
// may be submitted already when the process starts, so create a new context
|
||||
// with orm populated
|
||||
ctxx := orm.NewContext(context.Background(), c.ormCreator.Create())
|
||||
err := c.flowCtl.Start(ctxx, id, policy, resource)
|
||||
if err == nil {
|
||||
// no err, return directly
|
||||
return
|
||||
}
|
||||
// got error, try to stop the execution first in case that some tasks are already created
|
||||
if err := c.execMgr.StopAndWait(ctxx, id, 10*time.Second); err != nil {
|
||||
logger.Errorf("failed to stop the execution %d: %v", id, err)
|
||||
}
|
||||
if err := c.execMgr.MarkError(ctxx, id, err.Error()); err != nil {
|
||||
logger.Errorf("failed to mark error for the execution %d: %v", id, err)
|
||||
}
|
||||
}()
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (c *controller) Stop(ctx context.Context, id int64) error {
|
||||
return c.execMgr.Stop(ctx, id)
|
||||
}
|
||||
|
||||
func (c *controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
|
||||
query = q.MustClone(query)
|
||||
query.Keywords["VendorType"] = job.Replication
|
||||
return c.execMgr.Count(ctx, query)
|
||||
}
|
||||
|
||||
func (c *controller) ListExecutions(ctx context.Context, query *q.Query) ([]*Execution, error) {
|
||||
// as the following logic may change the content of the query, clone it first
|
||||
query = q.MustClone(query)
|
||||
query.Keywords["VendorType"] = job.Replication
|
||||
// convert the query keyword "PolicyID" or "policy_id" to the "VendorID"
|
||||
if value, exist := query.Keywords["PolicyID"]; exist {
|
||||
query.Keywords["VendorID"] = value
|
||||
delete(query.Keywords, "PolicyID")
|
||||
}
|
||||
if value, exist := query.Keywords["policy_id"]; exist {
|
||||
query.Keywords["VendorID"] = value
|
||||
delete(query.Keywords, "policy_id")
|
||||
}
|
||||
|
||||
execs, err := c.execMgr.List(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var executions []*Execution
|
||||
for _, exec := range execs {
|
||||
executions = append(executions, convertExecution(exec))
|
||||
}
|
||||
return executions, nil
|
||||
}
|
||||
|
||||
func (c *controller) GetExecution(ctx context.Context, id int64) (*Execution, error) {
|
||||
execs, err := c.execMgr.List(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ID": id,
|
||||
"VendorType": job.Replication,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(execs) == 0 {
|
||||
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
|
||||
WithMessage("replication execution %d not found", id)
|
||||
}
|
||||
return convertExecution(execs[0]), nil
|
||||
}
|
||||
|
||||
func (c *controller) TaskCount(ctx context.Context, query *q.Query) (int64, error) {
|
||||
query = q.MustClone(query)
|
||||
query.Keywords["VendorType"] = job.Replication
|
||||
return c.taskMgr.Count(ctx, query)
|
||||
}
|
||||
|
||||
func (c *controller) ListTasks(ctx context.Context, query *q.Query) ([]*Task, error) {
|
||||
query = q.MustClone(query)
|
||||
query.Keywords["VendorType"] = job.Replication
|
||||
tks, err := c.taskMgr.List(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var tasks []*Task
|
||||
for _, tk := range tks {
|
||||
tasks = append(tasks, convertTask(tk))
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (c *controller) GetTask(ctx context.Context, id int64) (*Task, error) {
|
||||
tasks, err := c.taskMgr.List(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ID": id,
|
||||
"VendorType": job.Replication,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
|
||||
WithMessage("replication task %d not found", id)
|
||||
}
|
||||
return convertTask(tasks[0]), nil
|
||||
}
|
||||
|
||||
func (c *controller) GetTaskLog(ctx context.Context, id int64) ([]byte, error) {
|
||||
// make sure the task specified by ID is replication task
|
||||
_, err := c.GetTask(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.taskMgr.GetLog(ctx, id)
|
||||
}
|
||||
|
||||
func convertExecution(exec *task.Execution) *Execution {
|
||||
return &Execution{
|
||||
ID: exec.ID,
|
||||
PolicyID: exec.VendorID,
|
||||
Status: exec.Status,
|
||||
StatusMessage: exec.StatusMessage,
|
||||
Metrics: exec.Metrics,
|
||||
Trigger: exec.Trigger,
|
||||
StartTime: exec.StartTime,
|
||||
EndTime: exec.EndTime,
|
||||
}
|
||||
}
|
||||
|
||||
func convertTask(task *task.Task) *Task {
|
||||
return &Task{
|
||||
ID: task.ID,
|
||||
ExecutionID: task.ExecutionID,
|
||||
Status: task.Status,
|
||||
StatusMessage: task.StatusMessage,
|
||||
RunCount: task.RunCount,
|
||||
ResourceType: task.GetStringFromExtraAttrs("resource_type"),
|
||||
SourceResource: task.GetStringFromExtraAttrs("source_resource"),
|
||||
DestinationResource: task.GetStringFromExtraAttrs("destination_resource"),
|
||||
Operation: task.GetStringFromExtraAttrs("operation"),
|
||||
JobID: task.JobID,
|
||||
CreationTime: task.CreationTime,
|
||||
StartTime: task.StartTime,
|
||||
UpdateTime: task.UpdateTime,
|
||||
EndTime: task.EndTime,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,223 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package replication
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/pkg/task/dao"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/testing/lib/orm"
|
||||
"github.com/goharbor/harbor/src/testing/mock"
|
||||
testingTask "github.com/goharbor/harbor/src/testing/pkg/task"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type replicationTestSuite struct {
|
||||
suite.Suite
|
||||
ctl *controller
|
||||
execMgr *testingTask.ExecutionManager
|
||||
taskMgr *testingTask.Manager
|
||||
flowCtl *flowController
|
||||
ormCreator *orm.Creator
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) SetupSuite() {
|
||||
r.execMgr = &testingTask.ExecutionManager{}
|
||||
r.taskMgr = &testingTask.Manager{}
|
||||
r.flowCtl = &flowController{}
|
||||
r.ormCreator = &orm.Creator{}
|
||||
r.ctl = &controller{
|
||||
execMgr: r.execMgr,
|
||||
taskMgr: r.taskMgr,
|
||||
flowCtl: r.flowCtl,
|
||||
ormCreator: r.ormCreator,
|
||||
wp: lib.NewWorkerPool(1024),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestStart() {
|
||||
// policy is disabled
|
||||
id, err := r.ctl.Start(context.Background(), &model.Policy{Enabled: false}, nil, task.ExecutionTriggerManual)
|
||||
r.Require().NotNil(err)
|
||||
|
||||
// got error when running the replication flow
|
||||
r.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
r.execMgr.On("StopAndWait", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
r.execMgr.On("MarkError", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
r.flowCtl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("error"))
|
||||
r.ormCreator.On("Create").Return(nil)
|
||||
id, err = r.ctl.Start(context.Background(), &model.Policy{Enabled: true}, nil, task.ExecutionTriggerManual)
|
||||
r.Require().Nil(err)
|
||||
r.Equal(int64(1), id)
|
||||
time.Sleep(1 * time.Second) // wait the functions called in the goroutine
|
||||
r.execMgr.AssertExpectations(r.T())
|
||||
r.flowCtl.AssertExpectations(r.T())
|
||||
r.ormCreator.AssertExpectations(r.T())
|
||||
|
||||
// reset the mocks
|
||||
r.SetupSuite()
|
||||
|
||||
// got no error when running the replication flow
|
||||
r.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
r.flowCtl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
r.ormCreator.On("Create").Return(nil)
|
||||
id, err = r.ctl.Start(context.Background(), &model.Policy{Enabled: true}, nil, task.ExecutionTriggerManual)
|
||||
r.Require().Nil(err)
|
||||
r.Equal(int64(1), id)
|
||||
time.Sleep(1 * time.Second) // wait the functions called in the goroutine
|
||||
r.execMgr.AssertExpectations(r.T())
|
||||
r.flowCtl.AssertExpectations(r.T())
|
||||
r.ormCreator.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestStop() {
|
||||
r.execMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||
err := r.ctl.Stop(nil, 1)
|
||||
r.Require().Nil(err)
|
||||
r.execMgr.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestExecutionCount() {
|
||||
r.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
total, err := r.ctl.ExecutionCount(nil, nil)
|
||||
r.Require().Nil(err)
|
||||
r.Equal(int64(1), total)
|
||||
r.execMgr.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestListExecutions() {
|
||||
r.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
|
||||
{
|
||||
ID: 1,
|
||||
VendorType: job.Replication,
|
||||
VendorID: 1,
|
||||
Status: job.RunningStatus.String(),
|
||||
Metrics: &dao.Metrics{
|
||||
TaskCount: 1,
|
||||
RunningTaskCount: 1,
|
||||
},
|
||||
Trigger: task.ExecutionTriggerManual,
|
||||
StartTime: time.Time{},
|
||||
EndTime: time.Time{},
|
||||
},
|
||||
}, nil)
|
||||
executions, err := r.ctl.ListExecutions(nil, nil)
|
||||
r.Require().Nil(err)
|
||||
r.Require().Len(executions, 1)
|
||||
r.Equal(int64(1), executions[0].ID)
|
||||
r.Equal(int64(1), executions[0].PolicyID)
|
||||
r.execMgr.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestGetExecution() {
|
||||
r.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
|
||||
{
|
||||
ID: 1,
|
||||
VendorType: job.Replication,
|
||||
VendorID: 1,
|
||||
Status: job.RunningStatus.String(),
|
||||
Metrics: &dao.Metrics{
|
||||
TaskCount: 1,
|
||||
RunningTaskCount: 1,
|
||||
},
|
||||
Trigger: task.ExecutionTriggerManual,
|
||||
StartTime: time.Time{},
|
||||
EndTime: time.Time{},
|
||||
},
|
||||
}, nil)
|
||||
execution, err := r.ctl.GetExecution(nil, 1)
|
||||
r.Require().Nil(err)
|
||||
r.Equal(int64(1), execution.ID)
|
||||
r.Equal(int64(1), execution.PolicyID)
|
||||
r.execMgr.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestTaskCount() {
|
||||
r.taskMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
total, err := r.ctl.TaskCount(nil, nil)
|
||||
r.Require().Nil(err)
|
||||
r.Equal(int64(1), total)
|
||||
r.taskMgr.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestListTasks() {
|
||||
r.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
|
||||
{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
Status: job.RunningStatus.String(),
|
||||
ExtraAttrs: map[string]interface{}{
|
||||
"resource_type": "artifact",
|
||||
"source_resource": "library/hello-world",
|
||||
"destination_resource": "library/hello-world",
|
||||
"operation": "copy",
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
tasks, err := r.ctl.ListTasks(nil, nil)
|
||||
r.Require().Nil(err)
|
||||
r.Require().Len(tasks, 1)
|
||||
r.Equal(int64(1), tasks[0].ID)
|
||||
r.Equal(int64(1), tasks[0].ExecutionID)
|
||||
r.Equal("artifact", tasks[0].ResourceType)
|
||||
r.Equal("library/hello-world", tasks[0].SourceResource)
|
||||
r.Equal("library/hello-world", tasks[0].DestinationResource)
|
||||
r.Equal("copy", tasks[0].Operation)
|
||||
r.taskMgr.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestGetTask() {
|
||||
r.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
|
||||
{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
Status: job.RunningStatus.String(),
|
||||
ExtraAttrs: map[string]interface{}{
|
||||
"resource_type": "artifact",
|
||||
"source_resource": "library/hello-world",
|
||||
"destination_resource": "library/hello-world",
|
||||
"operation": "copy",
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
task, err := r.ctl.GetTask(nil, 1)
|
||||
r.Require().Nil(err)
|
||||
r.Equal(int64(1), task.ID)
|
||||
r.Equal(int64(1), task.ExecutionID)
|
||||
r.Equal("artifact", task.ResourceType)
|
||||
r.Equal("library/hello-world", task.SourceResource)
|
||||
r.Equal("library/hello-world", task.DestinationResource)
|
||||
r.Equal("copy", task.Operation)
|
||||
r.taskMgr.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func (r *replicationTestSuite) TestGetTaskLog() {
|
||||
r.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte{'a'}, nil)
|
||||
data, err := r.ctl.GetTaskLog(nil, 1)
|
||||
r.Require().Nil(err)
|
||||
r.Equal([]byte{'a'}, data)
|
||||
r.taskMgr.AssertExpectations(r.T())
|
||||
}
|
||||
|
||||
func TestReplicationTestSuite(t *testing.T) {
|
||||
suite.Run(t, &replicationTestSuite{})
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
// Flow defines a specific replication flow
|
||||
type Flow interface {
|
||||
Run(ctx context.Context) (err error)
|
||||
}
|
||||
|
||||
// Controller controls the replication flow
|
||||
type Controller interface {
|
||||
Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) (err error)
|
||||
}
|
||||
|
||||
// NewController returns an instance of the default flow controller
|
||||
func NewController() Controller {
|
||||
return &controller{}
|
||||
}
|
||||
|
||||
type controller struct{}
|
||||
|
||||
func (c *controller) Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) error {
|
||||
// deletion flow
|
||||
if resource != nil && resource.Deleted {
|
||||
return NewDeletionFlow(executionID, policy, resource).Run(ctx)
|
||||
}
|
||||
// copy flow
|
||||
resources := []*model.Resource{}
|
||||
if resource != nil {
|
||||
resources = append(resources, resource)
|
||||
}
|
||||
return NewCopyFlow(executionID, policy, resources...).Run(ctx)
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
type copyFlow struct {
|
||||
executionID int64
|
||||
resources []*model.Resource
|
||||
policy *model.Policy
|
||||
executionMgr task.ExecutionManager
|
||||
taskMgr task.Manager
|
||||
}
|
||||
|
||||
// NewCopyFlow returns an instance of the copy flow which replicates the resources from
|
||||
// the source registry to the destination registry. If the parameter "resources" isn't provided,
|
||||
// will fetch the resources first
|
||||
func NewCopyFlow(executionID int64, policy *model.Policy, resources ...*model.Resource) Flow {
|
||||
return ©Flow{
|
||||
executionMgr: task.ExecMgr,
|
||||
taskMgr: task.Mgr,
|
||||
executionID: executionID,
|
||||
policy: policy,
|
||||
resources: resources,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *copyFlow) Run(ctx context.Context) error {
|
||||
logger := log.GetLogger(ctx)
|
||||
srcAdapter, dstAdapter, err := initialize(c.policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var srcResources []*model.Resource
|
||||
if len(c.resources) > 0 {
|
||||
srcResources, err = filterResources(c.resources, c.policy.Filters)
|
||||
} else {
|
||||
srcResources, err = fetchResources(srcAdapter, c.policy)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isStopped, err := c.isExecutionStopped(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isStopped {
|
||||
logger.Debugf("the execution %d is stopped, stop the flow", c.executionID)
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(srcResources) == 0 {
|
||||
// no candidates, mark the execution as done directly
|
||||
if err := c.executionMgr.MarkDone(ctx, c.executionID, "no resources need to be replicated"); err != nil {
|
||||
logger.Errorf("failed to mark done for the execution %d: %v", c.executionID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
srcResources = assembleSourceResources(srcResources, c.policy)
|
||||
dstResources := assembleDestinationResources(srcResources, c.policy)
|
||||
|
||||
if err = prepareForPush(dstAdapter, dstResources); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return c.createTasks(ctx, srcResources, dstResources)
|
||||
}
|
||||
|
||||
func (c *copyFlow) isExecutionStopped(ctx context.Context) (bool, error) {
|
||||
execution, err := c.executionMgr.Get(ctx, c.executionID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return execution.Status == job.StoppedStatus.String(), nil
|
||||
}
|
||||
|
||||
func (c *copyFlow) createTasks(ctx context.Context, srcResources, dstResources []*model.Resource) error {
|
||||
for i, resource := range srcResources {
|
||||
src, err := json.Marshal(resource)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dest, err := json.Marshal(dstResources[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
job := &task.Job{
|
||||
Name: job.Replication,
|
||||
Metadata: &job.Metadata{
|
||||
JobKind: job.KindGeneric,
|
||||
},
|
||||
Parameters: map[string]interface{}{
|
||||
"src_resource": string(src),
|
||||
"dst_resource": string(dest),
|
||||
},
|
||||
}
|
||||
|
||||
if _, err = c.taskMgr.Create(ctx, c.executionID, job, map[string]interface{}{
|
||||
"operation": "copy",
|
||||
"resource_type": string(resource.Type),
|
||||
"source_resource": getResourceName(resource),
|
||||
"destination_resource": getResourceName(dstResources[i])}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/replication/adapter"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
testingTask "github.com/goharbor/harbor/src/testing/pkg/task"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type copyFlowTestSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func (c *copyFlowTestSuite) TestRun() {
|
||||
adp := &mockAdapter{}
|
||||
factory := &mockFactory{}
|
||||
factory.On("AdapterPattern").Return(nil)
|
||||
factory.On("Create", mock.Anything).Return(adp, nil)
|
||||
adapter.RegisterFactory("TEST_FOR_COPY_FLOW", factory)
|
||||
|
||||
adp.On("Info").Return(&model.RegistryInfo{
|
||||
SupportedResourceTypes: []model.ResourceType{
|
||||
model.ResourceTypeArtifact,
|
||||
},
|
||||
}, nil)
|
||||
adp.On("FetchArtifacts", mock.Anything).Return([]*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}, nil)
|
||||
adp.On("PrepareForPush", mock.Anything).Return(nil)
|
||||
|
||||
execMgr := &testingTask.ExecutionManager{}
|
||||
execMgr.On("Get", mock.Anything, mock.Anything).Return(&task.Execution{
|
||||
Status: job.RunningStatus.String(),
|
||||
}, nil)
|
||||
|
||||
taskMgr := &testingTask.Manager{}
|
||||
taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
policy := &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
Type: "TEST_FOR_COPY_FLOW",
|
||||
},
|
||||
DestRegistry: &model.Registry{
|
||||
Type: "TEST_FOR_COPY_FLOW",
|
||||
},
|
||||
}
|
||||
flow := ©Flow{
|
||||
executionID: 1,
|
||||
policy: policy,
|
||||
executionMgr: execMgr,
|
||||
taskMgr: taskMgr,
|
||||
}
|
||||
err := flow.Run(context.Background())
|
||||
c.Require().Nil(err)
|
||||
}
|
||||
|
||||
func TestCopyFlowTestSuite(t *testing.T) {
|
||||
suite.Run(t, ©FlowTestSuite{})
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
type deletionFlow struct {
|
||||
executionID int64
|
||||
policy *model.Policy
|
||||
executionMgr task.ExecutionManager
|
||||
taskMgr task.Manager
|
||||
resources []*model.Resource
|
||||
}
|
||||
|
||||
// NewDeletionFlow returns an instance of the delete flow which deletes the resources
|
||||
// on the destination registry
|
||||
func NewDeletionFlow(executionID int64, policy *model.Policy, resources ...*model.Resource) Flow {
|
||||
return &deletionFlow{
|
||||
executionMgr: task.ExecMgr,
|
||||
taskMgr: task.Mgr,
|
||||
executionID: executionID,
|
||||
policy: policy,
|
||||
resources: resources,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deletionFlow) Run(ctx context.Context) error {
|
||||
logger := log.GetLogger(ctx)
|
||||
srcResources, err := filterResources(d.resources, d.policy.Filters)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(srcResources) == 0 {
|
||||
// no candidates, mark the execution as done directly
|
||||
if err := d.executionMgr.MarkDone(ctx, d.executionID, "no resources need to be replicated"); err != nil {
|
||||
logger.Errorf("failed to mark done for the execution %d: %v", d.executionID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
srcResources = assembleSourceResources(srcResources, d.policy)
|
||||
dstResources := assembleDestinationResources(srcResources, d.policy)
|
||||
|
||||
return d.createTasks(ctx, srcResources, dstResources)
|
||||
}
|
||||
|
||||
func (d *deletionFlow) createTasks(ctx context.Context, srcResources, dstResources []*model.Resource) error {
|
||||
for i, resource := range srcResources {
|
||||
src, err := json.Marshal(resource)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dest, err := json.Marshal(dstResources[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
job := &task.Job{
|
||||
Name: job.Replication,
|
||||
Metadata: &job.Metadata{
|
||||
JobKind: job.KindGeneric,
|
||||
},
|
||||
Parameters: map[string]interface{}{
|
||||
"src_resource": string(src),
|
||||
"dst_resource": string(dest),
|
||||
},
|
||||
}
|
||||
|
||||
operation := "deletion"
|
||||
if dstResources[i].IsDeleteTag {
|
||||
operation = "tag deletion"
|
||||
}
|
||||
|
||||
if _, err = d.taskMgr.Create(ctx, d.executionID, job, map[string]interface{}{
|
||||
"operation": operation,
|
||||
"resource_type": string(resource.Type),
|
||||
"source_resource": getResourceName(resource),
|
||||
"destination_resource": getResourceName(dstResources[i])}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -15,16 +15,23 @@
|
|||
package flow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/goharbor/harbor/src/testing/pkg/task"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
func TestRunOfDeletionFlow(t *testing.T) {
|
||||
scheduler := &fakedScheduler{}
|
||||
executionMgr := &fakedExecutionManager{}
|
||||
type deletionFlowTestSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func (d *deletionFlowTestSuite) TestRun() {
|
||||
taskMgr := &task.Manager{}
|
||||
taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
|
||||
policy := &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
|
|
@ -47,8 +54,16 @@ func TestRunOfDeletionFlow(t *testing.T) {
|
|||
},
|
||||
},
|
||||
}
|
||||
flow := NewDeletionFlow(executionMgr, scheduler, 1, policy, resources...)
|
||||
n, err := flow.Run(nil)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
flow := &deletionFlow{
|
||||
executionID: 1,
|
||||
policy: policy,
|
||||
taskMgr: taskMgr,
|
||||
resources: resources,
|
||||
}
|
||||
err := flow.Run(context.Background())
|
||||
d.Require().Nil(err)
|
||||
}
|
||||
|
||||
func TestDeletionFlowTestSuite(t *testing.T) {
|
||||
suite.Run(t, &deletionFlowTestSuite{})
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"github.com/goharbor/harbor/src/replication/adapter"
|
||||
)
|
||||
|
||||
// define a new interface to combine the two interfaces of adapter for mockery to generate the mocks
|
||||
type registryAdapter interface {
|
||||
adapter.Adapter
|
||||
adapter.ArtifactRegistry
|
||||
}
|
||||
|
||||
//go:generate mockery --dir . --name registryAdapter --output . --outpkg flow --filename mock_adapter_test.go --structname mockAdapter
|
||||
//go:generate mockery --dir ../../../replication/adapter --name Factory --output . --outpkg flow --filename mock_adapter_factory_test.go --structname mockFactory
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
// Code generated by mockery v2.1.0. DO NOT EDIT.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
adapter "github.com/goharbor/harbor/src/replication/adapter"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
model "github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
// mockFactory is an autogenerated mock type for the Factory type
|
||||
type mockFactory struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// AdapterPattern provides a mock function with given fields:
|
||||
func (_m *mockFactory) AdapterPattern() *model.AdapterPattern {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 *model.AdapterPattern
|
||||
if rf, ok := ret.Get(0).(func() *model.AdapterPattern); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*model.AdapterPattern)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// Create provides a mock function with given fields: _a0
|
||||
func (_m *mockFactory) Create(_a0 *model.Registry) (adapter.Adapter, error) {
|
||||
ret := _m.Called(_a0)
|
||||
|
||||
var r0 adapter.Adapter
|
||||
if rf, ok := ret.Get(0).(func(*model.Registry) adapter.Adapter); ok {
|
||||
r0 = rf(_a0)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(adapter.Adapter)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(*model.Registry) error); ok {
|
||||
r1 = rf(_a0)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
|
@ -0,0 +1,278 @@
|
|||
// Code generated by mockery v2.1.0. DO NOT EDIT.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
distribution "github.com/docker/distribution"
|
||||
|
||||
io "io"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
model "github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
// mockAdapter is an autogenerated mock type for the registryAdapter type
|
||||
type mockAdapter struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// BlobExist provides a mock function with given fields: repository, digest
|
||||
func (_m *mockAdapter) BlobExist(repository string, digest string) (bool, error) {
|
||||
ret := _m.Called(repository, digest)
|
||||
|
||||
var r0 bool
|
||||
if rf, ok := ret.Get(0).(func(string, string) bool); ok {
|
||||
r0 = rf(repository, digest)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string, string) error); ok {
|
||||
r1 = rf(repository, digest)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// DeleteManifest provides a mock function with given fields: repository, reference
|
||||
func (_m *mockAdapter) DeleteManifest(repository string, reference string) error {
|
||||
ret := _m.Called(repository, reference)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string) error); ok {
|
||||
r0 = rf(repository, reference)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// DeleteTag provides a mock function with given fields: repository, tag
|
||||
func (_m *mockAdapter) DeleteTag(repository string, tag string) error {
|
||||
ret := _m.Called(repository, tag)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string) error); ok {
|
||||
r0 = rf(repository, tag)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// FetchArtifacts provides a mock function with given fields: filters
|
||||
func (_m *mockAdapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) {
|
||||
ret := _m.Called(filters)
|
||||
|
||||
var r0 []*model.Resource
|
||||
if rf, ok := ret.Get(0).(func([]*model.Filter) []*model.Resource); ok {
|
||||
r0 = rf(filters)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]*model.Resource)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func([]*model.Filter) error); ok {
|
||||
r1 = rf(filters)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// HealthCheck provides a mock function with given fields:
|
||||
func (_m *mockAdapter) HealthCheck() (model.HealthStatus, error) {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 model.HealthStatus
|
||||
if rf, ok := ret.Get(0).(func() model.HealthStatus); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Get(0).(model.HealthStatus)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func() error); ok {
|
||||
r1 = rf()
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Info provides a mock function with given fields:
|
||||
func (_m *mockAdapter) Info() (*model.RegistryInfo, error) {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 *model.RegistryInfo
|
||||
if rf, ok := ret.Get(0).(func() *model.RegistryInfo); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*model.RegistryInfo)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func() error); ok {
|
||||
r1 = rf()
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ManifestExist provides a mock function with given fields: repository, reference
|
||||
func (_m *mockAdapter) ManifestExist(repository string, reference string) (bool, string, error) {
|
||||
ret := _m.Called(repository, reference)
|
||||
|
||||
var r0 bool
|
||||
if rf, ok := ret.Get(0).(func(string, string) bool); ok {
|
||||
r0 = rf(repository, reference)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
var r1 string
|
||||
if rf, ok := ret.Get(1).(func(string, string) string); ok {
|
||||
r1 = rf(repository, reference)
|
||||
} else {
|
||||
r1 = ret.Get(1).(string)
|
||||
}
|
||||
|
||||
var r2 error
|
||||
if rf, ok := ret.Get(2).(func(string, string) error); ok {
|
||||
r2 = rf(repository, reference)
|
||||
} else {
|
||||
r2 = ret.Error(2)
|
||||
}
|
||||
|
||||
return r0, r1, r2
|
||||
}
|
||||
|
||||
// PrepareForPush provides a mock function with given fields: _a0
|
||||
func (_m *mockAdapter) PrepareForPush(_a0 []*model.Resource) error {
|
||||
ret := _m.Called(_a0)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func([]*model.Resource) error); ok {
|
||||
r0 = rf(_a0)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// PullBlob provides a mock function with given fields: repository, digest
|
||||
func (_m *mockAdapter) PullBlob(repository string, digest string) (int64, io.ReadCloser, error) {
|
||||
ret := _m.Called(repository, digest)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(string, string) int64); ok {
|
||||
r0 = rf(repository, digest)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 io.ReadCloser
|
||||
if rf, ok := ret.Get(1).(func(string, string) io.ReadCloser); ok {
|
||||
r1 = rf(repository, digest)
|
||||
} else {
|
||||
if ret.Get(1) != nil {
|
||||
r1 = ret.Get(1).(io.ReadCloser)
|
||||
}
|
||||
}
|
||||
|
||||
var r2 error
|
||||
if rf, ok := ret.Get(2).(func(string, string) error); ok {
|
||||
r2 = rf(repository, digest)
|
||||
} else {
|
||||
r2 = ret.Error(2)
|
||||
}
|
||||
|
||||
return r0, r1, r2
|
||||
}
|
||||
|
||||
// PullManifest provides a mock function with given fields: repository, reference, accepttedMediaTypes
|
||||
func (_m *mockAdapter) PullManifest(repository string, reference string, accepttedMediaTypes ...string) (distribution.Manifest, string, error) {
|
||||
_va := make([]interface{}, len(accepttedMediaTypes))
|
||||
for _i := range accepttedMediaTypes {
|
||||
_va[_i] = accepttedMediaTypes[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, repository, reference)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
var r0 distribution.Manifest
|
||||
if rf, ok := ret.Get(0).(func(string, string, ...string) distribution.Manifest); ok {
|
||||
r0 = rf(repository, reference, accepttedMediaTypes...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(distribution.Manifest)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 string
|
||||
if rf, ok := ret.Get(1).(func(string, string, ...string) string); ok {
|
||||
r1 = rf(repository, reference, accepttedMediaTypes...)
|
||||
} else {
|
||||
r1 = ret.Get(1).(string)
|
||||
}
|
||||
|
||||
var r2 error
|
||||
if rf, ok := ret.Get(2).(func(string, string, ...string) error); ok {
|
||||
r2 = rf(repository, reference, accepttedMediaTypes...)
|
||||
} else {
|
||||
r2 = ret.Error(2)
|
||||
}
|
||||
|
||||
return r0, r1, r2
|
||||
}
|
||||
|
||||
// PushBlob provides a mock function with given fields: repository, digest, size, blob
|
||||
func (_m *mockAdapter) PushBlob(repository string, digest string, size int64, blob io.Reader) error {
|
||||
ret := _m.Called(repository, digest, size, blob)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string, int64, io.Reader) error); ok {
|
||||
r0 = rf(repository, digest, size, blob)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// PushManifest provides a mock function with given fields: repository, reference, mediaType, payload
|
||||
func (_m *mockAdapter) PushManifest(repository string, reference string, mediaType string, payload []byte) (string, error) {
|
||||
ret := _m.Called(repository, reference, mediaType, payload)
|
||||
|
||||
var r0 string
|
||||
if rf, ok := ret.Get(0).(func(string, string, string, []byte) string); ok {
|
||||
r0 = rf(repository, reference, mediaType, payload)
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string, string, string, []byte) error); ok {
|
||||
r1 = rf(repository, reference, mediaType, payload)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
|
@ -15,17 +15,12 @@
|
|||
package flow
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/goharbor/harbor/src/replication/filter"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
adp "github.com/goharbor/harbor/src/replication/adapter"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/filter"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/operation/execution"
|
||||
"github.com/goharbor/harbor/src/replication/operation/scheduler"
|
||||
"github.com/goharbor/harbor/src/replication/util"
|
||||
)
|
||||
|
||||
|
|
@ -171,108 +166,6 @@ func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// preprocess
|
||||
func preprocess(scheduler scheduler.Scheduler, srcResources, dstResources []*model.Resource) ([]*scheduler.ScheduleItem, error) {
|
||||
items, err := scheduler.Preprocess(srcResources, dstResources)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to preprocess the resources: %v", err)
|
||||
}
|
||||
log.Debug("preprocess the resources completed")
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// create task records in database
|
||||
func createTasks(mgr execution.Manager, executionID int64, items []*scheduler.ScheduleItem) error {
|
||||
for _, item := range items {
|
||||
operation := "copy"
|
||||
if item.DstResource.Deleted {
|
||||
operation = "deletion"
|
||||
if item.DstResource.IsDeleteTag {
|
||||
operation = "tag deletion"
|
||||
}
|
||||
}
|
||||
|
||||
task := &models.Task{
|
||||
ExecutionID: executionID,
|
||||
Status: models.TaskStatusInitialized,
|
||||
ResourceType: string(item.SrcResource.Type),
|
||||
SrcResource: getResourceName(item.SrcResource),
|
||||
DstResource: getResourceName(item.DstResource),
|
||||
Operation: operation,
|
||||
}
|
||||
|
||||
id, err := mgr.CreateTask(task)
|
||||
if err != nil {
|
||||
// if failed to create the task for one of the items,
|
||||
// the whole execution is marked as failure and all
|
||||
// the items will not be submitted
|
||||
return fmt.Errorf("failed to create task records for the execution %d: %v", executionID, err)
|
||||
}
|
||||
|
||||
item.TaskID = id
|
||||
log.Debugf("task record %d for the execution %d created", id, executionID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// schedule the replication tasks and update the task's status
|
||||
// returns the count of tasks which have been scheduled and the error
|
||||
func schedule(scheduler scheduler.Scheduler, executionMgr execution.Manager, items []*scheduler.ScheduleItem) (int, error) {
|
||||
results, err := scheduler.Schedule(items)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to schedule the tasks: %v", err)
|
||||
}
|
||||
|
||||
allFailed := true
|
||||
n := len(results)
|
||||
for _, result := range results {
|
||||
// if the task is failed to be submitted, update the status of the
|
||||
// task as failure
|
||||
now := time.Now()
|
||||
if result.Error != nil {
|
||||
log.Errorf("failed to schedule the task %d: %v", result.TaskID, result.Error)
|
||||
if err = executionMgr.UpdateTask(&models.Task{
|
||||
ID: result.TaskID,
|
||||
Status: models.TaskStatusFailed,
|
||||
EndTime: now,
|
||||
}, "Status", "EndTime"); err != nil {
|
||||
log.Errorf("failed to update the task status %d: %v", result.TaskID, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
allFailed = false
|
||||
// if the task is submitted successfully, update the status, job ID and start time
|
||||
if err = executionMgr.UpdateTaskStatus(result.TaskID, models.TaskStatusPending, 0, models.TaskStatusInitialized); err != nil {
|
||||
log.Errorf("failed to update the task status %d: %v", result.TaskID, err)
|
||||
}
|
||||
if err = executionMgr.UpdateTask(&models.Task{
|
||||
ID: result.TaskID,
|
||||
JobID: result.JobID,
|
||||
StartTime: now,
|
||||
}, "JobID", "StartTime"); err != nil {
|
||||
log.Errorf("failed to update the task %d: %v", result.TaskID, err)
|
||||
}
|
||||
log.Debugf("the task %d scheduled", result.TaskID)
|
||||
}
|
||||
// if all the tasks are failed, return err
|
||||
if allFailed {
|
||||
return n, errors.New("all tasks are failed")
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// check whether the execution is stopped
|
||||
func isExecutionStopped(mgr execution.Manager, id int64) (bool, error) {
|
||||
execution, err := mgr.Get(id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if execution == nil {
|
||||
return false, fmt.Errorf("execution %d not found", id)
|
||||
}
|
||||
return execution.Status == models.ExecutionStatusStopped, nil
|
||||
}
|
||||
|
||||
// return the name with format "res_name" or "res_name:[vtag1,vtag2,vtag3]"
|
||||
// if the resource has vtags
|
||||
func getResourceName(res *model.Resource) string {
|
||||
|
|
@ -0,0 +1,221 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/replication/adapter"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/testing/mock"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type stageTestSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func (s *stageTestSuite) SetupTest() {
|
||||
}
|
||||
|
||||
func (s *stageTestSuite) TestInitialize() {
|
||||
factory := &mockFactory{}
|
||||
factory.On("AdapterPattern").Return(nil)
|
||||
adapter.RegisterFactory(model.RegistryTypeHarbor, factory)
|
||||
|
||||
policy := &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
DestRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
}
|
||||
factory.On("Create", mock.Anything).Return(&mockAdapter{}, nil)
|
||||
_, _, err := initialize(policy)
|
||||
s.Nil(err)
|
||||
factory.AssertExpectations(s.T())
|
||||
}
|
||||
|
||||
func (s *stageTestSuite) TestFetchResources() {
|
||||
adapter := &mockAdapter{}
|
||||
adapter.On("Info").Return(&model.RegistryInfo{
|
||||
SupportedResourceTypes: []model.ResourceType{
|
||||
model.ResourceTypeArtifact,
|
||||
},
|
||||
}, nil)
|
||||
adapter.On("FetchArtifacts", mock.Anything).Return([]*model.Resource{
|
||||
{},
|
||||
{},
|
||||
}, nil)
|
||||
policy := &model.Policy{}
|
||||
resources, err := fetchResources(adapter, policy)
|
||||
s.Require().Nil(err)
|
||||
s.Len(resources, 2)
|
||||
adapter.AssertExpectations(s.T())
|
||||
}
|
||||
|
||||
func (s *stageTestSuite) TestFilterResources() {
|
||||
resources := []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeImage,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Artifacts: []*model.Artifact{
|
||||
{
|
||||
Tags: []string{"latest"},
|
||||
},
|
||||
},
|
||||
},
|
||||
Deleted: true,
|
||||
Override: true,
|
||||
},
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/harbor",
|
||||
},
|
||||
Artifacts: []*model.Artifact{
|
||||
{
|
||||
Tags: []string{"0.2.0"},
|
||||
},
|
||||
{
|
||||
Tags: []string{"0.3.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
Deleted: true,
|
||||
Override: true,
|
||||
},
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/mysql",
|
||||
},
|
||||
Artifacts: []*model.Artifact{
|
||||
{
|
||||
Tags: []string{"1.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
Deleted: true,
|
||||
Override: true,
|
||||
},
|
||||
}
|
||||
filters := []*model.Filter{
|
||||
{
|
||||
Type: model.FilterTypeResource,
|
||||
Value: model.ResourceTypeChart,
|
||||
},
|
||||
{
|
||||
Type: model.FilterTypeName,
|
||||
Value: "library/*",
|
||||
},
|
||||
{
|
||||
Type: model.FilterTypeName,
|
||||
Value: "library/harbor",
|
||||
},
|
||||
{
|
||||
Type: model.FilterTypeTag,
|
||||
Value: "0.2.?",
|
||||
},
|
||||
}
|
||||
res, err := filterResources(resources, filters)
|
||||
s.Require().Nil(err)
|
||||
s.Len(res, 1)
|
||||
s.Equal("library/harbor", res[0].Metadata.Repository.Name)
|
||||
s.Equal(1, len(res[0].Metadata.Artifacts))
|
||||
s.Equal("0.2.0", res[0].Metadata.Artifacts[0].Tags[0])
|
||||
}
|
||||
|
||||
func (s *stageTestSuite) TestAssembleSourceResources() {
|
||||
resources := []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}
|
||||
policy := &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
ID: 1,
|
||||
},
|
||||
}
|
||||
res := assembleSourceResources(resources, policy)
|
||||
s.Len(res, 1)
|
||||
s.Equal(int64(1), res[0].Registry.ID)
|
||||
}
|
||||
|
||||
func (s *stageTestSuite) TestAssembleDestinationResources() {
|
||||
resources := []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}
|
||||
policy := &model.Policy{
|
||||
DestRegistry: &model.Registry{},
|
||||
DestNamespace: "test",
|
||||
Override: true,
|
||||
}
|
||||
res := assembleDestinationResources(resources, policy)
|
||||
s.Len(res, 1)
|
||||
s.Equal(model.ResourceTypeChart, res[0].Type)
|
||||
s.Equal("test/hello-world", res[0].Metadata.Repository.Name)
|
||||
s.Equal(1, len(res[0].Metadata.Vtags))
|
||||
s.Equal("latest", res[0].Metadata.Vtags[0])
|
||||
}
|
||||
|
||||
func (s *stageTestSuite) TestReplaceNamespace() {
|
||||
// empty namespace
|
||||
repository := "c"
|
||||
namespace := ""
|
||||
result := replaceNamespace(repository, namespace)
|
||||
s.Equal("c", result)
|
||||
// repository contains no "/"
|
||||
repository = "c"
|
||||
namespace = "n"
|
||||
result = replaceNamespace(repository, namespace)
|
||||
s.Equal("n/c", result)
|
||||
// repository contains only one "/"
|
||||
repository = "b/c"
|
||||
namespace = "n"
|
||||
result = replaceNamespace(repository, namespace)
|
||||
s.Equal("n/c", result)
|
||||
// repository contains more than one "/"
|
||||
repository = "a/b/c"
|
||||
namespace = "n"
|
||||
result = replaceNamespace(repository, namespace)
|
||||
s.Equal("n/c", result)
|
||||
}
|
||||
|
||||
func TestStage(t *testing.T) {
|
||||
suite.Run(t, &stageTestSuite{})
|
||||
}
|
||||
|
|
@ -12,19 +12,6 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package task
|
||||
package replication
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestRegisterCheckInProcessor(t *testing.T) {
|
||||
err := RegisterCheckInProcessor("test", nil)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// already exist
|
||||
err = RegisterCheckInProcessor("test", nil)
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
//go:generate mockery --dir ./flow --name Controller --output . --outpkg replication --filename mock_flow_controller_test.go --structname flowController
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
// Code generated by mockery v2.1.0. DO NOT EDIT.
|
||||
|
||||
package replication
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
model "github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
// flowController is an autogenerated mock type for the Controller type
|
||||
type flowController struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// Start provides a mock function with given fields: ctx, executionID, policy, resource
|
||||
func (_m *flowController) Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) error {
|
||||
ret := _m.Called(ctx, executionID, policy, resource)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, *model.Policy, *model.Resource) error); ok {
|
||||
r0 = rf(ctx, executionID, policy, resource)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package replication
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/pkg/task/dao"
|
||||
)
|
||||
|
||||
// Execution model for replication
|
||||
type Execution struct {
|
||||
ID int64
|
||||
PolicyID int64
|
||||
Status string
|
||||
StatusMessage string
|
||||
Metrics *dao.Metrics
|
||||
Trigger string
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
}
|
||||
|
||||
// Task model for replication
|
||||
type Task struct {
|
||||
ID int64
|
||||
ExecutionID int64
|
||||
Status string
|
||||
StatusMessage string
|
||||
RunCount int32
|
||||
ResourceType string
|
||||
SourceResource string
|
||||
DestinationResource string
|
||||
Operation string
|
||||
JobID string
|
||||
CreationTime time.Time
|
||||
StartTime time.Time
|
||||
UpdateTime time.Time
|
||||
EndTime time.Time
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
|
@ -30,7 +31,6 @@ import (
|
|||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/lib/orm"
|
||||
"github.com/goharbor/harbor/src/pkg/project"
|
||||
"github.com/goharbor/harbor/src/pkg/repository"
|
||||
"github.com/goharbor/harbor/src/pkg/retention"
|
||||
|
|
@ -194,13 +194,9 @@ func Init() error {
|
|||
|
||||
retentionController = retention.NewAPIController(retentionMgr, projectMgr, repository.Mgr, scheduler.Sched, retentionLauncher)
|
||||
|
||||
retentionCallbackFun := func(p interface{}) error {
|
||||
str, ok := p.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("the type of param %v isn't string", p)
|
||||
}
|
||||
retentionCallbackFun := func(ctx context.Context, p string) error {
|
||||
param := &retention.TriggerParam{}
|
||||
if err := json.Unmarshal([]byte(str), param); err != nil {
|
||||
if err := json.Unmarshal([]byte(p), param); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal the param: %v", err)
|
||||
}
|
||||
_, err := retentionController.TriggerRetentionExec(param.PolicyID, param.Trigger, false)
|
||||
|
|
@ -211,16 +207,12 @@ func Init() error {
|
|||
return err
|
||||
}
|
||||
|
||||
p2pPreheatCallbackFun := func(p interface{}) error {
|
||||
str, ok := p.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("the type of param %v isn't string", p)
|
||||
}
|
||||
p2pPreheatCallbackFun := func(ctx context.Context, p string) error {
|
||||
param := &preheat.TriggerParam{}
|
||||
if err := json.Unmarshal([]byte(str), param); err != nil {
|
||||
if err := json.Unmarshal([]byte(p), param); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal the param: %v", err)
|
||||
}
|
||||
_, err := preheat.Enf.EnforcePolicy(orm.Context(), param.PolicyID)
|
||||
_, err := preheat.Enf.EnforcePolicy(ctx, param.PolicyID)
|
||||
return err
|
||||
}
|
||||
err = scheduler.RegisterCallbackFunc(preheat.SchedulerCallback, p2pPreheatCallbackFun)
|
||||
|
|
|
|||
|
|
@ -137,10 +137,6 @@ func init() {
|
|||
beego.Router("/api/projects/:pid([0-9]+)/robots/:id([0-9]+)", &RobotAPI{}, "get:Get;put:Put;delete:Delete")
|
||||
|
||||
beego.Router("/api/replication/adapters", &ReplicationAdapterAPI{}, "get:List")
|
||||
beego.Router("/api/replication/executions", &ReplicationOperationAPI{}, "get:ListExecutions;post:CreateExecution")
|
||||
beego.Router("/api/replication/executions/:id([0-9]+)", &ReplicationOperationAPI{}, "get:GetExecution;put:StopExecution")
|
||||
beego.Router("/api/replication/executions/:id([0-9]+)/tasks", &ReplicationOperationAPI{}, "get:ListTasks")
|
||||
beego.Router("/api/replication/executions/:id([0-9]+)/tasks/:tid([0-9]+)/log", &ReplicationOperationAPI{}, "get:GetTaskLog")
|
||||
|
||||
beego.Router("/api/replication/policies", &ReplicationPolicyAPI{}, "get:List;post:Create")
|
||||
beego.Router("/api/replication/policies/:id([0-9]+)", &ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete")
|
||||
|
|
|
|||
|
|
@ -1,248 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
common_http "github.com/goharbor/harbor/src/common/http"
|
||||
"github.com/goharbor/harbor/src/replication"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/event"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
// ReplicationOperationAPI handles the replication operation requests
|
||||
type ReplicationOperationAPI struct {
|
||||
BaseController
|
||||
execution *models.Execution
|
||||
task *models.Task
|
||||
}
|
||||
|
||||
// Prepare ...
|
||||
func (r *ReplicationOperationAPI) Prepare() {
|
||||
r.BaseController.Prepare()
|
||||
// As we delegate the jobservice to trigger the scheduled replication,
|
||||
// we need to allow the jobservice to call the API
|
||||
if !(r.SecurityCtx.IsSysAdmin() || r.SecurityCtx.IsSolutionUser()) {
|
||||
if !r.SecurityCtx.IsAuthenticated() {
|
||||
r.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
||||
return
|
||||
}
|
||||
r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
|
||||
return
|
||||
}
|
||||
|
||||
// check the existence of execution if execution ID is provided in the request path
|
||||
executionIDStr := r.GetStringFromPath(":id")
|
||||
if len(executionIDStr) > 0 {
|
||||
executionID, err := strconv.ParseInt(executionIDStr, 10, 64)
|
||||
if err != nil || executionID <= 0 {
|
||||
r.SendBadRequestError(fmt.Errorf("invalid execution ID: %s", executionIDStr))
|
||||
return
|
||||
}
|
||||
execution, err := replication.OperationCtl.GetExecution(executionID)
|
||||
if err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to get execution %d: %v", executionID, err))
|
||||
return
|
||||
}
|
||||
if execution == nil {
|
||||
r.SendNotFoundError(fmt.Errorf("execution %d not found", executionID))
|
||||
return
|
||||
}
|
||||
r.execution = execution
|
||||
|
||||
// check the existence of task if task ID is provided in the request path
|
||||
taskIDStr := r.GetStringFromPath(":tid")
|
||||
if len(taskIDStr) > 0 {
|
||||
taskID, err := strconv.ParseInt(taskIDStr, 10, 64)
|
||||
if err != nil || taskID <= 0 {
|
||||
r.SendBadRequestError(fmt.Errorf("invalid task ID: %s", taskIDStr))
|
||||
return
|
||||
}
|
||||
task, err := replication.OperationCtl.GetTask(taskID)
|
||||
if err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to get task %d: %v", taskID, err))
|
||||
return
|
||||
}
|
||||
if task == nil || task.ExecutionID != executionID {
|
||||
r.SendNotFoundError(fmt.Errorf("task %d not found", taskID))
|
||||
return
|
||||
}
|
||||
r.task = task
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The API is open only for system admin currently, we can use
|
||||
// the code commentted below to make the API available to the
|
||||
// users who have permission for all projects that the policy
|
||||
// refers
|
||||
/*
|
||||
func (r *ReplicationOperationAPI) authorized(policy *model.Policy, resource rbac.Resource, action rbac.Action) bool {
|
||||
|
||||
projects := []string{}
|
||||
// pull mode
|
||||
if policy.SrcRegistryID != 0 {
|
||||
projects = append(projects, policy.DestNamespace)
|
||||
} else {
|
||||
// push mode
|
||||
projects = append(projects, policy.SrcNamespaces...)
|
||||
}
|
||||
|
||||
for _, project := range projects {
|
||||
resource := rbac.NewProjectNamespace(project).Resource(resource)
|
||||
if !r.SecurityCtx.Can(action, resource) {
|
||||
r.HandleForbidden(r.SecurityCtx.GetUsername())
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
*/
|
||||
|
||||
// ListExecutions ...
|
||||
func (r *ReplicationOperationAPI) ListExecutions() {
|
||||
query := &models.ExecutionQuery{
|
||||
Trigger: r.GetString("trigger"),
|
||||
}
|
||||
|
||||
if len(r.GetString("status")) > 0 {
|
||||
query.Statuses = []string{r.GetString("status")}
|
||||
}
|
||||
if len(r.GetString("policy_id")) > 0 {
|
||||
policyID, err := r.GetInt64("policy_id")
|
||||
if err != nil || policyID <= 0 {
|
||||
r.SendBadRequestError(fmt.Errorf("invalid policy_id %s", r.GetString("policy_id")))
|
||||
return
|
||||
}
|
||||
query.PolicyID = policyID
|
||||
}
|
||||
page, size, err := r.GetPaginationParams()
|
||||
if err != nil {
|
||||
r.SendBadRequestError(err)
|
||||
return
|
||||
}
|
||||
query.Page = page
|
||||
query.Size = size
|
||||
|
||||
total, executions, err := replication.OperationCtl.ListExecutions(query)
|
||||
if err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to list executions: %v", err))
|
||||
return
|
||||
}
|
||||
r.SetPaginationHeader(total, query.Page, query.Size)
|
||||
r.WriteJSONData(executions)
|
||||
}
|
||||
|
||||
// CreateExecution starts a replication
|
||||
func (r *ReplicationOperationAPI) CreateExecution() {
|
||||
execution := &models.Execution{}
|
||||
if err := r.DecodeJSONReq(execution); err != nil {
|
||||
r.SendBadRequestError(err)
|
||||
return
|
||||
}
|
||||
|
||||
policy, err := replication.PolicyCtl.Get(execution.PolicyID)
|
||||
if err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", execution.PolicyID, err))
|
||||
return
|
||||
}
|
||||
if policy == nil {
|
||||
r.SendNotFoundError(fmt.Errorf("policy %d not found", execution.PolicyID))
|
||||
return
|
||||
}
|
||||
if !policy.Enabled {
|
||||
r.SendBadRequestError(fmt.Errorf("the policy %d is disabled", execution.PolicyID))
|
||||
return
|
||||
}
|
||||
if err = event.PopulateRegistries(replication.RegistryMgr, policy); err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to populate registries for policy %d: %v", execution.PolicyID, err))
|
||||
return
|
||||
}
|
||||
|
||||
trigger := r.GetString("trigger", string(model.TriggerTypeManual))
|
||||
executionID, err := replication.OperationCtl.StartReplication(policy, nil, model.TriggerType(trigger))
|
||||
if err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to start replication for policy %d: %v", execution.PolicyID, err))
|
||||
return
|
||||
}
|
||||
r.Redirect(http.StatusCreated, strconv.FormatInt(executionID, 10))
|
||||
}
|
||||
|
||||
// GetExecution gets one execution of the replication
|
||||
func (r *ReplicationOperationAPI) GetExecution() {
|
||||
r.WriteJSONData(r.execution)
|
||||
}
|
||||
|
||||
// StopExecution stops one execution of the replication
|
||||
func (r *ReplicationOperationAPI) StopExecution() {
|
||||
if err := replication.OperationCtl.StopReplication(r.execution.ID); err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to stop execution %d: %v", r.execution.ID, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ListTasks ...
|
||||
func (r *ReplicationOperationAPI) ListTasks() {
|
||||
query := &models.TaskQuery{
|
||||
ExecutionID: r.execution.ID,
|
||||
ResourceType: r.GetString("resource_type"),
|
||||
}
|
||||
status := r.GetString("status")
|
||||
if len(status) > 0 {
|
||||
query.Statuses = []string{status}
|
||||
}
|
||||
page, size, err := r.GetPaginationParams()
|
||||
if err != nil {
|
||||
r.SendBadRequestError(err)
|
||||
return
|
||||
}
|
||||
query.Page = page
|
||||
query.Size = size
|
||||
total, tasks, err := replication.OperationCtl.ListTasks(query)
|
||||
if err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to list tasks: %v", err))
|
||||
return
|
||||
}
|
||||
r.SetPaginationHeader(total, query.Page, query.Size)
|
||||
r.WriteJSONData(tasks)
|
||||
}
|
||||
|
||||
// GetTaskLog ...
|
||||
func (r *ReplicationOperationAPI) GetTaskLog() {
|
||||
logBytes, err := replication.OperationCtl.GetTaskLog(r.task.ID)
|
||||
if err != nil {
|
||||
if httpErr, ok := err.(*common_http.Error); ok {
|
||||
if ok && httpErr.Code == http.StatusNotFound {
|
||||
r.SendNotFoundError(fmt.Errorf("the log of task %d not found", r.task.ID))
|
||||
return
|
||||
}
|
||||
}
|
||||
r.SendInternalServerError(fmt.Errorf("failed to get log of task %d: %v", r.task.ID, err))
|
||||
return
|
||||
}
|
||||
r.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
|
||||
r.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
|
||||
_, err = r.Ctx.ResponseWriter.Write(logBytes)
|
||||
if err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to write log of task %d: %v", r.task.ID, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -1,443 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/replication"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
type fakedOperationController struct{}
|
||||
|
||||
func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
|
||||
return 1, nil
|
||||
}
|
||||
func (f *fakedOperationController) StopReplication(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
|
||||
return 1, []*models.Execution{
|
||||
{
|
||||
ID: 1,
|
||||
PolicyID: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedOperationController) GetExecution(id int64) (*models.Execution, error) {
|
||||
if id == 1 {
|
||||
return &models.Execution{
|
||||
ID: 1,
|
||||
PolicyID: 1,
|
||||
}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
|
||||
return 2, []*models.Task{
|
||||
{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedOperationController) GetTask(id int64) (*models.Task, error) {
|
||||
if id == 1 {
|
||||
return &models.Task{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
}, nil
|
||||
}
|
||||
if id == 3 {
|
||||
return &models.Task{
|
||||
ID: 3,
|
||||
ExecutionID: 2,
|
||||
}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) {
|
||||
return []byte("success"), nil
|
||||
}
|
||||
|
||||
type fakedPolicyManager struct{}
|
||||
|
||||
func (f *fakedPolicyManager) Create(*model.Policy) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (f *fakedPolicyManager) List(...*model.PolicyQuery) (int64, []*model.Policy, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) {
|
||||
if id == 1 {
|
||||
return &model.Policy{
|
||||
ID: 1,
|
||||
Enabled: true,
|
||||
SrcRegistry: &model.Registry{
|
||||
ID: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
if id == 2 {
|
||||
return &model.Policy{
|
||||
ID: 2,
|
||||
Enabled: false,
|
||||
SrcRegistry: &model.Registry{
|
||||
ID: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedPolicyManager) GetByName(name string) (*model.Policy, error) {
|
||||
if name == "duplicate_name" {
|
||||
return &model.Policy{
|
||||
Name: "duplicate_name",
|
||||
}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedPolicyManager) Update(*model.Policy) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedPolicyManager) Remove(int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestListExecutions(t *testing.T) {
|
||||
operationCtl := replication.OperationCtl
|
||||
defer func() {
|
||||
replication.OperationCtl = operationCtl
|
||||
}()
|
||||
replication.OperationCtl = &fakedOperationController{}
|
||||
|
||||
cases := []*codeCheckingCase{
|
||||
// 401
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions",
|
||||
},
|
||||
code: http.StatusUnauthorized,
|
||||
},
|
||||
// 403
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions",
|
||||
credential: nonSysAdmin,
|
||||
},
|
||||
code: http.StatusForbidden,
|
||||
},
|
||||
// 200
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusOK,
|
||||
},
|
||||
}
|
||||
|
||||
runCodeCheckingCases(t, cases...)
|
||||
}
|
||||
|
||||
func TestCreateExecution(t *testing.T) {
|
||||
operationCtl := replication.OperationCtl
|
||||
policyMgr := replication.PolicyCtl
|
||||
registryMgr := replication.RegistryMgr
|
||||
defer func() {
|
||||
replication.OperationCtl = operationCtl
|
||||
replication.PolicyCtl = policyMgr
|
||||
replication.RegistryMgr = registryMgr
|
||||
}()
|
||||
replication.OperationCtl = &fakedOperationController{}
|
||||
replication.PolicyCtl = &fakedPolicyManager{}
|
||||
replication.RegistryMgr = &fakedRegistryManager{}
|
||||
|
||||
cases := []*codeCheckingCase{
|
||||
// 401
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPost,
|
||||
url: "/api/replication/executions",
|
||||
},
|
||||
code: http.StatusUnauthorized,
|
||||
},
|
||||
// 403
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPost,
|
||||
url: "/api/replication/executions",
|
||||
credential: nonSysAdmin,
|
||||
},
|
||||
code: http.StatusForbidden,
|
||||
},
|
||||
// 404
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPost,
|
||||
url: "/api/replication/executions",
|
||||
bodyJSON: &models.Execution{
|
||||
PolicyID: 3,
|
||||
},
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusNotFound,
|
||||
},
|
||||
// 400
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPost,
|
||||
url: "/api/replication/executions",
|
||||
bodyJSON: &models.Execution{
|
||||
PolicyID: 2,
|
||||
},
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusBadRequest,
|
||||
},
|
||||
// 201
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPost,
|
||||
url: "/api/replication/executions",
|
||||
bodyJSON: &models.Execution{
|
||||
PolicyID: 1,
|
||||
},
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusCreated,
|
||||
},
|
||||
}
|
||||
|
||||
runCodeCheckingCases(t, cases...)
|
||||
}
|
||||
|
||||
func TestGetExecution(t *testing.T) {
|
||||
operationCtl := replication.OperationCtl
|
||||
defer func() {
|
||||
replication.OperationCtl = operationCtl
|
||||
}()
|
||||
replication.OperationCtl = &fakedOperationController{}
|
||||
|
||||
cases := []*codeCheckingCase{
|
||||
// 401
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1",
|
||||
},
|
||||
code: http.StatusUnauthorized,
|
||||
},
|
||||
// 403
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1",
|
||||
credential: nonSysAdmin,
|
||||
},
|
||||
code: http.StatusForbidden,
|
||||
},
|
||||
// 404
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/2",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusNotFound,
|
||||
},
|
||||
// 200
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusOK,
|
||||
},
|
||||
}
|
||||
|
||||
runCodeCheckingCases(t, cases...)
|
||||
}
|
||||
func TestStopExecution(t *testing.T) {
|
||||
operationCtl := replication.OperationCtl
|
||||
defer func() {
|
||||
replication.OperationCtl = operationCtl
|
||||
}()
|
||||
replication.OperationCtl = &fakedOperationController{}
|
||||
|
||||
cases := []*codeCheckingCase{
|
||||
// 401
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPut,
|
||||
url: "/api/replication/executions/1",
|
||||
},
|
||||
code: http.StatusUnauthorized,
|
||||
},
|
||||
// 403
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPut,
|
||||
url: "/api/replication/executions/1",
|
||||
credential: nonSysAdmin,
|
||||
},
|
||||
code: http.StatusForbidden,
|
||||
},
|
||||
// 404
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPut,
|
||||
url: "/api/replication/executions/2",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusNotFound,
|
||||
},
|
||||
// 200
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodPut,
|
||||
url: "/api/replication/executions/1",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusOK,
|
||||
},
|
||||
}
|
||||
|
||||
runCodeCheckingCases(t, cases...)
|
||||
}
|
||||
|
||||
func TestListTasks(t *testing.T) {
|
||||
operationCtl := replication.OperationCtl
|
||||
defer func() {
|
||||
replication.OperationCtl = operationCtl
|
||||
}()
|
||||
replication.OperationCtl = &fakedOperationController{}
|
||||
|
||||
cases := []*codeCheckingCase{
|
||||
// 401
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1/tasks",
|
||||
},
|
||||
code: http.StatusUnauthorized,
|
||||
},
|
||||
// 403
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1/tasks",
|
||||
credential: nonSysAdmin,
|
||||
},
|
||||
code: http.StatusForbidden,
|
||||
},
|
||||
// 404
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/2/tasks",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusNotFound,
|
||||
},
|
||||
// 200
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1/tasks",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusOK,
|
||||
},
|
||||
}
|
||||
|
||||
runCodeCheckingCases(t, cases...)
|
||||
}
|
||||
|
||||
func TestGetTaskLog(t *testing.T) {
|
||||
operationCtl := replication.OperationCtl
|
||||
defer func() {
|
||||
replication.OperationCtl = operationCtl
|
||||
}()
|
||||
replication.OperationCtl = &fakedOperationController{}
|
||||
|
||||
cases := []*codeCheckingCase{
|
||||
// 401
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1/tasks/1/log",
|
||||
},
|
||||
code: http.StatusUnauthorized,
|
||||
},
|
||||
// 403
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1/tasks/1/log",
|
||||
credential: nonSysAdmin,
|
||||
},
|
||||
code: http.StatusForbidden,
|
||||
},
|
||||
// 404, execution not found
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/2/tasks/1/log",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusNotFound,
|
||||
},
|
||||
// 404, task not found
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1/tasks/2/log",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusNotFound,
|
||||
},
|
||||
// 404, task not found
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1/tasks/3/log",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusNotFound,
|
||||
},
|
||||
// 200
|
||||
{
|
||||
request: &testingRequest{
|
||||
method: http.MethodGet,
|
||||
url: "/api/replication/executions/1/tasks/1/log",
|
||||
credential: sysAdmin,
|
||||
},
|
||||
code: http.StatusOK,
|
||||
},
|
||||
}
|
||||
|
||||
runCodeCheckingCases(t, cases...)
|
||||
}
|
||||
|
|
@ -17,11 +17,15 @@ package api
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/orm"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
replica "github.com/goharbor/harbor/src/controller/replication"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/replication"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/event"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/registry"
|
||||
|
|
@ -219,16 +223,29 @@ func (r *ReplicationPolicyAPI) Delete() {
|
|||
return
|
||||
}
|
||||
|
||||
isRunning, err := hasRunningExecutions(id)
|
||||
ctx := orm.Context()
|
||||
executions, err := replica.Ctl.ListExecutions(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"PolicyID": id,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to check the execution status of policy %d: %v", id, err))
|
||||
r.SendInternalServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if isRunning {
|
||||
for _, execution := range executions {
|
||||
if execution.Status != job.RunningStatus.String() {
|
||||
continue
|
||||
}
|
||||
r.SendPreconditionFailedError(fmt.Errorf("the policy %d has running executions, can not be deleted", id))
|
||||
return
|
||||
}
|
||||
for _, execution := range executions {
|
||||
if err = task.ExecMgr.Delete(ctx, execution.ID); err != nil {
|
||||
r.SendInternalServerError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := replication.PolicyCtl.Remove(id); err != nil {
|
||||
r.SendInternalServerError(fmt.Errorf("failed to delete the policy %d: %v", id, err))
|
||||
|
|
@ -236,22 +253,6 @@ func (r *ReplicationPolicyAPI) Delete() {
|
|||
}
|
||||
}
|
||||
|
||||
func hasRunningExecutions(policyID int64) (bool, error) {
|
||||
_, executions, err := replication.OperationCtl.ListExecutions(&models.ExecutionQuery{
|
||||
PolicyID: policyID,
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, execution := range executions {
|
||||
if execution.Status != models.ExecutionStatusInProgress {
|
||||
continue
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// ignore the credential for the registries
|
||||
func populateRegistries(registryMgr registry.Manager, policy *model.Policy) error {
|
||||
if err := event.PopulateRegistries(registryMgr, policy); err != nil {
|
||||
|
|
|
|||
|
|
@ -54,6 +54,50 @@ func (f *fakedRegistryManager) HealthCheck() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
type fakedPolicyManager struct{}
|
||||
|
||||
func (f *fakedPolicyManager) Create(*model.Policy) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (f *fakedPolicyManager) List(...*model.PolicyQuery) (int64, []*model.Policy, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) {
|
||||
if id == 1 {
|
||||
return &model.Policy{
|
||||
ID: 1,
|
||||
Enabled: true,
|
||||
SrcRegistry: &model.Registry{
|
||||
ID: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
if id == 2 {
|
||||
return &model.Policy{
|
||||
ID: 2,
|
||||
Enabled: false,
|
||||
SrcRegistry: &model.Registry{
|
||||
ID: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedPolicyManager) GetByName(name string) (*model.Policy, error) {
|
||||
if name == "duplicate_name" {
|
||||
return &model.Policy{
|
||||
Name: "duplicate_name",
|
||||
}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedPolicyManager) Update(*model.Policy) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedPolicyManager) Remove(int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestReplicationPolicyAPIList(t *testing.T) {
|
||||
policyMgr := replication.PolicyCtl
|
||||
defer func() {
|
||||
|
|
|
|||
|
|
@ -31,9 +31,6 @@ import (
|
|||
"github.com/goharbor/harbor/src/pkg/notifier/event"
|
||||
"github.com/goharbor/harbor/src/pkg/retention"
|
||||
sc "github.com/goharbor/harbor/src/pkg/scan"
|
||||
"github.com/goharbor/harbor/src/replication"
|
||||
"github.com/goharbor/harbor/src/replication/operation/hook"
|
||||
"github.com/goharbor/harbor/src/replication/policy/scheduler"
|
||||
)
|
||||
|
||||
var statusMap = map[string]string{
|
||||
|
|
@ -141,44 +138,6 @@ func (h *Handler) HandleScan() {
|
|||
}
|
||||
}
|
||||
|
||||
// HandleReplicationScheduleJob handles the webhook of replication schedule job
|
||||
func (h *Handler) HandleReplicationScheduleJob() {
|
||||
log.Debugf("received replication schedule job status update event: schedule-job-%d, status-%s", h.id, h.status)
|
||||
if err := scheduler.UpdateStatus(h.id, h.status); err != nil {
|
||||
log.Errorf("Failed to update job status, id: %d, status: %s", h.id, h.status)
|
||||
h.SendInternalServerError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// HandleReplicationTask handles the webhook of replication task
|
||||
func (h *Handler) HandleReplicationTask() {
|
||||
log.Debugf("received replication task status update event: task-%d, status-%s", h.id, h.status)
|
||||
|
||||
if err := hook.UpdateTask(replication.OperationCtl, h.id, h.rawStatus, h.revision); err != nil {
|
||||
log.Errorf("failed to update the status of the replication task %d: %v", h.id, err)
|
||||
h.SendInternalServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Trigger artifict webhook event only for JobFinished and JobError status
|
||||
if h.status == models.JobFinished || h.status == models.JobError || h.status == models.JobStopped {
|
||||
e := &event.Event{}
|
||||
metaData := &metadata.ReplicationMetaData{
|
||||
ReplicationTaskID: h.id,
|
||||
Status: h.rawStatus,
|
||||
}
|
||||
|
||||
if err := e.Build(metaData); err == nil {
|
||||
if err := e.Publish(); err != nil {
|
||||
log.Error(errors.Wrap(err, "replication job hook handler: event publish"))
|
||||
}
|
||||
} else {
|
||||
log.Error(errors.Wrap(err, "replication job hook handler: event publish"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HandleRetentionTask handles the webhook of retention task
|
||||
func (h *Handler) HandleRetentionTask() {
|
||||
taskID := h.id
|
||||
|
|
|
|||
|
|
@ -1,51 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/goharbor/harbor/src/core/service/notifications"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
)
|
||||
|
||||
// Handler handles the scheduler requests
|
||||
type Handler struct {
|
||||
notifications.BaseHandler
|
||||
}
|
||||
|
||||
// Handle ...
|
||||
func (h *Handler) Handle() {
|
||||
log.Debugf("received scheduler hook event for schedule %s", h.GetStringFromPath(":id"))
|
||||
|
||||
var data job.StatusChange
|
||||
if err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data); err != nil {
|
||||
log.Errorf("failed to decode hook event: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
schedulerID, err := h.GetInt64FromPath(":id")
|
||||
if err != nil {
|
||||
log.Errorf("failed to get the schedule ID: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = scheduler.HandleLegacyHook(h.Ctx.Request.Context(), schedulerID, &data); err != nil {
|
||||
log.Errorf("failed to handle the legacy hook: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -35,8 +35,11 @@ func WithLogger(ctx context.Context, logger *Logger) context.Context {
|
|||
// GetLogger retrieves the current logger from the context.
|
||||
// If no logger is available, the default logger is returned.
|
||||
func GetLogger(ctx context.Context) *Logger {
|
||||
logger := ctx.Value(loggerKey{})
|
||||
if ctx == nil {
|
||||
return L
|
||||
}
|
||||
|
||||
logger := ctx.Value(loggerKey{})
|
||||
if logger == nil {
|
||||
return L
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,25 +12,28 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
package orm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
import "github.com/astaxie/beego/orm"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
var (
|
||||
// Crt is a global instance of ORM creator
|
||||
Crt = NewCreator()
|
||||
)
|
||||
|
||||
type fakedFlow struct{}
|
||||
|
||||
func (f *fakedFlow) Run(interface{}) (int, error) {
|
||||
return 1, nil
|
||||
// NewCreator creates an ORM creator
|
||||
func NewCreator() Creator {
|
||||
return &creator{}
|
||||
}
|
||||
|
||||
func TestStart(t *testing.T) {
|
||||
flow := &fakedFlow{}
|
||||
controller := NewController()
|
||||
n, err := controller.Start(flow)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
// Creator creates ORMer
|
||||
// Introducing the "Creator" interface to eliminate the dependency on database
|
||||
type Creator interface {
|
||||
Create() orm.Ormer
|
||||
}
|
||||
|
||||
type creator struct{}
|
||||
|
||||
func (c *creator) Create() orm.Ormer {
|
||||
return orm.NewOrm()
|
||||
}
|
||||
|
|
@ -37,11 +37,18 @@ func New(kw KeyWords) *Query {
|
|||
// MustClone returns the clone of query when it's not nil
|
||||
// or returns a new Query instance
|
||||
func MustClone(query *Query) *Query {
|
||||
if query != nil {
|
||||
clone := *query
|
||||
return &clone
|
||||
q := &Query{
|
||||
Keywords: map[string]interface{}{},
|
||||
}
|
||||
return New(KeyWords{})
|
||||
if query != nil {
|
||||
q.PageNumber = query.PageNumber
|
||||
q.PageSize = query.PageSize
|
||||
q.Sorting = query.Sorting
|
||||
for k, v := range query.Keywords {
|
||||
q.Keywords[k] = v
|
||||
}
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// Range query
|
||||
|
|
|
|||
|
|
@ -12,26 +12,26 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
package lib
|
||||
|
||||
// Flow defines the replication flow
|
||||
type Flow interface {
|
||||
// returns the count of tasks which have been scheduled and the error
|
||||
Run(interface{}) (int, error)
|
||||
// NewWorkerPool creates a new worker pool with specified size
|
||||
func NewWorkerPool(size int32) *WorkerPool {
|
||||
wp := &WorkerPool{}
|
||||
wp.queue = make(chan struct{}, size)
|
||||
return wp
|
||||
}
|
||||
|
||||
// Controller is the controller that controls the replication flows
|
||||
type Controller interface {
|
||||
Start(Flow) (int, error)
|
||||
// WorkerPool controls the concurrency limit of task/process
|
||||
type WorkerPool struct {
|
||||
queue chan struct{}
|
||||
}
|
||||
|
||||
// NewController returns an instance of the default flow controller
|
||||
func NewController() Controller {
|
||||
return &controller{}
|
||||
// GetWorker hangs until a free worker available
|
||||
func (w *WorkerPool) GetWorker() {
|
||||
w.queue <- struct{}{}
|
||||
}
|
||||
|
||||
type controller struct{}
|
||||
|
||||
func (c *controller) Start(flow Flow) (int, error) {
|
||||
return flow.Run(nil)
|
||||
// ReleaseWorker hangs until the worker return back into the pool
|
||||
func (w *WorkerPool) ReleaseWorker() {
|
||||
<-w.queue
|
||||
}
|
||||
|
|
@ -18,7 +18,6 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
|
|
@ -29,7 +28,7 @@ var (
|
|||
)
|
||||
|
||||
// CallbackFunc defines the function that the scheduler calls when triggered
|
||||
type CallbackFunc func(interface{}) error
|
||||
type CallbackFunc func(ctx context.Context, param string) error
|
||||
|
||||
func init() {
|
||||
if err := task.RegisterCheckInProcessor(JobNameScheduler, triggerCallback); err != nil {
|
||||
|
|
@ -68,7 +67,7 @@ func callbackFuncExist(name string) bool {
|
|||
return exist
|
||||
}
|
||||
|
||||
func triggerCallback(ctx context.Context, task *task.Task, change *job.StatusChange) (err error) {
|
||||
func triggerCallback(ctx context.Context, task *task.Task, data string) (err error) {
|
||||
execution, err := Sched.(*scheduler).execMgr.Get(ctx, task.ExecutionID)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -85,5 +84,5 @@ func triggerCallback(ctx context.Context, task *task.Task, change *job.StatusCha
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return callbackFunc(schedule.CallbackFuncParam)
|
||||
return callbackFunc(ctx, schedule.CallbackFuncParam)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
|
@ -26,7 +27,7 @@ type callbackTestSuite struct {
|
|||
|
||||
func (c *callbackTestSuite) SetupTest() {
|
||||
registry = map[string]CallbackFunc{}
|
||||
err := RegisterCallbackFunc("callback", func(interface{}) error { return nil })
|
||||
err := RegisterCallbackFunc("callback", func(context.Context, string) error { return nil })
|
||||
c.Require().Nil(err)
|
||||
}
|
||||
|
||||
|
|
@ -40,11 +41,11 @@ func (c *callbackTestSuite) TestRegisterCallbackFunc() {
|
|||
c.NotNil(err)
|
||||
|
||||
// pass
|
||||
err = RegisterCallbackFunc("test", func(interface{}) error { return nil })
|
||||
err = RegisterCallbackFunc("test", func(context.Context, string) error { return nil })
|
||||
c.Nil(err)
|
||||
|
||||
// duplicate name
|
||||
err = RegisterCallbackFunc("test", func(interface{}) error { return nil })
|
||||
err = RegisterCallbackFunc("test", func(context.Context, string) error { return nil })
|
||||
c.NotNil(err)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -289,38 +289,3 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S
|
|||
}
|
||||
return schd, nil
|
||||
}
|
||||
|
||||
// HandleLegacyHook handles the legacy web hook for scheduler
|
||||
// We rewrite the implementation of scheduler with task manager mechanism in v2.1,
|
||||
// this method is used to handle the job status hook for the legacy implementation
|
||||
// We can remove the method and the hook endpoint after several releases
|
||||
func HandleLegacyHook(ctx context.Context, scheduleID int64, sc *job.StatusChange) error {
|
||||
scheduler := Sched.(*scheduler)
|
||||
executions, err := scheduler.execMgr.List(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"VendorType": JobNameScheduler,
|
||||
"VendorID": scheduleID,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(executions) == 0 {
|
||||
return errors.New(nil).WithCode(errors.NotFoundCode).
|
||||
WithMessage("no execution found for the schedule %d", scheduleID)
|
||||
}
|
||||
|
||||
tasks, err := scheduler.taskMgr.List(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ExecutionID": executions[0].ID,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return errors.New(nil).WithCode(errors.NotFoundCode).
|
||||
WithMessage("no task found for the execution %d", executions[0].ID)
|
||||
}
|
||||
return task.NewHookHandler().Handle(ctx, tasks[0].ID, sc)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,13 +15,15 @@
|
|||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/testing/mock"
|
||||
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type schedulerTestSuite struct {
|
||||
|
|
@ -34,7 +36,7 @@ type schedulerTestSuite struct {
|
|||
|
||||
func (s *schedulerTestSuite) SetupTest() {
|
||||
registry = map[string]CallbackFunc{}
|
||||
err := RegisterCallbackFunc("callback", func(interface{}) error { return nil })
|
||||
err := RegisterCallbackFunc("callback", func(context.Context, string) error { return nil })
|
||||
s.Require().Nil(err)
|
||||
|
||||
s.dao = &mockDAO{}
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
)
|
||||
|
||||
var (
|
||||
registry = map[string]CheckInProcessor{}
|
||||
)
|
||||
|
||||
// CheckInProcessor is the processor to process the check in data which is sent by jobservice via webhook
|
||||
type CheckInProcessor func(ctx context.Context, task *Task, change *job.StatusChange) (err error)
|
||||
|
||||
// RegisterCheckInProcessor registers check in processor for the specific vendor type
|
||||
func RegisterCheckInProcessor(vendorType string, processor CheckInProcessor) error {
|
||||
if _, exist := registry[vendorType]; exist {
|
||||
return fmt.Errorf("check in processor for %s already exists", vendorType)
|
||||
}
|
||||
registry[vendorType] = processor
|
||||
return nil
|
||||
}
|
||||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"strings"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
|
|
@ -28,8 +29,10 @@ import (
|
|||
// ExecutionDAO is the data access object interface for execution
|
||||
type ExecutionDAO interface {
|
||||
// Count returns the total count of executions according to the query
|
||||
// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
|
||||
Count(ctx context.Context, query *q.Query) (count int64, err error)
|
||||
// List the executions according to the query
|
||||
// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
|
||||
List(ctx context.Context, query *q.Query) (executions []*Execution, err error)
|
||||
// Get the specified execution
|
||||
Get(ctx context.Context, id int64) (execution *Execution, err error)
|
||||
|
|
@ -43,7 +46,9 @@ type ExecutionDAO interface {
|
|||
GetMetrics(ctx context.Context, id int64) (metrics *Metrics, err error)
|
||||
// RefreshStatus refreshes the status of the specified execution according to it's tasks. If it's status
|
||||
// is final, update the end time as well
|
||||
RefreshStatus(ctx context.Context, id int64) (err error)
|
||||
// If the status is changed, the returning "statusChanged" is set as "true" and the current status indicates
|
||||
// the changed status
|
||||
RefreshStatus(ctx context.Context, id int64) (statusChanged bool, currentStatus string, err error)
|
||||
}
|
||||
|
||||
// NewExecutionDAO returns an instance of ExecutionDAO
|
||||
|
|
@ -64,7 +69,7 @@ func (e *executionDAO) Count(ctx context.Context, query *q.Query) (int64, error)
|
|||
Keywords: query.Keywords,
|
||||
}
|
||||
}
|
||||
qs, err := orm.QuerySetter(ctx, &Execution{}, query)
|
||||
qs, err := e.querySetter(ctx, query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
|
@ -73,7 +78,7 @@ func (e *executionDAO) Count(ctx context.Context, query *q.Query) (int64, error)
|
|||
|
||||
func (e *executionDAO) List(ctx context.Context, query *q.Query) ([]*Execution, error) {
|
||||
executions := []*Execution{}
|
||||
qs, err := orm.QuerySetter(ctx, &Execution{}, query)
|
||||
qs, err := e.querySetter(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -178,33 +183,39 @@ func (e *executionDAO) GetMetrics(ctx context.Context, id int64) (*Metrics, erro
|
|||
metrics.ScheduledTaskCount + metrics.StoppedTaskCount
|
||||
return metrics, nil
|
||||
}
|
||||
func (e *executionDAO) RefreshStatus(ctx context.Context, id int64) error {
|
||||
|
||||
func (e *executionDAO) RefreshStatus(ctx context.Context, id int64) (bool, string, error) {
|
||||
// as the status of the execution can be refreshed by multiple operators concurrently
|
||||
// we use the optimistic locking to avoid the conflict and retry 5 times at most
|
||||
for i := 0; i < 5; i++ {
|
||||
retry, err := e.refreshStatus(ctx, id)
|
||||
statusChanged, currentStatus, retry, err := e.refreshStatus(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, "", err
|
||||
}
|
||||
if !retry {
|
||||
return nil
|
||||
return statusChanged, currentStatus, nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("failed to refresh the status of the execution %d after %d retries", id, 5)
|
||||
return false, "", fmt.Errorf("failed to refresh the status of the execution %d after %d retries", id, 5)
|
||||
}
|
||||
|
||||
func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error) {
|
||||
// the returning values:
|
||||
// 1. bool: is the status changed
|
||||
// 2. string: the current status if changed
|
||||
// 3. bool: whether a retry is needed
|
||||
// 4. error: the error
|
||||
func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, string, bool, error) {
|
||||
execution, err := e.Get(ctx, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", false, err
|
||||
}
|
||||
metrics, err := e.GetMetrics(ctx, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", false, err
|
||||
}
|
||||
// no task, return directly
|
||||
if metrics.TaskCount == 0 {
|
||||
return false, nil
|
||||
return false, "", false, nil
|
||||
}
|
||||
|
||||
var status string
|
||||
|
|
@ -220,20 +231,22 @@ func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error
|
|||
|
||||
ormer, err := orm.FromContext(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", false, err
|
||||
}
|
||||
|
||||
sql := `update execution set status = ?, revision = revision+1 where id = ? and revision = ?`
|
||||
result, err := ormer.Raw(sql, status, id, execution.Revision).Exec()
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", false, err
|
||||
}
|
||||
n, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, "", false, err
|
||||
}
|
||||
|
||||
// if the count of affected rows is 0, that means the execution is updating by others, retry
|
||||
if n == 0 {
|
||||
return true, nil
|
||||
return false, "", true, nil
|
||||
}
|
||||
|
||||
/* this is another solution to solve the concurrency issue for refreshing the execution status
|
||||
|
|
@ -290,5 +303,28 @@ func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error
|
|||
where id=?`
|
||||
sql = fmt.Sprintf(sql, job.ErrorStatus.String(), job.StoppedStatus.String(), job.SuccessStatus.String())
|
||||
_, err = ormer.Raw(sql, id, id).Exec()
|
||||
return false, err
|
||||
return status != execution.Status, status, false, err
|
||||
}
|
||||
|
||||
func (e *executionDAO) querySetter(ctx context.Context, query *q.Query) (orm.QuerySeter, error) {
|
||||
qs, err := orm.QuerySetter(ctx, &Execution{}, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// append the filter for "extra attrs"
|
||||
if query != nil && len(query.Keywords) > 0 {
|
||||
for key, value := range query.Keywords {
|
||||
if strings.HasPrefix(key, "ExtraAttrs.") {
|
||||
qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from execution where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "ExtraAttrs."), value))
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(key, "extra_attrs.") {
|
||||
qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from execution where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "extra_attrs."), value))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return qs, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ func (e *executionDAOTestSuite) SetupTest() {
|
|||
id, err := e.executionDAO.Create(e.ctx, &Execution{
|
||||
VendorType: "test",
|
||||
Trigger: "test",
|
||||
ExtraAttrs: "{}",
|
||||
ExtraAttrs: `{"key":"value"}`,
|
||||
})
|
||||
e.Require().Nil(err)
|
||||
e.executionID = id
|
||||
|
|
@ -62,22 +62,42 @@ func (e *executionDAOTestSuite) TearDownTest() {
|
|||
func (e *executionDAOTestSuite) TestCount() {
|
||||
count, err := e.executionDAO.Count(e.ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"VendorType": "test",
|
||||
"VendorType": "test",
|
||||
"ExtraAttrs.key": "value",
|
||||
},
|
||||
})
|
||||
e.Require().Nil(err)
|
||||
e.Equal(int64(1), count)
|
||||
|
||||
count, err = e.executionDAO.Count(e.ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"VendorType": "test",
|
||||
"ExtraAttrs.key": "incorrect-value",
|
||||
},
|
||||
})
|
||||
e.Require().Nil(err)
|
||||
e.Equal(int64(0), count)
|
||||
}
|
||||
|
||||
func (e *executionDAOTestSuite) TestList() {
|
||||
executions, err := e.executionDAO.List(e.ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"VendorType": "test",
|
||||
"VendorType": "test",
|
||||
"ExtraAttrs.key": "value",
|
||||
},
|
||||
})
|
||||
e.Require().Nil(err)
|
||||
e.Require().Len(executions, 1)
|
||||
e.Equal(e.executionID, executions[0].ID)
|
||||
|
||||
executions, err = e.executionDAO.List(e.ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"VendorType": "test",
|
||||
"ExtraAttrs.key": "incorrect-value",
|
||||
},
|
||||
})
|
||||
e.Require().Nil(err)
|
||||
e.Require().Len(executions, 0)
|
||||
}
|
||||
|
||||
func (e *executionDAOTestSuite) TestGet() {
|
||||
|
|
@ -200,8 +220,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() {
|
|||
e.Require().Nil(err)
|
||||
defer e.taskDao.Delete(e.ctx, taskID01)
|
||||
|
||||
err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
statusChanged, currentStatus, err := e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.True(statusChanged)
|
||||
e.Equal(job.SuccessStatus.String(), currentStatus)
|
||||
execution, err := e.executionDAO.Get(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.Equal(job.SuccessStatus.String(), execution.Status)
|
||||
|
|
@ -218,8 +240,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() {
|
|||
e.Require().Nil(err)
|
||||
defer e.taskDao.Delete(e.ctx, taskID02)
|
||||
|
||||
err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.True(statusChanged)
|
||||
e.Equal(job.StoppedStatus.String(), currentStatus)
|
||||
execution, err = e.executionDAO.Get(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.Equal(job.StoppedStatus.String(), execution.Status)
|
||||
|
|
@ -236,8 +260,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() {
|
|||
e.Require().Nil(err)
|
||||
defer e.taskDao.Delete(e.ctx, taskID03)
|
||||
|
||||
err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.True(statusChanged)
|
||||
e.Equal(job.ErrorStatus.String(), currentStatus)
|
||||
execution, err = e.executionDAO.Get(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.Equal(job.ErrorStatus.String(), execution.Status)
|
||||
|
|
@ -271,8 +297,29 @@ func (e *executionDAOTestSuite) TestRefreshStatus() {
|
|||
e.Require().Nil(err)
|
||||
defer e.taskDao.Delete(e.ctx, taskID06)
|
||||
|
||||
err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.True(statusChanged)
|
||||
e.Equal(job.RunningStatus.String(), currentStatus)
|
||||
execution, err = e.executionDAO.Get(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.Equal(job.RunningStatus.String(), execution.Status)
|
||||
e.Empty(execution.EndTime)
|
||||
|
||||
// add another running task, the status shouldn't be changed
|
||||
taskID07, err := e.taskDao.Create(e.ctx, &Task{
|
||||
ExecutionID: e.executionID,
|
||||
Status: job.RunningStatus.String(),
|
||||
StatusCode: job.RunningStatus.Code(),
|
||||
ExtraAttrs: "{}",
|
||||
})
|
||||
e.Require().Nil(err)
|
||||
defer e.taskDao.Delete(e.ctx, taskID07)
|
||||
|
||||
statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.False(statusChanged)
|
||||
e.Equal(job.RunningStatus.String(), currentStatus)
|
||||
execution, err = e.executionDAO.Get(e.ctx, e.executionID)
|
||||
e.Require().Nil(err)
|
||||
e.Equal(job.RunningStatus.String(), execution.Status)
|
||||
|
|
|
|||
|
|
@ -56,13 +56,14 @@ type Metrics struct {
|
|||
// Task database model
|
||||
type Task struct {
|
||||
ID int64 `orm:"pk;auto;column(id)"`
|
||||
VendorType string `orm:"column(vendor_type)"`
|
||||
ExecutionID int64 `orm:"column(execution_id)"`
|
||||
JobID string `orm:"column(job_id)"`
|
||||
Status string `orm:"column(status)"`
|
||||
StatusCode int `orm:"column(status_code)"`
|
||||
StatusRevision int64 `orm:"column(status_revision)"`
|
||||
StatusMessage string `orm:"column(status_message)"`
|
||||
RunCount int `orm:"column(run_count)"`
|
||||
RunCount int32 `orm:"column(run_count)"`
|
||||
ExtraAttrs string `orm:"column(extra_attrs)"` // json string
|
||||
CreationTime time.Time `orm:"column(creation_time)"`
|
||||
StartTime time.Time `orm:"column(start_time)"`
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ package dao
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
|
|
@ -27,8 +29,10 @@ import (
|
|||
// TaskDAO is the data access object interface for task
|
||||
type TaskDAO interface {
|
||||
// Count returns the total count of tasks according to the query
|
||||
// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
|
||||
Count(ctx context.Context, query *q.Query) (count int64, err error)
|
||||
// List the tasks according to the query
|
||||
// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
|
||||
List(ctx context.Context, query *q.Query) (tasks []*Task, err error)
|
||||
// Get the specified task
|
||||
Get(ctx context.Context, id int64) (task *Task, err error)
|
||||
|
|
@ -60,7 +64,7 @@ func (t *taskDAO) Count(ctx context.Context, query *q.Query) (int64, error) {
|
|||
Keywords: query.Keywords,
|
||||
}
|
||||
}
|
||||
qs, err := orm.QuerySetter(ctx, &Task{}, query)
|
||||
qs, err := t.querySetter(ctx, query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
|
@ -69,7 +73,7 @@ func (t *taskDAO) Count(ctx context.Context, query *q.Query) (int64, error) {
|
|||
|
||||
func (t *taskDAO) List(ctx context.Context, query *q.Query) ([]*Task, error) {
|
||||
tasks := []*Task{}
|
||||
qs, err := orm.QuerySetter(ctx, &Task{}, query)
|
||||
qs, err := t.querySetter(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -203,3 +207,26 @@ func (t *taskDAO) GetMaxEndTime(ctx context.Context, executionID int64) (time.Ti
|
|||
QueryRow(&endTime)
|
||||
return endTime, nil
|
||||
}
|
||||
|
||||
func (t *taskDAO) querySetter(ctx context.Context, query *q.Query) (orm.QuerySeter, error) {
|
||||
qs, err := orm.QuerySetter(ctx, &Task{}, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// append the filter for "extra attrs"
|
||||
if query != nil && len(query.Keywords) > 0 {
|
||||
for key, value := range query.Keywords {
|
||||
if strings.HasPrefix(key, "ExtraAttrs.") {
|
||||
qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from task where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "ExtraAttrs."), value))
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(key, "extra_attrs.") {
|
||||
qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from task where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "extra_attrs."), value))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return qs, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ func (t *taskDAOTestSuite) SetupTest() {
|
|||
ExecutionID: t.executionID,
|
||||
Status: "success",
|
||||
StatusCode: 1,
|
||||
ExtraAttrs: "{}",
|
||||
ExtraAttrs: `{"key":"value"}`,
|
||||
})
|
||||
t.Require().Nil(err)
|
||||
t.taskID = id
|
||||
|
|
@ -70,22 +70,42 @@ func (t *taskDAOTestSuite) TearDownTest() {
|
|||
func (t *taskDAOTestSuite) TestCount() {
|
||||
count, err := t.taskDAO.Count(t.ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ExecutionID": t.executionID,
|
||||
"ExecutionID": t.executionID,
|
||||
"ExtraAttrs.key": "value",
|
||||
},
|
||||
})
|
||||
t.Require().Nil(err)
|
||||
t.Equal(int64(1), count)
|
||||
|
||||
count, err = t.taskDAO.Count(t.ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ExecutionID": t.executionID,
|
||||
"ExtraAttrs.key": "incorrect-value",
|
||||
},
|
||||
})
|
||||
t.Require().Nil(err)
|
||||
t.Equal(int64(0), count)
|
||||
}
|
||||
|
||||
func (t *taskDAOTestSuite) TestList() {
|
||||
tasks, err := t.taskDAO.List(t.ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ExecutionID": t.executionID,
|
||||
"ExecutionID": t.executionID,
|
||||
"ExtraAttrs.key": "value",
|
||||
},
|
||||
})
|
||||
t.Require().Nil(err)
|
||||
t.Require().Len(tasks, 1)
|
||||
t.Equal(t.taskID, tasks[0].ID)
|
||||
|
||||
tasks, err = t.taskDAO.List(t.ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ExecutionID": t.executionID,
|
||||
"ExtraAttrs.key": "incorrect-value",
|
||||
},
|
||||
})
|
||||
t.Require().Nil(err)
|
||||
t.Require().Len(tasks, 0)
|
||||
}
|
||||
|
||||
func (t *taskDAOTestSuite) TestGet() {
|
||||
|
|
@ -140,7 +160,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() {
|
|||
|
||||
task, err := t.taskDAO.Get(t.ctx, t.taskID)
|
||||
t.Require().Nil(err)
|
||||
t.Equal(1, task.RunCount)
|
||||
t.Equal(int32(1), task.RunCount)
|
||||
t.True(time.Unix(statusRevision, 0).Equal(task.StartTime))
|
||||
t.Equal(status, task.Status)
|
||||
t.Equal(job.RunningStatus.Code(), task.StatusCode)
|
||||
|
|
@ -155,7 +175,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() {
|
|||
|
||||
task, err = t.taskDAO.Get(t.ctx, t.taskID)
|
||||
t.Require().Nil(err)
|
||||
t.Equal(1, task.RunCount)
|
||||
t.Equal(int32(1), task.RunCount)
|
||||
t.True(time.Unix(statusRevision, 0).Equal(task.StartTime))
|
||||
t.Equal(status, task.Status)
|
||||
t.Equal(job.SuccessStatus.Code(), task.StatusCode)
|
||||
|
|
@ -170,7 +190,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() {
|
|||
|
||||
task, err = t.taskDAO.Get(t.ctx, t.taskID)
|
||||
t.Require().Nil(err)
|
||||
t.Equal(2, task.RunCount)
|
||||
t.Equal(int32(2), task.RunCount)
|
||||
t.True(time.Unix(statusRevision, 0).Equal(task.StartTime))
|
||||
t.Equal(status, task.Status)
|
||||
t.Equal(job.RunningStatus.Code(), task.StatusCode)
|
||||
|
|
|
|||
|
|
@ -62,8 +62,10 @@ type ExecutionManager interface {
|
|||
// Get the specified execution
|
||||
Get(ctx context.Context, id int64) (execution *Execution, err error)
|
||||
// List executions according to the query
|
||||
// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
|
||||
List(ctx context.Context, query *q.Query) (executions []*Execution, err error)
|
||||
// Count counts total.
|
||||
// Count counts total of executions according to the query.
|
||||
// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
|
||||
Count(ctx context.Context, query *q.Query) (int64, error)
|
||||
}
|
||||
|
||||
|
|
@ -100,6 +102,7 @@ func (e *executionManager) Create(ctx context.Context, vendorType string, vendor
|
|||
execution := &dao.Execution{
|
||||
VendorType: vendorType,
|
||||
VendorID: vendorID,
|
||||
Status: job.RunningStatus.String(),
|
||||
Trigger: trigger,
|
||||
ExtraAttrs: string(data),
|
||||
StartTime: time.Now(),
|
||||
|
|
@ -156,7 +159,11 @@ func (e *executionManager) Stop(ctx context.Context, id int64) error {
|
|||
continue
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
// refresh the status explicitly in case that the execution status
|
||||
// isn't refreshed by task status change hook
|
||||
_, _, err = e.executionDAO.RefreshStatus(ctx, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *executionManager) StopAndWait(ctx context.Context, id int64, timeout time.Duration) error {
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ func (e *executionManagerTestSuite) TestStop() {
|
|||
},
|
||||
}, nil)
|
||||
e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||
e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil)
|
||||
err = e.execMgr.Stop(nil, 1)
|
||||
e.Require().Nil(err)
|
||||
e.taskDAO.AssertExpectations(e.T())
|
||||
|
|
@ -124,6 +125,7 @@ func (e *executionManagerTestSuite) TestStopAndWait() {
|
|||
},
|
||||
}, nil)
|
||||
e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||
e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil)
|
||||
err := e.execMgr.StopAndWait(nil, 1, 1*time.Second)
|
||||
e.Require().NotNil(err)
|
||||
e.taskDAO.AssertExpectations(e.T())
|
||||
|
|
@ -145,6 +147,7 @@ func (e *executionManagerTestSuite) TestStopAndWait() {
|
|||
},
|
||||
}, nil)
|
||||
e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||
e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil)
|
||||
err = e.execMgr.StopAndWait(nil, 1, 1*time.Second)
|
||||
e.Require().Nil(err)
|
||||
e.taskDAO.AssertExpectations(e.T())
|
||||
|
|
|
|||
|
|
@ -19,9 +19,17 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/task/dao"
|
||||
)
|
||||
|
||||
var (
|
||||
// HkHandler is a global instance of the HookHandler
|
||||
HkHandler = NewHookHandler()
|
||||
)
|
||||
|
||||
// NewHookHandler creates a hook handler instance
|
||||
func NewHookHandler() *HookHandler {
|
||||
return &HookHandler{
|
||||
|
|
@ -37,31 +45,68 @@ type HookHandler struct {
|
|||
}
|
||||
|
||||
// Handle the job status changing webhook
|
||||
func (h *HookHandler) Handle(ctx context.Context, taskID int64, sc *job.StatusChange) error {
|
||||
task, err := h.taskDAO.Get(ctx, taskID)
|
||||
func (h *HookHandler) Handle(ctx context.Context, sc *job.StatusChange) error {
|
||||
logger := log.GetLogger(ctx)
|
||||
|
||||
jobID := sc.JobID
|
||||
// when a "KindScheduled" job is scheduled by a periodical job, it's "JobID" field
|
||||
// is set as "87bbdee19bed5ce09c48a149@1605104520" which contains "@". In this case,
|
||||
// read the parent periodical job ID from "sc.Metadata.UpstreamJobID"
|
||||
if sc.Metadata.JobKind == job.KindScheduled {
|
||||
if len(sc.Metadata.UpstreamJobID) > 0 {
|
||||
jobID = sc.Metadata.UpstreamJobID
|
||||
}
|
||||
}
|
||||
tasks, err := h.taskDAO.List(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"JobID": jobID,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return errors.New(nil).WithCode(errors.NotFoundCode).
|
||||
WithMessage("task with job ID %s not found", sc.JobID)
|
||||
}
|
||||
task := tasks[0]
|
||||
execution, err := h.executionDAO.Get(ctx, task.ExecutionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// process check in data
|
||||
if len(sc.CheckIn) > 0 {
|
||||
execution, err := h.executionDAO.Get(ctx, task.ExecutionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
processor, exist := registry[execution.VendorType]
|
||||
processor, exist := checkInProcessorRegistry[execution.VendorType]
|
||||
if !exist {
|
||||
return fmt.Errorf("the check in processor for task %d not found", taskID)
|
||||
return fmt.Errorf("the check in processor for task %d not found", task.ID)
|
||||
}
|
||||
t := &Task{}
|
||||
t.From(task)
|
||||
return processor(ctx, t, sc)
|
||||
return processor(ctx, t, sc.CheckIn)
|
||||
}
|
||||
|
||||
// update task status
|
||||
if err = h.taskDAO.UpdateStatus(ctx, taskID, sc.Status, sc.Metadata.Revision); err != nil {
|
||||
if err = h.taskDAO.UpdateStatus(ctx, task.ID, sc.Status, sc.Metadata.Revision); err != nil {
|
||||
return err
|
||||
}
|
||||
// run the status change post function
|
||||
if fc, exist := statusChangePostFuncRegistry[execution.VendorType]; exist {
|
||||
if err = fc(ctx, task.ID, sc.Status); err != nil {
|
||||
logger.Errorf("failed to run the task status change post function for task %d: %v", task.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// update execution status
|
||||
return h.executionDAO.RefreshStatus(ctx, task.ExecutionID)
|
||||
statusChanged, currentStatus, err := h.executionDAO.RefreshStatus(ctx, task.ExecutionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// run the status change post function
|
||||
if fc, exist := executionStatusChangePostFuncRegistry[execution.VendorType]; exist && statusChanged {
|
||||
if err = fc(ctx, task.ExecutionID, currentStatus); err != nil {
|
||||
logger.Errorf("failed to run the execution status change post function for execution %d: %v", task.ExecutionID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,19 +43,23 @@ func (h *hookHandlerTestSuite) SetupTest() {
|
|||
|
||||
func (h *hookHandlerTestSuite) TestHandle() {
|
||||
// handle check in data
|
||||
registry["test"] = func(ctx context.Context, task *Task, change *job.StatusChange) (err error) { return nil }
|
||||
h.taskDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Task{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
checkInProcessorRegistry["test"] = func(ctx context.Context, task *Task, data string) (err error) { return nil }
|
||||
defer delete(checkInProcessorRegistry, "test")
|
||||
h.taskDAO.On("List", mock.Anything, mock.Anything).Return([]*dao.Task{
|
||||
{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
},
|
||||
}, nil)
|
||||
h.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{
|
||||
ID: 1,
|
||||
VendorType: "test",
|
||||
}, nil)
|
||||
sc := &job.StatusChange{
|
||||
CheckIn: "data",
|
||||
CheckIn: "data",
|
||||
Metadata: &job.StatsInfo{},
|
||||
}
|
||||
err := h.handler.Handle(nil, 1, sc)
|
||||
err := h.handler.Handle(nil, sc)
|
||||
h.Require().Nil(err)
|
||||
h.taskDAO.AssertExpectations(h.T())
|
||||
h.execDAO.AssertExpectations(h.T())
|
||||
|
|
@ -64,21 +68,28 @@ func (h *hookHandlerTestSuite) TestHandle() {
|
|||
h.SetupTest()
|
||||
|
||||
// handle status changing
|
||||
h.taskDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Task{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
h.taskDAO.On("List", mock.Anything, mock.Anything).Return([]*dao.Task{
|
||||
{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
},
|
||||
}, nil)
|
||||
h.taskDAO.On("UpdateStatus", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
h.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(nil)
|
||||
h.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{
|
||||
ID: 1,
|
||||
VendorType: "test",
|
||||
}, nil)
|
||||
h.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(true, job.RunningStatus.String(), nil)
|
||||
sc = &job.StatusChange{
|
||||
Status: job.SuccessStatus.String(),
|
||||
Metadata: &job.StatsInfo{
|
||||
Revision: time.Now().Unix(),
|
||||
},
|
||||
}
|
||||
err = h.handler.Handle(nil, 1, sc)
|
||||
err = h.handler.Handle(nil, sc)
|
||||
h.Require().Nil(err)
|
||||
h.taskDAO.AssertExpectations(h.T())
|
||||
h.execDAO.AssertExpectations(h.T())
|
||||
}
|
||||
|
||||
func TestHookHandlerTestSuite(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -142,17 +142,31 @@ func (_m *mockExecutionDAO) List(ctx context.Context, query *q.Query) ([]*dao.Ex
|
|||
}
|
||||
|
||||
// RefreshStatus provides a mock function with given fields: ctx, id
|
||||
func (_m *mockExecutionDAO) RefreshStatus(ctx context.Context, id int64) error {
|
||||
func (_m *mockExecutionDAO) RefreshStatus(ctx context.Context, id int64) (bool, string, error) {
|
||||
ret := _m.Called(ctx, id)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
|
||||
var r0 bool
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64) bool); ok {
|
||||
r0 = rf(ctx, id)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
return r0
|
||||
var r1 string
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64) string); ok {
|
||||
r1 = rf(ctx, id)
|
||||
} else {
|
||||
r1 = ret.Get(1).(string)
|
||||
}
|
||||
|
||||
var r2 error
|
||||
if rf, ok := ret.Get(2).(func(context.Context, int64) error); ok {
|
||||
r2 = rf(ctx, id)
|
||||
} else {
|
||||
r2 = ret.Error(2)
|
||||
}
|
||||
|
||||
return r0, r1, r2
|
||||
}
|
||||
|
||||
// Update provides a mock function with given fields: ctx, execution, props
|
||||
|
|
|
|||
|
|
@ -25,13 +25,6 @@ import (
|
|||
|
||||
// const definitions
|
||||
const (
|
||||
ExecutionVendorTypeReplication = "REPLICATION"
|
||||
ExecutionVendorTypeGarbageCollection = "GARBAGE_COLLECTION"
|
||||
ExecutionVendorTypeRetention = "RETENTION"
|
||||
ExecutionVendorTypeScan = "SCAN"
|
||||
ExecutionVendorTypeScanAll = "SCAN_ALL"
|
||||
ExecutionVendorTypeScheduler = "SCHEDULER"
|
||||
|
||||
ExecutionTriggerManual = "MANUAL"
|
||||
ExecutionTriggerSchedule = "SCHEDULE"
|
||||
ExecutionTriggerEvent = "EVENT"
|
||||
|
|
@ -61,14 +54,18 @@ type Execution struct {
|
|||
|
||||
// Task is the unit for running. It stores the jobservice job records and related information
|
||||
type Task struct {
|
||||
ID int64 `json:"id"`
|
||||
ID int64 `json:"id"`
|
||||
// indicate the task type: replication/GC/retention/scan/etc.
|
||||
VendorType string `json:"vendor_type"`
|
||||
ExecutionID int64 `json:"execution_id"`
|
||||
Status string `json:"status"`
|
||||
// the detail message to explain the status in some cases. e.g.
|
||||
// When the job is failed to submit to jobservice, this field can be used to explain the reason
|
||||
StatusMessage string `json:"status_message"`
|
||||
// the underlying job may retry several times
|
||||
RunCount int `json:"run_count"`
|
||||
RunCount int32 `json:"run_count"`
|
||||
// the ID of jobservice job
|
||||
JobID string `json:"job_id"`
|
||||
// the customized attributes for different kinds of consumers
|
||||
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
|
||||
// the time that the task record created
|
||||
|
|
@ -82,10 +79,12 @@ type Task struct {
|
|||
// From constructs a task from DAO model
|
||||
func (t *Task) From(task *dao.Task) {
|
||||
t.ID = task.ID
|
||||
t.VendorType = task.VendorType
|
||||
t.ExecutionID = task.ExecutionID
|
||||
t.Status = task.Status
|
||||
t.StatusMessage = task.StatusMessage
|
||||
t.RunCount = task.RunCount
|
||||
t.JobID = task.JobID
|
||||
t.CreationTime = task.CreationTime
|
||||
t.StartTime = task.StartTime
|
||||
t.UpdateTime = task.UpdateTime
|
||||
|
|
@ -100,6 +99,22 @@ func (t *Task) From(task *dao.Task) {
|
|||
}
|
||||
}
|
||||
|
||||
// GetStringFromExtraAttrs returns the string value specified by key
|
||||
func (t *Task) GetStringFromExtraAttrs(key string) string {
|
||||
if len(t.ExtraAttrs) == 0 {
|
||||
return ""
|
||||
}
|
||||
rt, exist := t.ExtraAttrs[key]
|
||||
if !exist {
|
||||
return ""
|
||||
}
|
||||
str, ok := rt.(string)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// Job is the model represents the requested jobservice job
|
||||
type Job struct {
|
||||
Name string
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var (
|
||||
checkInProcessorRegistry = map[string]CheckInProcessor{}
|
||||
statusChangePostFuncRegistry = map[string]StatusChangePostFunc{}
|
||||
executionStatusChangePostFuncRegistry = map[string]ExecutionStatusChangePostFunc{}
|
||||
)
|
||||
|
||||
// CheckInProcessor is the processor to process the check in data which is sent by jobservice via webhook
|
||||
type CheckInProcessor func(ctx context.Context, task *Task, data string) (err error)
|
||||
|
||||
// StatusChangePostFunc is the function called after the task status changed
|
||||
type StatusChangePostFunc func(ctx context.Context, taskID int64, status string) (err error)
|
||||
|
||||
// ExecutionStatusChangePostFunc is the function called after the execution status changed
|
||||
type ExecutionStatusChangePostFunc func(ctx context.Context, executionID int64, status string) (err error)
|
||||
|
||||
// RegisterCheckInProcessor registers check in processor for the specific vendor type
|
||||
func RegisterCheckInProcessor(vendorType string, processor CheckInProcessor) error {
|
||||
if _, exist := checkInProcessorRegistry[vendorType]; exist {
|
||||
return fmt.Errorf("check in processor for %s already exists", vendorType)
|
||||
}
|
||||
checkInProcessorRegistry[vendorType] = processor
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterTaskStatusChangePostFunc registers a task status change post function for the specific vendor type
|
||||
func RegisterTaskStatusChangePostFunc(vendorType string, fc StatusChangePostFunc) error {
|
||||
if _, exist := statusChangePostFuncRegistry[vendorType]; exist {
|
||||
return fmt.Errorf("the task status change post function for %s already exists", vendorType)
|
||||
}
|
||||
statusChangePostFuncRegistry[vendorType] = fc
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterExecutionStatusChangePostFunc registers an execution status change post function for the specific vendor type
|
||||
func RegisterExecutionStatusChangePostFunc(vendorType string, fc ExecutionStatusChangePostFunc) error {
|
||||
if _, exist := executionStatusChangePostFuncRegistry[vendorType]; exist {
|
||||
return fmt.Errorf("the execution status change post function for %s already exists", vendorType)
|
||||
}
|
||||
executionStatusChangePostFuncRegistry[vendorType] = fc
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package task
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestRegisterCheckInProcessor(t *testing.T) {
|
||||
err := RegisterCheckInProcessor("test", nil)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// already exist
|
||||
err = RegisterCheckInProcessor("test", nil)
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
|
||||
func TestRegisterTaskStatusChangePostFunc(t *testing.T) {
|
||||
err := RegisterTaskStatusChangePostFunc("test", nil)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// already exist
|
||||
err = RegisterTaskStatusChangePostFunc("test", nil)
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
|
||||
func TestRegisterExecutionStatusChangePostFunc(t *testing.T) {
|
||||
err := RegisterExecutionStatusChangePostFunc("test", nil)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// already exist
|
||||
err = RegisterExecutionStatusChangePostFunc("test", nil)
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
|
|
@ -47,10 +47,12 @@ type Manager interface {
|
|||
// Get the specified task
|
||||
Get(ctx context.Context, id int64) (task *Task, err error)
|
||||
// List the tasks according to the query
|
||||
// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
|
||||
List(ctx context.Context, query *q.Query) (tasks []*Task, err error)
|
||||
// Get the log of the specified task
|
||||
GetLog(ctx context.Context, id int64) (log []byte, err error)
|
||||
// Count counts total.
|
||||
// Count counts total of tasks according to the query.
|
||||
// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
|
||||
Count(ctx context.Context, query *q.Query) (int64, error)
|
||||
}
|
||||
|
||||
|
|
@ -58,6 +60,7 @@ type Manager interface {
|
|||
func NewManager() Manager {
|
||||
return &manager{
|
||||
dao: dao.NewTaskDAO(),
|
||||
execDAO: dao.NewExecutionDAO(),
|
||||
jsClient: cjob.GlobalClient,
|
||||
coreURL: config.GetCoreURL(),
|
||||
}
|
||||
|
|
@ -65,6 +68,7 @@ func NewManager() Manager {
|
|||
|
||||
type manager struct {
|
||||
dao dao.TaskDAO
|
||||
execDAO dao.ExecutionDAO
|
||||
jsClient cjob.Client
|
||||
coreURL string
|
||||
}
|
||||
|
|
@ -84,22 +88,11 @@ func (m *manager) Create(ctx context.Context, executionID int64, jb *Job, extraA
|
|||
// submit job to jobservice
|
||||
jobID, err := m.submitJob(ctx, id, jb)
|
||||
if err != nil {
|
||||
// failed to submit job to jobservice, update the status of task to error
|
||||
err = fmt.Errorf("failed to submit job to jobservice: %v", err)
|
||||
log.Error(err)
|
||||
now := time.Now()
|
||||
err = m.dao.Update(ctx, &dao.Task{
|
||||
ID: id,
|
||||
Status: job.ErrorStatus.String(),
|
||||
StatusCode: job.ErrorStatus.Code(),
|
||||
StatusMessage: err.Error(),
|
||||
UpdateTime: now,
|
||||
EndTime: now,
|
||||
}, "Status", "StatusCode", "StatusMessage", "UpdateTime", "EndTime")
|
||||
if err != nil {
|
||||
log.Errorf("failed to update task %d: %v", id, err)
|
||||
// failed to submit job to jobservice, delete the task record
|
||||
if err := m.dao.Delete(ctx, id); err != nil {
|
||||
log.Errorf("failed to delete the task %d: %v", id, err)
|
||||
}
|
||||
return id, nil
|
||||
return 0, err
|
||||
}
|
||||
|
||||
log.Debugf("the task %d is submitted to jobservice, the job ID is %s", id, jobID)
|
||||
|
|
@ -116,6 +109,10 @@ func (m *manager) Create(ctx context.Context, executionID int64, jb *Job, extraA
|
|||
}
|
||||
|
||||
func (m *manager) createTaskRecord(ctx context.Context, executionID int64, extraAttrs ...map[string]interface{}) (int64, error) {
|
||||
exec, err := m.execDAO.Get(ctx, executionID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
extras := map[string]interface{}{}
|
||||
if len(extraAttrs) > 0 && extraAttrs[0] != nil {
|
||||
extras = extraAttrs[0]
|
||||
|
|
@ -127,6 +124,7 @@ func (m *manager) createTaskRecord(ctx context.Context, executionID int64, extra
|
|||
|
||||
now := time.Now()
|
||||
return m.dao.Create(ctx, &dao.Task{
|
||||
VendorType: exec.VendorType,
|
||||
ExecutionID: executionID,
|
||||
Status: job.PendingStatus.String(),
|
||||
StatusCode: job.PendingStatus.Code(),
|
||||
|
|
|
|||
|
|
@ -31,14 +31,17 @@ type taskManagerTestSuite struct {
|
|||
suite.Suite
|
||||
mgr *manager
|
||||
dao *mockTaskDAO
|
||||
execDAO *mockExecutionDAO
|
||||
jsClient *mockJobserviceClient
|
||||
}
|
||||
|
||||
func (t *taskManagerTestSuite) SetupTest() {
|
||||
t.dao = &mockTaskDAO{}
|
||||
t.execDAO = &mockExecutionDAO{}
|
||||
t.jsClient = &mockJobserviceClient{}
|
||||
t.mgr = &manager{
|
||||
dao: t.dao,
|
||||
execDAO: t.execDAO,
|
||||
jsClient: t.jsClient,
|
||||
}
|
||||
}
|
||||
|
|
@ -53,6 +56,7 @@ func (t *taskManagerTestSuite) TestCount() {
|
|||
|
||||
func (t *taskManagerTestSuite) TestCreate() {
|
||||
// success to submit job to jobservice
|
||||
t.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{}, nil)
|
||||
t.dao.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
t.jsClient.On("SubmitJob", mock.Anything).Return("1", nil)
|
||||
t.dao.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
|
|
@ -61,21 +65,22 @@ func (t *taskManagerTestSuite) TestCreate() {
|
|||
t.Require().Nil(err)
|
||||
t.Equal(int64(1), id)
|
||||
t.dao.AssertExpectations(t.T())
|
||||
t.execDAO.AssertExpectations(t.T())
|
||||
t.jsClient.AssertExpectations(t.T())
|
||||
|
||||
// reset mock
|
||||
t.SetupTest()
|
||||
|
||||
// failed to submit job to jobservice
|
||||
t.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{}, nil)
|
||||
t.dao.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
t.jsClient.On("SubmitJob", mock.Anything).Return("", errors.New("error"))
|
||||
t.dao.On("Update", mock.Anything, mock.Anything, mock.Anything,
|
||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
t.dao.On("Delete", mock.Anything, mock.Anything).Return(nil)
|
||||
|
||||
id, err = t.mgr.Create(nil, 1, &Job{}, map[string]interface{}{"a": "b"})
|
||||
t.Require().Nil(err)
|
||||
t.Equal(int64(1), id)
|
||||
t.Require().NotNil(err)
|
||||
t.dao.AssertExpectations(t.T())
|
||||
t.execDAO.AssertExpectations(t.T())
|
||||
t.jsClient.AssertExpectations(t.T())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,9 +23,7 @@ var (
|
|||
type Configuration struct {
|
||||
CoreURL string
|
||||
TokenServiceURL string
|
||||
JobserviceURL string
|
||||
SecretKey string
|
||||
// TODO consider to use a specified secret for replication
|
||||
CoreSecret string
|
||||
JobserviceSecret string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
package dao
|
||||
|
||||
import "github.com/astaxie/beego/orm"
|
||||
|
||||
func paginateForQuerySetter(qs orm.QuerySeter, page, size int64) orm.QuerySeter {
|
||||
if size > 0 {
|
||||
qs = qs.Limit(size)
|
||||
if page > 0 {
|
||||
qs = qs.Offset((page - 1) * size)
|
||||
}
|
||||
}
|
||||
return qs
|
||||
}
|
||||
|
|
@ -1,355 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package dao
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/astaxie/beego/orm"
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
)
|
||||
|
||||
// AddExecution ...
|
||||
func AddExecution(execution *models.Execution) (int64, error) {
|
||||
o := dao.GetOrmer()
|
||||
now := time.Now()
|
||||
execution.StartTime = now
|
||||
|
||||
return o.Insert(execution)
|
||||
}
|
||||
|
||||
// GetTotalOfExecutions returns the total count of replication execution
|
||||
func GetTotalOfExecutions(query ...*models.ExecutionQuery) (int64, error) {
|
||||
qs := executionQueryConditions(query...)
|
||||
return qs.Count()
|
||||
}
|
||||
|
||||
// GetExecutions ...
|
||||
func GetExecutions(query ...*models.ExecutionQuery) ([]*models.Execution, error) {
|
||||
executions := []*models.Execution{}
|
||||
|
||||
qs := executionQueryConditions(query...)
|
||||
if len(query) > 0 && query[0] != nil {
|
||||
qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size)
|
||||
}
|
||||
|
||||
qs = qs.OrderBy("-StartTime")
|
||||
|
||||
_, err := qs.All(&executions)
|
||||
if err != nil || len(executions) == 0 {
|
||||
return executions, err
|
||||
}
|
||||
for _, e := range executions {
|
||||
fillExecution(e)
|
||||
}
|
||||
return executions, err
|
||||
}
|
||||
|
||||
func executionQueryConditions(query ...*models.ExecutionQuery) orm.QuerySeter {
|
||||
qs := dao.GetOrmer().QueryTable(new(models.Execution))
|
||||
if len(query) == 0 || query[0] == nil {
|
||||
return qs
|
||||
}
|
||||
|
||||
q := query[0]
|
||||
if q.PolicyID != 0 {
|
||||
qs = qs.Filter("PolicyID", q.PolicyID)
|
||||
}
|
||||
if len(q.Trigger) > 0 {
|
||||
qs = qs.Filter("Trigger", q.Trigger)
|
||||
}
|
||||
if len(q.Statuses) > 0 {
|
||||
qs = qs.Filter("Status__in", q.Statuses)
|
||||
}
|
||||
return qs
|
||||
}
|
||||
|
||||
// GetExecution ...
|
||||
func GetExecution(id int64) (*models.Execution, error) {
|
||||
o := dao.GetOrmer()
|
||||
t := models.Execution{ID: id}
|
||||
err := o.Read(&t)
|
||||
if err == orm.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fillExecution(&t)
|
||||
return &t, err
|
||||
}
|
||||
|
||||
// fillExecution will fill the statistics data and status by tasks data
|
||||
func fillExecution(execution *models.Execution) error {
|
||||
if executionFinished(execution.Status) {
|
||||
return nil
|
||||
}
|
||||
|
||||
o := dao.GetOrmer()
|
||||
sql := `select status, count(*) as c from replication_task where execution_id = ? group by status`
|
||||
queryParam := make([]interface{}, 1)
|
||||
queryParam = append(queryParam, execution.ID)
|
||||
|
||||
dt := []*models.TaskStat{}
|
||||
count, err := o.Raw(sql, queryParam).QueryRows(&dt)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("Query tasks error execution %d: %v", execution.ID, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
total := 0
|
||||
for _, d := range dt {
|
||||
status, _ := getStatus(d.Status)
|
||||
updateStatusCount(execution, status, d.C)
|
||||
total += d.C
|
||||
}
|
||||
|
||||
if execution.Total != total {
|
||||
log.Debugf("execution task count inconsistent and fixed, executionID=%d, execution.total=%d, tasks.count=%d",
|
||||
execution.ID, execution.Total, total)
|
||||
execution.Total = total
|
||||
}
|
||||
resetExecutionStatus(execution)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getStatus(status string) (string, error) {
|
||||
switch status {
|
||||
case models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress:
|
||||
return models.ExecutionStatusInProgress, nil
|
||||
case models.TaskStatusSucceed:
|
||||
return models.ExecutionStatusSucceed, nil
|
||||
case models.TaskStatusStopped:
|
||||
return models.ExecutionStatusStopped, nil
|
||||
case models.TaskStatusFailed:
|
||||
return models.ExecutionStatusFailed, nil
|
||||
}
|
||||
return "", fmt.Errorf("Not support task status ")
|
||||
}
|
||||
|
||||
func updateStatusCount(execution *models.Execution, status string, delta int) error {
|
||||
switch status {
|
||||
case models.ExecutionStatusInProgress:
|
||||
execution.InProgress += delta
|
||||
case models.ExecutionStatusSucceed:
|
||||
execution.Succeed += delta
|
||||
case models.ExecutionStatusStopped:
|
||||
execution.Stopped += delta
|
||||
case models.ExecutionStatusFailed:
|
||||
execution.Failed += delta
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resetExecutionStatus(execution *models.Execution) error {
|
||||
execution.Status = generateStatus(execution)
|
||||
if executionFinished(execution.Status) {
|
||||
o := dao.GetOrmer()
|
||||
sql := `select max(end_time) from replication_task where execution_id = ?`
|
||||
queryParam := make([]interface{}, 1)
|
||||
queryParam = append(queryParam, execution.ID)
|
||||
|
||||
var et time.Time
|
||||
err := o.Raw(sql, queryParam).QueryRow(&et)
|
||||
if err != nil {
|
||||
log.Errorf("Query end_time from tasks error execution %d: %v", execution.ID, err)
|
||||
et = time.Now()
|
||||
}
|
||||
execution.EndTime = et
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func generateStatus(execution *models.Execution) string {
|
||||
if execution.InProgress > 0 {
|
||||
return models.ExecutionStatusInProgress
|
||||
} else if execution.Failed > 0 {
|
||||
return models.ExecutionStatusFailed
|
||||
} else if execution.Stopped > 0 {
|
||||
return models.ExecutionStatusStopped
|
||||
}
|
||||
return models.ExecutionStatusSucceed
|
||||
}
|
||||
|
||||
func executionFinished(status string) bool {
|
||||
if status == models.ExecutionStatusStopped ||
|
||||
status == models.ExecutionStatusSucceed ||
|
||||
status == models.ExecutionStatusFailed {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DeleteExecution ...
|
||||
func DeleteExecution(id int64) error {
|
||||
o := dao.GetOrmer()
|
||||
_, err := o.Delete(&models.Execution{ID: id})
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteAllExecutions ...
|
||||
func DeleteAllExecutions(policyID int64) error {
|
||||
o := dao.GetOrmer()
|
||||
_, err := o.Delete(&models.Execution{PolicyID: policyID}, "PolicyID")
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateExecution ...
|
||||
func UpdateExecution(execution *models.Execution, props ...string) (int64, error) {
|
||||
if execution.ID == 0 {
|
||||
return 0, fmt.Errorf("execution ID is empty")
|
||||
}
|
||||
o := dao.GetOrmer()
|
||||
return o.Update(execution, props...)
|
||||
}
|
||||
|
||||
// AddTask ...
|
||||
func AddTask(task *models.Task) (int64, error) {
|
||||
o := dao.GetOrmer()
|
||||
now := time.Now()
|
||||
task.StartTime = now
|
||||
|
||||
return o.Insert(task)
|
||||
}
|
||||
|
||||
// GetTask ...
|
||||
func GetTask(id int64) (*models.Task, error) {
|
||||
o := dao.GetOrmer()
|
||||
sql := `select * from replication_task where id = ?`
|
||||
|
||||
var task models.Task
|
||||
|
||||
if err := o.Raw(sql, id).QueryRow(&task); err != nil {
|
||||
if err == orm.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &task, nil
|
||||
}
|
||||
|
||||
// GetTotalOfTasks ...
|
||||
func GetTotalOfTasks(query ...*models.TaskQuery) (int64, error) {
|
||||
qs := taskQueryConditions(query...)
|
||||
return qs.Count()
|
||||
}
|
||||
|
||||
// GetTasks ...
|
||||
func GetTasks(query ...*models.TaskQuery) ([]*models.Task, error) {
|
||||
tasks := []*models.Task{}
|
||||
|
||||
qs := taskQueryConditions(query...)
|
||||
if len(query) > 0 && query[0] != nil {
|
||||
qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size)
|
||||
}
|
||||
|
||||
qs = qs.OrderBy("-StartTime")
|
||||
|
||||
_, err := qs.All(&tasks)
|
||||
return tasks, err
|
||||
}
|
||||
|
||||
func taskQueryConditions(query ...*models.TaskQuery) orm.QuerySeter {
|
||||
qs := dao.GetOrmer().QueryTable(new(models.Task))
|
||||
if len(query) == 0 || query[0] == nil {
|
||||
return qs
|
||||
}
|
||||
|
||||
q := query[0]
|
||||
if q.ExecutionID != 0 {
|
||||
qs = qs.Filter("ExecutionID", q.ExecutionID)
|
||||
}
|
||||
if len(q.JobID) > 0 {
|
||||
qs = qs.Filter("JobID", q.JobID)
|
||||
}
|
||||
if len(q.ResourceType) > 0 {
|
||||
qs = qs.Filter("ResourceType", q.ResourceType)
|
||||
}
|
||||
if len(q.Statuses) > 0 {
|
||||
qs = qs.Filter("Status__in", q.Statuses)
|
||||
}
|
||||
return qs
|
||||
}
|
||||
|
||||
// DeleteTask ...
|
||||
func DeleteTask(id int64) error {
|
||||
o := dao.GetOrmer()
|
||||
_, err := o.Delete(&models.Task{ID: id})
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteAllTasks ...
|
||||
func DeleteAllTasks(executionID int64) error {
|
||||
o := dao.GetOrmer()
|
||||
_, err := o.Delete(&models.Task{ExecutionID: executionID}, "ExecutionID")
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateTask ...
|
||||
func UpdateTask(task *models.Task, props ...string) (int64, error) {
|
||||
if task.ID == 0 {
|
||||
return 0, fmt.Errorf("task ID is empty")
|
||||
}
|
||||
o := dao.GetOrmer()
|
||||
return o.Update(task, props...)
|
||||
}
|
||||
|
||||
// UpdateTaskStatus updates the status of task.
|
||||
// The implementation uses raw sql rather than QuerySetter.Filter... as QuerySetter
|
||||
// will generate sql like:
|
||||
// `UPDATE "replication_task" SET "end_time" = $1, "status" = $2
|
||||
// WHERE "id" IN ( SELECT T0."id" FROM "replication_task" T0 WHERE T0."id" = $3
|
||||
// AND T0."status" IN ($4, $5, $6))]`
|
||||
// which is not a "single" sql statement, this will cause issues when running in concurrency
|
||||
func UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) (int64, error) {
|
||||
params := []interface{}{}
|
||||
sql := `update replication_task set status = ?, status_revision = ?, end_time = ? `
|
||||
params = append(params, status, statusRevision)
|
||||
var t time.Time
|
||||
// when the task is in final status, update the endtime
|
||||
// when the task re-runs again, the endtime should be cleared
|
||||
// so set the endtime to null if the task isn't in final status
|
||||
if taskFinished(status) {
|
||||
t = time.Now()
|
||||
}
|
||||
params = append(params, t)
|
||||
|
||||
sql += fmt.Sprintf(`where id = ? and (status_revision < ? or status_revision = ? and status in (%s)) `, dao.ParamPlaceholderForIn(len(statusCondition)))
|
||||
params = append(params, id, statusRevision, statusRevision, statusCondition)
|
||||
|
||||
result, err := dao.GetOrmer().Raw(sql, params...).Exec()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, _ := result.RowsAffected()
|
||||
if n > 0 {
|
||||
log.Debugf("update task status %d: -> %s", id, status)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func taskFinished(status string) bool {
|
||||
return status == models.TaskStatusFailed || status == models.TaskStatusStopped || status == models.TaskStatusSucceed
|
||||
}
|
||||
|
|
@ -1,294 +0,0 @@
|
|||
package dao
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMethodOfExecution(t *testing.T) {
|
||||
execution1 := &models.Execution{
|
||||
PolicyID: 11209,
|
||||
Status: "InProgress",
|
||||
StatusText: "None",
|
||||
Total: 12,
|
||||
Failed: 0,
|
||||
Succeed: 7,
|
||||
InProgress: 5,
|
||||
Stopped: 0,
|
||||
Trigger: "Event",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
execution2 := &models.Execution{
|
||||
PolicyID: 11209,
|
||||
Status: "Failed",
|
||||
StatusText: "Network error",
|
||||
Total: 9,
|
||||
Failed: 1,
|
||||
Succeed: 8,
|
||||
InProgress: 0,
|
||||
Stopped: 0,
|
||||
Trigger: "Manual",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
|
||||
// test add
|
||||
id1, err := AddExecution(execution1)
|
||||
require.Nil(t, err)
|
||||
|
||||
_, err = AddExecution(execution2)
|
||||
require.Nil(t, err)
|
||||
|
||||
// test list
|
||||
query := &models.ExecutionQuery{
|
||||
Statuses: []string{"InProgress", "Failed"},
|
||||
Pagination: models.Pagination{
|
||||
Page: 1,
|
||||
Size: 10,
|
||||
},
|
||||
}
|
||||
executions, err := GetExecutions(query)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 2, len(executions))
|
||||
|
||||
total, err := GetTotalOfExecutions(query)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(2), total)
|
||||
|
||||
// test get
|
||||
execution, err := GetExecution(id1)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, execution1.Status, execution.Status)
|
||||
|
||||
// test update
|
||||
executionNew := &models.Execution{
|
||||
ID: id1,
|
||||
Status: "Succeed",
|
||||
Succeed: 12,
|
||||
InProgress: 0,
|
||||
EndTime: time.Now(),
|
||||
}
|
||||
n, err := UpdateExecution(executionNew, models.ExecutionPropsName.Status, models.ExecutionPropsName.Succeed, models.ExecutionPropsName.InProgress,
|
||||
models.ExecutionPropsName.EndTime)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), n)
|
||||
|
||||
// test delete
|
||||
require.Nil(t, DeleteExecution(execution1.ID))
|
||||
execution, err = GetExecution(execution1.ID)
|
||||
require.Nil(t, err)
|
||||
require.Nil(t, execution)
|
||||
|
||||
// test delete all
|
||||
require.Nil(t, DeleteAllExecutions(execution1.PolicyID))
|
||||
query = &models.ExecutionQuery{}
|
||||
n, err = GetTotalOfExecutions(query)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(0), n)
|
||||
}
|
||||
|
||||
func TestMethodOfTask(t *testing.T) {
|
||||
now := time.Now()
|
||||
task1 := &models.Task{
|
||||
ExecutionID: 112200,
|
||||
ResourceType: "resourceType1",
|
||||
SrcResource: "srcResource1",
|
||||
DstResource: "dstResource1",
|
||||
JobID: "jobID1",
|
||||
Status: "Initialized",
|
||||
StatusRevision: 1,
|
||||
StartTime: now,
|
||||
}
|
||||
task2 := &models.Task{
|
||||
ExecutionID: 112200,
|
||||
ResourceType: "resourceType2",
|
||||
SrcResource: "srcResource2",
|
||||
DstResource: "dstResource2",
|
||||
JobID: "jobID2",
|
||||
Status: "Stopped",
|
||||
StatusRevision: 1,
|
||||
StartTime: now,
|
||||
EndTime: now,
|
||||
}
|
||||
|
||||
// test add
|
||||
id1, err := AddTask(task1)
|
||||
require.Nil(t, err)
|
||||
|
||||
_, err = AddTask(task2)
|
||||
require.Nil(t, err)
|
||||
|
||||
// test list
|
||||
query := &models.TaskQuery{
|
||||
ResourceType: "resourceType1",
|
||||
Pagination: models.Pagination{
|
||||
Page: 1,
|
||||
Size: 10,
|
||||
},
|
||||
}
|
||||
tasks, err := GetTasks(query)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, len(tasks))
|
||||
|
||||
total, err := GetTotalOfTasks(query)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), total)
|
||||
|
||||
// test get
|
||||
task, err := GetTask(id1)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, task1.Status, task.Status)
|
||||
|
||||
// test update
|
||||
taskNew := &models.Task{
|
||||
ID: id1,
|
||||
Status: "Failed",
|
||||
EndTime: now,
|
||||
}
|
||||
n, err := UpdateTask(taskNew, models.TaskPropsName.Status, models.TaskPropsName.EndTime)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), n)
|
||||
|
||||
// test update status
|
||||
n, err = UpdateTaskStatus(id1, "Succeed", 2, "Initialized")
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), n)
|
||||
task, _ = GetTask(id1)
|
||||
assert.Equal(t, "Succeed", task.Status)
|
||||
assert.Equal(t, int64(2), task.StatusRevision)
|
||||
|
||||
// test delete
|
||||
require.Nil(t, DeleteTask(id1))
|
||||
task, err = GetTask(id1)
|
||||
require.Nil(t, err)
|
||||
require.Nil(t, task)
|
||||
|
||||
// test delete all
|
||||
require.Nil(t, DeleteAllTasks(task1.ExecutionID))
|
||||
query = &models.TaskQuery{}
|
||||
n, err = GetTotalOfTasks(query)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(0), n)
|
||||
}
|
||||
|
||||
func TestExecutionFill(t *testing.T) {
|
||||
now := time.Now()
|
||||
execution := &models.Execution{
|
||||
PolicyID: 11209,
|
||||
Status: "InProgress",
|
||||
StatusText: "None",
|
||||
Total: 2,
|
||||
Trigger: "Event",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
executionID, _ := AddExecution(execution)
|
||||
et1, _ := time.Parse("2006-01-02 15:04:05", "2019-03-21 08:01:01")
|
||||
et2, _ := time.Parse("2006-01-02 15:04:05", "2019-04-01 10:11:53")
|
||||
task1 := &models.Task{
|
||||
ID: 20191,
|
||||
ExecutionID: executionID,
|
||||
ResourceType: "resourceType1",
|
||||
SrcResource: "srcResource1",
|
||||
DstResource: "dstResource1",
|
||||
JobID: "jobID1",
|
||||
Status: "Succeed",
|
||||
StartTime: now,
|
||||
EndTime: et1,
|
||||
}
|
||||
task2 := &models.Task{
|
||||
ID: 20192,
|
||||
ExecutionID: executionID,
|
||||
ResourceType: "resourceType2",
|
||||
SrcResource: "srcResource2",
|
||||
DstResource: "dstResource2",
|
||||
JobID: "jobID2",
|
||||
Status: "Stopped",
|
||||
StartTime: now,
|
||||
EndTime: et2,
|
||||
}
|
||||
AddTask(task1)
|
||||
AddTask(task2)
|
||||
|
||||
defer func() {
|
||||
DeleteAllTasks(executionID)
|
||||
DeleteAllExecutions(11209)
|
||||
}()
|
||||
|
||||
// query and fill
|
||||
exe, err := GetExecution(executionID)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, "Stopped", exe.Status)
|
||||
assert.Equal(t, 0, exe.InProgress)
|
||||
assert.Equal(t, 1, exe.Stopped)
|
||||
assert.Equal(t, 0, exe.Failed)
|
||||
assert.Equal(t, 1, exe.Succeed)
|
||||
assert.Equal(t, et2.Second(), exe.EndTime.Second())
|
||||
}
|
||||
|
||||
func TestExecutionFill2(t *testing.T) {
|
||||
now := time.Now()
|
||||
execution := &models.Execution{
|
||||
PolicyID: 11209,
|
||||
Status: "InProgress",
|
||||
StatusText: "None",
|
||||
Total: 2,
|
||||
Trigger: "Event",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
executionID, _ := AddExecution(execution)
|
||||
task1 := &models.Task{
|
||||
ID: 20191,
|
||||
ExecutionID: executionID,
|
||||
ResourceType: "resourceType1",
|
||||
SrcResource: "srcResource1",
|
||||
DstResource: "dstResource1",
|
||||
JobID: "jobID1",
|
||||
Status: models.TaskStatusInProgress,
|
||||
StatusRevision: 1,
|
||||
StartTime: now,
|
||||
}
|
||||
task2 := &models.Task{
|
||||
ID: 20192,
|
||||
ExecutionID: executionID,
|
||||
ResourceType: "resourceType2",
|
||||
SrcResource: "srcResource2",
|
||||
DstResource: "dstResource2",
|
||||
JobID: "jobID2",
|
||||
Status: "Stopped",
|
||||
StatusRevision: 1,
|
||||
StartTime: now,
|
||||
EndTime: now,
|
||||
}
|
||||
taskID1, _ := AddTask(task1)
|
||||
AddTask(task2)
|
||||
|
||||
defer func() {
|
||||
DeleteAllTasks(executionID)
|
||||
DeleteAllExecutions(11209)
|
||||
}()
|
||||
|
||||
// query and fill
|
||||
exe, err := GetExecution(executionID)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, models.ExecutionStatusInProgress, exe.Status)
|
||||
assert.Equal(t, 1, exe.InProgress)
|
||||
assert.Equal(t, 1, exe.Stopped)
|
||||
assert.Equal(t, 0, exe.Failed)
|
||||
assert.Equal(t, 0, exe.Succeed)
|
||||
|
||||
// update task status and query and fill
|
||||
UpdateTaskStatus(taskID1, models.TaskStatusFailed, 2, models.TaskStatusInProgress)
|
||||
exes, err := GetExecutions(&models.ExecutionQuery{
|
||||
PolicyID: 11209,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, len(exes))
|
||||
assert.Equal(t, models.ExecutionStatusFailed, exes[0].Status)
|
||||
assert.Equal(t, 0, exes[0].InProgress)
|
||||
assert.Equal(t, 1, exes[0].Stopped)
|
||||
assert.Equal(t, 1, exes[0].Failed)
|
||||
assert.Equal(t, 0, exes[0].Succeed)
|
||||
}
|
||||
|
|
@ -7,14 +7,5 @@ import (
|
|||
func init() {
|
||||
orm.RegisterModel(
|
||||
new(Registry),
|
||||
new(RepPolicy),
|
||||
new(Execution),
|
||||
new(Task),
|
||||
new(ScheduleJob))
|
||||
}
|
||||
|
||||
// Pagination ...
|
||||
type Pagination struct {
|
||||
Page int64
|
||||
Size int64
|
||||
new(RepPolicy))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,156 +0,0 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
const (
|
||||
// ExecutionTable is the table name for replication executions
|
||||
ExecutionTable = "replication_execution"
|
||||
// TaskTable is table name for replication tasks
|
||||
TaskTable = "replication_task"
|
||||
)
|
||||
|
||||
// execution/task status/trigger const
|
||||
const (
|
||||
ExecutionStatusFailed string = "Failed"
|
||||
ExecutionStatusSucceed string = "Succeed"
|
||||
ExecutionStatusStopped string = "Stopped"
|
||||
ExecutionStatusInProgress string = "InProgress"
|
||||
|
||||
ExecutionTriggerManual string = "Manual"
|
||||
ExecutionTriggerEvent string = "Event"
|
||||
ExecutionTriggerSchedule string = "Schedule"
|
||||
|
||||
// The task has been persisted in db but not submitted to Jobservice
|
||||
TaskStatusInitialized string = "Initialized"
|
||||
TaskStatusPending string = "Pending"
|
||||
TaskStatusInProgress string = "InProgress"
|
||||
TaskStatusSucceed string = "Succeed"
|
||||
TaskStatusFailed string = "Failed"
|
||||
TaskStatusStopped string = "Stopped"
|
||||
)
|
||||
|
||||
// ExecutionPropsName defines the names of fields of Execution
|
||||
var ExecutionPropsName = ExecutionFieldsName{
|
||||
ID: "ID",
|
||||
PolicyID: "PolicyID",
|
||||
Status: "Status",
|
||||
StatusText: "StatusText",
|
||||
Total: "Total",
|
||||
Failed: "Failed",
|
||||
Succeed: "Succeed",
|
||||
InProgress: "InProgress",
|
||||
Stopped: "Stopped",
|
||||
Trigger: "Trigger",
|
||||
StartTime: "StartTime",
|
||||
EndTime: "EndTime",
|
||||
}
|
||||
|
||||
// ExecutionFieldsName defines the props of Execution
|
||||
type ExecutionFieldsName struct {
|
||||
ID string
|
||||
PolicyID string
|
||||
Status string
|
||||
StatusText string
|
||||
Total string
|
||||
Failed string
|
||||
Succeed string
|
||||
InProgress string
|
||||
Stopped string
|
||||
Trigger string
|
||||
StartTime string
|
||||
EndTime string
|
||||
}
|
||||
|
||||
// Execution holds information about once replication execution.
|
||||
type Execution struct {
|
||||
ID int64 `orm:"pk;auto;column(id)" json:"id"`
|
||||
PolicyID int64 `orm:"column(policy_id)" json:"policy_id"`
|
||||
Status string `orm:"column(status)" json:"status"`
|
||||
StatusText string `orm:"column(status_text)" json:"status_text"`
|
||||
Total int `orm:"column(total)" json:"total"`
|
||||
Failed int `orm:"column(failed)" json:"failed"`
|
||||
Succeed int `orm:"column(succeed)" json:"succeed"`
|
||||
InProgress int `orm:"column(in_progress)" json:"in_progress"`
|
||||
Stopped int `orm:"column(stopped)" json:"stopped"`
|
||||
Trigger model.TriggerType `orm:"column(trigger)" json:"trigger"`
|
||||
StartTime time.Time `orm:"column(start_time)" json:"start_time"`
|
||||
EndTime time.Time `orm:"column(end_time)" json:"end_time"`
|
||||
}
|
||||
|
||||
// TaskPropsName defines the names of fields of Task
|
||||
var TaskPropsName = TaskFieldsName{
|
||||
ID: "ID",
|
||||
ExecutionID: "ExecutionID",
|
||||
ResourceType: "ResourceType",
|
||||
SrcResource: "SrcResource",
|
||||
DstResource: "DstResource",
|
||||
JobID: "JobID",
|
||||
Status: "Status",
|
||||
StartTime: "StartTime",
|
||||
EndTime: "EndTime",
|
||||
}
|
||||
|
||||
// TaskFieldsName defines the props of Task
|
||||
type TaskFieldsName struct {
|
||||
ID string
|
||||
ExecutionID string
|
||||
ResourceType string
|
||||
SrcResource string
|
||||
DstResource string
|
||||
JobID string
|
||||
Status string
|
||||
StartTime string
|
||||
EndTime string
|
||||
}
|
||||
|
||||
// Task represent the tasks in one execution.
|
||||
type Task struct {
|
||||
ID int64 `orm:"pk;auto;column(id)" json:"id"`
|
||||
ExecutionID int64 `orm:"column(execution_id)" json:"execution_id"`
|
||||
ResourceType string `orm:"column(resource_type)" json:"resource_type"`
|
||||
SrcResource string `orm:"column(src_resource)" json:"src_resource"`
|
||||
DstResource string `orm:"column(dst_resource)" json:"dst_resource"`
|
||||
Operation string `orm:"column(operation)" json:"operation"`
|
||||
JobID string `orm:"column(job_id)" json:"job_id"`
|
||||
Status string `orm:"column(status)" json:"status"`
|
||||
StatusRevision int64 `orm:"column(status_revision)"`
|
||||
StartTime time.Time `orm:"column(start_time)" json:"start_time"`
|
||||
EndTime time.Time `orm:"column(end_time)" json:"end_time,omitempty"`
|
||||
}
|
||||
|
||||
// TableName is required by by beego orm to map Execution to table replication_execution
|
||||
func (r *Execution) TableName() string {
|
||||
return ExecutionTable
|
||||
}
|
||||
|
||||
// TableName is required by by beego orm to map Task to table replication_task
|
||||
func (r *Task) TableName() string {
|
||||
return TaskTable
|
||||
}
|
||||
|
||||
// ExecutionQuery holds the query conditions for replication executions
|
||||
type ExecutionQuery struct {
|
||||
PolicyID int64
|
||||
Statuses []string
|
||||
Trigger string
|
||||
Pagination
|
||||
}
|
||||
|
||||
// TaskQuery holds the query conditions for replication task
|
||||
type TaskQuery struct {
|
||||
ExecutionID int64
|
||||
JobID string
|
||||
Statuses []string
|
||||
ResourceType string
|
||||
Pagination
|
||||
}
|
||||
|
||||
// TaskStat holds statistics of task by status
|
||||
type TaskStat struct {
|
||||
Status string `orm:"column(status)"`
|
||||
C int `orm:"column(c)"`
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// TODO rename the package name to model
|
||||
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
// ScheduleJob is the persistent model for the schedule job which is
|
||||
// used as a scheduler
|
||||
type ScheduleJob struct {
|
||||
ID int64 `orm:"pk;auto;column(id)" json:"id"`
|
||||
PolicyID int64 `orm:"column(policy_id)" json:"policy_id"`
|
||||
JobID string `orm:"column(job_id)" json:"job_id"`
|
||||
Status string `orm:"column(status)" json:"status"`
|
||||
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
|
||||
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
|
||||
}
|
||||
|
||||
// TableName is required by by beego orm to map the object to the database table
|
||||
func (s *ScheduleJob) TableName() string {
|
||||
return "replication_schedule_job"
|
||||
}
|
||||
|
||||
// ScheduleJobQuery is the query used to list schedule jobs
|
||||
type ScheduleJobQuery struct {
|
||||
PolicyID int64
|
||||
}
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package dao
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/astaxie/beego/orm"
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
)
|
||||
|
||||
// ScheduleJob is the DAO for schedule job
|
||||
var ScheduleJob ScheduleJobDAO = &scheduleJobDAO{}
|
||||
|
||||
// ScheduleJobDAO ...
|
||||
type ScheduleJobDAO interface {
|
||||
Add(*models.ScheduleJob) (int64, error)
|
||||
Get(int64) (*models.ScheduleJob, error)
|
||||
Update(*models.ScheduleJob, ...string) error
|
||||
Delete(int64) error
|
||||
List(...*models.ScheduleJobQuery) ([]*models.ScheduleJob, error)
|
||||
}
|
||||
|
||||
type scheduleJobDAO struct{}
|
||||
|
||||
func (s *scheduleJobDAO) Add(sj *models.ScheduleJob) (int64, error) {
|
||||
now := time.Now()
|
||||
sj.CreationTime = now
|
||||
sj.UpdateTime = now
|
||||
return dao.GetOrmer().Insert(sj)
|
||||
}
|
||||
|
||||
func (s *scheduleJobDAO) Get(id int64) (*models.ScheduleJob, error) {
|
||||
sj := &models.ScheduleJob{
|
||||
ID: id,
|
||||
}
|
||||
if err := dao.GetOrmer().Read(sj); err != nil {
|
||||
if err == orm.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return sj, nil
|
||||
}
|
||||
|
||||
func (s *scheduleJobDAO) Update(sj *models.ScheduleJob, props ...string) error {
|
||||
if sj.UpdateTime.IsZero() {
|
||||
now := time.Now()
|
||||
sj.UpdateTime = now
|
||||
if len(props) > 0 {
|
||||
props = append(props, "UpdateTime")
|
||||
}
|
||||
}
|
||||
|
||||
_, err := dao.GetOrmer().Update(sj, props...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *scheduleJobDAO) Delete(id int64) error {
|
||||
_, err := dao.GetOrmer().Delete(&models.ScheduleJob{
|
||||
ID: id,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *scheduleJobDAO) List(query ...*models.ScheduleJobQuery) ([]*models.ScheduleJob, error) {
|
||||
qs := dao.GetOrmer().QueryTable(&models.ScheduleJob{})
|
||||
if len(query) > 0 && query[0] != nil {
|
||||
if query[0].PolicyID > 0 {
|
||||
qs = qs.Filter("PolicyID", query[0].PolicyID)
|
||||
}
|
||||
}
|
||||
sjs := []*models.ScheduleJob{}
|
||||
_, err := qs.All(&sjs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sjs, nil
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package dao
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var sjID int64
|
||||
|
||||
func TestAddScheduleJob(t *testing.T) {
|
||||
sj := &models.ScheduleJob{
|
||||
PolicyID: 1,
|
||||
JobID: "uuid",
|
||||
Status: "running",
|
||||
}
|
||||
id, err := ScheduleJob.Add(sj)
|
||||
require.Nil(t, err)
|
||||
sjID = id
|
||||
}
|
||||
|
||||
func TestUpdateScheduleJob(t *testing.T) {
|
||||
err := ScheduleJob.Update(&models.ScheduleJob{
|
||||
ID: sjID,
|
||||
Status: "success",
|
||||
}, "Status")
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestGetScheduleJob(t *testing.T) {
|
||||
sj, err := ScheduleJob.Get(sjID)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), sj.PolicyID)
|
||||
assert.Equal(t, "success", sj.Status)
|
||||
}
|
||||
|
||||
func TestListScheduleJobs(t *testing.T) {
|
||||
// nil query
|
||||
sjs, err := ScheduleJob.List()
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, len(sjs))
|
||||
|
||||
// query
|
||||
sjs, err = ScheduleJob.List(&models.ScheduleJobQuery{
|
||||
PolicyID: 1,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, len(sjs))
|
||||
|
||||
// query
|
||||
sjs, err = ScheduleJob.List(&models.ScheduleJobQuery{
|
||||
PolicyID: 2,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 0, len(sjs))
|
||||
}
|
||||
|
||||
func TestDeleteScheduleJob(t *testing.T) {
|
||||
err := ScheduleJob.Delete(sjID)
|
||||
require.Nil(t, err)
|
||||
|
||||
sj, err := ScheduleJob.Get(sjID)
|
||||
require.Nil(t, err)
|
||||
assert.Nil(t, sj)
|
||||
}
|
||||
|
|
@ -17,12 +17,14 @@ package event
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/goharbor/harbor/src/lib/orm"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
|
||||
commonthttp "github.com/goharbor/harbor/src/common/http"
|
||||
"github.com/goharbor/harbor/src/controller/replication"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/replication/config"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/operation"
|
||||
"github.com/goharbor/harbor/src/replication/policy"
|
||||
"github.com/goharbor/harbor/src/replication/registry"
|
||||
"github.com/goharbor/harbor/src/replication/util"
|
||||
|
|
@ -34,18 +36,18 @@ type Handler interface {
|
|||
}
|
||||
|
||||
// NewHandler ...
|
||||
func NewHandler(policyCtl policy.Controller, registryMgr registry.Manager, opCtl operation.Controller) Handler {
|
||||
func NewHandler(policyCtl policy.Controller, registryMgr registry.Manager) Handler {
|
||||
return &handler{
|
||||
policyCtl: policyCtl,
|
||||
registryMgr: registryMgr,
|
||||
opCtl: opCtl,
|
||||
ctl: replication.Ctl,
|
||||
}
|
||||
}
|
||||
|
||||
type handler struct {
|
||||
policyCtl policy.Controller
|
||||
registryMgr registry.Manager
|
||||
opCtl operation.Controller
|
||||
ctl replication.Controller
|
||||
}
|
||||
|
||||
func (h *handler) Handle(event *Event) error {
|
||||
|
|
@ -76,7 +78,7 @@ func (h *handler) Handle(event *Event) error {
|
|||
if err := PopulateRegistries(h.registryMgr, policy); err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := h.opCtl.StartReplication(policy, event.Resource, model.TriggerTypeEventBased)
|
||||
id, err := h.ctl.Start(orm.Context(), policy, event.Resource, task.ExecutionTriggerEvent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,43 +15,18 @@
|
|||
package event
|
||||
|
||||
import (
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/testing/controller/replication"
|
||||
"github.com/goharbor/harbor/src/testing/mock"
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/replication/config"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakedOperationController struct{}
|
||||
|
||||
func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
|
||||
return 1, nil
|
||||
}
|
||||
func (f *fakedOperationController) StopReplication(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) GetExecution(id int64) (*models.Execution, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) GetTask(id int64) (*models.Task, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type fakedPolicyController struct{}
|
||||
|
||||
func (f *fakedPolicyController) Create(*model.Policy) (int64, error) {
|
||||
|
|
@ -236,10 +211,16 @@ func TestGetRelatedPolicies(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestHandle(t *testing.T) {
|
||||
dao.PrepareTestForPostgresSQL()
|
||||
config.Config = &config.Configuration{}
|
||||
handler := NewHandler(&fakedPolicyController{},
|
||||
&fakedRegistryManager{},
|
||||
&fakedOperationController{})
|
||||
ctl := &replication.Controller{}
|
||||
ctl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
handler := &handler{
|
||||
policyCtl: &fakedPolicyController{},
|
||||
registryMgr: &fakedRegistryManager{},
|
||||
ctl: ctl,
|
||||
}
|
||||
|
||||
// nil event
|
||||
err := handler.Handle(nil)
|
||||
require.NotNil(t, err)
|
||||
|
|
|
|||
|
|
@ -1,243 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package operation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/job"
|
||||
hjob "github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/operation/execution"
|
||||
"github.com/goharbor/harbor/src/replication/operation/flow"
|
||||
"github.com/goharbor/harbor/src/replication/operation/scheduler"
|
||||
)
|
||||
|
||||
// Controller handles the replication-related operations: start,
|
||||
// stop, query, etc.
|
||||
type Controller interface {
|
||||
// trigger is used to specify what this replication is triggered by
|
||||
StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error)
|
||||
StopReplication(int64) error
|
||||
ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error)
|
||||
GetExecution(int64) (*models.Execution, error)
|
||||
ListTasks(...*models.TaskQuery) (int64, []*models.Task, error)
|
||||
GetTask(int64) (*models.Task, error)
|
||||
UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error
|
||||
GetTaskLog(int64) ([]byte, error)
|
||||
}
|
||||
|
||||
const (
|
||||
maxReplicators = 1024
|
||||
)
|
||||
|
||||
// NewController returns a controller implementation
|
||||
func NewController(js job.Client) Controller {
|
||||
ctl := &controller{
|
||||
replicators: make(chan struct{}, maxReplicators),
|
||||
executionMgr: execution.NewDefaultManager(),
|
||||
scheduler: scheduler.NewScheduler(js),
|
||||
flowCtl: flow.NewController(),
|
||||
}
|
||||
for i := 0; i < maxReplicators; i++ {
|
||||
ctl.replicators <- struct{}{}
|
||||
}
|
||||
return ctl
|
||||
}
|
||||
|
||||
type controller struct {
|
||||
replicators chan struct{}
|
||||
flowCtl flow.Controller
|
||||
executionMgr execution.Manager
|
||||
scheduler scheduler.Scheduler
|
||||
}
|
||||
|
||||
func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
|
||||
if !policy.Enabled {
|
||||
return 0, fmt.Errorf("the policy %d is disabled", policy.ID)
|
||||
}
|
||||
if len(trigger) == 0 {
|
||||
trigger = model.TriggerTypeManual
|
||||
}
|
||||
id, err := createExecution(c.executionMgr, policy.ID, trigger)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// control the count of concurrent replication requests
|
||||
log.Debugf("waiting for the available replicator ...")
|
||||
<-c.replicators
|
||||
log.Debugf("got an available replicator, starting the replication ...")
|
||||
go func() {
|
||||
defer func() {
|
||||
c.replicators <- struct{}{}
|
||||
}()
|
||||
flow := c.createFlow(id, policy, resource)
|
||||
if n, err := c.flowCtl.Start(flow); err != nil {
|
||||
// only update the execution when got error.
|
||||
// if got no error, it will be updated automatically
|
||||
// when listing the execution records
|
||||
if e := c.executionMgr.Update(&models.Execution{
|
||||
ID: id,
|
||||
Status: models.ExecutionStatusFailed,
|
||||
StatusText: err.Error(),
|
||||
Total: n,
|
||||
Failed: n,
|
||||
}, "Status", "StatusText", "Total", "Failed"); e != nil {
|
||||
log.Errorf("failed to update the execution %d: %v", id, e)
|
||||
}
|
||||
log.Errorf("the execution %d failed: %v", id, err)
|
||||
}
|
||||
}()
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// create different replication flows according to the input parameters
|
||||
func (c *controller) createFlow(executionID int64, policy *model.Policy, resource *model.Resource) flow.Flow {
|
||||
// replicate the deletion operation, so create a deletion flow
|
||||
if resource != nil && resource.Deleted {
|
||||
return flow.NewDeletionFlow(c.executionMgr, c.scheduler, executionID, policy, resource)
|
||||
}
|
||||
resources := []*model.Resource{}
|
||||
if resource != nil {
|
||||
resources = append(resources, resource)
|
||||
}
|
||||
return flow.NewCopyFlow(c.executionMgr, c.scheduler, executionID, policy, resources...)
|
||||
}
|
||||
|
||||
func (c *controller) StopReplication(executionID int64) error {
|
||||
_, tasks, err := c.ListTasks(&models.TaskQuery{
|
||||
ExecutionID: executionID,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// no tasks, just set its status to "stopped"
|
||||
if len(tasks) == 0 {
|
||||
execution, err := c.executionMgr.Get(executionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if execution == nil {
|
||||
return fmt.Errorf("the execution %d not found", executionID)
|
||||
}
|
||||
if execution.Status != models.ExecutionStatusInProgress {
|
||||
log.Debugf("the execution %d isn't in progress, no need to stop", executionID)
|
||||
return nil
|
||||
}
|
||||
if err = c.executionMgr.Update(&models.Execution{
|
||||
ID: executionID,
|
||||
Status: models.ExecutionStatusStopped,
|
||||
EndTime: time.Now(),
|
||||
}, models.ExecutionPropsName.Status, models.ExecutionPropsName.EndTime); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debugf("the status of execution %d is set to stopped", executionID)
|
||||
}
|
||||
// got tasks, stopping the tasks one by one
|
||||
for _, task := range tasks {
|
||||
if isTaskInFinalStatus(task) {
|
||||
log.Debugf("the task %d(job ID: %s) is in final status, its status is %s, skip", task.ID, task.JobID, task.Status)
|
||||
continue
|
||||
}
|
||||
if err = c.scheduler.Stop(task.JobID); err != nil {
|
||||
isStatusBehindError, ok := err.(*job.StatusBehindError)
|
||||
if ok {
|
||||
status := isStatusBehindError.Status()
|
||||
switch hjob.Status(status) {
|
||||
case hjob.ErrorStatus:
|
||||
status = models.TaskStatusFailed
|
||||
case hjob.SuccessStatus:
|
||||
status = models.TaskStatusSucceed
|
||||
}
|
||||
e := c.executionMgr.UpdateTask(&models.Task{
|
||||
ID: task.ID,
|
||||
Status: status,
|
||||
}, "Status")
|
||||
if e != nil {
|
||||
log.Errorf("failed to update the status the task %d(job ID: %s): %v", task.ID, task.JobID, e)
|
||||
} else {
|
||||
log.Debugf("got status behind error for task %d, update it's status to %s directly", task.ID, status)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err == job.ErrJobNotFound {
|
||||
e := c.executionMgr.UpdateTask(&models.Task{
|
||||
ID: task.ID,
|
||||
Status: models.ExecutionStatusStopped,
|
||||
}, "Status")
|
||||
if e != nil {
|
||||
log.Errorf("failed to update the status the task %d(job ID: %s): %v", task.ID, task.JobID, e)
|
||||
} else {
|
||||
log.Debugf("got job not found error for task %d, update it's status to %s directly", task.ID, models.ExecutionStatusStopped)
|
||||
}
|
||||
continue
|
||||
}
|
||||
log.Errorf("failed to stop the task %d(job ID: %s): %v", task.ID, task.JobID, err)
|
||||
continue
|
||||
}
|
||||
log.Debugf("the stop request for task %d(job ID: %s) sent", task.ID, task.JobID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isTaskInFinalStatus(task *models.Task) bool {
|
||||
if task == nil {
|
||||
return false
|
||||
}
|
||||
switch task.Status {
|
||||
case models.TaskStatusSucceed,
|
||||
models.TaskStatusStopped,
|
||||
models.TaskStatusFailed:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *controller) ListExecutions(query ...*models.ExecutionQuery) (int64, []*models.Execution, error) {
|
||||
return c.executionMgr.List(query...)
|
||||
}
|
||||
func (c *controller) GetExecution(executionID int64) (*models.Execution, error) {
|
||||
return c.executionMgr.Get(executionID)
|
||||
}
|
||||
func (c *controller) ListTasks(query ...*models.TaskQuery) (int64, []*models.Task, error) {
|
||||
return c.executionMgr.ListTasks(query...)
|
||||
}
|
||||
func (c *controller) GetTask(id int64) (*models.Task, error) {
|
||||
return c.executionMgr.GetTask(id)
|
||||
}
|
||||
func (c *controller) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
|
||||
return c.executionMgr.UpdateTaskStatus(id, status, statusRevision, statusCondition...)
|
||||
}
|
||||
func (c *controller) GetTaskLog(taskID int64) ([]byte, error) {
|
||||
return c.executionMgr.GetTaskLog(taskID)
|
||||
}
|
||||
|
||||
// create the execution record in database
|
||||
func createExecution(mgr execution.Manager, policyID int64, trigger model.TriggerType) (int64, error) {
|
||||
id, err := mgr.Create(&models.Execution{
|
||||
PolicyID: policyID,
|
||||
Trigger: trigger,
|
||||
Status: models.ExecutionStatusInProgress,
|
||||
StartTime: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create the execution record for replication based on policy %d: %v", policyID, err)
|
||||
}
|
||||
log.Debugf("an execution record for replication based on the policy %d created: %d", policyID, id)
|
||||
return id, nil
|
||||
}
|
||||
|
|
@ -1,390 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package operation
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution"
|
||||
"github.com/goharbor/harbor/src/replication/adapter"
|
||||
"github.com/goharbor/harbor/src/replication/config"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/operation/flow"
|
||||
"github.com/goharbor/harbor/src/replication/operation/scheduler"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakedExecutionManager struct{}
|
||||
|
||||
func (f *fakedExecutionManager) Create(*models.Execution) (int64, error) {
|
||||
return 1, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) List(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
|
||||
return 1, []*models.Execution{
|
||||
{
|
||||
ID: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) Get(int64) (*models.Execution, error) {
|
||||
return &models.Execution{
|
||||
ID: 1,
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) Update(*models.Execution, ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) Remove(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) RemoveAll(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) CreateTask(*models.Task) (int64, error) {
|
||||
return 1, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
|
||||
return 1, []*models.Task{
|
||||
{
|
||||
ID: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) GetTask(int64) (*models.Task, error) {
|
||||
return &models.Task{
|
||||
ID: 1,
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) UpdateTask(*models.Task, ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) UpdateTaskStatus(int64, string, int64, ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) RemoveTask(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) RemoveAllTasks(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) GetTaskLog(int64) ([]byte, error) {
|
||||
return []byte("message"), nil
|
||||
}
|
||||
|
||||
type fakedScheduler struct{}
|
||||
|
||||
func (f *fakedScheduler) Preprocess(src []*model.Resource, dst []*model.Resource) ([]*scheduler.ScheduleItem, error) {
|
||||
items := make([]*scheduler.ScheduleItem, 0)
|
||||
for i, res := range src {
|
||||
items = append(items, &scheduler.ScheduleItem{
|
||||
SrcResource: res,
|
||||
DstResource: dst[i],
|
||||
})
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
func (f *fakedScheduler) Schedule(items []*scheduler.ScheduleItem) ([]*scheduler.ScheduleResult, error) {
|
||||
results := make([]*scheduler.ScheduleResult, 0)
|
||||
for _, item := range items {
|
||||
results = append(results, &scheduler.ScheduleResult{
|
||||
TaskID: item.TaskID,
|
||||
Error: nil,
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
func (f *fakedScheduler) Stop(id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakedFactory struct {
|
||||
}
|
||||
|
||||
func (fakedFactory) Create(*model.Registry) (adapter.Adapter, error) {
|
||||
return &fakedAdapter{}, nil
|
||||
}
|
||||
|
||||
func (fakedFactory) AdapterPattern() *model.AdapterPattern {
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakedAdapter struct{}
|
||||
|
||||
func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
|
||||
return &model.RegistryInfo{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
SupportedResourceTypes: []model.ResourceType{
|
||||
model.ResourceTypeImage,
|
||||
model.ResourceTypeChart,
|
||||
},
|
||||
SupportedTriggers: []model.TriggerType{model.TriggerTypeManual},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
|
||||
return model.Healthy, nil
|
||||
}
|
||||
func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) {
|
||||
return []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeImage,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) {
|
||||
return false, "", nil
|
||||
}
|
||||
func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) {
|
||||
return nil, "", nil
|
||||
}
|
||||
func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) DeleteManifest(repository, digest string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) BlobExist(repository, digest string) (exist bool, err error) {
|
||||
return false, nil
|
||||
}
|
||||
func (f *fakedAdapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedAdapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
|
||||
return []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/harbor",
|
||||
},
|
||||
Vtags: []string{"0.2.0"},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedAdapter) ChartExist(name, version string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
func (f *fakedAdapter) DownloadChart(name, version string) (io.ReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedAdapter) UploadChart(name, version string, chart io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) DeleteChart(name, version string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var ctl *controller
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctl = &controller{
|
||||
replicators: make(chan struct{}, 1),
|
||||
executionMgr: &fakedExecutionManager{},
|
||||
scheduler: &fakedScheduler{},
|
||||
flowCtl: flow.NewController(),
|
||||
}
|
||||
ctl.replicators <- struct{}{}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestStartReplication(t *testing.T) {
|
||||
err := adapter.RegisterFactory(model.RegistryTypeHarbor, new(fakedFactory))
|
||||
require.Nil(t, err)
|
||||
config.Config = &config.Configuration{}
|
||||
|
||||
// policy is disabled
|
||||
policy := &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
DestRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
}
|
||||
resource := &model.Resource{
|
||||
Type: model.ResourceTypeImage,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"1.0", "2.0"},
|
||||
},
|
||||
}
|
||||
_, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased)
|
||||
require.NotNil(t, err)
|
||||
|
||||
// replicate resource deletion
|
||||
policy = &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
DestRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
Enabled: true,
|
||||
}
|
||||
resource = &model.Resource{
|
||||
Type: model.ResourceTypeImage,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"1.0", "2.0"},
|
||||
},
|
||||
Deleted: true,
|
||||
}
|
||||
id, err := ctl.StartReplication(policy, resource, model.TriggerTypeEventBased)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), id)
|
||||
|
||||
// replicate resource copy
|
||||
policy = &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
DestRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
Enabled: true,
|
||||
}
|
||||
resource = &model.Resource{
|
||||
Type: model.ResourceTypeImage,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"1.0", "2.0"},
|
||||
},
|
||||
Deleted: false,
|
||||
}
|
||||
id, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), id)
|
||||
|
||||
// nil resource
|
||||
policy = &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
DestRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
Enabled: true,
|
||||
}
|
||||
id, err = ctl.StartReplication(policy, nil, model.TriggerTypeEventBased)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), id)
|
||||
}
|
||||
|
||||
func TestStopReplication(t *testing.T) {
|
||||
err := ctl.StopReplication(1)
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestListExecutions(t *testing.T) {
|
||||
n, executions, err := ctl.ListExecutions()
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), n)
|
||||
assert.Equal(t, int64(1), executions[0].ID)
|
||||
}
|
||||
|
||||
func TestGetExecution(t *testing.T) {
|
||||
execution, err := ctl.GetExecution(1)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), execution.ID)
|
||||
}
|
||||
|
||||
func TestListTasks(t *testing.T) {
|
||||
n, tasks, err := ctl.ListTasks()
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), n)
|
||||
assert.Equal(t, int64(1), tasks[0].ID)
|
||||
}
|
||||
|
||||
func TestGetTask(t *testing.T) {
|
||||
task, err := ctl.GetTask(1)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), task.ID)
|
||||
}
|
||||
|
||||
func TestUpdateTaskStatus(t *testing.T) {
|
||||
err := ctl.UpdateTaskStatus(1, "running", 1)
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestGetTaskLog(t *testing.T) {
|
||||
log, err := ctl.GetTaskLog(1)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, "message", string(log))
|
||||
}
|
||||
|
||||
func TestIsTaskRunning(t *testing.T) {
|
||||
cases := []struct {
|
||||
task *models.Task
|
||||
isFinalStatus bool
|
||||
}{
|
||||
{
|
||||
task: nil,
|
||||
isFinalStatus: false,
|
||||
},
|
||||
{
|
||||
task: &models.Task{
|
||||
Status: models.TaskStatusSucceed,
|
||||
},
|
||||
isFinalStatus: true,
|
||||
},
|
||||
{
|
||||
task: &models.Task{
|
||||
Status: models.TaskStatusFailed,
|
||||
},
|
||||
isFinalStatus: true,
|
||||
},
|
||||
{
|
||||
task: &models.Task{
|
||||
Status: models.TaskStatusStopped,
|
||||
},
|
||||
isFinalStatus: true,
|
||||
},
|
||||
{
|
||||
task: &models.Task{
|
||||
Status: models.TaskStatusInProgress,
|
||||
},
|
||||
isFinalStatus: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
assert.Equal(t, c.isFinalStatus, isTaskInFinalStatus(c.task))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,182 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package execution
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/goharbor/harbor/src/core/utils"
|
||||
"github.com/goharbor/harbor/src/replication/dao"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
)
|
||||
|
||||
// Manager manages the executions
|
||||
type Manager interface {
|
||||
// Create a new execution
|
||||
Create(*models.Execution) (int64, error)
|
||||
// List the summaries of executions
|
||||
List(...*models.ExecutionQuery) (int64, []*models.Execution, error)
|
||||
// Get the specified execution
|
||||
Get(int64) (*models.Execution, error)
|
||||
// Update the data of the specified execution, the "props" are the
|
||||
// properties of execution that need to be updated
|
||||
Update(execution *models.Execution, props ...string) error
|
||||
// Remove the execution specified by the ID
|
||||
Remove(int64) error
|
||||
// Remove all executions of one policy specified by the policy ID
|
||||
RemoveAll(int64) error
|
||||
// Create a task
|
||||
CreateTask(*models.Task) (int64, error)
|
||||
// List the tasks according to the query
|
||||
ListTasks(...*models.TaskQuery) (int64, []*models.Task, error)
|
||||
// Get one specified task
|
||||
GetTask(int64) (*models.Task, error)
|
||||
// Update the task, the "props" are the properties of task
|
||||
// that need to be updated, it cannot include "status". If
|
||||
// you want to update the status, use "UpdateTaskStatus" instead
|
||||
UpdateTask(task *models.Task, props ...string) error
|
||||
// UpdateTaskStatus only updates the task status. If "statusCondition"
|
||||
// presents, only the tasks whose status equal to "statusCondition"
|
||||
// will be updated
|
||||
UpdateTaskStatus(taskID int64, status string, statusRevision int64, statusCondition ...string) error
|
||||
// Remove one task specified by task ID
|
||||
RemoveTask(int64) error
|
||||
// Remove all tasks of one execution specified by the execution ID
|
||||
RemoveAllTasks(int64) error
|
||||
// Get the log of one specific task
|
||||
GetTaskLog(int64) ([]byte, error)
|
||||
}
|
||||
|
||||
// DefaultManager ..
|
||||
type DefaultManager struct {
|
||||
}
|
||||
|
||||
// NewDefaultManager ...
|
||||
func NewDefaultManager() Manager {
|
||||
return &DefaultManager{}
|
||||
}
|
||||
|
||||
// Create a new execution
|
||||
func (dm *DefaultManager) Create(execution *models.Execution) (int64, error) {
|
||||
return dao.AddExecution(execution)
|
||||
}
|
||||
|
||||
// List the summaries of executions
|
||||
func (dm *DefaultManager) List(queries ...*models.ExecutionQuery) (int64, []*models.Execution, error) {
|
||||
total, err := dao.GetTotalOfExecutions(queries...)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
executions, err := dao.GetExecutions(queries...)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return total, executions, nil
|
||||
}
|
||||
|
||||
// Get the specified execution
|
||||
func (dm *DefaultManager) Get(id int64) (*models.Execution, error) {
|
||||
return dao.GetExecution(id)
|
||||
}
|
||||
|
||||
// Update ...
|
||||
func (dm *DefaultManager) Update(execution *models.Execution, props ...string) error {
|
||||
n, err := dao.UpdateExecution(execution, props...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return fmt.Errorf("Execution not found error: %d ", execution.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove the execution specified by the ID
|
||||
func (dm *DefaultManager) Remove(id int64) error {
|
||||
return dao.DeleteExecution(id)
|
||||
}
|
||||
|
||||
// RemoveAll executions of one policy specified by the policy ID
|
||||
func (dm *DefaultManager) RemoveAll(policyID int64) error {
|
||||
return dao.DeleteAllExecutions(policyID)
|
||||
}
|
||||
|
||||
// CreateTask used to create a task
|
||||
func (dm *DefaultManager) CreateTask(task *models.Task) (int64, error) {
|
||||
return dao.AddTask(task)
|
||||
}
|
||||
|
||||
// ListTasks list the tasks according to the query
|
||||
func (dm *DefaultManager) ListTasks(queries ...*models.TaskQuery) (int64, []*models.Task, error) {
|
||||
total, err := dao.GetTotalOfTasks(queries...)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
tasks, err := dao.GetTasks(queries...)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return total, tasks, nil
|
||||
}
|
||||
|
||||
// GetTask get one specified task
|
||||
func (dm *DefaultManager) GetTask(id int64) (*models.Task, error) {
|
||||
return dao.GetTask(id)
|
||||
}
|
||||
|
||||
// UpdateTask ...
|
||||
func (dm *DefaultManager) UpdateTask(task *models.Task, props ...string) error {
|
||||
n, err := dao.UpdateTask(task, props...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return fmt.Errorf("Task not found error: %d ", task.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateTaskStatus ...
|
||||
func (dm *DefaultManager) UpdateTaskStatus(taskID int64, status string, statusRevision int64, statusCondition ...string) error {
|
||||
if _, err := dao.UpdateTaskStatus(taskID, status, statusRevision, statusCondition...); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveTask remove one task specified by task ID
|
||||
func (dm *DefaultManager) RemoveTask(id int64) error {
|
||||
return dao.DeleteTask(id)
|
||||
}
|
||||
|
||||
// RemoveAllTasks of one execution specified by the execution ID
|
||||
func (dm *DefaultManager) RemoveAllTasks(executionID int64) error {
|
||||
return dao.DeleteAllTasks(executionID)
|
||||
}
|
||||
|
||||
// GetTaskLog get the log of one specific task
|
||||
func (dm *DefaultManager) GetTaskLog(taskID int64) ([]byte, error) {
|
||||
task, err := dao.GetTask(taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if task == nil {
|
||||
return nil, fmt.Errorf("Task not found %d ", taskID)
|
||||
}
|
||||
|
||||
return utils.GetJobServiceClient().GetJobLog(task.JobID)
|
||||
}
|
||||
|
|
@ -1,135 +0,0 @@
|
|||
package execution
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var executionManager = NewDefaultManager()
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
dao.PrepareTestForPostgresSQL()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestMethodOfExecutionManager(t *testing.T) {
|
||||
execution := &models.Execution{
|
||||
PolicyID: 11209,
|
||||
Status: "InProgress",
|
||||
StatusText: "None",
|
||||
Total: 12,
|
||||
Failed: 0,
|
||||
Succeed: 7,
|
||||
InProgress: 5,
|
||||
Stopped: 0,
|
||||
Trigger: "Event",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
|
||||
defer func() {
|
||||
executionManager.RemoveAll(execution.PolicyID)
|
||||
}()
|
||||
|
||||
// Create
|
||||
id, err := executionManager.Create(execution)
|
||||
require.Nil(t, err)
|
||||
|
||||
// List
|
||||
query := &models.ExecutionQuery{
|
||||
Statuses: []string{"InProgress", "Failed"},
|
||||
Pagination: models.Pagination{
|
||||
Page: 1,
|
||||
Size: 10,
|
||||
},
|
||||
}
|
||||
count, executions, err := executionManager.List(query)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), count)
|
||||
assert.Equal(t, 1, len(executions))
|
||||
|
||||
// Get
|
||||
_, err = executionManager.Get(id)
|
||||
require.Nil(t, err)
|
||||
|
||||
// Update
|
||||
executionNew := &models.Execution{
|
||||
ID: id,
|
||||
Status: "Failed",
|
||||
Succeed: 12,
|
||||
InProgress: 0,
|
||||
EndTime: time.Now(),
|
||||
}
|
||||
err = executionManager.Update(executionNew, models.ExecutionPropsName.Status, models.ExecutionPropsName.Succeed, models.ExecutionPropsName.InProgress,
|
||||
models.ExecutionPropsName.EndTime)
|
||||
require.Nil(t, err)
|
||||
|
||||
// Remove
|
||||
require.Nil(t, executionManager.Remove(id))
|
||||
}
|
||||
|
||||
func TestMethodOfTaskManager(t *testing.T) {
|
||||
now := time.Now()
|
||||
task := &models.Task{
|
||||
ExecutionID: 112200,
|
||||
ResourceType: "resourceType1",
|
||||
SrcResource: "srcResource1",
|
||||
DstResource: "dstResource1",
|
||||
JobID: "jobID1",
|
||||
Status: "Initialized",
|
||||
StatusRevision: 1,
|
||||
StartTime: now,
|
||||
}
|
||||
|
||||
defer func() {
|
||||
executionManager.RemoveAllTasks(task.ExecutionID)
|
||||
}()
|
||||
|
||||
// CreateTask
|
||||
id, err := executionManager.CreateTask(task)
|
||||
require.Nil(t, err)
|
||||
|
||||
// ListTasks
|
||||
query := &models.TaskQuery{
|
||||
ResourceType: "resourceType1",
|
||||
Pagination: models.Pagination{
|
||||
Page: 1,
|
||||
Size: 10,
|
||||
},
|
||||
}
|
||||
count, tasks, err := executionManager.ListTasks(query)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, len(tasks))
|
||||
assert.Equal(t, int64(1), count)
|
||||
|
||||
// GetTask
|
||||
_, err = executionManager.GetTask(id)
|
||||
require.Nil(t, err)
|
||||
|
||||
// UpdateTask
|
||||
taskNew := &models.Task{
|
||||
ID: id,
|
||||
SrcResource: "srcResourceChanged",
|
||||
}
|
||||
err = executionManager.UpdateTask(taskNew, models.TaskPropsName.SrcResource)
|
||||
require.Nil(t, err)
|
||||
taskUpdate, _ := executionManager.GetTask(id)
|
||||
assert.Equal(t, taskNew.SrcResource, taskUpdate.SrcResource)
|
||||
|
||||
// UpdateTaskStatus
|
||||
err = executionManager.UpdateTaskStatus(id, models.TaskStatusSucceed, 1, models.TaskStatusInitialized)
|
||||
require.Nil(t, err)
|
||||
taskUpdate, _ = executionManager.GetTask(id)
|
||||
assert.Equal(t, models.TaskStatusSucceed, taskUpdate.Status)
|
||||
|
||||
// Remove
|
||||
require.Nil(t, executionManager.RemoveTask(id))
|
||||
|
||||
// RemoveAll
|
||||
require.Nil(t, executionManager.RemoveAll(id))
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/operation/execution"
|
||||
"github.com/goharbor/harbor/src/replication/operation/scheduler"
|
||||
)
|
||||
|
||||
type copyFlow struct {
|
||||
executionID int64
|
||||
resources []*model.Resource
|
||||
policy *model.Policy
|
||||
executionMgr execution.Manager
|
||||
scheduler scheduler.Scheduler
|
||||
}
|
||||
|
||||
// NewCopyFlow returns an instance of the copy flow which replicates the resources from
|
||||
// the source registry to the destination registry. If the parameter "resources" isn't provided,
|
||||
// will fetch the resources first
|
||||
func NewCopyFlow(executionMgr execution.Manager, scheduler scheduler.Scheduler,
|
||||
executionID int64, policy *model.Policy, resources ...*model.Resource) Flow {
|
||||
return ©Flow{
|
||||
executionMgr: executionMgr,
|
||||
scheduler: scheduler,
|
||||
executionID: executionID,
|
||||
policy: policy,
|
||||
resources: resources,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *copyFlow) Run(interface{}) (int, error) {
|
||||
srcAdapter, dstAdapter, err := initialize(c.policy)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var srcResources []*model.Resource
|
||||
if len(c.resources) > 0 {
|
||||
srcResources, err = filterResources(c.resources, c.policy.Filters)
|
||||
} else {
|
||||
srcResources, err = fetchResources(srcAdapter, c.policy)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
isStopped, err := isExecutionStopped(c.executionMgr, c.executionID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if isStopped {
|
||||
log.Debugf("the execution %d is stopped, stop the flow", c.executionID)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if len(srcResources) == 0 {
|
||||
markExecutionSuccess(c.executionMgr, c.executionID, "no resources need to be replicated")
|
||||
log.Infof("no resources need to be replicated for the execution %d, skip", c.executionID)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
srcResources = assembleSourceResources(srcResources, c.policy)
|
||||
dstResources := assembleDestinationResources(srcResources, c.policy)
|
||||
|
||||
if err = prepareForPush(dstAdapter, dstResources); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
items, err := preprocess(c.scheduler, srcResources, dstResources)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err = createTasks(c.executionMgr, c.executionID, items); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return schedule(c.scheduler, c.executionMgr, items)
|
||||
}
|
||||
|
||||
// mark the execution as success in database
|
||||
func markExecutionSuccess(mgr execution.Manager, id int64, message string) {
|
||||
err := mgr.Update(
|
||||
&models.Execution{
|
||||
ID: id,
|
||||
Status: models.ExecutionStatusSucceed,
|
||||
StatusText: message,
|
||||
EndTime: time.Now(),
|
||||
}, "Status", "StatusText", "EndTime")
|
||||
if err != nil {
|
||||
log.Errorf("failed to update the execution %d: %v", id, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRunOfCopyFlow(t *testing.T) {
|
||||
scheduler := &fakedScheduler{}
|
||||
executionMgr := &fakedExecutionManager{}
|
||||
policy := &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
DestRegistry: &model.Registry{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
},
|
||||
}
|
||||
flow := NewCopyFlow(executionMgr, scheduler, 1, policy)
|
||||
n, err := flow.Run(nil)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 2, n)
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/operation/execution"
|
||||
"github.com/goharbor/harbor/src/replication/operation/scheduler"
|
||||
)
|
||||
|
||||
type deletionFlow struct {
|
||||
executionID int64
|
||||
policy *model.Policy
|
||||
executionMgr execution.Manager
|
||||
scheduler scheduler.Scheduler
|
||||
resources []*model.Resource
|
||||
}
|
||||
|
||||
// NewDeletionFlow returns an instance of the delete flow which deletes the resources
|
||||
// on the destination registry
|
||||
func NewDeletionFlow(executionMgr execution.Manager, scheduler scheduler.Scheduler,
|
||||
executionID int64, policy *model.Policy, resources ...*model.Resource) Flow {
|
||||
return &deletionFlow{
|
||||
executionMgr: executionMgr,
|
||||
scheduler: scheduler,
|
||||
executionID: executionID,
|
||||
policy: policy,
|
||||
resources: resources,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deletionFlow) Run(interface{}) (int, error) {
|
||||
srcResources, err := filterResources(d.resources, d.policy.Filters)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(srcResources) == 0 {
|
||||
markExecutionSuccess(d.executionMgr, d.executionID, "no resources need to be replicated")
|
||||
log.Infof("no resources need to be replicated for the execution %d, skip", d.executionID)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
srcResources = assembleSourceResources(srcResources, d.policy)
|
||||
dstResources := assembleDestinationResources(srcResources, d.policy)
|
||||
|
||||
items, err := preprocess(d.scheduler, srcResources, dstResources)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err = createTasks(d.executionMgr, d.executionID, items); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return schedule(d.scheduler, d.executionMgr, items)
|
||||
}
|
||||
|
|
@ -1,427 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package flow
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution"
|
||||
"github.com/goharbor/harbor/src/replication/adapter"
|
||||
"github.com/goharbor/harbor/src/replication/config"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/operation/scheduler"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakedFactory struct {
|
||||
}
|
||||
|
||||
func (fakedFactory) Create(*model.Registry) (adapter.Adapter, error) {
|
||||
return &fakedAdapter{}, nil
|
||||
}
|
||||
|
||||
func (fakedFactory) AdapterPattern() *model.AdapterPattern {
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakedAdapter struct{}
|
||||
|
||||
func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
|
||||
return &model.RegistryInfo{
|
||||
Type: model.RegistryTypeHarbor,
|
||||
SupportedResourceTypes: []model.ResourceType{
|
||||
model.ResourceTypeImage,
|
||||
model.ResourceTypeChart,
|
||||
},
|
||||
SupportedTriggers: []model.TriggerType{model.TriggerTypeManual},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
|
||||
return model.Healthy, nil
|
||||
}
|
||||
func (f *fakedAdapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) {
|
||||
return []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeImage,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) {
|
||||
return false, "", nil
|
||||
}
|
||||
func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error) {
|
||||
return nil, "", nil
|
||||
}
|
||||
func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (f *fakedAdapter) DeleteManifest(repository, digest string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) BlobExist(repository, digest string) (exist bool, err error) {
|
||||
return false, nil
|
||||
}
|
||||
func (f *fakedAdapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedAdapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) DeleteTag(repository, tag string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
|
||||
return []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/harbor",
|
||||
},
|
||||
Vtags: []string{"0.2.0"},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
func (f *fakedAdapter) ChartExist(name, version string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
func (f *fakedAdapter) DownloadChart(name, version string) (io.ReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedAdapter) UploadChart(name, version string, chart io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedAdapter) DeleteChart(name, version string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakedScheduler struct{}
|
||||
|
||||
func (f *fakedScheduler) Preprocess(src []*model.Resource, dst []*model.Resource) ([]*scheduler.ScheduleItem, error) {
|
||||
items := []*scheduler.ScheduleItem{}
|
||||
for i, res := range src {
|
||||
items = append(items, &scheduler.ScheduleItem{
|
||||
SrcResource: res,
|
||||
DstResource: dst[i],
|
||||
})
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
func (f *fakedScheduler) Schedule(items []*scheduler.ScheduleItem) ([]*scheduler.ScheduleResult, error) {
|
||||
results := []*scheduler.ScheduleResult{}
|
||||
for _, item := range items {
|
||||
results = append(results, &scheduler.ScheduleResult{
|
||||
TaskID: item.TaskID,
|
||||
Error: nil,
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
func (f *fakedScheduler) Stop(id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakedExecutionManager struct {
|
||||
taskID int64
|
||||
}
|
||||
|
||||
func (f *fakedExecutionManager) Create(*models.Execution) (int64, error) {
|
||||
return 1, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) List(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) Get(int64) (*models.Execution, error) {
|
||||
return &models.Execution{}, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) Update(*models.Execution, ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) Remove(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) RemoveAll(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) CreateTask(*models.Task) (int64, error) {
|
||||
f.taskID++
|
||||
id := f.taskID
|
||||
return id, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) GetTask(int64) (*models.Task, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedExecutionManager) UpdateTask(*models.Task, ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) UpdateTaskStatus(int64, string, int64, ...string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) RemoveTask(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) RemoveAllTasks(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedExecutionManager) GetTaskLog(int64) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
url := "https://registry.harbor.local"
|
||||
config.Config = &config.Configuration{
|
||||
CoreURL: url,
|
||||
}
|
||||
if err := adapter.RegisterFactory(model.RegistryTypeHarbor, new(fakedFactory)); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestFetchResources(t *testing.T) {
|
||||
adapter := &fakedAdapter{}
|
||||
policy := &model.Policy{}
|
||||
resources, err := fetchResources(adapter, policy)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 2, len(resources))
|
||||
}
|
||||
|
||||
func TestFilterResources(t *testing.T) {
|
||||
resources := []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeImage,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Artifacts: []*model.Artifact{
|
||||
{
|
||||
Tags: []string{"latest"},
|
||||
},
|
||||
},
|
||||
},
|
||||
Deleted: true,
|
||||
Override: true,
|
||||
},
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/harbor",
|
||||
},
|
||||
Artifacts: []*model.Artifact{
|
||||
{
|
||||
Tags: []string{"0.2.0"},
|
||||
},
|
||||
{
|
||||
Tags: []string{"0.3.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
Deleted: true,
|
||||
Override: true,
|
||||
},
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/mysql",
|
||||
},
|
||||
Artifacts: []*model.Artifact{
|
||||
{
|
||||
Tags: []string{"1.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
Deleted: true,
|
||||
Override: true,
|
||||
},
|
||||
}
|
||||
filters := []*model.Filter{
|
||||
{
|
||||
Type: model.FilterTypeResource,
|
||||
Value: model.ResourceTypeChart,
|
||||
},
|
||||
{
|
||||
Type: model.FilterTypeName,
|
||||
Value: "library/*",
|
||||
},
|
||||
{
|
||||
Type: model.FilterTypeName,
|
||||
Value: "library/harbor",
|
||||
},
|
||||
{
|
||||
Type: model.FilterTypeTag,
|
||||
Value: "0.2.?",
|
||||
},
|
||||
}
|
||||
res, err := filterResources(resources, filters)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, len(res))
|
||||
assert.Equal(t, "library/harbor", res[0].Metadata.Repository.Name)
|
||||
assert.Equal(t, 1, len(res[0].Metadata.Artifacts))
|
||||
assert.Equal(t, "0.2.0", res[0].Metadata.Artifacts[0].Tags[0])
|
||||
}
|
||||
|
||||
func TestAssembleSourceResources(t *testing.T) {
|
||||
resources := []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}
|
||||
policy := &model.Policy{
|
||||
SrcRegistry: &model.Registry{
|
||||
ID: 1,
|
||||
},
|
||||
}
|
||||
res := assembleSourceResources(resources, policy)
|
||||
assert.Equal(t, 1, len(res))
|
||||
assert.Equal(t, int64(1), res[0].Registry.ID)
|
||||
}
|
||||
|
||||
func TestAssembleDestinationResources(t *testing.T) {
|
||||
resources := []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}
|
||||
policy := &model.Policy{
|
||||
DestRegistry: &model.Registry{},
|
||||
DestNamespace: "test",
|
||||
Override: true,
|
||||
}
|
||||
res := assembleDestinationResources(resources, policy)
|
||||
assert.Equal(t, 1, len(res))
|
||||
assert.Equal(t, model.ResourceTypeChart, res[0].Type)
|
||||
assert.Equal(t, "test/hello-world", res[0].Metadata.Repository.Name)
|
||||
assert.Equal(t, 1, len(res[0].Metadata.Vtags))
|
||||
assert.Equal(t, "latest", res[0].Metadata.Vtags[0])
|
||||
}
|
||||
|
||||
func TestPreprocess(t *testing.T) {
|
||||
scheduler := &fakedScheduler{}
|
||||
srcResources := []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "library/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}
|
||||
dstResources := []*model.Resource{
|
||||
{
|
||||
Type: model.ResourceTypeChart,
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "test/hello-world",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Override: false,
|
||||
},
|
||||
}
|
||||
items, err := preprocess(scheduler, srcResources, dstResources)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, len(items))
|
||||
}
|
||||
|
||||
func TestCreateTasks(t *testing.T) {
|
||||
mgr := &fakedExecutionManager{}
|
||||
items := []*scheduler.ScheduleItem{
|
||||
{
|
||||
SrcResource: &model.Resource{},
|
||||
DstResource: &model.Resource{},
|
||||
},
|
||||
}
|
||||
err := createTasks(mgr, 1, items)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, int64(1), items[0].TaskID)
|
||||
}
|
||||
|
||||
func TestSchedule(t *testing.T) {
|
||||
sched := &fakedScheduler{}
|
||||
mgr := &fakedExecutionManager{}
|
||||
items := []*scheduler.ScheduleItem{
|
||||
{
|
||||
SrcResource: &model.Resource{},
|
||||
DstResource: &model.Resource{},
|
||||
TaskID: 1,
|
||||
},
|
||||
}
|
||||
n, err := schedule(sched, mgr, items)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
}
|
||||
|
||||
func TestReplaceNamespace(t *testing.T) {
|
||||
// empty namespace
|
||||
repository := "c"
|
||||
namespace := ""
|
||||
result := replaceNamespace(repository, namespace)
|
||||
assert.Equal(t, "c", result)
|
||||
// repository contains no "/"
|
||||
repository = "c"
|
||||
namespace = "n"
|
||||
result = replaceNamespace(repository, namespace)
|
||||
assert.Equal(t, "n/c", result)
|
||||
// repository contains only one "/"
|
||||
repository = "b/c"
|
||||
namespace = "n"
|
||||
result = replaceNamespace(repository, namespace)
|
||||
assert.Equal(t, "n/c", result)
|
||||
// repository contains more than one "/"
|
||||
repository = "a/b/c"
|
||||
namespace = "n"
|
||||
result = replaceNamespace(repository, namespace)
|
||||
assert.Equal(t, "n/c", result)
|
||||
}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package hook
|
||||
|
||||
import (
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/operation"
|
||||
)
|
||||
|
||||
// UpdateTask update the status of the task
|
||||
func UpdateTask(ctl operation.Controller, id int64, status string, statusRevision int64) error {
|
||||
jobStatus := job.Status(status)
|
||||
// convert the job status to task status
|
||||
s := ""
|
||||
preStatus := []string{}
|
||||
switch jobStatus {
|
||||
case job.PendingStatus:
|
||||
s = models.TaskStatusPending
|
||||
preStatus = append(preStatus, models.TaskStatusInitialized)
|
||||
case job.ScheduledStatus, job.RunningStatus:
|
||||
s = models.TaskStatusInProgress
|
||||
preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending)
|
||||
case job.StoppedStatus:
|
||||
s = models.TaskStatusStopped
|
||||
preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress)
|
||||
case job.ErrorStatus:
|
||||
s = models.TaskStatusFailed
|
||||
preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress)
|
||||
case job.SuccessStatus:
|
||||
s = models.TaskStatusSucceed
|
||||
preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress)
|
||||
}
|
||||
return ctl.UpdateTaskStatus(id, s, statusRevision, preStatus...)
|
||||
}
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package hook
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakedOperationController struct {
|
||||
status string
|
||||
}
|
||||
|
||||
func (f *fakedOperationController) StartReplication(*model.Policy, *model.Resource, model.TriggerType) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (f *fakedOperationController) StopReplication(int64) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) GetExecution(int64) (*models.Execution, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
|
||||
return 0, nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) GetTask(int64) (*models.Task, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
|
||||
f.status = status
|
||||
return nil
|
||||
}
|
||||
func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestUpdateTask(t *testing.T) {
|
||||
mgr := &fakedOperationController{}
|
||||
cases := []struct {
|
||||
inputStatus string
|
||||
expectedStatus string
|
||||
}{
|
||||
{
|
||||
inputStatus: job.PendingStatus.String(),
|
||||
expectedStatus: models.TaskStatusPending,
|
||||
},
|
||||
{
|
||||
inputStatus: job.ScheduledStatus.String(),
|
||||
expectedStatus: models.TaskStatusInProgress,
|
||||
},
|
||||
{
|
||||
inputStatus: job.RunningStatus.String(),
|
||||
expectedStatus: models.TaskStatusInProgress,
|
||||
},
|
||||
{
|
||||
inputStatus: job.StoppedStatus.String(),
|
||||
expectedStatus: models.TaskStatusStopped,
|
||||
},
|
||||
{
|
||||
inputStatus: job.ErrorStatus.String(),
|
||||
expectedStatus: models.TaskStatusFailed,
|
||||
},
|
||||
{
|
||||
inputStatus: job.SuccessStatus.String(),
|
||||
expectedStatus: models.TaskStatusSucceed,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
err := UpdateTask(mgr, 1, c.inputStatus, 1)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, c.expectedStatus, mgr.status)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
cjob "github.com/goharbor/harbor/src/common/job"
|
||||
"github.com/goharbor/harbor/src/common/job/models"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/replication/config"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
type defaultScheduler struct {
|
||||
client cjob.Client
|
||||
}
|
||||
|
||||
// NewScheduler returns an instance of Scheduler
|
||||
func NewScheduler(js cjob.Client) Scheduler {
|
||||
return &defaultScheduler{
|
||||
client: js,
|
||||
}
|
||||
}
|
||||
|
||||
// ScheduleItem is an item that can be scheduled
|
||||
type ScheduleItem struct {
|
||||
TaskID int64 // used as the param in the hook
|
||||
SrcResource *model.Resource
|
||||
DstResource *model.Resource
|
||||
}
|
||||
|
||||
// ScheduleResult is the result of the schedule for one item
|
||||
type ScheduleResult struct {
|
||||
TaskID int64
|
||||
JobID string
|
||||
Error error
|
||||
}
|
||||
|
||||
// Scheduler schedules
|
||||
type Scheduler interface {
|
||||
// Preprocess the resources and returns the item list that can be scheduled
|
||||
Preprocess([]*model.Resource, []*model.Resource) ([]*ScheduleItem, error)
|
||||
// Schedule the items. If got error when scheduling one of the items,
|
||||
// the error should be put in the corresponding ScheduleResult and the
|
||||
// returning error of this function should be nil
|
||||
Schedule([]*ScheduleItem) ([]*ScheduleResult, error)
|
||||
// Stop the job specified by ID
|
||||
Stop(id string) error
|
||||
}
|
||||
|
||||
// Preprocess the resources and returns the item list that can be scheduled
|
||||
func (d *defaultScheduler) Preprocess(srcResources []*model.Resource, destResources []*model.Resource) ([]*ScheduleItem, error) {
|
||||
if len(srcResources) != len(destResources) {
|
||||
err := errors.New("srcResources has different length with destResources")
|
||||
return nil, err
|
||||
}
|
||||
var items []*ScheduleItem
|
||||
for index, srcResource := range srcResources {
|
||||
destResource := destResources[index]
|
||||
item := &ScheduleItem{
|
||||
SrcResource: srcResource,
|
||||
DstResource: destResource,
|
||||
}
|
||||
items = append(items, item)
|
||||
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Schedule transfer the tasks to jobs,and then submit these jobs to job service.
|
||||
func (d *defaultScheduler) Schedule(items []*ScheduleItem) ([]*ScheduleResult, error) {
|
||||
var results []*ScheduleResult
|
||||
for _, item := range items {
|
||||
result := &ScheduleResult{
|
||||
TaskID: item.TaskID,
|
||||
}
|
||||
if item.TaskID == 0 {
|
||||
result.Error = errors.New("some tasks do not have a ID")
|
||||
results = append(results, result)
|
||||
continue
|
||||
}
|
||||
j := &models.JobData{
|
||||
Metadata: &models.JobMetadata{
|
||||
JobKind: job.KindGeneric,
|
||||
},
|
||||
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/replication/task/%d", config.Config.CoreURL, item.TaskID),
|
||||
}
|
||||
|
||||
j.Name = job.Replication
|
||||
src, err := json.Marshal(item.SrcResource)
|
||||
if err != nil {
|
||||
result.Error = err
|
||||
results = append(results, result)
|
||||
continue
|
||||
}
|
||||
dest, err := json.Marshal(item.DstResource)
|
||||
if err != nil {
|
||||
result.Error = err
|
||||
results = append(results, result)
|
||||
continue
|
||||
}
|
||||
j.Parameters = map[string]interface{}{
|
||||
"src_resource": string(src),
|
||||
"dst_resource": string(dest),
|
||||
}
|
||||
id, joberr := d.client.SubmitJob(j)
|
||||
if joberr != nil {
|
||||
result.Error = joberr
|
||||
results = append(results, result)
|
||||
continue
|
||||
}
|
||||
result.JobID = id
|
||||
results = append(results, result)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// Stop the transfer job
|
||||
func (d *defaultScheduler) Stop(id string) error {
|
||||
err := d.client.PostAction(id, string(job.StopCommand))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
package scheduler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/job/models"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
|
||||
var scheduler = &defaultScheduler{
|
||||
client: TestClient{},
|
||||
}
|
||||
|
||||
type TestClient struct {
|
||||
}
|
||||
|
||||
func (client TestClient) SubmitJob(*models.JobData) (string, error) {
|
||||
return "submited-uuid", nil
|
||||
}
|
||||
func (client TestClient) GetJobLog(uuid string) ([]byte, error) {
|
||||
return []byte("job log"), nil
|
||||
}
|
||||
func (client TestClient) PostAction(uuid, action string) error {
|
||||
return nil
|
||||
}
|
||||
func (client TestClient) GetExecutions(uuid string) ([]job.Stats, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestPreprocess(t *testing.T) {
|
||||
items, err := generateData()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
for _, item := range items {
|
||||
content, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
t.Log(string(content))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestStop(t *testing.T) {
|
||||
err := scheduler.Stop("id")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func generateData() ([]*ScheduleItem, error) {
|
||||
srcResource := &model.Resource{
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "namespace1",
|
||||
},
|
||||
Vtags: []string{"latest"},
|
||||
},
|
||||
Registry: &model.Registry{
|
||||
Credential: &model.Credential{},
|
||||
},
|
||||
}
|
||||
destResource := &model.Resource{
|
||||
Metadata: &model.ResourceMetadata{
|
||||
Repository: &model.Repository{
|
||||
Name: "namespace2",
|
||||
},
|
||||
Vtags: []string{"v1", "v2"},
|
||||
},
|
||||
Registry: &model.Registry{
|
||||
Credential: &model.Credential{},
|
||||
},
|
||||
}
|
||||
items, err := scheduler.Preprocess([]*model.Resource{srcResource}, []*model.Resource{destResource})
|
||||
return items, err
|
||||
}
|
||||
|
|
@ -17,20 +17,25 @@ package controller
|
|||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/job"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/lib/orm"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/replication/policy"
|
||||
"github.com/goharbor/harbor/src/replication/policy/manager"
|
||||
"github.com/goharbor/harbor/src/replication/policy/scheduler"
|
||||
)
|
||||
|
||||
// const definitions
|
||||
const (
|
||||
CallbackFuncName = "REPLICATION_CALLBACK"
|
||||
)
|
||||
|
||||
// NewController returns a policy controller which can CURD and schedule policies
|
||||
func NewController(js job.Client) policy.Controller {
|
||||
func NewController() policy.Controller {
|
||||
mgr := manager.NewDefaultManager()
|
||||
scheduler := scheduler.NewScheduler(js)
|
||||
ctl := &controller{
|
||||
scheduler: scheduler,
|
||||
scheduler: scheduler.Sched,
|
||||
}
|
||||
ctl.Controller = mgr
|
||||
return ctl
|
||||
|
|
@ -47,10 +52,7 @@ func (c *controller) Create(policy *model.Policy) (int64, error) {
|
|||
return 0, err
|
||||
}
|
||||
if isScheduledTrigger(policy) {
|
||||
// TODO: need a way to show the schedule status to users
|
||||
// maybe we can add a property "schedule status" for
|
||||
// listing policy API
|
||||
if err = c.scheduler.Schedule(id, policy.Trigger.Settings.Cron); err != nil {
|
||||
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id); err != nil {
|
||||
log.Errorf("failed to schedule the policy %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -72,7 +74,7 @@ func (c *controller) Update(policy *model.Policy) error {
|
|||
// need to reschedule the policy
|
||||
// unschedule first if needed
|
||||
if isScheduledTrigger(origin) {
|
||||
if err = c.scheduler.Unschedule(origin.ID); err != nil {
|
||||
if err = c.scheduler.UnScheduleByVendor(orm.Context(), job.Replication, origin.ID); err != nil {
|
||||
return fmt.Errorf("failed to unschedule the policy %d: %v", origin.ID, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -82,7 +84,7 @@ func (c *controller) Update(policy *model.Policy) error {
|
|||
}
|
||||
// schedule again if needed
|
||||
if isScheduledTrigger(policy) {
|
||||
if err = c.scheduler.Schedule(policy.ID, policy.Trigger.Settings.Cron); err != nil {
|
||||
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID); err != nil {
|
||||
return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -98,7 +100,7 @@ func (c *controller) Remove(policyID int64) error {
|
|||
return fmt.Errorf("policy %d not found", policyID)
|
||||
}
|
||||
if isScheduledTrigger(policy) {
|
||||
if err = c.scheduler.Unschedule(policyID); err != nil {
|
||||
if err = c.scheduler.UnScheduleByVendor(orm.Context(), job.Replication, policyID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,10 @@ package controller
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
"github.com/goharbor/harbor/src/testing/mock"
|
||||
"github.com/goharbor/harbor/src/testing/pkg/scheduler"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -45,20 +48,6 @@ func (f *fakedPolicyController) Remove(int64) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
type fakedScheduler struct {
|
||||
scheduled bool
|
||||
unscheduled bool
|
||||
}
|
||||
|
||||
func (f *fakedScheduler) Schedule(policyID int64, cron string) error {
|
||||
f.scheduled = true
|
||||
return nil
|
||||
}
|
||||
func (f *fakedScheduler) Unschedule(policyID int64) error {
|
||||
f.unscheduled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestIsScheduledTrigger(t *testing.T) {
|
||||
cases := []struct {
|
||||
policy *model.Policy
|
||||
|
|
@ -224,7 +213,8 @@ func TestIsScheduleTriggerChanged(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestCreate(t *testing.T) {
|
||||
scheduler := &fakedScheduler{}
|
||||
dao.PrepareTestForPostgresSQL()
|
||||
scheduler := &scheduler.Scheduler{}
|
||||
ctl := &controller{
|
||||
scheduler: scheduler,
|
||||
}
|
||||
|
|
@ -233,9 +223,10 @@ func TestCreate(t *testing.T) {
|
|||
// not scheduled trigger
|
||||
_, err := ctl.Create(&model.Policy{})
|
||||
require.Nil(t, err)
|
||||
assert.False(t, scheduler.scheduled)
|
||||
|
||||
// scheduled trigger
|
||||
scheduler.On("Schedule", mock.Anything, mock.Anything,
|
||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
_, err = ctl.Create(&model.Policy{
|
||||
Enabled: true,
|
||||
Trigger: &model.Trigger{
|
||||
|
|
@ -246,11 +237,11 @@ func TestCreate(t *testing.T) {
|
|||
},
|
||||
})
|
||||
require.Nil(t, err)
|
||||
assert.True(t, scheduler.scheduled)
|
||||
scheduler.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestUpdate(t *testing.T) {
|
||||
scheduler := &fakedScheduler{}
|
||||
scheduler := &scheduler.Scheduler{}
|
||||
c := &fakedPolicyController{}
|
||||
ctl := &controller{
|
||||
scheduler: scheduler,
|
||||
|
|
@ -275,10 +266,13 @@ func TestUpdate(t *testing.T) {
|
|||
current = origin
|
||||
err = ctl.Update(current)
|
||||
require.Nil(t, err)
|
||||
assert.False(t, scheduler.scheduled)
|
||||
assert.False(t, scheduler.unscheduled)
|
||||
|
||||
// the trigger changed
|
||||
scheduler.On("Schedule", mock.Anything, mock.Anything,
|
||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
|
||||
mock.Anything).Return(nil)
|
||||
|
||||
origin = &model.Policy{
|
||||
ID: 1,
|
||||
Enabled: true,
|
||||
|
|
@ -301,12 +295,11 @@ func TestUpdate(t *testing.T) {
|
|||
}
|
||||
err = ctl.Update(current)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, scheduler.unscheduled)
|
||||
assert.True(t, scheduler.scheduled)
|
||||
scheduler.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestRemove(t *testing.T) {
|
||||
scheduler := &fakedScheduler{}
|
||||
scheduler := &scheduler.Scheduler{}
|
||||
c := &fakedPolicyController{}
|
||||
ctl := &controller{
|
||||
scheduler: scheduler,
|
||||
|
|
@ -328,9 +321,10 @@ func TestRemove(t *testing.T) {
|
|||
c.policy = policy
|
||||
err = ctl.Remove(1)
|
||||
require.Nil(t, err)
|
||||
assert.False(t, scheduler.unscheduled)
|
||||
|
||||
// the trigger type is scheduled
|
||||
scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
|
||||
mock.Anything).Return(nil)
|
||||
policy = &model.Policy{
|
||||
ID: 1,
|
||||
Enabled: true,
|
||||
|
|
@ -344,5 +338,5 @@ func TestRemove(t *testing.T) {
|
|||
c.policy = policy
|
||||
err = ctl.Remove(1)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, scheduler.unscheduled)
|
||||
scheduler.AssertExpectations(t)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,121 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
commonHttp "github.com/goharbor/harbor/src/common/http"
|
||||
"github.com/goharbor/harbor/src/common/job"
|
||||
jobModels "github.com/goharbor/harbor/src/common/job/models"
|
||||
jsJob "github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/replication/config"
|
||||
"github.com/goharbor/harbor/src/replication/dao"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
)
|
||||
|
||||
// Scheduler can be used to schedule or unschedule a scheduled policy
|
||||
// Currently, the default scheduler implements its capabilities by delegating
|
||||
// the scheduled job of jobservice
|
||||
type Scheduler interface {
|
||||
Schedule(policyID int64, cron string) error
|
||||
Unschedule(policyID int64) error
|
||||
}
|
||||
|
||||
// NewScheduler returns an instance of scheduler
|
||||
func NewScheduler(js job.Client) Scheduler {
|
||||
return &scheduler{
|
||||
jobservice: js,
|
||||
}
|
||||
}
|
||||
|
||||
type scheduler struct {
|
||||
jobservice job.Client
|
||||
}
|
||||
|
||||
func (s *scheduler) Schedule(policyID int64, cron string) error {
|
||||
now := time.Now()
|
||||
id, err := dao.ScheduleJob.Add(&models.ScheduleJob{
|
||||
PolicyID: policyID,
|
||||
Status: job.JobServiceStatusPending,
|
||||
CreationTime: now,
|
||||
UpdateTime: now,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debugf("the schedule job record %d added", id)
|
||||
|
||||
statusHookURL := fmt.Sprintf("%s/service/notifications/jobs/replication/%d", config.Config.CoreURL, id)
|
||||
jobID, err := s.jobservice.SubmitJob(&jobModels.JobData{
|
||||
Name: jsJob.ReplicationScheduler,
|
||||
Parameters: map[string]interface{}{
|
||||
"url": config.Config.CoreURL,
|
||||
"policy_id": policyID,
|
||||
},
|
||||
Metadata: &jobModels.JobMetadata{
|
||||
JobKind: job.JobKindPeriodic,
|
||||
Cron: cron,
|
||||
},
|
||||
StatusHook: statusHookURL,
|
||||
})
|
||||
if err != nil {
|
||||
// clean up the record in database
|
||||
if e := dao.ScheduleJob.Delete(id); e != nil {
|
||||
log.Errorf("failed to delete the schedule job %d: %v", id, e)
|
||||
} else {
|
||||
log.Debugf("the schedule job record %d deleted", id)
|
||||
}
|
||||
return err
|
||||
}
|
||||
log.Debugf("the schedule job for policy %d submitted to the jobservice", policyID)
|
||||
|
||||
err = dao.ScheduleJob.Update(&models.ScheduleJob{
|
||||
ID: id,
|
||||
JobID: jobID,
|
||||
}, "JobID")
|
||||
log.Debugf("the policy %d scheduled", policyID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *scheduler) Unschedule(policyID int64) error {
|
||||
sjs, err := dao.ScheduleJob.List(&models.ScheduleJobQuery{
|
||||
PolicyID: policyID,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, sj := range sjs {
|
||||
if err = s.jobservice.PostAction(sj.JobID, job.JobActionStop); err != nil {
|
||||
// if the job specified by jobID is not found in jobservice, just delete
|
||||
// the record from database
|
||||
if e, ok := err.(*commonHttp.Error); !ok || (e.Code != http.StatusNotFound &&
|
||||
!strings.Contains(e.Message, "no valid periodic job policy found")) {
|
||||
return err
|
||||
}
|
||||
log.Debugf("the stop action for schedule job %s submitted to the jobservice", sj.JobID)
|
||||
}
|
||||
if err = dao.ScheduleJob.Delete(sj.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debugf("the schedule job record %d deleted", sj.ID)
|
||||
}
|
||||
log.Debugf("the policy %d unscheduled", policyID)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,172 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/job/models"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/replication/config"
|
||||
"github.com/goharbor/harbor/src/replication/dao"
|
||||
rep_models "github.com/goharbor/harbor/src/replication/dao/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TODO share the faked implementation in a separated common package?
|
||||
// TODO can we use a mock framework?
|
||||
|
||||
var (
|
||||
uuid = "uuid"
|
||||
policyID int64 = 100
|
||||
)
|
||||
|
||||
type fakedJobserviceClient struct {
|
||||
jobData *models.JobData
|
||||
stopped bool
|
||||
}
|
||||
|
||||
func (f *fakedJobserviceClient) SubmitJob(jobData *models.JobData) (string, error) {
|
||||
f.jobData = jobData
|
||||
return uuid, nil
|
||||
}
|
||||
func (f *fakedJobserviceClient) GetJobLog(uuid string) ([]byte, error) {
|
||||
f.stopped = true
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakedJobserviceClient) PostAction(uuid, action string) error {
|
||||
f.stopped = true
|
||||
return nil
|
||||
}
|
||||
func (f *fakedJobserviceClient) GetExecutions(uuid string) ([]job.Stats, error) {
|
||||
f.stopped = true
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type fakedScheduleJobDAO struct {
|
||||
idCounter int64
|
||||
sjs map[int64]*rep_models.ScheduleJob
|
||||
}
|
||||
|
||||
func (f *fakedScheduleJobDAO) Add(sj *rep_models.ScheduleJob) (int64, error) {
|
||||
if f.sjs == nil {
|
||||
f.sjs = make(map[int64]*rep_models.ScheduleJob)
|
||||
}
|
||||
id := f.idCounter + 1
|
||||
sj.ID = id
|
||||
f.sjs[id] = sj
|
||||
return id, nil
|
||||
}
|
||||
func (f *fakedScheduleJobDAO) Get(id int64) (*rep_models.ScheduleJob, error) {
|
||||
if f.sjs == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return f.sjs[id], nil
|
||||
}
|
||||
func (f *fakedScheduleJobDAO) Update(sj *rep_models.ScheduleJob, props ...string) error {
|
||||
err := fmt.Errorf("schedule job %d not found", sj.ID)
|
||||
if f.sjs == nil {
|
||||
return err
|
||||
}
|
||||
j, exist := f.sjs[sj.ID]
|
||||
if !exist {
|
||||
return err
|
||||
}
|
||||
if len(props) == 0 {
|
||||
f.sjs[sj.ID] = sj
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, prop := range props {
|
||||
switch prop {
|
||||
case "PolicyID":
|
||||
j.PolicyID = sj.PolicyID
|
||||
case "JobID":
|
||||
j.JobID = sj.JobID
|
||||
case "Status":
|
||||
j.Status = sj.Status
|
||||
case "UpdateTime":
|
||||
j.UpdateTime = sj.UpdateTime
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f *fakedScheduleJobDAO) Delete(id int64) error {
|
||||
if f.sjs == nil {
|
||||
return nil
|
||||
}
|
||||
delete(f.sjs, id)
|
||||
return nil
|
||||
}
|
||||
func (f *fakedScheduleJobDAO) List(query ...*rep_models.ScheduleJobQuery) ([]*rep_models.ScheduleJob, error) {
|
||||
var policyID int64
|
||||
if len(query) > 0 {
|
||||
policyID = query[0].PolicyID
|
||||
}
|
||||
sjs := []*rep_models.ScheduleJob{}
|
||||
for _, sj := range f.sjs {
|
||||
if policyID == 0 {
|
||||
sjs = append(sjs, sj)
|
||||
continue
|
||||
}
|
||||
if sj.PolicyID == policyID {
|
||||
sjs = append(sjs, sj)
|
||||
}
|
||||
}
|
||||
return sjs, nil
|
||||
}
|
||||
|
||||
func TestSchedule(t *testing.T) {
|
||||
config.Config = &config.Configuration{}
|
||||
dao.ScheduleJob = &fakedScheduleJobDAO{}
|
||||
js := &fakedJobserviceClient{}
|
||||
scheduler := NewScheduler(js)
|
||||
err := scheduler.Schedule(policyID, "1 * * * *")
|
||||
require.Nil(t, err)
|
||||
|
||||
sjs, err := dao.ScheduleJob.List(&rep_models.ScheduleJobQuery{
|
||||
PolicyID: policyID,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 1, len(sjs))
|
||||
assert.Equal(t, uuid, sjs[0].JobID)
|
||||
|
||||
policyID, ok := js.jobData.Parameters["policy_id"].(int64)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, policyID, policyID)
|
||||
}
|
||||
|
||||
func TestUnschedule(t *testing.T) {
|
||||
config.Config = &config.Configuration{}
|
||||
dao.ScheduleJob = &fakedScheduleJobDAO{}
|
||||
_, err := dao.ScheduleJob.Add(&rep_models.ScheduleJob{
|
||||
PolicyID: policyID,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
js := &fakedJobserviceClient{}
|
||||
scheduler := NewScheduler(js)
|
||||
err = scheduler.Unschedule(policyID)
|
||||
require.Nil(t, err)
|
||||
|
||||
sjs, err := dao.ScheduleJob.List(&rep_models.ScheduleJobQuery{
|
||||
PolicyID: policyID,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 0, len(sjs))
|
||||
|
||||
assert.True(t, js.stopped)
|
||||
}
|
||||
|
|
@ -15,14 +15,18 @@
|
|||
package replication
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/job"
|
||||
"github.com/goharbor/harbor/src/controller/replication"
|
||||
cfg "github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/replication/config"
|
||||
"github.com/goharbor/harbor/src/replication/event"
|
||||
"github.com/goharbor/harbor/src/replication/operation"
|
||||
"github.com/goharbor/harbor/src/replication/policy"
|
||||
"github.com/goharbor/harbor/src/replication/policy/controller"
|
||||
"github.com/goharbor/harbor/src/replication/registry"
|
||||
|
|
@ -60,12 +64,37 @@ var (
|
|||
PolicyCtl policy.Controller
|
||||
// RegistryMgr is a global registry manager
|
||||
RegistryMgr registry.Manager
|
||||
// OperationCtl is a global operation controller
|
||||
OperationCtl operation.Controller
|
||||
// EventHandler handles images/chart pull/push events
|
||||
EventHandler event.Handler
|
||||
)
|
||||
|
||||
func init() {
|
||||
callbackFunc := func(ctx context.Context, param string) error {
|
||||
policyID, err := strconv.ParseInt(param, 10, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
policy, err := PolicyCtl.Get(policyID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if policy == nil {
|
||||
return fmt.Errorf("policy %d not found", policyID)
|
||||
}
|
||||
if err = event.PopulateRegistries(RegistryMgr, policy); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = replication.Ctl.Start(ctx, policy, nil, task.ExecutionTriggerSchedule)
|
||||
return err
|
||||
}
|
||||
err := scheduler.RegisterCallbackFunc(controller.CallbackFuncName, callbackFunc)
|
||||
if err != nil {
|
||||
log.Errorf("failed to register the callback function for replication: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Init the global variables and configurations
|
||||
func Init(closing, done chan struct{}) error {
|
||||
// init config
|
||||
|
|
@ -76,21 +105,15 @@ func Init(closing, done chan struct{}) error {
|
|||
config.Config = &config.Configuration{
|
||||
CoreURL: cfg.InternalCoreURL(),
|
||||
TokenServiceURL: cfg.InternalTokenServiceEndpoint(),
|
||||
JobserviceURL: cfg.InternalJobServiceURL(),
|
||||
SecretKey: secretKey,
|
||||
CoreSecret: cfg.CoreSecret(),
|
||||
JobserviceSecret: cfg.JobserviceSecret(),
|
||||
}
|
||||
// TODO use a global http transport
|
||||
js := job.NewDefaultClient(config.Config.JobserviceURL, config.Config.CoreSecret)
|
||||
// init registry manager
|
||||
RegistryMgr = registry.NewDefaultManager()
|
||||
// init policy controller
|
||||
PolicyCtl = controller.NewController(js)
|
||||
// init operation controller
|
||||
OperationCtl = operation.NewController(js)
|
||||
PolicyCtl = controller.NewController()
|
||||
// init event handler
|
||||
EventHandler = event.NewHandler(PolicyCtl, RegistryMgr, OperationCtl)
|
||||
EventHandler = event.NewHandler(PolicyCtl, RegistryMgr)
|
||||
log.Debug("the replication initialization completed")
|
||||
|
||||
// Start health checker for registries
|
||||
|
|
|
|||
|
|
@ -38,6 +38,5 @@ func TestInit(t *testing.T) {
|
|||
require.Nil(t, err)
|
||||
assert.NotNil(t, PolicyCtl)
|
||||
assert.NotNil(t, RegistryMgr)
|
||||
assert.NotNil(t, OperationCtl)
|
||||
assert.NotNil(t, EventHandler)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,20 +17,18 @@ package handler
|
|||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
libhttp "github.com/goharbor/harbor/src/lib/http"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/server/router"
|
||||
)
|
||||
|
||||
// NewJobStatusHandler creates a handler to handle the job status changing
|
||||
func NewJobStatusHandler() http.Handler {
|
||||
return &jobStatusHandler{
|
||||
handler: task.NewHookHandler(),
|
||||
handler: task.HkHandler,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -40,23 +38,16 @@ type jobStatusHandler struct {
|
|||
|
||||
func (j *jobStatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
taskIDParam := router.Param(r.Context(), ":id")
|
||||
taskID, err := strconv.ParseInt(taskIDParam, 10, 64)
|
||||
if err != nil {
|
||||
libhttp.SendError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
sc := &job.StatusChange{}
|
||||
if err = json.NewDecoder(r.Body).Decode(sc); err != nil {
|
||||
if err := json.NewDecoder(r.Body).Decode(sc); err != nil {
|
||||
libhttp.SendError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = j.handler.Handle(r.Context(), taskID, sc); err != nil {
|
||||
if err := j.handler.Handle(r.Context(), sc); err != nil {
|
||||
// ignore the not found error to avoid the jobservice re-sending the hook
|
||||
if errors.IsNotFoundErr(err) {
|
||||
log.Warningf("task %d does not exist, ignore the not found error to avoid subsequent retrying webhooks from jobservice", taskID)
|
||||
log.Warningf("got not found error: %v, ignore it to avoid subsequent retrying webhooks from jobservice", err)
|
||||
return
|
||||
}
|
||||
libhttp.SendError(w, err)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/astaxie/beego"
|
||||
"github.com/goharbor/harbor/src/common"
|
||||
"github.com/goharbor/harbor/src/core/api"
|
||||
|
|
@ -22,11 +24,9 @@ import (
|
|||
"github.com/goharbor/harbor/src/core/controllers"
|
||||
"github.com/goharbor/harbor/src/core/service/notifications/admin"
|
||||
"github.com/goharbor/harbor/src/core/service/notifications/jobs"
|
||||
"github.com/goharbor/harbor/src/core/service/notifications/scheduler"
|
||||
"github.com/goharbor/harbor/src/core/service/token"
|
||||
"github.com/goharbor/harbor/src/server/handler"
|
||||
"github.com/goharbor/harbor/src/server/router"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func registerRoutes() {
|
||||
|
|
@ -47,12 +47,11 @@ func registerRoutes() {
|
|||
beego.Router("/api/internal/syncquota", &api.InternalAPI{}, "post:SyncQuota")
|
||||
|
||||
beego.Router("/service/notifications/jobs/adminjob/:id([0-9]+)", &admin.Handler{}, "post:HandleAdminJob")
|
||||
beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationScheduleJob")
|
||||
beego.Router("/service/notifications/jobs/replication/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationTask")
|
||||
beego.Router("/service/notifications/jobs/webhook/:id([0-9]+)", &jobs.Handler{}, "post:HandleNotificationJob")
|
||||
beego.Router("/service/notifications/jobs/retention/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleRetentionTask")
|
||||
beego.Router("/service/notifications/schedules/:id([0-9]+)", &scheduler.Handler{}, "post:Handle")
|
||||
beego.Router("/service/notifications/jobs/scan/:uuid", &jobs.Handler{}, "post:HandleScan")
|
||||
router.NewRoute().Method(http.MethodPost).Path("/service/notifications/schedules/:id([0-9]+)").Handler(handler.NewJobStatusHandler()) // legacy job status hook endpoint for scheduler
|
||||
router.NewRoute().Method(http.MethodPost).Path("/service/notifications/jobs/replication/:id([0-9]+)").Handler(handler.NewJobStatusHandler()) // legacy job status hook endpoint for replication scheduler
|
||||
router.NewRoute().Method(http.MethodPost).Path("/service/notifications/tasks/:id").Handler(handler.NewJobStatusHandler())
|
||||
|
||||
beego.Router("/service/token", &token.Handler{})
|
||||
|
|
|
|||
|
|
@ -29,13 +29,14 @@ import (
|
|||
// New returns http handler for API V2.0
|
||||
func New() http.Handler {
|
||||
h, api, err := restapi.HandlerAPI(restapi.Config{
|
||||
ArtifactAPI: newArtifactAPI(),
|
||||
RepositoryAPI: newRepositoryAPI(),
|
||||
AuditlogAPI: newAuditLogAPI(),
|
||||
ScanAPI: newScanAPI(),
|
||||
ProjectAPI: newProjectAPI(),
|
||||
PreheatAPI: newPreheatAPI(),
|
||||
IconAPI: newIconAPI(),
|
||||
ArtifactAPI: newArtifactAPI(),
|
||||
RepositoryAPI: newRepositoryAPI(),
|
||||
AuditlogAPI: newAuditLogAPI(),
|
||||
ScanAPI: newScanAPI(),
|
||||
ProjectAPI: newProjectAPI(),
|
||||
PreheatAPI: newPreheatAPI(),
|
||||
IconAPI: newIconAPI(),
|
||||
ReplicationAPI: newReplicationAPI(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
|
|
|||
|
|
@ -663,7 +663,7 @@ func convertTaskToPayload(model *task.Task) (*models.Task, error) {
|
|||
ExecutionID: model.ExecutionID,
|
||||
ExtraAttrs: model.ExtraAttrs,
|
||||
ID: model.ID,
|
||||
RunCount: int64(model.RunCount),
|
||||
RunCount: model.RunCount,
|
||||
StartTime: model.StartTime.String(),
|
||||
Status: model.Status,
|
||||
StatusMessage: model.StatusMessage,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,297 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-openapi/runtime/middleware"
|
||||
"github.com/go-openapi/strfmt"
|
||||
"github.com/goharbor/harbor/src/controller/replication"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
replica "github.com/goharbor/harbor/src/replication"
|
||||
"github.com/goharbor/harbor/src/replication/event"
|
||||
"github.com/goharbor/harbor/src/replication/policy"
|
||||
"github.com/goharbor/harbor/src/replication/policy/manager"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/replication"
|
||||
)
|
||||
|
||||
func newReplicationAPI() *replicationAPI {
|
||||
return &replicationAPI{
|
||||
ctl: replication.Ctl,
|
||||
policyMgr: manager.NewDefaultManager(),
|
||||
}
|
||||
}
|
||||
|
||||
type replicationAPI struct {
|
||||
BaseAPI
|
||||
ctl replication.Controller
|
||||
policyMgr policy.Controller
|
||||
}
|
||||
|
||||
func (r *replicationAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder {
|
||||
if err := r.RequireSysAdmin(ctx); err != nil {
|
||||
r.SendError(ctx, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *replicationAPI) StartReplication(ctx context.Context, params operation.StartReplicationParams) middleware.Responder {
|
||||
// TODO move the following logic to the replication controller after refactoring the policy management part with the new programming model
|
||||
policy, err := r.policyMgr.Get(params.Execution.PolicyID)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
if policy == nil {
|
||||
return r.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode).
|
||||
WithMessage("the replication policy %d not found", params.Execution.PolicyID))
|
||||
}
|
||||
if err = event.PopulateRegistries(replica.RegistryMgr, policy); err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
|
||||
// the legacy replication scheduler job("src/jobservice/job/impl/replication/scheduler.go") calls the start replication API
|
||||
// to trigger the scheduled replication, a query string "trigger" is added when sending the request
|
||||
// here is the logic to cover this part
|
||||
trigger := task.ExecutionTriggerManual
|
||||
if params.HTTPRequest.URL.Query().Get("trigger") == "scheduled" {
|
||||
trigger = task.ExecutionTriggerSchedule
|
||||
}
|
||||
|
||||
executionID, err := r.ctl.Start(ctx, policy, nil, trigger)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
location := strings.TrimSuffix(params.HTTPRequest.URL.Path, "/") + "/" + strconv.FormatInt(executionID, 10)
|
||||
return operation.NewStartReplicationCreated().WithLocation(location)
|
||||
}
|
||||
|
||||
func (r *replicationAPI) StopReplication(ctx context.Context, params operation.StopReplicationParams) middleware.Responder {
|
||||
if err := r.ctl.Stop(ctx, params.ID); err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *replicationAPI) ListReplicationExecutions(ctx context.Context, params operation.ListReplicationExecutionsParams) middleware.Responder {
|
||||
query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
if params.PolicyID != nil {
|
||||
query.Keywords["PolicyID"] = *params.PolicyID
|
||||
}
|
||||
if params.Status != nil {
|
||||
status := *params.Status
|
||||
// as we convert the status when responding requests to keep the backward compatibility,
|
||||
// here we need to reverse-convert the status
|
||||
switch status {
|
||||
case "InProgress":
|
||||
status = job.RunningStatus.String()
|
||||
case "Succeed":
|
||||
status = job.SuccessStatus.String()
|
||||
case "Stopped":
|
||||
status = job.StoppedStatus.String()
|
||||
case "Failed":
|
||||
status = job.ErrorStatus.String()
|
||||
}
|
||||
query.Keywords["Status"] = status
|
||||
}
|
||||
if params.Trigger != nil {
|
||||
trigger := *params.Trigger
|
||||
// as we convert the trigger when responding requests to keep the backward compatibility,
|
||||
// here we need to reverse-convert the trigger
|
||||
switch trigger {
|
||||
case "manual":
|
||||
trigger = task.ExecutionTriggerManual
|
||||
case "scheduled":
|
||||
trigger = task.ExecutionTriggerSchedule
|
||||
case "event_based":
|
||||
trigger = task.ExecutionTriggerEvent
|
||||
}
|
||||
query.Keywords["Trigger"] = trigger
|
||||
}
|
||||
|
||||
total, err := r.ctl.ExecutionCount(ctx, query)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
executions, err := r.ctl.ListExecutions(ctx, query)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
|
||||
var execs []*models.ReplicationExecution
|
||||
for _, execution := range executions {
|
||||
execs = append(execs, convertExecution(execution))
|
||||
}
|
||||
|
||||
return operation.NewListReplicationExecutionsOK().
|
||||
WithXTotalCount(total).
|
||||
WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
|
||||
WithPayload(execs)
|
||||
}
|
||||
|
||||
func (r *replicationAPI) GetReplicationExecution(ctx context.Context, params operation.GetReplicationExecutionParams) middleware.Responder {
|
||||
execution, err := r.ctl.GetExecution(ctx, params.ID)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
return operation.NewGetReplicationExecutionOK().WithPayload(convertExecution(execution))
|
||||
}
|
||||
|
||||
func (r *replicationAPI) ListReplicationTasks(ctx context.Context, params operation.ListReplicationTasksParams) middleware.Responder {
|
||||
query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
query.Keywords["ExecutionID"] = params.ID
|
||||
if params.Status != nil {
|
||||
var status interface{} = *params.Status
|
||||
// as we convert the status when responding requests to keep the backward compatibility,
|
||||
// here we need to reverse-convert the status
|
||||
// the status "pending" and "stopped" is same with jobservice, no need to convert
|
||||
switch status {
|
||||
case "InProgress":
|
||||
status = &q.OrList{
|
||||
Values: []interface{}{
|
||||
job.ScheduledStatus.String(),
|
||||
job.RunningStatus.String(),
|
||||
},
|
||||
}
|
||||
case "Succeed":
|
||||
status = job.SuccessStatus.String()
|
||||
case "Failed":
|
||||
status = job.ErrorStatus.String()
|
||||
}
|
||||
query.Keywords["Status"] = status
|
||||
}
|
||||
if params.ResourceType != nil {
|
||||
query.Keywords["ExtraAttrs.resource_type"] = *params.ResourceType
|
||||
}
|
||||
|
||||
total, err := r.ctl.TaskCount(ctx, query)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
|
||||
tasks, err := r.ctl.ListTasks(ctx, query)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
|
||||
var tks []*models.ReplicationTask
|
||||
for _, task := range tasks {
|
||||
tks = append(tks, convertTask(task))
|
||||
}
|
||||
|
||||
return operation.NewListReplicationTasksOK().
|
||||
WithXTotalCount(total).
|
||||
WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
|
||||
WithPayload(tks)
|
||||
}
|
||||
|
||||
func (r *replicationAPI) GetReplicationLog(ctx context.Context, params operation.GetReplicationLogParams) middleware.Responder {
|
||||
execution, err := r.ctl.GetExecution(ctx, params.ID)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
task, err := r.ctl.GetTask(ctx, params.TaskID)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
if execution.ID != task.ExecutionID {
|
||||
return r.SendError(ctx, errors.New(nil).
|
||||
WithCode(errors.NotFoundCode).
|
||||
WithMessage("execution %d contains no task with ID %d", params.ID, params.TaskID))
|
||||
}
|
||||
log, err := r.ctl.GetTaskLog(ctx, params.TaskID)
|
||||
if err != nil {
|
||||
return r.SendError(ctx, err)
|
||||
}
|
||||
return operation.NewGetReplicationLogOK().WithContentType("text/plain").WithPayload(string(log))
|
||||
}
|
||||
|
||||
func convertExecution(execution *replication.Execution) *models.ReplicationExecution {
|
||||
exec := &models.ReplicationExecution{
|
||||
ID: execution.ID,
|
||||
PolicyID: execution.PolicyID,
|
||||
StatusText: execution.StatusMessage,
|
||||
StartTime: strfmt.DateTime(execution.StartTime),
|
||||
EndTime: strfmt.DateTime(execution.EndTime),
|
||||
}
|
||||
// keep backward compatibility
|
||||
if execution.Metrics != nil {
|
||||
exec.Total = execution.Metrics.TaskCount
|
||||
exec.Succeed = execution.Metrics.SuccessTaskCount
|
||||
exec.Failed = execution.Metrics.ErrorTaskCount
|
||||
exec.InProgress = execution.Metrics.PendingTaskCount +
|
||||
execution.Metrics.ScheduledTaskCount + execution.Metrics.RunningTaskCount
|
||||
exec.Stopped = execution.Metrics.StoppedTaskCount
|
||||
}
|
||||
switch execution.Trigger {
|
||||
case task.ExecutionTriggerManual:
|
||||
exec.Trigger = "manual"
|
||||
case task.ExecutionTriggerSchedule:
|
||||
exec.Trigger = "scheduled"
|
||||
case task.ExecutionTriggerEvent:
|
||||
exec.Trigger = "event_based"
|
||||
}
|
||||
switch execution.Status {
|
||||
case job.RunningStatus.String():
|
||||
exec.Status = "InProgress"
|
||||
case job.SuccessStatus.String():
|
||||
exec.Status = "Succeed"
|
||||
case job.StoppedStatus.String():
|
||||
exec.Status = "Stopped"
|
||||
case job.ErrorStatus.String():
|
||||
exec.Status = "Failed"
|
||||
}
|
||||
|
||||
return exec
|
||||
}
|
||||
|
||||
func convertTask(task *replication.Task) *models.ReplicationTask {
|
||||
tk := &models.ReplicationTask{
|
||||
ID: task.ID,
|
||||
ExecutionID: task.ExecutionID,
|
||||
JobID: task.JobID,
|
||||
Operation: task.Operation,
|
||||
ResourceType: task.ResourceType,
|
||||
SrcResource: task.SourceResource,
|
||||
DstResource: task.DestinationResource,
|
||||
StartTime: strfmt.DateTime(task.StartTime),
|
||||
EndTime: strfmt.DateTime(task.EndTime),
|
||||
}
|
||||
// keep backward compatibility
|
||||
switch task.Status {
|
||||
case job.ScheduledStatus.String(), job.RunningStatus.String():
|
||||
tk.Status = "InProgress"
|
||||
case job.SuccessStatus.String():
|
||||
tk.Status = "Succeed"
|
||||
case job.ErrorStatus.String():
|
||||
tk.Status = "Failed"
|
||||
// the status "pending" and "stopped" is same with jobservice, no need to convert
|
||||
default:
|
||||
tk.Status = task.Status
|
||||
}
|
||||
return tk
|
||||
}
|
||||
|
|
@ -58,10 +58,6 @@ func registerLegacyRoutes() {
|
|||
|
||||
beego.Router("/api/"+version+"/replication/adapters", &api.ReplicationAdapterAPI{}, "get:List")
|
||||
beego.Router("/api/"+version+"/replication/adapterinfos", &api.ReplicationAdapterAPI{}, "get:ListAdapterInfos")
|
||||
beego.Router("/api/"+version+"/replication/executions", &api.ReplicationOperationAPI{}, "get:ListExecutions;post:CreateExecution")
|
||||
beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)", &api.ReplicationOperationAPI{}, "get:GetExecution;put:StopExecution")
|
||||
beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)/tasks", &api.ReplicationOperationAPI{}, "get:ListTasks")
|
||||
beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)/tasks/:tid([0-9]+)/log", &api.ReplicationOperationAPI{}, "get:GetTaskLog")
|
||||
beego.Router("/api/"+version+"/replication/policies", &api.ReplicationPolicyAPI{}, "get:List;post:Create")
|
||||
beego.Router("/api/"+version+"/replication/policies/:id([0-9]+)", &api.ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete")
|
||||
|
||||
|
|
|
|||
|
|
@ -21,3 +21,4 @@ package controller
|
|||
//go:generate mockery --case snake --dir ../../controller/scan --name Controller --output ./scan --outpkg scan
|
||||
//go:generate mockery --case snake --dir ../../controller/scan --name Checker --output ./scan --outpkg scan
|
||||
//go:generate mockery --case snake --dir ../../controller/scanner --name Controller --output ./scanner --outpkg scanner
|
||||
//go:generate mockery --case snake --dir ../../controller/replication --name Controller --output ./replication --outpkg replication
|
||||
|
|
|
|||
|
|
@ -0,0 +1,211 @@
|
|||
// Code generated by mockery v2.1.0. DO NOT EDIT.
|
||||
|
||||
package replication
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
model "github.com/goharbor/harbor/src/replication/model"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
q "github.com/goharbor/harbor/src/lib/q"
|
||||
|
||||
replication "github.com/goharbor/harbor/src/controller/replication"
|
||||
)
|
||||
|
||||
// Controller is an autogenerated mock type for the Controller type
|
||||
type Controller struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// ExecutionCount provides a mock function with given fields: ctx, query
|
||||
func (_m *Controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
|
||||
ret := _m.Called(ctx, query)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
|
||||
r0 = rf(ctx, query)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
|
||||
r1 = rf(ctx, query)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// GetExecution provides a mock function with given fields: ctx, executionID
|
||||
func (_m *Controller) GetExecution(ctx context.Context, executionID int64) (*replication.Execution, error) {
|
||||
ret := _m.Called(ctx, executionID)
|
||||
|
||||
var r0 *replication.Execution
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Execution); ok {
|
||||
r0 = rf(ctx, executionID)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*replication.Execution)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
|
||||
r1 = rf(ctx, executionID)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// GetTask provides a mock function with given fields: ctx, taskID
|
||||
func (_m *Controller) GetTask(ctx context.Context, taskID int64) (*replication.Task, error) {
|
||||
ret := _m.Called(ctx, taskID)
|
||||
|
||||
var r0 *replication.Task
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Task); ok {
|
||||
r0 = rf(ctx, taskID)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*replication.Task)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
|
||||
r1 = rf(ctx, taskID)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// GetTaskLog provides a mock function with given fields: ctx, taskID
|
||||
func (_m *Controller) GetTaskLog(ctx context.Context, taskID int64) ([]byte, error) {
|
||||
ret := _m.Called(ctx, taskID)
|
||||
|
||||
var r0 []byte
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64) []byte); ok {
|
||||
r0 = rf(ctx, taskID)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]byte)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
|
||||
r1 = rf(ctx, taskID)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ListExecutions provides a mock function with given fields: ctx, query
|
||||
func (_m *Controller) ListExecutions(ctx context.Context, query *q.Query) ([]*replication.Execution, error) {
|
||||
ret := _m.Called(ctx, query)
|
||||
|
||||
var r0 []*replication.Execution
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Execution); ok {
|
||||
r0 = rf(ctx, query)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]*replication.Execution)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
|
||||
r1 = rf(ctx, query)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ListTasks provides a mock function with given fields: ctx, query
|
||||
func (_m *Controller) ListTasks(ctx context.Context, query *q.Query) ([]*replication.Task, error) {
|
||||
ret := _m.Called(ctx, query)
|
||||
|
||||
var r0 []*replication.Task
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Task); ok {
|
||||
r0 = rf(ctx, query)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]*replication.Task)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
|
||||
r1 = rf(ctx, query)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Start provides a mock function with given fields: ctx, policy, resource, trigger
|
||||
func (_m *Controller) Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (int64, error) {
|
||||
ret := _m.Called(ctx, policy, resource, trigger)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *model.Policy, *model.Resource, string) int64); ok {
|
||||
r0 = rf(ctx, policy, resource, trigger)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *model.Policy, *model.Resource, string) error); ok {
|
||||
r1 = rf(ctx, policy, resource, trigger)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Stop provides a mock function with given fields: ctx, executionID
|
||||
func (_m *Controller) Stop(ctx context.Context, executionID int64) error {
|
||||
ret := _m.Called(ctx, executionID)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
|
||||
r0 = rf(ctx, executionID)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// TaskCount provides a mock function with given fields: ctx, query
|
||||
func (_m *Controller) TaskCount(ctx context.Context, query *q.Query) (int64, error) {
|
||||
ret := _m.Called(ctx, query)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
|
||||
r0 = rf(ctx, query)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
|
||||
r1 = rf(ctx, query)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
|
@ -12,17 +12,6 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scheduler
|
||||
package lib
|
||||
|
||||
import (
|
||||
"github.com/goharbor/harbor/src/replication/dao"
|
||||
"github.com/goharbor/harbor/src/replication/dao/models"
|
||||
)
|
||||
|
||||
// UpdateStatus updates the schedule job status
|
||||
func UpdateStatus(id int64, status string) error {
|
||||
return dao.ScheduleJob.Update(&models.ScheduleJob{
|
||||
ID: id,
|
||||
Status: status,
|
||||
}, "Status")
|
||||
}
|
||||
//go:generate mockery --case snake --dir ../../lib/orm --name Creator --output ./orm --outpkg orm
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
// Code generated by mockery v2.1.0. DO NOT EDIT.
|
||||
|
||||
package orm
|
||||
|
||||
import (
|
||||
orm "github.com/astaxie/beego/orm"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// Creator is an autogenerated mock type for the Creator type
|
||||
type Creator struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// Create provides a mock function with given fields:
|
||||
func (_m *Creator) Create() orm.Ormer {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 orm.Ormer
|
||||
if rf, ok := ret.Get(0).(func() orm.Ormer); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(orm.Ormer)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
|
@ -28,7 +28,7 @@ def get_endpoint():
|
|||
|
||||
def _create_client(server, credential, debug, api_type="products"):
|
||||
cfg = None
|
||||
if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat'):
|
||||
if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat', 'replication'):
|
||||
cfg = v2_swagger_client.Configuration()
|
||||
else:
|
||||
cfg = swagger_client.Configuration()
|
||||
|
|
@ -59,6 +59,7 @@ def _create_client(server, credential, debug, api_type="products"):
|
|||
"repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
|
||||
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
|
||||
}.get(api_type,'Error: Wrong API type')
|
||||
|
||||
def _assert_status_code(expect_code, return_code):
|
||||
|
|
|
|||
|
|
@ -45,41 +45,7 @@ class Replication(base.Base):
|
|||
#else:
|
||||
# raise Exception(r"Check replication rule trigger failed, expect <{}> actual <{}>.".format(expect_trigger, get_trigger))
|
||||
|
||||
|
||||
def start_replication(self, rule_id, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
return client.replications_post(swagger_client.Replication(int(rule_id)))
|
||||
|
||||
def list_replication_jobs(self, rule_id, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
return client.jobs_replication_get(int(rule_id))
|
||||
|
||||
def wait_until_jobs_finish(self, rule_id, retry=10, interval=5, **kwargs):
|
||||
Succeed = False
|
||||
for i in range(retry):
|
||||
Succeed = False
|
||||
jobs = self.get_replication_executions(rule_id, **kwargs)
|
||||
for job in jobs:
|
||||
if job.status == "Succeed":
|
||||
return
|
||||
if not Succeed:
|
||||
time.sleep(interval)
|
||||
if not Succeed:
|
||||
raise Exception("The jobs not Succeed")
|
||||
|
||||
def delete_replication_rule(self, rule_id, expect_status_code = 200, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
_, status_code, _ = client.replication_policies_id_delete_with_http_info(rule_id)
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
|
||||
def trigger_replication_executions(self, rule_id, expect_status_code = 201, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
_, status_code, _ = client.replication_executions_post_with_http_info({"policy_id":rule_id})
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
|
||||
|
||||
def get_replication_executions(self, rule_id, expect_status_code = 200, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
data, status_code, _ = client.replication_executions_get_with_http_info(policy_id=rule_id)
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
return data
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import base
|
||||
import v2_swagger_client
|
||||
from v2_swagger_client.rest import ApiException
|
||||
|
||||
class ReplicationV2(base.Base, object):
|
||||
def __init__(self):
|
||||
super(ReplicationV2,self).__init__(api_type = "replication")
|
||||
|
||||
def wait_until_jobs_finish(self, rule_id, retry=10, interval=5, **kwargs):
|
||||
Succeed = False
|
||||
for i in range(retry):
|
||||
Succeed = False
|
||||
jobs = self.get_replication_executions(rule_id, **kwargs)
|
||||
for job in jobs:
|
||||
if job.status == "Succeed":
|
||||
return
|
||||
if not Succeed:
|
||||
time.sleep(interval)
|
||||
if not Succeed:
|
||||
raise Exception("The jobs not Succeed")
|
||||
|
||||
def trigger_replication_executions(self, rule_id, expect_status_code = 201, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
_, status_code, _ = client.start_replication_with_http_info({"policy_id":rule_id})
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
|
||||
def get_replication_executions(self, rule_id, expect_status_code = 200, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
data, status_code, _ = client.list_replication_executions_with_http_info(policy_id=rule_id)
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
return data
|
||||
|
||||
|
|
@ -9,6 +9,7 @@ from library.replication import Replication
|
|||
from library.registry import Registry
|
||||
from library.artifact import Artifact
|
||||
from library.repository import Repository
|
||||
from library.replication_v2 import ReplicationV2
|
||||
import swagger_client
|
||||
|
||||
class TestProjects(unittest.TestCase):
|
||||
|
|
@ -17,6 +18,7 @@ class TestProjects(unittest.TestCase):
|
|||
self.project = Project()
|
||||
self.user = User()
|
||||
self.replication = Replication()
|
||||
self.replication_v2 = ReplicationV2()
|
||||
self.registry = Registry()
|
||||
self.artifact = Artifact()
|
||||
self.repo = Repository()
|
||||
|
|
@ -87,10 +89,10 @@ class TestProjects(unittest.TestCase):
|
|||
self.replication.check_replication_rule_should_exist(TestProjects.rule_id, rule_name, **ADMIN_CLIENT)
|
||||
|
||||
#6. Trigger the rule;
|
||||
self.replication.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT)
|
||||
self.replication_v2.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT)
|
||||
|
||||
#7. Wait for completion of this replication job;
|
||||
self.replication.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT)
|
||||
self.replication_v2.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT)
|
||||
|
||||
#8. Check image is replicated into target project successfully.
|
||||
artifact = self.artifact.get_reference_info(TestProjects.project_name, self.image, self.tag, **ADMIN_CLIENT)
|
||||
|
|
|
|||
Loading…
Reference in New Issue