Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(retention) refactor to use go swagger api 

Signed-off-by: Ziming Zhang <zziming@vmware.com>
This commit is contained in:
Ziming Zhang 2021-01-08 08:09:34 +08:00 committed by Ziming
parent f566748c77
commit 39fb500318
25 changed files with 1465 additions and 1780 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

500
api/v2.0/legacy_swagger.yaml
View File

@ -2334,316 +2334,6 @@ paths:
description: User have no permission to delete immutable tags of the project.
'500':
description: Internal server errors.
'/retentions/metadatas':
get:
summary: Get Retention Metadatas
description: Get Retention Metadatas.
tags:
- Products
- Retention
responses:
'200':
description: Get Retention Metadatas successfully.
schema:
$ref: '#/definitions/RetentionMetadata'
'/retentions':
post:
summary: Create Retention Policy
description: |
Create Retention Policy, you can reference metadatas API for the policy model.
You can check project metadatas to find whether a retention policy is already binded.
This method should only be called when no retention policy binded to project yet.
tags:
- Products
- Retention
parameters:
- name: policy
in: body
description: Create Retention Policy successfully.
required: true
schema:
$ref: '#/definitions/RetentionPolicy'
responses:
'201':
description: Project created successfully.
headers:
Location:
type: string
description: The URL of the created resource
'400':
description: Illegal format of provided ID value.
'401':
description: User need to log in first.
'403':
description: User have no permission.
'500':
description: Unexpected internal errors.
'/retentions/{id}':
get:
summary: Get Retention Policy
description: Get Retention Policy.
tags:
- Products
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
responses:
'200':
description: Get Retention Policy successfully.
schema:
$ref: '#/definitions/RetentionPolicy'
'401':
description: User need to log in first.
'403':
description: User have no permission.
'500':
description: Unexpected internal errors.
put:
summary: Update Retention Policy
description: |
Update Retention Policy, you can reference metadatas API for the policy model.
You can check project metadatas to find whether a retention policy is already binded.
This method should only be called when retention policy has already binded to project.
tags:
- Products
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: policy
in: body
required: true
schema:
$ref: '#/definitions/RetentionPolicy'
responses:
'200':
description: Update Retention Policy successfully.
'401':
description: User need to log in first.
'403':
description: User have no permission.
'500':
description: Unexpected internal errors.
'/retentions/{id}/executions':
post:
summary: Trigger a Retention job
description: Trigger a Retention job, if dry_run is True, nothing would be deleted actually.
tags:
- Products
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: action
in: body
required: true
schema:
type: object
properties:
dry_run:
type: boolean
responses:
'200':
description: Trigger a Retention job successfully.
'401':
description: User need to log in first.
'403':
description: User have no permission.
'500':
description: Unexpected internal errors.
get:
summary: Get a Retention job
description: Get a Retention job, job status may be delayed before job service schedule it up.
tags:
- Products
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: page
in: query
type: integer
format: int32
required: false
description: The page number.
- name: page_size
in: query
type: integer
format: int32
required: false
description: The size of per page.
responses:
'200':
description: Get a Retention job successfully.
schema:
type: array
items:
type: object
$ref: '#/definitions/RetentionExecution'
headers:
X-Total-Count:
description: The total count of available items
type: integer
Link:
description: Link to previous page and next page
type: string
'401':
description: User need to log in first.
'403':
description: User have no permission.
'500':
description: Unexpected internal errors.
'/retentions/{id}/executions/{eid}':
patch:
summary: Stop a Retention job
description: Stop a Retention job, only support "stop" action now.
tags:
- Products
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: eid
in: path
type: integer
format: int64
required: true
description: Retention execution ID.
- name: action
in: body
description: The action, only support "stop" now.
required: true
schema:
type: object
properties:
action:
type: string
responses:
'200':
description: Stop a Retention job successfully.
'401':
description: User need to log in first.
'403':
description: User have no permission.
'500':
description: Unexpected internal errors.
'/retentions/{id}/executions/{eid}/tasks':
get:
summary: Get Retention job tasks
description: Get Retention job tasks, each repository as a task.
tags:
- Products
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: eid
in: path
type: integer
format: int64
required: true
description: Retention execution ID.
- name: page
in: query
type: integer
format: int32
required: false
description: The page number.
- name: page_size
in: query
type: integer
format: int32
required: false
description: The size of per page.
responses:
'200':
description: Get Retention job tasks successfully.
schema:
type: array
items:
type: object
$ref: '#/definitions/RetentionExecutionTask'
headers:
X-Total-Count:
description: The total count of available items
type: integer
Link:
description: Link to previous page and next page
type: string
'401':
description: User need to log in first.
'403':
description: User have no permission.
'500':
description: Unexpected internal errors.
'/retentions/{id}/executions/{eid}/tasks/{tid}':
get:
summary: Get Retention job task log
description: Get Retention job task log, tags ratain or deletion detail will be shown in a table.
tags:
- Products
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: eid
in: path
type: integer
format: int64
required: true
description: Retention execution ID.
- name: tid
in: path
type: integer
format: int64
required: true
description: Retention execution ID.
responses:
'200':
description: Get Retention job task log successfully.
schema:
type: string
'401':
description: User need to log in first.
'403':
description: User have no permission.
'500':
description: Unexpected internal errors.
'/scanners':
get:
summary: List scanner registrations
@ -4279,196 +3969,6 @@ definitions:
type: string
description: The webhook job update time.
RetentionMetadata:
type: object
description: the tag retention metadata
properties:
templates:
type: array
description: templates
items:
$ref: '#/definitions/RetentionRuleMetadata'
scope_selectors:
type: array
description: supported scope selectors
items:
$ref: '#/definitions/RetentionSelectorMetadata'
tag_selectors:
type: array
description: supported tag selectors
items:
$ref: '#/definitions/RetentionSelectorMetadata'
RetentionRuleMetadata:
type: object
description: the tag retention rule metadata
properties:
rule_template:
type: string
description: rule id
display_text:
type: string
description: rule display text
action:
type: string
description: rule action
params:
type: array
description: rule params
items:
$ref: '#/definitions/RetentionRuleParamMetadata'
RetentionRuleParamMetadata:
type: object
description: rule param
properties:
type:
type: string
unit:
type: string
required:
type: boolean
RetentionSelectorMetadata:
type: object
description: retention selector
properties:
display_text:
type: string
kind:
type: string
decorations:
type: array
items:
type: string
RetentionPolicy:
type: object
description: retention policy
properties:
id:
type: integer
format: int64
algorithm:
type: string
rules:
type: array
items:
$ref: '#/definitions/RetentionRule'
trigger:
type: object
$ref: '#/definitions/RetentionRuleTrigger'
scope:
type: object
$ref: '#/definitions/RetentionPolicyScope'
RetentionRuleTrigger:
type: object
properties:
kind:
type: string
settings:
type: object
references:
type: object
RetentionPolicyScope:
type: object
properties:
level:
type: string
ref:
type: integer
RetentionRule:
type: object
properties:
id:
type: integer
priority:
type: integer
disabled:
type: boolean
action:
type: string
template:
type: string
params:
type: object
additionalProperties:
type: object
tag_selectors:
type: array
items:
$ref: '#/definitions/RetentionSelector'
scope_selectors:
type: object
additionalProperties:
type: array
items:
$ref: '#/definitions/RetentionSelector'
RetentionSelector:
type: object
properties:
kind:
type: string
decoration:
type: string
pattern:
type: string
extras:
type: string
RetentionExecution:
type: object
properties:
id:
type: integer
format: int64
policy_id:
type: integer
format: int64
start_time:
type: string
end_time:
type: string
status:
type: string
trigger:
type: string
dry_run:
type: boolean
RetentionExecutionTask:
type: object
properties:
id:
type: integer
format: int64
execution_id:
type: integer
format: int64
repository:
type: string
job_id:
type: string
status:
type: string
status_code:
type: integer
status_revision:
type: integer
format: int64
start_time:
type: string
end_time:
type: string
total:
type: integer
retained:
type: integer
QuotaSwitcher:
type: object
properties:

507
api/v2.0/swagger.yaml
View File

@ -2343,6 +2343,323 @@ paths:
description: The API server is alive
schema:
type: string
/retentions/metadatas:
get:
summary: Get Retention Metadatas
description: Get Retention Metadatas.
operationId: getRentenitionMetadata
tags:
- Retention
responses:
'200':
description: Get Retention Metadatas successfully.
schema:
$ref: '#/definitions/RetentionMetadata'
/retentions:
post:
summary: Create Retention Policy
operationId: createRetention
description: >-
Create Retention Policy, you can reference metadatas API for the policy model.
You can check project metadatas to find whether a retention policy is already binded.
This method should only be called when no retention policy binded to project yet.
tags:
- Retention
parameters:
- name: policy
in: body
description: Create Retention Policy successfully.
required: true
schema:
$ref: '#/definitions/RetentionPolicy'
responses:
'201':
$ref: '#/responses/201'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/retentions/{id}:
get:
summary: Get Retention Policy
operationId: getRetention
description: Get Retention Policy.
tags:
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
responses:
'200':
description: Get Retention Policy successfully.
schema:
$ref: '#/definitions/RetentionPolicy'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
put:
summary: Update Retention Policy
operationId: updateRetention
description: >-
Update Retention Policy, you can reference metadatas API for the policy model.
You can check project metadatas to find whether a retention policy is already binded.
This method should only be called when retention policy has already binded to project.
tags:
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: policy
in: body
required: true
schema:
$ref: '#/definitions/RetentionPolicy'
responses:
'200':
description: Update Retention Policy successfully.
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/retentions/{id}/executions:
post:
summary: Trigger a Retention job
operationId: triggerRetentionJob
description: Trigger a Retention job, if dry_run is True, nothing would be deleted actually.
tags:
- Retention
produces:
- text/plain
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: body
in: body
required: true
schema:
type: object
properties:
dry_run:
type: boolean
responses:
'200':
description: Trigger a Retention job successfully.
'201':
description: Retention job created successfully.
headers:
Location:
type: string
description: The URL of the created resource
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
get:
summary: Get Retention jobs
operationId: listRetentionJob
description: Get Retention jobs, job status may be delayed before job service schedule it up.
tags:
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: page
in: query
type: integer
format: int64
required: false
description: The page number.
- name: page_size
in: query
type: integer
format: int64
required: false
description: The size of per page.
responses:
'200':
description: Get a Retention job successfully.
schema:
type: array
items:
type: object
$ref: '#/definitions/RetentionExecution'
headers:
X-Total-Count:
description: The total count of available items
type: integer
Link:
description: Link to previous page and next page
type: string
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/retentions/{id}/executions/{eid}:
patch:
summary: Stop a Retention job
operationId: operateRetentionJob
description: Stop a Retention job, only support "stop" action now.
tags:
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: eid
in: path
type: integer
format: int64
required: true
description: Retention execution ID.
- name: body
in: body
description: The action, only support "stop" now.
required: true
schema:
type: object
properties:
action:
type: string
responses:
'200':
description: Stop a Retention job successfully.
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/retentions/{id}/executions/{eid}/tasks:
get:
summary: Get Retention job tasks
operationId: listRetentionTasks
description: Get Retention job tasks, each repository as a task.
tags:
- Retention
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: eid
in: path
type: integer
format: int64
required: true
description: Retention execution ID.
- name: page
in: query
type: integer
format: int64
required: false
description: The page number.
- name: page_size
in: query
type: integer
format: int64
required: false
description: The size of per page.
responses:
'200':
description: Get Retention job tasks successfully.
schema:
type: array
items:
type: object
$ref: '#/definitions/RetentionExecutionTask'
headers:
X-Total-Count:
description: The total count of available items
type: integer
Link:
description: Link to previous page and next page
type: string
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/retentions/{id}/executions/{eid}/tasks/{tid}:
get:
summary: Get Retention job task log
operationId: getRetentionTaskLog
description: Get Retention job task log, tags ratain or deletion detail will be shown in a table.
tags:
- Retention
produces:
- text/plain
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Retention ID.
- name: eid
in: path
type: integer
format: int64
required: true
description: Retention execution ID.
- name: tid
in: path
type: integer
format: int64
required: true
description: Retention execution ID.
responses:
'200':
description: Get Retention job task log successfully.
schema:
type: string
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
parameters:
query:
name: q
@ -3822,3 +4139,193 @@ definitions:
- Manual
- Schedule
- Event
RetentionMetadata:
type: object
description: the tag retention metadata
properties:
templates:
type: array
description: templates
items:
$ref: '#/definitions/RetentionRuleMetadata'
scope_selectors:
type: array
description: supported scope selectors
items:
$ref: '#/definitions/RetentionSelectorMetadata'
tag_selectors:
type: array
description: supported tag selectors
items:
$ref: '#/definitions/RetentionSelectorMetadata'
RetentionRuleMetadata:
type: object
description: the tag retention rule metadata
properties:
rule_template:
type: string
description: rule id
display_text:
type: string
description: rule display text
action:
type: string
description: rule action
params:
type: array
description: rule params
items:
$ref: '#/definitions/RetentionRuleParamMetadata'
RetentionRuleParamMetadata:
type: object
description: rule param
properties:
type:
type: string
unit:
type: string
required:
type: boolean
RetentionSelectorMetadata:
type: object
description: retention selector
properties:
display_text:
type: string
kind:
type: string
decorations:
type: array
items:
type: string
RetentionPolicy:
type: object
description: retention policy
properties:
id:
type: integer
format: int64
algorithm:
type: string
rules:
type: array
items:
$ref: '#/definitions/RetentionRule'
trigger:
type: object
$ref: '#/definitions/RetentionRuleTrigger'
scope:
type: object
$ref: '#/definitions/RetentionPolicyScope'
RetentionRuleTrigger:
type: object
properties:
kind:
type: string
settings:
type: object
references:
type: object
RetentionPolicyScope:
type: object
properties:
level:
type: string
ref:
type: integer
RetentionRule:
type: object
properties:
id:
type: integer
priority:
type: integer
disabled:
type: boolean
action:
type: string
template:
type: string
params:
type: object
additionalProperties:
type: object
tag_selectors:
type: array
items:
$ref: '#/definitions/RetentionSelector'
scope_selectors:
type: object
additionalProperties:
type: array
items:
$ref: '#/definitions/RetentionSelector'
RetentionSelector:
type: object
properties:
kind:
type: string
decoration:
type: string
pattern:
type: string
extras:
type: string
RetentionExecution:
type: object
properties:
id:
type: integer
format: int64
policy_id:
type: integer
format: int64
start_time:
type: string
end_time:
type: string
status:
type: string
trigger:
type: string
dry_run:
type: boolean
RetentionExecutionTask:
type: object
properties:
id:
type: integer
format: int64
execution_id:
type: integer
format: int64
repository:
type: string
job_id:
type: string
status:
type: string
status_code:
type: integer
status_revision:
type: integer
format: int64
start_time:
type: string
end_time:
type: string
total:
type: integer
retained:
type: integer

2
src/common/dao/testutils.go
View File

@ -96,7 +96,7 @@ func initDatabaseForTest(db *models.Database) {
}
if alias != "default" {
if err = globalOrm.Using(alias); err != nil {
if err = GetOrmer().Using(alias); err != nil {
log.Fatalf("failed to create new orm: %v", err)
}
}

2
src/controller/event/handler/init.go
View File

@ -34,7 +34,7 @@ func init() {
notifier.Subscribe(event.TopicScanningCompleted, &scan.Handler{})
notifier.Subscribe(event.TopicDeleteArtifact, &scan.DelArtHandler{Context: orm.Context})
notifier.Subscribe(event.TopicReplication, &artifact.ReplicationHandler{})
notifier.Subscribe(event.TopicTagRetention, &artifact.RetentionHandler{RetentionController: artifact.DefaultRetentionControllerFunc})
notifier.Subscribe(event.TopicTagRetention, &artifact.RetentionHandler{})
// replication
notifier.Subscribe(event.TopicPushArtifact, &replication.Handler{})

20
src/controller/event/handler/webhook/artifact/retention.go
View File

@ -2,31 +2,22 @@ package artifact
import (
"fmt"
"github.com/goharbor/harbor/src/controller/retention"
"github.com/goharbor/harbor/src/lib/orm"
"strings"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/controller/event/handler/util"
evtModel "github.com/goharbor/harbor/src/controller/event/model"
"github.com/goharbor/harbor/src/core/api"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/notifier/model"
"github.com/goharbor/harbor/src/pkg/retention"
)
// RetentionHandler preprocess tag retention event data
type RetentionHandler struct {
RetentionController func() retention.APIController
}
// DefaultRetentionControllerFunc ...
var DefaultRetentionControllerFunc = NewRetentionController
// NewRetentionController ...
func NewRetentionController() retention.APIController {
return api.GetRetentionController()
}
// Name ...
@ -85,7 +76,8 @@ func (r *RetentionHandler) IsStateful() bool {
}
func (r *RetentionHandler) constructRetentionPayload(event *event.RetentionEvent) (*model.Payload, bool, int64, error) {
task, err := r.RetentionController().GetRetentionExecTask(event.TaskID)
ctx := orm.Context()
task, err := retention.Ctl.GetRetentionExecTask(ctx, event.TaskID)
if err != nil {
log.Errorf("failed to get retention task %d: error: %v", event.TaskID, err)
return nil, false, 0, err
@ -94,7 +86,7 @@ func (r *RetentionHandler) constructRetentionPayload(event *event.RetentionEvent
return nil, false, 0, fmt.Errorf("task %d not found with retention event", event.TaskID)
}
execution, err := r.RetentionController().GetRetentionExec(task.ExecutionID)
execution, err := retention.Ctl.GetRetentionExec(ctx, task.ExecutionID)
if err != nil {
log.Errorf("failed to get retention execution %d: error: %v", task.ExecutionID, err)
return nil, false, 0, err
@ -107,7 +99,7 @@ func (r *RetentionHandler) constructRetentionPayload(event *event.RetentionEvent
return nil, true, 0, nil
}
md, err := r.RetentionController().GetRetention(execution.PolicyID)
md, err := retention.Ctl.GetRetention(ctx, execution.PolicyID)
if err != nil {
log.Errorf("failed to get tag retention policy %d: error: %v", execution.PolicyID, err)
return nil, false, 0, err

39
src/controller/event/handler/webhook/artifact/retention_test.go
View File

@ -1,32 +1,54 @@
package artifact
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/controller/retention"
ret "github.com/goharbor/harbor/src/pkg/retention"
"github.com/stretchr/testify/mock"
"os"
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/retention"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/pkg/notification"
retentiontesting "github.com/goharbor/harbor/src/testing/controller/retention"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRetentionHandler_Handle(t *testing.T) {
config.Init()
handler := &RetentionHandler{RetentionController: DefaultRetentionControllerFunc}
handler := &RetentionHandler{}
policyMgr := notification.PolicyMgr
retentionCtlFunc := handler.RetentionController
oldretentionCtl := retention.Ctl
defer func() {
notification.PolicyMgr = policyMgr
handler.RetentionController = retentionCtlFunc
retention.Ctl = oldretentionCtl
}()
notification.PolicyMgr = &fakedNotificationPolicyMgr{}
handler.RetentionController = retention.FakedRetentionControllerFunc
retentionCtl := &retentiontesting.Controller{}
retention.Ctl = retentionCtl
retentionCtl.On("GetRetentionExecTask", mock.Anything, mock.Anything).
Return(&ret.Task{
ID: 1,
ExecutionID: 1,
Status: "Success",
StartTime: time.Now(),
EndTime: time.Now(),
}, nil)
retentionCtl.On("GetRetentionExec", mock.Anything, mock.Anything).Return(&ret.Execution{
ID: 1,
PolicyID: 1,
Status: "Success",
Trigger: "Manual",
DryRun: true,
StartTime: time.Now(),
EndTime: time.Now(),
}, nil)
type args struct {
data interface{}
@ -80,3 +102,8 @@ func TestRetentionHandler_IsStateful(t *testing.T) {
handler := &RetentionHandler{}
assert.False(t, handler.IsStateful())
}
func TestMain(m *testing.M) {
dao.PrepareTestForPostgresSQL()
os.Exit(m.Run())
}

26
src/controller/retention/callback.go Normal file
View File

@ -0,0 +1,26 @@
package retention
import (
"context"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/scheduler"
)
func init() {
err := scheduler.RegisterCallbackFunc(SchedulerCallback, retentionCallback)
if err != nil {
log.Fatalf("failed to register retention callback, %v", err)
}
}
func retentionCallback(ctx context.Context, p string) error {
param := &TriggerParam{}
if err := json.Unmarshal([]byte(p), param); err != nil {
return fmt.Errorf("failed to unmarshal the param: %v", err)
}
_, err := Ctl.TriggerRetentionExec(ctx, param.PolicyID, param.Trigger, false)
return err
}

134
src/pkg/retention/controller.go → src/controller/retention/controller.go
View File

@ -19,53 +19,58 @@ import (
"fmt"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/retention"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
"time"
)
// go:generate mockery -name APIController -case snake
// go:generate mockery -name Controller -case snake
// APIController to handle the requests related with retention
type APIController interface {
GetRetention(id int64) (*policy.Metadata, error)
// Controller to handle the requests related with retention
type Controller interface {
GetRetention(ctx context.Context, id int64) (*policy.Metadata, error)
CreateRetention(p *policy.Metadata) (int64, error)
CreateRetention(ctx context.Context, p *policy.Metadata) (int64, error)
UpdateRetention(p *policy.Metadata) error
UpdateRetention(ctx context.Context, p *policy.Metadata) error
DeleteRetention(id int64) error
DeleteRetention(ctx context.Context, id int64) error
TriggerRetentionExec(policyID int64, trigger string, dryRun bool) (int64, error)
TriggerRetentionExec(ctx context.Context, policyID int64, trigger string, dryRun bool) (int64, error)
OperateRetentionExec(eid int64, action string) error
OperateRetentionExec(ctx context.Context, eid int64, action string) error
GetRetentionExec(eid int64) (*Execution, error)
GetRetentionExec(ctx context.Context, eid int64) (*retention.Execution, error)
ListRetentionExecs(policyID int64, query *q.Query) ([]*Execution, error)
ListRetentionExecs(ctx context.Context, policyID int64, query *q.Query) ([]*retention.Execution, error)
GetTotalOfRetentionExecs(policyID int64) (int64, error)
GetTotalOfRetentionExecs(ctx context.Context, policyID int64) (int64, error)
ListRetentionExecTasks(executionID int64, query *q.Query) ([]*Task, error)
ListRetentionExecTasks(ctx context.Context, executionID int64, query *q.Query) ([]*retention.Task, error)
GetTotalOfRetentionExecTasks(executionID int64) (int64, error)
GetTotalOfRetentionExecTasks(ctx context.Context, executionID int64) (int64, error)
GetRetentionExecTaskLog(taskID int64) ([]byte, error)
GetRetentionExecTaskLog(ctx context.Context, taskID int64) ([]byte, error)
GetRetentionExecTask(taskID int64) (*Task, error)
GetRetentionExecTask(ctx context.Context, taskID int64) (*retention.Task, error)
}
// DefaultAPIController ...
type DefaultAPIController struct {
manager Manager
var (
// Ctl is a global retention controller instance
Ctl = NewController()
)
// defaultController ...
type defaultController struct {
manager retention.Manager
execMgr task.ExecutionManager
taskMgr task.Manager
launcher Launcher
launcher retention.Launcher
projectManager project.Manager
repositoryMgr repository.Manager
scheduler scheduler.Scheduler
@ -84,12 +89,12 @@ type TriggerParam struct {
}
// GetRetention Get Retention
func (r *DefaultAPIController) GetRetention(id int64) (*policy.Metadata, error) {
func (r *defaultController) GetRetention(ctx context.Context, id int64) (*policy.Metadata, error) {
return r.manager.GetPolicy(id)
}
// CreateRetention Create Retention
func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error) {
func (r *defaultController) CreateRetention(ctx context.Context, p *policy.Metadata) (int64, error) {
id, err := r.manager.CreatePolicy(p)
if err != nil {
return 0, err
@ -99,9 +104,9 @@ func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
if ok && len(cron.(string)) > 0 {
extras := make(map[string]interface{})
if _, err = r.scheduler.Schedule(orm.Context(), schedulerVendorType, id, "", cron.(string), SchedulerCallback, TriggerParam{
if _, err = r.scheduler.Schedule(ctx, schedulerVendorType, id, "", cron.(string), SchedulerCallback, TriggerParam{
PolicyID: id,
Trigger: ExecutionTriggerSchedule,
Trigger: retention.ExecutionTriggerSchedule,
}, extras); err != nil {
return 0, err
}
@ -112,7 +117,7 @@ func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error
}
// UpdateRetention Update Retention
func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
func (r *defaultController) UpdateRetention(ctx context.Context, p *policy.Metadata) error {
p0, err := r.manager.GetPolicy(p.ID)
if err != nil {
return err
@ -152,16 +157,16 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
return err
}
if needUn {
err = r.scheduler.UnScheduleByVendor(orm.Context(), schedulerVendorType, p.ID)
err = r.scheduler.UnScheduleByVendor(ctx, schedulerVendorType, p.ID)
if err != nil {
return err
}
}
if needSch {
extras := make(map[string]interface{})
_, err := r.scheduler.Schedule(orm.Context(), schedulerVendorType, p.ID, "", p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{
_, err := r.scheduler.Schedule(ctx, schedulerVendorType, p.ID, "", p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{
PolicyID: p.ID,
Trigger: ExecutionTriggerSchedule,
Trigger: retention.ExecutionTriggerSchedule,
}, extras)
if err != nil {
return err
@ -172,19 +177,18 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
}
// DeleteRetention Delete Retention
func (r *DefaultAPIController) DeleteRetention(id int64) error {
func (r *defaultController) DeleteRetention(ctx context.Context, id int64) error {
p, err := r.manager.GetPolicy(id)
if err != nil {
return err
}
if p.Trigger.Kind == policy.TriggerKindSchedule && len(p.Trigger.Settings[policy.TriggerSettingsCron].(string)) > 0 {
err = r.scheduler.UnScheduleByVendor(orm.Context(), schedulerVendorType, id)
err = r.scheduler.UnScheduleByVendor(ctx, schedulerVendorType, id)
if err != nil {
return err
}
}
ctx := orm.Context()
err = r.deleteExecs(ctx, id)
if err != nil {
return err
@ -193,7 +197,7 @@ func (r *DefaultAPIController) DeleteRetention(id int64) error {
}
// deleteExecs delete executions
func (r *DefaultAPIController) deleteExecs(ctx context.Context, vendorID int64) error {
func (r *defaultController) deleteExecs(ctx context.Context, vendorID int64) error {
executions, err := r.execMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"VendorType": job.Retention,
@ -215,19 +219,18 @@ func (r *DefaultAPIController) deleteExecs(ctx context.Context, vendorID int64)
}
// TriggerRetentionExec Trigger Retention Execution
func (r *DefaultAPIController) TriggerRetentionExec(policyID int64, trigger string, dryRun bool) (int64, error) {
func (r *defaultController) TriggerRetentionExec(ctx context.Context, policyID int64, trigger string, dryRun bool) (int64, error) {
p, err := r.manager.GetPolicy(policyID)
if err != nil {
return 0, err
}
ctx := orm.Context()
id, err := r.execMgr.Create(ctx, job.Retention, policyID, trigger,
map[string]interface{}{
"dry_run": dryRun,
},
)
if _, err = r.launcher.Launch(p, id, dryRun); err != nil {
if _, err = r.launcher.Launch(ctx, p, id, dryRun); err != nil {
if err1 := r.execMgr.StopAndWait(ctx, id, 10*time.Second); err1 != nil {
logger.Errorf("failed to stop the retention execution %d: %v", id, err1)
}
@ -241,8 +244,7 @@ func (r *DefaultAPIController) TriggerRetentionExec(policyID int64, trigger stri
}
// OperateRetentionExec Operate Retention Execution
func (r *DefaultAPIController) OperateRetentionExec(eid int64, action string) error {
ctx := orm.Context()
func (r *defaultController) OperateRetentionExec(ctx context.Context, eid int64, action string) error {
e, err := r.execMgr.Get(ctx, eid)
if err != nil {
return err
@ -252,15 +254,14 @@ func (r *DefaultAPIController) OperateRetentionExec(eid int64, action string) er
}
switch action {
case "stop":
return r.launcher.Stop(eid)
return r.launcher.Stop(ctx, eid)
default:
return fmt.Errorf("not support action %s", action)
}
}
// GetRetentionExec Get Retention Execution
func (r *DefaultAPIController) GetRetentionExec(executionID int64) (*Execution, error) {
ctx := orm.Context()
func (r *defaultController) GetRetentionExec(ctx context.Context, executionID int64) (*retention.Execution, error) {
e, err := r.execMgr.Get(ctx, executionID)
if err != nil {
return nil, err
@ -270,8 +271,7 @@ func (r *DefaultAPIController) GetRetentionExec(executionID int64) (*Execution,
}
// ListRetentionExecs List Retention Executions
func (r *DefaultAPIController) ListRetentionExecs(policyID int64, query *q.Query) ([]*Execution, error) {
ctx := orm.Context()
func (r *defaultController) ListRetentionExecs(ctx context.Context, policyID int64, query *q.Query) ([]*retention.Execution, error) {
query = q.MustClone(query)
query.Keywords["VendorType"] = job.Retention
query.Keywords["VendorID"] = policyID
@ -279,15 +279,15 @@ func (r *DefaultAPIController) ListRetentionExecs(policyID int64, query *q.Query
if err != nil {
return nil, err
}
var executions []*Execution
var executions []*retention.Execution
for _, exec := range execs {
executions = append(executions, convertExecution(exec))
}
return executions, nil
}
func convertExecution(exec *task.Execution) *Execution {
return &Execution{
func convertExecution(exec *task.Execution) *retention.Execution {
return &retention.Execution{
ID: exec.ID,
PolicyID: exec.VendorID,
StartTime: exec.StartTime,
@ -299,8 +299,7 @@ func convertExecution(exec *task.Execution) *Execution {
}
// GetTotalOfRetentionExecs Count Retention Executions
func (r *DefaultAPIController) GetTotalOfRetentionExecs(policyID int64) (int64, error) {
ctx := orm.Context()
func (r *defaultController) GetTotalOfRetentionExecs(ctx context.Context, policyID int64) (int64, error) {
return r.execMgr.Count(ctx, &q.Query{
Keywords: map[string]interface{}{
"VendorType": job.Retention,
@ -310,8 +309,7 @@ func (r *DefaultAPIController) GetTotalOfRetentionExecs(policyID int64) (int64,
}
// ListRetentionExecTasks List Retention Execution Histories
func (r *DefaultAPIController) ListRetentionExecTasks(executionID int64, query *q.Query) ([]*Task, error) {
ctx := orm.Context()
func (r *defaultController) ListRetentionExecTasks(ctx context.Context, executionID int64, query *q.Query) ([]*retention.Task, error) {
query = q.MustClone(query)
query.Keywords["VendorType"] = job.Retention
query.Keywords["ExecutionID"] = executionID
@ -319,15 +317,15 @@ func (r *DefaultAPIController) ListRetentionExecTasks(executionID int64, query *
if err != nil {
return nil, err
}
var tasks []*Task
var tasks []*retention.Task
for _, tk := range tks {
tasks = append(tasks, convertTask(tk))
}
return tasks, nil
}
func convertTask(t *task.Task) *Task {
return &Task{
func convertTask(t *task.Task) *retention.Task {
return &retention.Task{
ID: t.ID,
ExecutionID: t.ExecutionID,
Repository: t.GetStringFromExtraAttrs("repository"),
@ -342,8 +340,7 @@ func convertTask(t *task.Task) *Task {
}
// GetTotalOfRetentionExecTasks Count Retention Execution Histories
func (r *DefaultAPIController) GetTotalOfRetentionExecTasks(executionID int64) (int64, error) {
ctx := orm.Context()
func (r *defaultController) GetTotalOfRetentionExecTasks(ctx context.Context, executionID int64) (int64, error) {
return r.taskMgr.Count(ctx, &q.Query{
Keywords: map[string]interface{}{
"VendorType": job.Retention,
@ -353,14 +350,12 @@ func (r *DefaultAPIController) GetTotalOfRetentionExecTasks(executionID int64) (
}
// GetRetentionExecTaskLog Get Retention Execution Task Log
func (r *DefaultAPIController) GetRetentionExecTaskLog(taskID int64) ([]byte, error) {
ctx := orm.Context()
func (r *defaultController) GetRetentionExecTaskLog(ctx context.Context, taskID int64) ([]byte, error) {
return r.taskMgr.GetLog(ctx, taskID)
}
// GetRetentionExecTask Get Retention Execution Task
func (r *DefaultAPIController) GetRetentionExecTask(taskID int64) (*Task, error) {
ctx := orm.Context()
func (r *defaultController) GetRetentionExecTask(ctx context.Context, taskID int64) (*retention.Task, error) {
t, err := r.taskMgr.Get(ctx, taskID)
if err != nil {
return nil, err
@ -370,8 +365,7 @@ func (r *DefaultAPIController) GetRetentionExecTask(taskID int64) (*Task, error)
}
// UpdateTaskInfo Update task info
func (r *DefaultAPIController) UpdateTaskInfo(taskID int64, total int, retained int) error {
ctx := orm.Context()
func (r *defaultController) UpdateTaskInfo(ctx context.Context, taskID int64, total int, retained int) error {
t, err := r.taskMgr.Get(ctx, taskID)
if err != nil {
return err
@ -383,15 +377,17 @@ func (r *DefaultAPIController) UpdateTaskInfo(taskID int64, total int, retained
return r.taskMgr.UpdateExtraAttrs(ctx, taskID, t.ExtraAttrs)
}
// NewAPIController ...
func NewAPIController(retentionMgr Manager, projectManager project.Manager, repositoryMgr repository.Manager, scheduler scheduler.Scheduler, retentionLauncher Launcher, execMgr task.ExecutionManager, taskMgr task.Manager) APIController {
return &DefaultAPIController{
// NewController ...
func NewController() Controller {
retentionMgr := retention.NewManager()
retentionLauncher := retention.NewLauncher(project.Mgr, repository.Mgr, retentionMgr, task.ExecMgr, task.Mgr)
return &defaultController{
manager: retentionMgr,
execMgr: execMgr,
taskMgr: taskMgr,
execMgr: task.ExecMgr,
taskMgr: task.Mgr,
launcher: retentionLauncher,
projectManager: projectManager,
repositoryMgr: repositoryMgr,
scheduler: scheduler,
projectManager: project.Mgr,
repositoryMgr: repository.Mgr,
scheduler: scheduler.Sched,
}
}

68
src/pkg/retention/controller_test.go → src/controller/retention/controller_test.go
View File

@ -16,8 +16,15 @@ package retention
import (
"context"
"os"
"strings"
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/retention"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
@ -28,8 +35,6 @@ import (
testingTask "github.com/goharbor/harbor/src/testing/pkg/task"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"strings"
"testing"
)
type ControllerTestSuite struct {
@ -43,6 +48,11 @@ func (s *ControllerTestSuite) SetupSuite() {
}
func TestMain(m *testing.M) {
dao.PrepareTestForPostgresSQL()
os.Exit(m.Run())
}
// TestController ...
func TestController(t *testing.T) {
suite.Run(t, new(ControllerTestSuite))
@ -55,7 +65,7 @@ func (s *ControllerTestSuite) TestPolicy() {
retentionLauncher := &fakeLauncher{}
execMgr := &testingTask.ExecutionManager{}
taskMgr := &testingTask.Manager{}
retentionMgr := NewManager()
retentionMgr := retention.NewManager()
execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
execMgr.On("Delete", mock.Anything, mock.Anything).Return(nil)
execMgr.On("Get", mock.Anything, mock.Anything).Return(&task.Execution{
@ -83,7 +93,15 @@ func (s *ControllerTestSuite) TestPolicy() {
taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
c := NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher, execMgr, taskMgr)
c := defaultController{
manager: retentionMgr,
execMgr: execMgr,
taskMgr: taskMgr,
launcher: retentionLauncher,
projectManager: projectMgr,
repositoryMgr: repositoryMgr,
scheduler: retentionScheduler,
}
p1 := &policy.Metadata{
Algorithm: "or",
@ -150,26 +168,27 @@ func (s *ControllerTestSuite) TestPolicy() {
},
}
id, err := c.CreateRetention(p1)
ctx := orm.Context()
id, err := c.CreateRetention(ctx, p1)
s.Require().Nil(err)
s.Require().True(id > 0)
p1, err = c.GetRetention(id)
p1, err = c.GetRetention(ctx, id)
s.Require().Nil(err)
s.Require().EqualValues("project", p1.Scope.Level)
s.Require().True(p1.ID > 0)
p1.Scope.Level = "test"
err = c.UpdateRetention(p1)
err = c.UpdateRetention(ctx, p1)
s.Require().Nil(err)
p1, err = c.GetRetention(id)
p1, err = c.GetRetention(ctx, id)
s.Require().Nil(err)
s.Require().EqualValues("test", p1.Scope.Level)
err = c.DeleteRetention(id)
err = c.DeleteRetention(ctx, id)
s.Require().Nil(err)
p1, err = c.GetRetention(id)
p1, err = c.GetRetention(ctx, id)
s.Require().NotNil(err)
s.Require().True(strings.Contains(err.Error(), "no such Retention policy"))
s.Require().Nil(p1)
@ -182,7 +201,7 @@ func (s *ControllerTestSuite) TestExecution() {
retentionLauncher := &fakeLauncher{}
execMgr := &testingTask.ExecutionManager{}
taskMgr := &testingTask.Manager{}
retentionMgr := NewManager()
retentionMgr := retention.NewManager()
execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
execMgr.On("Get", mock.Anything, mock.Anything).Return(&task.Execution{
ID: 1,
@ -209,7 +228,15 @@ func (s *ControllerTestSuite) TestExecution() {
taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
m := NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher, execMgr, taskMgr)
m := defaultController{
manager: retentionMgr,
execMgr: execMgr,
taskMgr: taskMgr,
launcher: retentionLauncher,
projectManager: projectMgr,
repositoryMgr: repositoryMgr,
scheduler: retentionScheduler,
}
p1 := &policy.Metadata{
Algorithm: "or",
@ -251,27 +278,28 @@ func (s *ControllerTestSuite) TestExecution() {
},
}
policyID, err := m.CreateRetention(p1)
ctx := orm.Context()
policyID, err := m.CreateRetention(ctx, p1)
s.Require().Nil(err)
s.Require().True(policyID > 0)
id, err := m.TriggerRetentionExec(policyID, ExecutionTriggerManual, false)
id, err := m.TriggerRetentionExec(ctx, policyID, retention.ExecutionTriggerManual, false)
s.Require().Nil(err)
s.Require().True(id > 0)
e1, err := m.GetRetentionExec(id)
e1, err := m.GetRetentionExec(ctx, id)
s.Require().Nil(err)
s.Require().NotNil(e1)
s.Require().EqualValues(id, e1.ID)
err = m.OperateRetentionExec(id, "stop")
err = m.OperateRetentionExec(ctx, id, "stop")
s.Require().Nil(err)
es, err := m.ListRetentionExecs(policyID, nil)
es, err := m.ListRetentionExecs(ctx, policyID, nil)
s.Require().Nil(err)
s.Require().EqualValues(1, len(es))
ts, err := m.ListRetentionExecTasks(id, nil)
ts, err := m.ListRetentionExecTasks(nil, id, nil)
s.Require().Nil(err)
s.Require().EqualValues(1, len(ts))
@ -301,10 +329,10 @@ func (f *fakeRetentionScheduler) ListSchedules(ctx context.Context, q *q.Query)
type fakeLauncher struct {
}
func (f *fakeLauncher) Stop(executionID int64) error {
func (f *fakeLauncher) Stop(ctx context.Context, executionID int64) error {
return nil
}
func (f *fakeLauncher) Launch(policy *policy.Metadata, executionID int64, isDryRun bool) (int64, error) {
func (f *fakeLauncher) Launch(ctx context.Context, policy *policy.Metadata, executionID int64, isDryRun bool) (int64, error) {
return 0, nil
}

43
src/core/api/base.go
View File

@ -18,7 +18,6 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/pkg/task"
"net/http"
"github.com/ghodss/yaml"
@ -32,9 +31,6 @@ import (
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/retention"
"github.com/goharbor/harbor/src/pkg/scheduler"
)
@ -43,17 +39,6 @@ const (
userSessionKey = "user"
)
// the managers/controllers used globally
var (
projectMgr project.Manager
retentionController retention.APIController
)
// GetRetentionController returns the retention API controller
func GetRetentionController() retention.APIController {
return retentionController
}
// BaseController ...
type BaseController struct {
api.BaseAPI
@ -182,28 +167,6 @@ func Init() error {
return err
}
// init project manager
initProjectManager()
retentionMgr := retention.NewManager()
retentionLauncher := retention.NewLauncher(projectMgr, repository.Mgr, retentionMgr, task.ExecMgr, task.Mgr)
retentionController = retention.NewAPIController(retentionMgr, projectMgr, repository.Mgr, scheduler.Sched, retentionLauncher, task.ExecMgr, task.Mgr)
retentionCallbackFun := func(ctx context.Context, p string) error {
param := &retention.TriggerParam{}
if err := json.Unmarshal([]byte(p), param); err != nil {
return fmt.Errorf("failed to unmarshal the param: %v", err)
}
_, err := retentionController.TriggerRetentionExec(param.PolicyID, param.Trigger, false)
return err
}
err := scheduler.RegisterCallbackFunc(retention.SchedulerCallback, retentionCallbackFun)
if err != nil {
return err
}
p2pPreheatCallbackFun := func(ctx context.Context, p string) error {
param := &preheat.TriggerParam{}
if err := json.Unmarshal([]byte(p), param); err != nil {
@ -212,7 +175,7 @@ func Init() error {
_, err := preheat.Enf.EnforcePolicy(ctx, param.PolicyID)
return err
}
err = scheduler.RegisterCallbackFunc(preheat.SchedulerCallback, p2pPreheatCallbackFun)
err := scheduler.RegisterCallbackFunc(preheat.SchedulerCallback, p2pPreheatCallbackFun)
return err
}
@ -231,7 +194,3 @@ func initChartController() error {
chartController = chartCtl
return nil
}
func initProjectManager() {
projectMgr = project.Mgr
}

10
src/core/api/harborapi_test.go
View File

@ -129,16 +129,6 @@ func init() {
beego.Router("/api/replication/policies", &ReplicationPolicyAPI{}, "get:List;post:Create")
beego.Router("/api/replication/policies/:id([0-9]+)", &ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete")
beego.Router("/api/retentions/metadatas", &RetentionAPI{}, "get:GetMetadatas")
beego.Router("/api/retentions/:id", &RetentionAPI{}, "get:GetRetention")
beego.Router("/api/retentions", &RetentionAPI{}, "post:CreateRetention")
beego.Router("/api/retentions/:id", &RetentionAPI{}, "put:UpdateRetention")
beego.Router("/api/retentions/:id/executions", &RetentionAPI{}, "post:TriggerRetentionExec")
beego.Router("/api/retentions/:id/executions/:eid", &RetentionAPI{}, "patch:OperateRetentionExec")
beego.Router("/api/retentions/:id/executions", &RetentionAPI{}, "get:ListRetentionExecs")
beego.Router("/api/retentions/:id/executions/:eid/tasks", &RetentionAPI{}, "get:ListRetentionExecTasks")
beego.Router("/api/retentions/:id/executions/:eid/tasks/:tid", &RetentionAPI{}, "get:GetRetentionExecTaskLog")
beego.Router("/api/projects/:pid([0-9]+)/webhook/policies", &NotificationPolicyAPI{}, "get:List;post:Post")
beego.Router("/api/projects/:pid([0-9]+)/webhook/policies/:id([0-9]+)", &NotificationPolicyAPI{})
beego.Router("/api/projects/:pid([0-9]+)/webhook/policies/test", &NotificationPolicyAPI{}, "post:Test")

453
src/core/api/retention.go
View File

@ -1,453 +0,0 @@
package api
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/common/rbac/system"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/task"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/project/metadata"
"github.com/goharbor/harbor/src/pkg/retention/policy"
)
// RetentionAPI ...
type RetentionAPI struct {
BaseController
metaMgr metadata.Manager
}
// Prepare validates the user
func (r *RetentionAPI) Prepare() {
r.BaseController.Prepare()
if !r.SecurityCtx.IsAuthenticated() {
r.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
r.metaMgr = metadata.Mgr
}
// GetMetadatas Get Metadatas
func (r *RetentionAPI) GetMetadatas() {
data := `
{
"templates": [
{
"rule_template": "latestPushedK",
"display_text": "the most recently pushed # artifacts",
"action": "retain",
"params": [
{
"type": "int",
"unit": "COUNT",
"required": true
}
]
},
{
"rule_template": "latestPulledN",
"display_text": "the most recently pulled # artifacts",
"action": "retain",
"params": [
{
"type": "int",
"unit": "COUNT",
"required": true
}
]
},
{
"rule_template": "nDaysSinceLastPush",
"display_text": "pushed within the last # days",
"action": "retain",
"params": [
{
"type": "int",
"unit": "DAYS",
"required": true
}
]
},
{
"rule_template": "nDaysSinceLastPull",
"display_text": "pulled within the last # days",
"action": "retain",
"params": [
{
"type": "int",
"unit": "DAYS",
"required": true
}
]
},
{
"rule_template": "always",
"display_text": "always",
"action": "retain",
"params": []
}
],
"scope_selectors": [
{
"display_text": "Repositories",
"kind": "doublestar",
"decorations": [
"repoMatches",
"repoExcludes"
]
}
],
"tag_selectors": [
{
"display_text": "Tags",
"kind": "doublestar",
"decorations": [
"matches",
"excludes"
]
}
]
}
`
w := r.Ctx.ResponseWriter
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write([]byte(data))
}
// GetRetention Get Retention
func (r *RetentionAPI) GetRetention() {
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
p, err := retentionController.GetRetention(id)
if err != nil {
r.SendBadRequestError(err)
return
}
if !r.requireAccess(p, rbac.ActionRead) {
return
}
r.WriteJSONData(p)
}
// CreateRetention Create Retention
func (r *RetentionAPI) CreateRetention() {
p := &policy.Metadata{}
isValid, err := r.DecodeJSONReqAndValidate(p)
if !isValid {
r.SendBadRequestError(err)
return
}
if len(p.Rules) > 15 {
r.SendBadRequestError(errors.New("only 15 rules are allowed at most"))
return
}
if err = r.checkRuleConflict(p); err != nil {
r.SendConflictError(err)
return
}
if !r.requireAccess(p, rbac.ActionCreate) {
return
}
switch p.Scope.Level {
case policy.ScopeLevelProject:
if p.Scope.Reference <= 0 {
r.SendBadRequestError(fmt.Errorf("invalid Project id %d", p.Scope.Reference))
return
}
if _, err := r.ProjectCtl.Get(r.Context(), p.Scope.Reference); err != nil {
if errors.IsNotFoundErr(err) {
r.SendBadRequestError(fmt.Errorf("invalid Project id %d", p.Scope.Reference))
} else {
r.SendBadRequestError(err)
}
return
}
default:
r.SendBadRequestError(fmt.Errorf("scope %s is not support", p.Scope.Level))
return
}
old, err := r.metaMgr.Get(r.Context(), p.Scope.Reference, "retention_id")
if err != nil {
r.SendInternalServerError(err)
return
}
if old != nil && len(old) > 0 {
r.SendBadRequestError(fmt.Errorf("project %v already has retention policy %v", p.Scope.Reference, old["retention_id"]))
return
}
id, err := retentionController.CreateRetention(p)
if err != nil {
r.SendInternalServerError(err)
return
}
if err := r.metaMgr.Add(r.Context(), p.Scope.Reference,
map[string]string{"retention_id": strconv.FormatInt(id, 10)}); err != nil {
r.SendInternalServerError(err)
}
r.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}
// UpdateRetention Update Retention
func (r *RetentionAPI) UpdateRetention() {
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
p := &policy.Metadata{}
isValid, err := r.DecodeJSONReqAndValidate(p)
if !isValid {
r.SendBadRequestError(err)
return
}
p.ID = id
if len(p.Rules) > 15 {
r.SendBadRequestError(errors.New("only 15 rules are allowed at most"))
return
}
if err = r.checkRuleConflict(p); err != nil {
r.SendConflictError(err)
return
}
if !r.requireAccess(p, rbac.ActionUpdate) {
return
}
if err = retentionController.UpdateRetention(p); err != nil {
r.SendInternalServerError(err)
return
}
}
func (r *RetentionAPI) checkRuleConflict(p *policy.Metadata) error {
temp := make(map[string]int)
for n, rule := range p.Rules {
rule.ID = 0
bs, _ := json.Marshal(rule)
if old, exists := temp[string(bs)]; exists {
return fmt.Errorf("rule %d is conflict with rule %d", n, old)
}
temp[string(bs)] = n
rule.ID = n
}
return nil
}
// TriggerRetentionExec Trigger Retention Execution
func (r *RetentionAPI) TriggerRetentionExec() {
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
d := &struct {
DryRun bool `json:"dry_run"`
}{
DryRun: false,
}
isValid, err := r.DecodeJSONReqAndValidate(d)
if !isValid {
r.SendBadRequestError(err)
return
}
p, err := retentionController.GetRetention(id)
if err != nil {
r.SendBadRequestError(err)
return
}
if !r.requireAccess(p, rbac.ActionUpdate) {
return
}
eid, err := retentionController.TriggerRetentionExec(id, task.ExecutionTriggerManual, d.DryRun)
if err != nil {
r.SendInternalServerError(err)
return
}
r.Redirect(http.StatusCreated, strconv.FormatInt(eid, 10))
}
// OperateRetentionExec Operate Retention Execution
func (r *RetentionAPI) OperateRetentionExec() {
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
eid, err := r.GetInt64FromPath(":eid")
if err != nil {
r.SendBadRequestError(err)
return
}
a := &struct {
Action string `json:"action" valid:"Required;Match(stop)"`
}{}
isValid, err := r.DecodeJSONReqAndValidate(a)
if !isValid {
r.SendBadRequestError(err)
return
}
p, err := retentionController.GetRetention(id)
if err != nil {
r.SendBadRequestError(err)
return
}
if !r.requireAccess(p, rbac.ActionUpdate) {
return
}
if err = retentionController.OperateRetentionExec(eid, a.Action); err != nil {
r.SendInternalServerError(err)
return
}
}
// ListRetentionExecs List Retention Execution
func (r *RetentionAPI) ListRetentionExecs() {
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
page, size, err := r.GetPaginationParams()
if err != nil {
r.SendInternalServerError(err)
return
}
query := &q.Query{
PageNumber: page,
PageSize: size,
}
p, err := retentionController.GetRetention(id)
if err != nil {
r.SendBadRequestError(err)
return
}
if !r.requireAccess(p, rbac.ActionList) {
return
}
execs, err := retentionController.ListRetentionExecs(id, query)
if err != nil {
r.SendInternalServerError(err)
return
}
total, err := retentionController.GetTotalOfRetentionExecs(id)
if err != nil {
r.SendInternalServerError(err)
return
}
r.SetPaginationHeader(total, query.PageNumber, query.PageSize)
r.WriteJSONData(execs)
}
// ListRetentionExecTasks List Retention Execution Tasks
func (r *RetentionAPI) ListRetentionExecTasks() {
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
eid, err := r.GetInt64FromPath(":eid")
if err != nil {
r.SendBadRequestError(err)
return
}
page, size, err := r.GetPaginationParams()
if err != nil {
r.SendInternalServerError(err)
return
}
query := &q.Query{
PageNumber: page,
PageSize: size,
Keywords: map[string]interface{}{
"VendorID": id,
"VendorType": job.Retention,
},
}
p, err := retentionController.GetRetention(id)
if err != nil {
r.SendBadRequestError(err)
return
}
if !r.requireAccess(p, rbac.ActionList) {
return
}
his, err := retentionController.ListRetentionExecTasks(eid, query)
if err != nil {
r.SendInternalServerError(err)
return
}
total, err := retentionController.GetTotalOfRetentionExecTasks(eid)
if err != nil {
r.SendInternalServerError(err)
return
}
r.SetPaginationHeader(total, query.PageNumber, query.PageSize)
r.WriteJSONData(his)
}
// GetRetentionExecTaskLog Get Retention Execution Task log
func (r *RetentionAPI) GetRetentionExecTaskLog() {
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
tid, err := r.GetInt64FromPath(":tid")
if err != nil {
r.SendBadRequestError(err)
return
}
p, err := retentionController.GetRetention(id)
if err != nil {
r.SendBadRequestError(err)
return
}
if !r.requireAccess(p, rbac.ActionRead) {
return
}
log, err := retentionController.GetRetentionExecTaskLog(tid)
if err != nil {
r.SendInternalServerError(err)
return
}
w := r.Ctx.ResponseWriter
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
w.Write(log)
}
func (r *RetentionAPI) requireAccess(p *policy.Metadata, action rbac.Action, subresources ...rbac.Resource) bool {
var hasPermission bool
switch p.Scope.Level {
case "project":
if len(subresources) == 0 {
subresources = append(subresources, rbac.ResourceTagRetention)
}
hasPermission, _ = r.HasProjectPermission(p.Scope.Reference, action, subresources...)
default:
resource := system.NewNamespace().Resource(rbac.ResourceTagRetention)
hasPermission = r.SecurityCtx.Can(r.Context(), action, resource)
}
if !hasPermission {
if !r.SecurityCtx.IsAuthenticated() {
r.SendUnAuthorizedError(errors.New("UnAuthorized"))
} else {
r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
}
return false
}
return true
}

468
src/core/api/retention_test.go
View File

@ -1,468 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"encoding/json"
"fmt"
"net/http"
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/retention/dao"
"github.com/goharbor/harbor/src/pkg/retention/dao/models"
"github.com/goharbor/harbor/src/pkg/retention/mocks"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestGetMetadatas(t *testing.T) {
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/retentions/metadatas",
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestCreatePolicy(t *testing.T) {
// mock retention api controller
mockController := &mocks.APIController{}
mockController.On("CreateRetention", mock.AnythingOfType("*policy.Metadata")).Return(int64(1), nil)
controller := retentionController
retentionController = mockController
defer func() {
retentionController = controller
}()
p1 := &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "latestPushedK",
Action: "retain",
Parameters: rule.Parameters{
"latestPushedK": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
}
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/retentions",
},
code: http.StatusUnauthorized,
},
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/retentions",
bodyJSON: p1,
credential: sysAdmin,
},
code: http.StatusCreated,
},
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/retentions",
bodyJSON: &policy.Metadata{
Algorithm: "NODEF",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "latestPushedK",
Action: "retain",
Parameters: rule.Parameters{
"latestPushedK": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/retentions",
bodyJSON: &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "latestPushedK",
Action: "retain",
Parameters: rule.Parameters{
"latestPushedK": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
{
ID: 2,
Priority: 1,
Template: "latestPushedK",
Action: "retain",
Parameters: rule.Parameters{
"latestPushedK": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
},
credential: sysAdmin,
},
code: http.StatusConflict,
},
}
runCodeCheckingCases(t, cases...)
}
func TestPolicy(t *testing.T) {
p := &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "latestPushedK",
Action: "retain",
Parameters: rule.Parameters{
"latestPushedK": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
}
p1 := &models.RetentionPolicy{
ScopeLevel: p.Scope.Level,
TriggerKind: p.Trigger.Kind,
CreateTime: time.Now(),
UpdateTime: time.Now(),
}
data, _ := json.Marshal(p)
p1.Data = string(data)
id, err := dao.CreatePolicy(p1)
require.Nil(t, err)
require.True(t, id > 0)
// mock retention api controller
mockController := &mocks.APIController{}
mockController.On("GetRetention", mock.AnythingOfType("int64")).Return(p, nil)
mockController.On("UpdateRetention", mock.AnythingOfType("*policy.Metadata")).Return(nil)
mockController.On("TriggerRetentionExec",
mock.AnythingOfType("int64"),
mock.AnythingOfType("string"),
mock.AnythingOfType("bool")).Return(int64(1), nil)
mockController.On("ListRetentionExecs", mock.AnythingOfType("int64"), mock.AnythingOfType("*q.Query")).Return(nil, nil)
mockController.On("GetTotalOfRetentionExecs", mock.AnythingOfType("int64")).Return(int64(0), nil)
controller := retentionController
retentionController = mockController
defer func() {
retentionController = controller
}()
cases := []*codeCheckingCase{
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("/api/retentions/%d", id),
credential: sysAdmin,
},
code: http.StatusOK,
},
{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("/api/retentions/%d", id),
bodyJSON: &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "latestPushedK",
Action: "retain",
Parameters: rule.Parameters{
"latestPushedK": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "b.+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
},
credential: sysAdmin,
},
code: http.StatusOK,
},
{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("/api/retentions/%d", id),
bodyJSON: &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "latestPushedK",
Action: "retain",
Parameters: rule.Parameters{
"latestPushedK": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "b.+",
},
},
},
},
{
ID: 2,
Priority: 1,
Template: "latestPushedK",
Action: "retain",
Parameters: rule.Parameters{
"latestPushedK": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "b.+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
},
credential: sysAdmin,
},
code: http.StatusConflict,
},
{
request: &testingRequest{
method: http.MethodPost,
url: fmt.Sprintf("/api/retentions/%d/executions", id),
bodyJSON: &struct {
DryRun bool `json:"dry_run"`
}{
DryRun: false,
},
credential: sysAdmin,
},
code: http.StatusCreated,
},
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("/api/retentions/%d/executions", id),
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}

96
src/pkg/retention/faked_controller.go
View File

@ -1,96 +0,0 @@
package retention
import (
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/retention/policy"
)
// FakedRetentionController ...
type FakedRetentionController struct {
}
// FakedRetentionControllerFunc ...
var FakedRetentionControllerFunc = NewFakedRetentionController
// NewFakedRetentionController ...
func NewFakedRetentionController() APIController {
return &FakedRetentionController{}
}
// GetRetention ...
func (f *FakedRetentionController) GetRetention(id int64) (*policy.Metadata, error) {
return &policy.Metadata{
ID: 1,
Algorithm: "",
Rules: nil,
Trigger: nil,
Scope: nil,
}, nil
}
// CreateRetention ...
func (f *FakedRetentionController) CreateRetention(p *policy.Metadata) (int64, error) {
return 0, nil
}
// UpdateRetention ...
func (f *FakedRetentionController) UpdateRetention(p *policy.Metadata) error {
return nil
}
// DeleteRetention ...
func (f *FakedRetentionController) DeleteRetention(id int64) error {
return nil
}
// TriggerRetentionExec ...
func (f *FakedRetentionController) TriggerRetentionExec(policyID int64, trigger string, dryRun bool) (int64, error) {
return 0, nil
}
// OperateRetentionExec ...
func (f *FakedRetentionController) OperateRetentionExec(eid int64, action string) error {
return nil
}
// GetRetentionExec ...
func (f *FakedRetentionController) GetRetentionExec(eid int64) (*Execution, error) {
return &Execution{
DryRun: false,
PolicyID: 1,
}, nil
}
// ListRetentionExecs ...
func (f *FakedRetentionController) ListRetentionExecs(policyID int64, query *q.Query) ([]*Execution, error) {
return nil, nil
}
// GetTotalOfRetentionExecs ...
func (f *FakedRetentionController) GetTotalOfRetentionExecs(policyID int64) (int64, error) {
return 0, nil
}
// ListRetentionExecTasks ...
func (f *FakedRetentionController) ListRetentionExecTasks(executionID int64, query *q.Query) ([]*Task, error) {
return nil, nil
}
// GetTotalOfRetentionExecTasks ...
func (f *FakedRetentionController) GetTotalOfRetentionExecTasks(executionID int64) (int64, error) {
return 0, nil
}
// GetRetentionExecTaskLog ...
func (f *FakedRetentionController) GetRetentionExecTaskLog(taskID int64) ([]byte, error) {
return nil, nil
}
// GetRetentionExecTask ...
func (f *FakedRetentionController) GetRetentionExecTask(taskID int64) (*Task, error) {
return &Task{
ID: 1,
ExecutionID: 1,
}, nil
}

77
src/pkg/retention/launcher.go
View File

@ -15,9 +15,8 @@
package retention
import (
"context"
"fmt"
beegoorm "github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/pkg/task"
@ -26,7 +25,6 @@ import (
cjob "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
pq "github.com/goharbor/harbor/src/lib/q"
@ -58,7 +56,7 @@ type Launcher interface {
// Returns:
// int64 : the count of tasks
// error : common error if any errors occurred
Launch(policy *policy.Metadata, executionID int64, isDryRun bool) (int64, error)
Launch(ctx context.Context, policy *policy.Metadata, executionID int64, isDryRun bool) (int64, error)
// Stop the jobs for one execution
//
// Arguments:
@ -66,21 +64,19 @@ type Launcher interface {
//
// Returns:
// error : common error if any errors occurred
Stop(executionID int64) error
Stop(ctx context.Context, executionID int64) error
}
// NewLauncher returns an instance of Launcher
func NewLauncher(projectMgr project.Manager, repositoryMgr repository.Manager,
retentionMgr Manager, execMgr task.ExecutionManager, taskMgr task.Manager) Launcher {
return &launcher{
projectMgr: projectMgr,
repositoryMgr: repositoryMgr,
retentionMgr: retentionMgr,
execMgr: execMgr,
taskMgr: taskMgr,
jobserviceClient: cjob.GlobalClient,
internalCoreURL: config.InternalCoreURL(),
chartServerEnabled: config.WithChartMuseum(),
projectMgr: projectMgr,
repositoryMgr: repositoryMgr,
retentionMgr: retentionMgr,
execMgr: execMgr,
taskMgr: taskMgr,
jobserviceClient: cjob.GlobalClient,
}
}
@ -92,17 +88,15 @@ type jobData struct {
}
type launcher struct {
retentionMgr Manager
taskMgr task.Manager
execMgr task.ExecutionManager
projectMgr project.Manager
repositoryMgr repository.Manager
jobserviceClient cjob.Client
internalCoreURL string
chartServerEnabled bool
retentionMgr Manager
taskMgr task.Manager
execMgr task.ExecutionManager
projectMgr project.Manager
repositoryMgr repository.Manager
jobserviceClient cjob.Client
}
func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool) (int64, error) {
func (l *launcher) Launch(ctx context.Context, ply *policy.Metadata, executionID int64, isDryRun bool) (int64, error) {
if ply == nil {
return 0, launcherError(fmt.Errorf("the policy is nil"))
}
@ -121,7 +115,7 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool
var err error
if level == "system" {
// get projects
allProjects, err = getProjects(l.projectMgr)
allProjects, err = getProjects(ctx, l.projectMgr)
if err != nil {
return 0, launcherError(err)
}
@ -156,7 +150,7 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool
var repositoryCandidates []*selector.Candidate
// get repositories of projects
for _, projectCandidate := range projectCandidates {
repositories, err := getRepositories(l.projectMgr, l.repositoryMgr, projectCandidate.NamespaceID, l.chartServerEnabled)
repositories, err := getRepositories(ctx, l.projectMgr, l.repositoryMgr, projectCandidate.NamespaceID)
if err != nil {
return 0, launcherError(err)
}
@ -207,7 +201,7 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool
}
// submit tasks to jobservice
if err = l.submitTasks(executionID, jobDatas); err != nil {
if err = l.submitTasks(ctx, executionID, jobDatas); err != nil {
return 0, launcherError(err)
}
@ -241,8 +235,7 @@ func createJobs(repositoryRules map[selector.Repository]*lwp.Metadata, isDryRun
return jobDatas, nil
}
func (l *launcher) submitTasks(executionID int64, jobDatas []*jobData) error {
ctx := orm.Context()
func (l *launcher) submitTasks(ctx context.Context, executionID int64, jobDatas []*jobData) error {
for _, jobData := range jobDatas {
_, err := l.taskMgr.Create(ctx, executionID, &task.Job{
Name: jobData.JobName,
@ -262,11 +255,10 @@ func (l *launcher) submitTasks(executionID int64, jobDatas []*jobData) error {
return nil
}
func (l *launcher) Stop(executionID int64) error {
func (l *launcher) Stop(ctx context.Context, executionID int64) error {
if executionID <= 0 {
return launcherError(fmt.Errorf("invalid execution ID: %d", executionID))
}
ctx := orm.Context()
return l.execMgr.Stop(ctx, executionID)
}
@ -274,8 +266,8 @@ func launcherError(err error) error {
return errors.Wrap(err, "launcher")
}
func getProjects(projectMgr project.Manager) ([]*selector.Candidate, error) {
projects, err := projectMgr.List(orm.Context(), nil)
func getProjects(ctx context.Context, projectMgr project.Manager) ([]*selector.Candidate, error) {
projects, err := projectMgr.List(ctx, nil)
if err != nil {
return nil, err
}
@ -289,8 +281,7 @@ func getProjects(projectMgr project.Manager) ([]*selector.Candidate, error) {
return candidates, nil
}
func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manager,
projectID int64, chartServerEnabled bool) ([]*selector.Candidate, error) {
func getRepositories(ctx context.Context, projectMgr project.Manager, repositoryMgr repository.Manager, projectID int64) ([]*selector.Candidate, error) {
var candidates []*selector.Candidate
/*
pro, err := projectMgr.Get(projectID)
@ -299,8 +290,7 @@ func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manage
}
*/
// get image repositories
// TODO set the context which contains the ORM
imageRepositories, err := repositoryMgr.List(orm.NewContext(nil, beegoorm.NewOrm()), &pq.Query{
imageRepositories, err := repositoryMgr.List(ctx, &pq.Query{
Keywords: map[string]interface{}{
"ProjectID": projectID,
},
@ -317,23 +307,6 @@ func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manage
Kind: "image",
})
}
// currently, doesn't support retention for chart
/*
if chartServerEnabled {
// get chart repositories when chart server is enabled
chartRepositories, err := repositoryMgr.ListChartRepositories(projectID)
if err != nil {
return nil, err
}
for _, r := range chartRepositories {
candidates = append(candidates, &art.Candidate{
Namespace: pro.Name,
Repository: r.Name,
Kind: "chart",
})
}
}
*/
return candidates, nil
}

34
src/pkg/retention/launcher_test.go
View File

@ -15,6 +15,7 @@
package retention
import (
"github.com/goharbor/harbor/src/lib/orm"
"testing"
"github.com/goharbor/harbor/src/common/job"
@ -136,7 +137,8 @@ func (l *launchTestSuite) SetupTest() {
}
func (l *launchTestSuite) TestGetProjects() {
projects, err := getProjects(l.projectMgr)
ctx := orm.Context()
projects, err := getProjects(ctx, l.projectMgr)
require.Nil(l.T(), err)
assert.Equal(l.T(), 2, len(projects))
assert.Equal(l.T(), int64(1), projects[0].NamespaceID)
@ -151,7 +153,8 @@ func (l *launchTestSuite) TestGetRepositories() {
Name: "library/image",
},
}, nil)
repositories, err := getRepositories(l.projectMgr, l.repositoryMgr, 1, true)
ctx := orm.Context()
repositories, err := getRepositories(ctx, l.projectMgr, l.repositoryMgr, 1)
require.Nil(l.T(), err)
l.repositoryMgr.AssertExpectations(l.T())
assert.Equal(l.T(), 1, len(repositories))
@ -166,23 +169,23 @@ func (l *launchTestSuite) TestLaunch() {
l.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
launcher := &launcher{
projectMgr: l.projectMgr,
repositoryMgr: l.repositoryMgr,
retentionMgr: l.retentionMgr,
execMgr: l.execMgr,
taskMgr: l.taskMgr,
jobserviceClient: l.jobserviceClient,
chartServerEnabled: true,
projectMgr: l.projectMgr,
repositoryMgr: l.repositoryMgr,
retentionMgr: l.retentionMgr,
execMgr: l.execMgr,
taskMgr: l.taskMgr,
jobserviceClient: l.jobserviceClient,
}
ctx := orm.Context()
var ply *policy.Metadata
// nil policy
n, err := launcher.Launch(ply, 1, false)
n, err := launcher.Launch(ctx, ply, 1, false)
require.NotNil(l.T(), err)
// nil rules
ply = &policy.Metadata{}
n, err = launcher.Launch(ply, 1, false)
n, err = launcher.Launch(ctx, ply, 1, false)
require.Nil(l.T(), err)
assert.Equal(l.T(), int64(0), n)
@ -192,7 +195,7 @@ func (l *launchTestSuite) TestLaunch() {
{},
},
}
_, err = launcher.Launch(ply, 1, false)
_, err = launcher.Launch(ctx, ply, 1, false)
require.NotNil(l.T(), err)
// system scope
@ -247,7 +250,7 @@ func (l *launchTestSuite) TestLaunch() {
},
},
}
n, err = launcher.Launch(ply, 1, false)
n, err = launcher.Launch(ctx, ply, 1, false)
require.Nil(l.T(), err)
l.repositoryMgr.AssertExpectations(l.T())
assert.Equal(l.T(), int64(1), n)
@ -264,11 +267,12 @@ func (l *launchTestSuite) TestStop() {
taskMgr: l.taskMgr,
jobserviceClient: l.jobserviceClient,
}
ctx := orm.Context()
// invalid execution ID
err := launcher.Stop(0)
err := launcher.Stop(ctx, 0)
require.NotNil(t, err)
err = launcher.Stop(1)
err = launcher.Stop(ctx, 1)
require.Nil(t, err)
}

27
src/pkg/retention/mocks/api_controller.go
View File

@ -3,6 +3,7 @@
package mocks
import (
"context"
"github.com/goharbor/harbor/src/lib/q"
policy "github.com/goharbor/harbor/src/pkg/retention/policy"
mock "github.com/stretchr/testify/mock"
@ -16,7 +17,7 @@ type APIController struct {
}
// CreateRetention provides a mock function with given fields: p
func (_m *APIController) CreateRetention(p *policy.Metadata) (int64, error) {
func (_m *APIController) CreateRetention(ctx context.Context, p *policy.Metadata) (int64, error) {
ret := _m.Called(p)
var r0 int64
@ -37,7 +38,7 @@ func (_m *APIController) CreateRetention(p *policy.Metadata) (int64, error) {
}
// DeleteRetention provides a mock function with given fields: id
func (_m *APIController) DeleteRetention(id int64) error {
func (_m *APIController) DeleteRetention(ctx context.Context, id int64) error {
ret := _m.Called(id)
var r0 error
@ -51,7 +52,7 @@ func (_m *APIController) DeleteRetention(id int64) error {
}
// GetRetention provides a mock function with given fields: id
func (_m *APIController) GetRetention(id int64) (*policy.Metadata, error) {
func (_m *APIController) GetRetention(ctx context.Context, id int64) (*policy.Metadata, error) {
ret := _m.Called(id)
var r0 *policy.Metadata
@ -74,7 +75,7 @@ func (_m *APIController) GetRetention(id int64) (*policy.Metadata, error) {
}
// GetRetentionExec provides a mock function with given fields: eid
func (_m *APIController) GetRetentionExec(eid int64) (*retention.Execution, error) {
func (_m *APIController) GetRetentionExec(ctx context.Context, eid int64) (*retention.Execution, error) {
ret := _m.Called(eid)
var r0 *retention.Execution
@ -97,7 +98,7 @@ func (_m *APIController) GetRetentionExec(eid int64) (*retention.Execution, erro
}
// GetRetentionExecTaskLog provides a mock function with given fields: taskID
func (_m *APIController) GetRetentionExecTaskLog(taskID int64) ([]byte, error) {
func (_m *APIController) GetRetentionExecTaskLog(ctx context.Context, taskID int64) ([]byte, error) {
ret := _m.Called(taskID)
var r0 []byte
@ -120,7 +121,7 @@ func (_m *APIController) GetRetentionExecTaskLog(taskID int64) ([]byte, error) {
}
// GetRetentionExecTask provides a mock function with given fields: taskID
func (_m *APIController) GetRetentionExecTask(taskID int64) (*retention.Task, error) {
func (_m *APIController) GetRetentionExecTask(ctx context.Context, taskID int64) (*retention.Task, error) {
return &retention.Task{
ID: 1,
ExecutionID: 1,
@ -128,7 +129,7 @@ func (_m *APIController) GetRetentionExecTask(taskID int64) (*retention.Task, er
}
// GetTotalOfRetentionExecTasks provides a mock function with given fields: executionID
func (_m *APIController) GetTotalOfRetentionExecTasks(executionID int64) (int64, error) {
func (_m *APIController) GetTotalOfRetentionExecTasks(ctx context.Context, executionID int64) (int64, error) {
ret := _m.Called(executionID)
var r0 int64
@ -149,7 +150,7 @@ func (_m *APIController) GetTotalOfRetentionExecTasks(executionID int64) (int64,
}
// GetTotalOfRetentionExecs provides a mock function with given fields: policyID
func (_m *APIController) GetTotalOfRetentionExecs(policyID int64) (int64, error) {
func (_m *APIController) GetTotalOfRetentionExecs(ctx context.Context, policyID int64) (int64, error) {
ret := _m.Called(policyID)
var r0 int64
@ -170,7 +171,7 @@ func (_m *APIController) GetTotalOfRetentionExecs(policyID int64) (int64, error)
}
// ListRetentionExecTasks provides a mock function with given fields: executionID, query
func (_m *APIController) ListRetentionExecTasks(executionID int64, query *q.Query) ([]*retention.Task, error) {
func (_m *APIController) ListRetentionExecTasks(ctx context.Context, executionID int64, query *q.Query) ([]*retention.Task, error) {
ret := _m.Called(executionID, query)
var r0 []*retention.Task
@ -193,7 +194,7 @@ func (_m *APIController) ListRetentionExecTasks(executionID int64, query *q.Quer
}
// ListRetentionExecs provides a mock function with given fields: policyID, query
func (_m *APIController) ListRetentionExecs(policyID int64, query *q.Query) ([]*retention.Execution, error) {
func (_m *APIController) ListRetentionExecs(ctx context.Context, policyID int64, query *q.Query) ([]*retention.Execution, error) {
ret := _m.Called(policyID, query)
var r0 []*retention.Execution
@ -216,7 +217,7 @@ func (_m *APIController) ListRetentionExecs(policyID int64, query *q.Query) ([]*
}
// OperateRetentionExec provides a mock function with given fields: eid, action
func (_m *APIController) OperateRetentionExec(eid int64, action string) error {
func (_m *APIController) OperateRetentionExec(ctx context.Context, eid int64, action string) error {
ret := _m.Called(eid, action)
var r0 error
@ -230,7 +231,7 @@ func (_m *APIController) OperateRetentionExec(eid int64, action string) error {
}
// TriggerRetentionExec provides a mock function with given fields: policyID, trigger, dryRun
func (_m *APIController) TriggerRetentionExec(policyID int64, trigger string, dryRun bool) (int64, error) {
func (_m *APIController) TriggerRetentionExec(ctx context.Context, policyID int64, trigger string, dryRun bool) (int64, error) {
ret := _m.Called(policyID, trigger, dryRun)
var r0 int64
@ -251,7 +252,7 @@ func (_m *APIController) TriggerRetentionExec(policyID int64, trigger string, dr
}
// UpdateRetention provides a mock function with given fields: p
func (_m *APIController) UpdateRetention(p *policy.Metadata) error {
func (_m *APIController) UpdateRetention(ctx context.Context, p *policy.Metadata) error {
ret := _m.Called(p)
var r0 error

1
src/server/v2.0/handler/handler.go
View File

@ -42,6 +42,7 @@ func New() http.Handler {
SysteminfoAPI: newSystemInfoAPI(),
PingAPI: newPingAPI(),
GCAPI: newGCAPI(),
RetentionAPI: newRetentionAPI(),
})
if err != nil {
log.Fatal(err)

74
src/server/v2.0/handler/model/retention.go Normal file
View File

@ -0,0 +1,74 @@
package model
import (
"encoding/json"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/pkg/retention"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/server/v2.0/models"
)
// RetentionPolicy ...
type RetentionPolicy struct {
*policy.Metadata
}
// ToSwagger ...
func (s *RetentionPolicy) ToSwagger() *models.RetentionPolicy {
var result models.RetentionPolicy
lib.JSONCopy(&result, s)
return &result
}
// NewRetentionPolicyFromSwagger ...
func NewRetentionPolicyFromSwagger(policy *models.RetentionPolicy) *RetentionPolicy {
data, err := json.Marshal(policy)
if err != nil {
return nil
}
var result RetentionPolicy
err = json.Unmarshal(data, &result)
if err != nil {
return nil
}
return &result
}
// NewRetentionPolicy ...
func NewRetentionPolicy(policy *policy.Metadata) *RetentionPolicy {
return &RetentionPolicy{policy}
}
// RetentionExec ...
type RetentionExec struct {
*retention.Execution
}
// ToSwagger ...
func (e *RetentionExec) ToSwagger() *models.RetentionExecution {
var result models.RetentionExecution
lib.JSONCopy(&result, e)
return &result
}
// NewRetentionExec ...
func NewRetentionExec(exec *retention.Execution) *RetentionExec {
return &RetentionExec{exec}
}
// RetentionTask ...
type RetentionTask struct {
*retention.Task
}
// ToSwagger ...
func (e *RetentionTask) ToSwagger() *models.RetentionExecutionTask {
var result models.RetentionExecutionTask
lib.JSONCopy(&result, e)
return &result
}
// NewRetentionTask ...
func NewRetentionTask(task *retention.Task) *RetentionTask {
return &RetentionTask{task}
}

7
src/server/v2.0/handler/project.go
View File

@ -3,6 +3,7 @@ package handler
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/controller/retention"
"strconv"
"strings"
"sync"
@ -50,6 +51,7 @@ func newProjectAPI() *projectAPI {
quotaCtl: quota.Ctl,
robotMgr: robot.Mgr,
preheatCtl: preheat.Ctl,
retentionCtl: retention.Ctl,
}
}
@ -63,6 +65,7 @@ type projectAPI struct {
quotaCtl quota.Controller
robotMgr robot.Manager
preheatCtl preheat.Controller
retentionCtl retention.Controller
}
func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateProjectParams) middleware.Responder {
@ -170,9 +173,7 @@ func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateP
// create a default retention policy for proxy project
if req.RegistryID != nil {
plc := policy.WithNDaysSinceLastPull(projectID, defaultDaysToRetentionForProxyCacheProject)
// TODO: move the retention controller to `src/controller/retention` and
// change to use the default retention controller in `src/controller/retention`
retentionID, err := api.GetRetentionController().CreateRetention(plc)
retentionID, err := a.retentionCtl.CreateRetention(ctx, plc)
if err != nil {
return a.SendError(ctx, err)
}

356
src/server/v2.0/handler/retention.go Normal file
View File

@ -0,0 +1,356 @@
package handler
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/common/rbac"
projectCtl "github.com/goharbor/harbor/src/controller/project"
retentionCtl "github.com/goharbor/harbor/src/controller/retention"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/project/metadata"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/retention"
)
func newRetentionAPI() *retentionAPI {
return &retentionAPI{
projectCtl: projectCtl.Ctl,
retentionCtl: retentionCtl.Ctl,
proMetaMgr: metadata.Mgr,
}
}
// RetentionAPI ...
type retentionAPI struct {
BaseAPI
proMetaMgr metadata.Manager
retentionCtl retentionCtl.Controller
projectCtl projectCtl.Controller
}
var (
rentenitionMetadataPayload = &models.RetentionMetadata{
Templates: []*models.RetentionRuleMetadata{
{
Action: "retain",
DisplayText: "the most recently pushed # artifacts",
RuleTemplate: "latestPushedK",
Params: []*models.RetentionRuleParamMetadata{
{
Required: true,
Type: "int",
Unit: "COUNT",
},
},
},
{
RuleTemplate: "latestPulledN",
DisplayText: "the most recently pulled # artifacts",
Action: "retain",
Params: []*models.RetentionRuleParamMetadata{
{
Type: "int",
Unit: "COUNT",
Required: true,
},
},
},
{
RuleTemplate: "nDaysSinceLastPush",
DisplayText: "pushed within the last # days",
Action: "retain",
Params: []*models.RetentionRuleParamMetadata{
{
Type: "int",
Unit: "DAYS",
Required: true,
},
},
},
{
RuleTemplate: "nDaysSinceLastPull",
DisplayText: "pulled within the last # days",
Action: "retain",
Params: []*models.RetentionRuleParamMetadata{
{
Type: "int",
Unit: "DAYS",
Required: true,
},
},
},
{
RuleTemplate: "always",
DisplayText: "always",
Action: "retain",
Params: []*models.RetentionRuleParamMetadata{},
},
},
ScopeSelectors: []*models.RetentionSelectorMetadata{
{
DisplayText: "Repositories",
Kind: "doublestar",
Decorations: []string{
"repoMatches",
"repoExcludes",
},
},
},
TagSelectors: []*models.RetentionSelectorMetadata{
{
DisplayText: "Tags",
Kind: "doublestar",
Decorations: []string{
"matches",
"excludes",
},
},
},
}
)
func (r *retentionAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder {
if err := r.RequireAuthenticated(ctx); err != nil {
return r.SendError(ctx, err)
}
return nil
}
func (r *retentionAPI) GetRentenitionMetadata(ctx context.Context, params operation.GetRentenitionMetadataParams) middleware.Responder {
return operation.NewGetRentenitionMetadataOK().WithPayload(rentenitionMetadataPayload)
}
func (r *retentionAPI) GetRetention(ctx context.Context, params operation.GetRetentionParams) middleware.Responder {
id := params.ID
p, err := r.retentionCtl.GetRetention(ctx, id)
if err != nil {
return r.SendError(ctx, err)
}
err = r.requireAccess(ctx, p, rbac.ActionRead)
if err != nil {
return r.SendError(ctx, err)
}
return operation.NewGetRetentionOK().WithPayload(model.NewRetentionPolicy(p).ToSwagger())
}
func (r *retentionAPI) CreateRetention(ctx context.Context, params operation.CreateRetentionParams) middleware.Responder {
p := model.NewRetentionPolicyFromSwagger(params.Policy).Metadata
if len(p.Rules) > 15 {
return r.SendError(ctx, errors.BadRequestError(fmt.Errorf("only 15 rules are allowed at most")))
}
if err := r.checkRuleConflict(p); err != nil {
return r.SendError(ctx, errors.ConflictError(err))
}
err := r.requireAccess(ctx, p, rbac.ActionCreate)
if err != nil {
return r.SendError(ctx, err)
}
switch p.Scope.Level {
case policy.ScopeLevelProject:
if p.Scope.Reference <= 0 {
return r.SendError(ctx, errors.BadRequestError(fmt.Errorf("invalid Project id %d", p.Scope.Reference)))
}
if _, err := r.projectCtl.Get(ctx, p.Scope.Reference); err != nil {
if errors.IsNotFoundErr(err) {
return r.SendError(ctx, errors.BadRequestError(fmt.Errorf("invalid Project id %d", p.Scope.Reference)))
}
return r.SendError(ctx, errors.BadRequestError(err))
}
default:
return r.SendError(ctx, errors.BadRequestError(fmt.Errorf("scope %s is not support", p.Scope.Level)))
}
old, err := r.proMetaMgr.Get(ctx, p.Scope.Reference, "retention_id")
if err != nil {
return r.SendError(ctx, err)
}
if old != nil && len(old) > 0 {
return r.SendError(ctx, errors.BadRequestError(fmt.Errorf("project %v already has retention policy %v", p.Scope.Reference, old["retention_id"])))
}
id, err := r.retentionCtl.CreateRetention(ctx, p)
if err != nil {
return r.SendError(ctx, err)
}
if err := r.proMetaMgr.Add(ctx, p.Scope.Reference,
map[string]string{"retention_id": strconv.FormatInt(id, 10)}); err != nil {
return r.SendError(ctx, err)
}
location := fmt.Sprintf("%s/%d", strings.TrimSuffix(params.HTTPRequest.URL.Path, "/"), id)
return operation.NewCreateRetentionCreated().WithLocation(location)
}
func (r *retentionAPI) UpdateRetention(ctx context.Context, params operation.UpdateRetentionParams) middleware.Responder {
p := model.NewRetentionPolicyFromSwagger(params.Policy).Metadata
p.ID = params.ID
if len(p.Rules) > 15 {
return r.SendError(ctx, errors.BadRequestError(fmt.Errorf("only 15 rules are allowed at most")))
}
if err := r.checkRuleConflict(p); err != nil {
return r.SendError(ctx, errors.ConflictError(err))
}
err := r.requireAccess(ctx, p, rbac.ActionUpdate)
if err != nil {
return r.SendError(ctx, err)
}
if err = r.retentionCtl.UpdateRetention(ctx, p); err != nil {
return r.SendError(ctx, err)
}
return operation.NewUpdateRetentionOK()
}
func (r *retentionAPI) checkRuleConflict(p *policy.Metadata) error {
temp := make(map[string]int)
for n, rule := range p.Rules {
rule.ID = 0
bs, _ := json.Marshal(rule)
if old, exists := temp[string(bs)]; exists {
return fmt.Errorf("rule %d is conflict with rule %d", n, old)
}
temp[string(bs)] = n
rule.ID = n
}
return nil
}
func (r *retentionAPI) TriggerRetentionJob(ctx context.Context, params operation.TriggerRetentionJobParams) middleware.Responder {
p, err := r.retentionCtl.GetRetention(ctx, params.ID)
if err != nil {
return r.SendError(ctx, errors.BadRequestError((err)))
}
err = r.requireAccess(ctx, p, rbac.ActionUpdate)
if err != nil {
return r.SendError(ctx, err)
}
eid, err := r.retentionCtl.TriggerRetentionExec(ctx, params.ID, task.ExecutionTriggerManual, params.Body.DryRun)
if err != nil {
return r.SendError(ctx, err)
}
location := fmt.Sprintf("%s/%d", strings.TrimSuffix(params.HTTPRequest.URL.Path, "/"), eid)
return operation.NewTriggerRetentionJobCreated().WithLocation(location)
}
func (r *retentionAPI) OperateRetentionJob(ctx context.Context, params operation.OperateRetentionJobParams) middleware.Responder {
if params.Body.Action != "stop" {
return r.SendError(ctx, errors.BadRequestError((fmt.Errorf("action should be 'stop'"))))
}
p, err := r.retentionCtl.GetRetention(ctx, params.ID)
if err != nil {
return r.SendError(ctx, errors.BadRequestError(err))
}
err = r.requireAccess(ctx, p, rbac.ActionUpdate)
if err != nil {
return r.SendError(ctx, err)
}
if err = r.retentionCtl.OperateRetentionExec(ctx, params.Eid, params.Body.Action); err != nil {
return r.SendError(ctx, err)
}
return operation.NewOperateRetentionJobOK()
}
func (r *retentionAPI) ListRetentionJob(ctx context.Context, params operation.ListRetentionJobParams) middleware.Responder {
query := &q.Query{
PageNumber: *params.Page,
PageSize: *params.PageSize,
}
p, err := r.retentionCtl.GetRetention(ctx, params.ID)
if err != nil {
return r.SendError(ctx, errors.BadRequestError(err))
}
err = r.requireAccess(ctx, p, rbac.ActionList)
if err != nil {
return r.SendError(ctx, err)
}
execs, err := r.retentionCtl.ListRetentionExecs(ctx, params.ID, query)
if err != nil {
return r.SendError(ctx, err)
}
total, err := r.retentionCtl.GetTotalOfRetentionExecs(ctx, params.ID)
if err != nil {
return r.SendError(ctx, err)
}
var payload []*models.RetentionExecution
for _, e := range execs {
payload = append(payload, model.NewRetentionExec(e).ToSwagger())
}
return operation.NewListRetentionJobOK().WithXTotalCount(total).
WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
WithPayload(payload)
}
func (r *retentionAPI) ListRetentionTasks(ctx context.Context, params operation.ListRetentionTasksParams) middleware.Responder {
query := &q.Query{
PageNumber: *params.Page,
PageSize: *params.PageSize,
}
p, err := r.retentionCtl.GetRetention(ctx, params.ID)
if err != nil {
return r.SendError(ctx, errors.BadRequestError(err))
}
err = r.requireAccess(ctx, p, rbac.ActionList)
if err != nil {
return r.SendError(ctx, err)
}
tasks, err := r.retentionCtl.ListRetentionExecTasks(ctx, params.Eid, query)
if err != nil {
return r.SendError(ctx, err)
}
total, err := r.retentionCtl.GetTotalOfRetentionExecTasks(ctx, params.Eid)
if err != nil {
return r.SendError(ctx, err)
}
var payload []*models.RetentionExecutionTask
for _, t := range tasks {
payload = append(payload, model.NewRetentionTask(t).ToSwagger())
}
return operation.NewListRetentionTasksOK().WithXTotalCount(total).
WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
WithPayload(payload)
}
func (r *retentionAPI) GetRetentionTaskLog(ctx context.Context, params operation.GetRetentionTaskLogParams) middleware.Responder {
p, err := r.retentionCtl.GetRetention(ctx, params.ID)
if err != nil {
return r.SendError(ctx, errors.BadRequestError(err))
}
err = r.requireAccess(ctx, p, rbac.ActionRead)
if err != nil {
return r.SendError(ctx, err)
}
log, err := r.retentionCtl.GetRetentionExecTaskLog(ctx, params.Tid)
if err != nil {
return r.SendError(ctx, err)
}
return operation.NewGetRetentionTaskLogOK().WithPayload(string(log))
}
func (r *retentionAPI) requireAccess(ctx context.Context, p *policy.Metadata, action rbac.Action, subresources ...rbac.Resource) error {
switch p.Scope.Level {
case "project":
if len(subresources) == 0 {
subresources = append(subresources, rbac.ResourceTagRetention)
}
err := r.RequireProjectAccess(ctx, p.Scope.Reference, action, subresources...)
return err
}
return r.RequireSystemAccess(ctx, action, rbac.ResourceTagRetention)
}

9
src/server/v2.0/route/legacy.go
View File

@ -74,15 +74,6 @@ func registerLegacyRoutes() {
beego.Router("/api/"+version+"/registries/:id/info", &api.RegistryAPI{}, "get:GetInfo")
beego.Router("/api/"+version+"/registries/:id/namespace", &api.RegistryAPI{}, "get:GetNamespace")
beego.Router("/api/"+version+"/retentions/metadatas", &api.RetentionAPI{}, "get:GetMetadatas")
beego.Router("/api/"+version+"/retentions/:id", &api.RetentionAPI{}, "get:GetRetention")
beego.Router("/api/"+version+"/retentions", &api.RetentionAPI{}, "post:CreateRetention")
beego.Router("/api/"+version+"/retentions/:id", &api.RetentionAPI{}, "put:UpdateRetention")
beego.Router("/api/"+version+"/retentions/:id/executions", &api.RetentionAPI{}, "post:TriggerRetentionExec")
beego.Router("/api/"+version+"/retentions/:id/executions/:eid", &api.RetentionAPI{}, "patch:OperateRetentionExec")
beego.Router("/api/"+version+"/retentions/:id/executions", &api.RetentionAPI{}, "get:ListRetentionExecs")
beego.Router("/api/"+version+"/retentions/:id/executions/:eid/tasks", &api.RetentionAPI{}, "get:ListRetentionExecTasks")
beego.Router("/api/"+version+"/retentions/:id/executions/:eid/tasks/:tid", &api.RetentionAPI{}, "get:GetRetentionExecTaskLog")
beego.Router("/api/"+version+"/projects/:pid([0-9]+)/immutabletagrules", &api.ImmutableTagRuleAPI{}, "get:List;post:Post")
beego.Router("/api/"+version+"/projects/:pid([0-9]+)/immutabletagrules/:id([0-9]+)", &api.ImmutableTagRuleAPI{})

1
src/testing/controller/controller.go
View File

@ -24,3 +24,4 @@ package controller
//go:generate mockery --case snake --dir ../../controller/replication --name Controller --output ./replication --outpkg replication
//go:generate mockery --case snake --dir ../../controller/robot --name Controller --output ./robot --outpkg robot
//go:generate mockery --case snake --dir ../../controller/proxy --name RemoteInterface --output ./proxy --outpkg proxy
//go:generate mockery --case snake --dir ../../controller/retention --name Controller --output ./retention --outpkg retention

283
src/testing/controller/retention/controller.go Normal file
View File

@ -0,0 +1,283 @@
// Code generated by mockery v2.1.0. DO NOT EDIT.
package retention
import (
context "context"
pkgretention "github.com/goharbor/harbor/src/pkg/retention"
mock "github.com/stretchr/testify/mock"
policy "github.com/goharbor/harbor/src/pkg/retention/policy"
q "github.com/goharbor/harbor/src/lib/q"
)
// Controller is an autogenerated mock type for the Controller type
type Controller struct {
mock.Mock
}
// CreateRetention provides a mock function with given fields: ctx, p
func (_m *Controller) CreateRetention(ctx context.Context, p *policy.Metadata) (int64, error) {
ret := _m.Called(ctx, p)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, *policy.Metadata) int64); ok {
r0 = rf(ctx, p)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *policy.Metadata) error); ok {
r1 = rf(ctx, p)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// DeleteRetention provides a mock function with given fields: ctx, id
func (_m *Controller) DeleteRetention(ctx context.Context, id int64) error {
ret := _m.Called(ctx, id)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// GetRetention provides a mock function with given fields: ctx, id
func (_m *Controller) GetRetention(ctx context.Context, id int64) (*policy.Metadata, error) {
ret := _m.Called(ctx, id)
var r0 *policy.Metadata
if rf, ok := ret.Get(0).(func(context.Context, int64) *policy.Metadata); ok {
r0 = rf(ctx, id)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*policy.Metadata)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetRetentionExec provides a mock function with given fields: ctx, eid
func (_m *Controller) GetRetentionExec(ctx context.Context, eid int64) (*pkgretention.Execution, error) {
ret := _m.Called(ctx, eid)
var r0 *pkgretention.Execution
if rf, ok := ret.Get(0).(func(context.Context, int64) *pkgretention.Execution); ok {
r0 = rf(ctx, eid)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*pkgretention.Execution)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, eid)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetRetentionExecTask provides a mock function with given fields: ctx, taskID
func (_m *Controller) GetRetentionExecTask(ctx context.Context, taskID int64) (*pkgretention.Task, error) {
ret := _m.Called(ctx, taskID)
var r0 *pkgretention.Task
if rf, ok := ret.Get(0).(func(context.Context, int64) *pkgretention.Task); ok {
r0 = rf(ctx, taskID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*pkgretention.Task)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, taskID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetRetentionExecTaskLog provides a mock function with given fields: ctx, taskID
func (_m *Controller) GetRetentionExecTaskLog(ctx context.Context, taskID int64) ([]byte, error) {
ret := _m.Called(ctx, taskID)
var r0 []byte
if rf, ok := ret.Get(0).(func(context.Context, int64) []byte); ok {
r0 = rf(ctx, taskID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]byte)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, taskID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetTotalOfRetentionExecTasks provides a mock function with given fields: ctx, executionID
func (_m *Controller) GetTotalOfRetentionExecTasks(ctx context.Context, executionID int64) (int64, error) {
ret := _m.Called(ctx, executionID)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, int64) int64); ok {
r0 = rf(ctx, executionID)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, executionID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetTotalOfRetentionExecs provides a mock function with given fields: ctx, policyID
func (_m *Controller) GetTotalOfRetentionExecs(ctx context.Context, policyID int64) (int64, error) {
ret := _m.Called(ctx, policyID)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, int64) int64); ok {
r0 = rf(ctx, policyID)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, policyID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListRetentionExecTasks provides a mock function with given fields: ctx, executionID, query
func (_m *Controller) ListRetentionExecTasks(ctx context.Context, executionID int64, query *q.Query) ([]*pkgretention.Task, error) {
ret := _m.Called(ctx, executionID, query)
var r0 []*pkgretention.Task
if rf, ok := ret.Get(0).(func(context.Context, int64, *q.Query) []*pkgretention.Task); ok {
r0 = rf(ctx, executionID, query)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*pkgretention.Task)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64, *q.Query) error); ok {
r1 = rf(ctx, executionID, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListRetentionExecs provides a mock function with given fields: ctx, policyID, query
func (_m *Controller) ListRetentionExecs(ctx context.Context, policyID int64, query *q.Query) ([]*pkgretention.Execution, error) {
ret := _m.Called(ctx, policyID, query)
var r0 []*pkgretention.Execution
if rf, ok := ret.Get(0).(func(context.Context, int64, *q.Query) []*pkgretention.Execution); ok {
r0 = rf(ctx, policyID, query)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*pkgretention.Execution)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64, *q.Query) error); ok {
r1 = rf(ctx, policyID, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// OperateRetentionExec provides a mock function with given fields: ctx, eid, action
func (_m *Controller) OperateRetentionExec(ctx context.Context, eid int64, action string) error {
ret := _m.Called(ctx, eid, action)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int64, string) error); ok {
r0 = rf(ctx, eid, action)
} else {
r0 = ret.Error(0)
}
return r0
}
// TriggerRetentionExec provides a mock function with given fields: ctx, policyID, trigger, dryRun
func (_m *Controller) TriggerRetentionExec(ctx context.Context, policyID int64, trigger string, dryRun bool) (int64, error) {
ret := _m.Called(ctx, policyID, trigger, dryRun)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, int64, string, bool) int64); ok {
r0 = rf(ctx, policyID, trigger, dryRun)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64, string, bool) error); ok {
r1 = rf(ctx, policyID, trigger, dryRun)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// UpdateRetention provides a mock function with given fields: ctx, p
func (_m *Controller) UpdateRetention(ctx context.Context, p *policy.Metadata) error {
ret := _m.Called(ctx, p)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, *policy.Metadata) error); ok {
r0 = rf(ctx, p)
} else {
r0 = ret.Error(0)
}
return r0
}

8
tests/robot-cases/Group0-BAT/API_DB.robot
View File

@ -85,10 +85,6 @@ Test Case - Project Level CVE Allowlist
[Tags] pro_cve
Harbor API Test ./tests/apitests/python/test_project_level_cve_allowlist.py
Test Case - Tag Retention
[Tags] tag_retention
Harbor API Test ./tests/apitests/python/test_retention.py
Test Case - Health Check
[Tags] health
Harbor API Test ./tests/apitests/python/test_health_check.py
@ -149,10 +145,6 @@ Test Case - Proxy Cache
[Tags] proxy_cache
Harbor API Test ./tests/apitests/python/test_proxy_cache.py
Test Case - Tag Immutability
[Tags] tag_immutability
Harbor API Test ./tests/apitests/python/test_tag_immutability.py
Test Case - P2P
[Tags] p2p
Harbor API Test ./tests/apitests/python/test_p2p.py