Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-26 04:05:40 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7306 from ywk253100/190404_cleanup

Browse Source 
Remove the useless replication code
This commit is contained in:
Wenkai Yin 2019-04-04 21:18:04 +08:00 committed by GitHub
commit 4116433de8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
91 changed files with 30 additions and 7685 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -63,5 +63,5 @@ script:
- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_run.sh $IP; fi
# TODO(ChenDe): Enable API test when API test problems resolved
#- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_run.sh DB $IP; fi
- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_run.sh LDAP $IP; fi
#- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_run.sh LDAP $IP; fi
- if [ "$OFFLINE" == true ]; then bash ./tests/travis/distro_installer.sh; fi

492
docs/swagger.yaml
View File

@ -1746,337 +1746,6 @@ paths:
description: Resource requested does not exist.
'500':
description: Unexpected internal errors.
/jobs/replication:
get:
summary: List filters jobs according to the policy and repository
description: |
This endpoint let user list filters jobs according to the policy and repository. (if start_time and end_time are both null, list jobs of last 10 days)
tags:
- Products
parameters:
- name: policy_id
in: query
type: integer
format: int
required: true
description: The ID of the policy that triggered this job.
- name: op_uuid
in: query
type: string
required: false
description: The UUID of one trigger of replication policy.
- name: num
in: query
type: integer
format: int32
required: false
description: The return list length number.
- name: end_time
in: query
type: integer
format: int64
required: false
description: The end time of jobs done. (Timestamp)
- name: start_time
in: query
type: integer
format: int64
required: false
description: The start time of jobs. (Timestamp)
- name: repository
in: query
type: string
required: false
description: The respond jobs list filter by repository name.
- name: status
in: query
type: string
required: false
description: The respond jobs list filter by status.
- name: page
in: query
type: integer
format: int32
required: false
description: 'The page nubmer, default is 1.'
- name: page_size
in: query
type: integer
format: int32
required: false
description: 'The size of per page, default is 10, maximum is 100.'
responses:
'200':
description: Get the required logs successfully.
schema:
type: array
items:
$ref: '#/definitions/JobStatus'
headers:
X-Total-Count:
description: The total count of jobs
type: integer
Link:
description: Link refers to the previous page and next page
type: string
'400':
description: Bad request because of invalid parameters.
'401':
description: User need to login first.
'500':
description: Unexpected internal errors.
put:
summary: Update status of jobs. Only stop is supported for now.
description: |
The endpoint is used to stop the replication jobs of a policy.
tags:
- Products
parameters:
- name: policyinfo
in: body
description: The policy ID and status.
required: true
schema:
$ref: '#/definitions/UpdateJobs'
responses:
'200':
description: Update the status successfully.
'400':
description: Bad request because of invalid parameters.
'401':
description: User need to login first.
'403':
description: User has no privilege for the operation.
'404':
description: Resource requested does not exist.
'500':
description: Unexpected internal errors.
'/jobs/replication/{id}':
delete:
summary: Delete specific ID job.
description: |
This endpoint is aimed to remove specific ID job from jobservice.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Delete job ID.
tags:
- Products
responses:
'200':
description: Job deleted successfully.
'400':
description: Job ID is invalid or can't remove this job.
'401':
description: Only admin has this authority.
'404':
description: Project ID does not exist.
'500':
description: Unexpected internal errors.
'/jobs/replication/{id}/log':
get:
summary: Get job logs.
description: |
This endpoint let user search job logs filtered by specific ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Relevant job ID
tags:
- Products
responses:
'200':
description: Get job log successfully.
'400':
description: Illegal format of provided ID value.
'401':
description: User need to log in first.
'404':
description: The specific repository ID's log does not exist.
'500':
description: Unexpected internal errors.
'/jobs/scan/{id}/log':
get:
summary: Get job logs.
description: |
This endpoint let user get scan job logs filtered by specific ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Relevant job ID
tags:
- Products
responses:
'200':
description: Get job log successfully.
'400':
description: Illegal format of provided ID value.
'401':
description: User need to log in first.
'404':
description: The specific repository ID's log does not exist.
'500':
description: Unexpected internal errors.
/policies/replication:
get:
summary: List filters policies by name and project_id
description: |
This endpoint let user list filters policies by name and project_id, if name and project_id are nil, list returns all policies
parameters:
- name: name
in: query
type: string
required: false
description: The replication's policy name.
- name: project_id
in: query
type: integer
format: int64
required: false
description: Relevant project ID.
- name: page
in: query
type: integer
format: int32
required: false
description: The page nubmer.
- name: page_size
in: query
type: integer
format: int32
required: false
description: The size of per page.
tags:
- Products
responses:
'200':
description: Get policy successfully.
schema:
type: array
items:
$ref: '#/definitions/RepPolicy'
'400':
description: Invalid project ID.
'401':
description: User need to log in first.
'500':
description: Unexpected internal errors.
post:
summary: Post creates a policy
description: |
This endpoint let user creates a policy, and if it is enabled, the replication will be triggered right now.
parameters:
- name: policyinfo
in: body
description: Create new policy.
required: true
schema:
$ref: '#/definitions/RepPolicy'
tags:
- Products
responses:
'201':
description: Create policy successfully.
'400':
description: Invalid project ID or target ID.
'401':
description: User need to log in first.
'409':
description: Policy name already used or policy already exists with the same project and target.
'415':
$ref: '#/responses/UnsupportedMediaType'
'500':
description: Unexpected internal errors.
'/policies/replication/{id}':
get:
summary: Get replication policy.
description: |
This endpoint let user search replication policy by specific ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: policy ID
tags:
- Products
responses:
'200':
description: Get job policy successfully.
schema:
$ref: '#/definitions/RepPolicy'
'401':
description: User need to log in first.
'404':
description: The specific repository ID's policy does not exist.
'500':
description: Unexpected internal errors.
put:
summary: 'Put modifies name, description, target and enablement of policy.'
description: |
This endpoint let user update policy name, description, target and enablement.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: policy ID
- name: policyupdate
in: body
description: Updated properties of the replication policy.
required: true
schema:
$ref: '#/definitions/RepPolicy'
tags:
- Products
responses:
'200':
description: Update job policy content successfully.
'400':
description: policy is enabled or target does not exist
'401':
description: User need to log in first.
'404':
description: The specific repository ID's policy does not exist.
'409':
description: Policy name already used or policy already exists with the same project and target.
'500':
description: Unexpected internal errors.
delete:
summary: Delete the replication policy specified by ID.
description: |
Delete the replication policy specified by ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Replication policy ID
tags:
- Products
responses:
'200':
description: Delete successfully.
'400':
description: Invalid parameters.
'401':
description: User need to log in first.
'404':
description: The resource does not exist.
'500':
description: Unexpected internal errors.
/replication/policies:
get:
summary: List replication policies
@ -2421,33 +2090,6 @@ paths:
description: The resource does not exist.
'500':
description: Unexpected internal errors.
/replications:
post:
summary: Trigger the replication according to the specified policy.
description: |
This endpoint is used to trigger a replication.
parameters:
- name: policy ID
in: body
description: The ID of replication policy.
required: true
schema:
$ref: '#/definitions/Replication'
tags:
- Products
responses:
'200':
description: Trigger the replication successfully.
schema:
$ref: '#/definitions/ReplicationResponse'
'401':
description: User need to log in first.
'404':
description: The policy does not exist.
'415':
$ref: '#/responses/UnsupportedMediaType'
'500':
description: Unexpected internal errors.
/replication/adapters:
get:
summary: List supported adapters.
@ -2622,33 +2264,6 @@ paths:
description: Registry not found
'500':
description: Unexpected internal errors.
'/targets/{id}/policies/':
get:
summary: List the target relevant policies.
description: |
This endpoint list policies filter with specific replication's target ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: The replication's target ID.
tags:
- Products
responses:
'200':
description: Get relevant policies successfully.
schema:
type: array
items:
$ref: '#/definitions/RepPolicy'
'401':
description: User need to log in first.
'404':
description: Replication's target not found
'500':
description: Unexpected internal errors.
/internal/syncregistry:
post:
summary: Sync repositories from registry to DB.
@ -4081,51 +3696,6 @@ definitions:
tag:
type: string
description: The repository's used tag.
RepPolicy:
type: object
properties:
id:
type: integer
format: int64
description: The policy ID.
name:
type: string
description: The policy name.
description:
type: string
description: The description of the policy.
projects:
type: array
description: The project list that the policy applys to.
items:
$ref: '#/definitions/Project'
registries:
type: array
description: The target list.
items:
$ref: '#/definitions/Registry'
trigger:
$ref: '#/definitions/RepTrigger'
filters:
type: array
description: The replication policy filter array.
items:
$ref: '#/definitions/RepFilter'
replicate_existing_image_now:
type: boolean
description: Whether to replicate the existing images now.
replicate_deletion:
type: boolean
description: Whether to replicate the deletion operation.
creation_time:
type: string
description: The create time of the policy.
update_time:
type: string
description: The update time of the policy.
error_job_count:
type: integer
description: The error job count number for the policy.
ReplicationPolicy:
type: object
properties:
@ -4154,7 +3724,7 @@ definitions:
type: string
description: The destination namespace.
trigger:
$ref: '#/definitions/RepTrigger'
$ref: '#/definitions/ReplicationTrigger'
filters:
type: array
description: The replication policy filter array.
@ -4175,43 +3745,20 @@ definitions:
update_time:
type: string
description: The update time of the policy.
RepTrigger:
type: object
properties:
kind:
type: string
description: 'The replication policy trigger kind. The valid values are manual, immediate and schedule.'
schedule_param:
$ref: '#/definitions/ScheduleParam'
ScheduleParam:
ReplicationTrigger:
type: object
properties:
type:
type: string
description: The schedule type. The valid values are daily and weekly.
weekday:
type: integer
format: int8
description: 'Optional, only used when the type is weedly. The valid values are 1-7.'
offtime:
type: integer
format: int64
description: 'The time offset with the UTC 00:00 in seconds.'
RepFilter:
description: 'The replication policy trigger type. The valid values are manual, event_based and scheduled.'
trigger_settings:
$ref: '#/definitions/TriggerSettings'
TriggerSettings:
type: object
properties:
kind:
cron:
type: string
description: 'The replication policy filter kind. The valid values are project, repository and tag.'
value:
type: string
description: 'The value of replication policy filter. When creating repository and tag filter, filling it with the pattern as string. When creating label filter, filling it with label ID as integer.'
pattern:
type: string
description: 'Depraceted, use value instead. The replication policy filter pattern.'
metadata:
type: object
description: This map object is the replication policy filter metadata.
description: The cron string for scheduled trigger
ReplicationFilter:
type: object
properties:
@ -4782,33 +4329,12 @@ definitions:
type: integer
description: 'The offest in seconds of UTC 0 o''clock, only valid when the policy type is "daily"'
description: 'The parameters of the policy, the values are dependant on the type of the policy.'
Replication:
type: object
properties:
policy_id:
type: integer
description: The ID of replication policy
ReplicationResponse:
type: object
properties:
uuid:
type: string
description: UUID of the replication
RepositoryDescription:
type: object
properties:
description:
type: string
description: The description of the repository.
UpdateJobs:
type: object
properties:
policy_id:
type: integer
description: The ID of replication policy
status:
type: string
description: The status of jobs. The only valid value is stop for now.
Label:
type: object
properties:
@ -4910,7 +4436,7 @@ definitions:
type: array
description: The replication policy list.
items:
$ref: '#/definitions/RepPolicy'
$ref: '#/definitions/ReplicationPolicy'
StringConfigItem:
type: object
properties:

287
src/common/dao/dao_test.go
View File

@ -26,7 +26,6 @@ import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func execUpdate(o orm.Ormer, sql string, params ...interface{}) error {
@ -675,292 +674,6 @@ func TestChangeUserProfile(t *testing.T) {
var targetID, policyID, policyID2, policyID3, jobID, jobID2, jobID3 int64
func TestAddRepPolicy(t *testing.T) {
policy := models.RepPolicy{
ProjectID: 1,
TargetID: targetID,
Description: "whatever",
Name: "mypolicy",
}
id, err := AddRepPolicy(policy)
t.Logf("added policy, id: %d", id)
if err != nil {
t.Errorf("Error occurred in AddRepPolicy: %v", err)
} else {
policyID = id
}
p, err := GetRepPolicy(id)
if err != nil {
t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id)
}
if p == nil {
t.Errorf("Unable to find a policy with id: %d", id)
}
if p.Name != "mypolicy" || p.TargetID != targetID || p.Description != "whatever" {
t.Errorf("The data does not match, expected: Name: mypolicy, TargetID: %d, Description: whatever;\n result: Name: %s, TargetID: %d, Description: %s",
targetID, p.Name, p.TargetID, p.Description)
}
}
func TestGetRepPolicyByTarget(t *testing.T) {
policies, err := GetRepPolicyByTarget(targetID)
if err != nil {
t.Fatalf("failed to get policy according target %d: %v", targetID, err)
}
if len(policies) == 0 {
t.Fatal("unexpected length of policies 0, expected is >0")
}
if policies[0].ID != policyID {
t.Fatalf("unexpected policy: %d, expected: %d", policies[0].ID, policyID)
}
}
func TestGetRepPolicyByProjectAndTarget(t *testing.T) {
policies, err := GetRepPolicyByProjectAndTarget(1, targetID)
if err != nil {
t.Fatalf("failed to get policy according project %d and target %d: %v", 1, targetID, err)
}
if len(policies) == 0 {
t.Fatal("unexpected length of policies 0, expected is >0")
}
if policies[0].ID != policyID {
t.Fatalf("unexpected policy: %d, expected: %d", policies[0].ID, policyID)
}
}
func TestGetRepPolicyByName(t *testing.T) {
policy, err := GetRepPolicy(policyID)
if err != nil {
t.Fatalf("failed to get policy %d: %v", policyID, err)
}
policy2, err := GetRepPolicyByName(policy.Name)
if err != nil {
t.Fatalf("failed to get policy %s: %v", policy.Name, err)
}
if policy.Name != policy2.Name {
t.Errorf("unexpected name: %s, expected: %s", policy2.Name, policy.Name)
}
}
func TestAddRepJob(t *testing.T) {
job := models.RepJob{
Repository: "library/ubuntu",
PolicyID: policyID,
Operation: "transfer",
TagList: []string{"12.01", "14.04", "latest"},
}
id, err := AddRepJob(job)
if err != nil {
t.Errorf("Error occurred in AddRepJob: %v", err)
return
}
jobID = id
j, err := GetRepJob(id)
if err != nil {
t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, id)
return
}
if j == nil {
t.Errorf("Unable to find a job with id: %d", id)
return
}
if j.Status != models.JobPending || j.Repository != "library/ubuntu" || j.PolicyID != policyID || j.Operation != "transfer" || len(j.TagList) != 3 {
t.Errorf("Expected data of job, id: %d, Status: %s, Repository: library/ubuntu, PolicyID: %d, Operation: transfer, taglist length 3"+
"but in returned data:, Status: %s, Repository: %s, Operation: %s, PolicyID: %d, TagList: %v", id, models.JobPending, policyID, j.Status, j.Repository, j.Operation, j.PolicyID, j.TagList)
return
}
}
func TestSetRepJobUUID(t *testing.T) {
uuid := "u-rep-job-uuid"
assert := assert.New(t)
err := SetRepJobUUID(jobID, uuid)
assert.Nil(err)
j, err := GetRepJob(jobID)
assert.Nil(err)
assert.Equal(uuid, j.UUID)
}
func TestUpdateRepJobStatus(t *testing.T) {
err := UpdateRepJobStatus(jobID, models.JobFinished)
if err != nil {
t.Errorf("Error occurred in UpdateRepJobStatus, error: %v, id: %d", err, jobID)
return
}
j, err := GetRepJob(jobID)
if err != nil {
t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, jobID)
}
if j == nil {
t.Errorf("Unable to find a job with id: %d", jobID)
}
if j.Status != models.JobFinished {
t.Errorf("Job's status: %s, expected: %s, id: %d", j.Status, models.JobFinished, jobID)
}
err = UpdateRepJobStatus(jobID, models.JobPending)
if err != nil {
t.Errorf("Error occurred in UpdateRepJobStatus when update it back to status pending, error: %v, id: %d", err, jobID)
return
}
}
func TestGetRepPolicyByProject(t *testing.T) {
p1, err := GetRepPolicyByProject(99)
if err != nil {
t.Errorf("Error occurred in GetRepPolicyByProject:%v, project ID: %d", err, 99)
return
}
if len(p1) > 0 {
t.Errorf("Unexpected length of policy list, expected: 0, in fact: %d, project id: %d", len(p1), 99)
return
}
p2, err := GetRepPolicyByProject(1)
if err != nil {
t.Errorf("Error occuered in GetRepPolicyByProject:%v, project ID: %d", err, 2)
return
}
if len(p2) != 1 {
t.Errorf("Unexpected length of policy list, expected: 1, in fact: %d, project id: %d", len(p2), 1)
return
}
if p2[0].ID != policyID {
t.Errorf("Unexpecred policy id in result, expected: %d, in fact: %d", policyID, p2[0].ID)
return
}
}
func TestGetRepJobs(t *testing.T) {
var policyID int64 = 10000
repository := "repository_for_test_get_rep_jobs"
operation := "operation_for_test"
status := "status_for_test"
now := time.Now().Add(1 * time.Minute)
id, err := AddRepJob(models.RepJob{
PolicyID: policyID,
Repository: repository,
Operation: operation,
Status: status,
CreationTime: now,
UpdateTime: now,
})
require.Nil(t, err)
defer DeleteRepJob(id)
// no query
jobs, err := GetRepJobs()
require.Nil(t, err)
found := false
for _, job := range jobs {
if job.ID == id {
found = true
break
}
}
assert.True(t, found)
// query by policy ID
jobs, err = GetRepJobs(&models.RepJobQuery{
PolicyID: policyID,
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
// query by repository
jobs, err = GetRepJobs(&models.RepJobQuery{
Repository: repository,
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
// query by operation
jobs, err = GetRepJobs(&models.RepJobQuery{
Operations: []string{operation},
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
// query by status
jobs, err = GetRepJobs(&models.RepJobQuery{
Statuses: []string{status},
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
// query by creation time
jobs, err = GetRepJobs(&models.RepJobQuery{
StartTime: &now,
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
}
func TestDeleteRepJob(t *testing.T) {
err := DeleteRepJob(jobID)
if err != nil {
t.Errorf("Error occurred in DeleteRepJob: %v, id: %d", err, jobID)
return
}
t.Logf("deleted rep job, id: %d", jobID)
j, err := GetRepJob(jobID)
if err != nil {
t.Errorf("Error occurred in GetRepJob:%v", err)
return
}
if j != nil {
t.Errorf("Able to find rep job after deletion, id: %d", jobID)
return
}
}
func TestGetTotalOfRepPolicies(t *testing.T) {
_, err := GetTotalOfRepPolicies("", 1)
require.Nil(t, err)
}
func TestFilterRepPolicies(t *testing.T) {
_, err := FilterRepPolicies("name", 0, 0, 0)
if err != nil {
t.Fatalf("failed to filter policy: %v", err)
}
}
func TestUpdateRepPolicy(t *testing.T) {
policy := &models.RepPolicy{
ID: policyID,
Name: "new_policy_name",
}
if err := UpdateRepPolicy(policy); err != nil {
t.Fatalf("failed to update policy")
}
}
func TestDeleteRepPolicy(t *testing.T) {
err := DeleteRepPolicy(policyID)
if err != nil {
t.Errorf("Error occurred in DeleteRepPolicy: %v, id: %d", err, policyID)
return
}
t.Logf("delete rep policy, id: %d", policyID)
p, err := GetRepPolicy(policyID)
require.Nil(t, err)
assert.Nil(t, p)
}
func TestGetOrmer(t *testing.T) {
o := GetOrmer()
if o == nil {

333
src/common/dao/replication_job.go
View File

@ -1,333 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"time"
"strings"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
)
// AddRepPolicy ...
func AddRepPolicy(policy models.RepPolicy) (int64, error) {
o := GetOrmer()
sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, creation_time, update_time, filters, replicate_deletion)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id`
params := []interface{}{}
now := time.Now()
params = append(params, policy.Name, policy.ProjectID, policy.TargetID, true,
policy.Description, policy.Trigger, now, now, policy.Filters,
policy.ReplicateDeletion)
var policyID int64
err := o.Raw(sql, params...).QueryRow(&policyID)
if err != nil {
return 0, err
}
return policyID, nil
}
// GetRepPolicy ...
func GetRepPolicy(id int64) (*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where id = ? and deleted = false`
var policy models.RepPolicy
if err := o.Raw(sql, id).QueryRow(&policy); err != nil {
if err == orm.ErrNoRows {
return nil, nil
}
return nil, err
}
return &policy, nil
}
// GetTotalOfRepPolicies returns the total count of replication policies
func GetTotalOfRepPolicies(name string, projectID int64) (int64, error) {
qs := GetOrmer().QueryTable(&models.RepPolicy{}).Filter("deleted", false)
if len(name) != 0 {
qs = qs.Filter("name__icontains", name)
}
if projectID != 0 {
qs = qs.Filter("project_id", projectID)
}
return qs.Count()
}
// FilterRepPolicies filters policies by name and project ID
func FilterRepPolicies(name string, projectID, page, pageSize int64) ([]*models.RepPolicy, error) {
o := GetOrmer()
var args []interface{}
sql := `select rp.id, rp.project_id, rp.target_id,
rt.name as target_name, rp.name, rp.description,
rp.cron_str, rp.filters, rp.replicate_deletion,
rp.creation_time, rp.update_time,
count(rj.status) as error_job_count
from replication_policy rp
left join replication_target rt on rp.target_id=rt.id
left join replication_job rj on rp.id=rj.policy_id and (rj.status='error'
or rj.status='retrying')
where rp.deleted = false `
if len(name) != 0 && projectID != 0 {
sql += `and rp.name like ? and rp.project_id = ? `
args = append(args, "%"+Escape(name)+"%")
args = append(args, projectID)
} else if len(name) != 0 {
sql += `and rp.name like ? `
args = append(args, "%"+Escape(name)+"%")
} else if projectID != 0 {
sql += `and rp.project_id = ? `
args = append(args, projectID)
}
sql += `group by rt.name, rp.id order by rp.creation_time`
if page > 0 && pageSize > 0 {
sql += ` limit ? offset ?`
args = append(args, pageSize, (page-1)*pageSize)
}
var policies []*models.RepPolicy
if _, err := o.Raw(sql, args).QueryRows(&policies); err != nil {
return nil, err
}
return policies, nil
}
// GetRepPolicyByName ...
func GetRepPolicyByName(name string) (*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where deleted = false and name = ?`
var policy models.RepPolicy
if err := o.Raw(sql, name).QueryRow(&policy); err != nil {
if err == orm.ErrNoRows {
return nil, nil
}
return nil, err
}
return &policy, nil
}
// GetRepPolicyByProject ...
func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where deleted = false and project_id = ?`
var policies []*models.RepPolicy
if _, err := o.Raw(sql, projectID).QueryRows(&policies); err != nil {
return nil, err
}
return policies, nil
}
// GetRepPolicyByTarget ...
func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where deleted = false and target_id = ?`
var policies []*models.RepPolicy
if _, err := o.Raw(sql, targetID).QueryRows(&policies); err != nil {
return nil, err
}
return policies, nil
}
// GetRepPolicyByProjectAndTarget ...
func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where deleted = false and project_id = ? and target_id = ?`
var policies []*models.RepPolicy
if _, err := o.Raw(sql, projectID, targetID).QueryRows(&policies); err != nil {
return nil, err
}
return policies, nil
}
// UpdateRepPolicy ...
func UpdateRepPolicy(policy *models.RepPolicy) error {
o := GetOrmer()
sql := `update replication_policy
set project_id = ?, target_id = ?, name = ?, description = ?, cron_str = ?, filters = ?, replicate_deletion = ?, update_time = ?
where id = ?`
_, err := o.Raw(sql, policy.ProjectID, policy.TargetID, policy.Name, policy.Description, policy.Trigger, policy.Filters, policy.ReplicateDeletion, time.Now(), policy.ID).Exec()
return err
}
// DeleteRepPolicy ...
func DeleteRepPolicy(id int64) error {
_, err := GetOrmer().Delete(&models.RepPolicy{
ID: id,
})
return err
}
// AddRepJob ...
func AddRepJob(job models.RepJob) (int64, error) {
o := GetOrmer()
if len(job.Status) == 0 {
job.Status = models.JobPending
}
if len(job.TagList) > 0 {
job.Tags = strings.Join(job.TagList, ",")
}
return o.Insert(&job)
}
// GetRepJob ...
func GetRepJob(id int64) (*models.RepJob, error) {
o := GetOrmer()
j := models.RepJob{ID: id}
err := o.Read(&j)
if err == orm.ErrNoRows {
return nil, nil
}
genTagListForJob(&j)
return &j, nil
}
// GetTotalCountOfRepJobs ...
func GetTotalCountOfRepJobs(query ...*models.RepJobQuery) (int64, error) {
qs := repJobQueryConditions(query...)
return qs.Count()
}
// GetRepJobs ...
func GetRepJobs(query ...*models.RepJobQuery) ([]*models.RepJob, error) {
jobs := []*models.RepJob{}
qs := repJobQueryConditions(query...)
if len(query) > 0 && query[0] != nil {
qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size)
}
qs = qs.OrderBy("-UpdateTime")
if _, err := qs.All(&jobs); err != nil {
return jobs, err
}
genTagListForJob(jobs...)
return jobs, nil
}
func repJobQueryConditions(query ...*models.RepJobQuery) orm.QuerySeter {
qs := GetOrmer().QueryTable(new(models.RepJob))
if len(query) == 0 || query[0] == nil {
return qs
}
q := query[0]
if q.PolicyID != 0 {
qs = qs.Filter("PolicyID", q.PolicyID)
}
if len(q.OpUUID) > 0 {
qs = qs.Filter("OpUUID__exact", q.OpUUID)
}
if len(q.Repository) > 0 {
qs = qs.Filter("Repository__icontains", q.Repository)
}
if len(q.Statuses) > 0 {
qs = qs.Filter("Status__in", q.Statuses)
}
if len(q.Operations) > 0 {
qs = qs.Filter("Operation__in", q.Operations)
}
if q.StartTime != nil {
qs = qs.Filter("CreationTime__gte", q.StartTime)
}
if q.EndTime != nil {
qs = qs.Filter("CreationTime__lte", q.EndTime)
}
return qs
}
// DeleteRepJob ...
func DeleteRepJob(id int64) error {
o := GetOrmer()
_, err := o.Delete(&models.RepJob{ID: id})
return err
}
// DeleteRepJobs deletes replication jobs by policy ID
func DeleteRepJobs(policyID int64) error {
_, err := GetOrmer().QueryTable(&models.RepJob{}).Filter("PolicyID", policyID).Delete()
return err
}
// UpdateRepJobStatus ...
func UpdateRepJobStatus(id int64, status string) error {
o := GetOrmer()
j := models.RepJob{
ID: id,
Status: status,
UpdateTime: time.Now(),
}
n, err := o.Update(&j, "Status", "UpdateTime")
if n == 0 {
log.Warningf("no records are updated when updating replication job %d", id)
}
return err
}
// SetRepJobUUID ...
func SetRepJobUUID(id int64, uuid string) error {
o := GetOrmer()
j := models.RepJob{
ID: id,
UUID: uuid,
}
n, err := o.Update(&j, "UUID")
if n == 0 {
log.Warningf("no records are updated when updating replication job %d", id)
}
return err
}
func genTagListForJob(jobs ...*models.RepJob) {
for _, j := range jobs {
if len(j.Tags) > 0 {
j.TagList = strings.Split(j.Tags, ",")
}
}
}

56
src/common/dao/replication_job_test.go
View File

@ -1,56 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDeleteRepJobs(t *testing.T) {
var policyID int64 = 999
_, err := AddRepJob(models.RepJob{
PolicyID: policyID,
Repository: "library/hello-world",
Operation: "delete",
Status: "success",
})
require.Nil(t, err)
_, err = AddRepJob(models.RepJob{
PolicyID: policyID,
Repository: "library/hello-world",
Operation: "delete",
Status: "success",
})
require.Nil(t, err)
jobs, err := GetRepJobs(&models.RepJobQuery{
PolicyID: policyID,
})
require.Nil(t, err)
require.Equal(t, 2, len(jobs))
err = DeleteRepJobs(policyID)
require.Nil(t, err)
jobs, err = GetRepJobs(&models.RepJobQuery{
PolicyID: policyID,
})
require.Nil(t, err)
assert.Equal(t, 0, len(jobs))
}

71
src/common/dao/watch_item.go
View File

@ -1,71 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"time"
"github.com/goharbor/harbor/src/common/models"
)
// DefaultDatabaseWatchItemDAO is an instance of DatabaseWatchItemDAO
var DefaultDatabaseWatchItemDAO WatchItemDAO = &DatabaseWatchItemDAO{}
// WatchItemDAO defines operations about WatchItem
type WatchItemDAO interface {
Add(*models.WatchItem) (int64, error)
DeleteByPolicyID(int64) error
Get(namespace, operation string) ([]models.WatchItem, error)
}
// DatabaseWatchItemDAO implements interface WatchItemDAO for database
type DatabaseWatchItemDAO struct{}
// Add a WatchItem
func (d *DatabaseWatchItemDAO) Add(item *models.WatchItem) (int64, error) {
o := GetOrmer()
var triggerID int64
now := time.Now()
sql := "insert into replication_immediate_trigger (policy_id, namespace, on_deletion, on_push, creation_time, update_time) values (?, ?, ?, ?, ?, ?) RETURNING id"
err := o.Raw(sql, item.PolicyID, item.Namespace, item.OnDeletion, item.OnPush, now, now).QueryRow(&triggerID)
if err != nil {
return 0, err
}
return triggerID, nil
}
// DeleteByPolicyID deletes the WatchItem specified by policy ID
func (d *DatabaseWatchItemDAO) DeleteByPolicyID(policyID int64) error {
_, err := GetOrmer().QueryTable(&models.WatchItem{}).Filter("PolicyID", policyID).Delete()
return err
}
// Get returns WatchItem list according to the namespace and operation
func (d *DatabaseWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) {
qs := GetOrmer().QueryTable(&models.WatchItem{}).Filter("Namespace", namespace)
if operation == "push" {
qs = qs.Filter("OnPush", true)
} else if operation == "delete" {
qs = qs.Filter("OnDeletion", true)
}
items := []models.WatchItem{}
_, err := qs.All(&items)
return items, err
}

77
src/common/dao/watch_item_test.go
View File

@ -1,77 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
// TODO: This UT makes common DAO depends on replication ng DAOs, comment it out temporarily here
/*
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
common_models "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/replication/ng/dao"
"github.com/goharbor/harbor/src/replication/ng/dao/models"
)
func TestMethodsOfWatchItem(t *testing.T) {
registryID, err := dao.AddRegistry(&models.Registry{
Name: "test_target_for_watch_item",
URL: "http://127.0.0.1",
})
require.Nil(t, err)
defer dao.DeleteRegistry(registryID)
policyID, err := AddRepPolicy(common_models.RepPolicy{
Name: "test_policy_for_watch_item",
ProjectID: 1,
TargetID: targetID,
})
require.Nil(t, err)
defer DeleteRepPolicy(policyID)
item := &common_models.WatchItem{
PolicyID: policyID,
Namespace: "library",
OnPush: false,
OnDeletion: true,
}
// test Add
id, err := DefaultDatabaseWatchItemDAO.Add(item)
require.Nil(t, err)
// test Get: operation-push
items, err := DefaultDatabaseWatchItemDAO.Get("library", "push")
require.Nil(t, err)
assert.Equal(t, 0, len(items))
// test Get: operation-delete
items, err = DefaultDatabaseWatchItemDAO.Get("library", "delete")
require.Nil(t, err)
assert.Equal(t, 1, len(items))
assert.Equal(t, id, items[0].ID)
assert.Equal(t, "library", items[0].Namespace)
assert.True(t, items[0].OnDeletion)
// test DeleteByPolicyID
err = DefaultDatabaseWatchItemDAO.DeleteByPolicyID(policyID)
require.Nil(t, err)
items, err = DefaultDatabaseWatchItemDAO.Get("library", "delete")
require.Nil(t, err)
assert.Equal(t, 0, len(items))
}
*/

3
src/common/models/base.go
View File

@ -20,8 +20,6 @@ import (
func init() {
orm.RegisterModel(
new(RepPolicy),
new(RepJob),
new(User),
new(Project),
new(Role),
@ -30,7 +28,6 @@ func init() {
new(RepoRecord),
new(ImgScanOverview),
new(ClairVulnTimestamp),
new(WatchItem),
new(ProjectMetadata),
new(ConfigEntry),
new(Label),

87
src/common/models/replication_job.go
View File

@ -1,87 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"time"
)
const (
// RepOpTransfer represents the operation of a job to transfer repository to a remote registry/harbor instance.
RepOpTransfer string = "transfer"
// RepOpDelete represents the operation of a job to remove repository from a remote registry/harbor instance.
RepOpDelete string = "delete"
// RepOpSchedule represents the operation of a job to schedule the real replication process
RepOpSchedule string = "schedule"
// RegistryTable is the table name for registry
RegistryTable = "registry"
// RepJobTable is the table name for replication jobs
RepJobTable = "replication_job"
// RepPolicyTable is table name for replication policies
RepPolicyTable = "replication_policy"
)
// RepPolicy is the model for a replication policy, which associate to a project and a target (destination)
type RepPolicy struct {
ID int64 `orm:"pk;auto;column(id)"`
ProjectID int64 `orm:"column(project_id)" `
TargetID int64 `orm:"column(target_id)"`
Name string `orm:"column(name)"`
Description string `orm:"column(description)"`
Trigger string `orm:"column(cron_str)"`
Filters string `orm:"column(filters)"`
ReplicateDeletion bool `orm:"column(replicate_deletion)"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add"`
UpdateTime time.Time `orm:"column(update_time);auto_now"`
Deleted bool `orm:"column(deleted)"`
}
// RepJob is the model for a replication job, which is the execution unit on job service, currently it is used to transfer/remove
// a repository to/from a remote registry instance.
type RepJob struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Status string `orm:"column(status)" json:"status"`
Repository string `orm:"column(repository)" json:"repository"`
PolicyID int64 `orm:"column(policy_id)" json:"policy_id"`
OpUUID string `orm:"column(op_uuid)" json:"op_uuid"`
Operation string `orm:"column(operation)" json:"operation"`
Tags string `orm:"column(tags)" json:"-"`
TagList []string `orm:"-" json:"tags"`
UUID string `orm:"column(job_uuid)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// TableName is required by by beego orm to map RepJob to table replication_job
func (r *RepJob) TableName() string {
return RepJobTable
}
// TableName is required by by beego orm to map RepPolicy to table replication_policy
func (r *RepPolicy) TableName() string {
return RepPolicyTable
}
// RepJobQuery holds query conditions for replication job
type RepJobQuery struct {
PolicyID int64
OpUUID string
Repository string
Statuses []string
Operations []string
StartTime *time.Time
EndTime *time.Time
Pagination
}

35
src/common/models/watch_item.go
View File

@ -1,35 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"time"
)
// WatchItem ...
type WatchItem struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
PolicyID int64 `orm:"column(policy_id)" json:"policy_id"`
Namespace string `orm:"column(namespace)" json:"namespace"`
OnDeletion bool `orm:"column(on_deletion)" json:"on_deletion"`
OnPush bool `orm:"column(on_push)" json:"on_push"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// TableName ...
func (w *WatchItem) TableName() string {
return "replication_immediate_trigger"
}

45
src/common/utils/test/policy_manager.go
View File

@ -1,45 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
)
type FakePolicyManager struct {
}
func (f *FakePolicyManager) GetPolicies(query models.QueryParameter) (*models.ReplicationPolicyQueryResult, error) {
return &models.ReplicationPolicyQueryResult{}, nil
}
func (f *FakePolicyManager) GetPolicy(id int64) (models.ReplicationPolicy, error) {
return models.ReplicationPolicy{
ID: 1,
Trigger: &models.Trigger{
Kind: replication.TriggerKindManual,
},
}, nil
}
func (f *FakePolicyManager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) {
return 1, nil
}
func (f *FakePolicyManager) UpdatePolicy(models.ReplicationPolicy) error {
return nil
}
func (f *FakePolicyManager) RemovePolicy(int64) error {
return nil
}

30
src/common/utils/test/replication_controllter.go
View File

@ -1,30 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
// FakeReplicatoinController ...
type FakeReplicatoinController struct {
FakePolicyManager
}
// Init initialize replication controller
func (f *FakeReplicatoinController) Init(closing chan struct{}) error {
return nil
}
// Replicate ...
func (f *FakeReplicatoinController) Replicate(policyID int64, metadata ...map[string]interface{}) error {
return nil
}

65
src/common/utils/test/watch_item.go
View File

@ -1,65 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"github.com/goharbor/harbor/src/common/models"
)
// FakeWatchItemDAO is the fake implement for the dao.WatchItemDAO
type FakeWatchItemDAO struct {
items []models.WatchItem
}
// Add ...
func (f *FakeWatchItemDAO) Add(item *models.WatchItem) (int64, error) {
f.items = append(f.items, *item)
return int64(len(f.items) + 1), nil
}
// DeleteByPolicyID : delete the WatchItem specified by policy ID
func (f *FakeWatchItemDAO) DeleteByPolicyID(policyID int64) error {
for i, item := range f.items {
if item.PolicyID == policyID {
f.items = append(f.items[:i], f.items[i+1:]...)
break
}
}
return nil
}
// Get returns WatchItem list according to the namespace and operation
func (f *FakeWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) {
items := []models.WatchItem{}
for _, item := range f.items {
if item.Namespace != namespace {
continue
}
if operation == "push" {
if item.OnPush {
items = append(items, item)
}
}
if operation == "delete" {
if item.OnDeletion {
items = append(items, item)
}
}
}
return items, nil
}

10
src/core/api/harborapi_test.go
View File

@ -39,8 +39,6 @@ import (
_ "github.com/goharbor/harbor/src/core/auth/ldap"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/src/replication/core"
_ "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/ng/model"
"github.com/goharbor/harbor/tests/apitests/apilib"
)
@ -127,9 +125,6 @@ func init() {
beego.Router("/api/repositories/top", &RepositoryAPI{}, "get:GetTopRepos")
beego.Router("/api/registries", &RegistryAPI{}, "get:List;post:Post")
beego.Router("/api/registries/:id([0-9]+)", &RegistryAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/policies/replication/:id([0-9]+)", &RepPolicyAPI{})
beego.Router("/api/policies/replication", &RepPolicyAPI{}, "get:List")
beego.Router("/api/policies/replication", &RepPolicyAPI{}, "post:Post;delete:Delete")
beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo")
beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo")
beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert")
@ -140,7 +135,6 @@ func init() {
beego.Router("/api/configurations", &ConfigAPI{})
beego.Router("/api/configs", &ConfigAPI{}, "get:GetInternalConfig")
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
beego.Router("/api/replications", &ReplicationAPI{})
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/labels/:id([0-9]+)/resources", &LabelAPI{}, "get:ListResources")
@ -183,10 +177,6 @@ func init() {
beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel")
beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel")
if err := core.Init(make(chan struct{})); err != nil {
log.Fatalf("failed to initialize GlobalController: %v", err)
}
// syncRegistry
if err := SyncRegistry(config.GlobalProjectMgr); err != nil {
log.Fatalf("failed to sync repositories from registry: %v", err)

37
src/core/api/label.go
View File

@ -23,9 +23,6 @@ import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/core"
rep_models "github.com/goharbor/harbor/src/replication/models"
)
// LabelAPI handles requests for label management
@ -313,26 +310,28 @@ func (l *LabelAPI) ListResources() {
return
}
result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{})
if err != nil {
l.HandleInternalServerError(fmt.Sprintf("failed to get policies: %v", err))
return
}
policies := []*rep_models.ReplicationPolicy{}
if result != nil {
for _, policy := range result.Policies {
for _, filter := range policy.Filters {
if filter.Kind != replication.FilterItemKindLabel {
continue
}
if filter.Value.(int64) == label.ID {
policies = append(policies, policy)
/*
result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{})
if err != nil {
l.HandleInternalServerError(fmt.Sprintf("failed to get policies: %v", err))
return
}
policies := []*rep_models.ReplicationPolicy{}
if result != nil {
for _, policy := range result.Policies {
for _, filter := range policy.Filters {
if filter.Kind != replication.FilterItemKindLabel {
continue
}
if filter.Value.(int64) == label.ID {
policies = append(policies, policy)
}
}
}
}
}
*/
resources := map[string]interface{}{}
resources["replication_policies"] = policies
resources["replication_policies"] = nil
l.Data["json"] = resources
l.ServeJSON()
}

107
src/core/api/label_test.go
View File

@ -21,12 +21,7 @@ import (
"testing"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/replication"
rep_models "github.com/goharbor/harbor/src/replication/models"
rep_dao "github.com/goharbor/harbor/src/replication/ng/dao"
dao_models "github.com/goharbor/harbor/src/replication/ng/dao/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -438,105 +433,3 @@ func TestLabelAPIDelete(t *testing.T) {
runCodeCheckingCases(t, cases...)
}
func TestListResources(t *testing.T) {
// global level label
globalLabelID, err := dao.AddLabel(&models.Label{
Name: "globel_level_label",
Scope: common.LabelScopeGlobal,
})
require.Nil(t, err)
defer dao.DeleteLabel(globalLabelID)
// project level label
projectLabelID, err := dao.AddLabel(&models.Label{
Name: "project_level_label",
Scope: common.LabelScopeProject,
ProjectID: 1,
})
require.Nil(t, err)
defer dao.DeleteLabel(projectLabelID)
registryID, err := rep_dao.AddRegistry(&dao_models.Registry{
Name: "target_for_testing_label_resource",
URL: "https://192.168.0.1",
})
require.Nil(t, err)
defer rep_dao.DeleteRegistry(registryID)
// create a policy references both global and project labels
policyID, err := dao.AddRepPolicy(models.RepPolicy{
Name: "policy_for_testing_label_resource",
ProjectID: 1,
TargetID: registryID,
Trigger: fmt.Sprintf(`{"kind":"%s"}`, replication.TriggerKindManual),
Filters: fmt.Sprintf(`[{"kind":"%s","value":%d}, {"kind":"%s","value":%d}]`,
replication.FilterItemKindLabel, globalLabelID,
replication.FilterItemKindLabel, projectLabelID),
})
require.Nil(t, err)
defer dao.DeleteRepPolicy(policyID)
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID),
},
code: http.StatusUnauthorized,
},
// 404
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, 10000),
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 403: global level label
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID),
credential: projAdmin,
},
code: http.StatusForbidden,
},
// 403: project level label
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, projectLabelID),
credential: projDeveloper,
},
code: http.StatusForbidden,
},
}
runCodeCheckingCases(t, cases...)
// 200: global level label
resources := map[string][]rep_models.ReplicationPolicy{}
err = handleAndParse(&testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID),
credential: sysAdmin,
}, &resources)
require.Nil(t, err)
policies := resources["replication_policies"]
require.Equal(t, 1, len(policies))
assert.Equal(t, policyID, policies[0].ID)
// 200: project level label
resources = map[string][]rep_models.ReplicationPolicy{}
err = handleAndParse(&testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, projectLabelID),
credential: projAdmin,
}, &resources)
require.Nil(t, err)
policies = resources["replication_policies"]
require.Equal(t, 1, len(policies))
assert.Equal(t, policyID, policies[0].ID)
}

36
src/core/api/models/replication.go
View File

@ -1,36 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"github.com/astaxie/beego/validation"
)
// Replication defines the properties of model used in replication API
type Replication struct {
PolicyID int64 `json:"policy_id"`
}
// ReplicationResponse describes response of a replication request, it gives
type ReplicationResponse struct {
UUID string `json:"uuid"`
}
// Valid ...
func (r *Replication) Valid(v *validation.Validation) {
if r.PolicyID <= 0 {
v.SetError("policy_id", "invalid value")
}
}

35
src/core/api/models/replication_job.go
View File

@ -1,35 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"github.com/astaxie/beego/validation"
)
// StopJobsReq holds information needed to stop the jobs for a replication rule
type StopJobsReq struct {
PolicyID int64 `json:"policy_id"`
Status string `json:"status"`
}
// Valid ...
func (s *StopJobsReq) Valid(v *validation.Validation) {
if s.PolicyID <= 0 {
v.SetError("policy_id", "invalid value")
}
if s.Status != "stop" {
v.SetError("status", "invalid status, valid values: [stop]")
}
}

69
src/core/api/models/replication_policy.go
View File

@ -1,69 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"time"
"github.com/astaxie/beego/validation"
common_models "github.com/goharbor/harbor/src/common/models"
rep_models "github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/ng/dao/models"
)
// ReplicationPolicy defines the data model used in API level
type ReplicationPolicy struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Filters []rep_models.Filter `json:"filters"`
ReplicateDeletion bool `json:"replicate_deletion"`
Trigger *rep_models.Trigger `json:"trigger"`
Projects []*common_models.Project `json:"projects"`
Registries []*models.Registry `json:"registries"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
ReplicateExistingImageNow bool `json:"replicate_existing_image_now"`
ErrorJobCount int64 `json:"error_job_count"`
}
// Valid ...
func (r *ReplicationPolicy) Valid(v *validation.Validation) {
if len(r.Name) == 0 {
v.SetError("name", "can not be empty")
}
if len(r.Name) > 256 {
v.SetError("name", "max length is 256")
}
if len(r.Projects) == 0 {
v.SetError("projects", "can not be empty")
}
if len(r.Registries) == 0 {
v.SetError("targets", "can not be empty")
}
for i := range r.Filters {
r.Filters[i].Valid(v)
}
if r.Trigger == nil {
v.SetError("trigger", "can not be empty")
} else {
r.Trigger.Valid(v)
}
}

12
src/core/api/project.go
View File

@ -284,18 +284,6 @@ func (p *ProjectAPI) deletable(projectID int64) (*deletableResp, error) {
}, nil
}
policies, err := dao.GetRepPolicyByProject(projectID)
if err != nil {
return nil, err
}
if len(policies) > 0 {
return &deletableResp{
Deletable: false,
Message: "the project contains replication rules, can not be deleted",
}, nil
}
// Check helm charts number
if config.WithChartMuseum() {
charts, err := chartController.ListCharts(p.project.Name)

5
src/core/api/registry.go
View File

@ -7,7 +7,6 @@ import (
"github.com/goharbor/harbor/src/replication/ng/event"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/replication/ng"
@ -198,8 +197,8 @@ func (t *RegistryAPI) Delete() {
return
}
// TODO: Use PolicyManager instead
policies, err := dao.GetRepPolicyByTarget(id)
// TODO: filter the policies by registry ID
_, policies, err := ng.PolicyCtl.List()
if err != nil {
msg := fmt.Sprintf("Get policies related to registry %d error: %v", id, err)
log.Error(msg)

107
src/core/api/replication.go
View File

@ -1,107 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"strings"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/replication/core"
"github.com/goharbor/harbor/src/replication/event/notification"
"github.com/goharbor/harbor/src/replication/event/topic"
"github.com/docker/distribution/uuid"
)
// ReplicationAPI handles API calls for replication
type ReplicationAPI struct {
BaseController
}
// Prepare does authentication and authorization works
func (r *ReplicationAPI) Prepare() {
r.BaseController.Prepare()
if !r.SecurityCtx.IsAuthenticated() {
r.HandleUnauthorized()
return
}
if !r.SecurityCtx.IsSysAdmin() && !r.SecurityCtx.IsSolutionUser() {
r.HandleForbidden(r.SecurityCtx.GetUsername())
return
}
}
// Post trigger a replication according to the specified policy
func (r *ReplicationAPI) Post() {
replication := &api_models.Replication{}
r.DecodeJSONReqAndValidate(replication)
policy, err := core.GlobalController.GetPolicy(replication.PolicyID)
if err != nil {
r.HandleInternalServerError(fmt.Sprintf("failed to get replication policy %d: %v", replication.PolicyID, err))
return
}
if policy.ID == 0 {
r.HandleNotFound(fmt.Sprintf("replication policy %d not found", replication.PolicyID))
return
}
count, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
PolicyID: replication.PolicyID,
Statuses: []string{models.JobPending, models.JobRunning},
Operations: []string{models.RepOpTransfer, models.RepOpDelete},
})
if err != nil {
r.HandleInternalServerError(fmt.Sprintf("failed to filter jobs of policy %d: %v",
replication.PolicyID, err))
return
}
if count > 0 {
r.RenderError(http.StatusPreconditionFailed, "policy has running/pending jobs, new replication can not be triggered")
return
}
opUUID, err := startReplication(replication.PolicyID)
if err != nil {
r.HandleInternalServerError(fmt.Sprintf("failed to publish replication topic for policy %d: %v", replication.PolicyID, err))
return
}
log.Infof("replication signal for policy %d sent", replication.PolicyID)
r.Data["json"] = api_models.ReplicationResponse{
UUID: opUUID,
}
r.ServeJSON()
}
// startReplication triggers a replication and return the uuid of this replication.
func startReplication(policyID int64) (string, error) {
opUUID := strings.Replace(uuid.Generate().String(), "-", "", -1)
return opUUID, notifier.Publish(topic.StartReplicationTopic,
notification.StartReplicationNotification{
PolicyID: policyID,
Metadata: map[string]interface{}{
"op_uuid": opUUID,
},
})
}

254
src/core/api/replication_job.go
View File

@ -1,254 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"strconv"
"time"
"github.com/goharbor/harbor/src/common/dao"
common_http "github.com/goharbor/harbor/src/common/http"
common_job "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils/log"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/replication/core"
)
// RepJobAPI handles request to /api/replicationJobs /api/replicationJobs/:id/log
type RepJobAPI struct {
BaseController
jobID int64
}
// Prepare validates that whether user has system admin role
func (ra *RepJobAPI) Prepare() {
ra.BaseController.Prepare()
if !ra.SecurityCtx.IsAuthenticated() {
ra.HandleUnauthorized()
return
}
if !(ra.Ctx.Request.Method == http.MethodGet || ra.SecurityCtx.IsSysAdmin()) {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
if len(ra.GetStringFromPath(":id")) != 0 {
id, err := ra.GetInt64FromPath(":id")
if err != nil {
ra.HandleBadRequest(fmt.Sprintf("invalid ID: %s", ra.GetStringFromPath(":id")))
return
}
ra.jobID = id
}
}
// List filters jobs according to the parameters
func (ra *RepJobAPI) List() {
policyID, err := ra.GetInt64("policy_id")
if err != nil || policyID <= 0 {
ra.HandleBadRequest(fmt.Sprintf("invalid policy_id: %s", ra.GetString("policy_id")))
return
}
policy, err := core.GlobalController.GetPolicy(policyID)
if err != nil {
log.Errorf("failed to get policy %d: %v", policyID, err)
ra.CustomAbort(http.StatusInternalServerError, "")
}
if policy.ID == 0 {
ra.HandleNotFound(fmt.Sprintf("policy %d not found", policyID))
return
}
resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplicationJob)
if !ra.SecurityCtx.Can(rbac.ActionList, resource) {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
query := &models.RepJobQuery{
PolicyID: policyID,
// hide the schedule job, the schedule job is used to trigger replication
// for scheduled policy
Operations: []string{models.RepOpTransfer, models.RepOpDelete},
}
query.Repository = ra.GetString("repository")
query.Statuses = ra.GetStrings("status")
query.OpUUID = ra.GetString("op_uuid")
startTimeStr := ra.GetString("start_time")
if len(startTimeStr) != 0 {
i, err := strconv.ParseInt(startTimeStr, 10, 64)
if err != nil {
ra.HandleBadRequest(fmt.Sprintf("invalid start_time: %s", startTimeStr))
return
}
t := time.Unix(i, 0)
query.StartTime = &t
}
endTimeStr := ra.GetString("end_time")
if len(endTimeStr) != 0 {
i, err := strconv.ParseInt(endTimeStr, 10, 64)
if err != nil {
ra.HandleBadRequest(fmt.Sprintf("invalid end_time: %s", endTimeStr))
return
}
t := time.Unix(i, 0)
query.EndTime = &t
}
query.Page, query.Size = ra.GetPaginationParams()
total, err := dao.GetTotalCountOfRepJobs(query)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to get total count of repository jobs of policy %d: %v", policyID, err))
return
}
jobs, err := dao.GetRepJobs(query)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to get repository jobs, query: %v :%v", query, err))
return
}
ra.SetPaginationHeader(total, query.Page, query.Size)
ra.Data["json"] = jobs
ra.ServeJSON()
}
// Delete ...
func (ra *RepJobAPI) Delete() {
if ra.jobID == 0 {
ra.HandleBadRequest("ID is nil")
return
}
job, err := dao.GetRepJob(ra.jobID)
if err != nil {
log.Errorf("failed to get job %d: %v", ra.jobID, err)
ra.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if job == nil {
ra.HandleNotFound(fmt.Sprintf("job %d not found", ra.jobID))
return
}
if job.Status == models.JobPending || job.Status == models.JobRunning {
ra.HandleBadRequest(fmt.Sprintf("job is %s, can not be deleted", job.Status))
return
}
if err = dao.DeleteRepJob(ra.jobID); err != nil {
log.Errorf("failed to deleted job %d: %v", ra.jobID, err)
ra.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
// GetLog ...
func (ra *RepJobAPI) GetLog() {
if ra.jobID == 0 {
ra.HandleBadRequest("ID is nil")
return
}
job, err := dao.GetRepJob(ra.jobID)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to get replication job %d: %v", ra.jobID, err))
return
}
if job == nil {
ra.HandleNotFound(fmt.Sprintf("replication job %d not found", ra.jobID))
return
}
policy, err := core.GlobalController.GetPolicy(job.PolicyID)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to get policy %d: %v", job.PolicyID, err))
return
}
resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplicationJob)
if !ra.SecurityCtx.Can(rbac.ActionRead, resource) {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
logBytes, err := utils.GetJobServiceClient().GetJobLog(job.UUID)
if err != nil {
if httpErr, ok := err.(*common_http.Error); ok {
ra.RenderError(httpErr.Code, "")
log.Errorf(fmt.Sprintf("failed to get log of job %d: %d %s",
ra.jobID, httpErr.Code, httpErr.Message))
return
}
ra.HandleInternalServerError(fmt.Sprintf("failed to get log of job %s: %v",
job.UUID, err))
return
}
ra.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
ra.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
_, err = ra.Ctx.ResponseWriter.Write(logBytes)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to write log of job %s: %v", job.UUID, err))
return
}
}
// StopJobs stop replication jobs for the policy
func (ra *RepJobAPI) StopJobs() {
req := &api_models.StopJobsReq{}
ra.DecodeJSONReqAndValidate(req)
policy, err := core.GlobalController.GetPolicy(req.PolicyID)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to get policy %d: %v", req.PolicyID, err))
return
}
if policy.ID == 0 {
ra.HandleNotFound(fmt.Sprintf("policy %d not found", req.PolicyID))
return
}
jobs, err := dao.GetRepJobs(&models.RepJobQuery{
PolicyID: policy.ID,
Operations: []string{models.RepOpTransfer, models.RepOpDelete},
})
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to list jobs of policy %d: %v", policy.ID, err))
return
}
for _, job := range jobs {
if err = utils.GetJobServiceClient().PostAction(job.UUID, common_job.JobActionStop); err != nil {
log.Errorf("failed to stop job id-%d uuid-%s: %v", job.ID, job.UUID, err)
continue
}
}
}
// TODO:add Post handler to call job service API to submit jobs by policy

443
src/core/api/replication_policy.go
View File

@ -1,443 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils/log"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/core"
rep_models "github.com/goharbor/harbor/src/replication/models"
rep_dao "github.com/goharbor/harbor/src/replication/ng/dao"
)
// RepPolicyAPI handles /api/replicationPolicies /api/replicationPolicies/:id/enablement
type RepPolicyAPI struct {
BaseController
}
// Prepare validates whether the user has system admin role
func (pa *RepPolicyAPI) Prepare() {
pa.BaseController.Prepare()
if !pa.SecurityCtx.IsAuthenticated() {
pa.HandleUnauthorized()
return
}
if !(pa.Ctx.Request.Method == http.MethodGet || pa.SecurityCtx.IsSysAdmin()) {
pa.HandleForbidden(pa.SecurityCtx.GetUsername())
return
}
}
// Get ...
func (pa *RepPolicyAPI) Get() {
id := pa.GetIDFromURL()
policy, err := core.GlobalController.GetPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if policy.ID == 0 {
pa.HandleNotFound(fmt.Sprintf("policy %d not found", id))
return
}
resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplication)
if !pa.SecurityCtx.Can(rbac.ActionRead, resource) {
pa.HandleForbidden(pa.SecurityCtx.GetUsername())
return
}
ply, err := convertFromRepPolicy(pa.ProjectMgr, policy)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
return
}
pa.Data["json"] = ply
pa.ServeJSON()
}
// List ...
func (pa *RepPolicyAPI) List() {
queryParam := rep_models.QueryParameter{
Name: pa.GetString("name"),
}
projectIDStr := pa.GetString("project_id")
if len(projectIDStr) > 0 {
projectID, err := strconv.ParseInt(projectIDStr, 10, 64)
if err != nil || projectID <= 0 {
pa.HandleBadRequest(fmt.Sprintf("invalid project ID: %s", projectIDStr))
return
}
queryParam.ProjectID = projectID
}
queryParam.Page, queryParam.PageSize = pa.GetPaginationParams()
result, err := core.GlobalController.GetPolicies(queryParam)
if err != nil {
log.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
var total int64
policies := []*api_models.ReplicationPolicy{}
if result != nil {
total = result.Total
for _, policy := range result.Policies {
resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplication)
if !pa.SecurityCtx.Can(rbac.ActionRead, resource) {
continue
}
ply, err := convertFromRepPolicy(pa.ProjectMgr, *policy)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
return
}
policies = append(policies, ply)
}
}
pa.SetPaginationHeader(total, queryParam.Page, queryParam.PageSize)
pa.Data["json"] = policies
pa.ServeJSON()
}
// Post creates a replicartion policy
func (pa *RepPolicyAPI) Post() {
policy := &api_models.ReplicationPolicy{}
pa.DecodeJSONReqAndValidate(policy)
// check the name
exist, err := exist(policy.Name)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to check the existence of policy %s: %v", policy.Name, err))
return
}
if exist {
pa.HandleConflict(fmt.Sprintf("name %s is already used", policy.Name))
return
}
// check the existence of projects
for _, project := range policy.Projects {
pro, err := pa.ProjectMgr.Get(project.ProjectID)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
return
}
if pro == nil {
pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID))
return
}
project.Name = pro.Name
}
// check the existence of targets
for _, r := range policy.Registries {
t, err := rep_dao.GetRegistry(r.ID)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", r.ID, err))
return
}
if t == nil {
pa.HandleNotFound(fmt.Sprintf("target %d not found", r.ID))
return
}
}
// check the existence of labels
for _, filter := range policy.Filters {
if filter.Kind == replication.FilterItemKindLabel {
labelID := filter.Value.(int64)
label, err := dao.GetLabel(labelID)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to get label %d: %v", labelID, err))
return
}
if label == nil || label.Deleted {
pa.HandleNotFound(fmt.Sprintf("label %d not found", labelID))
return
}
}
}
id, err := core.GlobalController.CreatePolicy(convertToRepPolicy(policy))
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to create policy: %v", err))
return
}
if policy.ReplicateExistingImageNow {
go func() {
if _, err = startReplication(id); err != nil {
log.Errorf("failed to send replication signal for policy %d: %v", id, err)
return
}
log.Infof("replication signal for policy %d sent", id)
}()
}
pa.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}
func exist(name string) (bool, error) {
result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{
Name: name,
})
if err != nil {
return false, err
}
for _, policy := range result.Policies {
if policy.Name == name {
return true, nil
}
}
return false, nil
}
// Put updates the replication policy
func (pa *RepPolicyAPI) Put() {
id := pa.GetIDFromURL()
originalPolicy, err := core.GlobalController.GetPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if originalPolicy.ID == 0 {
pa.HandleNotFound(fmt.Sprintf("policy %d not found", id))
return
}
policy := &api_models.ReplicationPolicy{}
pa.DecodeJSONReqAndValidate(policy)
policy.ID = id
// check the name
if policy.Name != originalPolicy.Name {
exist, err := exist(policy.Name)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to check the existence of policy %s: %v", policy.Name, err))
return
}
if exist {
pa.HandleConflict(fmt.Sprintf("name %s is already used", policy.Name))
return
}
}
// check the existence of projects
for _, project := range policy.Projects {
pro, err := pa.ProjectMgr.Get(project.ProjectID)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
return
}
if pro == nil {
pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID))
return
}
project.Name = pro.Name
}
// check the existence of targets
for _, r := range policy.Registries {
t, err := rep_dao.GetRegistry(r.ID)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", r.ID, err))
return
}
if t == nil {
pa.HandleNotFound(fmt.Sprintf("target %d not found", r.ID))
return
}
}
// check the existence of labels
for _, filter := range policy.Filters {
if filter.Kind == replication.FilterItemKindLabel {
labelID := filter.Value.(int64)
label, err := dao.GetLabel(labelID)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to get label %d: %v", labelID, err))
return
}
if label == nil || label.Deleted {
pa.HandleNotFound(fmt.Sprintf("label %d not found", labelID))
return
}
}
}
if err = core.GlobalController.UpdatePolicy(convertToRepPolicy(policy)); err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err))
return
}
if policy.ReplicateExistingImageNow {
go func() {
if _, err = startReplication(id); err != nil {
log.Errorf("failed to send replication signal for policy %d: %v", id, err)
return
}
log.Infof("replication signal for policy %d sent", id)
}()
}
}
// Delete the replication policy
func (pa *RepPolicyAPI) Delete() {
id := pa.GetIDFromURL()
policy, err := core.GlobalController.GetPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if policy.ID == 0 {
pa.HandleNotFound(fmt.Sprintf("policy %d not found", id))
return
}
count, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
PolicyID: id,
Statuses: []string{models.JobRunning, models.JobRetrying, models.JobPending},
// only get the transfer and delete jobs, do not get schedule job
Operations: []string{models.RepOpTransfer, models.RepOpDelete},
})
if err != nil {
log.Errorf("failed to filter jobs of policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, "")
}
if count > 0 {
pa.CustomAbort(http.StatusPreconditionFailed, "policy has running/retrying/pending jobs, can not be deleted")
}
if err = core.GlobalController.RemovePolicy(id); err != nil {
log.Errorf("failed to delete policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, "")
}
}
func convertFromRepPolicy(projectMgr promgr.ProjectManager, policy rep_models.ReplicationPolicy) (*api_models.ReplicationPolicy, error) {
if policy.ID == 0 {
return nil, nil
}
// populate simple properties
ply := &api_models.ReplicationPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
ReplicateDeletion: policy.ReplicateDeletion,
Trigger: policy.Trigger,
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
// populate projects
for _, projectID := range policy.ProjectIDs {
project, err := projectMgr.Get(projectID)
if err != nil {
return nil, err
}
ply.Projects = append(ply.Projects, project)
}
// populate targets
for _, targetID := range policy.TargetIDs {
r, err := rep_dao.GetRegistry(targetID)
if err != nil {
return nil, err
}
r.AccessSecret = ""
ply.Registries = append(ply.Registries, r)
}
// populate label used in label filter
for _, filter := range policy.Filters {
if filter.Kind == replication.FilterItemKindLabel {
labelID := filter.Value.(int64)
label, err := dao.GetLabel(labelID)
if err != nil {
return nil, err
}
filter.Value = label
}
ply.Filters = append(ply.Filters, filter)
}
// TODO call the method from replication controller
errJobCount, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
PolicyID: policy.ID,
Statuses: []string{models.JobError},
})
if err != nil {
return nil, err
}
ply.ErrorJobCount = errJobCount
return ply, nil
}
func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.ReplicationPolicy {
if policy == nil {
return rep_models.ReplicationPolicy{}
}
ply := rep_models.ReplicationPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
Filters: policy.Filters,
ReplicateDeletion: policy.ReplicateDeletion,
Trigger: policy.Trigger,
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
for _, project := range policy.Projects {
ply.ProjectIDs = append(ply.ProjectIDs, project.ProjectID)
ply.Namespaces = append(ply.Namespaces, project.Name)
}
for _, r := range policy.Registries {
ply.TargetIDs = append(ply.TargetIDs, r.ID)
}
return ply
}

711
src/core/api/replication_policy_test.go
View File

@ -1,711 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/goharbor/harbor/src/common/dao/project"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/replication"
rep_models "github.com/goharbor/harbor/src/replication/models"
dao_models "github.com/goharbor/harbor/src/replication/ng/dao/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var (
repPolicyAPIBasePath = "/api/policies/replication"
policyName = "testPolicy"
projectID int64 = 1
targetID int64
policyID int64
labelID2 int64
)
func TestRepPolicyAPIPost(t *testing.T) {
postFunc := func(resp *httptest.ResponseRecorder) error {
id, err := parseResourceID(resp)
if err != nil {
return err
}
policyID = id
return nil
}
CommonAddRegistry()
targetID = int64(CommonGetRegistry())
var err error
labelID2, err = dao.AddLabel(&models.Label{
Name: "label_for_replication_filter",
Scope: "g",
})
require.Nil(t, err)
defer dao.DeleteLabel(labelID2)
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
// 400, invalid name
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 400, invalid projects
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 400, invalid targets
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 400, invalid filters
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
Registries: []*dao_models.Registry{
{
ID: targetID,
},
},
Filters: []rep_models.Filter{
{
Kind: "invalid_filter_kind",
Pattern: "",
},
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 400, invalid trigger
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
Registries: []*dao_models.Registry{
{
ID: targetID,
},
},
Filters: []rep_models.Filter{
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: "invalid_trigger_kind",
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 404, project not found
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: 10000,
},
},
Registries: []*dao_models.Registry{
{
ID: targetID,
},
},
Filters: []rep_models.Filter{
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindManual,
},
},
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 404, target not found
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
Registries: []*dao_models.Registry{
{
ID: 10000,
},
},
Filters: []rep_models.Filter{
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindManual,
},
},
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 404, label not found
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
Registries: []*dao_models.Registry{
{
ID: targetID,
},
},
Filters: []rep_models.Filter{
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
},
{
Kind: replication.FilterItemKindLabel,
Value: 10000,
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindManual,
},
},
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 201
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
Registries: []*dao_models.Registry{
{
ID: targetID,
},
},
Filters: []rep_models.Filter{
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
},
{
Kind: replication.FilterItemKindLabel,
Value: labelID2,
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindManual,
},
},
credential: sysAdmin,
},
code: http.StatusCreated,
postFunc: postFunc,
},
// 409
{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
Registries: []*dao_models.Registry{
{
ID: targetID,
},
},
Filters: []rep_models.Filter{
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
},
{
Kind: replication.FilterItemKindLabel,
Value: labelID2,
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindManual,
},
},
credential: sysAdmin,
},
code: http.StatusConflict,
},
}
runCodeCheckingCases(t, cases...)
}
func TestRepPolicyAPIGet(t *testing.T) {
cases := []*codeCheckingCase{
// 404
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000),
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
},
code: http.StatusUnauthorized,
},
}
runCodeCheckingCases(t, cases...)
// 200
policy := &api_models.ReplicationPolicy{}
err := handleAndParse(
&testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
credential: sysAdmin,
}, policy)
require.Nil(t, err)
assert.Equal(t, policyID, policy.ID)
assert.Equal(t, policyName, policy.Name)
assert.Equal(t, 2, len(policy.Filters))
found := false
for _, filter := range policy.Filters {
if filter.Kind == replication.FilterItemKindLabel {
found = true
label, ok := filter.Value.(map[string]interface{})
if assert.True(t, ok) {
id := int64(label["id"].(float64))
deleted := label["deleted"].(bool)
assert.Equal(t, labelID2, id)
assert.True(t, deleted)
}
}
}
assert.True(t, found)
}
func TestRepPolicyAPIList(t *testing.T) {
projectAdmin := models.User{
Username: "project_admin",
Password: "ProjectAdmin",
Email: "project_admin@test.com",
}
projectDev := models.User{
Username: "project_dev",
Password: "ProjectDev",
Email: "project_dev@test.com",
}
var proAdminPMID, proDevPMID int
proAdminID, err := dao.Register(projectAdmin)
if err != nil {
panic(err)
}
defer dao.DeleteUser(int(proAdminID))
if proAdminPMID, err = project.AddProjectMember(models.Member{
ProjectID: 1,
Role: models.PROJECTADMIN,
EntityID: int(proAdminID),
EntityType: common.UserMember,
}); err != nil {
panic(err)
}
defer project.DeleteProjectMemberByID(proAdminPMID)
proDevID, err := dao.Register(projectDev)
if err != nil {
panic(err)
}
defer dao.DeleteUser(int(proDevID))
if proDevPMID, err = project.AddProjectMember(models.Member{
ProjectID: 1,
Role: models.DEVELOPER,
EntityID: int(proDevID),
EntityType: common.UserMember,
}); err != nil {
panic(err)
}
defer project.DeleteProjectMemberByID(proDevPMID)
// 400: invalid project ID
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
method: http.MethodGet,
url: repPolicyAPIBasePath,
queryStruct: struct {
ProjectID int64 `url:"project_id"`
}{
ProjectID: -1,
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
})
// 200 system admin
policies := []*api_models.ReplicationPolicy{}
err = handleAndParse(
&testingRequest{
method: http.MethodGet,
url: repPolicyAPIBasePath,
queryStruct: struct {
ProjectID int64 `url:"project_id"`
Name string `url:"name"`
}{
ProjectID: projectID,
Name: policyName,
},
credential: sysAdmin,
}, &policies)
require.Nil(t, err)
require.Equal(t, 1, len(policies))
assert.Equal(t, policyID, policies[0].ID)
assert.Equal(t, policyName, policies[0].Name)
// 200 project admin
policies = []*api_models.ReplicationPolicy{}
err = handleAndParse(
&testingRequest{
method: http.MethodGet,
url: repPolicyAPIBasePath,
queryStruct: struct {
ProjectID int64 `url:"project_id"`
Name string `url:"name"`
}{
ProjectID: projectID,
Name: policyName,
},
credential: &usrInfo{
Name: projectAdmin.Username,
Passwd: projectAdmin.Password,
},
}, &policies)
require.Nil(t, err)
require.Equal(t, 1, len(policies))
assert.Equal(t, policyID, policies[0].ID)
assert.Equal(t, policyName, policies[0].Name)
// 200 project developer
policies = []*api_models.ReplicationPolicy{}
err = handleAndParse(
&testingRequest{
method: http.MethodGet,
url: repPolicyAPIBasePath,
queryStruct: struct {
ProjectID int64 `url:"project_id"`
Name string `url:"name"`
}{
ProjectID: projectID,
Name: policyName,
},
credential: &usrInfo{
Name: projectDev.Username,
Passwd: projectDev.Password,
},
}, &policies)
require.Nil(t, err)
require.Equal(t, 0, len(policies))
// 200
policies = []*api_models.ReplicationPolicy{}
err = handleAndParse(
&testingRequest{
method: http.MethodGet,
url: repPolicyAPIBasePath,
queryStruct: struct {
ProjectID int64 `url:"project_id"`
Name string `url:"name"`
}{
ProjectID: projectID,
Name: "non_exist_policy",
},
credential: sysAdmin,
}, &policies)
require.Nil(t, err)
require.Equal(t, 0, len(policies))
}
func TestRepPolicyAPIPut(t *testing.T) {
cases := []*codeCheckingCase{
// 404
{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000),
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 400, invalid trigger
{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
Registries: []*dao_models.Registry{
{
ID: targetID,
},
},
Filters: []rep_models.Filter{
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: "invalid_trigger_kind",
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 200
{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
{
ProjectID: projectID,
},
},
Registries: []*dao_models.Registry{
{
ID: targetID,
},
},
Filters: []rep_models.Filter{
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindImmediate,
},
},
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestRepPolicyAPIDelete(t *testing.T) {
cases := []*codeCheckingCase{
// 404
{
request: &testingRequest{
method: http.MethodDelete,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000),
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 200
{
request: &testingRequest{
method: http.MethodDelete,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestConvertToRepPolicy(t *testing.T) {
cases := []struct {
input *api_models.ReplicationPolicy
expected rep_models.ReplicationPolicy
}{
{
input: nil,
expected: rep_models.ReplicationPolicy{},
},
{
input: &api_models.ReplicationPolicy{
ID: 1,
Name: "policy",
Description: "description",
Filters: []rep_models.Filter{
{
Kind: "filter_kind_01",
Pattern: "*",
},
},
ReplicateDeletion: true,
Trigger: &rep_models.Trigger{
Kind: "trigger_kind_01",
},
Projects: []*models.Project{
{
ProjectID: 1,
Name: "library",
},
},
Registries: []*dao_models.Registry{
{
ID: 1,
},
},
},
expected: rep_models.ReplicationPolicy{
ID: 1,
Name: "policy",
Description: "description",
Filters: []rep_models.Filter{
{
Kind: "filter_kind_01",
Pattern: "*",
},
},
ReplicateDeletion: true,
Trigger: &rep_models.Trigger{
Kind: "trigger_kind_01",
},
ProjectIDs: []int64{1},
Namespaces: []string{"library"},
TargetIDs: []int64{1},
},
},
}
for _, c := range cases {
assert.EqualValues(t, c.expected, convertToRepPolicy(c.input))
}
}

95
src/core/api/replication_test.go
View File

@ -1,95 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"testing"
"github.com/stretchr/testify/require"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/replication"
rep_dao "github.com/goharbor/harbor/src/replication/ng/dao"
dao_models "github.com/goharbor/harbor/src/replication/ng/dao/models"
)
const (
replicationAPIBaseURL = "/api/replications"
)
func TestReplicationAPIPost(t *testing.T) {
registryID, err := rep_dao.AddRegistry(
&dao_models.Registry{
Name: "test_replication_target",
URL: "127.0.0.1",
AccessKey: "username",
AccessSecret: "password",
})
require.Nil(t, err)
defer rep_dao.DeleteRegistry(registryID)
policyID, err := dao.AddRepPolicy(
models.RepPolicy{
Name: "test_replication_policy",
ProjectID: 1,
TargetID: targetID,
Trigger: fmt.Sprintf("{\"kind\":\"%s\"}", replication.TriggerKindManual),
})
require.Nil(t, err)
defer dao.DeleteRepPolicy(policyID)
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodPost,
url: replicationAPIBaseURL,
bodyJSON: &api_models.Replication{
PolicyID: policyID,
},
},
code: http.StatusUnauthorized,
},
// 404
{
request: &testingRequest{
method: http.MethodPost,
url: replicationAPIBaseURL,
bodyJSON: &api_models.Replication{
PolicyID: 10000,
},
credential: admin,
},
code: http.StatusNotFound,
},
// 200
{
request: &testingRequest{
method: http.MethodPost,
url: replicationAPIBaseURL,
bodyJSON: &api_models.Replication{
PolicyID: policyID,
},
credential: admin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}

6
src/core/main.go
View File

@ -38,8 +38,6 @@ import (
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/src/core/proxy"
"github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/replication/core"
_ "github.com/goharbor/harbor/src/replication/event"
)
const (
@ -134,10 +132,6 @@ func main() {
closing := make(chan struct{})
go gracefulShutdown(closing)
if err := core.Init(closing); err != nil {
log.Errorf("failed to initialize the replication controller: %v", err)
}
filter.Init()
beego.InsertFilter("/*", beego.BeforeRouter, filter.SecurityFilter)
beego.InsertFilter("/*", beego.BeforeRouter, filter.ReadonlyFilter)

7
src/core/router.go
View File

@ -88,9 +88,6 @@ func initRouters() {
beego.Router("/api/repositories/*/tags/:tag/manifest", &api.RepositoryAPI{}, "get:GetManifests")
beego.Router("/api/repositories/*/signatures", &api.RepositoryAPI{}, "get:GetSignatures")
beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos")
beego.Router("/api/jobs/replication/", &api.RepJobAPI{}, "get:List;put:StopJobs")
beego.Router("/api/jobs/replication/:id([0-9]+)", &api.RepJobAPI{})
beego.Router("/api/jobs/replication/:id([0-9]+)/log", &api.RepJobAPI{}, "get:GetLog")
beego.Router("/api/jobs/scan/:id([0-9]+)/log", &api.ScanJobAPI{}, "get:GetLog")
beego.Router("/api/system/gc", &api.GCAPI{}, "get:List")
@ -99,9 +96,6 @@ func initRouters() {
beego.Router("/api/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/policies/replication/:id([0-9]+)", &api.RepPolicyAPI{})
beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "get:List")
beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "post:Post")
beego.Router("/api/logs", &api.LogAPI{})
beego.Router("/api/replication/adapters", &api.ReplicationAdapterAPI{}, "get:List")
@ -116,7 +110,6 @@ func initRouters() {
beego.Router("/api/internal/configurations", &api.ConfigAPI{}, "get:GetInternalConfig;put:Put")
beego.Router("/api/configurations", &api.ConfigAPI{}, "get:Get;put:Put")
beego.Router("/api/statistics", &api.StatisticAPI{})
beego.Router("/api/replications", &api.ReplicationAPI{})
beego.Router("/api/labels", &api.LabelAPI{}, "post:Post;get:List")
beego.Router("/api/labels/:id([0-9]+)", &api.LabelAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/labels/:id([0-9]+)/resources", &api.LabelAPI{}, "get:ListResources")

27
src/replication/consts.go
View File

@ -1,27 +0,0 @@
package replication
const (
// FilterItemKindProject : Kind of filter item is 'project'
FilterItemKindProject = "project"
// FilterItemKindRepository : Kind of filter item is 'repository'
FilterItemKindRepository = "repository"
// FilterItemKindTag : Kind of filter item is 'tag'
FilterItemKindTag = "tag"
// FilterItemKindLabel : Kind of filter item is 'label'
FilterItemKindLabel = "label"
// AdaptorKindHarbor : Kind of adaptor of Harbor
AdaptorKindHarbor = "Harbor"
// TriggerKindImmediate : Kind of trigger is 'Immediate'
TriggerKindImmediate = "Immediate"
// TriggerKindSchedule : Kind of trigger is 'Scheduled'
TriggerKindSchedule = "Scheduled"
// TriggerKindManual : Kind of trigger is 'Manual'
TriggerKindManual = "Manual"
// TriggerScheduleDaily : type of scheduling is 'Daily'
TriggerScheduleDaily = "Daily"
// TriggerScheduleWeekly : type of scheduling is 'Weekly'
TriggerScheduleWeekly = "Weekly"
)

337
src/replication/core/controller.go
View File

@ -1,337 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package core
import (
"fmt"
"reflect"
"strings"
"time"
common_models "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/ng/model"
"github.com/goharbor/harbor/src/replication/ng/registry"
"github.com/goharbor/harbor/src/replication/policy"
"github.com/goharbor/harbor/src/replication/replicator"
"github.com/goharbor/harbor/src/replication/source"
"github.com/goharbor/harbor/src/replication/trigger"
"github.com/docker/distribution/uuid"
"github.com/goharbor/harbor/src/replication/ng"
)
// Controller defines the methods that a replicatoin controllter should implement
type Controller interface {
policy.Manager
Init(closing chan struct{}) error
Replicate(policyID int64, metadata ...map[string]interface{}) error
}
// DefaultController is core module to cordinate and control the overall workflow of the
// replication modules.
type DefaultController struct {
// Indicate whether the controller has been initialized or not
initialized bool
// Manage the policies
policyManager policy.Manager
// Manage the registries
registryManager registry.Manager
// Handle the things related with source
sourcer *source.Sourcer
// Manage the triggers of policies
triggerManager *trigger.Manager
// Handle the replication work
replicator replicator.Replicator
}
// Keep controller as singleton instance
var (
GlobalController Controller
)
// ControllerConfig includes related configurations required by the controller
type ControllerConfig struct {
// The capacity of the cache storing enabled triggers
CacheCapacity int
}
// NewDefaultController is the constructor of DefaultController.
func NewDefaultController(cfg ControllerConfig) *DefaultController {
// Controller refer the default instances
ctl := &DefaultController{
policyManager: policy.NewDefaultManager(),
registryManager: registry.NewDefaultManager(),
sourcer: source.NewSourcer(),
triggerManager: trigger.NewManager(cfg.CacheCapacity),
}
ctl.replicator = replicator.NewDefaultReplicator(utils.GetJobServiceClient())
return ctl
}
// Init initializes GlobalController and replication related managers
func Init(closing chan struct{}) error {
GlobalController = NewDefaultController(ControllerConfig{})
err := GlobalController.Init(closing)
if err != nil {
return err
}
return ng.Init()
}
// Init will initialize the controller and the sub components
func (ctl *DefaultController) Init(closing chan struct{}) error {
if ctl.initialized {
return nil
}
// Initialize sourcer
ctl.sourcer.Init()
ctl.initialized = true
// Start registry health checker to regularly check registries' health status
go registry.NewHealthChecker(time.Second*30, closing).Run()
return nil
}
// CreatePolicy is used to create a new policy and enable it if necessary
func (ctl *DefaultController) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, error) {
id, err := ctl.policyManager.CreatePolicy(newPolicy)
if err != nil {
return 0, err
}
newPolicy.ID = id
if err = ctl.triggerManager.SetupTrigger(&newPolicy); err != nil {
return 0, err
}
return id, nil
}
// UpdatePolicy will update the policy with new content.
// Parameter updatedPolicy must have the ID of the updated policy.
func (ctl *DefaultController) UpdatePolicy(updatedPolicy models.ReplicationPolicy) error {
id := updatedPolicy.ID
originPolicy, err := ctl.policyManager.GetPolicy(id)
if err != nil {
return err
}
if originPolicy.ID == 0 {
return fmt.Errorf("policy %d not found", id)
}
reset := false
if updatedPolicy.Trigger.Kind != originPolicy.Trigger.Kind {
reset = true
} else {
switch updatedPolicy.Trigger.Kind {
case replication.TriggerKindSchedule:
if !originPolicy.Trigger.ScheduleParam.Equal(updatedPolicy.Trigger.ScheduleParam) {
reset = true
}
case replication.TriggerKindImmediate:
// Always reset immediate trigger as it is relevant with namespaces
reset = true
default:
// manual trigger, no need to reset
}
}
if err = ctl.policyManager.UpdatePolicy(updatedPolicy); err != nil {
return err
}
if reset {
if err = ctl.triggerManager.UnsetTrigger(&originPolicy); err != nil {
return err
}
return ctl.triggerManager.SetupTrigger(&updatedPolicy)
}
return nil
}
// RemovePolicy will remove the specified policy and clean the related settings
func (ctl *DefaultController) RemovePolicy(policyID int64) error {
// TODO check pre-conditions
policy, err := ctl.policyManager.GetPolicy(policyID)
if err != nil {
return err
}
if policy.ID == 0 {
return fmt.Errorf("policy %d not found", policyID)
}
if err = ctl.triggerManager.UnsetTrigger(&policy); err != nil {
return err
}
return ctl.policyManager.RemovePolicy(policyID)
}
// GetPolicy is delegation of GetPolicy of Policy.Manager
func (ctl *DefaultController) GetPolicy(policyID int64) (models.ReplicationPolicy, error) {
return ctl.policyManager.GetPolicy(policyID)
}
// GetPolicies is delegation of GetPoliciemodels.ReplicationPolicy{}s of Policy.Manager
func (ctl *DefaultController) GetPolicies(query models.QueryParameter) (*models.ReplicationPolicyQueryResult, error) {
return ctl.policyManager.GetPolicies(query)
}
// Replicate starts one replication defined in the specified policy;
// Can be launched by the API layer and related triggers.
func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]interface{}) error {
policy, err := ctl.GetPolicy(policyID)
if err != nil {
return err
}
if policy.ID == 0 {
return fmt.Errorf("policy %d not found", policyID)
}
// prepare candidates for replication
candidates := getCandidates(&policy, ctl.sourcer, metadata...)
if len(candidates) == 0 {
log.Debugf("replication candidates are null, no further action needed")
}
registries := []*model.Registry{}
for _, targetID := range policy.TargetIDs {
r, err := ctl.registryManager.Get(targetID)
if err != nil {
return err
}
registries = append(registries, r)
}
// Get operation uuid from metadata, if none provided, generate one.
opUUID, err := getOpUUID(metadata...)
if err != nil {
return err
}
// submit the replication
return ctl.replicator.Replicate(&replicator.Replication{
PolicyID: policyID,
OpUUID: opUUID,
Candidates: candidates,
Registries: registries,
})
}
func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer,
metadata ...map[string]interface{}) []models.FilterItem {
candidates := []models.FilterItem{}
if len(metadata) > 0 {
meta := metadata[0]["candidates"]
if meta != nil {
cands, ok := meta.([]models.FilterItem)
if ok {
candidates = append(candidates, cands...)
}
}
}
if len(candidates) == 0 {
for _, namespace := range policy.Namespaces {
candidates = append(candidates, models.FilterItem{
Kind: replication.FilterItemKindProject,
Value: namespace,
Operation: common_models.RepOpTransfer,
})
}
}
filterChain := buildFilterChain(policy, sourcer)
return filterChain.DoFilter(candidates)
}
func buildFilterChain(policy *models.ReplicationPolicy, sourcer *source.Sourcer) source.FilterChain {
filters := []source.Filter{}
fm := map[string][]models.Filter{}
for _, filter := range policy.Filters {
fm[filter.Kind] = append(fm[filter.Kind], filter)
}
registry := sourcer.GetAdaptor(replication.AdaptorKindHarbor)
// repository filter
pattern := ""
repoFilters := fm[replication.FilterItemKindRepository]
if len(repoFilters) > 0 {
pattern = repoFilters[0].Value.(string)
}
filters = append(filters,
source.NewRepositoryFilter(pattern, registry))
// tag filter
pattern = ""
tagFilters := fm[replication.FilterItemKindTag]
if len(tagFilters) > 0 {
pattern = tagFilters[0].Value.(string)
}
filters = append(filters,
source.NewTagFilter(pattern, registry))
// label filters
var labelID int64
for _, labelFilter := range fm[replication.FilterItemKindLabel] {
labelID = labelFilter.Value.(int64)
filters = append(filters, source.NewLabelFilter(labelID))
}
return source.NewDefaultFilterChain(filters)
}
// getOpUUID get operation uuid from metadata or generate one if none found.
func getOpUUID(metadata ...map[string]interface{}) (string, error) {
if len(metadata) <= 0 {
return strings.Replace(uuid.Generate().String(), "-", "", -1), nil
}
opUUID, ok := metadata[0]["op_uuid"]
if !ok {
return strings.Replace(uuid.Generate().String(), "-", "", -1), nil
}
id, ok := opUUID.(string)
if !ok {
return "", fmt.Errorf("operation uuid should have type 'string', but got '%s'", reflect.TypeOf(opUUID).Name())
}
if id == "" {
return "", fmt.Errorf("provided operation uuid is empty")
}
return id, nil
}

181
src/replication/core/controller_test.go
View File

@ -1,181 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package core
import (
"os"
"testing"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/ng/registry"
"github.com/goharbor/harbor/src/replication/source"
"github.com/goharbor/harbor/src/replication/trigger"
"github.com/stretchr/testify/assert"
)
func TestMain(m *testing.M) {
GlobalController = &DefaultController{
policyManager: &test.FakePolicyManager{},
registryManager: registry.NewDefaultManager(),
sourcer: source.NewSourcer(),
triggerManager: trigger.NewManager(0),
}
os.Exit(m.Run())
}
func TestInit(t *testing.T) {
assert.Nil(t, GlobalController.Init(make(chan struct{})))
}
func TestCreatePolicy(t *testing.T) {
_, err := GlobalController.CreatePolicy(models.ReplicationPolicy{
Trigger: &models.Trigger{
Kind: replication.TriggerKindManual,
},
})
assert.Nil(t, err)
}
func TestUpdatePolicy(t *testing.T) {
assert.Nil(t, GlobalController.UpdatePolicy(models.ReplicationPolicy{
ID: 2,
Trigger: &models.Trigger{
Kind: replication.TriggerKindManual,
},
}))
}
func TestRemovePolicy(t *testing.T) {
assert.Nil(t, GlobalController.RemovePolicy(1))
}
func TestGetPolicy(t *testing.T) {
_, err := GlobalController.GetPolicy(1)
assert.Nil(t, err)
}
func TestGetPolicies(t *testing.T) {
_, err := GlobalController.GetPolicies(models.QueryParameter{})
assert.Nil(t, err)
}
func TestReplicate(t *testing.T) {
// TODO
}
func TestGetCandidates(t *testing.T) {
policy := &models.ReplicationPolicy{
ID: 1,
Filters: []models.Filter{
{
Kind: replication.FilterItemKindTag,
Value: "*",
},
},
Trigger: &models.Trigger{
Kind: replication.TriggerKindImmediate,
},
}
sourcer := source.NewSourcer()
candidates := []models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world:release-1.0",
},
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world:latest",
},
}
metadata := map[string]interface{}{
"candidates": candidates,
}
result := getCandidates(policy, sourcer, metadata)
assert.Equal(t, 2, len(result))
policy.Filters = []models.Filter{
{
Kind: replication.FilterItemKindTag,
Value: "release-*",
},
}
result = getCandidates(policy, sourcer, metadata)
assert.Equal(t, 1, len(result))
// test label filter
test.InitDatabaseFromEnv()
policy.Filters = []models.Filter{
{
Kind: replication.FilterItemKindLabel,
Value: int64(1),
},
}
result = getCandidates(policy, sourcer, metadata)
assert.Equal(t, 0, len(result))
}
func TestBuildFilterChain(t *testing.T) {
policy := &models.ReplicationPolicy{
ID: 1,
Filters: []models.Filter{
{
Kind: replication.FilterItemKindRepository,
Value: "*",
},
{
Kind: replication.FilterItemKindTag,
Value: "*",
},
{
Kind: replication.FilterItemKindLabel,
Value: int64(1),
},
},
}
sourcer := source.NewSourcer()
chain := buildFilterChain(policy, sourcer)
assert.Equal(t, 3, len(chain.Filters()))
}
func TestGetOpUUID(t *testing.T) {
uuid, err := getOpUUID()
assert.Nil(t, err)
assert.NotEmpty(t, uuid)
uuid, err = getOpUUID(map[string]interface{}{
"name": "test",
})
assert.Nil(t, err)
assert.NotEmpty(t, uuid)
uuid, err = getOpUUID(map[string]interface{}{
"op_uuid": 0,
})
assert.NotNil(t, err)
uuid, err = getOpUUID(map[string]interface{}{
"op_uuid": "0",
})
assert.Nil(t, err)
assert.Equal(t, uuid, "0")
}

39
src/replication/event/init.go
View File

@ -1,39 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/replication/event/topic"
)
// Subscribe related topics
func init() {
// Listen the related event topics
handlers := map[string]notifier.NotificationHandler{
topic.StartReplicationTopic: &StartReplicationHandler{},
topic.ReplicationEventTopicOnPush: &OnPushHandler{},
topic.ReplicationEventTopicOnDeletion: &OnDeletionHandler{},
}
for topic, handler := range handlers {
if err := notifier.Subscribe(topic, handler); err != nil {
log.Errorf("failed to subscribe topic %s: %v", topic, err)
continue
}
log.Debugf("topic %s is subscribed", topic)
}
}

34
src/replication/event/notification/notification.go
View File

@ -1,34 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package notification
// OnPushNotification contains the data required by this handler
type OnPushNotification struct {
// The name of the image that is being pushed
Image string
}
// OnDeletionNotification contains the data required by this handler
type OnDeletionNotification struct {
// The name of the image that is being deleted
Image string
}
// StartReplicationNotification contains data required by this handler
type StartReplicationNotification struct {
// ID of the policy
PolicyID int64
Metadata map[string]interface{}
}

48
src/replication/event/on_deletion_handler.go
View File

@ -1,48 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"errors"
"fmt"
"reflect"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/replication/event/notification"
)
// OnDeletionHandler implements the notification handler interface to handle image on push event.
type OnDeletionHandler struct{}
// Handle implements the same method of notification handler interface
func (oph *OnDeletionHandler) Handle(value interface{}) error {
if value == nil {
return errors.New("OnDeletionHandler can not handle nil value")
}
vType := reflect.TypeOf(value)
if vType.Kind() != reflect.Struct || vType.String() != "notification.OnDeletionNotification" {
return fmt.Errorf("Mismatch value type of OnDeletionHandler, expect %s but got %s", "notification.OnDeletionNotification", vType.String())
}
notification := value.(notification.OnDeletionNotification)
return checkAndTriggerReplication(notification.Image, models.RepOpDelete)
}
// IsStateful implements the same method of notification handler interface
func (oph *OnDeletionHandler) IsStateful() bool {
// Statless
return false
}

43
src/replication/event/on_deletion_handler_test.go
View File

@ -1,43 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication/event/notification"
"github.com/stretchr/testify/assert"
)
func TestHandleOfOnDeletionHandler(t *testing.T) {
dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{}
handler := &OnDeletionHandler{}
assert.NotNil(t, handler.Handle(nil))
assert.NotNil(t, handler.Handle(map[string]string{}))
assert.NotNil(t, handler.Handle(struct{}{}))
assert.Nil(t, handler.Handle(notification.OnDeletionNotification{
Image: "library/hello-world:latest",
}))
}
func TestIsStatefulOfOnDeletionHandler(t *testing.T) {
handler := &OnDeletionHandler{}
assert.False(t, handler.IsStateful())
}

91
src/replication/event/on_push_handler.go
View File

@ -1,91 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"errors"
"fmt"
"reflect"
common_models "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/event/notification"
"github.com/goharbor/harbor/src/replication/event/topic"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/trigger"
)
// OnPushHandler implements the notification handler interface to handle image on push event.
type OnPushHandler struct{}
// Handle implements the same method of notification handler interface
func (oph *OnPushHandler) Handle(value interface{}) error {
if value == nil {
return errors.New("OnPushHandler can not handle nil value")
}
vType := reflect.TypeOf(value)
if vType.Kind() != reflect.Struct || vType.String() != "notification.OnPushNotification" {
return fmt.Errorf("Mismatch value type of OnPushHandler, expect %s but got %s", "notification.OnPushNotification", vType.String())
}
notification := value.(notification.OnPushNotification)
return checkAndTriggerReplication(notification.Image, common_models.RepOpTransfer)
}
// IsStateful implements the same method of notification handler interface
func (oph *OnPushHandler) IsStateful() bool {
// Statless
return false
}
// checks whether replication policy is set on the resource, if is, trigger the replication
func checkAndTriggerReplication(image, operation string) error {
project, _ := utils.ParseRepository(image)
watchItems, err := trigger.DefaultWatchList.Get(project, operation)
if err != nil {
return fmt.Errorf("failed to get watch list for resource %s, operation %s: %v",
image, operation, err)
}
if len(watchItems) == 0 {
log.Debugf("no replication should be triggered for resource %s, operation %s, skip", image, operation)
return nil
}
for _, watchItem := range watchItems {
item := models.FilterItem{
Kind: replication.FilterItemKindTag,
Value: image,
Operation: operation,
}
if err := notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{
PolicyID: watchItem.PolicyID,
Metadata: map[string]interface{}{
"candidates": []models.FilterItem{item},
},
}); err != nil {
return fmt.Errorf("failed to publish replication topic for resource %s, operation %s, policy %d: %v",
image, operation, watchItem.PolicyID, err)
}
log.Infof("replication topic for resource %s, operation %s, policy %d triggered",
image, operation, watchItem.PolicyID)
}
return nil
}

43
src/replication/event/on_push_handler_test.go
View File

@ -1,43 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication/event/notification"
"github.com/stretchr/testify/assert"
)
func TestHandleOfOnPushHandler(t *testing.T) {
dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{}
handler := &OnPushHandler{}
assert.NotNil(t, handler.Handle(nil))
assert.NotNil(t, handler.Handle(map[string]string{}))
assert.NotNil(t, handler.Handle(struct{}{}))
assert.Nil(t, handler.Handle(notification.OnPushNotification{
Image: "library/hello-world:latest",
}))
}
func TestIsStatefulOfOnPushHandler(t *testing.T) {
handler := &OnPushHandler{}
assert.False(t, handler.IsStateful())
}

53
src/replication/event/start_replication_handler.go
View File

@ -1,53 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"errors"
"fmt"
"reflect"
"github.com/goharbor/harbor/src/replication/core"
"github.com/goharbor/harbor/src/replication/event/notification"
)
// StartReplicationHandler implements the notification handler interface to handle start replication requests.
type StartReplicationHandler struct{}
// Handle implements the same method of notification handler interface
func (srh *StartReplicationHandler) Handle(value interface{}) error {
if value == nil {
return errors.New("StartReplicationHandler can not handle nil value")
}
vType := reflect.TypeOf(value)
if vType.Kind() != reflect.Struct || vType.String() != "notification.StartReplicationNotification" {
return fmt.Errorf("Mismatch value type of StartReplicationHandler, expect %s but got %s", "notification.StartReplicationNotification", vType.String())
}
notification := value.(notification.StartReplicationNotification)
if notification.PolicyID <= 0 {
return errors.New("Invalid policy")
}
// Start replication
return core.GlobalController.Replicate(notification.PolicyID, notification.Metadata)
}
// IsStateful implements the same method of notification handler interface
func (srh *StartReplicationHandler) IsStateful() bool {
// Stateless
return false
}

45
src/replication/event/start_replication_handler_test.go
View File

@ -1,45 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"testing"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication/core"
"github.com/goharbor/harbor/src/replication/event/notification"
"github.com/stretchr/testify/assert"
)
func TestHandle(t *testing.T) {
core.GlobalController = &test.FakeReplicatoinController{}
handler := &StartReplicationHandler{}
assert.NotNil(t, handler.Handle(nil))
assert.NotNil(t, handler.Handle(map[string]string{}))
assert.NotNil(t, handler.Handle(struct{}{}))
assert.NotNil(t, handler.Handle(notification.StartReplicationNotification{
PolicyID: -1,
}))
assert.Nil(t, handler.Handle(notification.StartReplicationNotification{
PolicyID: 1,
}))
}
func TestIsStateful(t *testing.T) {
handler := &StartReplicationHandler{}
assert.False(t, handler.IsStateful())
}

12
src/replication/event/topic/topics.go
View File

@ -1,12 +0,0 @@
package topic
const (
// ReplicationEventTopicOnPush : OnPush event
ReplicationEventTopicOnPush = "OnPush"
// ReplicationEventTopicOnDeletion : OnDeletion event
ReplicationEventTopicOnDeletion = "OnDeletion"
// StartReplicationTopic : Start application request
StartReplicationTopic = "StartReplication"
)

73
src/replication/models/filter.go
View File

@ -1,73 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"fmt"
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/replication"
)
// Filter is the data model represents the filter defined by user.
type Filter struct {
Kind string `json:"kind"`
Pattern string `json:"pattern"` // deprecated, use Value instead
Value interface{} `json:"value"`
}
// Valid ...
func (f *Filter) Valid(v *validation.Validation) {
switch f.Kind {
case replication.FilterItemKindProject,
replication.FilterItemKindRepository,
replication.FilterItemKindTag:
if f.Value == nil {
// check the Filter.Pattern if the Filter.Value is nil for compatibility
if len(f.Pattern) == 0 {
v.SetError("value", "the value can not be empty")
}
return
}
pattern, ok := f.Value.(string)
if !ok {
v.SetError("value", "the type of value should be string for project, repository and image filter")
return
}
if len(pattern) == 0 {
v.SetError("value", "the value can not be empty")
return
}
case replication.FilterItemKindLabel:
if f.Value == nil {
v.SetError("value", "the value can not be empty")
return
}
labelID, ok := f.Value.(float64)
i := int64(labelID)
if !ok || float64(i) != labelID {
v.SetError("value", "the type of value should be integer for label filter")
return
}
if i <= 0 {
v.SetError("value", fmt.Sprintf("invalid label ID: %d", i))
return
}
f.Value = i
default:
v.SetError("kind", fmt.Sprintf("invalid filter kind: %s", f.Kind))
return
}
}

7
src/replication/models/filter_config.go
View File

@ -1,7 +0,0 @@
package models
// FilterConfig is data model to provide configurations to the filters.
type FilterConfig struct {
// The pattern for fuzzy matching
Pattern string
}

35
src/replication/models/filter_item.go
View File

@ -1,35 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
// FilterItem is the general data model represents the filtering resources which are used as input and output for the filters.
type FilterItem struct {
// The kind of the filtering resources. Support 'project','repository' and 'tag' etc.
Kind string `json:"kind"`
// The key value of resource which can be used to filter out the resource matched with specified pattern.
// E.g:
// kind == 'project', value will be project name;
// kind == 'repository', value will be repository name
// kind == 'tag', value will be tag name.
Value string `json:"value"`
Operation string `json:"operation"`
// Extension placeholder.
// To append more additional information if required by the filter.
Metadata map[string]interface{} `json:"metadata"`
}

68
src/replication/models/filter_test.go
View File

@ -1,68 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"testing"
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/replication"
"github.com/stretchr/testify/assert"
)
func TestValid(t *testing.T) {
cases := map[*Filter]bool{
{}: true,
{
Kind: "invalid_kind",
}: true,
{
Kind: replication.FilterItemKindRepository,
}: true,
{
Kind: replication.FilterItemKindRepository,
Pattern: "*",
}: false,
{
Kind: replication.FilterItemKindRepository,
Value: "*",
}: false,
{
Kind: replication.FilterItemKindLabel,
}: true,
{
Kind: replication.FilterItemKindLabel,
Value: "",
}: true,
{
Kind: replication.FilterItemKindLabel,
Value: 1.2,
}: true,
{
Kind: replication.FilterItemKindLabel,
Value: -1,
}: true,
{
Kind: replication.FilterItemKindLabel,
Value: 1,
}: true,
}
for filter, hasError := range cases {
v := &validation.Validation{}
filter.Valid(v)
assert.Equal(t, hasError, v.HasErrors())
}
}

55
src/replication/models/policy.go
View File

@ -1,55 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"time"
)
// ReplicationPolicy defines the structure of a replication policy.
type ReplicationPolicy struct {
ID int64 // UUID of the policy
Name string
Description string
Filters []Filter
ReplicateDeletion bool
Trigger *Trigger // The trigger of the replication
ProjectIDs []int64 // Projects attached to this policy
TargetIDs []int64
Namespaces []string // The namespaces are used to set immediate trigger
CreationTime time.Time
UpdateTime time.Time
}
// QueryParameter defines the parameters used to do query selection.
type QueryParameter struct {
// Query by page, couple with pageSize
Page int64
// Size of each page, couple with page
PageSize int64
// Query by project ID
ProjectID int64
// Query by name
Name string
}
// ReplicationPolicyQueryResult is the query result of replication policy
type ReplicationPolicyQueryResult struct {
Total int64
Policies []*ReplicationPolicy
}

34
src/replication/models/registry_models.go
View File

@ -1,34 +0,0 @@
package models
// Namespace is the resource group/scope like project in Harbor and organization in docker hub.
type Namespace struct {
// Name of the namespace
Name string
// Extensions to provide flexibility
Metadata map[string]interface{}
}
// Repository is to keep the info of image repository.
type Repository struct {
// Name of the repository
Name string
// Project reference of this repository belongs to
Namespace Namespace
// Extensions to provide flexibility
Metadata map[string]interface{}
}
// Tag keeps the info of image with specified version
type Tag struct {
// Name of the tag
Name string
// The repository reference of this tag belongs to
Repository Repository
// Extensions to provide flexibility
Metadata map[string]interface{}
}

79
src/replication/models/trigger.go
View File

@ -1,79 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"fmt"
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/replication"
)
// Trigger is replication launching approach definition
type Trigger struct {
Kind string `json:"kind"` // the type of the trigger
ScheduleParam *ScheduleParam `json:"schedule_param"` // optional, only used when kind is 'schedule'
}
// Valid ...
func (t *Trigger) Valid(v *validation.Validation) {
if !(t.Kind == replication.TriggerKindImmediate ||
t.Kind == replication.TriggerKindManual ||
t.Kind == replication.TriggerKindSchedule) {
v.SetError("kind", fmt.Sprintf("invalid trigger kind: %s", t.Kind))
}
if t.Kind == replication.TriggerKindSchedule {
if t.ScheduleParam == nil {
v.SetError("schedule_param", "empty schedule_param")
} else {
t.ScheduleParam.Valid(v)
}
}
}
// ScheduleParam defines the parameters used by schedule trigger
type ScheduleParam struct {
Type string `json:"type"` // daily or weekly
Weekday int8 `json:"weekday"` // Optional, only used when type is 'weekly'
Offtime int64 `json:"offtime"` // The time offset with the UTC 00:00 in seconds
}
// Valid ...
func (s *ScheduleParam) Valid(v *validation.Validation) {
if !(s.Type == replication.TriggerScheduleDaily ||
s.Type == replication.TriggerScheduleWeekly) {
v.SetError("type", fmt.Sprintf("invalid schedule trigger parameter type: %s", s.Type))
}
if s.Type == replication.TriggerScheduleWeekly {
if s.Weekday < 1 || s.Weekday > 7 {
v.SetError("weekday", fmt.Sprintf("invalid schedule trigger parameter weekday: %d", s.Weekday))
}
}
if s.Offtime < 0 || s.Offtime > 3600*24 {
v.SetError("offtime", fmt.Sprintf("invalid schedule trigger parameter offtime: %d", s.Offtime))
}
}
// Equal ...
func (s *ScheduleParam) Equal(param *ScheduleParam) bool {
if param == nil {
return false
}
return s.Type == param.Type && s.Weekday == param.Weekday && s.Offtime == param.Offtime
}

77
src/replication/models/trigger_test.go
View File

@ -1,77 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"testing"
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/replication"
"github.com/stretchr/testify/assert"
)
func TestValidOfTrigger(t *testing.T) {
cases := map[*Trigger]bool{
{}: true,
{
Kind: "invalid_kind",
}: true,
{
Kind: replication.TriggerKindImmediate,
}: false,
{
Kind: replication.TriggerKindSchedule,
}: true,
}
for filter, hasError := range cases {
v := &validation.Validation{}
filter.Valid(v)
assert.Equal(t, hasError, v.HasErrors())
}
}
func TestValidOfScheduleParam(t *testing.T) {
cases := map[*ScheduleParam]bool{
{}: true,
{
Type: "invalid_type",
}: true,
{
Type: replication.TriggerScheduleDaily,
Offtime: 3600*24 + 1,
}: true,
{
Type: replication.TriggerScheduleDaily,
Offtime: 3600 * 2,
}: false,
{
Type: replication.TriggerScheduleWeekly,
Weekday: 0,
Offtime: 3600 * 2,
}: true,
{
Type: replication.TriggerScheduleWeekly,
Weekday: 7,
Offtime: 3600 * 2,
}: false,
}
for param, hasError := range cases {
v := &validation.Validation{}
param.Valid(v)
assert.Equal(t, hasError, v.HasErrors())
}
}

205
src/replication/policy/manager.go
View File

@ -1,205 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package policy
import (
"encoding/json"
"time"
"github.com/goharbor/harbor/src/common/dao"
persist_models "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
)
// Manager defines the method a policy manger should implement
type Manager interface {
GetPolicies(models.QueryParameter) (*models.ReplicationPolicyQueryResult, error)
GetPolicy(int64) (models.ReplicationPolicy, error)
CreatePolicy(models.ReplicationPolicy) (int64, error)
UpdatePolicy(models.ReplicationPolicy) error
RemovePolicy(int64) error
}
// DefaultManager provides replication policy CURD capabilities.
type DefaultManager struct{}
// NewDefaultManager is the constructor of DefaultManager.
func NewDefaultManager() *DefaultManager {
return &DefaultManager{}
}
// GetPolicies returns all the policies
func (m *DefaultManager) GetPolicies(query models.QueryParameter) (*models.ReplicationPolicyQueryResult, error) {
result := &models.ReplicationPolicyQueryResult{
Policies: []*models.ReplicationPolicy{},
}
total, err := dao.GetTotalOfRepPolicies(query.Name, query.ProjectID)
if err != nil {
return nil, err
}
result.Total = total
policies, err := dao.FilterRepPolicies(query.Name, query.ProjectID, query.Page, query.PageSize)
if err != nil {
return nil, err
}
for _, policy := range policies {
ply, err := convertFromPersistModel(policy)
if err != nil {
return nil, err
}
result.Policies = append(result.Policies, &ply)
}
return result, nil
}
// GetPolicy returns the policy with the specified ID
func (m *DefaultManager) GetPolicy(policyID int64) (models.ReplicationPolicy, error) {
policy, err := dao.GetRepPolicy(policyID)
if err != nil {
return models.ReplicationPolicy{}, err
}
return convertFromPersistModel(policy)
}
func convertFromPersistModel(policy *persist_models.RepPolicy) (models.ReplicationPolicy, error) {
if policy == nil {
return models.ReplicationPolicy{}, nil
}
ply := models.ReplicationPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
ReplicateDeletion: policy.ReplicateDeletion,
ProjectIDs: []int64{policy.ProjectID},
TargetIDs: []int64{policy.TargetID},
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
project, err := config.GlobalProjectMgr.Get(policy.ProjectID)
if err != nil {
return models.ReplicationPolicy{}, err
}
ply.Namespaces = []string{project.Name}
if len(policy.Filters) > 0 {
filters := []models.Filter{}
if err := json.Unmarshal([]byte(policy.Filters), &filters); err != nil {
return models.ReplicationPolicy{}, err
}
for i := range filters {
if filters[i].Value == nil && len(filters[i].Pattern) > 0 {
filters[i].Value = filters[i].Pattern
}
// convert the type of Value to int64 as the default type of
// json Unmarshal for number is float64
if filters[i].Kind == replication.FilterItemKindLabel {
filters[i].Value = int64(filters[i].Value.(float64))
}
}
ply.Filters = filters
}
if len(policy.Trigger) > 0 {
trigger := &models.Trigger{}
if err := json.Unmarshal([]byte(policy.Trigger), trigger); err != nil {
return models.ReplicationPolicy{}, err
}
ply.Trigger = trigger
}
return ply, nil
}
func convertToPersistModel(policy models.ReplicationPolicy) (*persist_models.RepPolicy, error) {
ply := &persist_models.RepPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
ReplicateDeletion: policy.ReplicateDeletion,
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
if len(policy.ProjectIDs) > 0 {
ply.ProjectID = policy.ProjectIDs[0]
}
if len(policy.TargetIDs) > 0 {
ply.TargetID = policy.TargetIDs[0]
}
if policy.Trigger != nil {
trigger, err := json.Marshal(policy.Trigger)
if err != nil {
return nil, err
}
ply.Trigger = string(trigger)
}
if len(policy.Filters) > 0 {
filters, err := json.Marshal(policy.Filters)
if err != nil {
return nil, err
}
ply.Filters = string(filters)
}
return ply, nil
}
// CreatePolicy creates a new policy with the provided data;
// If creating failed, error will be returned;
// If creating succeed, ID of the new created policy will be returned.
func (m *DefaultManager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) {
now := time.Now()
policy.CreationTime = now
policy.UpdateTime = now
ply, err := convertToPersistModel(policy)
if err != nil {
return 0, err
}
return dao.AddRepPolicy(*ply)
}
// UpdatePolicy updates the policy;
// If updating failed, error will be returned.
func (m *DefaultManager) UpdatePolicy(policy models.ReplicationPolicy) error {
policy.UpdateTime = time.Now()
ply, err := convertToPersistModel(policy)
if err != nil {
return err
}
return dao.UpdateRepPolicy(ply)
}
// RemovePolicy removes the specified policy;
// If removing failed, error will be returned.
func (m *DefaultManager) RemovePolicy(policyID int64) error {
// delete replication jobs
if err := dao.DeleteRepJobs(policyID); err != nil {
return err
}
// delete the replication policy
return dao.DeleteRepPolicy(policyID)
}

60
src/replication/policy/manager_test.go
View File

@ -1,60 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package policy
import (
"encoding/json"
"testing"
"github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvertToPersistModel(t *testing.T) {
var id, projectID, targetID int64 = 1, 1, 1
name := "policy01"
replicateDeletion := true
trigger := &models.Trigger{
Kind: "trigger_kind",
}
filters := []models.Filter{
{
Kind: "filter_kind",
Pattern: "filter_pattern",
},
}
policy := models.ReplicationPolicy{
ID: id,
Name: name,
ReplicateDeletion: replicateDeletion,
ProjectIDs: []int64{projectID},
TargetIDs: []int64{targetID},
Trigger: trigger,
Filters: filters,
}
ply, err := convertToPersistModel(policy)
require.Nil(t, err)
assert.Equal(t, id, ply.ID)
assert.Equal(t, name, ply.Name)
assert.Equal(t, replicateDeletion, ply.ReplicateDeletion)
assert.Equal(t, projectID, ply.ProjectID)
assert.Equal(t, targetID, ply.TargetID)
tg, _ := json.Marshal(trigger)
assert.Equal(t, string(tg), ply.Trigger)
ft, _ := json.Marshal(filters)
assert.Equal(t, string(ft), ply.Filters)
}

34
src/replication/registry/adaptor.go
View File

@ -1,34 +0,0 @@
package registry
import (
"github.com/goharbor/harbor/src/replication/models"
)
// Adaptor defines the unified operations for all the supported registries such as Harbor or DockerHub.
// It's used to adapt the different interfaces provied by the different registry providers.
// Use external registry with restful api providing as example, these intrefaces may depends on the
// related restful apis like:
// /api/vx/repositories/{namespace}/{repositoryName}/tags/{name}
// /api/v0/accounts/{namespace}
type Adaptor interface {
// Return the unique kind identifier of the adaptor
Kind() string
// Get all the namespaces
GetNamespaces() []models.Namespace
// Get the namespace with the specified name
GetNamespace(name string) models.Namespace
// Get all the repositories under the specified namespace
GetRepositories(namespace string) []models.Repository
// Get the repository with the specified name under the specified namespace
GetRepository(name string, namespace string) models.Repository
// Get all the tags of the specified repository under the namespace
GetTags(repositoryName string, namespace string) []models.Tag
// Get the tag with the specified name of the repository under the namespace
GetTag(name string, repositoryName string, namespace string) models.Tag
}

83
src/replication/registry/harbor_adaptor.go
View File

@ -1,83 +0,0 @@
package registry
import (
"github.com/goharbor/harbor/src/common/dao"
common_models "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
)
// TODO refacotor the methods of HarborAdaptor by caling Harbor's API
// HarborAdaptor is defined to adapt the Harbor registry
type HarborAdaptor struct{}
// Kind returns the unique kind identifier of the adaptor
func (ha *HarborAdaptor) Kind() string {
return replication.AdaptorKindHarbor
}
// GetNamespaces is ued to get all the namespaces
func (ha *HarborAdaptor) GetNamespaces() []models.Namespace {
return nil
}
// GetNamespace is used to get the namespace with the specified name
func (ha *HarborAdaptor) GetNamespace(name string) models.Namespace {
return models.Namespace{}
}
// GetRepositories is used to get all the repositories under the specified namespace
func (ha *HarborAdaptor) GetRepositories(namespace string) []models.Repository {
repos, err := dao.GetRepositories(&common_models.RepositoryQuery{
ProjectName: namespace,
})
if err != nil {
log.Errorf("failed to get repositories under namespace %s: %v", namespace, err)
return nil
}
repositories := []models.Repository{}
for _, repo := range repos {
repositories = append(repositories, models.Repository{
Name: repo.Name,
})
}
return repositories
}
// GetRepository is used to get the repository with the specified name under the specified namespace
func (ha *HarborAdaptor) GetRepository(name string, namespace string) models.Repository {
return models.Repository{}
}
// GetTags is used to get all the tags of the specified repository under the namespace
func (ha *HarborAdaptor) GetTags(repositoryName string, namespace string) []models.Tag {
client, err := utils.NewRepositoryClientForUI("harbor-core", repositoryName)
if err != nil {
log.Errorf("failed to create registry client: %v", err)
return nil
}
ts, err := client.ListTag()
if err != nil {
log.Errorf("failed to get tags of repository %s: %v", repositoryName, err)
return nil
}
tags := []models.Tag{}
for _, t := range ts {
tags = append(tags, models.Tag{
Name: t,
})
}
return tags
}
// GetTag is used to get the tag with the specified name of the repository under the namespace
func (ha *HarborAdaptor) GetTag(name string, repositoryName string, namespace string) models.Tag {
return models.Tag{}
}

137
src/replication/replicator/replicator.go
View File

@ -1,137 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replicator
import (
"fmt"
"strings"
"github.com/goharbor/harbor/src/common/dao"
common_job "github.com/goharbor/harbor/src/common/job"
job_models "github.com/goharbor/harbor/src/common/job/models"
common_models "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/ng/model"
)
// Replication holds information for a replication
type Replication struct {
PolicyID int64
OpUUID string
Candidates []models.FilterItem
Registries []*model.Registry
Operation string
}
// Replicator submits the replication work to the jobservice
type Replicator interface {
Replicate(*Replication) error
}
// DefaultReplicator provides a default implement for Replicator
type DefaultReplicator struct {
client common_job.Client
}
// NewDefaultReplicator returns an instance of DefaultReplicator
func NewDefaultReplicator(client common_job.Client) *DefaultReplicator {
return &DefaultReplicator{
client: client,
}
}
// Replicate ...
func (d *DefaultReplicator) Replicate(replication *Replication) error {
repositories := map[string][]string{}
// TODO the operation of all candidates are same for now. Update it after supporting
// replicate deletion
operation := ""
for _, candidate := range replication.Candidates {
strs := strings.SplitN(candidate.Value, ":", 2)
if len(strs) != 2 {
return fmt.Errorf("malforld image '%s'", candidate.Value)
}
repositories[strs[0]] = append(repositories[strs[0]], strs[1])
operation = candidate.Operation
}
for _, registry := range replication.Registries {
for repository, tags := range repositories {
// create job in database
id, err := dao.AddRepJob(common_models.RepJob{
PolicyID: replication.PolicyID,
OpUUID: replication.OpUUID,
Repository: repository,
TagList: tags,
Operation: operation,
})
if err != nil {
return err
}
// submit job to jobservice
log.Debugf("submiting replication job to jobservice, repository: %s, tags: %v, operation: %s, target: %s",
repository, tags, operation, registry.URL)
job := &job_models.JobData{
Metadata: &job_models.JobMetadata{
JobKind: common_job.JobKindGeneric,
},
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/replication/%d",
config.InternalCoreURL(), id),
}
if operation == common_models.RepOpTransfer {
job.Name = common_job.ImageTransfer
job.Parameters = map[string]interface{}{
"repository": repository,
"tags": tags,
"src_registry_url": config.InternalCoreURL(),
"src_registry_insecure": false,
"src_token_service_url": config.InternalTokenServiceEndpoint(),
"dst_registry_url": registry.URL,
"dst_registry_insecure": registry.Insecure,
"dst_registry_username": registry.Credential.AccessKey,
"dst_registry_password": registry.Credential.AccessSecret,
}
} else {
job.Name = common_job.ImageDelete
job.Parameters = map[string]interface{}{
"repository": repository,
"tags": tags,
"dst_registry_url": registry.URL,
"dst_registry_insecure": registry.Insecure,
"dst_registry_username": registry.Credential.AccessKey,
"dst_registry_password": registry.Credential.AccessSecret,
}
}
uuid, err := d.client.SubmitJob(job)
if err != nil {
if er := dao.UpdateRepJobStatus(id, common_models.JobError); er != nil {
log.Errorf("failed to update the status of job %d: %s", id, er)
}
return err
}
// create the mapping relationship between the jobs in database and jobservice
if err = dao.SetRepJobUUID(id, uuid); err != nil {
return err
}
}
}
return nil
}

23
src/replication/replicator/replicator_test.go
View File

@ -1,23 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replicator
import (
"testing"
)
func TestNewDefaultReplicator(t *testing.T) {
NewDefaultReplicator(nil)
}

16
src/replication/source/convertor.go
View File

@ -1,16 +0,0 @@
package source
import (
"github.com/goharbor/harbor/src/replication/models"
)
// Converter is designed to covert the format of output from upstream filter to the input format
// required by the downstream filter if needed.
// Each converter covers only one specified conversion process between the two filters.
// E.g:
// If project filter connects to repository filter, then one converter should be defined for this connection;
// If project filter connects to tag filter, then another one should be defined. The above one can not be reused.
type Converter interface {
// Accept the items from upstream filter as input and then covert them to the required format and returned.
Convert(itemsOfUpstream []models.FilterItem) (itemsOfDownstream []models.FilterItem)
}

57
src/replication/source/default_filter_chain.go
View File

@ -1,57 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"github.com/goharbor/harbor/src/replication/models"
)
// DefaultFilterChain provides a default implement for interface FilterChain
type DefaultFilterChain struct {
filters []Filter
}
// NewDefaultFilterChain returns an instance of DefaultFilterChain
func NewDefaultFilterChain(filters []Filter) *DefaultFilterChain {
return &DefaultFilterChain{
filters: filters,
}
}
// Build nil implement now
func (d *DefaultFilterChain) Build(filters []Filter) error {
return nil
}
// Filters returns the filter list
func (d *DefaultFilterChain) Filters() []Filter {
return d.filters
}
// DoFilter does the filter works for filterItems
func (d *DefaultFilterChain) DoFilter(filterItems []models.FilterItem) []models.FilterItem {
if len(filterItems) == 0 {
return []models.FilterItem{}
}
for _, filter := range d.filters {
converter := filter.GetConverter()
if converter != nil {
filterItems = converter.Convert(filterItems)
}
filterItems = filter.DoFilter(filterItems)
}
return filterItems
}

75
src/replication/source/default_filter_chain_test.go
View File

@ -1,75 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"testing"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBuild(t *testing.T) {
chain := NewDefaultFilterChain(nil)
require.Nil(t, chain.Build(nil))
}
func TestFilters(t *testing.T) {
filters := []Filter{NewPatternFilter("project", "*")}
chain := NewDefaultFilterChain(filters)
assert.EqualValues(t, filters, chain.Filters())
}
func TestDoFilter(t *testing.T) {
projectFilter := NewPatternFilter(replication.FilterItemKindProject, "library*")
repositoryFilter := NewPatternFilter(replication.FilterItemKindRepository,
"library/ubuntu*", &fakeRepositoryConverter{})
filters := []Filter{projectFilter, repositoryFilter}
items := []models.FilterItem{
{
Kind: replication.FilterItemKindProject,
Value: "library",
},
{
Kind: replication.FilterItemKindProject,
Value: "test",
},
}
chain := NewDefaultFilterChain(filters)
items = chain.DoFilter(items)
assert.EqualValues(t, []models.FilterItem{
{
Kind: replication.FilterItemKindRepository,
Value: "library/ubuntu",
},
}, items)
}
type fakeRepositoryConverter struct{}
func (f *fakeRepositoryConverter) Convert(items []models.FilterItem) []models.FilterItem {
result := []models.FilterItem{}
for _, item := range items {
result = append(result, models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: item.Value + "/ubuntu",
})
}
return result
}

18
src/replication/source/filter.go
View File

@ -1,18 +0,0 @@
package source
import (
"github.com/goharbor/harbor/src/replication/models"
)
// Filter define the operations of selecting the matched resources from the candidates
// according to the specified pattern.
type Filter interface {
// Initialize the filter
Init() error
// Return the converter if existing or nil if never set
GetConverter() Converter
// Filter the items
DoFilter(filterItems []models.FilterItem) []models.FilterItem
}

21
src/replication/source/filter_chain.go
View File

@ -1,21 +0,0 @@
package source
import (
"github.com/goharbor/harbor/src/replication/models"
)
// FilterChain is the interface to define the operations of coordinating multiple filters
// to work together as a whole pipeline.
// E.g:
// (original resources)---->[project filter]---->[repository filter]---->[tag filter]---->[......]---->(filter resources)
type FilterChain interface {
// Build the filter chain with the filters provided;
// if failed, an error will be returned.
Build(filter []Filter) error
// Return all the filters in the chain.
Filters() []Filter
// Filter the items and returned the filtered items via the appended filters in the chain.
DoFilter(filterItems []models.FilterItem) []models.FilterItem
}

88
src/replication/source/label_filter.go
View File

@ -1,88 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"fmt"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
)
// LabelFilter filter resources according to label
type LabelFilter struct {
labelID int64
}
// Init ...
func (l *LabelFilter) Init() error {
return nil
}
// GetConverter ...
func (l *LabelFilter) GetConverter() Converter {
return nil
}
// NewLabelFilter returns an instance of LabelFilter
func NewLabelFilter(labelID int64) *LabelFilter {
return &LabelFilter{
labelID: labelID,
}
}
// DoFilter filter the resources according to the label
func (l *LabelFilter) DoFilter(items []models.FilterItem) []models.FilterItem {
candidates := []string{}
for _, item := range items {
candidates = append(candidates, item.Value)
}
log.Debugf("label filter candidates: %v", candidates)
result := []models.FilterItem{}
for _, item := range items {
hasLabel, err := hasLabel(item, l.labelID)
if err != nil {
log.Errorf("failed to check the label of resouce %v: %v, skip it", item, err)
continue
}
if hasLabel {
log.Debugf("has label %d, add %s to the label filter result list", l.labelID, item.Value)
result = append(result, item)
}
}
return result
}
func hasLabel(resource models.FilterItem, labelID int64) (bool, error) {
rType := ""
switch resource.Kind {
case replication.FilterItemKindProject:
rType = common.ResourceTypeProject
case replication.FilterItemKindRepository:
rType = common.ResourceTypeRepository
case replication.FilterItemKindTag:
rType = common.ResourceTypeImage
default:
return false, fmt.Errorf("invalid resource type: %s", resource.Kind)
}
rl, err := dao.GetResourceLabel(rType, resource.Value, labelID)
if err != nil {
return false, err
}
return rl != nil, nil
}

48
src/replication/source/label_filter_test.go
View File

@ -1,48 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"testing"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
)
func TestInitOfLabelFilter(t *testing.T) {
filter := NewLabelFilter(1)
assert.Nil(t, filter.Init())
}
func TestGetConverterOfLabelFilter(t *testing.T) {
filter := NewLabelFilter(1)
assert.Nil(t, filter.GetConverter())
}
func TestDoFilterOfLabelFilter(t *testing.T) {
test.InitDatabaseFromEnv()
filter := NewLabelFilter(1)
items := []models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world:latest",
},
}
result := filter.DoFilter(items)
assert.Equal(t, 0, len(result))
}

23
src/replication/source/match.go
View File

@ -1,23 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"path/filepath"
)
func match(pattern, str string) (bool, error) {
return filepath.Match(pattern, str)
}

43
src/replication/source/match_test.go
View File

@ -1,43 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMatch(t *testing.T) {
cases := []struct {
pattern string
str string
matched bool
}{
{"", "", true},
{"*", "library", true},
{"library/*", "library/mysql", true},
{"library/*", "library/mysql/5.6", false},
{"library/mysq?", "library/mysql", true},
{"library/mysq?", "library/mysqld", false},
}
for _, c := range cases {
matched, err := match(c.pattern, c.str)
require.Nil(t, err)
assert.Equal(t, c.matched, matched)
}
}

84
src/replication/source/pattern_filter.go
View File

@ -1,84 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"regexp"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/models"
)
// PatternFilter implements Filter interface for pattern filter
type PatternFilter struct {
kind string
pattern string
converter Converter
}
// NewPatternFilter returns an instance of PatternFilter
func NewPatternFilter(kind, pattern string, converter ...Converter) *PatternFilter {
filer := &PatternFilter{
kind: kind,
pattern: pattern,
}
if len(converter) > 0 {
filer.converter = converter[0]
}
return filer
}
// Init the filter. nil implement for now
func (p *PatternFilter) Init() error {
return nil
}
// GetConverter returns the converter
func (p *PatternFilter) GetConverter() Converter {
return p.converter
}
// DoFilter filters resources
func (p *PatternFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem {
items := []models.FilterItem{}
for _, item := range filterItems {
if item.Kind != p.kind {
log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip",
p.kind, item.Kind)
continue
}
matched, err := regexp.MatchString(p.pattern, item.Value)
if err != nil {
log.Errorf("failed to match pattern %s, value %s: %v, skip",
p.pattern, item.Value, err)
continue
}
if !matched {
log.Debugf("%s does not match to the %s filter %s, skip",
item.Value, p.kind, p.pattern)
continue
}
log.Debugf("add %s to the result of %s filter %s",
item.Value, p.kind, p.pattern)
items = append(items, item)
}
return items
}

63
src/replication/source/pattern_filter_test.go
View File

@ -1,63 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
"testing"
)
var pfilter = NewPatternFilter(replication.FilterItemKindTag, "library/ubuntu:release-*", nil)
func TestPatternFilterInit(t *testing.T) {
assert.Nil(t, pfilter.Init())
}
func TestPatternFilterGetConverter(t *testing.T) {
assert.Nil(t, pfilter.GetConverter())
}
func TestPatternFilterDoFilter(t *testing.T) {
items := []models.FilterItem{
{
Kind: replication.FilterItemKindProject,
},
{
Kind: replication.FilterItemKindRepository,
},
{
Kind: replication.FilterItemKindTag,
Value: "library/ubuntu:release-14.04",
},
{
Kind: replication.FilterItemKindTag,
Value: "library/ubuntu:release-16.04",
},
{
Kind: replication.FilterItemKindTag,
Value: "library/ubuntu:test",
},
}
result := pfilter.DoFilter(items)
assert.Equal(t, 2, len(result))
assert.Equal(t, replication.FilterItemKindTag, result[0].Kind)
assert.Equal(t, "library/ubuntu:release-14.04", result[0].Value)
assert.Equal(t, replication.FilterItemKindTag, result[1].Kind)
assert.Equal(t, "library/ubuntu:release-16.04", result[1].Value)
}

55
src/replication/source/repository_convertor.go
View File

@ -1,55 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/registry"
)
// RepositoryConverter implement Converter interface, convert projects to repositories
type RepositoryConverter struct {
registry registry.Adaptor
}
// NewRepositoryConverter returns an instance of RepositoryConverter
func NewRepositoryConverter(registry registry.Adaptor) *RepositoryConverter {
return &RepositoryConverter{
registry: registry,
}
}
// Convert projects to repositories
func (r *RepositoryConverter) Convert(items []models.FilterItem) []models.FilterItem {
result := []models.FilterItem{}
for _, item := range items {
// just put it to the result list if the item is not a project
if item.Kind != replication.FilterItemKindProject {
result = append(result, item)
continue
}
repositories := r.registry.GetRepositories(item.Value)
for _, repository := range repositories {
result = append(result, models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: repository.Name,
Operation: item.Operation,
})
}
}
return result
}

95
src/replication/source/repository_convertor_test.go
View File

@ -1,95 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"testing"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
)
func TestRepositoryConvert(t *testing.T) {
items := []models.FilterItem{
{
Kind: replication.FilterItemKindProject,
Value: "library",
},
{
Kind: replication.FilterItemKindRepository,
},
}
expected := []models.FilterItem{
{
Kind: replication.FilterItemKindRepository,
Value: "library/ubuntu",
},
{
Kind: replication.FilterItemKindRepository,
Value: "library/centos",
},
{
Kind: replication.FilterItemKindRepository,
},
}
converter := NewRepositoryConverter(&fakeRegistryAdaptor{})
assert.EqualValues(t, expected, converter.Convert(items))
}
type fakeRegistryAdaptor struct{}
func (f *fakeRegistryAdaptor) Kind() string {
return "fake"
}
func (f *fakeRegistryAdaptor) GetNamespaces() []models.Namespace {
return nil
}
func (f *fakeRegistryAdaptor) GetNamespace(name string) models.Namespace {
return models.Namespace{}
}
func (f *fakeRegistryAdaptor) GetRepositories(namespace string) []models.Repository {
return []models.Repository{
{
Name: "library/ubuntu",
},
{
Name: "library/centos",
},
}
}
func (f *fakeRegistryAdaptor) GetRepository(name string, namespace string) models.Repository {
return models.Repository{}
}
func (f *fakeRegistryAdaptor) GetTags(repositoryName string, namespace string) []models.Tag {
return []models.Tag{
{
Name: "14.04",
},
{
Name: "16.04",
},
}
}
func (f *fakeRegistryAdaptor) GetTag(name string, repositoryName string, namespace string) models.Tag {
return models.Tag{}
}

89
src/replication/source/repository_filter.go
View File

@ -1,89 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"strings"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/registry"
)
// RepositoryFilter implement Filter interface to filter repository
type RepositoryFilter struct {
pattern string
converter Converter
}
// NewRepositoryFilter returns an instance of RepositoryFilter
func NewRepositoryFilter(pattern string, registry registry.Adaptor) *RepositoryFilter {
return &RepositoryFilter{
pattern: pattern,
converter: NewRepositoryConverter(registry),
}
}
// Init ...
func (r *RepositoryFilter) Init() error {
return nil
}
// GetConverter ...
func (r *RepositoryFilter) GetConverter() Converter {
return r.converter
}
// DoFilter filters repository and image(according to the repository part) and drops any other resource types
func (r *RepositoryFilter) DoFilter(items []models.FilterItem) []models.FilterItem {
candidates := []string{}
for _, item := range items {
candidates = append(candidates, item.Value)
}
log.Debugf("repository filter candidates: %v", candidates)
result := []models.FilterItem{}
for _, item := range items {
if item.Kind != replication.FilterItemKindRepository && item.Kind != replication.FilterItemKindTag {
log.Warningf("unsupported type %s for repository filter, drop", item.Kind)
continue
}
repository := item.Value
if item.Kind == replication.FilterItemKindTag {
repository = strings.SplitN(repository, ":", 2)[0]
}
if len(r.pattern) == 0 {
log.Debugf("pattern is null, add %s to the repository filter result list", item.Value)
result = append(result, item)
} else {
// trim the project
_, repository = utils.ParseRepository(repository)
matched, err := match(r.pattern, repository)
if err != nil {
log.Errorf("failed to match pattern %s to value %s: %v, skip it", r.pattern, repository, err)
continue
}
if matched {
log.Debugf("pattern %s matched, add %s to the repository filter result list", r.pattern, item.Value)
result = append(result, item)
}
}
}
return result
}

75
src/replication/source/repository_filter_test.go
View File

@ -1,75 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"testing"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/registry"
"github.com/stretchr/testify/assert"
)
func TestInitOfRepositoryFilter(t *testing.T) {
filter := NewRepositoryFilter("", &registry.HarborAdaptor{})
assert.Nil(t, filter.Init())
}
func TestGetConverterOfRepositoryFilter(t *testing.T) {
filter := NewRepositoryFilter("", &registry.HarborAdaptor{})
assert.NotNil(t, filter.GetConverter())
}
func TestDoFilterOfRepositoryFilter(t *testing.T) {
// invalid filter item type
filter := NewRepositoryFilter("", &registry.HarborAdaptor{})
items := filter.DoFilter([]models.FilterItem{
{
Kind: "invalid_type",
},
})
assert.Equal(t, 0, len(items))
// empty pattern
filter = NewRepositoryFilter("", &registry.HarborAdaptor{})
items = filter.DoFilter([]models.FilterItem{
{
Kind: replication.FilterItemKindRepository,
Value: "library/hello-world",
},
})
assert.Equal(t, 1, len(items))
// non-empty pattern
filter = NewRepositoryFilter("*", &registry.HarborAdaptor{})
items = filter.DoFilter([]models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world",
},
})
assert.Equal(t, 1, len(items))
// non-empty pattern
filter = NewRepositoryFilter("*", &registry.HarborAdaptor{})
items = filter.DoFilter([]models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world:latest",
},
})
assert.Equal(t, 1, len(items))
}

36
src/replication/source/sourcer.go
View File

@ -1,36 +0,0 @@
package source
import (
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/registry"
)
// Sourcer is used to manage and/or handle all the artifacts and information related with source registry.
// All the things with replication source should be covered in this object.
type Sourcer struct {
// Keep the adaptors we support now
adaptors map[string]registry.Adaptor
}
// NewSourcer is the constructor of Sourcer
func NewSourcer() *Sourcer {
return &Sourcer{
adaptors: make(map[string]registry.Adaptor),
}
}
// Init will do some initialization work like registrying all the adaptors we support
func (sc *Sourcer) Init() {
// Register Harbor adaptor
sc.adaptors[replication.AdaptorKindHarbor] = &registry.HarborAdaptor{}
}
// GetAdaptor returns the required adaptor with the specified kind.
// If no adaptor with the specified kind existing, nil will be returned.
func (sc *Sourcer) GetAdaptor(kind string) registry.Adaptor {
if len(kind) == 0 {
return nil
}
return sc.adaptors[kind]
}

24
src/replication/source/sourcer_test.go
View File

@ -1,24 +0,0 @@
package source
import (
"testing"
"github.com/goharbor/harbor/src/replication"
)
func TestReplicationSourcer(t *testing.T) {
testingSourcer := NewSourcer()
if testingSourcer == nil {
t.Fatal("Failed to create sourcer")
}
testingSourcer.Init()
if testingSourcer.GetAdaptor("") != nil {
t.Fatal("Empty kind should not be supported")
}
if testingSourcer.GetAdaptor(replication.AdaptorKindHarbor) == nil {
t.Fatalf("%s adaptor should be existing", replication.AdaptorKindHarbor)
}
}

76
src/replication/source/tag_combination_filter.go
View File

@ -1,76 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"strings"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
)
// TagCombinationFilter implements Filter interface for merging tag filter items
// whose repository are same into one repository filter item
type TagCombinationFilter struct{}
// NewTagCombinationFilter returns an instance of TagCombinationFilter
func NewTagCombinationFilter() *TagCombinationFilter {
return &TagCombinationFilter{}
}
// Init the filter. nil implement for now
func (t *TagCombinationFilter) Init() error {
return nil
}
// GetConverter returns the converter
func (t *TagCombinationFilter) GetConverter() Converter {
return nil
}
// DoFilter filters resources
func (t *TagCombinationFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem {
repos := map[string][]string{}
for _, item := range filterItems {
if item.Kind != replication.FilterItemKindTag {
log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip",
replication.FilterItemKindTag, item.Kind)
continue
}
strs := strings.Split(item.Value, ":")
if len(strs) != 2 {
log.Warningf("unexpected image format: %s, skip", item.Value)
continue
}
repos[strs[0]] = append(repos[strs[0]], strs[1])
}
// TODO append operation
items := []models.FilterItem{}
for repo, tags := range repos {
items = append(items, models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: repo,
Metadata: map[string]interface{}{
"tags": tags,
},
})
}
return items
}

83
src/replication/source/tag_combination_filter_test.go
View File

@ -1,83 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
"testing"
)
var tcfilter = NewTagCombinationFilter()
func TestTagCombinationFilterInit(t *testing.T) {
assert.Nil(t, tcfilter.Init())
}
func TestTagCombinationFilterGetConverter(t *testing.T) {
assert.Nil(t, tcfilter.GetConverter())
}
func TestTagCombinationFilterDoFilter(t *testing.T) {
items := []models.FilterItem{
{
Kind: replication.FilterItemKindProject,
},
{
Kind: replication.FilterItemKindRepository,
},
{
Kind: replication.FilterItemKindTag,
Value: "library/ubuntu:invalid_tag:latest",
},
{
Kind: replication.FilterItemKindTag,
Value: "library/ubuntu:14.04",
},
{
Kind: replication.FilterItemKindTag,
Value: "library/ubuntu:16.04",
},
{
Kind: replication.FilterItemKindTag,
Value: "library/centos:7",
},
}
result := tcfilter.DoFilter(items)
assert.Equal(t, 2, len(result))
var ubuntu, centos models.FilterItem
if result[0].Value == "library/ubuntu" {
ubuntu = result[0]
centos = result[1]
} else {
centos = result[0]
ubuntu = result[1]
}
assert.Equal(t, replication.FilterItemKindRepository, ubuntu.Kind)
assert.Equal(t, "library/ubuntu", ubuntu.Value)
metadata, ok := ubuntu.Metadata["tags"].([]string)
assert.True(t, ok)
assert.EqualValues(t, []string{"14.04", "16.04"}, metadata)
assert.Equal(t, replication.FilterItemKindRepository, centos.Kind)
assert.Equal(t, "library/centos", centos.Value)
metadata, ok = centos.Metadata["tags"].([]string)
assert.True(t, ok)
assert.EqualValues(t, []string{"7"}, metadata)
}

55
src/replication/source/tag_convertor.go
View File

@ -1,55 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/registry"
)
// TagConverter implement Converter interface, convert repositories to tags
type TagConverter struct {
registry registry.Adaptor
}
// NewTagConverter returns an instance of TagConverter
func NewTagConverter(registry registry.Adaptor) *TagConverter {
return &TagConverter{
registry: registry,
}
}
// Convert repositories to tags
func (t *TagConverter) Convert(items []models.FilterItem) []models.FilterItem {
result := []models.FilterItem{}
for _, item := range items {
if item.Kind != replication.FilterItemKindRepository {
// just put it to the result list if the item is not a repository
result = append(result, item)
continue
}
tags := t.registry.GetTags(item.Value, "")
for _, tag := range tags {
result = append(result, models.FilterItem{
Kind: replication.FilterItemKindTag,
Value: item.Value + ":" + tag.Name,
Operation: item.Operation,
})
}
}
return result
}

51
src/replication/source/tag_convertor_test.go
View File

@ -1,51 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"testing"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
)
func TestTagConvert(t *testing.T) {
items := []models.FilterItem{
{
Kind: replication.FilterItemKindRepository,
Value: "library/ubuntu",
},
{
Kind: replication.FilterItemKindProject,
},
}
expected := []models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/ubuntu:14.04",
},
{
Kind: replication.FilterItemKindTag,
Value: "library/ubuntu:16.04",
},
{
Kind: replication.FilterItemKindProject,
},
}
converter := NewTagConverter(&fakeRegistryAdaptor{})
assert.EqualValues(t, expected, converter.Convert(items))
}

84
src/replication/source/tag_filter.go
View File

@ -1,84 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"strings"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/registry"
)
// TagFilter implements Filter interface to filter tag
type TagFilter struct {
pattern string
converter Converter
}
// NewTagFilter returns an instance of TagFilter
func NewTagFilter(pattern string, registry registry.Adaptor) *TagFilter {
return &TagFilter{
pattern: pattern,
converter: NewTagConverter(registry),
}
}
// Init ...
func (t *TagFilter) Init() error {
return nil
}
// GetConverter ...
func (t *TagFilter) GetConverter() Converter {
return t.converter
}
// DoFilter filters tag of the image
func (t *TagFilter) DoFilter(items []models.FilterItem) []models.FilterItem {
candidates := []string{}
for _, item := range items {
candidates = append(candidates, item.Value)
}
log.Debugf("tag filter candidates: %v", candidates)
result := []models.FilterItem{}
for _, item := range items {
if item.Kind != replication.FilterItemKindTag {
log.Warningf("unsupported type %s for tag filter, dropped", item.Kind)
continue
}
if len(t.pattern) == 0 {
log.Debugf("pattern is null, add %s to the tag filter result list", item.Value)
result = append(result, item)
continue
}
tag := strings.SplitN(item.Value, ":", 2)[1]
matched, err := match(t.pattern, tag)
if err != nil {
log.Errorf("failed to match pattern %s to value %s: %v, skip it", t.pattern, tag, err)
continue
}
if matched {
log.Debugf("pattern %s matched, add %s to the tag filter result list", t.pattern, item.Value)
result = append(result, item)
}
}
return result
}

85
src/replication/source/tag_filter_test.go
View File

@ -1,85 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"testing"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/registry"
"github.com/stretchr/testify/assert"
)
func TestInitOfTagFilter(t *testing.T) {
filter := NewTagFilter("", &registry.HarborAdaptor{})
assert.Nil(t, filter.Init())
}
func TestGetConverterOfTagFilter(t *testing.T) {
filter := NewTagFilter("", &registry.HarborAdaptor{})
assert.NotNil(t, filter.GetConverter())
}
func TestDoFilterOfTagFilter(t *testing.T) {
// invalid filter item type
filter := NewTagFilter("", &registry.HarborAdaptor{})
items := filter.DoFilter([]models.FilterItem{
{
Kind: "invalid_type",
},
})
assert.Equal(t, 0, len(items))
// empty pattern
filter = NewTagFilter("", &registry.HarborAdaptor{})
items = filter.DoFilter([]models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world:latest",
},
})
assert.Equal(t, 1, len(items))
// non-empty pattern
filter = NewTagFilter("l*t", &registry.HarborAdaptor{})
items = filter.DoFilter([]models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world:latest",
},
})
assert.Equal(t, 1, len(items))
// non-empty pattern
filter = NewTagFilter("lates?", &registry.HarborAdaptor{})
items = filter.DoFilter([]models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world:latest",
},
})
assert.Equal(t, 1, len(items))
// non-empty pattern
filter = NewTagFilter("latest?", &registry.HarborAdaptor{})
items = filter.DoFilter([]models.FilterItem{
{
Kind: replication.FilterItemKindTag,
Value: "library/hello-world:latest",
},
})
assert.Equal(t, 0, len(items))
}

212
src/replication/trigger/cache.go
View File

@ -1,212 +0,0 @@
package trigger
import (
"container/heap"
"fmt"
"sync"
"time"
)
const (
// The max count of items the cache can keep
defaultCapacity = 1000
)
// Item keeps more metadata of the triggers which are stored in the heap.
type Item struct {
// Which policy the trigger belong to
policyID int64
// Frequency of cache querying
// First compration factor
frequency int
// The timestamp of being put into heap
// Second compration factor
timestamp int64
// The index in the heap
index int
}
// MetaQueue implements heap.Interface and holds items which are metadata of trigger
type MetaQueue []*Item
// Len return the size of the queue
func (mq MetaQueue) Len() int {
return len(mq)
}
// Less is a comparator of heap
func (mq MetaQueue) Less(i, j int) bool {
return mq[i].frequency < mq[j].frequency ||
(mq[i].frequency == mq[j].frequency &&
mq[i].timestamp < mq[j].timestamp)
}
// Swap the items to rebuild heap
func (mq MetaQueue) Swap(i, j int) {
mq[i], mq[j] = mq[j], mq[i]
mq[i].index = i
mq[j].index = j
}
// Push item into heap
func (mq *MetaQueue) Push(x interface{}) {
item := x.(*Item)
n := len(*mq)
item.index = n
item.timestamp = time.Now().UTC().UnixNano()
*mq = append(*mq, item)
}
// Pop smallest item from heap
func (mq *MetaQueue) Pop() interface{} {
old := *mq
n := len(old)
item := old[n-1] // Smallest item
item.index = -1 // For safety
*mq = old[:n-1]
return item
}
// Update the frequency of item
func (mq *MetaQueue) Update(item *Item) {
item.frequency++
heap.Fix(mq, item.index)
}
// CacheItem is the data stored in the cache.
// It contains trigger and heap item references.
type CacheItem struct {
// The trigger reference
trigger Interface
// The heap item reference
item *Item
}
// Cache is used to cache the enabled triggers with specified capacity.
// If exceed the capacity, cached items will be adjusted with the following rules:
// The item with least usage frequency will be replaced;
// If multiple items with same usage frequency, the oldest one will be replaced.
type Cache struct {
// The max count of items this cache can keep
capacity int
// Lock to handle concurrent case
lock *sync.RWMutex
// Hash map for quick locating cached item
hash map[string]CacheItem
// Heap for quick locating the trigger with least usage
queue *MetaQueue
}
// NewCache is constructor of cache
func NewCache(capacity int) *Cache {
capa := capacity
if capa <= 0 {
capa = defaultCapacity
}
// Initialize heap
mq := make(MetaQueue, 0)
heap.Init(&mq)
return &Cache{
capacity: capa,
lock: new(sync.RWMutex),
hash: make(map[string]CacheItem),
queue: &mq,
}
}
// Get the trigger interface with the specified policy ID
func (c *Cache) Get(policyID int64) Interface {
if policyID <= 0 {
return nil
}
c.lock.RLock()
defer c.lock.RUnlock()
k := c.key(policyID)
if cacheItem, ok := c.hash[k]; ok {
// Update frequency
c.queue.Update(cacheItem.item)
return cacheItem.trigger
}
return nil
}
// Put the item into cache with ID of ploicy as key
func (c *Cache) Put(policyID int64, trigger Interface) {
if policyID <= 0 || trigger == nil {
return
}
c.lock.Lock()
defer c.lock.Unlock()
// Exceed the capacity?
if c.Size() >= c.capacity {
// Pop one for the new one
v := heap.Pop(c.queue)
item := v.(*Item)
// Remove from hash
delete(c.hash, c.key(item.policyID))
}
// Add to meta queue
item := &Item{
policyID: policyID,
frequency: 1,
}
heap.Push(c.queue, item)
// Cache
cacheItem := CacheItem{
trigger: trigger,
item: item,
}
k := c.key(policyID)
c.hash[k] = cacheItem
}
// Remove the trigger attached to the specified policy
func (c *Cache) Remove(policyID int64) Interface {
if policyID > 0 {
c.lock.Lock()
defer c.lock.Unlock()
// If existing
k := c.key(policyID)
if cacheItem, ok := c.hash[k]; ok {
// Remove from heap
heap.Remove(c.queue, cacheItem.item.index)
// Remove from hash
delete(c.hash, k)
return cacheItem.trigger
}
}
return nil
}
// Size return the count of triggers in the cache
func (c *Cache) Size() int {
return len(c.hash)
}
// Generate a hash key with the policy ID
func (c *Cache) key(policyID int64) string {
return fmt.Sprintf("trigger-%d", policyID)
}

53
src/replication/trigger/cache_test.go
View File

@ -1,53 +0,0 @@
package trigger
import "testing"
import "time"
func TestCache(t *testing.T) {
cache := NewCache(10)
trigger := NewImmediateTrigger(ImmediateParam{})
cache.Put(1, trigger)
if cache.Size() != 1 {
t.Fatalf("Invalid size, expect 1 but got %d", cache.Size())
}
tr := cache.Get(1)
if tr == nil {
t.Fatal("Should not get nil item")
}
tri := cache.Remove(1)
if tri == nil || cache.Size() > 0 {
t.Fatal("Failed to remove")
}
}
func TestCacheChange(t *testing.T) {
cache := NewCache(2)
trigger1 := NewImmediateTrigger(ImmediateParam{})
trigger2 := NewImmediateTrigger(ImmediateParam{})
cache.Put(1, trigger1)
cache.Put(2, trigger2)
if cache.Size() != 2 {
t.Fatalf("Invalid size, expect 2 but got %d", cache.Size())
}
if tr := cache.Get(2); tr == nil {
t.Fatal("Should not get nil item")
}
time.Sleep(100 * time.Microsecond)
trigger3 := NewImmediateTrigger(ImmediateParam{})
cache.Put(3, trigger3)
if cache.Size() != 2 {
t.Fatalf("Invalid size, expect 2 but got %d", cache.Size())
}
if tr := cache.Get(1); tr != nil {
t.Fatal("item1 should not exist")
}
}

46
src/replication/trigger/immediate.go
View File

@ -1,46 +0,0 @@
package trigger
import (
"github.com/goharbor/harbor/src/replication"
)
// ImmediateTrigger will setup watcher at the image pushing action to fire
// replication event at pushing happening time.
type ImmediateTrigger struct {
params ImmediateParam
}
// NewImmediateTrigger is constructor of ImmediateTrigger
func NewImmediateTrigger(params ImmediateParam) *ImmediateTrigger {
return &ImmediateTrigger{
params: params,
}
}
// Kind is the implementation of same method defined in Trigger interface
func (st *ImmediateTrigger) Kind() string {
return replication.TriggerKindImmediate
}
// Setup is the implementation of same method defined in Trigger interface
func (st *ImmediateTrigger) Setup() error {
// TODO: Need more complicated logic here to handle partial updates
for _, namespace := range st.params.Namespaces {
wt := WatchItem{
PolicyID: st.params.PolicyID,
Namespace: namespace,
OnDeletion: st.params.OnDeletion,
OnPush: true,
}
if err := DefaultWatchList.Add(wt); err != nil {
return err
}
}
return nil
}
// Unset is the implementation of same method defined in Trigger interface
func (st *ImmediateTrigger) Unset() error {
return DefaultWatchList.Remove(st.params.PolicyID)
}

57
src/replication/trigger/immediate_test.go
View File

@ -1,57 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package trigger
import (
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestKindOfImmediateTrigger(t *testing.T) {
trigger := NewImmediateTrigger(ImmediateParam{})
assert.Equal(t, replication.TriggerKindImmediate, trigger.Kind())
}
func TestSetupAndUnsetOfImmediateTrigger(t *testing.T) {
dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{}
param := ImmediateParam{}
param.PolicyID = 1
param.OnDeletion = true
param.Namespaces = []string{"library"}
trigger := NewImmediateTrigger(param)
err := trigger.Setup()
require.Nil(t, err)
items, err := DefaultWatchList.Get("library", "push")
require.Nil(t, err)
assert.Equal(t, 1, len(items))
items, err = DefaultWatchList.Get("library", "delete")
require.Nil(t, err)
assert.Equal(t, 1, len(items))
err = trigger.Unset()
require.Nil(t, err)
items, err = DefaultWatchList.Get("library", "delete")
require.Nil(t, err)
assert.Equal(t, 0, len(items))
}

13
src/replication/trigger/interface.go
View File

@ -1,13 +0,0 @@
package trigger
// Interface is certain mechanism to know when fire the replication operation.
type Interface interface {
// Kind indicates what type of the trigger is.
Kind() string
// Setup/enable the trigger; if failed, an error would be returned.
Setup() error
// Remove/disable the trigger; if failed, an error would be returned.
Unset() error
}

124
src/replication/trigger/manager.go
View File

@ -1,124 +0,0 @@
package trigger
import (
"fmt"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
)
// Manager provides unified methods to manage the triggers of policies;
// Cache the enabled triggers, setup/unset the trigger based on the parameters
// with json format.
type Manager struct {
// Cache for triggers
// cache *Cache
}
// NewManager is the constructor of trigger manager.
// capacity is the max number of trigger references manager can keep in memory
func NewManager(capacity int) *Manager {
return &Manager{
// cache: NewCache(capacity),
}
}
/*
// GetTrigger returns the enabled trigger reference if existing in the cache.
func (m *Manager) GetTrigger(policyID int64) Interface {
return m.cache.Get(policyID)
}
// RemoveTrigger will disable the trigger and remove it from the cache if existing.
func (m *Manager) RemoveTrigger(policyID int64) error {
trigger := m.cache.Get(policyID)
if trigger == nil {
return errors.New("Trigger is not cached, please use UnsetTrigger to disable the trigger")
}
// Unset trigger
if err := trigger.Unset(); err != nil {
return err
}
// Remove from cache
// No need to check the return of remove because the dirty item cached in the cache
// will be removed out finally after a certain while
m.cache.Remove(policyID)
return nil
}
*/
// SetupTrigger will create the new trigger based on the provided policy.
// If failed, an error will be returned.
func (m *Manager) SetupTrigger(policy *models.ReplicationPolicy) error {
trigger, err := createTrigger(policy)
if err != nil {
return err
}
// manual trigger, do nothing
if trigger == nil {
return nil
}
tg := trigger.(Interface)
if err = tg.Setup(); err != nil {
return err
}
log.Debugf("%s trigger for policy %d is set", tg.Kind(), policy.ID)
return nil
}
// UnsetTrigger will disable the trigger which is not cached in the trigger cache.
func (m *Manager) UnsetTrigger(policy *models.ReplicationPolicy) error {
trigger, err := createTrigger(policy)
if err != nil {
return err
}
// manual trigger, do nothing
if trigger == nil {
return nil
}
tg := trigger.(Interface)
if err = tg.Unset(); err != nil {
return err
}
log.Debugf("%s trigger for policy %d is unset", tg.Kind(), policy.ID)
return nil
}
func createTrigger(policy *models.ReplicationPolicy) (interface{}, error) {
if policy == nil || policy.Trigger == nil {
return nil, fmt.Errorf("empty policy or trigger")
}
trigger := policy.Trigger
switch trigger.Kind {
case replication.TriggerKindSchedule:
param := ScheduleParam{}
param.PolicyID = policy.ID
param.Type = trigger.ScheduleParam.Type
param.Weekday = trigger.ScheduleParam.Weekday
param.Offtime = trigger.ScheduleParam.Offtime
return NewScheduleTrigger(param), nil
case replication.TriggerKindImmediate:
param := ImmediateParam{}
param.PolicyID = policy.ID
param.OnDeletion = policy.ReplicateDeletion
param.Namespaces = policy.Namespaces
return NewImmediateTrigger(param), nil
case replication.TriggerKindManual:
return nil, nil
default:
return nil, fmt.Errorf("invalid trigger type: %s", trigger.Kind)
}
}

94
src/replication/trigger/manager_test.go
View File

@ -1,94 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package trigger
import (
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateTrigger(t *testing.T) {
// nil policy
_, err := createTrigger(nil)
require.NotNil(t, err)
// nil trigger
_, err = createTrigger(&models.ReplicationPolicy{})
require.NotNil(t, err)
// schedule trigger
trigger, err := createTrigger(&models.ReplicationPolicy{
Trigger: &models.Trigger{
Kind: replication.TriggerKindSchedule,
ScheduleParam: &models.ScheduleParam{
Type: replication.TriggerScheduleWeekly,
Weekday: 1,
Offtime: 1,
},
},
})
require.Nil(t, err)
assert.NotNil(t, trigger)
// immediate trigger
trigger, err = createTrigger(&models.ReplicationPolicy{
Trigger: &models.Trigger{
Kind: replication.TriggerKindImmediate,
},
})
require.Nil(t, err)
assert.NotNil(t, trigger)
// manual trigger
trigger, err = createTrigger(&models.ReplicationPolicy{
Trigger: &models.Trigger{
Kind: replication.TriggerKindManual,
},
})
require.Nil(t, err)
assert.Nil(t, trigger)
}
func TestSetupTrigger(t *testing.T) {
dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{}
mgr := NewManager(1)
err := mgr.SetupTrigger(&models.ReplicationPolicy{
Trigger: &models.Trigger{
Kind: replication.TriggerKindImmediate,
},
})
assert.Nil(t, err)
}
func TestUnsetTrigger(t *testing.T) {
dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{}
mgr := NewManager(1)
err := mgr.UnsetTrigger(&models.ReplicationPolicy{
Trigger: &models.Trigger{
Kind: replication.TriggerKindImmediate,
},
})
assert.Nil(t, err)
}

22
src/replication/trigger/param_immediate.go
View File

@ -1,22 +0,0 @@
package trigger
// NOTES: Whether replicate the existing images when the type of trigger is
// 'Immediate' is a once-effective setting which will not be persisted
// and kept as one parameter of 'Immediate' trigger. It will only be
// covered by the UI logic.
// ImmediateParam defines the parameter of immediate trigger
type ImmediateParam struct {
// Basic parameters
BasicParam
// Namepaces
Namespaces []string
}
// Parse is the implementation of same method in TriggerParam interface
// NOTES: No need to implement this method for 'Immediate' trigger as
// it does not have any parameters with json format.
func (ip ImmediateParam) Parse(param string) error {
return nil
}

30
src/replication/trigger/param_schedule.go
View File

@ -1,30 +0,0 @@
package trigger
import (
"encoding/json"
"errors"
)
// ScheduleParam defines the parameter of schedule trigger
type ScheduleParam struct {
// Basic parameters
BasicParam
// Daily or weekly
Type string
// Optional, only used when type is 'weekly'
Weekday int8
// The time offset with the UTC 00:00 in seconds
Offtime int64
}
// Parse is the implementation of same method in TriggerParam interface
func (stp ScheduleParam) Parse(param string) error {
if len(param) == 0 {
return errors.New("Parameter of schedule trigger should not be empty")
}
return json.Unmarshal([]byte(param), &stp)
}

107
src/replication/trigger/schedule.go
View File

@ -1,107 +0,0 @@
package trigger
import (
"fmt"
"net/http"
"github.com/goharbor/harbor/src/common/dao"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/job"
job_models "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/models"
common_utils "github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/replication"
)
// ScheduleTrigger will schedule a alternate policy to provide 'daily' and 'weekly' trigger ways.
type ScheduleTrigger struct {
params ScheduleParam
}
// NewScheduleTrigger is constructor of ScheduleTrigger
func NewScheduleTrigger(params ScheduleParam) *ScheduleTrigger {
return &ScheduleTrigger{
params: params,
}
}
// Kind is the implementation of same method defined in Trigger interface
func (st *ScheduleTrigger) Kind() string {
return replication.TriggerKindSchedule
}
// Setup is the implementation of same method defined in Trigger interface
func (st *ScheduleTrigger) Setup() error {
metadata := &job_models.JobMetadata{
JobKind: job.JobKindPeriodic,
}
switch st.params.Type {
case replication.TriggerScheduleDaily:
h, m, s := common_utils.ParseOfftime(st.params.Offtime)
metadata.Cron = fmt.Sprintf("%d %d %d * * *", s, m, h)
case replication.TriggerScheduleWeekly:
h, m, s := common_utils.ParseOfftime(st.params.Offtime)
metadata.Cron = fmt.Sprintf("%d %d %d * * %d", s, m, h, st.params.Weekday%7)
default:
return fmt.Errorf("unsupported schedule trigger type: %s", st.params.Type)
}
id, err := dao.AddRepJob(models.RepJob{
Repository: "N/A",
PolicyID: st.params.PolicyID,
Operation: models.RepOpSchedule,
})
if err != nil {
return err
}
uuid, err := utils.GetJobServiceClient().SubmitJob(&job_models.JobData{
Name: job.ImageReplicate,
Parameters: map[string]interface{}{
"policy_id": st.params.PolicyID,
"url": config.InternalCoreURL(),
"insecure": true,
},
Metadata: metadata,
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/replication/%d",
config.InternalCoreURL(), id),
})
if err != nil {
// clean up the job record in database
if e := dao.DeleteRepJob(id); e != nil {
log.Errorf("failed to delete job %d: %v", id, e)
}
return err
}
return dao.SetRepJobUUID(id, uuid)
}
// Unset is the implementation of same method defined in Trigger interface
func (st *ScheduleTrigger) Unset() error {
jobs, err := dao.GetRepJobs(&models.RepJobQuery{
PolicyID: st.params.PolicyID,
Operations: []string{models.RepOpSchedule},
})
if err != nil {
return err
}
if len(jobs) != 1 {
log.Warningf("only one job should be found, but found %d now", len(jobs))
}
for _, j := range jobs {
if err = utils.GetJobServiceClient().PostAction(j.UUID, job.JobActionStop); err != nil {
// if the job specified by UUID is not found in jobservice, delete the job
// record from database
if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound {
return err
}
}
if err = dao.DeleteRepJob(j.ID); err != nil {
return err
}
}
return nil
}

27
src/replication/trigger/schedule_test.go
View File

@ -1,27 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package trigger
import (
"testing"
"github.com/goharbor/harbor/src/replication"
"github.com/stretchr/testify/assert"
)
func TestKindOfScheduleTrigger(t *testing.T) {
trigger := NewScheduleTrigger(ScheduleParam{})
assert.Equal(t, replication.TriggerKindSchedule, trigger.Kind())
}

17
src/replication/trigger/trigger_param.go
View File

@ -1,17 +0,0 @@
package trigger
// BasicParam contains the general parameters for all triggers
type BasicParam struct {
// ID of the related policy
PolicyID int64
// Whether delete remote replicated images if local ones are deleted
OnDeletion bool
}
// Parameter defines operation of doing initialization from parameter json text
type Parameter interface {
// Decode parameter with json style to the owner struct
// If failed, an error will be returned
Parse(param string) error
}

65
src/replication/trigger/watch_list.go
View File

@ -1,65 +0,0 @@
package trigger
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
)
// DefaultWatchList is the default instance of WatchList
var DefaultWatchList = &WatchList{}
// WatchList contains the items which should be evaluated for replication
// when image pushing or deleting happens.
type WatchList struct{}
// WatchItem keeps the related data for evaluation in WatchList.
type WatchItem struct {
// ID of policy
PolicyID int64
// Corresponding namespace
Namespace string
// For deletion event
OnDeletion bool
// For pushing event
OnPush bool
}
// Add item to the list and persist into DB
func (wl *WatchList) Add(item WatchItem) error {
_, err := dao.DefaultDatabaseWatchItemDAO.Add(
&models.WatchItem{
PolicyID: item.PolicyID,
Namespace: item.Namespace,
OnPush: item.OnPush,
OnDeletion: item.OnDeletion,
})
return err
}
// Remove the specified watch item from list
func (wl *WatchList) Remove(policyID int64) error {
return dao.DefaultDatabaseWatchItemDAO.DeleteByPolicyID(policyID)
}
// Get the watch items according to the namespace and operation
func (wl *WatchList) Get(namespace, operation string) ([]WatchItem, error) {
items, err := dao.DefaultDatabaseWatchItemDAO.Get(namespace, operation)
if err != nil {
return nil, err
}
watchItems := []WatchItem{}
for _, item := range items {
watchItems = append(watchItems, WatchItem{
PolicyID: item.PolicyID,
Namespace: item.Namespace,
OnPush: item.OnPush,
OnDeletion: item.OnDeletion,
})
}
return watchItems, nil
}

64
src/replication/trigger/watch_list_test.go
View File

@ -1,64 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package trigger
import (
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMethodsOfWatchList(t *testing.T) {
dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{}
var policyID int64 = 1
// test Add
item := WatchItem{
PolicyID: policyID,
Namespace: "library",
OnDeletion: true,
OnPush: false,
}
err := DefaultWatchList.Add(item)
require.Nil(t, err)
// test Get: non-exist namespace
items, err := DefaultWatchList.Get("non-exist-namespace", "delete")
require.Nil(t, err)
assert.Equal(t, 0, len(items))
// test Get: non-exist operation
items, err = DefaultWatchList.Get("library", "non-exist-operation")
require.Nil(t, err)
assert.Equal(t, 0, len(items))
// test Get: valid params
items, err = DefaultWatchList.Get("library", "delete")
require.Nil(t, err)
assert.Equal(t, 1, len(items))
assert.Equal(t, policyID, items[0].PolicyID)
// test Remove
err = DefaultWatchList.Remove(policyID)
require.Nil(t, err)
items, err = DefaultWatchList.Get("library", "delete")
require.Nil(t, err)
assert.Equal(t, 0, len(items))
}