diff --git a/.travis.yml b/.travis.yml index f04b4c926..791cb7ec3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,5 +63,5 @@ script: - if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_run.sh $IP; fi # TODO(ChenDe): Enable API test when API test problems resolved #- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_run.sh DB $IP; fi -- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_run.sh LDAP $IP; fi +#- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_run.sh LDAP $IP; fi - if [ "$OFFLINE" == true ]; then bash ./tests/travis/distro_installer.sh; fi diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 8f7503ebb..eca24ab17 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1746,337 +1746,6 @@ paths: description: Resource requested does not exist. '500': description: Unexpected internal errors. - /jobs/replication: - get: - summary: List filters jobs according to the policy and repository - description: | - This endpoint let user list filters jobs according to the policy and repository. (if start_time and end_time are both null, list jobs of last 10 days) - tags: - - Products - parameters: - - name: policy_id - in: query - type: integer - format: int - required: true - description: The ID of the policy that triggered this job. - - name: op_uuid - in: query - type: string - required: false - description: The UUID of one trigger of replication policy. - - name: num - in: query - type: integer - format: int32 - required: false - description: The return list length number. - - name: end_time - in: query - type: integer - format: int64 - required: false - description: The end time of jobs done. (Timestamp) - - name: start_time - in: query - type: integer - format: int64 - required: false - description: The start time of jobs. (Timestamp) - - name: repository - in: query - type: string - required: false - description: The respond jobs list filter by repository name. - - name: status - in: query - type: string - required: false - description: The respond jobs list filter by status. - - name: page - in: query - type: integer - format: int32 - required: false - description: 'The page nubmer, default is 1.' - - name: page_size - in: query - type: integer - format: int32 - required: false - description: 'The size of per page, default is 10, maximum is 100.' - responses: - '200': - description: Get the required logs successfully. - schema: - type: array - items: - $ref: '#/definitions/JobStatus' - headers: - X-Total-Count: - description: The total count of jobs - type: integer - Link: - description: Link refers to the previous page and next page - type: string - '400': - description: Bad request because of invalid parameters. - '401': - description: User need to login first. - '500': - description: Unexpected internal errors. - put: - summary: Update status of jobs. Only stop is supported for now. - description: | - The endpoint is used to stop the replication jobs of a policy. - tags: - - Products - parameters: - - name: policyinfo - in: body - description: The policy ID and status. - required: true - schema: - $ref: '#/definitions/UpdateJobs' - responses: - '200': - description: Update the status successfully. - '400': - description: Bad request because of invalid parameters. - '401': - description: User need to login first. - '403': - description: User has no privilege for the operation. - '404': - description: Resource requested does not exist. - '500': - description: Unexpected internal errors. - '/jobs/replication/{id}': - delete: - summary: Delete specific ID job. - description: | - This endpoint is aimed to remove specific ID job from jobservice. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: Delete job ID. - tags: - - Products - responses: - '200': - description: Job deleted successfully. - '400': - description: Job ID is invalid or can't remove this job. - '401': - description: Only admin has this authority. - '404': - description: Project ID does not exist. - '500': - description: Unexpected internal errors. - '/jobs/replication/{id}/log': - get: - summary: Get job logs. - description: | - This endpoint let user search job logs filtered by specific ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: Relevant job ID - tags: - - Products - responses: - '200': - description: Get job log successfully. - '400': - description: Illegal format of provided ID value. - '401': - description: User need to log in first. - '404': - description: The specific repository ID's log does not exist. - '500': - description: Unexpected internal errors. - '/jobs/scan/{id}/log': - get: - summary: Get job logs. - description: | - This endpoint let user get scan job logs filtered by specific ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: Relevant job ID - tags: - - Products - responses: - '200': - description: Get job log successfully. - '400': - description: Illegal format of provided ID value. - '401': - description: User need to log in first. - '404': - description: The specific repository ID's log does not exist. - '500': - description: Unexpected internal errors. - /policies/replication: - get: - summary: List filters policies by name and project_id - description: | - This endpoint let user list filters policies by name and project_id, if name and project_id are nil, list returns all policies - parameters: - - name: name - in: query - type: string - required: false - description: The replication's policy name. - - name: project_id - in: query - type: integer - format: int64 - required: false - description: Relevant project ID. - - name: page - in: query - type: integer - format: int32 - required: false - description: The page nubmer. - - name: page_size - in: query - type: integer - format: int32 - required: false - description: The size of per page. - tags: - - Products - responses: - '200': - description: Get policy successfully. - schema: - type: array - items: - $ref: '#/definitions/RepPolicy' - '400': - description: Invalid project ID. - '401': - description: User need to log in first. - '500': - description: Unexpected internal errors. - post: - summary: Post creates a policy - description: | - This endpoint let user creates a policy, and if it is enabled, the replication will be triggered right now. - parameters: - - name: policyinfo - in: body - description: Create new policy. - required: true - schema: - $ref: '#/definitions/RepPolicy' - tags: - - Products - responses: - '201': - description: Create policy successfully. - '400': - description: Invalid project ID or target ID. - '401': - description: User need to log in first. - '409': - description: Policy name already used or policy already exists with the same project and target. - '415': - $ref: '#/responses/UnsupportedMediaType' - '500': - description: Unexpected internal errors. - '/policies/replication/{id}': - get: - summary: Get replication policy. - description: | - This endpoint let user search replication policy by specific ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: policy ID - tags: - - Products - responses: - '200': - description: Get job policy successfully. - schema: - $ref: '#/definitions/RepPolicy' - '401': - description: User need to log in first. - '404': - description: The specific repository ID's policy does not exist. - '500': - description: Unexpected internal errors. - put: - summary: 'Put modifies name, description, target and enablement of policy.' - description: | - This endpoint let user update policy name, description, target and enablement. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: policy ID - - name: policyupdate - in: body - description: Updated properties of the replication policy. - required: true - schema: - $ref: '#/definitions/RepPolicy' - tags: - - Products - responses: - '200': - description: Update job policy content successfully. - '400': - description: policy is enabled or target does not exist - '401': - description: User need to log in first. - '404': - description: The specific repository ID's policy does not exist. - '409': - description: Policy name already used or policy already exists with the same project and target. - '500': - description: Unexpected internal errors. - delete: - summary: Delete the replication policy specified by ID. - description: | - Delete the replication policy specified by ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: Replication policy ID - tags: - - Products - responses: - '200': - description: Delete successfully. - '400': - description: Invalid parameters. - '401': - description: User need to log in first. - '404': - description: The resource does not exist. - '500': - description: Unexpected internal errors. /replication/policies: get: summary: List replication policies @@ -2421,33 +2090,6 @@ paths: description: The resource does not exist. '500': description: Unexpected internal errors. - /replications: - post: - summary: Trigger the replication according to the specified policy. - description: | - This endpoint is used to trigger a replication. - parameters: - - name: policy ID - in: body - description: The ID of replication policy. - required: true - schema: - $ref: '#/definitions/Replication' - tags: - - Products - responses: - '200': - description: Trigger the replication successfully. - schema: - $ref: '#/definitions/ReplicationResponse' - '401': - description: User need to log in first. - '404': - description: The policy does not exist. - '415': - $ref: '#/responses/UnsupportedMediaType' - '500': - description: Unexpected internal errors. /replication/adapters: get: summary: List supported adapters. @@ -2622,33 +2264,6 @@ paths: description: Registry not found '500': description: Unexpected internal errors. - '/targets/{id}/policies/': - get: - summary: List the target relevant policies. - description: | - This endpoint list policies filter with specific replication's target ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: The replication's target ID. - tags: - - Products - responses: - '200': - description: Get relevant policies successfully. - schema: - type: array - items: - $ref: '#/definitions/RepPolicy' - '401': - description: User need to log in first. - '404': - description: Replication's target not found - '500': - description: Unexpected internal errors. /internal/syncregistry: post: summary: Sync repositories from registry to DB. @@ -4081,51 +3696,6 @@ definitions: tag: type: string description: The repository's used tag. - RepPolicy: - type: object - properties: - id: - type: integer - format: int64 - description: The policy ID. - name: - type: string - description: The policy name. - description: - type: string - description: The description of the policy. - projects: - type: array - description: The project list that the policy applys to. - items: - $ref: '#/definitions/Project' - registries: - type: array - description: The target list. - items: - $ref: '#/definitions/Registry' - trigger: - $ref: '#/definitions/RepTrigger' - filters: - type: array - description: The replication policy filter array. - items: - $ref: '#/definitions/RepFilter' - replicate_existing_image_now: - type: boolean - description: Whether to replicate the existing images now. - replicate_deletion: - type: boolean - description: Whether to replicate the deletion operation. - creation_time: - type: string - description: The create time of the policy. - update_time: - type: string - description: The update time of the policy. - error_job_count: - type: integer - description: The error job count number for the policy. ReplicationPolicy: type: object properties: @@ -4154,7 +3724,7 @@ definitions: type: string description: The destination namespace. trigger: - $ref: '#/definitions/RepTrigger' + $ref: '#/definitions/ReplicationTrigger' filters: type: array description: The replication policy filter array. @@ -4175,43 +3745,20 @@ definitions: update_time: type: string description: The update time of the policy. - RepTrigger: - type: object - properties: - kind: - type: string - description: 'The replication policy trigger kind. The valid values are manual, immediate and schedule.' - schedule_param: - $ref: '#/definitions/ScheduleParam' - ScheduleParam: + ReplicationTrigger: type: object properties: type: type: string - description: The schedule type. The valid values are daily and weekly. - weekday: - type: integer - format: int8 - description: 'Optional, only used when the type is weedly. The valid values are 1-7.' - offtime: - type: integer - format: int64 - description: 'The time offset with the UTC 00:00 in seconds.' - RepFilter: + description: 'The replication policy trigger type. The valid values are manual, event_based and scheduled.' + trigger_settings: + $ref: '#/definitions/TriggerSettings' + TriggerSettings: type: object properties: - kind: + cron: type: string - description: 'The replication policy filter kind. The valid values are project, repository and tag.' - value: - type: string - description: 'The value of replication policy filter. When creating repository and tag filter, filling it with the pattern as string. When creating label filter, filling it with label ID as integer.' - pattern: - type: string - description: 'Depraceted, use value instead. The replication policy filter pattern.' - metadata: - type: object - description: This map object is the replication policy filter metadata. + description: The cron string for scheduled trigger ReplicationFilter: type: object properties: @@ -4782,33 +4329,12 @@ definitions: type: integer description: 'The offest in seconds of UTC 0 o''clock, only valid when the policy type is "daily"' description: 'The parameters of the policy, the values are dependant on the type of the policy.' - Replication: - type: object - properties: - policy_id: - type: integer - description: The ID of replication policy - ReplicationResponse: - type: object - properties: - uuid: - type: string - description: UUID of the replication RepositoryDescription: type: object properties: description: type: string description: The description of the repository. - UpdateJobs: - type: object - properties: - policy_id: - type: integer - description: The ID of replication policy - status: - type: string - description: The status of jobs. The only valid value is stop for now. Label: type: object properties: @@ -4910,7 +4436,7 @@ definitions: type: array description: The replication policy list. items: - $ref: '#/definitions/RepPolicy' + $ref: '#/definitions/ReplicationPolicy' StringConfigItem: type: object properties: diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 0f540da1f..cb00ab09a 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -26,7 +26,6 @@ import ( "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func execUpdate(o orm.Ormer, sql string, params ...interface{}) error { @@ -675,292 +674,6 @@ func TestChangeUserProfile(t *testing.T) { var targetID, policyID, policyID2, policyID3, jobID, jobID2, jobID3 int64 -func TestAddRepPolicy(t *testing.T) { - policy := models.RepPolicy{ - ProjectID: 1, - TargetID: targetID, - Description: "whatever", - Name: "mypolicy", - } - id, err := AddRepPolicy(policy) - t.Logf("added policy, id: %d", id) - if err != nil { - t.Errorf("Error occurred in AddRepPolicy: %v", err) - } else { - policyID = id - } - p, err := GetRepPolicy(id) - if err != nil { - t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id) - } - if p == nil { - t.Errorf("Unable to find a policy with id: %d", id) - } - - if p.Name != "mypolicy" || p.TargetID != targetID || p.Description != "whatever" { - t.Errorf("The data does not match, expected: Name: mypolicy, TargetID: %d, Description: whatever;\n result: Name: %s, TargetID: %d, Description: %s", - targetID, p.Name, p.TargetID, p.Description) - } -} - -func TestGetRepPolicyByTarget(t *testing.T) { - policies, err := GetRepPolicyByTarget(targetID) - if err != nil { - t.Fatalf("failed to get policy according target %d: %v", targetID, err) - } - - if len(policies) == 0 { - t.Fatal("unexpected length of policies 0, expected is >0") - } - - if policies[0].ID != policyID { - t.Fatalf("unexpected policy: %d, expected: %d", policies[0].ID, policyID) - } -} - -func TestGetRepPolicyByProjectAndTarget(t *testing.T) { - policies, err := GetRepPolicyByProjectAndTarget(1, targetID) - if err != nil { - t.Fatalf("failed to get policy according project %d and target %d: %v", 1, targetID, err) - } - - if len(policies) == 0 { - t.Fatal("unexpected length of policies 0, expected is >0") - } - - if policies[0].ID != policyID { - t.Fatalf("unexpected policy: %d, expected: %d", policies[0].ID, policyID) - } -} - -func TestGetRepPolicyByName(t *testing.T) { - policy, err := GetRepPolicy(policyID) - if err != nil { - t.Fatalf("failed to get policy %d: %v", policyID, err) - } - - policy2, err := GetRepPolicyByName(policy.Name) - if err != nil { - t.Fatalf("failed to get policy %s: %v", policy.Name, err) - } - - if policy.Name != policy2.Name { - t.Errorf("unexpected name: %s, expected: %s", policy2.Name, policy.Name) - } - -} - -func TestAddRepJob(t *testing.T) { - job := models.RepJob{ - Repository: "library/ubuntu", - PolicyID: policyID, - Operation: "transfer", - TagList: []string{"12.01", "14.04", "latest"}, - } - id, err := AddRepJob(job) - if err != nil { - t.Errorf("Error occurred in AddRepJob: %v", err) - return - } - jobID = id - - j, err := GetRepJob(id) - if err != nil { - t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, id) - return - } - if j == nil { - t.Errorf("Unable to find a job with id: %d", id) - return - } - if j.Status != models.JobPending || j.Repository != "library/ubuntu" || j.PolicyID != policyID || j.Operation != "transfer" || len(j.TagList) != 3 { - t.Errorf("Expected data of job, id: %d, Status: %s, Repository: library/ubuntu, PolicyID: %d, Operation: transfer, taglist length 3"+ - "but in returned data:, Status: %s, Repository: %s, Operation: %s, PolicyID: %d, TagList: %v", id, models.JobPending, policyID, j.Status, j.Repository, j.Operation, j.PolicyID, j.TagList) - return - } -} - -func TestSetRepJobUUID(t *testing.T) { - uuid := "u-rep-job-uuid" - assert := assert.New(t) - err := SetRepJobUUID(jobID, uuid) - assert.Nil(err) - j, err := GetRepJob(jobID) - assert.Nil(err) - assert.Equal(uuid, j.UUID) -} - -func TestUpdateRepJobStatus(t *testing.T) { - err := UpdateRepJobStatus(jobID, models.JobFinished) - if err != nil { - t.Errorf("Error occurred in UpdateRepJobStatus, error: %v, id: %d", err, jobID) - return - } - j, err := GetRepJob(jobID) - if err != nil { - t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, jobID) - } - if j == nil { - t.Errorf("Unable to find a job with id: %d", jobID) - } - if j.Status != models.JobFinished { - t.Errorf("Job's status: %s, expected: %s, id: %d", j.Status, models.JobFinished, jobID) - } - err = UpdateRepJobStatus(jobID, models.JobPending) - if err != nil { - t.Errorf("Error occurred in UpdateRepJobStatus when update it back to status pending, error: %v, id: %d", err, jobID) - return - } -} - -func TestGetRepPolicyByProject(t *testing.T) { - p1, err := GetRepPolicyByProject(99) - if err != nil { - t.Errorf("Error occurred in GetRepPolicyByProject:%v, project ID: %d", err, 99) - return - } - if len(p1) > 0 { - t.Errorf("Unexpected length of policy list, expected: 0, in fact: %d, project id: %d", len(p1), 99) - return - } - - p2, err := GetRepPolicyByProject(1) - if err != nil { - t.Errorf("Error occuered in GetRepPolicyByProject:%v, project ID: %d", err, 2) - return - } - if len(p2) != 1 { - t.Errorf("Unexpected length of policy list, expected: 1, in fact: %d, project id: %d", len(p2), 1) - return - } - if p2[0].ID != policyID { - t.Errorf("Unexpecred policy id in result, expected: %d, in fact: %d", policyID, p2[0].ID) - return - } -} - -func TestGetRepJobs(t *testing.T) { - var policyID int64 = 10000 - repository := "repository_for_test_get_rep_jobs" - operation := "operation_for_test" - status := "status_for_test" - now := time.Now().Add(1 * time.Minute) - id, err := AddRepJob(models.RepJob{ - PolicyID: policyID, - Repository: repository, - Operation: operation, - Status: status, - CreationTime: now, - UpdateTime: now, - }) - require.Nil(t, err) - defer DeleteRepJob(id) - - // no query - jobs, err := GetRepJobs() - require.Nil(t, err) - found := false - for _, job := range jobs { - if job.ID == id { - found = true - break - } - } - assert.True(t, found) - - // query by policy ID - jobs, err = GetRepJobs(&models.RepJobQuery{ - PolicyID: policyID, - }) - require.Nil(t, err) - require.Equal(t, 1, len(jobs)) - assert.Equal(t, id, jobs[0].ID) - - // query by repository - jobs, err = GetRepJobs(&models.RepJobQuery{ - Repository: repository, - }) - require.Nil(t, err) - require.Equal(t, 1, len(jobs)) - assert.Equal(t, id, jobs[0].ID) - - // query by operation - jobs, err = GetRepJobs(&models.RepJobQuery{ - Operations: []string{operation}, - }) - require.Nil(t, err) - require.Equal(t, 1, len(jobs)) - assert.Equal(t, id, jobs[0].ID) - - // query by status - jobs, err = GetRepJobs(&models.RepJobQuery{ - Statuses: []string{status}, - }) - require.Nil(t, err) - require.Equal(t, 1, len(jobs)) - assert.Equal(t, id, jobs[0].ID) - - // query by creation time - jobs, err = GetRepJobs(&models.RepJobQuery{ - StartTime: &now, - }) - require.Nil(t, err) - require.Equal(t, 1, len(jobs)) - assert.Equal(t, id, jobs[0].ID) -} - -func TestDeleteRepJob(t *testing.T) { - err := DeleteRepJob(jobID) - if err != nil { - t.Errorf("Error occurred in DeleteRepJob: %v, id: %d", err, jobID) - return - } - t.Logf("deleted rep job, id: %d", jobID) - j, err := GetRepJob(jobID) - if err != nil { - t.Errorf("Error occurred in GetRepJob:%v", err) - return - } - if j != nil { - t.Errorf("Able to find rep job after deletion, id: %d", jobID) - return - } -} - -func TestGetTotalOfRepPolicies(t *testing.T) { - _, err := GetTotalOfRepPolicies("", 1) - require.Nil(t, err) -} - -func TestFilterRepPolicies(t *testing.T) { - _, err := FilterRepPolicies("name", 0, 0, 0) - if err != nil { - t.Fatalf("failed to filter policy: %v", err) - } -} - -func TestUpdateRepPolicy(t *testing.T) { - policy := &models.RepPolicy{ - ID: policyID, - Name: "new_policy_name", - } - if err := UpdateRepPolicy(policy); err != nil { - t.Fatalf("failed to update policy") - } -} - -func TestDeleteRepPolicy(t *testing.T) { - err := DeleteRepPolicy(policyID) - if err != nil { - t.Errorf("Error occurred in DeleteRepPolicy: %v, id: %d", err, policyID) - return - } - t.Logf("delete rep policy, id: %d", policyID) - p, err := GetRepPolicy(policyID) - require.Nil(t, err) - assert.Nil(t, p) -} - func TestGetOrmer(t *testing.T) { o := GetOrmer() if o == nil { diff --git a/src/common/dao/replication_job.go b/src/common/dao/replication_job.go deleted file mode 100644 index 06de6c34d..000000000 --- a/src/common/dao/replication_job.go +++ /dev/null @@ -1,333 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dao - -import ( - "time" - - "strings" - - "github.com/astaxie/beego/orm" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils/log" -) - -// AddRepPolicy ... -func AddRepPolicy(policy models.RepPolicy) (int64, error) { - o := GetOrmer() - sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, creation_time, update_time, filters, replicate_deletion) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id` - params := []interface{}{} - now := time.Now() - - params = append(params, policy.Name, policy.ProjectID, policy.TargetID, true, - policy.Description, policy.Trigger, now, now, policy.Filters, - policy.ReplicateDeletion) - - var policyID int64 - err := o.Raw(sql, params...).QueryRow(&policyID) - if err != nil { - return 0, err - } - - return policyID, nil -} - -// GetRepPolicy ... -func GetRepPolicy(id int64) (*models.RepPolicy, error) { - o := GetOrmer() - sql := `select * from replication_policy where id = ? and deleted = false` - - var policy models.RepPolicy - - if err := o.Raw(sql, id).QueryRow(&policy); err != nil { - if err == orm.ErrNoRows { - return nil, nil - } - return nil, err - } - - return &policy, nil -} - -// GetTotalOfRepPolicies returns the total count of replication policies -func GetTotalOfRepPolicies(name string, projectID int64) (int64, error) { - qs := GetOrmer().QueryTable(&models.RepPolicy{}).Filter("deleted", false) - - if len(name) != 0 { - qs = qs.Filter("name__icontains", name) - } - - if projectID != 0 { - qs = qs.Filter("project_id", projectID) - } - - return qs.Count() -} - -// FilterRepPolicies filters policies by name and project ID -func FilterRepPolicies(name string, projectID, page, pageSize int64) ([]*models.RepPolicy, error) { - o := GetOrmer() - - var args []interface{} - - sql := `select rp.id, rp.project_id, rp.target_id, - rt.name as target_name, rp.name, rp.description, - rp.cron_str, rp.filters, rp.replicate_deletion, - rp.creation_time, rp.update_time, - count(rj.status) as error_job_count - from replication_policy rp - left join replication_target rt on rp.target_id=rt.id - left join replication_job rj on rp.id=rj.policy_id and (rj.status='error' - or rj.status='retrying') - where rp.deleted = false ` - - if len(name) != 0 && projectID != 0 { - sql += `and rp.name like ? and rp.project_id = ? ` - args = append(args, "%"+Escape(name)+"%") - args = append(args, projectID) - } else if len(name) != 0 { - sql += `and rp.name like ? ` - args = append(args, "%"+Escape(name)+"%") - } else if projectID != 0 { - sql += `and rp.project_id = ? ` - args = append(args, projectID) - } - - sql += `group by rt.name, rp.id order by rp.creation_time` - - if page > 0 && pageSize > 0 { - sql += ` limit ? offset ?` - args = append(args, pageSize, (page-1)*pageSize) - } - - var policies []*models.RepPolicy - if _, err := o.Raw(sql, args).QueryRows(&policies); err != nil { - return nil, err - } - - return policies, nil -} - -// GetRepPolicyByName ... -func GetRepPolicyByName(name string) (*models.RepPolicy, error) { - o := GetOrmer() - sql := `select * from replication_policy where deleted = false and name = ?` - - var policy models.RepPolicy - - if err := o.Raw(sql, name).QueryRow(&policy); err != nil { - if err == orm.ErrNoRows { - return nil, nil - } - return nil, err - } - - return &policy, nil -} - -// GetRepPolicyByProject ... -func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) { - o := GetOrmer() - sql := `select * from replication_policy where deleted = false and project_id = ?` - - var policies []*models.RepPolicy - - if _, err := o.Raw(sql, projectID).QueryRows(&policies); err != nil { - return nil, err - } - - return policies, nil -} - -// GetRepPolicyByTarget ... -func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) { - o := GetOrmer() - sql := `select * from replication_policy where deleted = false and target_id = ?` - - var policies []*models.RepPolicy - - if _, err := o.Raw(sql, targetID).QueryRows(&policies); err != nil { - return nil, err - } - - return policies, nil -} - -// GetRepPolicyByProjectAndTarget ... -func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPolicy, error) { - o := GetOrmer() - sql := `select * from replication_policy where deleted = false and project_id = ? and target_id = ?` - - var policies []*models.RepPolicy - - if _, err := o.Raw(sql, projectID, targetID).QueryRows(&policies); err != nil { - return nil, err - } - - return policies, nil -} - -// UpdateRepPolicy ... -func UpdateRepPolicy(policy *models.RepPolicy) error { - o := GetOrmer() - - sql := `update replication_policy - set project_id = ?, target_id = ?, name = ?, description = ?, cron_str = ?, filters = ?, replicate_deletion = ?, update_time = ? - where id = ?` - - _, err := o.Raw(sql, policy.ProjectID, policy.TargetID, policy.Name, policy.Description, policy.Trigger, policy.Filters, policy.ReplicateDeletion, time.Now(), policy.ID).Exec() - - return err -} - -// DeleteRepPolicy ... -func DeleteRepPolicy(id int64) error { - _, err := GetOrmer().Delete(&models.RepPolicy{ - ID: id, - }) - return err -} - -// AddRepJob ... -func AddRepJob(job models.RepJob) (int64, error) { - o := GetOrmer() - if len(job.Status) == 0 { - job.Status = models.JobPending - } - if len(job.TagList) > 0 { - job.Tags = strings.Join(job.TagList, ",") - } - return o.Insert(&job) -} - -// GetRepJob ... -func GetRepJob(id int64) (*models.RepJob, error) { - o := GetOrmer() - j := models.RepJob{ID: id} - err := o.Read(&j) - if err == orm.ErrNoRows { - return nil, nil - } - genTagListForJob(&j) - return &j, nil -} - -// GetTotalCountOfRepJobs ... -func GetTotalCountOfRepJobs(query ...*models.RepJobQuery) (int64, error) { - qs := repJobQueryConditions(query...) - return qs.Count() -} - -// GetRepJobs ... -func GetRepJobs(query ...*models.RepJobQuery) ([]*models.RepJob, error) { - jobs := []*models.RepJob{} - - qs := repJobQueryConditions(query...) - if len(query) > 0 && query[0] != nil { - qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size) - } - - qs = qs.OrderBy("-UpdateTime") - - if _, err := qs.All(&jobs); err != nil { - return jobs, err - } - - genTagListForJob(jobs...) - - return jobs, nil -} - -func repJobQueryConditions(query ...*models.RepJobQuery) orm.QuerySeter { - qs := GetOrmer().QueryTable(new(models.RepJob)) - if len(query) == 0 || query[0] == nil { - return qs - } - - q := query[0] - if q.PolicyID != 0 { - qs = qs.Filter("PolicyID", q.PolicyID) - } - if len(q.OpUUID) > 0 { - qs = qs.Filter("OpUUID__exact", q.OpUUID) - } - if len(q.Repository) > 0 { - qs = qs.Filter("Repository__icontains", q.Repository) - } - if len(q.Statuses) > 0 { - qs = qs.Filter("Status__in", q.Statuses) - } - if len(q.Operations) > 0 { - qs = qs.Filter("Operation__in", q.Operations) - } - if q.StartTime != nil { - qs = qs.Filter("CreationTime__gte", q.StartTime) - } - if q.EndTime != nil { - qs = qs.Filter("CreationTime__lte", q.EndTime) - } - return qs -} - -// DeleteRepJob ... -func DeleteRepJob(id int64) error { - o := GetOrmer() - _, err := o.Delete(&models.RepJob{ID: id}) - return err -} - -// DeleteRepJobs deletes replication jobs by policy ID -func DeleteRepJobs(policyID int64) error { - _, err := GetOrmer().QueryTable(&models.RepJob{}).Filter("PolicyID", policyID).Delete() - return err -} - -// UpdateRepJobStatus ... -func UpdateRepJobStatus(id int64, status string) error { - o := GetOrmer() - j := models.RepJob{ - ID: id, - Status: status, - UpdateTime: time.Now(), - } - n, err := o.Update(&j, "Status", "UpdateTime") - if n == 0 { - log.Warningf("no records are updated when updating replication job %d", id) - } - return err -} - -// SetRepJobUUID ... -func SetRepJobUUID(id int64, uuid string) error { - o := GetOrmer() - j := models.RepJob{ - ID: id, - UUID: uuid, - } - n, err := o.Update(&j, "UUID") - if n == 0 { - log.Warningf("no records are updated when updating replication job %d", id) - } - return err -} - -func genTagListForJob(jobs ...*models.RepJob) { - for _, j := range jobs { - if len(j.Tags) > 0 { - j.TagList = strings.Split(j.Tags, ",") - } - } -} diff --git a/src/common/dao/replication_job_test.go b/src/common/dao/replication_job_test.go deleted file mode 100644 index 9bfb38689..000000000 --- a/src/common/dao/replication_job_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dao - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDeleteRepJobs(t *testing.T) { - var policyID int64 = 999 - _, err := AddRepJob(models.RepJob{ - PolicyID: policyID, - Repository: "library/hello-world", - Operation: "delete", - Status: "success", - }) - require.Nil(t, err) - _, err = AddRepJob(models.RepJob{ - PolicyID: policyID, - Repository: "library/hello-world", - Operation: "delete", - Status: "success", - }) - require.Nil(t, err) - - jobs, err := GetRepJobs(&models.RepJobQuery{ - PolicyID: policyID, - }) - require.Nil(t, err) - require.Equal(t, 2, len(jobs)) - - err = DeleteRepJobs(policyID) - require.Nil(t, err) - - jobs, err = GetRepJobs(&models.RepJobQuery{ - PolicyID: policyID, - }) - require.Nil(t, err) - assert.Equal(t, 0, len(jobs)) -} diff --git a/src/common/dao/watch_item.go b/src/common/dao/watch_item.go deleted file mode 100644 index 53ea74d3a..000000000 --- a/src/common/dao/watch_item.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dao - -import ( - "time" - - "github.com/goharbor/harbor/src/common/models" -) - -// DefaultDatabaseWatchItemDAO is an instance of DatabaseWatchItemDAO -var DefaultDatabaseWatchItemDAO WatchItemDAO = &DatabaseWatchItemDAO{} - -// WatchItemDAO defines operations about WatchItem -type WatchItemDAO interface { - Add(*models.WatchItem) (int64, error) - DeleteByPolicyID(int64) error - Get(namespace, operation string) ([]models.WatchItem, error) -} - -// DatabaseWatchItemDAO implements interface WatchItemDAO for database -type DatabaseWatchItemDAO struct{} - -// Add a WatchItem -func (d *DatabaseWatchItemDAO) Add(item *models.WatchItem) (int64, error) { - o := GetOrmer() - - var triggerID int64 - now := time.Now() - - sql := "insert into replication_immediate_trigger (policy_id, namespace, on_deletion, on_push, creation_time, update_time) values (?, ?, ?, ?, ?, ?) RETURNING id" - - err := o.Raw(sql, item.PolicyID, item.Namespace, item.OnDeletion, item.OnPush, now, now).QueryRow(&triggerID) - if err != nil { - return 0, err - } - - return triggerID, nil -} - -// DeleteByPolicyID deletes the WatchItem specified by policy ID -func (d *DatabaseWatchItemDAO) DeleteByPolicyID(policyID int64) error { - _, err := GetOrmer().QueryTable(&models.WatchItem{}).Filter("PolicyID", policyID).Delete() - return err -} - -// Get returns WatchItem list according to the namespace and operation -func (d *DatabaseWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) { - qs := GetOrmer().QueryTable(&models.WatchItem{}).Filter("Namespace", namespace) - if operation == "push" { - qs = qs.Filter("OnPush", true) - } else if operation == "delete" { - qs = qs.Filter("OnDeletion", true) - } - - items := []models.WatchItem{} - _, err := qs.All(&items) - return items, err -} diff --git a/src/common/dao/watch_item_test.go b/src/common/dao/watch_item_test.go deleted file mode 100644 index 339ff2d23..000000000 --- a/src/common/dao/watch_item_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dao - -// TODO: This UT makes common DAO depends on replication ng DAOs, comment it out temporarily here -/* -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - common_models "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/replication/ng/dao" - "github.com/goharbor/harbor/src/replication/ng/dao/models" -) - -func TestMethodsOfWatchItem(t *testing.T) { - registryID, err := dao.AddRegistry(&models.Registry{ - Name: "test_target_for_watch_item", - URL: "http://127.0.0.1", - }) - require.Nil(t, err) - defer dao.DeleteRegistry(registryID) - - policyID, err := AddRepPolicy(common_models.RepPolicy{ - Name: "test_policy_for_watch_item", - ProjectID: 1, - TargetID: targetID, - }) - require.Nil(t, err) - defer DeleteRepPolicy(policyID) - - item := &common_models.WatchItem{ - PolicyID: policyID, - Namespace: "library", - OnPush: false, - OnDeletion: true, - } - - // test Add - id, err := DefaultDatabaseWatchItemDAO.Add(item) - require.Nil(t, err) - - // test Get: operation-push - items, err := DefaultDatabaseWatchItemDAO.Get("library", "push") - require.Nil(t, err) - assert.Equal(t, 0, len(items)) - - // test Get: operation-delete - items, err = DefaultDatabaseWatchItemDAO.Get("library", "delete") - require.Nil(t, err) - assert.Equal(t, 1, len(items)) - assert.Equal(t, id, items[0].ID) - assert.Equal(t, "library", items[0].Namespace) - assert.True(t, items[0].OnDeletion) - - // test DeleteByPolicyID - err = DefaultDatabaseWatchItemDAO.DeleteByPolicyID(policyID) - require.Nil(t, err) - items, err = DefaultDatabaseWatchItemDAO.Get("library", "delete") - require.Nil(t, err) - assert.Equal(t, 0, len(items)) -} -*/ diff --git a/src/common/models/base.go b/src/common/models/base.go index edf7de596..be8877cb8 100644 --- a/src/common/models/base.go +++ b/src/common/models/base.go @@ -20,8 +20,6 @@ import ( func init() { orm.RegisterModel( - new(RepPolicy), - new(RepJob), new(User), new(Project), new(Role), @@ -30,7 +28,6 @@ func init() { new(RepoRecord), new(ImgScanOverview), new(ClairVulnTimestamp), - new(WatchItem), new(ProjectMetadata), new(ConfigEntry), new(Label), diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go deleted file mode 100644 index f1188dcf5..000000000 --- a/src/common/models/replication_job.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "time" -) - -const ( - // RepOpTransfer represents the operation of a job to transfer repository to a remote registry/harbor instance. - RepOpTransfer string = "transfer" - // RepOpDelete represents the operation of a job to remove repository from a remote registry/harbor instance. - RepOpDelete string = "delete" - // RepOpSchedule represents the operation of a job to schedule the real replication process - RepOpSchedule string = "schedule" - // RegistryTable is the table name for registry - RegistryTable = "registry" - // RepJobTable is the table name for replication jobs - RepJobTable = "replication_job" - // RepPolicyTable is table name for replication policies - RepPolicyTable = "replication_policy" -) - -// RepPolicy is the model for a replication policy, which associate to a project and a target (destination) -type RepPolicy struct { - ID int64 `orm:"pk;auto;column(id)"` - ProjectID int64 `orm:"column(project_id)" ` - TargetID int64 `orm:"column(target_id)"` - Name string `orm:"column(name)"` - Description string `orm:"column(description)"` - Trigger string `orm:"column(cron_str)"` - Filters string `orm:"column(filters)"` - ReplicateDeletion bool `orm:"column(replicate_deletion)"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add"` - UpdateTime time.Time `orm:"column(update_time);auto_now"` - Deleted bool `orm:"column(deleted)"` -} - -// RepJob is the model for a replication job, which is the execution unit on job service, currently it is used to transfer/remove -// a repository to/from a remote registry instance. -type RepJob struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - Status string `orm:"column(status)" json:"status"` - Repository string `orm:"column(repository)" json:"repository"` - PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` - OpUUID string `orm:"column(op_uuid)" json:"op_uuid"` - Operation string `orm:"column(operation)" json:"operation"` - Tags string `orm:"column(tags)" json:"-"` - TagList []string `orm:"-" json:"tags"` - UUID string `orm:"column(job_uuid)" json:"-"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` -} - -// TableName is required by by beego orm to map RepJob to table replication_job -func (r *RepJob) TableName() string { - return RepJobTable -} - -// TableName is required by by beego orm to map RepPolicy to table replication_policy -func (r *RepPolicy) TableName() string { - return RepPolicyTable -} - -// RepJobQuery holds query conditions for replication job -type RepJobQuery struct { - PolicyID int64 - OpUUID string - Repository string - Statuses []string - Operations []string - StartTime *time.Time - EndTime *time.Time - Pagination -} diff --git a/src/common/models/watch_item.go b/src/common/models/watch_item.go deleted file mode 100644 index 0912b1dd0..000000000 --- a/src/common/models/watch_item.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "time" -) - -// WatchItem ... -type WatchItem struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` - Namespace string `orm:"column(namespace)" json:"namespace"` - OnDeletion bool `orm:"column(on_deletion)" json:"on_deletion"` - OnPush bool `orm:"column(on_push)" json:"on_push"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` -} - -// TableName ... -func (w *WatchItem) TableName() string { - return "replication_immediate_trigger" -} diff --git a/src/common/utils/test/policy_manager.go b/src/common/utils/test/policy_manager.go deleted file mode 100644 index 9a8d36ebb..000000000 --- a/src/common/utils/test/policy_manager.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" -) - -type FakePolicyManager struct { -} - -func (f *FakePolicyManager) GetPolicies(query models.QueryParameter) (*models.ReplicationPolicyQueryResult, error) { - return &models.ReplicationPolicyQueryResult{}, nil -} - -func (f *FakePolicyManager) GetPolicy(id int64) (models.ReplicationPolicy, error) { - return models.ReplicationPolicy{ - ID: 1, - Trigger: &models.Trigger{ - Kind: replication.TriggerKindManual, - }, - }, nil -} -func (f *FakePolicyManager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) { - return 1, nil -} -func (f *FakePolicyManager) UpdatePolicy(models.ReplicationPolicy) error { - return nil -} -func (f *FakePolicyManager) RemovePolicy(int64) error { - return nil -} diff --git a/src/common/utils/test/replication_controllter.go b/src/common/utils/test/replication_controllter.go deleted file mode 100644 index 01306aa24..000000000 --- a/src/common/utils/test/replication_controllter.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -// FakeReplicatoinController ... -type FakeReplicatoinController struct { - FakePolicyManager -} - -// Init initialize replication controller -func (f *FakeReplicatoinController) Init(closing chan struct{}) error { - return nil -} - -// Replicate ... -func (f *FakeReplicatoinController) Replicate(policyID int64, metadata ...map[string]interface{}) error { - return nil -} diff --git a/src/common/utils/test/watch_item.go b/src/common/utils/test/watch_item.go deleted file mode 100644 index 929d1a486..000000000 --- a/src/common/utils/test/watch_item.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "github.com/goharbor/harbor/src/common/models" -) - -// FakeWatchItemDAO is the fake implement for the dao.WatchItemDAO -type FakeWatchItemDAO struct { - items []models.WatchItem -} - -// Add ... -func (f *FakeWatchItemDAO) Add(item *models.WatchItem) (int64, error) { - f.items = append(f.items, *item) - return int64(len(f.items) + 1), nil -} - -// DeleteByPolicyID : delete the WatchItem specified by policy ID -func (f *FakeWatchItemDAO) DeleteByPolicyID(policyID int64) error { - for i, item := range f.items { - if item.PolicyID == policyID { - f.items = append(f.items[:i], f.items[i+1:]...) - break - } - } - return nil -} - -// Get returns WatchItem list according to the namespace and operation -func (f *FakeWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) { - items := []models.WatchItem{} - for _, item := range f.items { - if item.Namespace != namespace { - continue - } - - if operation == "push" { - if item.OnPush { - items = append(items, item) - } - } - - if operation == "delete" { - if item.OnDeletion { - items = append(items, item) - } - } - } - - return items, nil -} diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 0ad8b39e3..1de53ddfd 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -39,8 +39,6 @@ import ( _ "github.com/goharbor/harbor/src/core/auth/ldap" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/filter" - "github.com/goharbor/harbor/src/replication/core" - _ "github.com/goharbor/harbor/src/replication/event" "github.com/goharbor/harbor/src/replication/ng/model" "github.com/goharbor/harbor/tests/apitests/apilib" ) @@ -127,9 +125,6 @@ func init() { beego.Router("/api/repositories/top", &RepositoryAPI{}, "get:GetTopRepos") beego.Router("/api/registries", &RegistryAPI{}, "get:List;post:Post") beego.Router("/api/registries/:id([0-9]+)", &RegistryAPI{}, "get:Get;put:Put;delete:Delete") - beego.Router("/api/policies/replication/:id([0-9]+)", &RepPolicyAPI{}) - beego.Router("/api/policies/replication", &RepPolicyAPI{}, "get:List") - beego.Router("/api/policies/replication", &RepPolicyAPI{}, "post:Post;delete:Delete") beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo") beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert") @@ -140,7 +135,6 @@ func init() { beego.Router("/api/configurations", &ConfigAPI{}) beego.Router("/api/configs", &ConfigAPI{}, "get:GetInternalConfig") beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping") - beego.Router("/api/replications", &ReplicationAPI{}) beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List") beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete") beego.Router("/api/labels/:id([0-9]+)/resources", &LabelAPI{}, "get:ListResources") @@ -183,10 +177,6 @@ func init() { beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel") beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel") - if err := core.Init(make(chan struct{})); err != nil { - log.Fatalf("failed to initialize GlobalController: %v", err) - } - // syncRegistry if err := SyncRegistry(config.GlobalProjectMgr); err != nil { log.Fatalf("failed to sync repositories from registry: %v", err) diff --git a/src/core/api/label.go b/src/core/api/label.go index 702e3cc37..8884e88c9 100644 --- a/src/core/api/label.go +++ b/src/core/api/label.go @@ -23,9 +23,6 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/rbac" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/core" - rep_models "github.com/goharbor/harbor/src/replication/models" ) // LabelAPI handles requests for label management @@ -313,26 +310,28 @@ func (l *LabelAPI) ListResources() { return } - result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{}) - if err != nil { - l.HandleInternalServerError(fmt.Sprintf("failed to get policies: %v", err)) - return - } - policies := []*rep_models.ReplicationPolicy{} - if result != nil { - for _, policy := range result.Policies { - for _, filter := range policy.Filters { - if filter.Kind != replication.FilterItemKindLabel { - continue - } - if filter.Value.(int64) == label.ID { - policies = append(policies, policy) + /* + result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{}) + if err != nil { + l.HandleInternalServerError(fmt.Sprintf("failed to get policies: %v", err)) + return + } + policies := []*rep_models.ReplicationPolicy{} + if result != nil { + for _, policy := range result.Policies { + for _, filter := range policy.Filters { + if filter.Kind != replication.FilterItemKindLabel { + continue + } + if filter.Value.(int64) == label.ID { + policies = append(policies, policy) + } } } } - } + */ resources := map[string]interface{}{} - resources["replication_policies"] = policies + resources["replication_policies"] = nil l.Data["json"] = resources l.ServeJSON() } diff --git a/src/core/api/label_test.go b/src/core/api/label_test.go index 3d2fc0c9b..671ddd043 100644 --- a/src/core/api/label_test.go +++ b/src/core/api/label_test.go @@ -21,12 +21,7 @@ import ( "testing" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/replication" - rep_models "github.com/goharbor/harbor/src/replication/models" - rep_dao "github.com/goharbor/harbor/src/replication/ng/dao" - dao_models "github.com/goharbor/harbor/src/replication/ng/dao/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -438,105 +433,3 @@ func TestLabelAPIDelete(t *testing.T) { runCodeCheckingCases(t, cases...) } - -func TestListResources(t *testing.T) { - // global level label - globalLabelID, err := dao.AddLabel(&models.Label{ - Name: "globel_level_label", - Scope: common.LabelScopeGlobal, - }) - require.Nil(t, err) - defer dao.DeleteLabel(globalLabelID) - - // project level label - projectLabelID, err := dao.AddLabel(&models.Label{ - Name: "project_level_label", - Scope: common.LabelScopeProject, - ProjectID: 1, - }) - require.Nil(t, err) - defer dao.DeleteLabel(projectLabelID) - - registryID, err := rep_dao.AddRegistry(&dao_models.Registry{ - Name: "target_for_testing_label_resource", - URL: "https://192.168.0.1", - }) - require.Nil(t, err) - defer rep_dao.DeleteRegistry(registryID) - - // create a policy references both global and project labels - policyID, err := dao.AddRepPolicy(models.RepPolicy{ - Name: "policy_for_testing_label_resource", - ProjectID: 1, - TargetID: registryID, - Trigger: fmt.Sprintf(`{"kind":"%s"}`, replication.TriggerKindManual), - Filters: fmt.Sprintf(`[{"kind":"%s","value":%d}, {"kind":"%s","value":%d}]`, - replication.FilterItemKindLabel, globalLabelID, - replication.FilterItemKindLabel, projectLabelID), - }) - require.Nil(t, err) - defer dao.DeleteRepPolicy(policyID) - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID), - }, - code: http.StatusUnauthorized, - }, - // 404 - { - request: &testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, 10000), - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 403: global level label - { - request: &testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID), - credential: projAdmin, - }, - code: http.StatusForbidden, - }, - // 403: project level label - { - request: &testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, projectLabelID), - credential: projDeveloper, - }, - code: http.StatusForbidden, - }, - } - runCodeCheckingCases(t, cases...) - - // 200: global level label - resources := map[string][]rep_models.ReplicationPolicy{} - err = handleAndParse(&testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID), - credential: sysAdmin, - }, &resources) - require.Nil(t, err) - policies := resources["replication_policies"] - require.Equal(t, 1, len(policies)) - assert.Equal(t, policyID, policies[0].ID) - - // 200: project level label - resources = map[string][]rep_models.ReplicationPolicy{} - err = handleAndParse(&testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, projectLabelID), - credential: projAdmin, - }, &resources) - require.Nil(t, err) - policies = resources["replication_policies"] - require.Equal(t, 1, len(policies)) - assert.Equal(t, policyID, policies[0].ID) -} diff --git a/src/core/api/models/replication.go b/src/core/api/models/replication.go deleted file mode 100644 index 84617300a..000000000 --- a/src/core/api/models/replication.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "github.com/astaxie/beego/validation" -) - -// Replication defines the properties of model used in replication API -type Replication struct { - PolicyID int64 `json:"policy_id"` -} - -// ReplicationResponse describes response of a replication request, it gives -type ReplicationResponse struct { - UUID string `json:"uuid"` -} - -// Valid ... -func (r *Replication) Valid(v *validation.Validation) { - if r.PolicyID <= 0 { - v.SetError("policy_id", "invalid value") - } -} diff --git a/src/core/api/models/replication_job.go b/src/core/api/models/replication_job.go deleted file mode 100644 index fb742f928..000000000 --- a/src/core/api/models/replication_job.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "github.com/astaxie/beego/validation" -) - -// StopJobsReq holds information needed to stop the jobs for a replication rule -type StopJobsReq struct { - PolicyID int64 `json:"policy_id"` - Status string `json:"status"` -} - -// Valid ... -func (s *StopJobsReq) Valid(v *validation.Validation) { - if s.PolicyID <= 0 { - v.SetError("policy_id", "invalid value") - } - if s.Status != "stop" { - v.SetError("status", "invalid status, valid values: [stop]") - } -} diff --git a/src/core/api/models/replication_policy.go b/src/core/api/models/replication_policy.go deleted file mode 100644 index da1aa2d88..000000000 --- a/src/core/api/models/replication_policy.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "time" - - "github.com/astaxie/beego/validation" - common_models "github.com/goharbor/harbor/src/common/models" - rep_models "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/ng/dao/models" -) - -// ReplicationPolicy defines the data model used in API level -type ReplicationPolicy struct { - ID int64 `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - Filters []rep_models.Filter `json:"filters"` - ReplicateDeletion bool `json:"replicate_deletion"` - Trigger *rep_models.Trigger `json:"trigger"` - Projects []*common_models.Project `json:"projects"` - Registries []*models.Registry `json:"registries"` - CreationTime time.Time `json:"creation_time"` - UpdateTime time.Time `json:"update_time"` - ReplicateExistingImageNow bool `json:"replicate_existing_image_now"` - ErrorJobCount int64 `json:"error_job_count"` -} - -// Valid ... -func (r *ReplicationPolicy) Valid(v *validation.Validation) { - if len(r.Name) == 0 { - v.SetError("name", "can not be empty") - } - - if len(r.Name) > 256 { - v.SetError("name", "max length is 256") - } - - if len(r.Projects) == 0 { - v.SetError("projects", "can not be empty") - } - - if len(r.Registries) == 0 { - v.SetError("targets", "can not be empty") - } - - for i := range r.Filters { - r.Filters[i].Valid(v) - } - - if r.Trigger == nil { - v.SetError("trigger", "can not be empty") - } else { - r.Trigger.Valid(v) - } -} diff --git a/src/core/api/project.go b/src/core/api/project.go index 3903f3861..bfbadf94a 100644 --- a/src/core/api/project.go +++ b/src/core/api/project.go @@ -284,18 +284,6 @@ func (p *ProjectAPI) deletable(projectID int64) (*deletableResp, error) { }, nil } - policies, err := dao.GetRepPolicyByProject(projectID) - if err != nil { - return nil, err - } - - if len(policies) > 0 { - return &deletableResp{ - Deletable: false, - Message: "the project contains replication rules, can not be deleted", - }, nil - } - // Check helm charts number if config.WithChartMuseum() { charts, err := chartController.ListCharts(p.project.Name) diff --git a/src/core/api/registry.go b/src/core/api/registry.go index 235a0de09..ed8b7b8f9 100644 --- a/src/core/api/registry.go +++ b/src/core/api/registry.go @@ -7,7 +7,6 @@ import ( "github.com/goharbor/harbor/src/replication/ng/event" - "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/api/models" "github.com/goharbor/harbor/src/replication/ng" @@ -198,8 +197,8 @@ func (t *RegistryAPI) Delete() { return } - // TODO: Use PolicyManager instead - policies, err := dao.GetRepPolicyByTarget(id) + // TODO: filter the policies by registry ID + _, policies, err := ng.PolicyCtl.List() if err != nil { msg := fmt.Sprintf("Get policies related to registry %d error: %v", id, err) log.Error(msg) diff --git a/src/core/api/replication.go b/src/core/api/replication.go deleted file mode 100644 index 11bf84497..000000000 --- a/src/core/api/replication.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "fmt" - "net/http" - "strings" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils/log" - api_models "github.com/goharbor/harbor/src/core/api/models" - "github.com/goharbor/harbor/src/core/notifier" - "github.com/goharbor/harbor/src/replication/core" - "github.com/goharbor/harbor/src/replication/event/notification" - "github.com/goharbor/harbor/src/replication/event/topic" - - "github.com/docker/distribution/uuid" -) - -// ReplicationAPI handles API calls for replication -type ReplicationAPI struct { - BaseController -} - -// Prepare does authentication and authorization works -func (r *ReplicationAPI) Prepare() { - r.BaseController.Prepare() - if !r.SecurityCtx.IsAuthenticated() { - r.HandleUnauthorized() - return - } - - if !r.SecurityCtx.IsSysAdmin() && !r.SecurityCtx.IsSolutionUser() { - r.HandleForbidden(r.SecurityCtx.GetUsername()) - return - } -} - -// Post trigger a replication according to the specified policy -func (r *ReplicationAPI) Post() { - replication := &api_models.Replication{} - r.DecodeJSONReqAndValidate(replication) - - policy, err := core.GlobalController.GetPolicy(replication.PolicyID) - if err != nil { - r.HandleInternalServerError(fmt.Sprintf("failed to get replication policy %d: %v", replication.PolicyID, err)) - return - } - - if policy.ID == 0 { - r.HandleNotFound(fmt.Sprintf("replication policy %d not found", replication.PolicyID)) - return - } - - count, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{ - PolicyID: replication.PolicyID, - Statuses: []string{models.JobPending, models.JobRunning}, - Operations: []string{models.RepOpTransfer, models.RepOpDelete}, - }) - if err != nil { - r.HandleInternalServerError(fmt.Sprintf("failed to filter jobs of policy %d: %v", - replication.PolicyID, err)) - return - } - if count > 0 { - r.RenderError(http.StatusPreconditionFailed, "policy has running/pending jobs, new replication can not be triggered") - return - } - - opUUID, err := startReplication(replication.PolicyID) - if err != nil { - r.HandleInternalServerError(fmt.Sprintf("failed to publish replication topic for policy %d: %v", replication.PolicyID, err)) - return - } - log.Infof("replication signal for policy %d sent", replication.PolicyID) - - r.Data["json"] = api_models.ReplicationResponse{ - UUID: opUUID, - } - r.ServeJSON() -} - -// startReplication triggers a replication and return the uuid of this replication. -func startReplication(policyID int64) (string, error) { - opUUID := strings.Replace(uuid.Generate().String(), "-", "", -1) - return opUUID, notifier.Publish(topic.StartReplicationTopic, - notification.StartReplicationNotification{ - PolicyID: policyID, - Metadata: map[string]interface{}{ - "op_uuid": opUUID, - }, - }) -} diff --git a/src/core/api/replication_job.go b/src/core/api/replication_job.go deleted file mode 100644 index 32ee1a7b5..000000000 --- a/src/core/api/replication_job.go +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "fmt" - "net/http" - "strconv" - "time" - - "github.com/goharbor/harbor/src/common/dao" - common_http "github.com/goharbor/harbor/src/common/http" - common_job "github.com/goharbor/harbor/src/common/job" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/rbac" - "github.com/goharbor/harbor/src/common/utils/log" - api_models "github.com/goharbor/harbor/src/core/api/models" - "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/replication/core" -) - -// RepJobAPI handles request to /api/replicationJobs /api/replicationJobs/:id/log -type RepJobAPI struct { - BaseController - jobID int64 -} - -// Prepare validates that whether user has system admin role -func (ra *RepJobAPI) Prepare() { - ra.BaseController.Prepare() - if !ra.SecurityCtx.IsAuthenticated() { - ra.HandleUnauthorized() - return - } - - if !(ra.Ctx.Request.Method == http.MethodGet || ra.SecurityCtx.IsSysAdmin()) { - ra.HandleForbidden(ra.SecurityCtx.GetUsername()) - return - } - - if len(ra.GetStringFromPath(":id")) != 0 { - id, err := ra.GetInt64FromPath(":id") - if err != nil { - ra.HandleBadRequest(fmt.Sprintf("invalid ID: %s", ra.GetStringFromPath(":id"))) - return - } - ra.jobID = id - } - -} - -// List filters jobs according to the parameters -func (ra *RepJobAPI) List() { - - policyID, err := ra.GetInt64("policy_id") - if err != nil || policyID <= 0 { - ra.HandleBadRequest(fmt.Sprintf("invalid policy_id: %s", ra.GetString("policy_id"))) - return - } - - policy, err := core.GlobalController.GetPolicy(policyID) - if err != nil { - log.Errorf("failed to get policy %d: %v", policyID, err) - ra.CustomAbort(http.StatusInternalServerError, "") - } - - if policy.ID == 0 { - ra.HandleNotFound(fmt.Sprintf("policy %d not found", policyID)) - return - } - - resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplicationJob) - if !ra.SecurityCtx.Can(rbac.ActionList, resource) { - ra.HandleForbidden(ra.SecurityCtx.GetUsername()) - return - } - - query := &models.RepJobQuery{ - PolicyID: policyID, - // hide the schedule job, the schedule job is used to trigger replication - // for scheduled policy - Operations: []string{models.RepOpTransfer, models.RepOpDelete}, - } - - query.Repository = ra.GetString("repository") - query.Statuses = ra.GetStrings("status") - query.OpUUID = ra.GetString("op_uuid") - - startTimeStr := ra.GetString("start_time") - if len(startTimeStr) != 0 { - i, err := strconv.ParseInt(startTimeStr, 10, 64) - if err != nil { - ra.HandleBadRequest(fmt.Sprintf("invalid start_time: %s", startTimeStr)) - return - } - t := time.Unix(i, 0) - query.StartTime = &t - } - - endTimeStr := ra.GetString("end_time") - if len(endTimeStr) != 0 { - i, err := strconv.ParseInt(endTimeStr, 10, 64) - if err != nil { - ra.HandleBadRequest(fmt.Sprintf("invalid end_time: %s", endTimeStr)) - return - } - t := time.Unix(i, 0) - query.EndTime = &t - } - - query.Page, query.Size = ra.GetPaginationParams() - - total, err := dao.GetTotalCountOfRepJobs(query) - if err != nil { - ra.HandleInternalServerError(fmt.Sprintf("failed to get total count of repository jobs of policy %d: %v", policyID, err)) - return - } - jobs, err := dao.GetRepJobs(query) - if err != nil { - ra.HandleInternalServerError(fmt.Sprintf("failed to get repository jobs, query: %v :%v", query, err)) - return - } - - ra.SetPaginationHeader(total, query.Page, query.Size) - - ra.Data["json"] = jobs - ra.ServeJSON() -} - -// Delete ... -func (ra *RepJobAPI) Delete() { - if ra.jobID == 0 { - ra.HandleBadRequest("ID is nil") - return - } - - job, err := dao.GetRepJob(ra.jobID) - if err != nil { - log.Errorf("failed to get job %d: %v", ra.jobID, err) - ra.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if job == nil { - ra.HandleNotFound(fmt.Sprintf("job %d not found", ra.jobID)) - return - } - - if job.Status == models.JobPending || job.Status == models.JobRunning { - ra.HandleBadRequest(fmt.Sprintf("job is %s, can not be deleted", job.Status)) - return - } - - if err = dao.DeleteRepJob(ra.jobID); err != nil { - log.Errorf("failed to deleted job %d: %v", ra.jobID, err) - ra.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } -} - -// GetLog ... -func (ra *RepJobAPI) GetLog() { - if ra.jobID == 0 { - ra.HandleBadRequest("ID is nil") - return - } - - job, err := dao.GetRepJob(ra.jobID) - if err != nil { - ra.HandleInternalServerError(fmt.Sprintf("failed to get replication job %d: %v", ra.jobID, err)) - return - } - - if job == nil { - ra.HandleNotFound(fmt.Sprintf("replication job %d not found", ra.jobID)) - return - } - - policy, err := core.GlobalController.GetPolicy(job.PolicyID) - if err != nil { - ra.HandleInternalServerError(fmt.Sprintf("failed to get policy %d: %v", job.PolicyID, err)) - return - } - - resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplicationJob) - if !ra.SecurityCtx.Can(rbac.ActionRead, resource) { - ra.HandleForbidden(ra.SecurityCtx.GetUsername()) - return - } - - logBytes, err := utils.GetJobServiceClient().GetJobLog(job.UUID) - if err != nil { - if httpErr, ok := err.(*common_http.Error); ok { - ra.RenderError(httpErr.Code, "") - log.Errorf(fmt.Sprintf("failed to get log of job %d: %d %s", - ra.jobID, httpErr.Code, httpErr.Message)) - return - } - ra.HandleInternalServerError(fmt.Sprintf("failed to get log of job %s: %v", - job.UUID, err)) - return - } - ra.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes))) - ra.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain") - _, err = ra.Ctx.ResponseWriter.Write(logBytes) - if err != nil { - ra.HandleInternalServerError(fmt.Sprintf("failed to write log of job %s: %v", job.UUID, err)) - return - } -} - -// StopJobs stop replication jobs for the policy -func (ra *RepJobAPI) StopJobs() { - req := &api_models.StopJobsReq{} - ra.DecodeJSONReqAndValidate(req) - - policy, err := core.GlobalController.GetPolicy(req.PolicyID) - if err != nil { - ra.HandleInternalServerError(fmt.Sprintf("failed to get policy %d: %v", req.PolicyID, err)) - return - } - - if policy.ID == 0 { - ra.HandleNotFound(fmt.Sprintf("policy %d not found", req.PolicyID)) - return - } - - jobs, err := dao.GetRepJobs(&models.RepJobQuery{ - PolicyID: policy.ID, - Operations: []string{models.RepOpTransfer, models.RepOpDelete}, - }) - if err != nil { - ra.HandleInternalServerError(fmt.Sprintf("failed to list jobs of policy %d: %v", policy.ID, err)) - return - } - for _, job := range jobs { - if err = utils.GetJobServiceClient().PostAction(job.UUID, common_job.JobActionStop); err != nil { - log.Errorf("failed to stop job id-%d uuid-%s: %v", job.ID, job.UUID, err) - continue - } - } -} - -// TODO:add Post handler to call job service API to submit jobs by policy diff --git a/src/core/api/replication_policy.go b/src/core/api/replication_policy.go deleted file mode 100644 index 81c94e3d8..000000000 --- a/src/core/api/replication_policy.go +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "fmt" - - "net/http" - "strconv" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/rbac" - "github.com/goharbor/harbor/src/common/utils/log" - api_models "github.com/goharbor/harbor/src/core/api/models" - "github.com/goharbor/harbor/src/core/promgr" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/core" - rep_models "github.com/goharbor/harbor/src/replication/models" - rep_dao "github.com/goharbor/harbor/src/replication/ng/dao" -) - -// RepPolicyAPI handles /api/replicationPolicies /api/replicationPolicies/:id/enablement -type RepPolicyAPI struct { - BaseController -} - -// Prepare validates whether the user has system admin role -func (pa *RepPolicyAPI) Prepare() { - pa.BaseController.Prepare() - if !pa.SecurityCtx.IsAuthenticated() { - pa.HandleUnauthorized() - return - } - - if !(pa.Ctx.Request.Method == http.MethodGet || pa.SecurityCtx.IsSysAdmin()) { - pa.HandleForbidden(pa.SecurityCtx.GetUsername()) - return - } -} - -// Get ... -func (pa *RepPolicyAPI) Get() { - id := pa.GetIDFromURL() - policy, err := core.GlobalController.GetPolicy(id) - if err != nil { - log.Errorf("failed to get policy %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if policy.ID == 0 { - pa.HandleNotFound(fmt.Sprintf("policy %d not found", id)) - return - } - - resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplication) - if !pa.SecurityCtx.Can(rbac.ActionRead, resource) { - pa.HandleForbidden(pa.SecurityCtx.GetUsername()) - return - } - - ply, err := convertFromRepPolicy(pa.ProjectMgr, policy) - if err != nil { - pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err) - return - } - - pa.Data["json"] = ply - pa.ServeJSON() -} - -// List ... -func (pa *RepPolicyAPI) List() { - queryParam := rep_models.QueryParameter{ - Name: pa.GetString("name"), - } - projectIDStr := pa.GetString("project_id") - if len(projectIDStr) > 0 { - projectID, err := strconv.ParseInt(projectIDStr, 10, 64) - if err != nil || projectID <= 0 { - pa.HandleBadRequest(fmt.Sprintf("invalid project ID: %s", projectIDStr)) - return - } - queryParam.ProjectID = projectID - } - queryParam.Page, queryParam.PageSize = pa.GetPaginationParams() - - result, err := core.GlobalController.GetPolicies(queryParam) - if err != nil { - log.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - var total int64 - policies := []*api_models.ReplicationPolicy{} - if result != nil { - total = result.Total - for _, policy := range result.Policies { - resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplication) - if !pa.SecurityCtx.Can(rbac.ActionRead, resource) { - continue - } - ply, err := convertFromRepPolicy(pa.ProjectMgr, *policy) - if err != nil { - pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err) - return - } - policies = append(policies, ply) - } - } - - pa.SetPaginationHeader(total, queryParam.Page, queryParam.PageSize) - - pa.Data["json"] = policies - pa.ServeJSON() -} - -// Post creates a replicartion policy -func (pa *RepPolicyAPI) Post() { - policy := &api_models.ReplicationPolicy{} - pa.DecodeJSONReqAndValidate(policy) - - // check the name - exist, err := exist(policy.Name) - if err != nil { - pa.HandleInternalServerError(fmt.Sprintf("failed to check the existence of policy %s: %v", policy.Name, err)) - return - } - - if exist { - pa.HandleConflict(fmt.Sprintf("name %s is already used", policy.Name)) - return - } - - // check the existence of projects - for _, project := range policy.Projects { - pro, err := pa.ProjectMgr.Get(project.ProjectID) - if err != nil { - pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err) - return - } - if pro == nil { - pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID)) - return - } - project.Name = pro.Name - } - - // check the existence of targets - for _, r := range policy.Registries { - t, err := rep_dao.GetRegistry(r.ID) - if err != nil { - pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", r.ID, err)) - return - } - - if t == nil { - pa.HandleNotFound(fmt.Sprintf("target %d not found", r.ID)) - return - } - } - - // check the existence of labels - for _, filter := range policy.Filters { - if filter.Kind == replication.FilterItemKindLabel { - labelID := filter.Value.(int64) - label, err := dao.GetLabel(labelID) - if err != nil { - pa.HandleInternalServerError(fmt.Sprintf("failed to get label %d: %v", labelID, err)) - return - } - if label == nil || label.Deleted { - pa.HandleNotFound(fmt.Sprintf("label %d not found", labelID)) - return - } - } - } - - id, err := core.GlobalController.CreatePolicy(convertToRepPolicy(policy)) - if err != nil { - pa.HandleInternalServerError(fmt.Sprintf("failed to create policy: %v", err)) - return - } - - if policy.ReplicateExistingImageNow { - go func() { - if _, err = startReplication(id); err != nil { - log.Errorf("failed to send replication signal for policy %d: %v", id, err) - return - } - log.Infof("replication signal for policy %d sent", id) - }() - } - - pa.Redirect(http.StatusCreated, strconv.FormatInt(id, 10)) -} - -func exist(name string) (bool, error) { - result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{ - Name: name, - }) - if err != nil { - return false, err - } - - for _, policy := range result.Policies { - if policy.Name == name { - return true, nil - } - } - return false, nil -} - -// Put updates the replication policy -func (pa *RepPolicyAPI) Put() { - id := pa.GetIDFromURL() - - originalPolicy, err := core.GlobalController.GetPolicy(id) - if err != nil { - log.Errorf("failed to get policy %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if originalPolicy.ID == 0 { - pa.HandleNotFound(fmt.Sprintf("policy %d not found", id)) - return - } - - policy := &api_models.ReplicationPolicy{} - pa.DecodeJSONReqAndValidate(policy) - - policy.ID = id - - // check the name - if policy.Name != originalPolicy.Name { - exist, err := exist(policy.Name) - if err != nil { - pa.HandleInternalServerError(fmt.Sprintf("failed to check the existence of policy %s: %v", policy.Name, err)) - return - } - - if exist { - pa.HandleConflict(fmt.Sprintf("name %s is already used", policy.Name)) - return - } - } - - // check the existence of projects - for _, project := range policy.Projects { - pro, err := pa.ProjectMgr.Get(project.ProjectID) - if err != nil { - pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err) - return - } - if pro == nil { - pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID)) - return - } - project.Name = pro.Name - } - - // check the existence of targets - for _, r := range policy.Registries { - t, err := rep_dao.GetRegistry(r.ID) - if err != nil { - pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", r.ID, err)) - return - } - - if t == nil { - pa.HandleNotFound(fmt.Sprintf("target %d not found", r.ID)) - return - } - } - - // check the existence of labels - for _, filter := range policy.Filters { - if filter.Kind == replication.FilterItemKindLabel { - labelID := filter.Value.(int64) - label, err := dao.GetLabel(labelID) - if err != nil { - pa.HandleInternalServerError(fmt.Sprintf("failed to get label %d: %v", labelID, err)) - return - } - if label == nil || label.Deleted { - pa.HandleNotFound(fmt.Sprintf("label %d not found", labelID)) - return - } - } - } - - if err = core.GlobalController.UpdatePolicy(convertToRepPolicy(policy)); err != nil { - pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err)) - return - } - - if policy.ReplicateExistingImageNow { - go func() { - if _, err = startReplication(id); err != nil { - log.Errorf("failed to send replication signal for policy %d: %v", id, err) - return - } - log.Infof("replication signal for policy %d sent", id) - }() - } -} - -// Delete the replication policy -func (pa *RepPolicyAPI) Delete() { - id := pa.GetIDFromURL() - - policy, err := core.GlobalController.GetPolicy(id) - if err != nil { - log.Errorf("failed to get policy %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if policy.ID == 0 { - pa.HandleNotFound(fmt.Sprintf("policy %d not found", id)) - return - } - - count, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{ - PolicyID: id, - Statuses: []string{models.JobRunning, models.JobRetrying, models.JobPending}, - // only get the transfer and delete jobs, do not get schedule job - Operations: []string{models.RepOpTransfer, models.RepOpDelete}, - }) - if err != nil { - log.Errorf("failed to filter jobs of policy %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, "") - } - if count > 0 { - pa.CustomAbort(http.StatusPreconditionFailed, "policy has running/retrying/pending jobs, can not be deleted") - } - - if err = core.GlobalController.RemovePolicy(id); err != nil { - log.Errorf("failed to delete policy %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, "") - } -} - -func convertFromRepPolicy(projectMgr promgr.ProjectManager, policy rep_models.ReplicationPolicy) (*api_models.ReplicationPolicy, error) { - if policy.ID == 0 { - return nil, nil - } - - // populate simple properties - ply := &api_models.ReplicationPolicy{ - ID: policy.ID, - Name: policy.Name, - Description: policy.Description, - ReplicateDeletion: policy.ReplicateDeletion, - Trigger: policy.Trigger, - CreationTime: policy.CreationTime, - UpdateTime: policy.UpdateTime, - } - - // populate projects - for _, projectID := range policy.ProjectIDs { - project, err := projectMgr.Get(projectID) - if err != nil { - return nil, err - } - - ply.Projects = append(ply.Projects, project) - } - - // populate targets - for _, targetID := range policy.TargetIDs { - r, err := rep_dao.GetRegistry(targetID) - if err != nil { - return nil, err - } - r.AccessSecret = "" - ply.Registries = append(ply.Registries, r) - } - - // populate label used in label filter - for _, filter := range policy.Filters { - if filter.Kind == replication.FilterItemKindLabel { - labelID := filter.Value.(int64) - label, err := dao.GetLabel(labelID) - if err != nil { - return nil, err - } - filter.Value = label - } - ply.Filters = append(ply.Filters, filter) - } - - // TODO call the method from replication controller - errJobCount, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{ - PolicyID: policy.ID, - Statuses: []string{models.JobError}, - }) - if err != nil { - return nil, err - } - ply.ErrorJobCount = errJobCount - - return ply, nil -} - -func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.ReplicationPolicy { - if policy == nil { - return rep_models.ReplicationPolicy{} - } - - ply := rep_models.ReplicationPolicy{ - ID: policy.ID, - Name: policy.Name, - Description: policy.Description, - Filters: policy.Filters, - ReplicateDeletion: policy.ReplicateDeletion, - Trigger: policy.Trigger, - CreationTime: policy.CreationTime, - UpdateTime: policy.UpdateTime, - } - - for _, project := range policy.Projects { - ply.ProjectIDs = append(ply.ProjectIDs, project.ProjectID) - ply.Namespaces = append(ply.Namespaces, project.Name) - } - - for _, r := range policy.Registries { - ply.TargetIDs = append(ply.TargetIDs, r.ID) - } - - return ply -} diff --git a/src/core/api/replication_policy_test.go b/src/core/api/replication_policy_test.go deleted file mode 100644 index 11d564ded..000000000 --- a/src/core/api/replication_policy_test.go +++ /dev/null @@ -1,711 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package api - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" - - "github.com/goharbor/harbor/src/common/dao/project" - - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - api_models "github.com/goharbor/harbor/src/core/api/models" - "github.com/goharbor/harbor/src/replication" - rep_models "github.com/goharbor/harbor/src/replication/models" - dao_models "github.com/goharbor/harbor/src/replication/ng/dao/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var ( - repPolicyAPIBasePath = "/api/policies/replication" - policyName = "testPolicy" - projectID int64 = 1 - targetID int64 - policyID int64 - labelID2 int64 -) - -func TestRepPolicyAPIPost(t *testing.T) { - postFunc := func(resp *httptest.ResponseRecorder) error { - id, err := parseResourceID(resp) - if err != nil { - return err - } - policyID = id - return nil - } - - CommonAddRegistry() - targetID = int64(CommonGetRegistry()) - - var err error - labelID2, err = dao.AddLabel(&models.Label{ - Name: "label_for_replication_filter", - Scope: "g", - }) - require.Nil(t, err) - defer dao.DeleteLabel(labelID2) - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - }, - code: http.StatusUnauthorized, - }, - // 403 - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - credential: nonSysAdmin, - }, - code: http.StatusForbidden, - }, - - // 400, invalid name - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{}, - credential: sysAdmin, - }, - code: http.StatusBadRequest, - }, - // 400, invalid projects - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - }, - credential: sysAdmin, - }, - code: http.StatusBadRequest, - }, - // 400, invalid targets - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - }, - credential: sysAdmin, - }, - code: http.StatusBadRequest, - }, - // 400, invalid filters - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: targetID, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: "invalid_filter_kind", - Pattern: "", - }, - }, - }, - credential: sysAdmin, - }, - code: http.StatusBadRequest, - }, - // 400, invalid trigger - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: targetID, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }, - }, - Trigger: &rep_models.Trigger{ - Kind: "invalid_trigger_kind", - }, - }, - credential: sysAdmin, - }, - code: http.StatusBadRequest, - }, - // 404, project not found - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: 10000, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: targetID, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }, - }, - Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindManual, - }, - }, - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 404, target not found - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: 10000, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }, - }, - Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindManual, - }, - }, - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 404, label not found - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: targetID, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }, - { - Kind: replication.FilterItemKindLabel, - Value: 10000, - }, - }, - Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindManual, - }, - }, - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 201 - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: targetID, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }, - { - Kind: replication.FilterItemKindLabel, - Value: labelID2, - }, - }, - Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindManual, - }, - }, - credential: sysAdmin, - }, - code: http.StatusCreated, - postFunc: postFunc, - }, - // 409 - { - request: &testingRequest{ - method: http.MethodPost, - url: repPolicyAPIBasePath, - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: targetID, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }, - { - Kind: replication.FilterItemKindLabel, - Value: labelID2, - }, - }, - Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindManual, - }, - }, - credential: sysAdmin, - }, - code: http.StatusConflict, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestRepPolicyAPIGet(t *testing.T) { - - cases := []*codeCheckingCase{ - // 404 - { - request: &testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000), - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 401 - { - request: &testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), - }, - code: http.StatusUnauthorized, - }, - } - - runCodeCheckingCases(t, cases...) - - // 200 - policy := &api_models.ReplicationPolicy{} - err := handleAndParse( - &testingRequest{ - method: http.MethodGet, - url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), - credential: sysAdmin, - }, policy) - require.Nil(t, err) - assert.Equal(t, policyID, policy.ID) - assert.Equal(t, policyName, policy.Name) - assert.Equal(t, 2, len(policy.Filters)) - found := false - for _, filter := range policy.Filters { - if filter.Kind == replication.FilterItemKindLabel { - found = true - label, ok := filter.Value.(map[string]interface{}) - if assert.True(t, ok) { - id := int64(label["id"].(float64)) - deleted := label["deleted"].(bool) - assert.Equal(t, labelID2, id) - assert.True(t, deleted) - } - } - } - assert.True(t, found) -} - -func TestRepPolicyAPIList(t *testing.T) { - projectAdmin := models.User{ - Username: "project_admin", - Password: "ProjectAdmin", - Email: "project_admin@test.com", - } - projectDev := models.User{ - Username: "project_dev", - Password: "ProjectDev", - Email: "project_dev@test.com", - } - var proAdminPMID, proDevPMID int - proAdminID, err := dao.Register(projectAdmin) - if err != nil { - panic(err) - } - defer dao.DeleteUser(int(proAdminID)) - if proAdminPMID, err = project.AddProjectMember(models.Member{ - ProjectID: 1, - Role: models.PROJECTADMIN, - EntityID: int(proAdminID), - EntityType: common.UserMember, - }); err != nil { - panic(err) - } - defer project.DeleteProjectMemberByID(proAdminPMID) - - proDevID, err := dao.Register(projectDev) - if err != nil { - panic(err) - } - defer dao.DeleteUser(int(proDevID)) - - if proDevPMID, err = project.AddProjectMember(models.Member{ - ProjectID: 1, - Role: models.DEVELOPER, - EntityID: int(proDevID), - EntityType: common.UserMember, - }); err != nil { - panic(err) - } - defer project.DeleteProjectMemberByID(proDevPMID) - - // 400: invalid project ID - runCodeCheckingCases(t, &codeCheckingCase{ - request: &testingRequest{ - method: http.MethodGet, - url: repPolicyAPIBasePath, - queryStruct: struct { - ProjectID int64 `url:"project_id"` - }{ - ProjectID: -1, - }, - credential: sysAdmin, - }, - code: http.StatusBadRequest, - }) - - // 200 system admin - policies := []*api_models.ReplicationPolicy{} - err = handleAndParse( - &testingRequest{ - method: http.MethodGet, - url: repPolicyAPIBasePath, - queryStruct: struct { - ProjectID int64 `url:"project_id"` - Name string `url:"name"` - }{ - ProjectID: projectID, - Name: policyName, - }, - credential: sysAdmin, - }, &policies) - require.Nil(t, err) - require.Equal(t, 1, len(policies)) - assert.Equal(t, policyID, policies[0].ID) - assert.Equal(t, policyName, policies[0].Name) - - // 200 project admin - policies = []*api_models.ReplicationPolicy{} - err = handleAndParse( - &testingRequest{ - method: http.MethodGet, - url: repPolicyAPIBasePath, - queryStruct: struct { - ProjectID int64 `url:"project_id"` - Name string `url:"name"` - }{ - ProjectID: projectID, - Name: policyName, - }, - credential: &usrInfo{ - Name: projectAdmin.Username, - Passwd: projectAdmin.Password, - }, - }, &policies) - require.Nil(t, err) - require.Equal(t, 1, len(policies)) - assert.Equal(t, policyID, policies[0].ID) - assert.Equal(t, policyName, policies[0].Name) - - // 200 project developer - policies = []*api_models.ReplicationPolicy{} - err = handleAndParse( - &testingRequest{ - method: http.MethodGet, - url: repPolicyAPIBasePath, - queryStruct: struct { - ProjectID int64 `url:"project_id"` - Name string `url:"name"` - }{ - ProjectID: projectID, - Name: policyName, - }, - credential: &usrInfo{ - Name: projectDev.Username, - Passwd: projectDev.Password, - }, - }, &policies) - require.Nil(t, err) - require.Equal(t, 0, len(policies)) - - // 200 - policies = []*api_models.ReplicationPolicy{} - err = handleAndParse( - &testingRequest{ - method: http.MethodGet, - url: repPolicyAPIBasePath, - queryStruct: struct { - ProjectID int64 `url:"project_id"` - Name string `url:"name"` - }{ - ProjectID: projectID, - Name: "non_exist_policy", - }, - credential: sysAdmin, - }, &policies) - require.Nil(t, err) - require.Equal(t, 0, len(policies)) -} - -func TestRepPolicyAPIPut(t *testing.T) { - cases := []*codeCheckingCase{ - // 404 - { - request: &testingRequest{ - method: http.MethodPut, - url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000), - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 400, invalid trigger - { - request: &testingRequest{ - method: http.MethodPut, - url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: targetID, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }, - }, - Trigger: &rep_models.Trigger{ - Kind: "invalid_trigger_kind", - }, - }, - credential: sysAdmin, - }, - code: http.StatusBadRequest, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodPut, - url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), - bodyJSON: &api_models.ReplicationPolicy{ - Name: policyName, - Projects: []*models.Project{ - { - ProjectID: projectID, - }, - }, - Registries: []*dao_models.Registry{ - { - ID: targetID, - }, - }, - Filters: []rep_models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }, - }, - Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindImmediate, - }, - }, - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestRepPolicyAPIDelete(t *testing.T) { - cases := []*codeCheckingCase{ - // 404 - { - request: &testingRequest{ - method: http.MethodDelete, - url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000), - credential: sysAdmin, - }, - code: http.StatusNotFound, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodDelete, - url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), - credential: sysAdmin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} - -func TestConvertToRepPolicy(t *testing.T) { - cases := []struct { - input *api_models.ReplicationPolicy - expected rep_models.ReplicationPolicy - }{ - { - input: nil, - expected: rep_models.ReplicationPolicy{}, - }, - { - input: &api_models.ReplicationPolicy{ - ID: 1, - Name: "policy", - Description: "description", - Filters: []rep_models.Filter{ - { - Kind: "filter_kind_01", - Pattern: "*", - }, - }, - ReplicateDeletion: true, - Trigger: &rep_models.Trigger{ - Kind: "trigger_kind_01", - }, - Projects: []*models.Project{ - { - ProjectID: 1, - Name: "library", - }, - }, - Registries: []*dao_models.Registry{ - { - ID: 1, - }, - }, - }, - expected: rep_models.ReplicationPolicy{ - ID: 1, - Name: "policy", - Description: "description", - Filters: []rep_models.Filter{ - { - Kind: "filter_kind_01", - Pattern: "*", - }, - }, - ReplicateDeletion: true, - Trigger: &rep_models.Trigger{ - Kind: "trigger_kind_01", - }, - ProjectIDs: []int64{1}, - Namespaces: []string{"library"}, - TargetIDs: []int64{1}, - }, - }, - } - - for _, c := range cases { - assert.EqualValues(t, c.expected, convertToRepPolicy(c.input)) - } -} diff --git a/src/core/api/replication_test.go b/src/core/api/replication_test.go deleted file mode 100644 index 6177ae6b5..000000000 --- a/src/core/api/replication_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package api - -import ( - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - api_models "github.com/goharbor/harbor/src/core/api/models" - "github.com/goharbor/harbor/src/replication" - rep_dao "github.com/goharbor/harbor/src/replication/ng/dao" - dao_models "github.com/goharbor/harbor/src/replication/ng/dao/models" -) - -const ( - replicationAPIBaseURL = "/api/replications" -) - -func TestReplicationAPIPost(t *testing.T) { - registryID, err := rep_dao.AddRegistry( - &dao_models.Registry{ - Name: "test_replication_target", - URL: "127.0.0.1", - AccessKey: "username", - AccessSecret: "password", - }) - require.Nil(t, err) - defer rep_dao.DeleteRegistry(registryID) - - policyID, err := dao.AddRepPolicy( - models.RepPolicy{ - Name: "test_replication_policy", - ProjectID: 1, - TargetID: targetID, - Trigger: fmt.Sprintf("{\"kind\":\"%s\"}", replication.TriggerKindManual), - }) - require.Nil(t, err) - defer dao.DeleteRepPolicy(policyID) - - cases := []*codeCheckingCase{ - // 401 - { - request: &testingRequest{ - method: http.MethodPost, - url: replicationAPIBaseURL, - bodyJSON: &api_models.Replication{ - PolicyID: policyID, - }, - }, - code: http.StatusUnauthorized, - }, - // 404 - { - request: &testingRequest{ - method: http.MethodPost, - url: replicationAPIBaseURL, - bodyJSON: &api_models.Replication{ - PolicyID: 10000, - }, - credential: admin, - }, - code: http.StatusNotFound, - }, - // 200 - { - request: &testingRequest{ - method: http.MethodPost, - url: replicationAPIBaseURL, - bodyJSON: &api_models.Replication{ - PolicyID: policyID, - }, - credential: admin, - }, - code: http.StatusOK, - }, - } - - runCodeCheckingCases(t, cases...) -} diff --git a/src/core/main.go b/src/core/main.go index 0f7f1e37f..1ea4a19a7 100644 --- a/src/core/main.go +++ b/src/core/main.go @@ -38,8 +38,6 @@ import ( "github.com/goharbor/harbor/src/core/filter" "github.com/goharbor/harbor/src/core/proxy" "github.com/goharbor/harbor/src/core/service/token" - "github.com/goharbor/harbor/src/replication/core" - _ "github.com/goharbor/harbor/src/replication/event" ) const ( @@ -134,10 +132,6 @@ func main() { closing := make(chan struct{}) go gracefulShutdown(closing) - if err := core.Init(closing); err != nil { - log.Errorf("failed to initialize the replication controller: %v", err) - } - filter.Init() beego.InsertFilter("/*", beego.BeforeRouter, filter.SecurityFilter) beego.InsertFilter("/*", beego.BeforeRouter, filter.ReadonlyFilter) diff --git a/src/core/router.go b/src/core/router.go index 237d6e3ad..27f8b15d2 100644 --- a/src/core/router.go +++ b/src/core/router.go @@ -88,9 +88,6 @@ func initRouters() { beego.Router("/api/repositories/*/tags/:tag/manifest", &api.RepositoryAPI{}, "get:GetManifests") beego.Router("/api/repositories/*/signatures", &api.RepositoryAPI{}, "get:GetSignatures") beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos") - beego.Router("/api/jobs/replication/", &api.RepJobAPI{}, "get:List;put:StopJobs") - beego.Router("/api/jobs/replication/:id([0-9]+)", &api.RepJobAPI{}) - beego.Router("/api/jobs/replication/:id([0-9]+)/log", &api.RepJobAPI{}, "get:GetLog") beego.Router("/api/jobs/scan/:id([0-9]+)/log", &api.ScanJobAPI{}, "get:GetLog") beego.Router("/api/system/gc", &api.GCAPI{}, "get:List") @@ -99,9 +96,6 @@ func initRouters() { beego.Router("/api/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post") - beego.Router("/api/policies/replication/:id([0-9]+)", &api.RepPolicyAPI{}) - beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "get:List") - beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "post:Post") beego.Router("/api/logs", &api.LogAPI{}) beego.Router("/api/replication/adapters", &api.ReplicationAdapterAPI{}, "get:List") @@ -116,7 +110,6 @@ func initRouters() { beego.Router("/api/internal/configurations", &api.ConfigAPI{}, "get:GetInternalConfig;put:Put") beego.Router("/api/configurations", &api.ConfigAPI{}, "get:Get;put:Put") beego.Router("/api/statistics", &api.StatisticAPI{}) - beego.Router("/api/replications", &api.ReplicationAPI{}) beego.Router("/api/labels", &api.LabelAPI{}, "post:Post;get:List") beego.Router("/api/labels/:id([0-9]+)", &api.LabelAPI{}, "get:Get;put:Put;delete:Delete") beego.Router("/api/labels/:id([0-9]+)/resources", &api.LabelAPI{}, "get:ListResources") diff --git a/src/replication/consts.go b/src/replication/consts.go deleted file mode 100644 index 70b1781c8..000000000 --- a/src/replication/consts.go +++ /dev/null @@ -1,27 +0,0 @@ -package replication - -const ( - // FilterItemKindProject : Kind of filter item is 'project' - FilterItemKindProject = "project" - // FilterItemKindRepository : Kind of filter item is 'repository' - FilterItemKindRepository = "repository" - // FilterItemKindTag : Kind of filter item is 'tag' - FilterItemKindTag = "tag" - // FilterItemKindLabel : Kind of filter item is 'label' - FilterItemKindLabel = "label" - - // AdaptorKindHarbor : Kind of adaptor of Harbor - AdaptorKindHarbor = "Harbor" - - // TriggerKindImmediate : Kind of trigger is 'Immediate' - TriggerKindImmediate = "Immediate" - // TriggerKindSchedule : Kind of trigger is 'Scheduled' - TriggerKindSchedule = "Scheduled" - // TriggerKindManual : Kind of trigger is 'Manual' - TriggerKindManual = "Manual" - - // TriggerScheduleDaily : type of scheduling is 'Daily' - TriggerScheduleDaily = "Daily" - // TriggerScheduleWeekly : type of scheduling is 'Weekly' - TriggerScheduleWeekly = "Weekly" -) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go deleted file mode 100644 index c165f699b..000000000 --- a/src/replication/core/controller.go +++ /dev/null @@ -1,337 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package core - -import ( - "fmt" - "reflect" - "strings" - "time" - - common_models "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/ng/model" - "github.com/goharbor/harbor/src/replication/ng/registry" - "github.com/goharbor/harbor/src/replication/policy" - "github.com/goharbor/harbor/src/replication/replicator" - "github.com/goharbor/harbor/src/replication/source" - "github.com/goharbor/harbor/src/replication/trigger" - - "github.com/docker/distribution/uuid" - "github.com/goharbor/harbor/src/replication/ng" -) - -// Controller defines the methods that a replicatoin controllter should implement -type Controller interface { - policy.Manager - Init(closing chan struct{}) error - Replicate(policyID int64, metadata ...map[string]interface{}) error -} - -// DefaultController is core module to cordinate and control the overall workflow of the -// replication modules. -type DefaultController struct { - // Indicate whether the controller has been initialized or not - initialized bool - - // Manage the policies - policyManager policy.Manager - - // Manage the registries - registryManager registry.Manager - - // Handle the things related with source - sourcer *source.Sourcer - - // Manage the triggers of policies - triggerManager *trigger.Manager - - // Handle the replication work - replicator replicator.Replicator -} - -// Keep controller as singleton instance -var ( - GlobalController Controller -) - -// ControllerConfig includes related configurations required by the controller -type ControllerConfig struct { - // The capacity of the cache storing enabled triggers - CacheCapacity int -} - -// NewDefaultController is the constructor of DefaultController. -func NewDefaultController(cfg ControllerConfig) *DefaultController { - // Controller refer the default instances - ctl := &DefaultController{ - policyManager: policy.NewDefaultManager(), - registryManager: registry.NewDefaultManager(), - sourcer: source.NewSourcer(), - triggerManager: trigger.NewManager(cfg.CacheCapacity), - } - - ctl.replicator = replicator.NewDefaultReplicator(utils.GetJobServiceClient()) - - return ctl -} - -// Init initializes GlobalController and replication related managers -func Init(closing chan struct{}) error { - GlobalController = NewDefaultController(ControllerConfig{}) - err := GlobalController.Init(closing) - if err != nil { - return err - } - - return ng.Init() -} - -// Init will initialize the controller and the sub components -func (ctl *DefaultController) Init(closing chan struct{}) error { - if ctl.initialized { - return nil - } - - // Initialize sourcer - ctl.sourcer.Init() - - ctl.initialized = true - - // Start registry health checker to regularly check registries' health status - go registry.NewHealthChecker(time.Second*30, closing).Run() - - return nil -} - -// CreatePolicy is used to create a new policy and enable it if necessary -func (ctl *DefaultController) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, error) { - id, err := ctl.policyManager.CreatePolicy(newPolicy) - if err != nil { - return 0, err - } - - newPolicy.ID = id - if err = ctl.triggerManager.SetupTrigger(&newPolicy); err != nil { - return 0, err - } - - return id, nil -} - -// UpdatePolicy will update the policy with new content. -// Parameter updatedPolicy must have the ID of the updated policy. -func (ctl *DefaultController) UpdatePolicy(updatedPolicy models.ReplicationPolicy) error { - id := updatedPolicy.ID - originPolicy, err := ctl.policyManager.GetPolicy(id) - if err != nil { - return err - } - - if originPolicy.ID == 0 { - return fmt.Errorf("policy %d not found", id) - } - - reset := false - if updatedPolicy.Trigger.Kind != originPolicy.Trigger.Kind { - reset = true - } else { - switch updatedPolicy.Trigger.Kind { - case replication.TriggerKindSchedule: - if !originPolicy.Trigger.ScheduleParam.Equal(updatedPolicy.Trigger.ScheduleParam) { - reset = true - } - case replication.TriggerKindImmediate: - // Always reset immediate trigger as it is relevant with namespaces - reset = true - default: - // manual trigger, no need to reset - } - } - - if err = ctl.policyManager.UpdatePolicy(updatedPolicy); err != nil { - return err - } - - if reset { - if err = ctl.triggerManager.UnsetTrigger(&originPolicy); err != nil { - return err - } - - return ctl.triggerManager.SetupTrigger(&updatedPolicy) - } - - return nil -} - -// RemovePolicy will remove the specified policy and clean the related settings -func (ctl *DefaultController) RemovePolicy(policyID int64) error { - // TODO check pre-conditions - - policy, err := ctl.policyManager.GetPolicy(policyID) - if err != nil { - return err - } - - if policy.ID == 0 { - return fmt.Errorf("policy %d not found", policyID) - } - - if err = ctl.triggerManager.UnsetTrigger(&policy); err != nil { - return err - } - - return ctl.policyManager.RemovePolicy(policyID) -} - -// GetPolicy is delegation of GetPolicy of Policy.Manager -func (ctl *DefaultController) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { - return ctl.policyManager.GetPolicy(policyID) -} - -// GetPolicies is delegation of GetPoliciemodels.ReplicationPolicy{}s of Policy.Manager -func (ctl *DefaultController) GetPolicies(query models.QueryParameter) (*models.ReplicationPolicyQueryResult, error) { - return ctl.policyManager.GetPolicies(query) -} - -// Replicate starts one replication defined in the specified policy; -// Can be launched by the API layer and related triggers. -func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]interface{}) error { - policy, err := ctl.GetPolicy(policyID) - if err != nil { - return err - } - if policy.ID == 0 { - return fmt.Errorf("policy %d not found", policyID) - } - - // prepare candidates for replication - candidates := getCandidates(&policy, ctl.sourcer, metadata...) - if len(candidates) == 0 { - log.Debugf("replication candidates are null, no further action needed") - } - - registries := []*model.Registry{} - for _, targetID := range policy.TargetIDs { - r, err := ctl.registryManager.Get(targetID) - if err != nil { - return err - } - registries = append(registries, r) - } - - // Get operation uuid from metadata, if none provided, generate one. - opUUID, err := getOpUUID(metadata...) - if err != nil { - return err - } - - // submit the replication - return ctl.replicator.Replicate(&replicator.Replication{ - PolicyID: policyID, - OpUUID: opUUID, - Candidates: candidates, - Registries: registries, - }) -} - -func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer, - metadata ...map[string]interface{}) []models.FilterItem { - candidates := []models.FilterItem{} - if len(metadata) > 0 { - meta := metadata[0]["candidates"] - if meta != nil { - cands, ok := meta.([]models.FilterItem) - if ok { - candidates = append(candidates, cands...) - } - } - } - - if len(candidates) == 0 { - for _, namespace := range policy.Namespaces { - candidates = append(candidates, models.FilterItem{ - Kind: replication.FilterItemKindProject, - Value: namespace, - Operation: common_models.RepOpTransfer, - }) - } - } - - filterChain := buildFilterChain(policy, sourcer) - - return filterChain.DoFilter(candidates) -} - -func buildFilterChain(policy *models.ReplicationPolicy, sourcer *source.Sourcer) source.FilterChain { - filters := []source.Filter{} - - fm := map[string][]models.Filter{} - for _, filter := range policy.Filters { - fm[filter.Kind] = append(fm[filter.Kind], filter) - } - - registry := sourcer.GetAdaptor(replication.AdaptorKindHarbor) - // repository filter - pattern := "" - repoFilters := fm[replication.FilterItemKindRepository] - if len(repoFilters) > 0 { - pattern = repoFilters[0].Value.(string) - } - filters = append(filters, - source.NewRepositoryFilter(pattern, registry)) - // tag filter - pattern = "" - tagFilters := fm[replication.FilterItemKindTag] - if len(tagFilters) > 0 { - pattern = tagFilters[0].Value.(string) - } - filters = append(filters, - source.NewTagFilter(pattern, registry)) - // label filters - var labelID int64 - for _, labelFilter := range fm[replication.FilterItemKindLabel] { - labelID = labelFilter.Value.(int64) - filters = append(filters, source.NewLabelFilter(labelID)) - } - - return source.NewDefaultFilterChain(filters) -} - -// getOpUUID get operation uuid from metadata or generate one if none found. -func getOpUUID(metadata ...map[string]interface{}) (string, error) { - if len(metadata) <= 0 { - return strings.Replace(uuid.Generate().String(), "-", "", -1), nil - } - - opUUID, ok := metadata[0]["op_uuid"] - if !ok { - return strings.Replace(uuid.Generate().String(), "-", "", -1), nil - } - - id, ok := opUUID.(string) - if !ok { - return "", fmt.Errorf("operation uuid should have type 'string', but got '%s'", reflect.TypeOf(opUUID).Name()) - } - - if id == "" { - return "", fmt.Errorf("provided operation uuid is empty") - } - - return id, nil -} diff --git a/src/replication/core/controller_test.go b/src/replication/core/controller_test.go deleted file mode 100644 index 8fad7ea97..000000000 --- a/src/replication/core/controller_test.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package core - -import ( - "os" - "testing" - - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/ng/registry" - "github.com/goharbor/harbor/src/replication/source" - "github.com/goharbor/harbor/src/replication/trigger" - "github.com/stretchr/testify/assert" -) - -func TestMain(m *testing.M) { - GlobalController = &DefaultController{ - policyManager: &test.FakePolicyManager{}, - registryManager: registry.NewDefaultManager(), - sourcer: source.NewSourcer(), - triggerManager: trigger.NewManager(0), - } - os.Exit(m.Run()) -} - -func TestInit(t *testing.T) { - assert.Nil(t, GlobalController.Init(make(chan struct{}))) -} - -func TestCreatePolicy(t *testing.T) { - _, err := GlobalController.CreatePolicy(models.ReplicationPolicy{ - Trigger: &models.Trigger{ - Kind: replication.TriggerKindManual, - }, - }) - assert.Nil(t, err) -} - -func TestUpdatePolicy(t *testing.T) { - assert.Nil(t, GlobalController.UpdatePolicy(models.ReplicationPolicy{ - ID: 2, - Trigger: &models.Trigger{ - Kind: replication.TriggerKindManual, - }, - })) -} - -func TestRemovePolicy(t *testing.T) { - assert.Nil(t, GlobalController.RemovePolicy(1)) -} - -func TestGetPolicy(t *testing.T) { - _, err := GlobalController.GetPolicy(1) - assert.Nil(t, err) -} - -func TestGetPolicies(t *testing.T) { - _, err := GlobalController.GetPolicies(models.QueryParameter{}) - assert.Nil(t, err) -} - -func TestReplicate(t *testing.T) { - // TODO -} - -func TestGetCandidates(t *testing.T) { - policy := &models.ReplicationPolicy{ - ID: 1, - Filters: []models.Filter{ - { - Kind: replication.FilterItemKindTag, - Value: "*", - }, - }, - Trigger: &models.Trigger{ - Kind: replication.TriggerKindImmediate, - }, - } - - sourcer := source.NewSourcer() - - candidates := []models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world:release-1.0", - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world:latest", - }, - } - metadata := map[string]interface{}{ - "candidates": candidates, - } - result := getCandidates(policy, sourcer, metadata) - assert.Equal(t, 2, len(result)) - - policy.Filters = []models.Filter{ - { - Kind: replication.FilterItemKindTag, - Value: "release-*", - }, - } - result = getCandidates(policy, sourcer, metadata) - assert.Equal(t, 1, len(result)) - - // test label filter - test.InitDatabaseFromEnv() - policy.Filters = []models.Filter{ - { - Kind: replication.FilterItemKindLabel, - Value: int64(1), - }, - } - result = getCandidates(policy, sourcer, metadata) - assert.Equal(t, 0, len(result)) -} - -func TestBuildFilterChain(t *testing.T) { - policy := &models.ReplicationPolicy{ - ID: 1, - Filters: []models.Filter{ - { - Kind: replication.FilterItemKindRepository, - Value: "*", - }, - - { - Kind: replication.FilterItemKindTag, - Value: "*", - }, - - { - Kind: replication.FilterItemKindLabel, - Value: int64(1), - }, - }, - } - - sourcer := source.NewSourcer() - - chain := buildFilterChain(policy, sourcer) - assert.Equal(t, 3, len(chain.Filters())) -} - -func TestGetOpUUID(t *testing.T) { - uuid, err := getOpUUID() - assert.Nil(t, err) - assert.NotEmpty(t, uuid) - - uuid, err = getOpUUID(map[string]interface{}{ - "name": "test", - }) - assert.Nil(t, err) - assert.NotEmpty(t, uuid) - - uuid, err = getOpUUID(map[string]interface{}{ - "op_uuid": 0, - }) - assert.NotNil(t, err) - - uuid, err = getOpUUID(map[string]interface{}{ - "op_uuid": "0", - }) - assert.Nil(t, err) - assert.Equal(t, uuid, "0") -} diff --git a/src/replication/event/init.go b/src/replication/event/init.go deleted file mode 100644 index a055177c8..000000000 --- a/src/replication/event/init.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package event - -import ( - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/notifier" - "github.com/goharbor/harbor/src/replication/event/topic" -) - -// Subscribe related topics -func init() { - // Listen the related event topics - handlers := map[string]notifier.NotificationHandler{ - topic.StartReplicationTopic: &StartReplicationHandler{}, - topic.ReplicationEventTopicOnPush: &OnPushHandler{}, - topic.ReplicationEventTopicOnDeletion: &OnDeletionHandler{}, - } - - for topic, handler := range handlers { - if err := notifier.Subscribe(topic, handler); err != nil { - log.Errorf("failed to subscribe topic %s: %v", topic, err) - continue - } - log.Debugf("topic %s is subscribed", topic) - } -} diff --git a/src/replication/event/notification/notification.go b/src/replication/event/notification/notification.go deleted file mode 100644 index e59b149f4..000000000 --- a/src/replication/event/notification/notification.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package notification - -// OnPushNotification contains the data required by this handler -type OnPushNotification struct { - // The name of the image that is being pushed - Image string -} - -// OnDeletionNotification contains the data required by this handler -type OnDeletionNotification struct { - // The name of the image that is being deleted - Image string -} - -// StartReplicationNotification contains data required by this handler -type StartReplicationNotification struct { - // ID of the policy - PolicyID int64 - Metadata map[string]interface{} -} diff --git a/src/replication/event/on_deletion_handler.go b/src/replication/event/on_deletion_handler.go deleted file mode 100644 index 1fe84cb7b..000000000 --- a/src/replication/event/on_deletion_handler.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package event - -import ( - "errors" - "fmt" - "reflect" - - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/replication/event/notification" -) - -// OnDeletionHandler implements the notification handler interface to handle image on push event. -type OnDeletionHandler struct{} - -// Handle implements the same method of notification handler interface -func (oph *OnDeletionHandler) Handle(value interface{}) error { - if value == nil { - return errors.New("OnDeletionHandler can not handle nil value") - } - - vType := reflect.TypeOf(value) - if vType.Kind() != reflect.Struct || vType.String() != "notification.OnDeletionNotification" { - return fmt.Errorf("Mismatch value type of OnDeletionHandler, expect %s but got %s", "notification.OnDeletionNotification", vType.String()) - } - - notification := value.(notification.OnDeletionNotification) - return checkAndTriggerReplication(notification.Image, models.RepOpDelete) -} - -// IsStateful implements the same method of notification handler interface -func (oph *OnDeletionHandler) IsStateful() bool { - // Statless - return false -} diff --git a/src/replication/event/on_deletion_handler_test.go b/src/replication/event/on_deletion_handler_test.go deleted file mode 100644 index 5133b9cb9..000000000 --- a/src/replication/event/on_deletion_handler_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package event - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/replication/event/notification" - "github.com/stretchr/testify/assert" -) - -func TestHandleOfOnDeletionHandler(t *testing.T) { - dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} - - handler := &OnDeletionHandler{} - - assert.NotNil(t, handler.Handle(nil)) - assert.NotNil(t, handler.Handle(map[string]string{})) - assert.NotNil(t, handler.Handle(struct{}{})) - - assert.Nil(t, handler.Handle(notification.OnDeletionNotification{ - Image: "library/hello-world:latest", - })) -} - -func TestIsStatefulOfOnDeletionHandler(t *testing.T) { - handler := &OnDeletionHandler{} - assert.False(t, handler.IsStateful()) -} diff --git a/src/replication/event/on_push_handler.go b/src/replication/event/on_push_handler.go deleted file mode 100644 index 2c002b7ad..000000000 --- a/src/replication/event/on_push_handler.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package event - -import ( - "errors" - "fmt" - "reflect" - - common_models "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/notifier" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/event/notification" - "github.com/goharbor/harbor/src/replication/event/topic" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/trigger" -) - -// OnPushHandler implements the notification handler interface to handle image on push event. -type OnPushHandler struct{} - -// Handle implements the same method of notification handler interface -func (oph *OnPushHandler) Handle(value interface{}) error { - if value == nil { - return errors.New("OnPushHandler can not handle nil value") - } - - vType := reflect.TypeOf(value) - if vType.Kind() != reflect.Struct || vType.String() != "notification.OnPushNotification" { - return fmt.Errorf("Mismatch value type of OnPushHandler, expect %s but got %s", "notification.OnPushNotification", vType.String()) - } - - notification := value.(notification.OnPushNotification) - - return checkAndTriggerReplication(notification.Image, common_models.RepOpTransfer) -} - -// IsStateful implements the same method of notification handler interface -func (oph *OnPushHandler) IsStateful() bool { - // Statless - return false -} - -// checks whether replication policy is set on the resource, if is, trigger the replication -func checkAndTriggerReplication(image, operation string) error { - project, _ := utils.ParseRepository(image) - watchItems, err := trigger.DefaultWatchList.Get(project, operation) - if err != nil { - return fmt.Errorf("failed to get watch list for resource %s, operation %s: %v", - image, operation, err) - } - if len(watchItems) == 0 { - log.Debugf("no replication should be triggered for resource %s, operation %s, skip", image, operation) - return nil - } - - for _, watchItem := range watchItems { - item := models.FilterItem{ - Kind: replication.FilterItemKindTag, - Value: image, - Operation: operation, - } - - if err := notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ - PolicyID: watchItem.PolicyID, - Metadata: map[string]interface{}{ - "candidates": []models.FilterItem{item}, - }, - }); err != nil { - return fmt.Errorf("failed to publish replication topic for resource %s, operation %s, policy %d: %v", - image, operation, watchItem.PolicyID, err) - } - log.Infof("replication topic for resource %s, operation %s, policy %d triggered", - image, operation, watchItem.PolicyID) - } - return nil -} diff --git a/src/replication/event/on_push_handler_test.go b/src/replication/event/on_push_handler_test.go deleted file mode 100644 index 630038658..000000000 --- a/src/replication/event/on_push_handler_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package event - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/replication/event/notification" - "github.com/stretchr/testify/assert" -) - -func TestHandleOfOnPushHandler(t *testing.T) { - dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} - - handler := &OnPushHandler{} - - assert.NotNil(t, handler.Handle(nil)) - assert.NotNil(t, handler.Handle(map[string]string{})) - assert.NotNil(t, handler.Handle(struct{}{})) - - assert.Nil(t, handler.Handle(notification.OnPushNotification{ - Image: "library/hello-world:latest", - })) -} - -func TestIsStatefulOfOnPushHandler(t *testing.T) { - handler := &OnPushHandler{} - assert.False(t, handler.IsStateful()) -} diff --git a/src/replication/event/start_replication_handler.go b/src/replication/event/start_replication_handler.go deleted file mode 100644 index f82bf4ac5..000000000 --- a/src/replication/event/start_replication_handler.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package event - -import ( - "errors" - "fmt" - "reflect" - - "github.com/goharbor/harbor/src/replication/core" - "github.com/goharbor/harbor/src/replication/event/notification" -) - -// StartReplicationHandler implements the notification handler interface to handle start replication requests. -type StartReplicationHandler struct{} - -// Handle implements the same method of notification handler interface -func (srh *StartReplicationHandler) Handle(value interface{}) error { - if value == nil { - return errors.New("StartReplicationHandler can not handle nil value") - } - - vType := reflect.TypeOf(value) - if vType.Kind() != reflect.Struct || vType.String() != "notification.StartReplicationNotification" { - return fmt.Errorf("Mismatch value type of StartReplicationHandler, expect %s but got %s", "notification.StartReplicationNotification", vType.String()) - } - - notification := value.(notification.StartReplicationNotification) - if notification.PolicyID <= 0 { - return errors.New("Invalid policy") - } - - // Start replication - return core.GlobalController.Replicate(notification.PolicyID, notification.Metadata) -} - -// IsStateful implements the same method of notification handler interface -func (srh *StartReplicationHandler) IsStateful() bool { - // Stateless - return false -} diff --git a/src/replication/event/start_replication_handler_test.go b/src/replication/event/start_replication_handler_test.go deleted file mode 100644 index 0369ab692..000000000 --- a/src/replication/event/start_replication_handler_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package event - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/replication/core" - "github.com/goharbor/harbor/src/replication/event/notification" - "github.com/stretchr/testify/assert" -) - -func TestHandle(t *testing.T) { - core.GlobalController = &test.FakeReplicatoinController{} - - handler := &StartReplicationHandler{} - - assert.NotNil(t, handler.Handle(nil)) - assert.NotNil(t, handler.Handle(map[string]string{})) - assert.NotNil(t, handler.Handle(struct{}{})) - assert.NotNil(t, handler.Handle(notification.StartReplicationNotification{ - PolicyID: -1, - })) - assert.Nil(t, handler.Handle(notification.StartReplicationNotification{ - PolicyID: 1, - })) -} - -func TestIsStateful(t *testing.T) { - handler := &StartReplicationHandler{} - assert.False(t, handler.IsStateful()) -} diff --git a/src/replication/event/topic/topics.go b/src/replication/event/topic/topics.go deleted file mode 100644 index 71bcf95cb..000000000 --- a/src/replication/event/topic/topics.go +++ /dev/null @@ -1,12 +0,0 @@ -package topic - -const ( - // ReplicationEventTopicOnPush : OnPush event - ReplicationEventTopicOnPush = "OnPush" - - // ReplicationEventTopicOnDeletion : OnDeletion event - ReplicationEventTopicOnDeletion = "OnDeletion" - - // StartReplicationTopic : Start application request - StartReplicationTopic = "StartReplication" -) diff --git a/src/replication/models/filter.go b/src/replication/models/filter.go deleted file mode 100644 index 49c8481a2..000000000 --- a/src/replication/models/filter.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "fmt" - - "github.com/astaxie/beego/validation" - "github.com/goharbor/harbor/src/replication" -) - -// Filter is the data model represents the filter defined by user. -type Filter struct { - Kind string `json:"kind"` - Pattern string `json:"pattern"` // deprecated, use Value instead - Value interface{} `json:"value"` -} - -// Valid ... -func (f *Filter) Valid(v *validation.Validation) { - switch f.Kind { - case replication.FilterItemKindProject, - replication.FilterItemKindRepository, - replication.FilterItemKindTag: - if f.Value == nil { - // check the Filter.Pattern if the Filter.Value is nil for compatibility - if len(f.Pattern) == 0 { - v.SetError("value", "the value can not be empty") - } - return - } - pattern, ok := f.Value.(string) - if !ok { - v.SetError("value", "the type of value should be string for project, repository and image filter") - return - } - if len(pattern) == 0 { - v.SetError("value", "the value can not be empty") - return - } - case replication.FilterItemKindLabel: - if f.Value == nil { - v.SetError("value", "the value can not be empty") - return - } - labelID, ok := f.Value.(float64) - i := int64(labelID) - if !ok || float64(i) != labelID { - v.SetError("value", "the type of value should be integer for label filter") - return - } - if i <= 0 { - v.SetError("value", fmt.Sprintf("invalid label ID: %d", i)) - return - } - f.Value = i - default: - v.SetError("kind", fmt.Sprintf("invalid filter kind: %s", f.Kind)) - return - } -} diff --git a/src/replication/models/filter_config.go b/src/replication/models/filter_config.go deleted file mode 100644 index 33e450d9a..000000000 --- a/src/replication/models/filter_config.go +++ /dev/null @@ -1,7 +0,0 @@ -package models - -// FilterConfig is data model to provide configurations to the filters. -type FilterConfig struct { - // The pattern for fuzzy matching - Pattern string -} diff --git a/src/replication/models/filter_item.go b/src/replication/models/filter_item.go deleted file mode 100644 index 7b93351c5..000000000 --- a/src/replication/models/filter_item.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -// FilterItem is the general data model represents the filtering resources which are used as input and output for the filters. -type FilterItem struct { - - // The kind of the filtering resources. Support 'project','repository' and 'tag' etc. - Kind string `json:"kind"` - - // The key value of resource which can be used to filter out the resource matched with specified pattern. - // E.g: - // kind == 'project', value will be project name; - // kind == 'repository', value will be repository name - // kind == 'tag', value will be tag name. - Value string `json:"value"` - - Operation string `json:"operation"` - - // Extension placeholder. - // To append more additional information if required by the filter. - Metadata map[string]interface{} `json:"metadata"` -} diff --git a/src/replication/models/filter_test.go b/src/replication/models/filter_test.go deleted file mode 100644 index 90580108a..000000000 --- a/src/replication/models/filter_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "testing" - - "github.com/astaxie/beego/validation" - "github.com/goharbor/harbor/src/replication" - "github.com/stretchr/testify/assert" -) - -func TestValid(t *testing.T) { - cases := map[*Filter]bool{ - {}: true, - { - Kind: "invalid_kind", - }: true, - { - Kind: replication.FilterItemKindRepository, - }: true, - { - Kind: replication.FilterItemKindRepository, - Pattern: "*", - }: false, - { - Kind: replication.FilterItemKindRepository, - Value: "*", - }: false, - { - Kind: replication.FilterItemKindLabel, - }: true, - { - Kind: replication.FilterItemKindLabel, - Value: "", - }: true, - { - Kind: replication.FilterItemKindLabel, - Value: 1.2, - }: true, - { - Kind: replication.FilterItemKindLabel, - Value: -1, - }: true, - { - Kind: replication.FilterItemKindLabel, - Value: 1, - }: true, - } - - for filter, hasError := range cases { - v := &validation.Validation{} - filter.Valid(v) - assert.Equal(t, hasError, v.HasErrors()) - } -} diff --git a/src/replication/models/policy.go b/src/replication/models/policy.go deleted file mode 100644 index b7e3847b0..000000000 --- a/src/replication/models/policy.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "time" -) - -// ReplicationPolicy defines the structure of a replication policy. -type ReplicationPolicy struct { - ID int64 // UUID of the policy - Name string - Description string - Filters []Filter - ReplicateDeletion bool - Trigger *Trigger // The trigger of the replication - ProjectIDs []int64 // Projects attached to this policy - TargetIDs []int64 - Namespaces []string // The namespaces are used to set immediate trigger - CreationTime time.Time - UpdateTime time.Time -} - -// QueryParameter defines the parameters used to do query selection. -type QueryParameter struct { - // Query by page, couple with pageSize - Page int64 - - // Size of each page, couple with page - PageSize int64 - - // Query by project ID - ProjectID int64 - - // Query by name - Name string -} - -// ReplicationPolicyQueryResult is the query result of replication policy -type ReplicationPolicyQueryResult struct { - Total int64 - Policies []*ReplicationPolicy -} diff --git a/src/replication/models/registry_models.go b/src/replication/models/registry_models.go deleted file mode 100644 index 6fa28c15b..000000000 --- a/src/replication/models/registry_models.go +++ /dev/null @@ -1,34 +0,0 @@ -package models - -// Namespace is the resource group/scope like project in Harbor and organization in docker hub. -type Namespace struct { - // Name of the namespace - Name string - - // Extensions to provide flexibility - Metadata map[string]interface{} -} - -// Repository is to keep the info of image repository. -type Repository struct { - // Name of the repository - Name string - - // Project reference of this repository belongs to - Namespace Namespace - - // Extensions to provide flexibility - Metadata map[string]interface{} -} - -// Tag keeps the info of image with specified version -type Tag struct { - // Name of the tag - Name string - - // The repository reference of this tag belongs to - Repository Repository - - // Extensions to provide flexibility - Metadata map[string]interface{} -} diff --git a/src/replication/models/trigger.go b/src/replication/models/trigger.go deleted file mode 100644 index 870241a43..000000000 --- a/src/replication/models/trigger.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "fmt" - - "github.com/astaxie/beego/validation" - "github.com/goharbor/harbor/src/replication" -) - -// Trigger is replication launching approach definition -type Trigger struct { - Kind string `json:"kind"` // the type of the trigger - ScheduleParam *ScheduleParam `json:"schedule_param"` // optional, only used when kind is 'schedule' -} - -// Valid ... -func (t *Trigger) Valid(v *validation.Validation) { - if !(t.Kind == replication.TriggerKindImmediate || - t.Kind == replication.TriggerKindManual || - t.Kind == replication.TriggerKindSchedule) { - v.SetError("kind", fmt.Sprintf("invalid trigger kind: %s", t.Kind)) - } - - if t.Kind == replication.TriggerKindSchedule { - if t.ScheduleParam == nil { - v.SetError("schedule_param", "empty schedule_param") - } else { - t.ScheduleParam.Valid(v) - } - } -} - -// ScheduleParam defines the parameters used by schedule trigger -type ScheduleParam struct { - Type string `json:"type"` // daily or weekly - Weekday int8 `json:"weekday"` // Optional, only used when type is 'weekly' - Offtime int64 `json:"offtime"` // The time offset with the UTC 00:00 in seconds -} - -// Valid ... -func (s *ScheduleParam) Valid(v *validation.Validation) { - if !(s.Type == replication.TriggerScheduleDaily || - s.Type == replication.TriggerScheduleWeekly) { - v.SetError("type", fmt.Sprintf("invalid schedule trigger parameter type: %s", s.Type)) - } - - if s.Type == replication.TriggerScheduleWeekly { - if s.Weekday < 1 || s.Weekday > 7 { - v.SetError("weekday", fmt.Sprintf("invalid schedule trigger parameter weekday: %d", s.Weekday)) - } - } - - if s.Offtime < 0 || s.Offtime > 3600*24 { - v.SetError("offtime", fmt.Sprintf("invalid schedule trigger parameter offtime: %d", s.Offtime)) - } -} - -// Equal ... -func (s *ScheduleParam) Equal(param *ScheduleParam) bool { - if param == nil { - return false - } - - return s.Type == param.Type && s.Weekday == param.Weekday && s.Offtime == param.Offtime -} diff --git a/src/replication/models/trigger_test.go b/src/replication/models/trigger_test.go deleted file mode 100644 index b0013b995..000000000 --- a/src/replication/models/trigger_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "testing" - - "github.com/astaxie/beego/validation" - "github.com/goharbor/harbor/src/replication" - "github.com/stretchr/testify/assert" -) - -func TestValidOfTrigger(t *testing.T) { - cases := map[*Trigger]bool{ - {}: true, - { - Kind: "invalid_kind", - }: true, - { - Kind: replication.TriggerKindImmediate, - }: false, - { - Kind: replication.TriggerKindSchedule, - }: true, - } - - for filter, hasError := range cases { - v := &validation.Validation{} - filter.Valid(v) - assert.Equal(t, hasError, v.HasErrors()) - } -} - -func TestValidOfScheduleParam(t *testing.T) { - cases := map[*ScheduleParam]bool{ - {}: true, - { - Type: "invalid_type", - }: true, - { - Type: replication.TriggerScheduleDaily, - Offtime: 3600*24 + 1, - }: true, - { - Type: replication.TriggerScheduleDaily, - Offtime: 3600 * 2, - }: false, - { - Type: replication.TriggerScheduleWeekly, - Weekday: 0, - Offtime: 3600 * 2, - }: true, - { - Type: replication.TriggerScheduleWeekly, - Weekday: 7, - Offtime: 3600 * 2, - }: false, - } - - for param, hasError := range cases { - v := &validation.Validation{} - param.Valid(v) - assert.Equal(t, hasError, v.HasErrors()) - } -} diff --git a/src/replication/policy/manager.go b/src/replication/policy/manager.go deleted file mode 100644 index 7210a1782..000000000 --- a/src/replication/policy/manager.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package policy - -import ( - "encoding/json" - "time" - - "github.com/goharbor/harbor/src/common/dao" - persist_models "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" -) - -// Manager defines the method a policy manger should implement -type Manager interface { - GetPolicies(models.QueryParameter) (*models.ReplicationPolicyQueryResult, error) - GetPolicy(int64) (models.ReplicationPolicy, error) - CreatePolicy(models.ReplicationPolicy) (int64, error) - UpdatePolicy(models.ReplicationPolicy) error - RemovePolicy(int64) error -} - -// DefaultManager provides replication policy CURD capabilities. -type DefaultManager struct{} - -// NewDefaultManager is the constructor of DefaultManager. -func NewDefaultManager() *DefaultManager { - return &DefaultManager{} -} - -// GetPolicies returns all the policies -func (m *DefaultManager) GetPolicies(query models.QueryParameter) (*models.ReplicationPolicyQueryResult, error) { - result := &models.ReplicationPolicyQueryResult{ - Policies: []*models.ReplicationPolicy{}, - } - total, err := dao.GetTotalOfRepPolicies(query.Name, query.ProjectID) - if err != nil { - return nil, err - } - result.Total = total - - policies, err := dao.FilterRepPolicies(query.Name, query.ProjectID, query.Page, query.PageSize) - if err != nil { - return nil, err - } - - for _, policy := range policies { - ply, err := convertFromPersistModel(policy) - if err != nil { - return nil, err - } - - result.Policies = append(result.Policies, &ply) - } - - return result, nil -} - -// GetPolicy returns the policy with the specified ID -func (m *DefaultManager) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { - policy, err := dao.GetRepPolicy(policyID) - if err != nil { - return models.ReplicationPolicy{}, err - } - - return convertFromPersistModel(policy) -} - -func convertFromPersistModel(policy *persist_models.RepPolicy) (models.ReplicationPolicy, error) { - if policy == nil { - return models.ReplicationPolicy{}, nil - } - - ply := models.ReplicationPolicy{ - ID: policy.ID, - Name: policy.Name, - Description: policy.Description, - ReplicateDeletion: policy.ReplicateDeletion, - ProjectIDs: []int64{policy.ProjectID}, - TargetIDs: []int64{policy.TargetID}, - CreationTime: policy.CreationTime, - UpdateTime: policy.UpdateTime, - } - - project, err := config.GlobalProjectMgr.Get(policy.ProjectID) - if err != nil { - return models.ReplicationPolicy{}, err - } - ply.Namespaces = []string{project.Name} - - if len(policy.Filters) > 0 { - filters := []models.Filter{} - if err := json.Unmarshal([]byte(policy.Filters), &filters); err != nil { - return models.ReplicationPolicy{}, err - } - for i := range filters { - if filters[i].Value == nil && len(filters[i].Pattern) > 0 { - filters[i].Value = filters[i].Pattern - } - // convert the type of Value to int64 as the default type of - // json Unmarshal for number is float64 - if filters[i].Kind == replication.FilterItemKindLabel { - filters[i].Value = int64(filters[i].Value.(float64)) - } - } - ply.Filters = filters - } - - if len(policy.Trigger) > 0 { - trigger := &models.Trigger{} - if err := json.Unmarshal([]byte(policy.Trigger), trigger); err != nil { - return models.ReplicationPolicy{}, err - } - ply.Trigger = trigger - } - - return ply, nil -} - -func convertToPersistModel(policy models.ReplicationPolicy) (*persist_models.RepPolicy, error) { - ply := &persist_models.RepPolicy{ - ID: policy.ID, - Name: policy.Name, - Description: policy.Description, - ReplicateDeletion: policy.ReplicateDeletion, - CreationTime: policy.CreationTime, - UpdateTime: policy.UpdateTime, - } - - if len(policy.ProjectIDs) > 0 { - ply.ProjectID = policy.ProjectIDs[0] - } - - if len(policy.TargetIDs) > 0 { - ply.TargetID = policy.TargetIDs[0] - } - - if policy.Trigger != nil { - trigger, err := json.Marshal(policy.Trigger) - if err != nil { - return nil, err - } - ply.Trigger = string(trigger) - } - - if len(policy.Filters) > 0 { - filters, err := json.Marshal(policy.Filters) - if err != nil { - return nil, err - } - ply.Filters = string(filters) - } - - return ply, nil -} - -// CreatePolicy creates a new policy with the provided data; -// If creating failed, error will be returned; -// If creating succeed, ID of the new created policy will be returned. -func (m *DefaultManager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) { - now := time.Now() - policy.CreationTime = now - policy.UpdateTime = now - ply, err := convertToPersistModel(policy) - if err != nil { - return 0, err - } - return dao.AddRepPolicy(*ply) -} - -// UpdatePolicy updates the policy; -// If updating failed, error will be returned. -func (m *DefaultManager) UpdatePolicy(policy models.ReplicationPolicy) error { - policy.UpdateTime = time.Now() - ply, err := convertToPersistModel(policy) - if err != nil { - return err - } - return dao.UpdateRepPolicy(ply) -} - -// RemovePolicy removes the specified policy; -// If removing failed, error will be returned. -func (m *DefaultManager) RemovePolicy(policyID int64) error { - // delete replication jobs - if err := dao.DeleteRepJobs(policyID); err != nil { - return err - } - // delete the replication policy - return dao.DeleteRepPolicy(policyID) -} diff --git a/src/replication/policy/manager_test.go b/src/replication/policy/manager_test.go deleted file mode 100644 index 49178e71f..000000000 --- a/src/replication/policy/manager_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package policy - -import ( - "encoding/json" - "testing" - - "github.com/goharbor/harbor/src/replication/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConvertToPersistModel(t *testing.T) { - var id, projectID, targetID int64 = 1, 1, 1 - name := "policy01" - replicateDeletion := true - trigger := &models.Trigger{ - Kind: "trigger_kind", - } - filters := []models.Filter{ - { - Kind: "filter_kind", - Pattern: "filter_pattern", - }, - } - policy := models.ReplicationPolicy{ - ID: id, - Name: name, - ReplicateDeletion: replicateDeletion, - ProjectIDs: []int64{projectID}, - TargetIDs: []int64{targetID}, - Trigger: trigger, - Filters: filters, - } - - ply, err := convertToPersistModel(policy) - require.Nil(t, err) - assert.Equal(t, id, ply.ID) - assert.Equal(t, name, ply.Name) - assert.Equal(t, replicateDeletion, ply.ReplicateDeletion) - assert.Equal(t, projectID, ply.ProjectID) - assert.Equal(t, targetID, ply.TargetID) - tg, _ := json.Marshal(trigger) - assert.Equal(t, string(tg), ply.Trigger) - ft, _ := json.Marshal(filters) - assert.Equal(t, string(ft), ply.Filters) -} diff --git a/src/replication/registry/adaptor.go b/src/replication/registry/adaptor.go deleted file mode 100644 index fc011edca..000000000 --- a/src/replication/registry/adaptor.go +++ /dev/null @@ -1,34 +0,0 @@ -package registry - -import ( - "github.com/goharbor/harbor/src/replication/models" -) - -// Adaptor defines the unified operations for all the supported registries such as Harbor or DockerHub. -// It's used to adapt the different interfaces provied by the different registry providers. -// Use external registry with restful api providing as example, these intrefaces may depends on the -// related restful apis like: -// /api/vx/repositories/{namespace}/{repositoryName}/tags/{name} -// /api/v0/accounts/{namespace} -type Adaptor interface { - // Return the unique kind identifier of the adaptor - Kind() string - - // Get all the namespaces - GetNamespaces() []models.Namespace - - // Get the namespace with the specified name - GetNamespace(name string) models.Namespace - - // Get all the repositories under the specified namespace - GetRepositories(namespace string) []models.Repository - - // Get the repository with the specified name under the specified namespace - GetRepository(name string, namespace string) models.Repository - - // Get all the tags of the specified repository under the namespace - GetTags(repositoryName string, namespace string) []models.Tag - - // Get the tag with the specified name of the repository under the namespace - GetTag(name string, repositoryName string, namespace string) models.Tag -} diff --git a/src/replication/registry/harbor_adaptor.go b/src/replication/registry/harbor_adaptor.go deleted file mode 100644 index a63b2c3f5..000000000 --- a/src/replication/registry/harbor_adaptor.go +++ /dev/null @@ -1,83 +0,0 @@ -package registry - -import ( - "github.com/goharbor/harbor/src/common/dao" - common_models "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" -) - -// TODO refacotor the methods of HarborAdaptor by caling Harbor's API - -// HarborAdaptor is defined to adapt the Harbor registry -type HarborAdaptor struct{} - -// Kind returns the unique kind identifier of the adaptor -func (ha *HarborAdaptor) Kind() string { - return replication.AdaptorKindHarbor -} - -// GetNamespaces is ued to get all the namespaces -func (ha *HarborAdaptor) GetNamespaces() []models.Namespace { - return nil -} - -// GetNamespace is used to get the namespace with the specified name -func (ha *HarborAdaptor) GetNamespace(name string) models.Namespace { - return models.Namespace{} -} - -// GetRepositories is used to get all the repositories under the specified namespace -func (ha *HarborAdaptor) GetRepositories(namespace string) []models.Repository { - repos, err := dao.GetRepositories(&common_models.RepositoryQuery{ - ProjectName: namespace, - }) - if err != nil { - log.Errorf("failed to get repositories under namespace %s: %v", namespace, err) - return nil - } - - repositories := []models.Repository{} - for _, repo := range repos { - repositories = append(repositories, models.Repository{ - Name: repo.Name, - }) - } - return repositories -} - -// GetRepository is used to get the repository with the specified name under the specified namespace -func (ha *HarborAdaptor) GetRepository(name string, namespace string) models.Repository { - return models.Repository{} -} - -// GetTags is used to get all the tags of the specified repository under the namespace -func (ha *HarborAdaptor) GetTags(repositoryName string, namespace string) []models.Tag { - client, err := utils.NewRepositoryClientForUI("harbor-core", repositoryName) - if err != nil { - log.Errorf("failed to create registry client: %v", err) - return nil - } - - ts, err := client.ListTag() - if err != nil { - log.Errorf("failed to get tags of repository %s: %v", repositoryName, err) - return nil - } - - tags := []models.Tag{} - for _, t := range ts { - tags = append(tags, models.Tag{ - Name: t, - }) - } - - return tags -} - -// GetTag is used to get the tag with the specified name of the repository under the namespace -func (ha *HarborAdaptor) GetTag(name string, repositoryName string, namespace string) models.Tag { - return models.Tag{} -} diff --git a/src/replication/replicator/replicator.go b/src/replication/replicator/replicator.go deleted file mode 100644 index 0fd4cef99..000000000 --- a/src/replication/replicator/replicator.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package replicator - -import ( - "fmt" - "strings" - - "github.com/goharbor/harbor/src/common/dao" - common_job "github.com/goharbor/harbor/src/common/job" - job_models "github.com/goharbor/harbor/src/common/job/models" - common_models "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/ng/model" -) - -// Replication holds information for a replication -type Replication struct { - PolicyID int64 - OpUUID string - Candidates []models.FilterItem - Registries []*model.Registry - Operation string -} - -// Replicator submits the replication work to the jobservice -type Replicator interface { - Replicate(*Replication) error -} - -// DefaultReplicator provides a default implement for Replicator -type DefaultReplicator struct { - client common_job.Client -} - -// NewDefaultReplicator returns an instance of DefaultReplicator -func NewDefaultReplicator(client common_job.Client) *DefaultReplicator { - return &DefaultReplicator{ - client: client, - } -} - -// Replicate ... -func (d *DefaultReplicator) Replicate(replication *Replication) error { - repositories := map[string][]string{} - // TODO the operation of all candidates are same for now. Update it after supporting - // replicate deletion - operation := "" - for _, candidate := range replication.Candidates { - strs := strings.SplitN(candidate.Value, ":", 2) - if len(strs) != 2 { - return fmt.Errorf("malforld image '%s'", candidate.Value) - } - repositories[strs[0]] = append(repositories[strs[0]], strs[1]) - operation = candidate.Operation - } - - for _, registry := range replication.Registries { - for repository, tags := range repositories { - // create job in database - id, err := dao.AddRepJob(common_models.RepJob{ - PolicyID: replication.PolicyID, - OpUUID: replication.OpUUID, - Repository: repository, - TagList: tags, - Operation: operation, - }) - if err != nil { - return err - } - - // submit job to jobservice - log.Debugf("submiting replication job to jobservice, repository: %s, tags: %v, operation: %s, target: %s", - repository, tags, operation, registry.URL) - job := &job_models.JobData{ - Metadata: &job_models.JobMetadata{ - JobKind: common_job.JobKindGeneric, - }, - StatusHook: fmt.Sprintf("%s/service/notifications/jobs/replication/%d", - config.InternalCoreURL(), id), - } - - if operation == common_models.RepOpTransfer { - job.Name = common_job.ImageTransfer - job.Parameters = map[string]interface{}{ - "repository": repository, - "tags": tags, - "src_registry_url": config.InternalCoreURL(), - "src_registry_insecure": false, - "src_token_service_url": config.InternalTokenServiceEndpoint(), - "dst_registry_url": registry.URL, - "dst_registry_insecure": registry.Insecure, - "dst_registry_username": registry.Credential.AccessKey, - "dst_registry_password": registry.Credential.AccessSecret, - } - } else { - job.Name = common_job.ImageDelete - job.Parameters = map[string]interface{}{ - "repository": repository, - "tags": tags, - "dst_registry_url": registry.URL, - "dst_registry_insecure": registry.Insecure, - "dst_registry_username": registry.Credential.AccessKey, - "dst_registry_password": registry.Credential.AccessSecret, - } - } - - uuid, err := d.client.SubmitJob(job) - if err != nil { - if er := dao.UpdateRepJobStatus(id, common_models.JobError); er != nil { - log.Errorf("failed to update the status of job %d: %s", id, er) - } - return err - } - - // create the mapping relationship between the jobs in database and jobservice - if err = dao.SetRepJobUUID(id, uuid); err != nil { - return err - } - } - } - return nil -} diff --git a/src/replication/replicator/replicator_test.go b/src/replication/replicator/replicator_test.go deleted file mode 100644 index e3ae9aac4..000000000 --- a/src/replication/replicator/replicator_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package replicator - -import ( - "testing" -) - -func TestNewDefaultReplicator(t *testing.T) { - NewDefaultReplicator(nil) -} diff --git a/src/replication/source/convertor.go b/src/replication/source/convertor.go deleted file mode 100644 index 0bc4cc34f..000000000 --- a/src/replication/source/convertor.go +++ /dev/null @@ -1,16 +0,0 @@ -package source - -import ( - "github.com/goharbor/harbor/src/replication/models" -) - -// Converter is designed to covert the format of output from upstream filter to the input format -// required by the downstream filter if needed. -// Each converter covers only one specified conversion process between the two filters. -// E.g: -// If project filter connects to repository filter, then one converter should be defined for this connection; -// If project filter connects to tag filter, then another one should be defined. The above one can not be reused. -type Converter interface { - // Accept the items from upstream filter as input and then covert them to the required format and returned. - Convert(itemsOfUpstream []models.FilterItem) (itemsOfDownstream []models.FilterItem) -} diff --git a/src/replication/source/default_filter_chain.go b/src/replication/source/default_filter_chain.go deleted file mode 100644 index adfa5e858..000000000 --- a/src/replication/source/default_filter_chain.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "github.com/goharbor/harbor/src/replication/models" -) - -// DefaultFilterChain provides a default implement for interface FilterChain -type DefaultFilterChain struct { - filters []Filter -} - -// NewDefaultFilterChain returns an instance of DefaultFilterChain -func NewDefaultFilterChain(filters []Filter) *DefaultFilterChain { - return &DefaultFilterChain{ - filters: filters, - } -} - -// Build nil implement now -func (d *DefaultFilterChain) Build(filters []Filter) error { - return nil -} - -// Filters returns the filter list -func (d *DefaultFilterChain) Filters() []Filter { - return d.filters -} - -// DoFilter does the filter works for filterItems -func (d *DefaultFilterChain) DoFilter(filterItems []models.FilterItem) []models.FilterItem { - if len(filterItems) == 0 { - return []models.FilterItem{} - } - - for _, filter := range d.filters { - converter := filter.GetConverter() - if converter != nil { - filterItems = converter.Convert(filterItems) - } - filterItems = filter.DoFilter(filterItems) - } - return filterItems -} diff --git a/src/replication/source/default_filter_chain_test.go b/src/replication/source/default_filter_chain_test.go deleted file mode 100644 index 85a760790..000000000 --- a/src/replication/source/default_filter_chain_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBuild(t *testing.T) { - chain := NewDefaultFilterChain(nil) - require.Nil(t, chain.Build(nil)) -} - -func TestFilters(t *testing.T) { - filters := []Filter{NewPatternFilter("project", "*")} - chain := NewDefaultFilterChain(filters) - assert.EqualValues(t, filters, chain.Filters()) -} - -func TestDoFilter(t *testing.T) { - projectFilter := NewPatternFilter(replication.FilterItemKindProject, "library*") - repositoryFilter := NewPatternFilter(replication.FilterItemKindRepository, - "library/ubuntu*", &fakeRepositoryConverter{}) - filters := []Filter{projectFilter, repositoryFilter} - - items := []models.FilterItem{ - { - Kind: replication.FilterItemKindProject, - Value: "library", - }, - { - Kind: replication.FilterItemKindProject, - Value: "test", - }, - } - chain := NewDefaultFilterChain(filters) - items = chain.DoFilter(items) - assert.EqualValues(t, []models.FilterItem{ - { - Kind: replication.FilterItemKindRepository, - Value: "library/ubuntu", - }, - }, items) - -} - -type fakeRepositoryConverter struct{} - -func (f *fakeRepositoryConverter) Convert(items []models.FilterItem) []models.FilterItem { - result := []models.FilterItem{} - for _, item := range items { - result = append(result, models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: item.Value + "/ubuntu", - }) - } - return result -} diff --git a/src/replication/source/filter.go b/src/replication/source/filter.go deleted file mode 100644 index 1d375fa86..000000000 --- a/src/replication/source/filter.go +++ /dev/null @@ -1,18 +0,0 @@ -package source - -import ( - "github.com/goharbor/harbor/src/replication/models" -) - -// Filter define the operations of selecting the matched resources from the candidates -// according to the specified pattern. -type Filter interface { - // Initialize the filter - Init() error - - // Return the converter if existing or nil if never set - GetConverter() Converter - - // Filter the items - DoFilter(filterItems []models.FilterItem) []models.FilterItem -} diff --git a/src/replication/source/filter_chain.go b/src/replication/source/filter_chain.go deleted file mode 100644 index 1e27f9fed..000000000 --- a/src/replication/source/filter_chain.go +++ /dev/null @@ -1,21 +0,0 @@ -package source - -import ( - "github.com/goharbor/harbor/src/replication/models" -) - -// FilterChain is the interface to define the operations of coordinating multiple filters -// to work together as a whole pipeline. -// E.g: -// (original resources)---->[project filter]---->[repository filter]---->[tag filter]---->[......]---->(filter resources) -type FilterChain interface { - // Build the filter chain with the filters provided; - // if failed, an error will be returned. - Build(filter []Filter) error - - // Return all the filters in the chain. - Filters() []Filter - - // Filter the items and returned the filtered items via the appended filters in the chain. - DoFilter(filterItems []models.FilterItem) []models.FilterItem -} diff --git a/src/replication/source/label_filter.go b/src/replication/source/label_filter.go deleted file mode 100644 index 6a65ad71d..000000000 --- a/src/replication/source/label_filter.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "fmt" - - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" -) - -// LabelFilter filter resources according to label -type LabelFilter struct { - labelID int64 -} - -// Init ... -func (l *LabelFilter) Init() error { - return nil -} - -// GetConverter ... -func (l *LabelFilter) GetConverter() Converter { - return nil -} - -// NewLabelFilter returns an instance of LabelFilter -func NewLabelFilter(labelID int64) *LabelFilter { - return &LabelFilter{ - labelID: labelID, - } -} - -// DoFilter filter the resources according to the label -func (l *LabelFilter) DoFilter(items []models.FilterItem) []models.FilterItem { - candidates := []string{} - for _, item := range items { - candidates = append(candidates, item.Value) - } - log.Debugf("label filter candidates: %v", candidates) - result := []models.FilterItem{} - for _, item := range items { - hasLabel, err := hasLabel(item, l.labelID) - if err != nil { - log.Errorf("failed to check the label of resouce %v: %v, skip it", item, err) - continue - } - if hasLabel { - log.Debugf("has label %d, add %s to the label filter result list", l.labelID, item.Value) - result = append(result, item) - } - } - return result -} - -func hasLabel(resource models.FilterItem, labelID int64) (bool, error) { - rType := "" - switch resource.Kind { - case replication.FilterItemKindProject: - rType = common.ResourceTypeProject - case replication.FilterItemKindRepository: - rType = common.ResourceTypeRepository - case replication.FilterItemKindTag: - rType = common.ResourceTypeImage - default: - return false, fmt.Errorf("invalid resource type: %s", resource.Kind) - } - rl, err := dao.GetResourceLabel(rType, resource.Value, labelID) - if err != nil { - return false, err - } - return rl != nil, nil -} diff --git a/src/replication/source/label_filter_test.go b/src/replication/source/label_filter_test.go deleted file mode 100644 index 2486c4af1..000000000 --- a/src/replication/source/label_filter_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/replication" - - "github.com/goharbor/harbor/src/replication/models" - "github.com/stretchr/testify/assert" -) - -func TestInitOfLabelFilter(t *testing.T) { - filter := NewLabelFilter(1) - assert.Nil(t, filter.Init()) -} - -func TestGetConverterOfLabelFilter(t *testing.T) { - filter := NewLabelFilter(1) - assert.Nil(t, filter.GetConverter()) -} - -func TestDoFilterOfLabelFilter(t *testing.T) { - test.InitDatabaseFromEnv() - filter := NewLabelFilter(1) - items := []models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world:latest", - }, - } - result := filter.DoFilter(items) - assert.Equal(t, 0, len(result)) -} diff --git a/src/replication/source/match.go b/src/replication/source/match.go deleted file mode 100644 index b2737e44d..000000000 --- a/src/replication/source/match.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "path/filepath" -) - -func match(pattern, str string) (bool, error) { - return filepath.Match(pattern, str) -} diff --git a/src/replication/source/match_test.go b/src/replication/source/match_test.go deleted file mode 100644 index 2f31b8055..000000000 --- a/src/replication/source/match_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMatch(t *testing.T) { - cases := []struct { - pattern string - str string - matched bool - }{ - {"", "", true}, - {"*", "library", true}, - {"library/*", "library/mysql", true}, - {"library/*", "library/mysql/5.6", false}, - {"library/mysq?", "library/mysql", true}, - {"library/mysq?", "library/mysqld", false}, - } - - for _, c := range cases { - matched, err := match(c.pattern, c.str) - require.Nil(t, err) - assert.Equal(t, c.matched, matched) - } -} diff --git a/src/replication/source/pattern_filter.go b/src/replication/source/pattern_filter.go deleted file mode 100644 index 9492738be..000000000 --- a/src/replication/source/pattern_filter.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "regexp" - - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/replication/models" -) - -// PatternFilter implements Filter interface for pattern filter -type PatternFilter struct { - kind string - pattern string - converter Converter -} - -// NewPatternFilter returns an instance of PatternFilter -func NewPatternFilter(kind, pattern string, converter ...Converter) *PatternFilter { - filer := &PatternFilter{ - kind: kind, - pattern: pattern, - } - - if len(converter) > 0 { - filer.converter = converter[0] - } - - return filer -} - -// Init the filter. nil implement for now -func (p *PatternFilter) Init() error { - return nil -} - -// GetConverter returns the converter -func (p *PatternFilter) GetConverter() Converter { - return p.converter -} - -// DoFilter filters resources -func (p *PatternFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem { - items := []models.FilterItem{} - for _, item := range filterItems { - if item.Kind != p.kind { - log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip", - p.kind, item.Kind) - continue - } - - matched, err := regexp.MatchString(p.pattern, item.Value) - if err != nil { - log.Errorf("failed to match pattern %s, value %s: %v, skip", - p.pattern, item.Value, err) - continue - } - - if !matched { - log.Debugf("%s does not match to the %s filter %s, skip", - item.Value, p.kind, p.pattern) - continue - } - - log.Debugf("add %s to the result of %s filter %s", - item.Value, p.kind, p.pattern) - items = append(items, item) - } - - return items -} diff --git a/src/replication/source/pattern_filter_test.go b/src/replication/source/pattern_filter_test.go deleted file mode 100644 index 59315dec1..000000000 --- a/src/replication/source/pattern_filter_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/stretchr/testify/assert" - - "testing" -) - -var pfilter = NewPatternFilter(replication.FilterItemKindTag, "library/ubuntu:release-*", nil) - -func TestPatternFilterInit(t *testing.T) { - assert.Nil(t, pfilter.Init()) -} - -func TestPatternFilterGetConverter(t *testing.T) { - assert.Nil(t, pfilter.GetConverter()) -} - -func TestPatternFilterDoFilter(t *testing.T) { - items := []models.FilterItem{ - { - Kind: replication.FilterItemKindProject, - }, - { - Kind: replication.FilterItemKindRepository, - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/ubuntu:release-14.04", - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/ubuntu:release-16.04", - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/ubuntu:test", - }, - } - result := pfilter.DoFilter(items) - assert.Equal(t, 2, len(result)) - assert.Equal(t, replication.FilterItemKindTag, result[0].Kind) - assert.Equal(t, "library/ubuntu:release-14.04", result[0].Value) - assert.Equal(t, replication.FilterItemKindTag, result[1].Kind) - assert.Equal(t, "library/ubuntu:release-16.04", result[1].Value) - -} diff --git a/src/replication/source/repository_convertor.go b/src/replication/source/repository_convertor.go deleted file mode 100644 index 5089eb10a..000000000 --- a/src/replication/source/repository_convertor.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/registry" -) - -// RepositoryConverter implement Converter interface, convert projects to repositories -type RepositoryConverter struct { - registry registry.Adaptor -} - -// NewRepositoryConverter returns an instance of RepositoryConverter -func NewRepositoryConverter(registry registry.Adaptor) *RepositoryConverter { - return &RepositoryConverter{ - registry: registry, - } -} - -// Convert projects to repositories -func (r *RepositoryConverter) Convert(items []models.FilterItem) []models.FilterItem { - result := []models.FilterItem{} - for _, item := range items { - // just put it to the result list if the item is not a project - if item.Kind != replication.FilterItemKindProject { - result = append(result, item) - continue - } - - repositories := r.registry.GetRepositories(item.Value) - for _, repository := range repositories { - result = append(result, models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: repository.Name, - Operation: item.Operation, - }) - } - } - return result -} diff --git a/src/replication/source/repository_convertor_test.go b/src/replication/source/repository_convertor_test.go deleted file mode 100644 index 334238868..000000000 --- a/src/replication/source/repository_convertor_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/stretchr/testify/assert" -) - -func TestRepositoryConvert(t *testing.T) { - items := []models.FilterItem{ - { - Kind: replication.FilterItemKindProject, - Value: "library", - }, - { - Kind: replication.FilterItemKindRepository, - }, - } - expected := []models.FilterItem{ - { - Kind: replication.FilterItemKindRepository, - Value: "library/ubuntu", - }, - { - Kind: replication.FilterItemKindRepository, - Value: "library/centos", - }, - { - Kind: replication.FilterItemKindRepository, - }, - } - - converter := NewRepositoryConverter(&fakeRegistryAdaptor{}) - assert.EqualValues(t, expected, converter.Convert(items)) -} - -type fakeRegistryAdaptor struct{} - -func (f *fakeRegistryAdaptor) Kind() string { - return "fake" -} - -func (f *fakeRegistryAdaptor) GetNamespaces() []models.Namespace { - return nil -} - -func (f *fakeRegistryAdaptor) GetNamespace(name string) models.Namespace { - return models.Namespace{} -} - -func (f *fakeRegistryAdaptor) GetRepositories(namespace string) []models.Repository { - return []models.Repository{ - { - Name: "library/ubuntu", - }, - { - Name: "library/centos", - }, - } -} - -func (f *fakeRegistryAdaptor) GetRepository(name string, namespace string) models.Repository { - return models.Repository{} -} - -func (f *fakeRegistryAdaptor) GetTags(repositoryName string, namespace string) []models.Tag { - return []models.Tag{ - { - Name: "14.04", - }, - { - Name: "16.04", - }, - } -} - -func (f *fakeRegistryAdaptor) GetTag(name string, repositoryName string, namespace string) models.Tag { - return models.Tag{} -} diff --git a/src/replication/source/repository_filter.go b/src/replication/source/repository_filter.go deleted file mode 100644 index b002a4575..000000000 --- a/src/replication/source/repository_filter.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "strings" - - "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/registry" -) - -// RepositoryFilter implement Filter interface to filter repository -type RepositoryFilter struct { - pattern string - converter Converter -} - -// NewRepositoryFilter returns an instance of RepositoryFilter -func NewRepositoryFilter(pattern string, registry registry.Adaptor) *RepositoryFilter { - return &RepositoryFilter{ - pattern: pattern, - converter: NewRepositoryConverter(registry), - } -} - -// Init ... -func (r *RepositoryFilter) Init() error { - return nil -} - -// GetConverter ... -func (r *RepositoryFilter) GetConverter() Converter { - return r.converter -} - -// DoFilter filters repository and image(according to the repository part) and drops any other resource types -func (r *RepositoryFilter) DoFilter(items []models.FilterItem) []models.FilterItem { - candidates := []string{} - for _, item := range items { - candidates = append(candidates, item.Value) - } - log.Debugf("repository filter candidates: %v", candidates) - - result := []models.FilterItem{} - for _, item := range items { - if item.Kind != replication.FilterItemKindRepository && item.Kind != replication.FilterItemKindTag { - log.Warningf("unsupported type %s for repository filter, drop", item.Kind) - continue - } - - repository := item.Value - if item.Kind == replication.FilterItemKindTag { - repository = strings.SplitN(repository, ":", 2)[0] - } - - if len(r.pattern) == 0 { - log.Debugf("pattern is null, add %s to the repository filter result list", item.Value) - result = append(result, item) - } else { - // trim the project - _, repository = utils.ParseRepository(repository) - matched, err := match(r.pattern, repository) - if err != nil { - log.Errorf("failed to match pattern %s to value %s: %v, skip it", r.pattern, repository, err) - continue - } - if matched { - log.Debugf("pattern %s matched, add %s to the repository filter result list", r.pattern, item.Value) - result = append(result, item) - } - } - } - return result -} diff --git a/src/replication/source/repository_filter_test.go b/src/replication/source/repository_filter_test.go deleted file mode 100644 index 6570411aa..000000000 --- a/src/replication/source/repository_filter_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/registry" - "github.com/stretchr/testify/assert" -) - -func TestInitOfRepositoryFilter(t *testing.T) { - filter := NewRepositoryFilter("", ®istry.HarborAdaptor{}) - assert.Nil(t, filter.Init()) -} - -func TestGetConverterOfRepositoryFilter(t *testing.T) { - filter := NewRepositoryFilter("", ®istry.HarborAdaptor{}) - assert.NotNil(t, filter.GetConverter()) -} - -func TestDoFilterOfRepositoryFilter(t *testing.T) { - // invalid filter item type - filter := NewRepositoryFilter("", ®istry.HarborAdaptor{}) - items := filter.DoFilter([]models.FilterItem{ - { - Kind: "invalid_type", - }, - }) - assert.Equal(t, 0, len(items)) - - // empty pattern - filter = NewRepositoryFilter("", ®istry.HarborAdaptor{}) - items = filter.DoFilter([]models.FilterItem{ - { - Kind: replication.FilterItemKindRepository, - Value: "library/hello-world", - }, - }) - assert.Equal(t, 1, len(items)) - - // non-empty pattern - filter = NewRepositoryFilter("*", ®istry.HarborAdaptor{}) - items = filter.DoFilter([]models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world", - }, - }) - assert.Equal(t, 1, len(items)) - - // non-empty pattern - filter = NewRepositoryFilter("*", ®istry.HarborAdaptor{}) - items = filter.DoFilter([]models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world:latest", - }, - }) - assert.Equal(t, 1, len(items)) -} diff --git a/src/replication/source/sourcer.go b/src/replication/source/sourcer.go deleted file mode 100644 index 622713303..000000000 --- a/src/replication/source/sourcer.go +++ /dev/null @@ -1,36 +0,0 @@ -package source - -import ( - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/registry" -) - -// Sourcer is used to manage and/or handle all the artifacts and information related with source registry. -// All the things with replication source should be covered in this object. -type Sourcer struct { - // Keep the adaptors we support now - adaptors map[string]registry.Adaptor -} - -// NewSourcer is the constructor of Sourcer -func NewSourcer() *Sourcer { - return &Sourcer{ - adaptors: make(map[string]registry.Adaptor), - } -} - -// Init will do some initialization work like registrying all the adaptors we support -func (sc *Sourcer) Init() { - // Register Harbor adaptor - sc.adaptors[replication.AdaptorKindHarbor] = ®istry.HarborAdaptor{} -} - -// GetAdaptor returns the required adaptor with the specified kind. -// If no adaptor with the specified kind existing, nil will be returned. -func (sc *Sourcer) GetAdaptor(kind string) registry.Adaptor { - if len(kind) == 0 { - return nil - } - - return sc.adaptors[kind] -} diff --git a/src/replication/source/sourcer_test.go b/src/replication/source/sourcer_test.go deleted file mode 100644 index fd9f11b5f..000000000 --- a/src/replication/source/sourcer_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package source - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication" -) - -func TestReplicationSourcer(t *testing.T) { - testingSourcer := NewSourcer() - if testingSourcer == nil { - t.Fatal("Failed to create sourcer") - } - - testingSourcer.Init() - - if testingSourcer.GetAdaptor("") != nil { - t.Fatal("Empty kind should not be supported") - } - - if testingSourcer.GetAdaptor(replication.AdaptorKindHarbor) == nil { - t.Fatalf("%s adaptor should be existing", replication.AdaptorKindHarbor) - } -} diff --git a/src/replication/source/tag_combination_filter.go b/src/replication/source/tag_combination_filter.go deleted file mode 100644 index ce7be8e6f..000000000 --- a/src/replication/source/tag_combination_filter.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "strings" - - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" -) - -// TagCombinationFilter implements Filter interface for merging tag filter items -// whose repository are same into one repository filter item -type TagCombinationFilter struct{} - -// NewTagCombinationFilter returns an instance of TagCombinationFilter -func NewTagCombinationFilter() *TagCombinationFilter { - return &TagCombinationFilter{} -} - -// Init the filter. nil implement for now -func (t *TagCombinationFilter) Init() error { - return nil -} - -// GetConverter returns the converter -func (t *TagCombinationFilter) GetConverter() Converter { - return nil -} - -// DoFilter filters resources -func (t *TagCombinationFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem { - repos := map[string][]string{} - for _, item := range filterItems { - if item.Kind != replication.FilterItemKindTag { - log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip", - replication.FilterItemKindTag, item.Kind) - continue - } - - strs := strings.Split(item.Value, ":") - if len(strs) != 2 { - log.Warningf("unexpected image format: %s, skip", item.Value) - continue - } - - repos[strs[0]] = append(repos[strs[0]], strs[1]) - } - - // TODO append operation - items := []models.FilterItem{} - for repo, tags := range repos { - items = append(items, models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: repo, - Metadata: map[string]interface{}{ - "tags": tags, - }, - }) - } - - return items -} diff --git a/src/replication/source/tag_combination_filter_test.go b/src/replication/source/tag_combination_filter_test.go deleted file mode 100644 index 336364471..000000000 --- a/src/replication/source/tag_combination_filter_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/stretchr/testify/assert" - - "testing" -) - -var tcfilter = NewTagCombinationFilter() - -func TestTagCombinationFilterInit(t *testing.T) { - assert.Nil(t, tcfilter.Init()) -} - -func TestTagCombinationFilterGetConverter(t *testing.T) { - assert.Nil(t, tcfilter.GetConverter()) -} - -func TestTagCombinationFilterDoFilter(t *testing.T) { - items := []models.FilterItem{ - { - Kind: replication.FilterItemKindProject, - }, - { - Kind: replication.FilterItemKindRepository, - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/ubuntu:invalid_tag:latest", - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/ubuntu:14.04", - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/ubuntu:16.04", - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/centos:7", - }, - } - result := tcfilter.DoFilter(items) - assert.Equal(t, 2, len(result)) - - var ubuntu, centos models.FilterItem - if result[0].Value == "library/ubuntu" { - ubuntu = result[0] - centos = result[1] - } else { - centos = result[0] - ubuntu = result[1] - } - - assert.Equal(t, replication.FilterItemKindRepository, ubuntu.Kind) - assert.Equal(t, "library/ubuntu", ubuntu.Value) - metadata, ok := ubuntu.Metadata["tags"].([]string) - assert.True(t, ok) - assert.EqualValues(t, []string{"14.04", "16.04"}, metadata) - - assert.Equal(t, replication.FilterItemKindRepository, centos.Kind) - assert.Equal(t, "library/centos", centos.Value) - metadata, ok = centos.Metadata["tags"].([]string) - assert.True(t, ok) - assert.EqualValues(t, []string{"7"}, metadata) -} diff --git a/src/replication/source/tag_convertor.go b/src/replication/source/tag_convertor.go deleted file mode 100644 index 2eec6de4d..000000000 --- a/src/replication/source/tag_convertor.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/registry" -) - -// TagConverter implement Converter interface, convert repositories to tags -type TagConverter struct { - registry registry.Adaptor -} - -// NewTagConverter returns an instance of TagConverter -func NewTagConverter(registry registry.Adaptor) *TagConverter { - return &TagConverter{ - registry: registry, - } -} - -// Convert repositories to tags -func (t *TagConverter) Convert(items []models.FilterItem) []models.FilterItem { - result := []models.FilterItem{} - for _, item := range items { - if item.Kind != replication.FilterItemKindRepository { - // just put it to the result list if the item is not a repository - result = append(result, item) - continue - } - - tags := t.registry.GetTags(item.Value, "") - for _, tag := range tags { - result = append(result, models.FilterItem{ - Kind: replication.FilterItemKindTag, - Value: item.Value + ":" + tag.Name, - Operation: item.Operation, - }) - } - } - return result -} diff --git a/src/replication/source/tag_convertor_test.go b/src/replication/source/tag_convertor_test.go deleted file mode 100644 index 835bd29be..000000000 --- a/src/replication/source/tag_convertor_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/stretchr/testify/assert" -) - -func TestTagConvert(t *testing.T) { - items := []models.FilterItem{ - { - Kind: replication.FilterItemKindRepository, - Value: "library/ubuntu", - }, - { - Kind: replication.FilterItemKindProject, - }, - } - expected := []models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/ubuntu:14.04", - }, - { - Kind: replication.FilterItemKindTag, - Value: "library/ubuntu:16.04", - }, - { - Kind: replication.FilterItemKindProject, - }, - } - - converter := NewTagConverter(&fakeRegistryAdaptor{}) - assert.EqualValues(t, expected, converter.Convert(items)) -} diff --git a/src/replication/source/tag_filter.go b/src/replication/source/tag_filter.go deleted file mode 100644 index f02fb0736..000000000 --- a/src/replication/source/tag_filter.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "strings" - - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/registry" -) - -// TagFilter implements Filter interface to filter tag -type TagFilter struct { - pattern string - converter Converter -} - -// NewTagFilter returns an instance of TagFilter -func NewTagFilter(pattern string, registry registry.Adaptor) *TagFilter { - return &TagFilter{ - pattern: pattern, - converter: NewTagConverter(registry), - } -} - -// Init ... -func (t *TagFilter) Init() error { - return nil -} - -// GetConverter ... -func (t *TagFilter) GetConverter() Converter { - return t.converter -} - -// DoFilter filters tag of the image -func (t *TagFilter) DoFilter(items []models.FilterItem) []models.FilterItem { - candidates := []string{} - for _, item := range items { - candidates = append(candidates, item.Value) - } - log.Debugf("tag filter candidates: %v", candidates) - - result := []models.FilterItem{} - for _, item := range items { - if item.Kind != replication.FilterItemKindTag { - log.Warningf("unsupported type %s for tag filter, dropped", item.Kind) - continue - } - - if len(t.pattern) == 0 { - log.Debugf("pattern is null, add %s to the tag filter result list", item.Value) - result = append(result, item) - continue - } - - tag := strings.SplitN(item.Value, ":", 2)[1] - matched, err := match(t.pattern, tag) - if err != nil { - log.Errorf("failed to match pattern %s to value %s: %v, skip it", t.pattern, tag, err) - continue - } - - if matched { - log.Debugf("pattern %s matched, add %s to the tag filter result list", t.pattern, item.Value) - result = append(result, item) - } - } - return result -} diff --git a/src/replication/source/tag_filter_test.go b/src/replication/source/tag_filter_test.go deleted file mode 100644 index 0b11de444..000000000 --- a/src/replication/source/tag_filter_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/goharbor/harbor/src/replication/registry" - "github.com/stretchr/testify/assert" -) - -func TestInitOfTagFilter(t *testing.T) { - filter := NewTagFilter("", ®istry.HarborAdaptor{}) - assert.Nil(t, filter.Init()) -} - -func TestGetConverterOfTagFilter(t *testing.T) { - filter := NewTagFilter("", ®istry.HarborAdaptor{}) - assert.NotNil(t, filter.GetConverter()) -} - -func TestDoFilterOfTagFilter(t *testing.T) { - // invalid filter item type - filter := NewTagFilter("", ®istry.HarborAdaptor{}) - items := filter.DoFilter([]models.FilterItem{ - { - Kind: "invalid_type", - }, - }) - assert.Equal(t, 0, len(items)) - - // empty pattern - filter = NewTagFilter("", ®istry.HarborAdaptor{}) - items = filter.DoFilter([]models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world:latest", - }, - }) - assert.Equal(t, 1, len(items)) - - // non-empty pattern - filter = NewTagFilter("l*t", ®istry.HarborAdaptor{}) - items = filter.DoFilter([]models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world:latest", - }, - }) - assert.Equal(t, 1, len(items)) - - // non-empty pattern - filter = NewTagFilter("lates?", ®istry.HarborAdaptor{}) - items = filter.DoFilter([]models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world:latest", - }, - }) - assert.Equal(t, 1, len(items)) - - // non-empty pattern - filter = NewTagFilter("latest?", ®istry.HarborAdaptor{}) - items = filter.DoFilter([]models.FilterItem{ - { - Kind: replication.FilterItemKindTag, - Value: "library/hello-world:latest", - }, - }) - assert.Equal(t, 0, len(items)) -} diff --git a/src/replication/trigger/cache.go b/src/replication/trigger/cache.go deleted file mode 100644 index 59b82664d..000000000 --- a/src/replication/trigger/cache.go +++ /dev/null @@ -1,212 +0,0 @@ -package trigger - -import ( - "container/heap" - "fmt" - "sync" - "time" -) - -const ( - // The max count of items the cache can keep - defaultCapacity = 1000 -) - -// Item keeps more metadata of the triggers which are stored in the heap. -type Item struct { - // Which policy the trigger belong to - policyID int64 - - // Frequency of cache querying - // First compration factor - frequency int - - // The timestamp of being put into heap - // Second compration factor - timestamp int64 - - // The index in the heap - index int -} - -// MetaQueue implements heap.Interface and holds items which are metadata of trigger -type MetaQueue []*Item - -// Len return the size of the queue -func (mq MetaQueue) Len() int { - return len(mq) -} - -// Less is a comparator of heap -func (mq MetaQueue) Less(i, j int) bool { - return mq[i].frequency < mq[j].frequency || - (mq[i].frequency == mq[j].frequency && - mq[i].timestamp < mq[j].timestamp) -} - -// Swap the items to rebuild heap -func (mq MetaQueue) Swap(i, j int) { - mq[i], mq[j] = mq[j], mq[i] - mq[i].index = i - mq[j].index = j -} - -// Push item into heap -func (mq *MetaQueue) Push(x interface{}) { - item := x.(*Item) - n := len(*mq) - item.index = n - item.timestamp = time.Now().UTC().UnixNano() - *mq = append(*mq, item) -} - -// Pop smallest item from heap -func (mq *MetaQueue) Pop() interface{} { - old := *mq - n := len(old) - item := old[n-1] // Smallest item - item.index = -1 // For safety - *mq = old[:n-1] - return item -} - -// Update the frequency of item -func (mq *MetaQueue) Update(item *Item) { - item.frequency++ - heap.Fix(mq, item.index) -} - -// CacheItem is the data stored in the cache. -// It contains trigger and heap item references. -type CacheItem struct { - // The trigger reference - trigger Interface - - // The heap item reference - item *Item -} - -// Cache is used to cache the enabled triggers with specified capacity. -// If exceed the capacity, cached items will be adjusted with the following rules: -// The item with least usage frequency will be replaced; -// If multiple items with same usage frequency, the oldest one will be replaced. -type Cache struct { - // The max count of items this cache can keep - capacity int - - // Lock to handle concurrent case - lock *sync.RWMutex - - // Hash map for quick locating cached item - hash map[string]CacheItem - - // Heap for quick locating the trigger with least usage - queue *MetaQueue -} - -// NewCache is constructor of cache -func NewCache(capacity int) *Cache { - capa := capacity - if capa <= 0 { - capa = defaultCapacity - } - - // Initialize heap - mq := make(MetaQueue, 0) - heap.Init(&mq) - - return &Cache{ - capacity: capa, - lock: new(sync.RWMutex), - hash: make(map[string]CacheItem), - queue: &mq, - } -} - -// Get the trigger interface with the specified policy ID -func (c *Cache) Get(policyID int64) Interface { - if policyID <= 0 { - return nil - } - - c.lock.RLock() - defer c.lock.RUnlock() - - k := c.key(policyID) - - if cacheItem, ok := c.hash[k]; ok { - // Update frequency - c.queue.Update(cacheItem.item) - return cacheItem.trigger - } - - return nil -} - -// Put the item into cache with ID of ploicy as key -func (c *Cache) Put(policyID int64, trigger Interface) { - if policyID <= 0 || trigger == nil { - return - } - - c.lock.Lock() - defer c.lock.Unlock() - - // Exceed the capacity? - if c.Size() >= c.capacity { - // Pop one for the new one - v := heap.Pop(c.queue) - item := v.(*Item) - // Remove from hash - delete(c.hash, c.key(item.policyID)) - } - - // Add to meta queue - item := &Item{ - policyID: policyID, - frequency: 1, - } - heap.Push(c.queue, item) - - // Cache - cacheItem := CacheItem{ - trigger: trigger, - item: item, - } - - k := c.key(policyID) - c.hash[k] = cacheItem -} - -// Remove the trigger attached to the specified policy -func (c *Cache) Remove(policyID int64) Interface { - if policyID > 0 { - c.lock.Lock() - defer c.lock.Unlock() - - // If existing - k := c.key(policyID) - if cacheItem, ok := c.hash[k]; ok { - // Remove from heap - heap.Remove(c.queue, cacheItem.item.index) - - // Remove from hash - delete(c.hash, k) - - return cacheItem.trigger - } - - } - - return nil -} - -// Size return the count of triggers in the cache -func (c *Cache) Size() int { - return len(c.hash) -} - -// Generate a hash key with the policy ID -func (c *Cache) key(policyID int64) string { - return fmt.Sprintf("trigger-%d", policyID) -} diff --git a/src/replication/trigger/cache_test.go b/src/replication/trigger/cache_test.go deleted file mode 100644 index b7348cf26..000000000 --- a/src/replication/trigger/cache_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package trigger - -import "testing" -import "time" - -func TestCache(t *testing.T) { - cache := NewCache(10) - trigger := NewImmediateTrigger(ImmediateParam{}) - - cache.Put(1, trigger) - if cache.Size() != 1 { - t.Fatalf("Invalid size, expect 1 but got %d", cache.Size()) - } - - tr := cache.Get(1) - if tr == nil { - t.Fatal("Should not get nil item") - } - - tri := cache.Remove(1) - if tri == nil || cache.Size() > 0 { - t.Fatal("Failed to remove") - } -} - -func TestCacheChange(t *testing.T) { - cache := NewCache(2) - trigger1 := NewImmediateTrigger(ImmediateParam{}) - trigger2 := NewImmediateTrigger(ImmediateParam{}) - cache.Put(1, trigger1) - cache.Put(2, trigger2) - - if cache.Size() != 2 { - t.Fatalf("Invalid size, expect 2 but got %d", cache.Size()) - } - - if tr := cache.Get(2); tr == nil { - t.Fatal("Should not get nil item") - } - - time.Sleep(100 * time.Microsecond) - - trigger3 := NewImmediateTrigger(ImmediateParam{}) - cache.Put(3, trigger3) - if cache.Size() != 2 { - t.Fatalf("Invalid size, expect 2 but got %d", cache.Size()) - } - - if tr := cache.Get(1); tr != nil { - t.Fatal("item1 should not exist") - } - -} diff --git a/src/replication/trigger/immediate.go b/src/replication/trigger/immediate.go deleted file mode 100644 index e373ecb2d..000000000 --- a/src/replication/trigger/immediate.go +++ /dev/null @@ -1,46 +0,0 @@ -package trigger - -import ( - "github.com/goharbor/harbor/src/replication" -) - -// ImmediateTrigger will setup watcher at the image pushing action to fire -// replication event at pushing happening time. -type ImmediateTrigger struct { - params ImmediateParam -} - -// NewImmediateTrigger is constructor of ImmediateTrigger -func NewImmediateTrigger(params ImmediateParam) *ImmediateTrigger { - return &ImmediateTrigger{ - params: params, - } -} - -// Kind is the implementation of same method defined in Trigger interface -func (st *ImmediateTrigger) Kind() string { - return replication.TriggerKindImmediate -} - -// Setup is the implementation of same method defined in Trigger interface -func (st *ImmediateTrigger) Setup() error { - // TODO: Need more complicated logic here to handle partial updates - for _, namespace := range st.params.Namespaces { - wt := WatchItem{ - PolicyID: st.params.PolicyID, - Namespace: namespace, - OnDeletion: st.params.OnDeletion, - OnPush: true, - } - - if err := DefaultWatchList.Add(wt); err != nil { - return err - } - } - return nil -} - -// Unset is the implementation of same method defined in Trigger interface -func (st *ImmediateTrigger) Unset() error { - return DefaultWatchList.Remove(st.params.PolicyID) -} diff --git a/src/replication/trigger/immediate_test.go b/src/replication/trigger/immediate_test.go deleted file mode 100644 index 2b6a2524a..000000000 --- a/src/replication/trigger/immediate_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package trigger - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/replication" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestKindOfImmediateTrigger(t *testing.T) { - trigger := NewImmediateTrigger(ImmediateParam{}) - assert.Equal(t, replication.TriggerKindImmediate, trigger.Kind()) -} - -func TestSetupAndUnsetOfImmediateTrigger(t *testing.T) { - dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} - - param := ImmediateParam{} - param.PolicyID = 1 - param.OnDeletion = true - param.Namespaces = []string{"library"} - trigger := NewImmediateTrigger(param) - - err := trigger.Setup() - require.Nil(t, err) - - items, err := DefaultWatchList.Get("library", "push") - require.Nil(t, err) - assert.Equal(t, 1, len(items)) - - items, err = DefaultWatchList.Get("library", "delete") - require.Nil(t, err) - assert.Equal(t, 1, len(items)) - - err = trigger.Unset() - require.Nil(t, err) - items, err = DefaultWatchList.Get("library", "delete") - require.Nil(t, err) - assert.Equal(t, 0, len(items)) -} diff --git a/src/replication/trigger/interface.go b/src/replication/trigger/interface.go deleted file mode 100644 index 34929ee6b..000000000 --- a/src/replication/trigger/interface.go +++ /dev/null @@ -1,13 +0,0 @@ -package trigger - -// Interface is certain mechanism to know when fire the replication operation. -type Interface interface { - // Kind indicates what type of the trigger is. - Kind() string - - // Setup/enable the trigger; if failed, an error would be returned. - Setup() error - - // Remove/disable the trigger; if failed, an error would be returned. - Unset() error -} diff --git a/src/replication/trigger/manager.go b/src/replication/trigger/manager.go deleted file mode 100644 index addeb7179..000000000 --- a/src/replication/trigger/manager.go +++ /dev/null @@ -1,124 +0,0 @@ -package trigger - -import ( - "fmt" - - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" -) - -// Manager provides unified methods to manage the triggers of policies; -// Cache the enabled triggers, setup/unset the trigger based on the parameters -// with json format. -type Manager struct { - // Cache for triggers - // cache *Cache -} - -// NewManager is the constructor of trigger manager. -// capacity is the max number of trigger references manager can keep in memory -func NewManager(capacity int) *Manager { - return &Manager{ - // cache: NewCache(capacity), - } -} - -/* -// GetTrigger returns the enabled trigger reference if existing in the cache. -func (m *Manager) GetTrigger(policyID int64) Interface { - return m.cache.Get(policyID) -} - -// RemoveTrigger will disable the trigger and remove it from the cache if existing. -func (m *Manager) RemoveTrigger(policyID int64) error { - trigger := m.cache.Get(policyID) - if trigger == nil { - return errors.New("Trigger is not cached, please use UnsetTrigger to disable the trigger") - } - - // Unset trigger - if err := trigger.Unset(); err != nil { - return err - } - - // Remove from cache - // No need to check the return of remove because the dirty item cached in the cache - // will be removed out finally after a certain while - m.cache.Remove(policyID) - - return nil -} -*/ - -// SetupTrigger will create the new trigger based on the provided policy. -// If failed, an error will be returned. -func (m *Manager) SetupTrigger(policy *models.ReplicationPolicy) error { - trigger, err := createTrigger(policy) - if err != nil { - return err - } - - // manual trigger, do nothing - if trigger == nil { - return nil - } - - tg := trigger.(Interface) - if err = tg.Setup(); err != nil { - return err - } - - log.Debugf("%s trigger for policy %d is set", tg.Kind(), policy.ID) - return nil -} - -// UnsetTrigger will disable the trigger which is not cached in the trigger cache. -func (m *Manager) UnsetTrigger(policy *models.ReplicationPolicy) error { - trigger, err := createTrigger(policy) - if err != nil { - return err - } - - // manual trigger, do nothing - if trigger == nil { - return nil - } - - tg := trigger.(Interface) - if err = tg.Unset(); err != nil { - return err - } - - log.Debugf("%s trigger for policy %d is unset", tg.Kind(), policy.ID) - return nil -} - -func createTrigger(policy *models.ReplicationPolicy) (interface{}, error) { - if policy == nil || policy.Trigger == nil { - return nil, fmt.Errorf("empty policy or trigger") - } - - trigger := policy.Trigger - switch trigger.Kind { - case replication.TriggerKindSchedule: - param := ScheduleParam{} - param.PolicyID = policy.ID - param.Type = trigger.ScheduleParam.Type - param.Weekday = trigger.ScheduleParam.Weekday - param.Offtime = trigger.ScheduleParam.Offtime - - return NewScheduleTrigger(param), nil - case replication.TriggerKindImmediate: - param := ImmediateParam{} - param.PolicyID = policy.ID - param.OnDeletion = policy.ReplicateDeletion - param.Namespaces = policy.Namespaces - - return NewImmediateTrigger(param), nil - case replication.TriggerKindManual: - return nil, nil - default: - return nil, fmt.Errorf("invalid trigger type: %s", trigger.Kind) - } -} diff --git a/src/replication/trigger/manager_test.go b/src/replication/trigger/manager_test.go deleted file mode 100644 index a9cf07f93..000000000 --- a/src/replication/trigger/manager_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package trigger - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/replication" - "github.com/goharbor/harbor/src/replication/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCreateTrigger(t *testing.T) { - // nil policy - _, err := createTrigger(nil) - require.NotNil(t, err) - - // nil trigger - _, err = createTrigger(&models.ReplicationPolicy{}) - require.NotNil(t, err) - - // schedule trigger - trigger, err := createTrigger(&models.ReplicationPolicy{ - Trigger: &models.Trigger{ - Kind: replication.TriggerKindSchedule, - ScheduleParam: &models.ScheduleParam{ - Type: replication.TriggerScheduleWeekly, - Weekday: 1, - Offtime: 1, - }, - }, - }) - require.Nil(t, err) - assert.NotNil(t, trigger) - - // immediate trigger - trigger, err = createTrigger(&models.ReplicationPolicy{ - Trigger: &models.Trigger{ - Kind: replication.TriggerKindImmediate, - }, - }) - require.Nil(t, err) - assert.NotNil(t, trigger) - - // manual trigger - trigger, err = createTrigger(&models.ReplicationPolicy{ - Trigger: &models.Trigger{ - Kind: replication.TriggerKindManual, - }, - }) - require.Nil(t, err) - assert.Nil(t, trigger) -} - -func TestSetupTrigger(t *testing.T) { - dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} - - mgr := NewManager(1) - - err := mgr.SetupTrigger(&models.ReplicationPolicy{ - Trigger: &models.Trigger{ - Kind: replication.TriggerKindImmediate, - }, - }) - assert.Nil(t, err) -} - -func TestUnsetTrigger(t *testing.T) { - dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} - - mgr := NewManager(1) - - err := mgr.UnsetTrigger(&models.ReplicationPolicy{ - Trigger: &models.Trigger{ - Kind: replication.TriggerKindImmediate, - }, - }) - assert.Nil(t, err) -} diff --git a/src/replication/trigger/param_immediate.go b/src/replication/trigger/param_immediate.go deleted file mode 100644 index dde053412..000000000 --- a/src/replication/trigger/param_immediate.go +++ /dev/null @@ -1,22 +0,0 @@ -package trigger - -// NOTES: Whether replicate the existing images when the type of trigger is -// 'Immediate' is a once-effective setting which will not be persisted -// and kept as one parameter of 'Immediate' trigger. It will only be -// covered by the UI logic. - -// ImmediateParam defines the parameter of immediate trigger -type ImmediateParam struct { - // Basic parameters - BasicParam - - // Namepaces - Namespaces []string -} - -// Parse is the implementation of same method in TriggerParam interface -// NOTES: No need to implement this method for 'Immediate' trigger as -// it does not have any parameters with json format. -func (ip ImmediateParam) Parse(param string) error { - return nil -} diff --git a/src/replication/trigger/param_schedule.go b/src/replication/trigger/param_schedule.go deleted file mode 100644 index 6af2f94fa..000000000 --- a/src/replication/trigger/param_schedule.go +++ /dev/null @@ -1,30 +0,0 @@ -package trigger - -import ( - "encoding/json" - "errors" -) - -// ScheduleParam defines the parameter of schedule trigger -type ScheduleParam struct { - // Basic parameters - BasicParam - - // Daily or weekly - Type string - - // Optional, only used when type is 'weekly' - Weekday int8 - - // The time offset with the UTC 00:00 in seconds - Offtime int64 -} - -// Parse is the implementation of same method in TriggerParam interface -func (stp ScheduleParam) Parse(param string) error { - if len(param) == 0 { - return errors.New("Parameter of schedule trigger should not be empty") - } - - return json.Unmarshal([]byte(param), &stp) -} diff --git a/src/replication/trigger/schedule.go b/src/replication/trigger/schedule.go deleted file mode 100644 index 129cce297..000000000 --- a/src/replication/trigger/schedule.go +++ /dev/null @@ -1,107 +0,0 @@ -package trigger - -import ( - "fmt" - "net/http" - - "github.com/goharbor/harbor/src/common/dao" - common_http "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/common/job" - job_models "github.com/goharbor/harbor/src/common/job/models" - "github.com/goharbor/harbor/src/common/models" - common_utils "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/replication" -) - -// ScheduleTrigger will schedule a alternate policy to provide 'daily' and 'weekly' trigger ways. -type ScheduleTrigger struct { - params ScheduleParam -} - -// NewScheduleTrigger is constructor of ScheduleTrigger -func NewScheduleTrigger(params ScheduleParam) *ScheduleTrigger { - return &ScheduleTrigger{ - params: params, - } -} - -// Kind is the implementation of same method defined in Trigger interface -func (st *ScheduleTrigger) Kind() string { - return replication.TriggerKindSchedule -} - -// Setup is the implementation of same method defined in Trigger interface -func (st *ScheduleTrigger) Setup() error { - metadata := &job_models.JobMetadata{ - JobKind: job.JobKindPeriodic, - } - switch st.params.Type { - case replication.TriggerScheduleDaily: - h, m, s := common_utils.ParseOfftime(st.params.Offtime) - metadata.Cron = fmt.Sprintf("%d %d %d * * *", s, m, h) - case replication.TriggerScheduleWeekly: - h, m, s := common_utils.ParseOfftime(st.params.Offtime) - metadata.Cron = fmt.Sprintf("%d %d %d * * %d", s, m, h, st.params.Weekday%7) - default: - return fmt.Errorf("unsupported schedule trigger type: %s", st.params.Type) - } - - id, err := dao.AddRepJob(models.RepJob{ - Repository: "N/A", - PolicyID: st.params.PolicyID, - Operation: models.RepOpSchedule, - }) - if err != nil { - return err - } - uuid, err := utils.GetJobServiceClient().SubmitJob(&job_models.JobData{ - Name: job.ImageReplicate, - Parameters: map[string]interface{}{ - "policy_id": st.params.PolicyID, - "url": config.InternalCoreURL(), - "insecure": true, - }, - Metadata: metadata, - StatusHook: fmt.Sprintf("%s/service/notifications/jobs/replication/%d", - config.InternalCoreURL(), id), - }) - if err != nil { - // clean up the job record in database - if e := dao.DeleteRepJob(id); e != nil { - log.Errorf("failed to delete job %d: %v", id, e) - } - return err - } - return dao.SetRepJobUUID(id, uuid) -} - -// Unset is the implementation of same method defined in Trigger interface -func (st *ScheduleTrigger) Unset() error { - jobs, err := dao.GetRepJobs(&models.RepJobQuery{ - PolicyID: st.params.PolicyID, - Operations: []string{models.RepOpSchedule}, - }) - if err != nil { - return err - } - if len(jobs) != 1 { - log.Warningf("only one job should be found, but found %d now", len(jobs)) - } - - for _, j := range jobs { - if err = utils.GetJobServiceClient().PostAction(j.UUID, job.JobActionStop); err != nil { - // if the job specified by UUID is not found in jobservice, delete the job - // record from database - if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound { - return err - } - } - if err = dao.DeleteRepJob(j.ID); err != nil { - return err - } - } - return nil -} diff --git a/src/replication/trigger/schedule_test.go b/src/replication/trigger/schedule_test.go deleted file mode 100644 index 1e22616af..000000000 --- a/src/replication/trigger/schedule_test.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package trigger - -import ( - "testing" - - "github.com/goharbor/harbor/src/replication" - "github.com/stretchr/testify/assert" -) - -func TestKindOfScheduleTrigger(t *testing.T) { - trigger := NewScheduleTrigger(ScheduleParam{}) - assert.Equal(t, replication.TriggerKindSchedule, trigger.Kind()) -} diff --git a/src/replication/trigger/trigger_param.go b/src/replication/trigger/trigger_param.go deleted file mode 100644 index 92449f032..000000000 --- a/src/replication/trigger/trigger_param.go +++ /dev/null @@ -1,17 +0,0 @@ -package trigger - -// BasicParam contains the general parameters for all triggers -type BasicParam struct { - // ID of the related policy - PolicyID int64 - - // Whether delete remote replicated images if local ones are deleted - OnDeletion bool -} - -// Parameter defines operation of doing initialization from parameter json text -type Parameter interface { - // Decode parameter with json style to the owner struct - // If failed, an error will be returned - Parse(param string) error -} diff --git a/src/replication/trigger/watch_list.go b/src/replication/trigger/watch_list.go deleted file mode 100644 index 5c2dc5248..000000000 --- a/src/replication/trigger/watch_list.go +++ /dev/null @@ -1,65 +0,0 @@ -package trigger - -import ( - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" -) - -// DefaultWatchList is the default instance of WatchList -var DefaultWatchList = &WatchList{} - -// WatchList contains the items which should be evaluated for replication -// when image pushing or deleting happens. -type WatchList struct{} - -// WatchItem keeps the related data for evaluation in WatchList. -type WatchItem struct { - // ID of policy - PolicyID int64 - - // Corresponding namespace - Namespace string - - // For deletion event - OnDeletion bool - - // For pushing event - OnPush bool -} - -// Add item to the list and persist into DB -func (wl *WatchList) Add(item WatchItem) error { - _, err := dao.DefaultDatabaseWatchItemDAO.Add( - &models.WatchItem{ - PolicyID: item.PolicyID, - Namespace: item.Namespace, - OnPush: item.OnPush, - OnDeletion: item.OnDeletion, - }) - return err -} - -// Remove the specified watch item from list -func (wl *WatchList) Remove(policyID int64) error { - return dao.DefaultDatabaseWatchItemDAO.DeleteByPolicyID(policyID) -} - -// Get the watch items according to the namespace and operation -func (wl *WatchList) Get(namespace, operation string) ([]WatchItem, error) { - items, err := dao.DefaultDatabaseWatchItemDAO.Get(namespace, operation) - if err != nil { - return nil, err - } - - watchItems := []WatchItem{} - for _, item := range items { - watchItems = append(watchItems, WatchItem{ - PolicyID: item.PolicyID, - Namespace: item.Namespace, - OnPush: item.OnPush, - OnDeletion: item.OnDeletion, - }) - } - - return watchItems, nil -} diff --git a/src/replication/trigger/watch_list_test.go b/src/replication/trigger/watch_list_test.go deleted file mode 100644 index ad2915d5c..000000000 --- a/src/replication/trigger/watch_list_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package trigger - -import ( - "testing" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMethodsOfWatchList(t *testing.T) { - dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} - - var policyID int64 = 1 - - // test Add - item := WatchItem{ - PolicyID: policyID, - Namespace: "library", - OnDeletion: true, - OnPush: false, - } - - err := DefaultWatchList.Add(item) - require.Nil(t, err) - - // test Get: non-exist namespace - items, err := DefaultWatchList.Get("non-exist-namespace", "delete") - require.Nil(t, err) - assert.Equal(t, 0, len(items)) - - // test Get: non-exist operation - items, err = DefaultWatchList.Get("library", "non-exist-operation") - require.Nil(t, err) - assert.Equal(t, 0, len(items)) - - // test Get: valid params - items, err = DefaultWatchList.Get("library", "delete") - require.Nil(t, err) - assert.Equal(t, 1, len(items)) - assert.Equal(t, policyID, items[0].PolicyID) - - // test Remove - err = DefaultWatchList.Remove(policyID) - require.Nil(t, err) - items, err = DefaultWatchList.Get("library", "delete") - require.Nil(t, err) - assert.Equal(t, 0, len(items)) -}