From 722e45b20b30385462a8df61aee704c7d52dff11 Mon Sep 17 00:00:00 2001 From: Ziming Zhang Date: Tue, 3 Sep 2019 15:52:41 +0800 Subject: [PATCH] add swagger for tag retention Signed-off-by: Ziming Zhang Change-Id: I0f3ed8085e231868de74c273ba85946826181d5b --- docs/swagger.yaml | 483 +++++++++++++++++- src/core/api/retention.go | 22 +- src/pkg/retention/controller_test.go | 4 +- src/pkg/retention/manager.go | 2 +- src/pkg/retention/manager_test.go | 4 +- src/pkg/retention/models.go | 2 +- src/pkg/retention/policy/models.go | 11 +- src/pkg/retention/policy/rule/models.go | 2 +- .../tag-retention.component.html | 2 +- .../tag-retention/tag-retention.component.ts | 7 + tests/apitests/python/library/base.py | 2 +- tests/apitests/python/library/docker_api.py | 43 +- tests/apitests/python/library/repository.py | 8 + tests/apitests/python/library/retention.py | 183 +++++++ tests/apitests/python/test_retention.py | 110 ++++ tests/apitests/python/testutils.py | 4 +- tests/robot-cases/Group0-BAT/API_DB.robot | 3 + 17 files changed, 862 insertions(+), 30 deletions(-) create mode 100644 tests/apitests/python/library/retention.py create mode 100644 tests/apitests/python/test_retention.py diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 8a492d0cd..687f92d15 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -80,7 +80,7 @@ paths: type: integer format: int32 required: false - description: 'The page nubmer, default is 1.' + description: 'The page number, default is 1.' - name: page_size in: query type: integer @@ -280,7 +280,7 @@ paths: type: integer format: int32 required: false - description: 'The page nubmer, default is 1.' + description: 'The page number, default is 1.' - name: page_size in: query type: integer @@ -679,7 +679,7 @@ paths: type: integer format: int32 required: false - description: 'The page nubmer, default is 1.' + description: 'The page number, default is 1.' - name: page_size in: query type: integer @@ -793,7 +793,7 @@ paths: type: integer format: int32 required: false - description: 'The page nubmer, default is 1.' + description: 'The page number, default is 1.' - name: page_size in: query type: integer @@ -1035,7 +1035,7 @@ paths: type: integer format: int32 required: false - description: 'The page nubmer, default is 1.' + description: 'The page number, default is 1.' - name: page_size in: query type: integer @@ -1598,7 +1598,7 @@ paths: type: integer format: int32 required: false - description: 'The page nubmer, default is 1.' + description: 'The page number, default is 1.' - name: page_size in: query type: integer @@ -3625,7 +3625,7 @@ paths: type: integer format: int32 required: false - description: 'The page nubmer, default is 1.' + description: 'The page number, default is 1.' - name: page_size in: query type: integer @@ -3968,6 +3968,277 @@ paths: description: User have no permission to list webhook jobs of the project. '500': description: Unexpected internal errors. + + '/retentions/metadatas': + get: + summary: Get Retention Metadatas + description: Get Retention Metadatas. + tags: + - Products + - Retention + responses: + '200': + description: Get Retention Metadatas successfully. + schema: + type: object + $ref: '#/definitions/RetentionMetadata' + + '/retentions': + post: + summary: Create Retention Policy + description: | + Create Retention Policy, you can reference metadatas API for the policy model. + You can check project metadatas to find whether a retention policy is already binded. + This method should only be called when no retention policy binded to project yet. + tags: + - Products + - Retention + parameters: + - name: policy + in: body + description: Create Retention Policy successfully. + required: true + schema: + $ref: '#/definitions/RetentionPolicy' + responses: + '201': + description: Project created successfully. + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission. + '500': + description: Unexpected internal errors. + + '/retentions/{id}': + get: + summary: Get Retention Policy + description: Get Retention Policy. + tags: + - Products + - Retention + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Retention ID. + responses: + '200': + description: Get Retention Policy successfully. + schema: + type: object + $ref: '#/definitions/RetentionPolicy' + '401': + description: User need to log in first. + '403': + description: User have no permission. + '500': + description: Unexpected internal errors. + put: + summary: Update Retention Policy + description: | + Update Retention Policy, you can reference metadatas API for the policy model. + You can check project metadatas to find whether a retention policy is already binded. + This method should only be called when retention policy has already binded to project. + tags: + - Products + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Retention ID. + - name: policy + in: body + required: true + schema: + $ref: '#/definitions/RetentionPolicy' + responses: + '200': + description: Update Retention Policy successfully. + '401': + description: User need to log in first. + '403': + description: User have no permission. + '500': + description: Unexpected internal errors. + + '/retentions/{id}/executions': + post: + summary: Trigger a Retention job + description: Trigger a Retention job, if dry_run is True, nothing would be deleted actually. + tags: + - Products + - Retention + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Retention ID. + - name: action + in: body + required: true + schema: + type: object + properties: + dry_run: + type: boolean + responses: + '200': + description: Trigger a Retention job successfully. + '401': + description: User need to log in first. + '403': + description: User have no permission. + '500': + description: Unexpected internal errors. + get: + summary: Get a Retention job + description: Get a Retention job, job status may be delayed before job service schedule it up. + tags: + - Products + - Retention + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Retention ID. + responses: + '200': + description: Get a Retention job successfully. + schema: + type: array + items: + type: object + $ref: '#/definitions/RetentionExecution' + '401': + description: User need to log in first. + '403': + description: User have no permission. + '500': + description: Unexpected internal errors. + + '/retentions/{id}/executions/{eid}': + patch: + summary: Stop a Retention job + description: Stop a Retention job, only support "stop" action now. + tags: + - Products + - Retention + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Retention ID. + - name: eid + in: path + type: integer + format: int64 + required: true + description: Retention execution ID. + - name: action + in: body + description: The action, only support "stop" now. + required: true + schema: + type: object + properties: + action: + type: string + responses: + '200': + description: Stop a Retention job successfully. + '401': + description: User need to log in first. + '403': + description: User have no permission. + '500': + description: Unexpected internal errors. + + '/retentions/{id}/executions/{eid}/tasks': + get: + summary: Get Retention job tasks + description: Get Retention job tasks, each repository as a task. + tags: + - Products + - Retention + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Retention ID. + - name: eid + in: path + type: integer + format: int64 + required: true + description: Retention execution ID. + responses: + '200': + description: Get Retention job tasks successfully. + schema: + type: array + items: + type: object + $ref: '#/definitions/RetentionExecutionTask' + '401': + description: User need to log in first. + '403': + description: User have no permission. + '500': + description: Unexpected internal errors. + + '/retentions/{id}/executions/{eid}/tasks/{tid}': + get: + summary: Get Retention job task log + description: Get Retention job task log, tags ratain or deletion detail will be shown in a table. + tags: + - Products + - Retention + parameters: + - name: id + in: path + type: integer + format: int64 + required: true + description: Retention ID. + - name: eid + in: path + type: integer + format: int64 + required: true + description: Retention execution ID. + - name: tid + in: path + type: integer + format: int64 + required: true + description: Retention execution ID. + responses: + '200': + description: Get Retention job task log successfully. + schema: + type: string + '401': + description: User need to log in first. + '403': + description: User have no permission. + '500': + description: Unexpected internal errors. + responses: OK: description: 'Success' @@ -4714,7 +4985,7 @@ definitions: description: The digest of the image. scan_status: type: string - description: 'The status of the scan job, it can be "pendnig", "running", "finished", "error".' + description: 'The status of the scan job, it can be "pending", "running", "finished", "error".' job_id: type: integer description: The ID of the job on jobservice to scan the image. @@ -4904,7 +5175,7 @@ definitions: properties: daily_time: type: integer - description: 'The offest in seconds of UTC 0 o''clock, only valid when the policy type is "daily"' + description: 'The offset in seconds of UTC 0 o''clock, only valid when the policy type is "daily"' description: 'The parameters of the policy, the values are dependant on the type of the policy.' ConfigurationsResponse: type: object @@ -5004,7 +5275,7 @@ definitions: properties: daily_time: type: integer - description: 'The offest in seconds of UTC 0 o''clock, only valid when the policy type is "daily"' + description: 'The offset in seconds of UTC 0 o''clock, only valid when the policy type is "daily"' description: 'The parameters of the policy, the values are dependant on the type of the policy.' RepositoryDescription: type: object @@ -5786,3 +6057,195 @@ definitions: update_time: type: string description: The webhook job update time. + + RetentionMetadata: + type: object + description: the tag retention metadata + properties: + templates: + type: array + description: templates + items: + $ref: '#/definitions/RetentionRuleMetadata' + scope_selectors: + type: array + description: supported scope selectors + items: + $ref: '#/definitions/RetentionSelectorMetadata' + tag_selectors: + type: array + description: supported tag selectors + items: + $ref: '#/definitions/RetentionSelectorMetadata' + + RetentionRuleMetadata: + type: object + description: the tag retention rule metadata + properties: + rule_template: + type: string + description: rule id + display_text: + type: string + description: rule display text + action: + type: string + description: rule action + params: + type: array + description: rule params + items: + $ref: '#/definitions/RetentionRuleParamMetadata' + + RetentionRuleParamMetadata: + type: object + description: rule param + properties: + type: + type: string + unit: + type: string + required: + type: boolean + + + RetentionSelectorMetadata: + type: object + description: retention selector + properties: + display_text: + type: string + kind: + type: string + decorations: + type: array + items: + type: string + + RetentionPolicy: + type: object + description: retention policy + properties: + id: + type: integer + format: int64 + algorithm: + type: string + rules: + type: array + items: + $ref: '#/definitions/RetentionRule' + trigger: + type: object + items: + $ref: '#/definitions/RetentionRuleTrigger' + scope: + type: object + items: + $ref: '#/definitions/RetentionPolicyScope' + + RetentionRuleTrigger: + type: object + properties: + kind: + type: string + settings: + type: object + references: + type: object + + RetentionPolicyScope: + type: object + properties: + level: + type: string + ref: + type: integer + + RetentionRule: + type: object + properties: + id: + type: integer + priority: + type: integer + disabled: + type: boolean + action: + type: string + template: + type: string + params: + type: object + additionalProperties: + type: object + tag_selectors: + type: array + items: + $ref: '#/definitions/RetentionSelector' + scope_selectors: + type: object + additionalProperties: + type: array + items: + $ref: '#/definitions/RetentionSelector' + + RetentionSelector: + type: object + properties: + kind: + type: string + decoration: + type: string + pattern: + type: string + + RetentionExecution: + type: object + properties: + id: + type: integer + format: int64 + policy_id: + type: integer + format: int64 + start_time: + type: string + end_time: + type: string + status: + type: string + trigger: + type: string + dry_run: + type: boolean + + RetentionExecutionTask: + type: object + properties: + id: + type: integer + format: int64 + execution_id: + type: integer + format: int64 + repository: + type: string + job_id: + type: string + status: + type: string + status_code: + type: integer + status_revision: + type: integer + format: int64 + start_time: + type: string + end_time: + type: string + total: + type: integer + retained: + type: integer + diff --git a/src/core/api/retention.go b/src/core/api/retention.go index a880db895..5fa8a6700 100644 --- a/src/core/api/retention.go +++ b/src/core/api/retention.go @@ -151,6 +151,10 @@ func (r *RetentionAPI) CreateRetention() { r.SendBadRequestError(err) return } + if len(p.Rules) > 15 { + r.SendBadRequestError(errors.New("only 15 rules are allowed at most")) + return + } if err = r.checkRuleConflict(p); err != nil { r.SendConflictError(err) return @@ -176,6 +180,15 @@ func (r *RetentionAPI) CreateRetention() { r.SendBadRequestError(fmt.Errorf("scope %s is not support", p.Scope.Level)) return } + old, err := r.pm.GetMetadataManager().Get(p.Scope.Reference, "retention_id") + if err != nil { + r.SendInternalServerError(err) + return + } + if old != nil && len(old) > 0 { + r.SendBadRequestError(fmt.Errorf("project %v already has retention policy %v", p.Scope.Reference, old["retention_id"])) + return + } id, err := retentionController.CreateRetention(p) if err != nil { r.SendInternalServerError(err) @@ -202,6 +215,10 @@ func (r *RetentionAPI) UpdateRetention() { return } p.ID = id + if len(p.Rules) > 15 { + r.SendBadRequestError(errors.New("only 15 rules are allowed at most")) + return + } if err = r.checkRuleConflict(p); err != nil { r.SendConflictError(err) return @@ -218,14 +235,13 @@ func (r *RetentionAPI) UpdateRetention() { func (r *RetentionAPI) checkRuleConflict(p *policy.Metadata) error { temp := make(map[string]int) for n, rule := range p.Rules { - tid := rule.ID rule.ID = 0 bs, _ := json.Marshal(rule) if old, exists := temp[string(bs)]; exists { return fmt.Errorf("rule %d is conflict with rule %d", n, old) } temp[string(bs)] = n - rule.ID = tid + rule.ID = n } return nil } @@ -276,7 +292,7 @@ func (r *RetentionAPI) OperateRetentionExec() { return } a := &struct { - Action string `json:"action" valid:"Required"` + Action string `json:"action" valid:"Required;Match(stop)"` }{} isValid, err := r.DecodeJSONReqAndValidate(a) if !isValid { diff --git a/src/pkg/retention/controller_test.go b/src/pkg/retention/controller_test.go index 692b17407..d2627c94f 100644 --- a/src/pkg/retention/controller_test.go +++ b/src/pkg/retention/controller_test.go @@ -1,6 +1,7 @@ package retention import ( + "strings" "testing" "github.com/goharbor/harbor/src/pkg/retention/dep" @@ -118,7 +119,8 @@ func (s *ControllerTestSuite) TestPolicy() { s.Require().Nil(err) p1, err = c.GetRetention(id) - s.Require().Nil(err) + s.Require().NotNil(err) + s.Require().True(strings.Contains(err.Error(), "no such Retention policy")) s.Require().Nil(p1) } diff --git a/src/pkg/retention/manager.go b/src/pkg/retention/manager.go index e0cc2d1ab..c123491cc 100644 --- a/src/pkg/retention/manager.go +++ b/src/pkg/retention/manager.go @@ -111,7 +111,7 @@ func (d *DefaultManager) GetPolicy(id int64) (*policy.Metadata, error) { p1, err := dao.GetPolicy(id) if err != nil { if err == orm.ErrNoRows { - return nil, nil + return nil, fmt.Errorf("no such Retention policy with id %v", id) } return nil, err } diff --git a/src/pkg/retention/manager_test.go b/src/pkg/retention/manager_test.go index 690fc2611..1a2f97c8d 100644 --- a/src/pkg/retention/manager_test.go +++ b/src/pkg/retention/manager_test.go @@ -2,6 +2,7 @@ package retention import ( "os" + "strings" "testing" "time" @@ -83,7 +84,8 @@ func TestPolicy(t *testing.T) { assert.Nil(t, err) p1, err = m.GetPolicy(id) - assert.Nil(t, err) + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "no such Retention policy")) assert.Nil(t, p1) } diff --git a/src/pkg/retention/models.go b/src/pkg/retention/models.go index 920b98e7d..a1bc1f689 100644 --- a/src/pkg/retention/models.go +++ b/src/pkg/retention/models.go @@ -37,7 +37,7 @@ type Execution struct { StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time,omitempty"` Status string `json:"status"` - Trigger string `json:"Trigger"` + Trigger string `json:"trigger"` DryRun bool `json:"dry_run"` } diff --git a/src/pkg/retention/policy/models.go b/src/pkg/retention/policy/models.go index 8cc31e834..e8fd86ad7 100644 --- a/src/pkg/retention/policy/models.go +++ b/src/pkg/retention/policy/models.go @@ -53,27 +53,24 @@ type Metadata struct { // Which scope the policy will be applied to Scope *Scope `json:"scope" valid:"Required"` - - // The max number of rules in a policy - Capacity int `json:"cap"` } // Valid Valid func (m *Metadata) Valid(v *validation.Validation) { if m.Trigger == nil { - _ = v.SetError("Trigger", "Trigger is required") + _ = v.SetError("Trigger", "Can not be empty") return } if m.Scope == nil { - _ = v.SetError("Scope", "Scope is required") + _ = v.SetError("Scope", "Can not be empty") return } if m.Trigger != nil && m.Trigger.Kind == TriggerKindSchedule { if m.Trigger.Settings == nil { - _ = v.SetError("Trigger.Settings", "Trigger.Settings is required") + _ = v.SetError("Trigger.Settings", "Can not be empty") } else { if _, ok := m.Trigger.Settings[TriggerSettingsCron]; !ok { - _ = v.SetError("Trigger.Settings", "cron in Trigger.Settings is required") + _ = v.SetError("Trigger.Settings.cron", "Can not be empty") } } } diff --git a/src/pkg/retention/policy/rule/models.go b/src/pkg/retention/policy/rule/models.go index 6dd49534b..cf90ff3a7 100644 --- a/src/pkg/retention/policy/rule/models.go +++ b/src/pkg/retention/policy/rule/models.go @@ -37,7 +37,7 @@ type Metadata struct { Template string `json:"template" valid:"Required"` // The parameters of this rule - Parameters Parameters `json:"params"` + Parameters Parameters `json:"params" valid:"Required"` // Selector attached to the rule for filtering tags TagSelectors []*Selector `json:"tag_selectors" valid:"Required"` diff --git a/src/portal/src/app/project/tag-retention/tag-retention.component.html b/src/portal/src/app/project/tag-retention/tag-retention.component.html index 649c4b7b2..4f9255e74 100644 --- a/src/portal/src/app/project/tag-retention/tag-retention.component.html +++ b/src/portal/src/app/project/tag-retention/tag-retention.component.html @@ -116,7 +116,7 @@ {{execution.dry_run ? 'YES' : 'NO'}} {{execution.Trigger}} + (click)="openDetail(i,execution.id)">{{execution.trigger}} {{execution.start_time|date:'medium'}} {{execution.duration}} diff --git a/src/portal/src/app/project/tag-retention/tag-retention.component.ts b/src/portal/src/app/project/tag-retention/tag-retention.component.ts index d424646fb..dcc80f795 100644 --- a/src/portal/src/app/project/tag-retention/tag-retention.component.ts +++ b/src/portal/src/app/project/tag-retention/tag-retention.component.ts @@ -166,6 +166,13 @@ export class TagRetentionComponent implements OnInit { if (this.retentionId) { this.tagRetentionService.getRetention(this.retentionId).subscribe( response => { + if (response && response.rules && response.rules.length > 0) { + response.rules.forEach(item => { + if (!item.params) { + item.params = {}; + } + }); + } this.retention = response; this.loadingRule = false; }, error => { diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index daaf8eba7..9083e82ba 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -39,7 +39,7 @@ def _random_name(prefix): def _get_id_from_header(header): try: location = header["Location"] - return location.split("/")[-1] + return int(location.split("/")[-1]) except Exception: return None diff --git a/tests/apitests/python/library/docker_api.py b/tests/apitests/python/library/docker_api.py index 9def39472..aa5a0e0db 100644 --- a/tests/apitests/python/library/docker_api.py +++ b/tests/apitests/python/library/docker_api.py @@ -12,6 +12,7 @@ except ImportError: class DockerAPI(object): def __init__(self): self.DCLIENT = docker.APIClient(base_url='unix://var/run/docker.sock',version='auto',timeout=10) + self.DCLIENT2 = docker.from_env() def docker_login(self, registry, username, password, expected_error_message = None): if expected_error_message is "": @@ -84,4 +85,44 @@ class DockerAPI(object): raise Exception(r" Failed to catch error [{}] when push image {}".format (expected_error_message, harbor_registry)) else: if str(ret).lower().find("errorDetail".lower()) >= 0: - raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, ret)) \ No newline at end of file + raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, ret)) + + def docker_image_build(self, harbor_registry, tags=None, size=1, expected_error_message = None): + caught_err = False + ret = "" + try: + baseimage='busybox:latest' + if not self.DCLIENT.images(name=baseimage): + self.DCLIENT.pull(baseimage) + c=self.DCLIENT.create_container(image='busybox:latest',command='dd if=/dev/urandom of=test bs=1M count=%d' % size ) + self.DCLIENT.start(c) + self.DCLIENT.wait(c) + if not tags: + tags=['latest'] + firstrepo="%s:%s" % (harbor_registry, tags[0]) + #self.DCLIENT.commit(c, firstrepo) + self.DCLIENT2.containers.get(c).commit(harbor_registry, tags[0]) + for tag in tags[1:]: + repo="%s:%s" % (harbor_registry, tag) + self.DCLIENT.tag(firstrepo, repo) + for tag in tags: + repo="%s:%s" % (harbor_registry, tag) + self.DCLIENT.push(repo) + print("build image %s with size %d" % (repo, size)) + self.DCLIENT.remove_image(repo) + self.DCLIENT.remove_container(c) + except Exception, err: + caught_err = True + if expected_error_message is not None: + print "docker image build error:", str(err) + if str(err).lower().find(expected_error_message.lower()) < 0: + raise Exception(r"Push image: Return message {} is not as expected {}".format(str(err), expected_error_message)) + else: + raise Exception(r" Docker build image {} failed, error is [{}]".format (harbor_registry, err.message)) + if caught_err == False: + if expected_error_message is not None: + if str(ret).lower().find(expected_error_message.lower()) < 0: + raise Exception(r" Failed to catch error [{}] when push image {}".format (expected_error_message, harbor_registry)) + else: + if str(ret).lower().find("errorDetail".lower()) >= 0: + raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, ret)) diff --git a/tests/apitests/python/library/repository.py b/tests/apitests/python/library/repository.py index 5d1462e54..ddef01b4b 100644 --- a/tests/apitests/python/library/repository.py +++ b/tests/apitests/python/library/repository.py @@ -30,6 +30,14 @@ def push_image_to_project(project_name, registry, username, password, image, tag return r'{}/{}'.format(project_name, image), new_tag +def push_special_image_to_project(project_name, registry, username, password, image, tags=None, size=1, expected_login_error_message=None, expected_error_message = None): + _docker_api = DockerAPI() + _docker_api.docker_login(registry, username, password, expected_error_message = expected_login_error_message) + time.sleep(2) + if expected_login_error_message != None: + return + _docker_api.docker_image_build(r'{}/{}/{}'.format(registry, project_name, image), tags = tags, size=size, expected_error_message=expected_error_message) + def is_repo_exist_in_project(repositories, repo_name): result = False for reop in repositories: diff --git a/tests/apitests/python/library/retention.py b/tests/apitests/python/library/retention.py new file mode 100644 index 000000000..90f9f19d7 --- /dev/null +++ b/tests/apitests/python/library/retention.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- + +import base +import swagger_client + +class Retention(base.Base): + def get_metadatas(self, expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + metadatas, status_code, _ = client.retentions_metadatas_get_with_http_info() + base._assert_status_code(expect_status_code, status_code) + return metadatas + + def create_retention_policy(self, project_id, selector_repository="**", selector_tag="**", expect_status_code = 201, **kwargs): + policy=swagger_client.RetentionPolicy( + algorithm='or', + rules=[ + swagger_client.RetentionRule( + disabled=False, + action="retain", + template="always", + params= { + + }, + scope_selectors={ + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": selector_repository + } + ] + }, + tag_selectors=[ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": selector_tag + } + ] + ) + ], + trigger= { + "kind": "Schedule", + "settings": { + "cron": "" + }, + "references": { + } + }, + scope= { + "level": "project", + "ref": project_id + }, + ) + client = self._get_client(**kwargs) + _, status_code, header = client.retentions_post_with_http_info(policy) + base._assert_status_code(expect_status_code, status_code) + return base._get_id_from_header(header) + + def get_retention_policy(self, retention_id, expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + policy, status_code, _ = client.retentions_id_get_with_http_info(retention_id) + base._assert_status_code(expect_status_code, status_code) + return policy + + def update_retention_policy(self, retention_id, selector_repository="**", selector_tag="**", expect_status_code = 200, **kwargs): + policy=swagger_client.RetentionPolicy( + id=retention_id, + algorithm='or', + rules=[ + swagger_client.RetentionRule( + disabled=False, + action="retain", + template="always", + params= { + + }, + scope_selectors={ + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": selector_repository + } + ] + }, + tag_selectors=[ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": selector_tag + } + ] + ) + ], + trigger= { + "kind": "Schedule", + "settings": { + "cron": "" + }, + "references": { + } + }, + scope= { + "level": "project", + "ref": project_id + }, + ) + client = self._get_client(**kwargs) + _, status_code, _ = client.retentions_id_put_with_http_info(retention_id, policy) + base._assert_status_code(expect_status_code, status_code) + + def update_retention_add_rule(self, retention_id, selector_repository="**", selector_tag="**", expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + policy, status_code, _ = client.retentions_id_get_with_http_info(retention_id) + base._assert_status_code(200, status_code) + policy.rules.append(swagger_client.RetentionRule( + disabled=False, + action="retain", + template="always", + params= { + + }, + scope_selectors={ + "repository": [ + { + "kind": "doublestar", + "decoration": "repoMatches", + "pattern": selector_repository + } + ] + }, + tag_selectors=[ + { + "kind": "doublestar", + "decoration": "matches", + "pattern": selector_tag + } + ] + )) + _, status_code, _ = client.retentions_id_put_with_http_info(retention_id, policy) + base._assert_status_code(expect_status_code, status_code) + + def trigger_retention_policy(self, retention_id, dry_run=False, expect_status_code = 201, **kwargs): + client = self._get_client(**kwargs) + + _, status_code, _ = client.retentions_id_executions_post_with_http_info(retention_id, {"dry_run":dry_run}) + base._assert_status_code(expect_status_code, status_code) + + def stop_retention_execution(self, retention_id, exec_id, expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + + r, status_code, _ = client.retentions_id_executions_eid_patch(retention_id, exec_id, {"action":"stop"}) + base._assert_status_code(expect_status_code, status_code) + return r + + def get_retention_executions(self, retention_id, expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + + r, status_code, _ = client.retentions_id_executions_get_with_http_info(retention_id) + base._assert_status_code(expect_status_code, status_code) + return r + + def get_retention_exec_tasks(self, retention_id, exec_id, expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + + r, status_code, _ = client.retentions_id_executions_eid_tasks_get_with_http_info(retention_id, exec_id) + base._assert_status_code(expect_status_code, status_code) + return r + + def get_retention_exec_task_log(self, retention_id, exec_id, task_id, expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + + r, status_code, _ = client.retentions_id_executions_eid_tasks_tid_get_with_http_info(retention_id, exec_id, task_id) + base._assert_status_code(expect_status_code, status_code) + return r + + def get_retention_metadatas(self, expect_status_code = 200, **kwargs): + client = self._get_client(**kwargs) + + r, status_code, _ = client.retentions_metadatas_get_with_http_info() + base._assert_status_code(expect_status_code, status_code) + return r diff --git a/tests/apitests/python/test_retention.py b/tests/apitests/python/test_retention.py new file mode 100644 index 000000000..06584af23 --- /dev/null +++ b/tests/apitests/python/test_retention.py @@ -0,0 +1,110 @@ +from __future__ import absolute_import + +import unittest +import time + +from testutils import ADMIN_CLIENT +from testutils import TEARDOWN +from testutils import harbor_server +from library.repository import push_special_image_to_project + +from library.retention import Retention +from library.project import Project +from library.repository import Repository +from library.user import User +from library.system import System + + +class TestProjects(unittest.TestCase): + """ + Test case: + Tag Retention + Setup: + Create Project test-retention + Push image test1:1.0, test1:2.0, test1:3.0,latest, test2:1.0, test2:latest, test3:1.0, test4:1.0 + Test Steps: + 1. Create Retention Policy + 2. Add rule "For the repositories matching **, retain always with tags matching latest*" + 3. Add rule "For the repositories matching test3*, retain always with tags matching **" + 4. Dry run, check execution and tasks + 5. Real run, check images retained + Tear Down: + 1. Delete project test-retention + """ + @classmethod + def setUpClass(self): + self.user = User() + self.system = System() + self.repo= Repository() + self.project = Project() + self.retention=Retention() + + def testTagRetention(self): + user_ra_password = "Aa123456" + user_ra_id, user_ra_name = self.user.create_user(user_password=user_ra_password, **ADMIN_CLIENT) + print("Created user: %s, id: %s" % (user_ra_name, user_ra_id)) + TestProjects.USER_RA_CLIENT = dict(endpoint=ADMIN_CLIENT["endpoint"], + username=user_ra_name, + password=user_ra_password) + TestProjects.user_ra_id = int(user_ra_id) + + TestProjects.project_src_repo_id, project_src_repo_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_RA_CLIENT) + + # Push image test1:1.0, test1:2.0, test1:3.0,latest, test2:1.0, test2:latest, test3:1.0 + push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['1.0']) + push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['2.0']) + push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['3.0','latest']) + push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['1.0']) + push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['latest']) + push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test3", ['1.0']) + push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test4", ['1.0']) + + resp=self.repo.get_repository(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) + self.assertEqual(len(resp), 4) + + # Create Retention Policy + retention_id = self.retention.create_retention_policy(TestProjects.project_src_repo_id, selector_repository="**", selector_tag="latest*", expect_status_code = 201, **TestProjects.USER_RA_CLIENT) + + # Add rule + self.retention.update_retention_add_rule(retention_id,selector_repository="test3*", selector_tag="**", expect_status_code = 200, **TestProjects.USER_RA_CLIENT) + + # Dry run + self.retention.trigger_retention_policy(retention_id, dry_run=True, **TestProjects.USER_RA_CLIENT) + time.sleep(2) + resp=self.retention.get_retention_executions(retention_id, **TestProjects.USER_RA_CLIENT) + self.assertTrue(len(resp)>0) + execution=resp[0] + resp=self.retention.get_retention_exec_tasks(retention_id,execution.id, **TestProjects.USER_RA_CLIENT) + self.assertEqual(len(resp), 4) + resp=self.retention.get_retention_exec_task_log(retention_id,execution.id,resp[0].id, **TestProjects.USER_RA_CLIENT) + print(resp) + + # Real run + self.retention.trigger_retention_policy(retention_id, dry_run=False, **TestProjects.USER_RA_CLIENT) + time.sleep(10) + resp=self.retention.get_retention_executions(retention_id, **TestProjects.USER_RA_CLIENT) + self.assertTrue(len(resp)>1) + execution=resp[0] + resp=self.retention.get_retention_exec_tasks(retention_id,execution.id, **TestProjects.USER_RA_CLIENT) + self.assertEqual(len(resp), 4) + resp=self.retention.get_retention_exec_task_log(retention_id,execution.id,resp[0].id, **TestProjects.USER_RA_CLIENT) + print(resp) + resp=self.repo.get_repository(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) + self.assertEqual(len(resp), 3) + + + @classmethod + def tearDownClass(self): + print "Case completed" + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def test_ClearData(self): + resp=self.repo.get_repository(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) + for repo in resp: + self.repo.delete_repoitory(repo.name, **TestProjects.USER_RA_CLIENT) + self.project.delete_project(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) + self.user.delete_user(TestProjects.user_ra_id, **ADMIN_CLIENT) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/apitests/python/testutils.py b/tests/apitests/python/testutils.py index 0bb2838b6..7417ca1bd 100644 --- a/tests/apitests/python/testutils.py +++ b/tests/apitests/python/testutils.py @@ -2,7 +2,7 @@ import time import os import sys -sys.path.append(os.environ["SWAGGER_CLIENT_PATH"]) +sys.path.insert(0, os.environ["SWAGGER_CLIENT_PATH"]) from swagger_client.rest import ApiException import swagger_client.models from pprint import pprint @@ -12,7 +12,7 @@ admin_pwd = "Harbor12345" harbor_server = os.environ["HARBOR_HOST"] #CLIENT=dict(endpoint="https://"+harbor_server+"/api") -ADMIN_CLIENT=dict(endpoint = "https://"+harbor_server+"/api", username = admin_user, password = admin_pwd) +ADMIN_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api", username = admin_user, password = admin_pwd) USER_ROLE=dict(admin=0,normal=1) TEARDOWN = True diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index 6679af892..0324e476c 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -53,3 +53,6 @@ Test Case - System Level CVE Whitelist Harbor API Test ./tests/apitests/python/test_sys_cve_whitelists.py Test Case - Project Level CVE Whitelist Harbor API Test ./tests/apitests/python/test_project_level_cve_whitelist.py +Test Case - Tag Retention + Harbor API Test ./tests/apitests/python/test_retention.py +