Merge pull request #8936 from bitsf/tag_retention_swagger

add swagger for tag retention

docs/swagger.yaml

@@ -80,7 +80,7 @@ paths:
           type: integer
           format: int32
           required: false
-          description: 'The page nubmer, default is 1.'
+          description: 'The page number, default is 1.'
         - name: page_size
           in: query
           type: integer
@@ -280,7 +280,7 @@ paths:
           type: integer
           format: int32
           required: false
-          description: 'The page nubmer, default is 1.'
+          description: 'The page number, default is 1.'
         - name: page_size
           in: query
           type: integer
@@ -679,7 +679,7 @@ paths:
           type: integer
           format: int32
           required: false
-          description: 'The page nubmer, default is 1.'
+          description: 'The page number, default is 1.'
         - name: page_size
           in: query
           type: integer
@@ -793,7 +793,7 @@ paths:
           type: integer
           format: int32
           required: false
-          description: 'The page nubmer, default is 1.'
+          description: 'The page number, default is 1.'
         - name: page_size
           in: query
           type: integer
@@ -1035,7 +1035,7 @@ paths:
           type: integer
           format: int32
           required: false
-          description: 'The page nubmer, default is 1.'
+          description: 'The page number, default is 1.'
         - name: page_size
           in: query
           type: integer
@@ -1598,7 +1598,7 @@ paths:
           type: integer
           format: int32
           required: false
-          description: 'The page nubmer, default is 1.'
+          description: 'The page number, default is 1.'
         - name: page_size
           in: query
           type: integer
@@ -3625,7 +3625,7 @@ paths:
           type: integer
           format: int32
           required: false
-          description: 'The page nubmer, default is 1.'
+          description: 'The page number, default is 1.'
         - name: page_size
           in: query
           type: integer
@@ -3968,6 +3968,277 @@ paths:
           description: User have no permission to list webhook jobs of the project.
         '500':
           description: Unexpected internal errors.
+
+  '/retentions/metadatas':
+    get:
+      summary: Get Retention Metadatas
+      description: Get Retention Metadatas.
+      tags:
+        - Products
+        - Retention
+      responses:
+        '200':
+          description: Get Retention Metadatas successfully.
+          schema:
+            type: object
+            $ref: '#/definitions/RetentionMetadata'
+
+  '/retentions':
+    post:
+      summary: Create Retention Policy
+      description: |
+        Create Retention Policy, you can reference metadatas API for the policy model.
+        You can check project metadatas to find whether a retention policy is already binded.
+        This method should only be called when no retention policy binded to project yet.
+      tags:
+        - Products
+        - Retention
+      parameters:
+        - name: policy
+          in: body
+          description: Create Retention Policy successfully.
+          required: true
+          schema:
+            $ref: '#/definitions/RetentionPolicy'
+      responses:
+        '201':
+          description: Project created successfully.
+        '400':
+          description: Illegal format of provided ID value.
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User have no permission.
+        '500':
+          description: Unexpected internal errors.
+
+  '/retentions/{id}':
+    get:
+      summary: Get Retention Policy
+      description: Get Retention Policy.
+      tags:
+        - Products
+        - Retention
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention ID.
+      responses:
+        '200':
+          description: Get Retention Policy successfully.
+          schema:
+            type: object
+            $ref: '#/definitions/RetentionPolicy'
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User have no permission.
+        '500':
+          description: Unexpected internal errors.
+    put:
+      summary: Update Retention Policy
+      description: |
+        Update Retention Policy, you can reference metadatas API for the policy model.
+        You can check project metadatas to find whether a retention policy is already binded.
+        This method should only be called when retention policy has already binded to project.
+      tags:
+        - Products
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention ID.
+        - name: policy
+          in: body
+          required: true
+          schema:
+            $ref: '#/definitions/RetentionPolicy'
+      responses:
+        '200':
+          description: Update Retention Policy successfully.
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User have no permission.
+        '500':
+          description: Unexpected internal errors.
+
+  '/retentions/{id}/executions':
+    post:
+      summary: Trigger a Retention job
+      description: Trigger a Retention job, if dry_run is True, nothing would be deleted actually.
+      tags:
+        - Products
+        - Retention
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention ID.
+        - name: action
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              dry_run:
+                type: boolean
+      responses:
+        '200':
+          description: Trigger a Retention job successfully.
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User have no permission.
+        '500':
+          description: Unexpected internal errors.
+    get:
+      summary: Get a Retention job
+      description: Get a Retention job, job status may be delayed before job service schedule it up.
+      tags:
+        - Products
+        - Retention
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention ID.
+      responses:
+        '200':
+          description: Get a Retention job successfully.
+          schema:
+            type: array
+            items:
+              type: object
+              $ref: '#/definitions/RetentionExecution'
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User have no permission.
+        '500':
+          description: Unexpected internal errors.
+
+  '/retentions/{id}/executions/{eid}':
+    patch:
+      summary: Stop a Retention job
+      description: Stop a Retention job, only support "stop" action now.
+      tags:
+        - Products
+        - Retention
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention ID.
+        - name: eid
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention execution ID.
+        - name: action
+          in: body
+          description: The action, only support "stop" now.
+          required: true
+          schema:
+            type: object
+            properties:
+              action:
+                type: string
+      responses:
+        '200':
+          description: Stop a Retention job successfully.
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User have no permission.
+        '500':
+          description: Unexpected internal errors.
+
+  '/retentions/{id}/executions/{eid}/tasks':
+    get:
+      summary: Get Retention job tasks
+      description: Get Retention job tasks, each repository as a task.
+      tags:
+        - Products
+        - Retention
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention ID.
+        - name: eid
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention execution ID.
+      responses:
+        '200':
+          description: Get Retention job tasks successfully.
+          schema:
+            type: array
+            items:
+              type: object
+              $ref: '#/definitions/RetentionExecutionTask'
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User have no permission.
+        '500':
+          description: Unexpected internal errors.
+
+  '/retentions/{id}/executions/{eid}/tasks/{tid}':
+    get:
+      summary: Get Retention job task log
+      description: Get Retention job task log, tags ratain or deletion detail will be shown in a table.
+      tags:
+        - Products
+        - Retention
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention ID.
+        - name: eid
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention execution ID.
+        - name: tid
+          in: path
+          type: integer
+          format: int64
+          required: true
+          description: Retention execution ID.
+      responses:
+        '200':
+          description: Get Retention job task log successfully.
+          schema:
+            type: string
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User have no permission.
+        '500':
+          description: Unexpected internal errors.
+
 responses:
   OK:
     description: 'Success'
@@ -4714,7 +4985,7 @@ definitions:
             description: The digest of the image.
           scan_status:
             type: string
-            description: 'The status of the scan job, it can be "pendnig", "running", "finished", "error".'
+            description: 'The status of the scan job, it can be "pending", "running", "finished", "error".'
           job_id:
             type: integer
             description: The ID of the job on jobservice to scan the image.
@@ -4904,7 +5175,7 @@ definitions:
             properties:
               daily_time:
                 type: integer
-                description: 'The offest in seconds of UTC 0 o''clock, only valid when the policy type is "daily"'
+                description: 'The offset in seconds of UTC 0 o''clock, only valid when the policy type is "daily"'
             description: 'The parameters of the policy, the values are dependant on the type of the policy.'
   ConfigurationsResponse:
     type: object
@@ -5004,7 +5275,7 @@ definitions:
             properties:
               daily_time:
                 type: integer
-                description: 'The offest in seconds of UTC 0 o''clock, only valid when the policy type is "daily"'
+                description: 'The offset in seconds of UTC 0 o''clock, only valid when the policy type is "daily"'
             description: 'The parameters of the policy, the values are dependant on the type of the policy.'
   RepositoryDescription:
     type: object
@@ -5786,3 +6057,195 @@ definitions:
       update_time:
         type: string
         description: The webhook job update time.
+
+  RetentionMetadata:
+    type: object
+    description: the tag retention metadata
+    properties:
+      templates:
+        type: array
+        description: templates
+        items:
+          $ref: '#/definitions/RetentionRuleMetadata'
+      scope_selectors:
+        type: array
+        description: supported scope selectors
+        items:
+          $ref: '#/definitions/RetentionSelectorMetadata'
+      tag_selectors:
+        type: array
+        description: supported tag selectors
+        items:
+          $ref: '#/definitions/RetentionSelectorMetadata'
+
+  RetentionRuleMetadata:
+    type: object
+    description: the tag retention rule metadata
+    properties:
+      rule_template:
+        type: string
+        description: rule id
+      display_text:
+        type: string
+        description: rule display text
+      action:
+        type: string
+        description: rule action
+      params:
+        type: array
+        description: rule params
+        items:
+          $ref: '#/definitions/RetentionRuleParamMetadata'
+
+  RetentionRuleParamMetadata:
+    type: object
+    description: rule param
+    properties:
+      type:
+        type: string
+      unit:
+        type: string
+      required:
+        type: boolean
+
+
+  RetentionSelectorMetadata:
+    type: object
+    description: retention selector
+    properties:
+      display_text:
+        type: string
+      kind:
+        type: string
+      decorations:
+        type: array
+        items:
+          type: string
+
+  RetentionPolicy:
+    type: object
+    description: retention policy
+    properties:
+      id:
+        type: integer
+        format: int64
+      algorithm:
+        type: string
+      rules:
+        type: array
+        items:
+          $ref: '#/definitions/RetentionRule'
+      trigger:
+        type: object
+        items:
+          $ref: '#/definitions/RetentionRuleTrigger'
+      scope:
+        type: object
+        items:
+          $ref: '#/definitions/RetentionPolicyScope'
+
+  RetentionRuleTrigger:
+    type: object
+    properties:
+      kind:
+        type: string
+      settings:
+        type: object
+      references:
+        type: object
+
+  RetentionPolicyScope:
+    type: object
+    properties:
+      level:
+        type: string
+      ref:
+        type: integer
+
+  RetentionRule:
+    type: object
+    properties:
+      id:
+        type: integer
+      priority:
+        type: integer
+      disabled:
+        type: boolean
+      action:
+        type: string
+      template:
+        type: string
+      params:
+        type: object
+        additionalProperties:
+          type: object
+      tag_selectors:
+        type: array
+        items:
+          $ref: '#/definitions/RetentionSelector'
+      scope_selectors:
+        type: object
+        additionalProperties:
+          type: array
+          items:
+            $ref: '#/definitions/RetentionSelector'
+
+  RetentionSelector:
+    type: object
+    properties:
+      kind:
+        type: string
+      decoration:
+        type: string
+      pattern:
+        type: string
+
+  RetentionExecution:
+    type: object
+    properties:
+      id:
+        type: integer
+        format: int64
+      policy_id:
+        type: integer
+        format: int64
+      start_time:
+        type: string
+      end_time:
+        type: string
+      status:
+        type: string
+      trigger:
+        type: string
+      dry_run:
+        type: boolean
+
+  RetentionExecutionTask:
+    type: object
+    properties:
+      id:
+        type: integer
+        format: int64
+      execution_id:
+        type: integer
+        format: int64
+      repository:
+        type: string
+      job_id:
+        type: string
+      status:
+        type: string
+      status_code:
+        type: integer
+      status_revision:
+        type: integer
+        format: int64
+      start_time:
+        type: string
+      end_time:
+        type: string
+      total:
+        type: integer
+      retained:
+        type: integer
+

src/core/api/retention.go

@@ -151,6 +151,10 @@ func (r *RetentionAPI) CreateRetention() {
 		r.SendBadRequestError(err)
 		return
 	}
+	if len(p.Rules) > 15 {
+		r.SendBadRequestError(errors.New("only 15 rules are allowed at most"))
+		return
+	}
 	if err = r.checkRuleConflict(p); err != nil {
 		r.SendConflictError(err)
 		return
@@ -176,6 +180,15 @@ func (r *RetentionAPI) CreateRetention() {
 		r.SendBadRequestError(fmt.Errorf("scope %s is not support", p.Scope.Level))
 		return
 	}
+	old, err := r.pm.GetMetadataManager().Get(p.Scope.Reference, "retention_id")
+	if err != nil {
+		r.SendInternalServerError(err)
+		return
+	}
+	if old != nil && len(old) > 0 {
+		r.SendBadRequestError(fmt.Errorf("project %v already has retention policy %v", p.Scope.Reference, old["retention_id"]))
+		return
+	}
 	id, err := retentionController.CreateRetention(p)
 	if err != nil {
 		r.SendInternalServerError(err)
@@ -202,6 +215,10 @@ func (r *RetentionAPI) UpdateRetention() {
 		return
 	}
 	p.ID = id
+	if len(p.Rules) > 15 {
+		r.SendBadRequestError(errors.New("only 15 rules are allowed at most"))
+		return
+	}
 	if err = r.checkRuleConflict(p); err != nil {
 		r.SendConflictError(err)
 		return
@@ -218,14 +235,13 @@ func (r *RetentionAPI) UpdateRetention() {
 func (r *RetentionAPI) checkRuleConflict(p *policy.Metadata) error {
 	temp := make(map[string]int)
 	for n, rule := range p.Rules {
-		tid := rule.ID
 		rule.ID = 0
 		bs, _ := json.Marshal(rule)
 		if old, exists := temp[string(bs)]; exists {
 			return fmt.Errorf("rule %d is conflict with rule %d", n, old)
 		}
 		temp[string(bs)] = n
-		rule.ID = tid
+		rule.ID = n
 	}
 	return nil
 }
@@ -276,7 +292,7 @@ func (r *RetentionAPI) OperateRetentionExec() {
 		return
 	}
 	a := &struct {
-		Action string `json:"action" valid:"Required"`
+		Action string `json:"action" valid:"Required;Match(stop)"`
 	}{}
 	isValid, err := r.DecodeJSONReqAndValidate(a)
 	if !isValid {

src/pkg/retention/controller_test.go

@@ -1,6 +1,7 @@
 package retention
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/goharbor/harbor/src/pkg/retention/dep"
@@ -118,7 +119,8 @@ func (s *ControllerTestSuite) TestPolicy() {
 	s.Require().Nil(err)
 
 	p1, err = c.GetRetention(id)
-	s.Require().Nil(err)
+	s.Require().NotNil(err)
+	s.Require().True(strings.Contains(err.Error(), "no such Retention policy"))
 	s.Require().Nil(p1)
 }
 

src/pkg/retention/manager.go

@@ -111,7 +111,7 @@ func (d *DefaultManager) GetPolicy(id int64) (*policy.Metadata, error) {
 	p1, err := dao.GetPolicy(id)
 	if err != nil {
 		if err == orm.ErrNoRows {
-			return nil, nil
+			return nil, fmt.Errorf("no such Retention policy with id %v", id)
 		}
 		return nil, err
 	}

src/pkg/retention/manager_test.go

@@ -2,6 +2,7 @@ package retention
 
 import (
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -83,7 +84,8 @@ func TestPolicy(t *testing.T) {
 	assert.Nil(t, err)
 
 	p1, err = m.GetPolicy(id)
-	assert.Nil(t, err)
+	assert.NotNil(t, err)
+	assert.True(t, strings.Contains(err.Error(), "no such Retention policy"))
 	assert.Nil(t, p1)
 }
 

src/pkg/retention/models.go

@@ -37,7 +37,7 @@ type Execution struct {
 	StartTime time.Time `json:"start_time"`
 	EndTime   time.Time `json:"end_time,omitempty"`
 	Status    string    `json:"status"`
-	Trigger   string    `json:"Trigger"`
+	Trigger   string    `json:"trigger"`
 	DryRun    bool      `json:"dry_run"`
 }
 

src/pkg/retention/policy/models.go

@@ -53,27 +53,24 @@ type Metadata struct {
 
 	// Which scope the policy will be applied to
 	Scope *Scope `json:"scope" valid:"Required"`
-
-	// The max number of rules in a policy
-	Capacity int `json:"cap"`
 }
 
 // Valid Valid
 func (m *Metadata) Valid(v *validation.Validation) {
 	if m.Trigger == nil {
-		_ = v.SetError("Trigger", "Trigger is required")
+		_ = v.SetError("Trigger", "Can not be empty")
 		return
 	}
 	if m.Scope == nil {
-		_ = v.SetError("Scope", "Scope is required")
+		_ = v.SetError("Scope", "Can not be empty")
 		return
 	}
 	if m.Trigger != nil && m.Trigger.Kind == TriggerKindSchedule {
 		if m.Trigger.Settings == nil {
-			_ = v.SetError("Trigger.Settings", "Trigger.Settings is required")
+			_ = v.SetError("Trigger.Settings", "Can not be empty")
 		} else {
 			if _, ok := m.Trigger.Settings[TriggerSettingsCron]; !ok {
-				_ = v.SetError("Trigger.Settings", "cron in Trigger.Settings is required")
+				_ = v.SetError("Trigger.Settings.cron", "Can not be empty")
 			}
 		}
 	}

src/pkg/retention/policy/rule/models.go

@@ -37,7 +37,7 @@ type Metadata struct {
 	Template string `json:"template" valid:"Required"`
 
 	// The parameters of this rule
-	Parameters Parameters `json:"params"`
+	Parameters Parameters `json:"params" valid:"Required"`
 
 	// Selector attached to the rule for filtering tags
 	TagSelectors []*Selector `json:"tag_selectors" valid:"Required"`

src/portal/src/app/project/tag-retention/tag-retention.component.html

@@ -116,7 +116,7 @@
                     <clr-dg-cell class="hand"
                                  (click)="openDetail(i,execution.id)">{{execution.dry_run ? 'YES' : 'NO'}}</clr-dg-cell>
                     <clr-dg-cell class="hand"
-                                 (click)="openDetail(i,execution.id)">{{execution.Trigger}}</clr-dg-cell>
+                                 (click)="openDetail(i,execution.id)">{{execution.trigger}}</clr-dg-cell>
                     <clr-dg-cell class="hand"
                                  (click)="openDetail(i,execution.id)">{{execution.start_time|date:'medium'}}</clr-dg-cell>
                     <clr-dg-cell class="hand" (click)="openDetail(i,execution.id)">{{execution.duration}}</clr-dg-cell>

src/portal/src/app/project/tag-retention/tag-retention.component.ts

@@ -166,6 +166,13 @@ export class TagRetentionComponent implements OnInit {
         if (this.retentionId) {
             this.tagRetentionService.getRetention(this.retentionId).subscribe(
                 response => {
+                    if (response && response.rules && response.rules.length > 0) {
+                        response.rules.forEach(item => {
+                            if (!item.params) {
+                                item.params = {};
+                            }
+                        });
+                    }
                     this.retention = response;
                     this.loadingRule = false;
                 }, error => {

tests/apitests/python/library/base.py

@@ -39,7 +39,7 @@ def _random_name(prefix):
 def _get_id_from_header(header):
     try:
         location = header["Location"]
-        return location.split("/")[-1]
+        return int(location.split("/")[-1])
     except Exception:
         return None
 

tests/apitests/python/library/docker_api.py

@@ -12,6 +12,7 @@ except ImportError:
 class DockerAPI(object):
     def __init__(self):
         self.DCLIENT = docker.APIClient(base_url='unix://var/run/docker.sock',version='auto',timeout=10)
+        self.DCLIENT2 = docker.from_env()
 
     def docker_login(self, registry, username, password, expected_error_message = None):
         if expected_error_message is "":
@@ -84,4 +85,44 @@ class DockerAPI(object):
                     raise Exception(r" Failed to catch error [{}] when push image {}".format (expected_error_message, harbor_registry))
             else:
                 if str(ret).lower().find("errorDetail".lower()) >= 0:
-                    raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, ret))
+                    raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, ret))
+
+    def docker_image_build(self, harbor_registry, tags=None, size=1, expected_error_message = None):
+        caught_err = False
+        ret = ""
+        try:
+            baseimage='busybox:latest'
+            if not self.DCLIENT.images(name=baseimage):
+                self.DCLIENT.pull(baseimage)
+            c=self.DCLIENT.create_container(image='busybox:latest',command='dd if=/dev/urandom of=test bs=1M count=%d' % size )
+            self.DCLIENT.start(c)
+            self.DCLIENT.wait(c)
+            if not tags:
+                tags=['latest']
+            firstrepo="%s:%s" % (harbor_registry, tags[0])
+            #self.DCLIENT.commit(c, firstrepo)
+            self.DCLIENT2.containers.get(c).commit(harbor_registry, tags[0])
+            for tag in tags[1:]:
+                repo="%s:%s" % (harbor_registry, tag)
+                self.DCLIENT.tag(firstrepo, repo)
+            for tag in tags:
+                repo="%s:%s" % (harbor_registry, tag)
+                self.DCLIENT.push(repo)
+                print("build image %s with size %d" % (repo, size))
+                self.DCLIENT.remove_image(repo)
+            self.DCLIENT.remove_container(c)
+        except Exception, err:
+            caught_err = True
+            if expected_error_message is not None:
+                print "docker image build error:", str(err)
+                if str(err).lower().find(expected_error_message.lower()) < 0:
+                    raise Exception(r"Push image: Return message {} is not as expected {}".format(str(err), expected_error_message))
+            else:
+                raise Exception(r" Docker build image {} failed, error is [{}]".format (harbor_registry, err.message))
+        if caught_err == False:
+            if expected_error_message is not None:
+                if str(ret).lower().find(expected_error_message.lower()) < 0:
+                    raise Exception(r" Failed to catch error [{}] when push image {}".format (expected_error_message, harbor_registry))
+            else:
+                if str(ret).lower().find("errorDetail".lower()) >= 0:
+                    raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, ret))

tests/apitests/python/library/repository.py

@@ -30,6 +30,14 @@ def push_image_to_project(project_name, registry, username, password, image, tag
 
     return r'{}/{}'.format(project_name, image), new_tag
 
+def push_special_image_to_project(project_name, registry, username, password, image, tags=None, size=1, expected_login_error_message=None, expected_error_message = None):
+    _docker_api = DockerAPI()
+    _docker_api.docker_login(registry, username, password, expected_error_message = expected_login_error_message)
+    time.sleep(2)
+    if expected_login_error_message != None:
+        return
+    _docker_api.docker_image_build(r'{}/{}/{}'.format(registry, project_name, image), tags = tags, size=size, expected_error_message=expected_error_message)
+
 def is_repo_exist_in_project(repositories, repo_name):
     result = False
     for reop in repositories:

tests/apitests/python/library/retention.py

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+
+import base
+import swagger_client
+
+class Retention(base.Base):
+    def get_metadatas(self, expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+        metadatas, status_code, _ = client.retentions_metadatas_get_with_http_info()
+        base._assert_status_code(expect_status_code, status_code)
+        return metadatas
+
+    def create_retention_policy(self, project_id, selector_repository="**", selector_tag="**", expect_status_code = 201, **kwargs):
+        policy=swagger_client.RetentionPolicy(
+            algorithm='or',
+            rules=[
+                swagger_client.RetentionRule(
+                    disabled=False,
+                    action="retain",
+                    template="always",
+                    params= {
+
+                    },
+                    scope_selectors={
+                        "repository": [
+                            {
+                                "kind": "doublestar",
+                                "decoration": "repoMatches",
+                                "pattern": selector_repository
+                            }
+                        ]
+                    },
+                    tag_selectors=[
+                        {
+                            "kind": "doublestar",
+                            "decoration": "matches",
+                            "pattern": selector_tag
+                        }
+                    ]
+                )
+            ],
+            trigger= {
+                "kind": "Schedule",
+                "settings": {
+                    "cron": ""
+                },
+                "references": {
+                }
+            },
+            scope= {
+                "level": "project",
+                "ref": project_id
+            },
+        )
+        client = self._get_client(**kwargs)
+        _, status_code, header = client.retentions_post_with_http_info(policy)
+        base._assert_status_code(expect_status_code, status_code)
+        return base._get_id_from_header(header)
+
+    def get_retention_policy(self, retention_id, expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+        policy, status_code, _ = client.retentions_id_get_with_http_info(retention_id)
+        base._assert_status_code(expect_status_code, status_code)
+        return policy
+
+    def update_retention_policy(self, retention_id, selector_repository="**", selector_tag="**", expect_status_code = 200, **kwargs):
+        policy=swagger_client.RetentionPolicy(
+            id=retention_id,
+            algorithm='or',
+            rules=[
+                swagger_client.RetentionRule(
+                    disabled=False,
+                    action="retain",
+                    template="always",
+                    params= {
+
+                    },
+                    scope_selectors={
+                        "repository": [
+                            {
+                                "kind": "doublestar",
+                                "decoration": "repoMatches",
+                                "pattern": selector_repository
+                            }
+                        ]
+                    },
+                    tag_selectors=[
+                        {
+                            "kind": "doublestar",
+                            "decoration": "matches",
+                            "pattern": selector_tag
+                        }
+                    ]
+                )
+            ],
+            trigger= {
+                "kind": "Schedule",
+                "settings": {
+                    "cron": ""
+                },
+                "references": {
+                }
+            },
+            scope= {
+                "level": "project",
+                "ref": project_id
+            },
+        )
+        client = self._get_client(**kwargs)
+        _, status_code, _ = client.retentions_id_put_with_http_info(retention_id, policy)
+        base._assert_status_code(expect_status_code, status_code)
+
+    def update_retention_add_rule(self, retention_id, selector_repository="**", selector_tag="**", expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+        policy, status_code, _ = client.retentions_id_get_with_http_info(retention_id)
+        base._assert_status_code(200, status_code)
+        policy.rules.append(swagger_client.RetentionRule(
+                                                disabled=False,
+                                                action="retain",
+                                                template="always",
+                                                params= {
+
+                                                },
+                                                scope_selectors={
+                                                    "repository": [
+                                                        {
+                                                            "kind": "doublestar",
+                                                            "decoration": "repoMatches",
+                                                            "pattern": selector_repository
+                                                        }
+                                                    ]
+                                                },
+                                                tag_selectors=[
+                                                    {
+                                                        "kind": "doublestar",
+                                                        "decoration": "matches",
+                                                        "pattern": selector_tag
+                                                    }
+                                                ]
+                                            ))
+        _, status_code, _ = client.retentions_id_put_with_http_info(retention_id, policy)
+        base._assert_status_code(expect_status_code, status_code)
+
+    def trigger_retention_policy(self, retention_id, dry_run=False, expect_status_code = 201, **kwargs):
+        client = self._get_client(**kwargs)
+
+        _, status_code, _ = client.retentions_id_executions_post_with_http_info(retention_id, {"dry_run":dry_run})
+        base._assert_status_code(expect_status_code, status_code)
+
+    def stop_retention_execution(self, retention_id, exec_id, expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+
+        r, status_code, _ = client.retentions_id_executions_eid_patch(retention_id, exec_id, {"action":"stop"})
+        base._assert_status_code(expect_status_code, status_code)
+        return r
+
+    def get_retention_executions(self, retention_id, expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+
+        r, status_code, _ = client.retentions_id_executions_get_with_http_info(retention_id)
+        base._assert_status_code(expect_status_code, status_code)
+        return r
+
+    def get_retention_exec_tasks(self, retention_id, exec_id, expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+
+        r, status_code, _ = client.retentions_id_executions_eid_tasks_get_with_http_info(retention_id, exec_id)
+        base._assert_status_code(expect_status_code, status_code)
+        return r
+
+    def get_retention_exec_task_log(self, retention_id, exec_id, task_id, expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+
+        r, status_code, _ = client.retentions_id_executions_eid_tasks_tid_get_with_http_info(retention_id, exec_id, task_id)
+        base._assert_status_code(expect_status_code, status_code)
+        return r
+
+    def get_retention_metadatas(self, expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+
+        r, status_code, _ = client.retentions_metadatas_get_with_http_info()
+        base._assert_status_code(expect_status_code, status_code)
+        return r

tests/apitests/python/test_retention.py

@@ -0,0 +1,110 @@
+from __future__ import absolute_import
+
+import unittest
+import time
+
+from testutils import ADMIN_CLIENT
+from testutils import TEARDOWN
+from testutils import harbor_server
+from library.repository import push_special_image_to_project
+
+from library.retention import Retention
+from library.project import Project
+from library.repository import Repository
+from library.user import User
+from library.system import System
+
+
+class TestProjects(unittest.TestCase):
+    """
+    Test case:
+        Tag Retention
+    Setup:
+        Create Project test-retention
+        Push image test1:1.0, test1:2.0, test1:3.0,latest, test2:1.0, test2:latest, test3:1.0, test4:1.0
+    Test Steps:
+        1. Create Retention Policy
+        2. Add rule "For the repositories matching **, retain always with tags matching latest*"
+        3. Add rule "For the repositories matching test3*, retain always with tags matching **"
+        4. Dry run, check execution and tasks
+        5. Real run, check images retained
+    Tear Down:
+        1. Delete project test-retention
+    """
+    @classmethod
+    def setUpClass(self):
+        self.user = User()
+        self.system = System()
+        self.repo= Repository()
+        self.project = Project()
+        self.retention=Retention()
+
+    def testTagRetention(self):
+        user_ra_password = "Aa123456"
+        user_ra_id, user_ra_name = self.user.create_user(user_password=user_ra_password, **ADMIN_CLIENT)
+        print("Created user: %s, id: %s" % (user_ra_name, user_ra_id))
+        TestProjects.USER_RA_CLIENT = dict(endpoint=ADMIN_CLIENT["endpoint"],
+                                   username=user_ra_name,
+                                   password=user_ra_password)
+        TestProjects.user_ra_id = int(user_ra_id)
+
+        TestProjects.project_src_repo_id, project_src_repo_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_RA_CLIENT)
+
+        # Push image test1:1.0, test1:2.0, test1:3.0,latest, test2:1.0, test2:latest, test3:1.0
+        push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['1.0'])
+        push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['2.0'])
+        push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['3.0','latest'])
+        push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['1.0'])
+        push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['latest'])
+        push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test3", ['1.0'])
+        push_special_image_to_project(project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test4", ['1.0'])
+
+        resp=self.repo.get_repository(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT)
+        self.assertEqual(len(resp), 4)
+
+        # Create Retention Policy
+        retention_id = self.retention.create_retention_policy(TestProjects.project_src_repo_id, selector_repository="**", selector_tag="latest*", expect_status_code = 201, **TestProjects.USER_RA_CLIENT)
+
+        # Add rule
+        self.retention.update_retention_add_rule(retention_id,selector_repository="test3*", selector_tag="**", expect_status_code = 200, **TestProjects.USER_RA_CLIENT)
+
+        # Dry run
+        self.retention.trigger_retention_policy(retention_id, dry_run=True, **TestProjects.USER_RA_CLIENT)
+        time.sleep(2)
+        resp=self.retention.get_retention_executions(retention_id, **TestProjects.USER_RA_CLIENT)
+        self.assertTrue(len(resp)>0)
+        execution=resp[0]
+        resp=self.retention.get_retention_exec_tasks(retention_id,execution.id, **TestProjects.USER_RA_CLIENT)
+        self.assertEqual(len(resp), 4)
+        resp=self.retention.get_retention_exec_task_log(retention_id,execution.id,resp[0].id, **TestProjects.USER_RA_CLIENT)
+        print(resp)
+
+        # Real run
+        self.retention.trigger_retention_policy(retention_id, dry_run=False, **TestProjects.USER_RA_CLIENT)
+        time.sleep(10)
+        resp=self.retention.get_retention_executions(retention_id, **TestProjects.USER_RA_CLIENT)
+        self.assertTrue(len(resp)>1)
+        execution=resp[0]
+        resp=self.retention.get_retention_exec_tasks(retention_id,execution.id, **TestProjects.USER_RA_CLIENT)
+        self.assertEqual(len(resp), 4)
+        resp=self.retention.get_retention_exec_task_log(retention_id,execution.id,resp[0].id, **TestProjects.USER_RA_CLIENT)
+        print(resp)
+        resp=self.repo.get_repository(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT)
+        self.assertEqual(len(resp), 3)
+
+
+    @classmethod
+    def tearDownClass(self):
+        print "Case completed"
+
+    @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
+    def test_ClearData(self):
+        resp=self.repo.get_repository(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT)
+        for repo in resp:
+            self.repo.delete_repoitory(repo.name, **TestProjects.USER_RA_CLIENT)
+        self.project.delete_project(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT)
+        self.user.delete_user(TestProjects.user_ra_id, **ADMIN_CLIENT)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/apitests/python/testutils.py

@@ -2,7 +2,7 @@ import time
 import os
 import sys
 
-sys.path.append(os.environ["SWAGGER_CLIENT_PATH"])
+sys.path.insert(0, os.environ["SWAGGER_CLIENT_PATH"])
 from swagger_client.rest import ApiException
 import swagger_client.models
 from pprint import pprint
@@ -12,7 +12,7 @@ admin_pwd = "Harbor12345"
 
 harbor_server = os.environ["HARBOR_HOST"]
 #CLIENT=dict(endpoint="https://"+harbor_server+"/api")
-ADMIN_CLIENT=dict(endpoint = "https://"+harbor_server+"/api", username = admin_user, password =  admin_pwd)
+ADMIN_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api", username = admin_user, password =  admin_pwd)
 USER_ROLE=dict(admin=0,normal=1)
 TEARDOWN = True
 

tests/robot-cases/Group0-BAT/API_DB.robot

@@ -53,3 +53,6 @@ Test Case - System Level CVE Whitelist
     Harbor API Test  ./tests/apitests/python/test_sys_cve_whitelists.py
 Test Case - Project Level CVE Whitelist
     Harbor API Test  ./tests/apitests/python/test_project_level_cve_whitelist.py
+Test Case - Tag Retention
+    Harbor API Test  ./tests/apitests/python/test_retention.py
+