add index for rule,selector and performer

Signed-off-by: Steven Zou <szou@vmware.com>

src/pkg/retention/policy/action/index.go

@@ -33,7 +33,7 @@ func Register(action string, factory PerformerFactory) {
 }
 
 // Get performer with the provided action
-func Get(action string) (Performer, error) {
+func Get(action string, params interface{}) (Performer, error) {
 	if len(action) == 0 {
 		return nil, errors.New("empty action")
 	}
@@ -48,5 +48,5 @@ func Get(action string) (Performer, error) {
 		return nil, errors.Errorf("invalid action performer registered for action %s", action)
 	}
 
-	return factory(), nil
+	return factory(params), nil
 }

src/pkg/retention/policy/action/performer.go

@@ -16,6 +16,11 @@ package action
 
 import "github.com/goharbor/harbor/src/pkg/retention/res"
 
+const (
+	// Retain artifacts
+	Retain = "retain"
+)
+
 // Performer performs the related actions targeting the candidates
 type Performer interface {
 	// Perform the action
@@ -30,10 +35,12 @@ type Performer interface {
 }
 
 // PerformerFactory is factory method for creating Performer
-type PerformerFactory func() Performer
+type PerformerFactory func(params interface{}) Performer
 
 // retainAction make sure all the candidates will be retained and others will be cleared
-type retainAction struct{}
+type retainAction struct {
+	all []*res.Candidate
+}
 
 // Perform the action
 func (ra *retainAction) Perform(candidates []*res.Candidate) ([]*res.Result, error) {
@@ -41,6 +48,21 @@ func (ra *retainAction) Perform(candidates []*res.Candidate) ([]*res.Result, err
 }
 
 // NewRetainAction is factory method for RetainAction
-func NewRetainAction() Performer {
-	return &retainAction{}
+func NewRetainAction(params interface{}) Performer {
+	if params != nil {
+		if all, ok := params.([]*res.Candidate); ok {
+			return &retainAction{
+				all: all,
+			}
+		}
+	}
+
+	return &retainAction{
+		all: make([]*res.Candidate, 0),
+	}
+}
+
+func init() {
+	// Register itself
+	Register(Retain, NewRetainAction)
 }

src/pkg/retention/policy/alg/or/processor.go

@@ -28,8 +28,8 @@ import (
 type processor struct {
 	performer action.Performer
 	// keep evaluator and its related selector if existing
-	// attentions here, the selector can be nil, that means match all "**"
-	evaluators map[*rule.Evaluator]res.Selector
+	// attentions here, the selectors can be empty/nil, that means match all "**"
+	evaluators map[*rule.Evaluator][]res.Selector
 }
 
 // New processor
@@ -85,10 +85,10 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) {
 	wg := new(sync.WaitGroup)
 	wg.Add(len(p.evaluators))
 
-	for eva, selector := range p.evaluators {
-		var evaluator rule.Evaluator = *eva
+	for eva, selectors := range p.evaluators {
+		var evaluator = *eva
 
-		go func(evaluator rule.Evaluator, selector res.Selector) {
+		go func(evaluator rule.Evaluator, selectors []res.Selector) {
 			var (
 				processed []*res.Candidate
 				err       error
@@ -98,10 +98,18 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) {
 				wg.Done()
 			}()
 
-			if selector != nil {
-				if processed, err = selector.Select(artifacts); err != nil {
-					errChan <- err
-					return
+			// init
+			// pass array copy to the selector
+			processed = append(processed, artifacts...)
+
+			if len(selectors) > 0 {
+				// selecting artifacts one by one
+				// `&&` mappings
+				for _, s := range selectors {
+					if processed, err = s.Select(processed); err != nil {
+						errChan <- err
+						return
+					}
 				}
 			}
 
@@ -114,7 +122,7 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) {
 				// Pass to the outside
 				resChan <- processed
 			}
-		}(evaluator, selector)
+		}(evaluator, selectors)
 	}
 
 	// waiting for all the rules are evaluated
@@ -128,9 +136,9 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) {
 }
 
 // AddEvaluator appends a rule evaluator for processing
-func (p *processor) AddEvaluator(evaluator rule.Evaluator, selector res.Selector) {
+func (p *processor) AddEvaluator(evaluator rule.Evaluator, selectors []res.Selector) {
 	if evaluator != nil {
-		p.evaluators[&evaluator] = selector
+		p.evaluators[&evaluator] = selectors
 	}
 }
 

src/pkg/retention/policy/alg/processor.go

@@ -37,9 +37,9 @@ type Processor interface {
 	// Add a rule evaluator for the processor
 	//
 	//  Arguments:
-	//    evaluator rule.Evaluator : a rule evaluator
-	//    selector res.Selector    : selector to narrow down the scope, optional
-	AddEvaluator(evaluator rule.Evaluator, selector res.Selector)
+	//    evaluator rule.Evaluator    : a rule evaluator
+	//    selectors []res.Selector    : selectors to narrow down the scope (&& adopted), optional
+	AddEvaluator(evaluator rule.Evaluator, selectors []res.Selector)
 
 	// Set performer for the processor
 	//

src/pkg/retention/policy/rule/lastx/evaluator.go

@@ -14,15 +14,64 @@
 
 package lastx
 
-import "github.com/goharbor/harbor/src/pkg/retention/res"
+import (
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/retention/policy/action"
+	"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+)
 
-// Evaluator for evaluating last x days
-type Evaluator struct {
+const (
+	// TemplateID of last x days rule
+	TemplateID = "lastXDays"
+	// ParameterX ...
+	ParameterX = TemplateID
+	// DefaultX defines the default X
+	DefaultX = 10
+)
+
+// evaluator for evaluating last x days
+type evaluator struct {
 	// last x days
 	x int
 }
 
 // Process the candidates based on the rule definition
-func (e *Evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
+func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
 	return nil, nil
 }
+
+// New a Evaluator
+func New(params rule.Parameters) rule.Evaluator {
+	if params != nil {
+		if param, ok := params[ParameterX]; ok {
+			if v, ok := param.(int); ok {
+				return &evaluator{
+					x: v,
+				}
+			}
+		}
+	}
+
+	log.Debugf("default parameter %d used for rule %s", DefaultX, TemplateID)
+
+	return &evaluator{
+		x: DefaultX,
+	}
+}
+
+func init() {
+	// Register itself
+	rule.Register(&rule.IndexMeta{
+		TemplateID: TemplateID,
+		Action:     action.Retain,
+		Parameters: []*rule.IndexedParam{
+			{
+				Name:     ParameterX,
+				Type:     "int",
+				Unit:     "days",
+				Required: true,
+			},
+		},
+	}, New)
+}

src/pkg/retention/policy/rule/latestk/evaluator.go

@@ -14,15 +14,64 @@
 
 package latestk
 
-import "github.com/goharbor/harbor/src/pkg/retention/res"
+import (
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/retention/policy/action"
+	"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+)
 
-// Evaluator for evaluating latest x tags
-type Evaluator struct {
-	// latest x
+const (
+	// TemplateID of latest k rule
+	TemplateID = "latestK"
+	// ParameterK ...
+	ParameterK = TemplateID
+	// DefaultK defines the default K
+	DefaultK = 10
+)
+
+// evaluator for evaluating latest k tags
+type evaluator struct {
+	// latest k
 	k int
 }
 
 // Process the candidates based on the rule definition
-func (e *Evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
+func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
 	return nil, nil
 }
+
+// New a Evaluator
+func New(params rule.Parameters) rule.Evaluator {
+	if params != nil {
+		if param, ok := params[ParameterK]; ok {
+			if v, ok := param.(int); ok {
+				return &evaluator{
+					k: v,
+				}
+			}
+		}
+	}
+
+	log.Debugf("default parameter %d used for rule %s", DefaultK, TemplateID)
+
+	return &evaluator{
+		k: DefaultK,
+	}
+}
+
+func init() {
+	// Register itself
+	rule.Register(&rule.IndexMeta{
+		TemplateID: TemplateID,
+		Action:     action.Retain,
+		Parameters: []*rule.IndexedParam{
+			{
+				Name:     ParameterK,
+				Type:     "int",
+				Unit:     "count",
+				Required: true,
+			},
+		},
+	}, New)
+}

src/pkg/retention/policy/rule/models.go

@@ -33,7 +33,7 @@ type Metadata struct {
 	Parameters Parameters `json:"params"`
 
 	// Selector attached to the rule for filtering tags
-	TagSelector *Selector `json:"tag_selector"`
+	TagSelectors []*Selector `json:"tag_selectors"`
 
 	// Selector attached to the rule for filtering scope (e.g: repositories or namespaces)
 	ScopeSelectors []*Selector `json:"scope_selectors"`
@@ -42,17 +42,16 @@ type Metadata struct {
 // Selector to narrow down the list
 type Selector struct {
 	// Kind of the selector
-	// "regularExpression", "label" or "list"
+	// "regularExpression" or "label"
 	Kind string `json:"kind"`
 
 	// Decorated the selector
 	// for "regularExpression" : "matches" and "excludes"
 	// for "label" : "with" and "without"
-	// for "list"  : "in" and "not in"
-	decoration string `json:"decoration"`
+	Decoration string `json:"decoration"`
 
 	// Param for the selector
-	Value Parameter `json:"param"`
+	Pattern string `json:"pattern"`
 }
 
 // Parameters of rule, indexed by the key

src/pkg/retention/res/list/selector.go

@@ -1,50 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package list
-
-import "github.com/goharbor/harbor/src/pkg/retention/res"
-
-const (
-	// InRepos for in [repositories]
-	InRepos = "in repositories"
-	// NotInRepos for not in [repositories]
-	NotInRepos = "not in repositories"
-	// InTags for in [tags]
-	InTags = "in tags"
-	// NotInTags for not in [tags]
-	NotInTags = "not in tags"
-)
-
-// selector for regular expression
-type selector struct {
-	// Pre defined pattern declarator
-	// "InRepo", "NotInRepo", "InTag" and "NotInTags"
-	decoration string
-	// The item list
-	values []string
-}
-
-// Select candidates by regular expressions
-func (s *selector) Select(artifacts []*res.Candidate) ([]*res.Candidate, error) {
-	return nil, nil
-}
-
-// New is factory method for list selector
-func New(decoration string, pattern interface{}) res.Selector {
-	return &selector{
-		decoration: decoration,
-		values:     pattern.([]string),
-	}
-}

src/pkg/retention/res/selector.go

@@ -27,4 +27,4 @@ type Selector interface {
 }
 
 // SelectorFactory is factory method to return a selector implementation
-type SelectorFactory func(decoration string, pattern interface{}) Selector
+type SelectorFactory func(decoration string, pattern string) Selector

src/pkg/retention/res/selector/index.go

@@ -1,55 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package selector
-
-import (
-	"fmt"
-	"github.com/goharbor/harbor/src/pkg/retention/res"
-	"github.com/pkg/errors"
-	"sync"
-)
-
-// index for keeping the mapping between selector meta and its implementation
-var index sync.Map
-
-// Register the selector with the corresponding selector kind and decoration
-func Register(kind, decoration string, factory res.SelectorFactory) {
-	id := fmt.Sprintf("%s:%s", kind, decoration)
-	if len(id) == 0 || factory == nil {
-		// do nothing
-		return
-	}
-
-	index.Store(id, factory)
-}
-
-// Get selector with the provided kind and decoration
-func Get(kind, decoration string, pattern interface{}) (res.Selector, error) {
-	if len(templateID) == 0 {
-		return nil, errors.New("empty rule template ID")
-	}
-
-	v, ok := index.Load(templateID)
-	if !ok {
-		return nil, errors.Errorf("rule evaluator %s is not registered", templateID)
-	}
-
-	factory, ok := v.(RuleFactory)
-	if !ok {
-		return nil, errors.Errorf("invalid rule evaluator registered for %s", templateID)
-	}
-
-	return factory(parameters), nil
-}

src/pkg/retention/res/selectors/index.go

@@ -0,0 +1,90 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package selectors
+
+import (
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/pkg/errors"
+	"sync"
+)
+
+// index for keeping the mapping between selector meta and its implementation
+var index sync.Map
+
+// IndexedMeta describes the indexed selector
+type IndexedMeta struct {
+	Kind        string   `json:"kind"`
+	Decorations []string `json:"decorations"`
+}
+
+// indexedItem defined item kept in the index
+type indexedItem struct {
+	Meta    *IndexedMeta
+	Factory res.SelectorFactory
+}
+
+// Register the selector with the corresponding selector kind and decoration
+func Register(kind string, decorations []string, factory res.SelectorFactory) {
+	if len(kind) == 0 || factory == nil {
+		// do nothing
+		return
+	}
+
+	index.Store(kind, &indexedItem{
+		Meta: &IndexedMeta{
+			Kind:        kind,
+			Decorations: decorations,
+		},
+		Factory: factory,
+	})
+}
+
+// Get selector with the provided kind and decoration
+func Get(kind, decoration, pattern string) (res.Selector, error) {
+	if len(kind) == 0 || len(decoration) == 0 {
+		return nil, errors.New("empty selector kind or decoration")
+	}
+
+	v, ok := index.Load(kind)
+	if !ok {
+		return nil, errors.Errorf("selector %s is not registered", kind)
+	}
+
+	item := v.(*indexedItem)
+	for _, dec := range item.Meta.Decorations {
+		if dec == decoration {
+			factory := item.Factory
+			return factory(decoration, pattern), nil
+		}
+	}
+
+	return nil, errors.Errorf("decoration %s of selector %s is not supported", decoration, kind)
+}
+
+// Index returns all the declarative selectors
+func Index() []*IndexedMeta {
+	all := make([]*IndexedMeta, 0)
+
+	index.Range(func(k, v interface{}) bool {
+		if item, ok := v.(*indexedItem); ok {
+			all = append(all, item.Meta)
+			return true
+		}
+
+		return false
+	})
+
+	return all
+}

src/pkg/retention/res/selectors/label/selector.go

@@ -14,11 +14,19 @@
 
 package label
 
-import "github.com/goharbor/harbor/src/pkg/retention/res"
+import (
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
+	"strings"
+)
 
 const (
-	With    = "with labels"
-	Without = "without labels"
+	// Kind ...
+	Kind = "label"
+	// With labels
+	With = "withLabels"
+	// Without labels
+	Without = "withoutLabels"
 )
 
 // selector is for label selector
@@ -36,9 +44,16 @@ func (s *selector) Select(artifacts []*res.Candidate) ([]*res.Candidate, error)
 }
 
 // New is factory method for list selector
-func New(decoration string, pattern interface{}) res.Selector {
+func New(decoration string, pattern string) res.Selector {
+	labels := strings.Split(pattern, ",")
+
 	return &selector{
 		decoration: decoration,
-		labels:     pattern.([]string),
+		labels:     labels,
 	}
 }
+
+func init() {
+	// Register regexp selector
+	selectors.Register(Kind, []string{With, Without}, New)
+}

src/pkg/retention/res/selectors/regexp/selector.go

@@ -14,9 +14,14 @@
 
 package regexp
 
-import "github.com/goharbor/harbor/src/pkg/retention/res"
+import (
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
+)
 
 const (
+	// Kind ...
+	Kind = "regularExpression"
 	// Matches [pattern]
 	Matches = "matches"
 	// Excludes [pattern]
@@ -38,9 +43,14 @@ func (s *selector) Select(artifacts []*res.Candidate) ([]*res.Candidate, error)
 }
 
 // New is factory method for regexp selector
-func New(decoration string, pattern interface{}) res.Selector {
+func New(decoration string, pattern string) res.Selector {
 	return &selector{
 		decoration: decoration,
-		pattern:    pattern.(string),
+		pattern:    pattern,
 	}
 }
+
+func init() {
+	// Register regexp selector
+	selectors.Register(Kind, []string{Matches, Excludes}, New)
+}