mirror of
https://github.com/goharbor/harbor.git
synced 2024-10-06 01:07:55 +02:00
Merge pull request #8445 from steven-zou/fix/tag_retention
refactor index registering processes
This commit is contained in:
commit
4bf7f7b3e4
@ -3,14 +3,15 @@ package api
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common/rbac"
|
"github.com/goharbor/harbor/src/common/rbac"
|
||||||
"github.com/goharbor/harbor/src/core/filter"
|
"github.com/goharbor/harbor/src/core/filter"
|
||||||
"github.com/goharbor/harbor/src/core/promgr"
|
"github.com/goharbor/harbor/src/core/promgr"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention"
|
"github.com/goharbor/harbor/src/pkg/retention"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy"
|
"github.com/goharbor/harbor/src/pkg/retention/policy"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/q"
|
"github.com/goharbor/harbor/src/pkg/retention/q"
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// RetentionAPI ...
|
// RetentionAPI ...
|
||||||
@ -65,7 +66,7 @@ func (r *RetentionAPI) GetMetadatas() {
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"rule_template": "latestK",
|
"rule_template": "latestPushedK",
|
||||||
"display_text": "the most recently pushed # images",
|
"display_text": "the most recently pushed # images",
|
||||||
"action": "retain",
|
"action": "retain",
|
||||||
"params": [
|
"params": [
|
||||||
@ -77,7 +78,7 @@ func (r *RetentionAPI) GetMetadatas() {
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"rule_template": "latestPulledK",
|
"rule_template": "latestPulledN",
|
||||||
"display_text": "the most recently pulled # images",
|
"display_text": "the most recently pulled # images",
|
||||||
"action": "retain",
|
"action": "retain",
|
||||||
"params": [
|
"params": [
|
||||||
|
@ -147,7 +147,7 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if needUn {
|
if needUn {
|
||||||
err = r.scheduler.UnSchedule((p0.Trigger.References[policy.TriggerReferencesJobid].(int64)))
|
err = r.scheduler.UnSchedule(p0.Trigger.References[policy.TriggerReferencesJobid].(int64))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -72,7 +72,7 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error {
|
|||||||
|
|
||||||
// Log stage: start
|
// Log stage: start
|
||||||
repoPath := fmt.Sprintf("%s/%s", repo.Namespace, repo.Name)
|
repoPath := fmt.Sprintf("%s/%s", repo.Namespace, repo.Name)
|
||||||
myLogger.Infof("Run retention process.\n Repository: %s \n Rule Algorithm: %s \n Dry Run: %f", repoPath, liteMeta.Algorithm, isDryRun)
|
myLogger.Infof("Run retention process.\n Repository: %s \n Rule Algorithm: %s \n Dry Run: %v", repoPath, liteMeta.Algorithm, isDryRun)
|
||||||
|
|
||||||
// Stop check point 1:
|
// Stop check point 1:
|
||||||
if isStopped(ctx) {
|
if isStopped(ctx) {
|
||||||
@ -125,13 +125,13 @@ func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Re
|
|||||||
op := func(art *res.Candidate) string {
|
op := func(art *res.Candidate) string {
|
||||||
if e, exists := hash[art.Hash()]; exists {
|
if e, exists := hash[art.Hash()]; exists {
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return "Err"
|
return "ERR"
|
||||||
}
|
}
|
||||||
|
|
||||||
return "X"
|
return "DEL"
|
||||||
}
|
}
|
||||||
|
|
||||||
return "√"
|
return "RETAIN"
|
||||||
}
|
}
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
@ -158,7 +158,7 @@ func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Re
|
|||||||
table.AppendBulk(data)
|
table.AppendBulk(data)
|
||||||
table.Render()
|
table.Render()
|
||||||
|
|
||||||
logger.Infof("%s", buf.String())
|
logger.Infof("\n%s", buf.String())
|
||||||
|
|
||||||
// log all the concrete errors if have
|
// log all the concrete errors if have
|
||||||
for _, r := range results {
|
for _, r := range results {
|
||||||
@ -214,15 +214,14 @@ func getParamRepo(params job.Parameters) (*res.Repository, error) {
|
|||||||
return nil, errors.Errorf("missing parameter: %s", ParamRepo)
|
return nil, errors.Errorf("missing parameter: %s", ParamRepo)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("%T", v)
|
repoJSON, ok := v.(string)
|
||||||
repoMap, ok := v.(map[string]interface{})
|
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.Errorf("invalid parameter: %s", ParamRepo)
|
return nil, errors.Errorf("invalid parameter: %s", ParamRepo)
|
||||||
}
|
}
|
||||||
|
|
||||||
repo := &res.Repository{}
|
repo := &res.Repository{}
|
||||||
if err := repo.FromMap(repoMap); err != nil {
|
if err := repo.FromJSON(repoJSON); err != nil {
|
||||||
return nil, fmt.Errorf("failed to convert map to repository: %v", err)
|
return nil, errors.Wrap(err, "parse repository from JSON")
|
||||||
}
|
}
|
||||||
|
|
||||||
return repo, nil
|
return repo, nil
|
||||||
@ -234,14 +233,14 @@ func getParamMeta(params job.Parameters) (*lwp.Metadata, error) {
|
|||||||
return nil, errors.Errorf("missing parameter: %s", ParamMeta)
|
return nil, errors.Errorf("missing parameter: %s", ParamMeta)
|
||||||
}
|
}
|
||||||
|
|
||||||
metaMap, ok := v.(map[string]interface{})
|
metaJSON, ok := v.(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.Errorf("invalid parameter: %s", ParamMeta)
|
return nil, errors.Errorf("invalid parameter: %s", ParamMeta)
|
||||||
}
|
}
|
||||||
|
|
||||||
meta := &lwp.Metadata{}
|
meta := &lwp.Metadata{}
|
||||||
if err := meta.FromMap(metaMap); err != nil {
|
if err := meta.FromJSON(metaJSON); err != nil {
|
||||||
return nil, fmt.Errorf("failed to convert map to metadata: %v", err)
|
return nil, errors.Wrap(err, "parse retention policy from JSON")
|
||||||
}
|
}
|
||||||
|
|
||||||
return meta, nil
|
return meta, nil
|
||||||
|
@ -16,7 +16,6 @@ package retention
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
@ -27,13 +26,10 @@ import (
|
|||||||
"github.com/goharbor/harbor/src/pkg/retention/dep"
|
"github.com/goharbor/harbor/src/pkg/retention/dep"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy"
|
"github.com/goharbor/harbor/src/pkg/retention/policy"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/alg/or"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@ -54,18 +50,6 @@ func TestJob(t *testing.T) {
|
|||||||
|
|
||||||
// SetupSuite ...
|
// SetupSuite ...
|
||||||
func (suite *JobTestSuite) SetupSuite() {
|
func (suite *JobTestSuite) SetupSuite() {
|
||||||
alg.Register(alg.AlgorithmOR, or.New)
|
|
||||||
selectors.Register(doublestar.Kind, []string{
|
|
||||||
doublestar.Matches,
|
|
||||||
doublestar.Excludes,
|
|
||||||
doublestar.RepoMatches,
|
|
||||||
doublestar.RepoExcludes,
|
|
||||||
doublestar.NSMatches,
|
|
||||||
doublestar.NSExcludes,
|
|
||||||
}, doublestar.New)
|
|
||||||
selectors.Register(label.Kind, []string{label.With, label.Without}, label.New)
|
|
||||||
action.Register(action.Retain, action.NewRetainAction)
|
|
||||||
|
|
||||||
suite.oldClient = dep.DefaultClient
|
suite.oldClient = dep.DefaultClient
|
||||||
dep.DefaultClient = &fakeRetentionClient{}
|
dep.DefaultClient = &fakeRetentionClient{}
|
||||||
}
|
}
|
||||||
@ -83,9 +67,9 @@ func (suite *JobTestSuite) TestRunSuccess() {
|
|||||||
Name: "harbor",
|
Name: "harbor",
|
||||||
Kind: res.Image,
|
Kind: res.Image,
|
||||||
}
|
}
|
||||||
repoMap, err := toMap(repository)
|
repoJSON, err := repository.ToJSON()
|
||||||
require.Nil(suite.T(), err)
|
require.Nil(suite.T(), err)
|
||||||
params[ParamRepo] = repoMap
|
params[ParamRepo] = repoJSON
|
||||||
|
|
||||||
scopeSelectors := make(map[string][]*rule.Selector)
|
scopeSelectors := make(map[string][]*rule.Selector)
|
||||||
scopeSelectors["project"] = []*rule.Selector{{
|
scopeSelectors["project"] = []*rule.Selector{{
|
||||||
@ -95,7 +79,7 @@ func (suite *JobTestSuite) TestRunSuccess() {
|
|||||||
}}
|
}}
|
||||||
|
|
||||||
ruleParams := make(rule.Parameters)
|
ruleParams := make(rule.Parameters)
|
||||||
ruleParams[latestk.ParameterK] = 10
|
ruleParams[latestps.ParameterK] = 10
|
||||||
|
|
||||||
meta := &lwp.Metadata{
|
meta := &lwp.Metadata{
|
||||||
Algorithm: policy.AlgorithmOR,
|
Algorithm: policy.AlgorithmOR,
|
||||||
@ -104,7 +88,7 @@ func (suite *JobTestSuite) TestRunSuccess() {
|
|||||||
ID: 1,
|
ID: 1,
|
||||||
Priority: 999,
|
Priority: 999,
|
||||||
Action: action.Retain,
|
Action: action.Retain,
|
||||||
Template: latestk.TemplateID,
|
Template: latestps.TemplateID,
|
||||||
Parameters: ruleParams,
|
Parameters: ruleParams,
|
||||||
TagSelectors: []*rule.Selector{{
|
TagSelectors: []*rule.Selector{{
|
||||||
Kind: label.Kind,
|
Kind: label.Kind,
|
||||||
@ -119,9 +103,9 @@ func (suite *JobTestSuite) TestRunSuccess() {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
metaMap, err := toMap(meta)
|
metaJSON, err := meta.ToJSON()
|
||||||
require.Nil(suite.T(), err)
|
require.Nil(suite.T(), err)
|
||||||
params[ParamMeta] = metaMap
|
params[ParamMeta] = metaJSON
|
||||||
|
|
||||||
j := &Job{}
|
j := &Job{}
|
||||||
err = j.Validate(params)
|
err = j.Validate(params)
|
||||||
@ -239,15 +223,3 @@ func (c *fakeJobContext) GetLogger() logger.Interface {
|
|||||||
func (c *fakeJobContext) Tracker() job.Tracker {
|
func (c *fakeJobContext) Tracker() job.Tracker {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func toMap(v interface{}) (map[string]interface{}, error) {
|
|
||||||
data, err := json.Marshal(v)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
m := map[string]interface{}{}
|
|
||||||
if err = json.Unmarshal(data, &m); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
@ -18,6 +18,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/index"
|
||||||
|
|
||||||
cjob "github.com/goharbor/harbor/src/common/job"
|
cjob "github.com/goharbor/harbor/src/common/job"
|
||||||
"github.com/goharbor/harbor/src/common/job/models"
|
"github.com/goharbor/harbor/src/common/job/models"
|
||||||
cmodels "github.com/goharbor/harbor/src/common/models"
|
cmodels "github.com/goharbor/harbor/src/common/models"
|
||||||
@ -31,7 +33,6 @@ import (
|
|||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/q"
|
"github.com/goharbor/harbor/src/pkg/retention/q"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -126,7 +127,7 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool
|
|||||||
case "system":
|
case "system":
|
||||||
// filter projects according to the project selectors
|
// filter projects according to the project selectors
|
||||||
for _, projectSelector := range rule.ScopeSelectors["project"] {
|
for _, projectSelector := range rule.ScopeSelectors["project"] {
|
||||||
selector, err := selectors.Get(projectSelector.Kind, projectSelector.Decoration,
|
selector, err := index.Get(projectSelector.Kind, projectSelector.Decoration,
|
||||||
projectSelector.Pattern)
|
projectSelector.Pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, launcherError(err)
|
return 0, launcherError(err)
|
||||||
@ -153,7 +154,7 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool
|
|||||||
}
|
}
|
||||||
// filter repositories according to the repository selectors
|
// filter repositories according to the repository selectors
|
||||||
for _, repositorySelector := range rule.ScopeSelectors["repository"] {
|
for _, repositorySelector := range rule.ScopeSelectors["repository"] {
|
||||||
selector, err := selectors.Get(repositorySelector.Kind, repositorySelector.Decoration,
|
selector, err := index.Get(repositorySelector.Kind, repositorySelector.Decoration,
|
||||||
repositorySelector.Pattern)
|
repositorySelector.Pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, launcherError(err)
|
return 0, launcherError(err)
|
||||||
@ -205,22 +206,38 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool
|
|||||||
allFailed := true
|
allFailed := true
|
||||||
for _, jobData := range jobDatas {
|
for _, jobData := range jobDatas {
|
||||||
j := &models.JobData{
|
j := &models.JobData{
|
||||||
|
Name: job.Retention,
|
||||||
Metadata: &models.JobMetadata{
|
Metadata: &models.JobMetadata{
|
||||||
JobKind: job.KindGeneric,
|
JobKind: job.KindGeneric,
|
||||||
},
|
},
|
||||||
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/retention/task/%d", l.internalCoreURL, jobData.taskID),
|
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/retention/task/%d", l.internalCoreURL, jobData.taskID),
|
||||||
|
Parameters: make(map[string]interface{}, 3),
|
||||||
}
|
}
|
||||||
j.Name = job.Retention
|
|
||||||
j.Parameters = map[string]interface{}{
|
var (
|
||||||
ParamRepo: jobData.repository,
|
repoJSON, policyJSON string
|
||||||
ParamMeta: jobData.policy,
|
)
|
||||||
ParamDryRun: isDryRun,
|
// Set dry run
|
||||||
|
j.Parameters[ParamDryRun] = isDryRun
|
||||||
|
// Set repository
|
||||||
|
if repoJSON, err = jobData.repository.ToJSON(); err == nil {
|
||||||
|
j.Parameters[ParamRepo] = repoJSON
|
||||||
|
// Set retention policy
|
||||||
|
if policyJSON, err = jobData.policy.ToJSON(); err == nil {
|
||||||
|
j.Parameters[ParamMeta] = policyJSON
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var jobID string
|
||||||
|
if err == nil {
|
||||||
|
// Submit job
|
||||||
|
jobID, err = l.jobserviceClient.SubmitJob(j)
|
||||||
|
}
|
||||||
|
|
||||||
task := &Task{
|
task := &Task{
|
||||||
ID: jobData.taskID,
|
ID: jobData.taskID,
|
||||||
}
|
}
|
||||||
props := []string{"Status"}
|
props := []string{"Status"}
|
||||||
jobID, err := l.jobserviceClient.SubmitJob(j)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(launcherError(fmt.Errorf("failed to submit task %d: %v", jobData.taskID, err)))
|
log.Error(launcherError(fmt.Errorf("failed to submit task %d: %v", jobData.taskID, err)))
|
||||||
task.Status = cmodels.JobError
|
task.Status = cmodels.JobError
|
||||||
|
@ -187,7 +187,7 @@ func (d *DefaultManager) ListTasks(query ...*q.TaskQuery) ([]*Task, error) {
|
|||||||
}
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
tasks := []*Task{}
|
tasks := make([]*Task, 0)
|
||||||
for _, t := range ts {
|
for _, t := range ts {
|
||||||
tasks = append(tasks, &Task{
|
tasks = append(tasks, &Task{
|
||||||
ID: t.ID,
|
ID: t.ID,
|
||||||
|
@ -6,6 +6,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common/dao"
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common/job"
|
"github.com/goharbor/harbor/src/common/job"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy"
|
"github.com/goharbor/harbor/src/pkg/retention/policy"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
|
@ -12,19 +12,26 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package action
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
// index for keeping the mapping action and its performer
|
// index for keeping the mapping action and its performer
|
||||||
var index sync.Map
|
var index sync.Map
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Register retain action
|
||||||
|
Register(action.Retain, action.NewRetainAction)
|
||||||
|
}
|
||||||
|
|
||||||
// Register the performer with the corresponding action
|
// Register the performer with the corresponding action
|
||||||
func Register(action string, factory PerformerFactory) {
|
func Register(action string, factory action.PerformerFactory) {
|
||||||
if len(action) == 0 || factory == nil {
|
if len(action) == 0 || factory == nil {
|
||||||
// do nothing
|
// do nothing
|
||||||
return
|
return
|
||||||
@ -34,19 +41,19 @@ func Register(action string, factory PerformerFactory) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get performer with the provided action
|
// Get performer with the provided action
|
||||||
func Get(action string, params interface{}, isDryRun bool) (Performer, error) {
|
func Get(act string, params interface{}, isDryRun bool) (action.Performer, error) {
|
||||||
if len(action) == 0 {
|
if len(act) == 0 {
|
||||||
return nil, errors.New("empty action")
|
return nil, errors.New("empty action")
|
||||||
}
|
}
|
||||||
|
|
||||||
v, ok := index.Load(action)
|
v, ok := index.Load(act)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.Errorf("action %s is not registered", action)
|
return nil, errors.Errorf("action %s is not registered", act)
|
||||||
}
|
}
|
||||||
|
|
||||||
factory, ok := v.(PerformerFactory)
|
factory, ok := v.(action.PerformerFactory)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.Errorf("invalid action performer registered for action %s", action)
|
return nil, errors.Errorf("invalid action performer registered for action %s", act)
|
||||||
}
|
}
|
||||||
|
|
||||||
return factory(params, isDryRun), nil
|
return factory(params, isDryRun), nil
|
@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package action
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
@ -22,6 +22,7 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
)
|
)
|
||||||
@ -88,7 +89,7 @@ func (p *fakePerformer) Perform(candidates []*res.Candidate) (results []*res.Res
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func newFakePerformer(params interface{}, isDryRun bool) Performer {
|
func newFakePerformer(params interface{}, isDryRun bool) action.Performer {
|
||||||
return &fakePerformer{
|
return &fakePerformer{
|
||||||
parameters: params,
|
parameters: params,
|
||||||
isDryRun: isDryRun,
|
isDryRun: isDryRun,
|
@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
// Retain artifacts
|
// Retain artifacts
|
||||||
Retain = "Retain"
|
Retain = "retain"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Performer performs the related actions targeting the candidates
|
// Performer performs the related actions targeting the candidates
|
||||||
@ -92,8 +92,3 @@ func NewRetainAction(params interface{}, isDryRun bool) Performer {
|
|||||||
isDryRun: isDryRun,
|
isDryRun: isDryRun,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
|
||||||
// Register itself
|
|
||||||
Register(Retain, NewRetainAction)
|
|
||||||
}
|
|
||||||
|
@ -12,11 +12,16 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package alg
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/pkg/errors"
|
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/alg/or"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -27,21 +32,26 @@ const (
|
|||||||
// index for keeping the mapping between algorithm and its processor
|
// index for keeping the mapping between algorithm and its processor
|
||||||
var index sync.Map
|
var index sync.Map
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Register or
|
||||||
|
Register(AlgorithmOR, or.New)
|
||||||
|
}
|
||||||
|
|
||||||
// Register processor with the algorithm
|
// Register processor with the algorithm
|
||||||
func Register(algorithm string, processor Factory) {
|
func Register(algorithm string, processor alg.Factory) {
|
||||||
if len(algorithm) > 0 && processor != nil {
|
if len(algorithm) > 0 && processor != nil {
|
||||||
index.Store(algorithm, processor)
|
index.Store(algorithm, processor)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get Processor
|
// Get Processor
|
||||||
func Get(algorithm string, params []*Parameter) (Processor, error) {
|
func Get(algorithm string, params []*alg.Parameter) (alg.Processor, error) {
|
||||||
v, ok := index.Load(algorithm)
|
v, ok := index.Load(algorithm)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.Errorf("no processor registered with algorithm: %s", algorithm)
|
return nil, errors.Errorf("no processor registered with algorithm: %s", algorithm)
|
||||||
}
|
}
|
||||||
|
|
||||||
if fac, ok := v.(Factory); ok {
|
if fac, ok := v.(alg.Factory); ok {
|
||||||
return fac(params), nil
|
return fac(params), nil
|
||||||
}
|
}
|
||||||
|
|
@ -15,13 +15,14 @@
|
|||||||
package or
|
package or
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"sync"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common/utils/log"
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"sync"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// processor to handle the rules with OR mapping ways
|
// processor to handle the rules with OR mapping ways
|
||||||
@ -153,13 +154,11 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(processed) > 0 {
|
|
||||||
// Pass to the outside
|
// Pass to the outside
|
||||||
resChan <- &chanItem{
|
resChan <- &chanItem{
|
||||||
action: evaluator.Action(),
|
action: evaluator.Action(),
|
||||||
processed: processed,
|
processed: processed,
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}(evaluator, selectors)
|
}(evaluator, selectors)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -204,10 +203,6 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) {
|
|||||||
return results, nil
|
return results, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
|
||||||
alg.Register(alg.AlgorithmOR, New)
|
|
||||||
}
|
|
||||||
|
|
||||||
type cHash map[string]*res.Candidate
|
type cHash map[string]*res.Candidate
|
||||||
|
|
||||||
func (ch cHash) toList() []*res.Candidate {
|
func (ch cHash) toList() []*res.Candidate {
|
||||||
|
@ -19,18 +19,21 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/dep"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/always"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
|
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/lastx"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/lastx"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/dep"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
@ -40,7 +43,6 @@ import (
|
|||||||
type ProcessorTestSuite struct {
|
type ProcessorTestSuite struct {
|
||||||
suite.Suite
|
suite.Suite
|
||||||
|
|
||||||
p alg.Processor
|
|
||||||
all []*res.Candidate
|
all []*res.Candidate
|
||||||
|
|
||||||
oldClient dep.Client
|
oldClient dep.Client
|
||||||
@ -72,10 +74,21 @@ func (suite *ProcessorTestSuite) SetupSuite() {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
params := make([]*alg.Parameter, 0)
|
suite.oldClient = dep.DefaultClient
|
||||||
|
dep.DefaultClient = &fakeRetentionClient{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TearDownSuite ...
|
||||||
|
func (suite *ProcessorTestSuite) TearDownSuite() {
|
||||||
|
dep.DefaultClient = suite.oldClient
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProcess tests process method
|
||||||
|
func (suite *ProcessorTestSuite) TestProcess() {
|
||||||
|
|
||||||
perf := action.NewRetainAction(suite.all, false)
|
perf := action.NewRetainAction(suite.all, false)
|
||||||
|
|
||||||
|
params := make([]*alg.Parameter, 0)
|
||||||
lastxParams := make(map[string]rule.Parameter)
|
lastxParams := make(map[string]rule.Parameter)
|
||||||
lastxParams[lastx.ParameterX] = 10
|
lastxParams[lastx.ParameterX] = 10
|
||||||
params = append(params, &alg.Parameter{
|
params = append(params, &alg.Parameter{
|
||||||
@ -88,32 +101,18 @@ func (suite *ProcessorTestSuite) SetupSuite() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
latestKParams := make(map[string]rule.Parameter)
|
latestKParams := make(map[string]rule.Parameter)
|
||||||
latestKParams[latestk.ParameterK] = 10
|
latestKParams[latestps.ParameterK] = 10
|
||||||
params = append(params, &alg.Parameter{
|
params = append(params, &alg.Parameter{
|
||||||
Evaluator: latestk.New(latestKParams),
|
Evaluator: latestps.New(latestKParams),
|
||||||
Selectors: []res.Selector{
|
Selectors: []res.Selector{
|
||||||
label.New(label.With, "L3"),
|
label.New(label.With, "L3"),
|
||||||
},
|
},
|
||||||
Performer: perf,
|
Performer: perf,
|
||||||
})
|
})
|
||||||
|
|
||||||
p, err := alg.Get(alg.AlgorithmOR, params)
|
p := New(params)
|
||||||
require.NoError(suite.T(), err)
|
|
||||||
|
|
||||||
suite.p = p
|
results, err := p.Process(suite.all)
|
||||||
|
|
||||||
suite.oldClient = dep.DefaultClient
|
|
||||||
dep.DefaultClient = &fakeRetentionClient{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TearDownSuite ...
|
|
||||||
func (suite *ProcessorTestSuite) TearDownSuite() {
|
|
||||||
dep.DefaultClient = suite.oldClient
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestProcess tests process method
|
|
||||||
func (suite *ProcessorTestSuite) TestProcess() {
|
|
||||||
results, err := suite.p.Process(suite.all)
|
|
||||||
require.NoError(suite.T(), err)
|
require.NoError(suite.T(), err)
|
||||||
assert.Equal(suite.T(), 1, len(results))
|
assert.Equal(suite.T(), 1, len(results))
|
||||||
assert.Condition(suite.T(), func() bool {
|
assert.Condition(suite.T(), func() bool {
|
||||||
@ -127,6 +126,43 @@ func (suite *ProcessorTestSuite) TestProcess() {
|
|||||||
}, "no errors in the returned result list")
|
}, "no errors in the returned result list")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestProcess2 ...
|
||||||
|
func (suite *ProcessorTestSuite) TestProcess2() {
|
||||||
|
perf := action.NewRetainAction(suite.all, false)
|
||||||
|
|
||||||
|
params := make([]*alg.Parameter, 0)
|
||||||
|
alwaysParams := make(map[string]rule.Parameter)
|
||||||
|
params = append(params, &alg.Parameter{
|
||||||
|
Evaluator: always.New(alwaysParams),
|
||||||
|
Selectors: []res.Selector{
|
||||||
|
doublestar.New(doublestar.Matches, "latest"),
|
||||||
|
label.New(label.With, ""),
|
||||||
|
},
|
||||||
|
Performer: perf,
|
||||||
|
})
|
||||||
|
|
||||||
|
p := New(params)
|
||||||
|
|
||||||
|
results, err := p.Process(suite.all)
|
||||||
|
require.NoError(suite.T(), err)
|
||||||
|
assert.Equal(suite.T(), 1, len(results))
|
||||||
|
assert.Condition(suite.T(), func() bool {
|
||||||
|
found := false
|
||||||
|
for _, r := range results {
|
||||||
|
if r.Error != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Target.Tag == "dev" {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return found
|
||||||
|
}, "no errors in the returned result list")
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
type fakeRetentionClient struct{}
|
type fakeRetentionClient struct{}
|
||||||
|
|
||||||
// GetCandidates ...
|
// GetCandidates ...
|
||||||
|
@ -17,12 +17,17 @@ package policy
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
index4 "github.com/goharbor/harbor/src/pkg/retention/policy/action/index"
|
||||||
|
|
||||||
|
index3 "github.com/goharbor/harbor/src/pkg/retention/policy/alg/index"
|
||||||
|
|
||||||
|
index2 "github.com/goharbor/harbor/src/pkg/retention/res/selectors/index"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/index"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -61,19 +66,19 @@ func (bb *basicBuilder) Build(policy *lwp.Metadata, isDryRun bool) (alg.Processo
|
|||||||
params := make([]*alg.Parameter, 0)
|
params := make([]*alg.Parameter, 0)
|
||||||
|
|
||||||
for _, r := range policy.Rules {
|
for _, r := range policy.Rules {
|
||||||
evaluator, err := rule.Get(r.Template, r.Parameters)
|
evaluator, err := index.Get(r.Template, r.Parameters)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
perf, err := action.Get(r.Action, bb.allCandidates, isDryRun)
|
perf, err := index4.Get(r.Action, bb.allCandidates, isDryRun)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "get action performer by metadata")
|
return nil, errors.Wrap(err, "get action performer by metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
sl := make([]res.Selector, 0)
|
sl := make([]res.Selector, 0)
|
||||||
for _, s := range r.TagSelectors {
|
for _, s := range r.TagSelectors {
|
||||||
sel, err := selectors.Get(s.Kind, s.Decoration, s.Pattern)
|
sel, err := index2.Get(s.Kind, s.Decoration, s.Pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "get selector by metadata")
|
return nil, errors.Wrap(err, "get selector by metadata")
|
||||||
}
|
}
|
||||||
@ -88,7 +93,7 @@ func (bb *basicBuilder) Build(policy *lwp.Metadata, isDryRun bool) (alg.Processo
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
p, err := alg.Get(policy.Algorithm, params)
|
p, err := index3.Get(policy.Algorithm, params)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, fmt.Sprintf("get processor for algorithm: %s", policy.Algorithm))
|
return nil, errors.Wrap(err, fmt.Sprintf("get processor for algorithm: %s", policy.Algorithm))
|
||||||
}
|
}
|
||||||
|
@ -18,19 +18,23 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
index3 "github.com/goharbor/harbor/src/pkg/retention/policy/action/index"
|
||||||
|
|
||||||
|
index2 "github.com/goharbor/harbor/src/pkg/retention/policy/alg/index"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/index"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/dep"
|
"github.com/goharbor/harbor/src/pkg/retention/dep"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/alg/or"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/alg/or"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
||||||
|
|
||||||
@ -83,8 +87,8 @@ func (suite *TestBuilderSuite) SetupSuite() {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
alg.Register(alg.AlgorithmOR, or.New)
|
index2.Register(index2.AlgorithmOR, or.New)
|
||||||
selectors.Register(doublestar.Kind, []string{
|
index.Register(doublestar.Kind, []string{
|
||||||
doublestar.Matches,
|
doublestar.Matches,
|
||||||
doublestar.Excludes,
|
doublestar.Excludes,
|
||||||
doublestar.RepoMatches,
|
doublestar.RepoMatches,
|
||||||
@ -92,8 +96,8 @@ func (suite *TestBuilderSuite) SetupSuite() {
|
|||||||
doublestar.NSMatches,
|
doublestar.NSMatches,
|
||||||
doublestar.NSExcludes,
|
doublestar.NSExcludes,
|
||||||
}, doublestar.New)
|
}, doublestar.New)
|
||||||
selectors.Register(label.Kind, []string{label.With, label.Without}, label.New)
|
index.Register(label.Kind, []string{label.With, label.Without}, label.New)
|
||||||
action.Register(action.Retain, action.NewRetainAction)
|
index3.Register(action.Retain, action.NewRetainAction)
|
||||||
|
|
||||||
suite.oldClient = dep.DefaultClient
|
suite.oldClient = dep.DefaultClient
|
||||||
dep.DefaultClient = &fakeRetentionClient{}
|
dep.DefaultClient = &fakeRetentionClient{}
|
||||||
@ -109,7 +113,7 @@ func (suite *TestBuilderSuite) TestBuild() {
|
|||||||
b := &basicBuilder{suite.all}
|
b := &basicBuilder{suite.all}
|
||||||
|
|
||||||
params := make(rule.Parameters)
|
params := make(rule.Parameters)
|
||||||
params[latestk.ParameterK] = 10
|
params[latestps.ParameterK] = 10
|
||||||
|
|
||||||
scopeSelectors := make(map[string][]*rule.Selector, 1)
|
scopeSelectors := make(map[string][]*rule.Selector, 1)
|
||||||
scopeSelectors["repository"] = []*rule.Selector{{
|
scopeSelectors["repository"] = []*rule.Selector{{
|
||||||
@ -124,14 +128,14 @@ func (suite *TestBuilderSuite) TestBuild() {
|
|||||||
ID: 1,
|
ID: 1,
|
||||||
Priority: 999,
|
Priority: 999,
|
||||||
Action: action.Retain,
|
Action: action.Retain,
|
||||||
Template: latestk.TemplateID,
|
Template: latestps.TemplateID,
|
||||||
Parameters: params,
|
Parameters: params,
|
||||||
ScopeSelectors: scopeSelectors,
|
ScopeSelectors: scopeSelectors,
|
||||||
TagSelectors: []*rule.Selector{
|
TagSelectors: []*rule.Selector{
|
||||||
{
|
{
|
||||||
Kind: label.Kind,
|
Kind: doublestar.Kind,
|
||||||
Decoration: label.With,
|
Decoration: doublestar.Matches,
|
||||||
Pattern: "L3",
|
Pattern: "latest",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
@ -141,19 +145,7 @@ func (suite *TestBuilderSuite) TestBuild() {
|
|||||||
require.NoError(suite.T(), err)
|
require.NoError(suite.T(), err)
|
||||||
require.NotNil(suite.T(), p)
|
require.NotNil(suite.T(), p)
|
||||||
|
|
||||||
artifacts := []*res.Candidate{
|
results, err := p.Process(suite.all)
|
||||||
{
|
|
||||||
NamespaceID: 1,
|
|
||||||
Namespace: "library",
|
|
||||||
Repository: "harbor",
|
|
||||||
Kind: "image",
|
|
||||||
Tag: "dev",
|
|
||||||
PushedTime: time.Now().Unix(),
|
|
||||||
Labels: []string{"L3"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
results, err := p.Process(artifacts)
|
|
||||||
require.NoError(suite.T(), err)
|
require.NoError(suite.T(), err)
|
||||||
assert.Equal(suite.T(), 1, len(results))
|
assert.Equal(suite.T(), 1, len(results))
|
||||||
assert.Condition(suite.T(), func() (success bool) {
|
assert.Condition(suite.T(), func() (success bool) {
|
||||||
@ -161,7 +153,7 @@ func (suite *TestBuilderSuite) TestBuild() {
|
|||||||
success = art.Error == nil &&
|
success = art.Error == nil &&
|
||||||
art.Target != nil &&
|
art.Target != nil &&
|
||||||
art.Target.Repository == "harbor" &&
|
art.Target.Repository == "harbor" &&
|
||||||
art.Target.Tag == "latest"
|
art.Target.Tag == "dev"
|
||||||
|
|
||||||
return
|
return
|
||||||
})
|
})
|
||||||
|
@ -18,6 +18,8 @@ package lwp
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -32,14 +34,21 @@ type Metadata struct {
|
|||||||
Rules []*rule.Metadata `json:"rules"`
|
Rules []*rule.Metadata `json:"rules"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// FromMap constructs the metadata struct from map
|
// ToJSON marshals metadata to JSON string
|
||||||
func (meta *Metadata) FromMap(m map[string]interface{}) error {
|
func (m *Metadata) ToJSON() (string, error) {
|
||||||
mdata, err := json.Marshal(&m)
|
jsonData, err := json.Marshal(m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", errors.Wrap(err, "marshal reporitory")
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(mdata, meta); err != nil {
|
|
||||||
return err
|
return string(jsonData), nil
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
|
// FromJSON constructs the metadata from json data
|
||||||
|
func (m *Metadata) FromJSON(jsonData string) error {
|
||||||
|
if len(jsonData) == 0 {
|
||||||
|
return errors.New("empty json data to construct repository")
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.Unmarshal([]byte(jsonData), m)
|
||||||
}
|
}
|
||||||
|
@ -40,11 +40,3 @@ func (e *evaluator) Action() string {
|
|||||||
func New(_ rule.Parameters) rule.Evaluator {
|
func New(_ rule.Parameters) rule.Evaluator {
|
||||||
return &evaluator{}
|
return &evaluator{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
|
||||||
rule.Register(&rule.IndexMeta{
|
|
||||||
TemplateID: TemplateID,
|
|
||||||
Action: action.Retain,
|
|
||||||
Parameters: []*rule.IndexedParam{},
|
|
||||||
}, New)
|
|
||||||
}
|
|
||||||
|
@ -12,18 +12,33 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package rule
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/pkg/errors"
|
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestpl"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/always"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/lastx"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
// index for keeping the mapping between template ID and evaluator
|
// index for keeping the mapping between template ID and evaluator
|
||||||
var index sync.Map
|
var index sync.Map
|
||||||
|
|
||||||
// IndexMeta defines metadata for rule registration
|
// Metadata defines metadata for rule registration
|
||||||
type IndexMeta struct {
|
type Metadata struct {
|
||||||
TemplateID string `json:"rule_template"`
|
TemplateID string `json:"rule_template"`
|
||||||
|
|
||||||
// Action of the rule performs
|
// Action of the rule performs
|
||||||
@ -48,13 +63,78 @@ type IndexedParam struct {
|
|||||||
|
|
||||||
// indexedItem is the item saved in the sync map
|
// indexedItem is the item saved in the sync map
|
||||||
type indexedItem struct {
|
type indexedItem struct {
|
||||||
Meta *IndexMeta
|
Meta *Metadata
|
||||||
|
|
||||||
Factory Factory
|
Factory rule.Factory
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Register latest pushed
|
||||||
|
Register(&Metadata{
|
||||||
|
TemplateID: latestps.TemplateID,
|
||||||
|
Action: action.Retain,
|
||||||
|
Parameters: []*IndexedParam{
|
||||||
|
{
|
||||||
|
Name: latestps.ParameterK,
|
||||||
|
Type: "int",
|
||||||
|
Unit: "count",
|
||||||
|
Required: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, latestps.New)
|
||||||
|
|
||||||
|
// Register latest pulled
|
||||||
|
Register(&Metadata{
|
||||||
|
TemplateID: latestpl.TemplateID,
|
||||||
|
Action: action.Retain,
|
||||||
|
Parameters: []*IndexedParam{
|
||||||
|
{
|
||||||
|
Name: latestpl.ParameterN,
|
||||||
|
Type: "int",
|
||||||
|
Unit: "count",
|
||||||
|
Required: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, latestpl.New)
|
||||||
|
|
||||||
|
// Register latest active
|
||||||
|
Register(&Metadata{
|
||||||
|
TemplateID: latestk.TemplateID,
|
||||||
|
Action: action.Retain,
|
||||||
|
Parameters: []*IndexedParam{
|
||||||
|
{
|
||||||
|
Name: latestk.ParameterK,
|
||||||
|
Type: "int",
|
||||||
|
Unit: "count",
|
||||||
|
Required: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, latestk.New)
|
||||||
|
|
||||||
|
// Register lastx
|
||||||
|
Register(&Metadata{
|
||||||
|
TemplateID: lastx.TemplateID,
|
||||||
|
Action: action.Retain,
|
||||||
|
Parameters: []*IndexedParam{
|
||||||
|
{
|
||||||
|
Name: lastx.ParameterX,
|
||||||
|
Type: "int",
|
||||||
|
Unit: "days",
|
||||||
|
Required: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, lastx.New)
|
||||||
|
|
||||||
|
// Register always
|
||||||
|
Register(&Metadata{
|
||||||
|
TemplateID: always.TemplateID,
|
||||||
|
Action: action.Retain,
|
||||||
|
Parameters: []*IndexedParam{},
|
||||||
|
}, always.New)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Register the rule evaluator with the corresponding rule template
|
// Register the rule evaluator with the corresponding rule template
|
||||||
func Register(meta *IndexMeta, factory Factory) {
|
func Register(meta *Metadata, factory rule.Factory) {
|
||||||
if meta == nil || factory == nil || len(meta.TemplateID) == 0 {
|
if meta == nil || factory == nil || len(meta.TemplateID) == 0 {
|
||||||
// do nothing
|
// do nothing
|
||||||
return
|
return
|
||||||
@ -67,7 +147,7 @@ func Register(meta *IndexMeta, factory Factory) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get rule evaluator with the provided template ID
|
// Get rule evaluator with the provided template ID
|
||||||
func Get(templateID string, parameters Parameters) (Evaluator, error) {
|
func Get(templateID string, parameters rule.Parameters) (rule.Evaluator, error) {
|
||||||
if len(templateID) == 0 {
|
if len(templateID) == 0 {
|
||||||
return nil, errors.New("empty rule template ID")
|
return nil, errors.New("empty rule template ID")
|
||||||
}
|
}
|
||||||
@ -100,8 +180,8 @@ func Get(templateID string, parameters Parameters) (Evaluator, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Index returns all the metadata of the registered rules
|
// Index returns all the metadata of the registered rules
|
||||||
func Index() []*IndexMeta {
|
func Index() []*Metadata {
|
||||||
res := make([]*IndexMeta, 0)
|
res := make([]*Metadata, 0)
|
||||||
|
|
||||||
index.Range(func(k, v interface{}) bool {
|
index.Range(func(k, v interface{}) bool {
|
||||||
if item, ok := v.(*indexedItem); ok {
|
if item, ok := v.(*indexedItem); ok {
|
@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package rule
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
@ -22,6 +22,7 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
|
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
@ -39,7 +40,7 @@ func TestIndexEntry(t *testing.T) {
|
|||||||
|
|
||||||
// SetupSuite ...
|
// SetupSuite ...
|
||||||
func (suite *IndexTestSuite) SetupSuite() {
|
func (suite *IndexTestSuite) SetupSuite() {
|
||||||
Register(&IndexMeta{
|
Register(&Metadata{
|
||||||
TemplateID: "fakeEvaluator",
|
TemplateID: "fakeEvaluator",
|
||||||
Action: "retain",
|
Action: "retain",
|
||||||
Parameters: []*IndexedParam{
|
Parameters: []*IndexedParam{
|
||||||
@ -56,7 +57,7 @@ func (suite *IndexTestSuite) SetupSuite() {
|
|||||||
// TestRegister tests register
|
// TestRegister tests register
|
||||||
func (suite *IndexTestSuite) TestGet() {
|
func (suite *IndexTestSuite) TestGet() {
|
||||||
|
|
||||||
params := make(Parameters)
|
params := make(rule.Parameters)
|
||||||
params["fakeParam"] = 99
|
params["fakeParam"] = 99
|
||||||
evaluator, err := Get("fakeEvaluator", params)
|
evaluator, err := Get("fakeEvaluator", params)
|
||||||
require.NoError(suite.T(), err)
|
require.NoError(suite.T(), err)
|
||||||
@ -71,11 +72,11 @@ func (suite *IndexTestSuite) TestGet() {
|
|||||||
Labels: []string{"L1", "L2"},
|
Labels: []string{"L1", "L2"},
|
||||||
}}
|
}}
|
||||||
|
|
||||||
res, err := evaluator.Process(candidates)
|
results, err := evaluator.Process(candidates)
|
||||||
require.NoError(suite.T(), err)
|
require.NoError(suite.T(), err)
|
||||||
assert.Equal(suite.T(), 1, len(res))
|
assert.Equal(suite.T(), 1, len(results))
|
||||||
assert.Condition(suite.T(), func() bool {
|
assert.Condition(suite.T(), func() bool {
|
||||||
c := res[0]
|
c := results[0]
|
||||||
return c.Repository == "harbor" && c.Tag == "latest"
|
return c.Repository == "harbor" && c.Tag == "latest"
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -83,13 +84,17 @@ func (suite *IndexTestSuite) TestGet() {
|
|||||||
// TestIndex tests Index
|
// TestIndex tests Index
|
||||||
func (suite *IndexTestSuite) TestIndex() {
|
func (suite *IndexTestSuite) TestIndex() {
|
||||||
metas := Index()
|
metas := Index()
|
||||||
require.Equal(suite.T(), 1, len(metas))
|
require.Equal(suite.T(), 6, len(metas))
|
||||||
assert.Condition(suite.T(), func() bool {
|
assert.Condition(suite.T(), func() bool {
|
||||||
m := metas[0]
|
for _, m := range metas {
|
||||||
return m.TemplateID == "fakeEvaluator" &&
|
if m.TemplateID == "fakeEvaluator" &&
|
||||||
m.Action == "retain" &&
|
m.Action == "retain" &&
|
||||||
len(m.Parameters) > 0
|
len(m.Parameters) > 0 {
|
||||||
})
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}, "check fake evaluator in index")
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeEvaluator struct {
|
type fakeEvaluator struct {
|
||||||
@ -107,7 +112,7 @@ func (e *fakeEvaluator) Action() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newFakeEvaluator is the factory of fakeEvaluator
|
// newFakeEvaluator is the factory of fakeEvaluator
|
||||||
func newFakeEvaluator(parameters Parameters) Evaluator {
|
func newFakeEvaluator(parameters rule.Parameters) rule.Evaluator {
|
||||||
i := 10
|
i := 10
|
||||||
if v, ok := parameters["fakeParam"]; ok {
|
if v, ok := parameters["fakeParam"]; ok {
|
||||||
i = v.(int)
|
i = v.(int)
|
@ -73,19 +73,3 @@ func New(params rule.Parameters) rule.Evaluator {
|
|||||||
x: DefaultX,
|
x: DefaultX,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
|
||||||
// Register itself
|
|
||||||
rule.Register(&rule.IndexMeta{
|
|
||||||
TemplateID: TemplateID,
|
|
||||||
Action: action.Retain,
|
|
||||||
Parameters: []*rule.IndexedParam{
|
|
||||||
{
|
|
||||||
Name: ParameterX,
|
|
||||||
Type: "int",
|
|
||||||
Unit: "days",
|
|
||||||
Required: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}, New)
|
|
||||||
}
|
|
||||||
|
@ -1,99 +0,0 @@
|
|||||||
// Copyright Project Harbor Authors
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package latestk
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
|
||||||
"github.com/stretchr/testify/suite"
|
|
||||||
)
|
|
||||||
|
|
||||||
type EvaluatorTestSuite struct {
|
|
||||||
suite.Suite
|
|
||||||
|
|
||||||
artifacts []*res.Candidate
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *EvaluatorTestSuite) SetupSuite() {
|
|
||||||
e.artifacts = []*res.Candidate{
|
|
||||||
{PulledTime: 1, PushedTime: 2},
|
|
||||||
{PulledTime: 3, PushedTime: 4},
|
|
||||||
{PulledTime: 6, PushedTime: 5},
|
|
||||||
{PulledTime: 8, PushedTime: 7},
|
|
||||||
{PulledTime: 9, PushedTime: 9},
|
|
||||||
{PulledTime: 10, PushedTime: 10},
|
|
||||||
{PulledTime: 0, PushedTime: 11},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *EvaluatorTestSuite) TestProcess() {
|
|
||||||
tests := []struct {
|
|
||||||
k int
|
|
||||||
expected int
|
|
||||||
minActiveTime int64
|
|
||||||
}{
|
|
||||||
{k: 0, expected: 0},
|
|
||||||
{k: 1, expected: 1, minActiveTime: 11},
|
|
||||||
{k: 2, expected: 2, minActiveTime: 10},
|
|
||||||
{k: 5, expected: 5, minActiveTime: 6},
|
|
||||||
{k: 6, expected: 6, minActiveTime: 3},
|
|
||||||
{k: 99, expected: len(e.artifacts)},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
e.T().Run(strconv.Itoa(tt.k), func(t *testing.T) {
|
|
||||||
sut := &evaluator{k: tt.k}
|
|
||||||
|
|
||||||
result, err := sut.Process(e.artifacts)
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, result, tt.expected)
|
|
||||||
|
|
||||||
for _, v := range result {
|
|
||||||
assert.True(t, activeTime(v) >= tt.minActiveTime)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *EvaluatorTestSuite) TestNew() {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
params rule.Parameters
|
|
||||||
expectedK int
|
|
||||||
}{
|
|
||||||
{name: "Valid", params: rule.Parameters{ParameterK: 5}, expectedK: 5},
|
|
||||||
{name: "Default If Negative", params: rule.Parameters{ParameterK: -5}, expectedK: DefaultK},
|
|
||||||
{name: "Default If Wrong Type", params: rule.Parameters{ParameterK: "5"}, expectedK: DefaultK},
|
|
||||||
{name: "Default If Wrong Key", params: rule.Parameters{"n": 5}, expectedK: DefaultK},
|
|
||||||
{name: "Default If Empty", params: rule.Parameters{}, expectedK: DefaultK},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
e.T().Run(tt.name, func(t *testing.T) {
|
|
||||||
sut := New(tt.params).(*evaluator)
|
|
||||||
|
|
||||||
require.Equal(t, tt.expectedK, sut.k)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEvaluatorSuite(t *testing.T) {
|
|
||||||
suite.Run(t, &EvaluatorTestSuite{})
|
|
||||||
}
|
|
@ -24,15 +24,15 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// TemplateID of latest k rule
|
// TemplateID of latest active k rule
|
||||||
TemplateID = "latestK"
|
TemplateID = "latestActiveK"
|
||||||
// ParameterK ...
|
// ParameterK ...
|
||||||
ParameterK = TemplateID
|
ParameterK = TemplateID
|
||||||
// DefaultK defines the default K
|
// DefaultK defines the default K
|
||||||
DefaultK = 10
|
DefaultK = 10
|
||||||
)
|
)
|
||||||
|
|
||||||
// evaluator for evaluating latest k tags
|
// evaluator for evaluating latest active k images
|
||||||
type evaluator struct {
|
type evaluator struct {
|
||||||
// latest k
|
// latest k
|
||||||
k int
|
k int
|
||||||
@ -40,9 +40,12 @@ type evaluator struct {
|
|||||||
|
|
||||||
// Process the candidates based on the rule definition
|
// Process the candidates based on the rule definition
|
||||||
func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
|
func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
|
||||||
// The updated proposal does not guarantee the order artifacts are provided, so we have to sort them first
|
// Sort artifacts by their "active time"
|
||||||
|
//
|
||||||
|
// Active time is defined as the selection of c.PulledTime or c.PushedTime,
|
||||||
|
// whichever is bigger, aka more recent.
|
||||||
sort.Slice(artifacts, func(i, j int) bool {
|
sort.Slice(artifacts, func(i, j int) bool {
|
||||||
return artifacts[i].PushedTime < artifacts[j].PushedTime
|
return activeTime(artifacts[i]) > activeTime(artifacts[j])
|
||||||
})
|
})
|
||||||
|
|
||||||
i := e.k
|
i := e.k
|
||||||
@ -77,18 +80,10 @@ func New(params rule.Parameters) rule.Evaluator {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func activeTime(c *res.Candidate) int64 {
|
||||||
// Register itself
|
if c.PulledTime > c.PushedTime {
|
||||||
rule.Register(&rule.IndexMeta{
|
return c.PulledTime
|
||||||
TemplateID: TemplateID,
|
}
|
||||||
Action: action.Retain,
|
|
||||||
Parameters: []*rule.IndexedParam{
|
return c.PushedTime
|
||||||
{
|
|
||||||
Name: ParameterK,
|
|
||||||
Type: "int",
|
|
||||||
Unit: "count",
|
|
||||||
Required: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}, New)
|
|
||||||
}
|
}
|
||||||
|
@ -1,71 +1,99 @@
|
|||||||
|
// Copyright Project Harbor Authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
package latestk
|
package latestk
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"math/rand"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/suite"
|
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
)
|
)
|
||||||
|
|
||||||
type EvaluatorTestSuite struct {
|
type EvaluatorTestSuite struct {
|
||||||
suite.Suite
|
suite.Suite
|
||||||
|
|
||||||
|
artifacts []*res.Candidate
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *EvaluatorTestSuite) TestNew() {
|
func (e *EvaluatorTestSuite) SetupSuite() {
|
||||||
tests := []struct {
|
e.artifacts = []*res.Candidate{
|
||||||
Name string
|
{PulledTime: 1, PushedTime: 2},
|
||||||
args rule.Parameters
|
{PulledTime: 3, PushedTime: 4},
|
||||||
expectedK int
|
{PulledTime: 6, PushedTime: 5},
|
||||||
}{
|
{PulledTime: 8, PushedTime: 7},
|
||||||
{Name: "Valid", args: map[string]rule.Parameter{ParameterK: 5}, expectedK: 5},
|
{PulledTime: 9, PushedTime: 9},
|
||||||
{Name: "Default If Negative", args: map[string]rule.Parameter{ParameterK: -1}, expectedK: DefaultK},
|
{PulledTime: 10, PushedTime: 10},
|
||||||
{Name: "Default If Not Set", args: map[string]rule.Parameter{}, expectedK: DefaultK},
|
{PulledTime: 0, PushedTime: 11},
|
||||||
{Name: "Default If Wrong Type", args: map[string]rule.Parameter{ParameterK: "foo"}, expectedK: DefaultK},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
e.T().Run(tt.Name, func(t *testing.T) {
|
|
||||||
e := New(tt.args).(*evaluator)
|
|
||||||
|
|
||||||
require.Equal(t, tt.expectedK, e.k)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *EvaluatorTestSuite) TestProcess() {
|
func (e *EvaluatorTestSuite) TestProcess() {
|
||||||
data := []*res.Candidate{{PushedTime: 0}, {PushedTime: 1}, {PushedTime: 2}, {PushedTime: 3}, {PushedTime: 4}}
|
|
||||||
rand.Shuffle(len(data), func(i, j int) {
|
|
||||||
data[i], data[j] = data[j], data[i]
|
|
||||||
})
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
k int
|
k int
|
||||||
expected int
|
expected int
|
||||||
|
minActiveTime int64
|
||||||
}{
|
}{
|
||||||
{k: 0, expected: 0},
|
{k: 0, expected: 0},
|
||||||
{k: 1, expected: 1},
|
{k: 1, expected: 1, minActiveTime: 11},
|
||||||
{k: 3, expected: 3},
|
{k: 2, expected: 2, minActiveTime: 10},
|
||||||
{k: 5, expected: 5},
|
{k: 5, expected: 5, minActiveTime: 6},
|
||||||
{k: 6, expected: 5},
|
{k: 6, expected: 6, minActiveTime: 3},
|
||||||
|
{k: 99, expected: len(e.artifacts)},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
e.T().Run(strconv.Itoa(tt.k), func(t *testing.T) {
|
e.T().Run(strconv.Itoa(tt.k), func(t *testing.T) {
|
||||||
e := New(map[string]rule.Parameter{ParameterK: tt.k})
|
sut := &evaluator{k: tt.k}
|
||||||
|
|
||||||
result, err := e.Process(data)
|
result, err := sut.Process(e.artifacts)
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Len(t, result, tt.expected)
|
require.Len(t, result, tt.expected)
|
||||||
|
|
||||||
|
for _, v := range result {
|
||||||
|
assert.True(t, activeTime(v) >= tt.minActiveTime)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEvaluator(t *testing.T) {
|
func (e *EvaluatorTestSuite) TestNew() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
params rule.Parameters
|
||||||
|
expectedK int
|
||||||
|
}{
|
||||||
|
{name: "Valid", params: rule.Parameters{ParameterK: 5}, expectedK: 5},
|
||||||
|
{name: "Default If Negative", params: rule.Parameters{ParameterK: -5}, expectedK: DefaultK},
|
||||||
|
{name: "Default If Wrong Type", params: rule.Parameters{ParameterK: "5"}, expectedK: DefaultK},
|
||||||
|
{name: "Default If Wrong Key", params: rule.Parameters{"n": 5}, expectedK: DefaultK},
|
||||||
|
{name: "Default If Empty", params: rule.Parameters{}, expectedK: DefaultK},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
e.T().Run(tt.name, func(t *testing.T) {
|
||||||
|
sut := New(tt.params).(*evaluator)
|
||||||
|
|
||||||
|
require.Equal(t, tt.expectedK, sut.k)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluatorSuite(t *testing.T) {
|
||||||
suite.Run(t, &EvaluatorTestSuite{})
|
suite.Run(t, &EvaluatorTestSuite{})
|
||||||
}
|
}
|
||||||
|
@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package lastpulled
|
package latestpl
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sort"
|
"sort"
|
||||||
@ -25,7 +25,7 @@ import (
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
// TemplateID of the rule
|
// TemplateID of the rule
|
||||||
TemplateID = "lastpulled"
|
TemplateID = "latestPulledN"
|
||||||
|
|
||||||
// ParameterN is the name of the metadata parameter for the N value
|
// ParameterN is the name of the metadata parameter for the N value
|
||||||
ParameterN = TemplateID
|
ParameterN = TemplateID
|
@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package lastpulled
|
package latestpl
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"math/rand"
|
"math/rand"
|
@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package latestk
|
package latestps
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sort"
|
"sort"
|
||||||
@ -24,15 +24,15 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// TemplateID of latest active k rule
|
// TemplateID of latest k rule
|
||||||
TemplateID = "latestActiveK"
|
TemplateID = "latestPushedK"
|
||||||
// ParameterK ...
|
// ParameterK ...
|
||||||
ParameterK = TemplateID
|
ParameterK = TemplateID
|
||||||
// DefaultK defines the default K
|
// DefaultK defines the default K
|
||||||
DefaultK = 10
|
DefaultK = 10
|
||||||
)
|
)
|
||||||
|
|
||||||
// evaluator for evaluating latest active k images
|
// evaluator for evaluating latest k tags
|
||||||
type evaluator struct {
|
type evaluator struct {
|
||||||
// latest k
|
// latest k
|
||||||
k int
|
k int
|
||||||
@ -40,12 +40,9 @@ type evaluator struct {
|
|||||||
|
|
||||||
// Process the candidates based on the rule definition
|
// Process the candidates based on the rule definition
|
||||||
func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
|
func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
|
||||||
// Sort artifacts by their "active time"
|
// The updated proposal does not guarantee the order artifacts are provided, so we have to sort them first
|
||||||
//
|
|
||||||
// Active time is defined as the selection of c.PulledTime or c.PushedTime,
|
|
||||||
// whichever is bigger, aka more recent.
|
|
||||||
sort.Slice(artifacts, func(i, j int) bool {
|
sort.Slice(artifacts, func(i, j int) bool {
|
||||||
return activeTime(artifacts[i]) > activeTime(artifacts[j])
|
return artifacts[i].PushedTime < artifacts[j].PushedTime
|
||||||
})
|
})
|
||||||
|
|
||||||
i := e.k
|
i := e.k
|
||||||
@ -79,27 +76,3 @@ func New(params rule.Parameters) rule.Evaluator {
|
|||||||
k: DefaultK,
|
k: DefaultK,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func activeTime(c *res.Candidate) int64 {
|
|
||||||
if c.PulledTime > c.PushedTime {
|
|
||||||
return c.PulledTime
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.PushedTime
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
// Register itself
|
|
||||||
rule.Register(&rule.IndexMeta{
|
|
||||||
TemplateID: TemplateID,
|
|
||||||
Action: action.Retain,
|
|
||||||
Parameters: []*rule.IndexedParam{
|
|
||||||
{
|
|
||||||
Name: ParameterK,
|
|
||||||
Type: "int",
|
|
||||||
Unit: "count",
|
|
||||||
Required: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}, New)
|
|
||||||
}
|
|
71
src/pkg/retention/policy/rule/latestps/evaluator_test.go
Normal file
71
src/pkg/retention/policy/rule/latestps/evaluator_test.go
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
package latestps
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math/rand"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type EvaluatorTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *EvaluatorTestSuite) TestNew() {
|
||||||
|
tests := []struct {
|
||||||
|
Name string
|
||||||
|
args rule.Parameters
|
||||||
|
expectedK int
|
||||||
|
}{
|
||||||
|
{Name: "Valid", args: map[string]rule.Parameter{ParameterK: 5}, expectedK: 5},
|
||||||
|
{Name: "Default If Negative", args: map[string]rule.Parameter{ParameterK: -1}, expectedK: DefaultK},
|
||||||
|
{Name: "Default If Not Set", args: map[string]rule.Parameter{}, expectedK: DefaultK},
|
||||||
|
{Name: "Default If Wrong Type", args: map[string]rule.Parameter{ParameterK: "foo"}, expectedK: DefaultK},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
e.T().Run(tt.Name, func(t *testing.T) {
|
||||||
|
e := New(tt.args).(*evaluator)
|
||||||
|
|
||||||
|
require.Equal(t, tt.expectedK, e.k)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *EvaluatorTestSuite) TestProcess() {
|
||||||
|
data := []*res.Candidate{{PushedTime: 0}, {PushedTime: 1}, {PushedTime: 2}, {PushedTime: 3}, {PushedTime: 4}}
|
||||||
|
rand.Shuffle(len(data), func(i, j int) {
|
||||||
|
data[i], data[j] = data[j], data[i]
|
||||||
|
})
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
k int
|
||||||
|
expected int
|
||||||
|
}{
|
||||||
|
{k: 0, expected: 0},
|
||||||
|
{k: 1, expected: 1},
|
||||||
|
{k: 3, expected: 3},
|
||||||
|
{k: 5, expected: 5},
|
||||||
|
{k: 6, expected: 5},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
e.T().Run(strconv.Itoa(tt.k), func(t *testing.T) {
|
||||||
|
e := New(map[string]rule.Parameter{ParameterK: tt.k})
|
||||||
|
|
||||||
|
result, err := e.Process(data)
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, result, tt.expected)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluator(t *testing.T) {
|
||||||
|
suite.Run(t, &EvaluatorTestSuite{})
|
||||||
|
}
|
@ -18,6 +18,8 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -30,24 +32,31 @@ const (
|
|||||||
// Repository of candidate
|
// Repository of candidate
|
||||||
type Repository struct {
|
type Repository struct {
|
||||||
// Namespace
|
// Namespace
|
||||||
Namespace string
|
Namespace string `json:"namespace"`
|
||||||
// Repository name
|
// Repository name
|
||||||
Name string
|
Name string `json:"name"`
|
||||||
// So far we need the kind of repository and retrieve candidates with different APIs
|
// So far we need the kind of repository and retrieve candidates with different APIs
|
||||||
// TODO: REMOVE IT IN THE FUTURE IF WE SUPPORT UNIFIED ARTIFACT MODEL
|
// TODO: REMOVE IT IN THE FUTURE IF WE SUPPORT UNIFIED ARTIFACT MODEL
|
||||||
Kind string
|
Kind string `json:"kind"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// FromMap constructs the repository struct from map
|
// ToJSON marshals repository to JSON string
|
||||||
func (r *Repository) FromMap(m map[string]interface{}) error {
|
func (r *Repository) ToJSON() (string, error) {
|
||||||
mdata, err := json.Marshal(&m)
|
jsonData, err := json.Marshal(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", errors.Wrap(err, "marshal reporitory")
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(mdata, r); err != nil {
|
|
||||||
return err
|
return string(jsonData), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromJSON constructs the repository from json data
|
||||||
|
func (r *Repository) FromJSON(jsonData string) error {
|
||||||
|
if len(jsonData) == 0 {
|
||||||
|
return errors.New("empty json data to construct repository")
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
|
return json.Unmarshal([]byte(jsonData), r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Candidate for retention processor to match
|
// Candidate for retention processor to match
|
||||||
|
@ -12,11 +12,13 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
package selectors
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
|
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
@ -32,6 +34,9 @@ func init() {
|
|||||||
doublestar.NSMatches,
|
doublestar.NSMatches,
|
||||||
doublestar.NSExcludes,
|
doublestar.NSExcludes,
|
||||||
}, doublestar.New)
|
}, doublestar.New)
|
||||||
|
|
||||||
|
// Register label selector
|
||||||
|
Register(label.Kind, []string{label.With, label.Without}, label.New)
|
||||||
}
|
}
|
||||||
|
|
||||||
// index for keeping the mapping between selector meta and its implementation
|
// index for keeping the mapping between selector meta and its implementation
|
@ -15,9 +15,9 @@
|
|||||||
package label
|
package label
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention/res"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -51,7 +51,10 @@ func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate
|
|||||||
|
|
||||||
// New is factory method for list selector
|
// New is factory method for list selector
|
||||||
func New(decoration string, pattern string) res.Selector {
|
func New(decoration string, pattern string) res.Selector {
|
||||||
labels := strings.Split(pattern, ",")
|
labels := make([]string, 0)
|
||||||
|
if len(pattern) > 0 {
|
||||||
|
labels = append(labels, strings.Split(pattern, ",")...)
|
||||||
|
}
|
||||||
|
|
||||||
return &selector{
|
return &selector{
|
||||||
decoration: decoration,
|
decoration: decoration,
|
||||||
@ -81,8 +84,3 @@ func isMatched(patternLbls []string, resLbls []string, decoration string) bool {
|
|||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
|
||||||
// Register doublestar selector
|
|
||||||
selectors.Register(Kind, []string{With, Without}, New)
|
|
||||||
}
|
|
||||||
|
Loading…
Reference in New Issue
Block a user