From 9a7df265cec6d57e1e4107973f80aa9a1be963d0 Mon Sep 17 00:00:00 2001 From: Nathan Lowe Date: Sun, 4 Aug 2019 20:05:05 -0400 Subject: [PATCH] Retention: New Evaluator: Pulled within the last N Days Signed-off-by: Nathan Lowe --- src/core/api/retention.go | 27 ++++- .../retention/policy/rule/dayspl/evaluator.go | 69 ++++++++++++ .../policy/rule/dayspl/evaluator_test.go | 104 ++++++++++++++++++ src/pkg/retention/policy/rule/index/index.go | 17 ++- .../retention/policy/rule/index/index_test.go | 2 +- 5 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 src/pkg/retention/policy/rule/dayspl/evaluator.go create mode 100644 src/pkg/retention/policy/rule/dayspl/evaluator_test.go diff --git a/src/core/api/retention.go b/src/core/api/retention.go index fed34a21b..7e9e113e7 100644 --- a/src/core/api/retention.go +++ b/src/core/api/retention.go @@ -88,14 +88,37 @@ func (r *RetentionAPI) GetMetadatas() { "required": true } ] - }, - { + }, + { "rule_template": "nothing", "display_text": "none", "action": "retain", + "params": [] + }, + { + "rule_template": "always", + "display_text": "always", + "action": "retain", "params": [ + { + "type": "int", + "unit": "COUNT", + "required": true + } ] }, + { + "rule_template": "dayspl", + "display_text": "pulled within the last # days", + "action": "retain", + "params": [ + { + "type": "int", + "unit": "DAYS", + "required": true + } + ] + }, { "rule_template": "daysps", "display_text": "pushed within the last # days", diff --git a/src/pkg/retention/policy/rule/dayspl/evaluator.go b/src/pkg/retention/policy/rule/dayspl/evaluator.go new file mode 100644 index 000000000..9b2fb34e9 --- /dev/null +++ b/src/pkg/retention/policy/rule/dayspl/evaluator.go @@ -0,0 +1,69 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dayspl + +import ( + "time" + + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/retention/policy/action" + "github.com/goharbor/harbor/src/pkg/retention/policy/rule" + "github.com/goharbor/harbor/src/pkg/retention/res" +) + +const ( + // TemplateID of the rule + TemplateID = "nDaysSinceLastPull" + + // ParameterN is the name of the metadata parameter for the N value + ParameterN = TemplateID + + // DefaultN is the default number of days that an artifact must have + // been pulled within to retain the tag or artifact. + DefaultN = 30 +) + +type evaluator struct { + n int +} + +func (e *evaluator) Process(artifacts []*res.Candidate) (result []*res.Candidate, err error) { + minPullTime := time.Now().UTC().Add(time.Duration(-1*24*e.n) * time.Hour).Unix() + for _, a := range artifacts { + if a.PulledTime >= minPullTime { + result = append(result, a) + } + } + + return +} + +func (e *evaluator) Action() string { + return action.Retain +} + +// New constructs a new 'Days Since Last Pull' evaluator +func New(params rule.Parameters) rule.Evaluator { + if params != nil { + if p, ok := params[ParameterN]; ok { + if v, ok := p.(int); ok && v >= 0 { + return &evaluator{n: v} + } + } + } + + log.Debugf("default parameter %d used for rule %s", DefaultN, TemplateID) + return &evaluator{n: DefaultN} +} diff --git a/src/pkg/retention/policy/rule/dayspl/evaluator_test.go b/src/pkg/retention/policy/rule/dayspl/evaluator_test.go new file mode 100644 index 000000000..0c8ba1ec1 --- /dev/null +++ b/src/pkg/retention/policy/rule/dayspl/evaluator_test.go @@ -0,0 +1,104 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dayspl + +import ( + "strconv" + "testing" + "time" + + "github.com/goharbor/harbor/src/pkg/retention/policy/rule" + "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type EvaluatorTestSuite struct { + suite.Suite +} + +func (e *EvaluatorTestSuite) TestNew() { + tests := []struct { + Name string + args rule.Parameters + expectedN int + }{ + {Name: "Valid", args: map[string]rule.Parameter{ParameterN: 5}, expectedN: 5}, + {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterN: -1}, expectedN: DefaultN}, + {Name: "Default If Not Set", args: map[string]rule.Parameter{}, expectedN: DefaultN}, + {Name: "Default If Wrong Type", args: map[string]rule.Parameter{ParameterN: "foo"}, expectedN: DefaultN}, + } + + for _, tt := range tests { + e.T().Run(tt.Name, func(t *testing.T) { + e := New(tt.args).(*evaluator) + + require.Equal(t, tt.expectedN, e.n) + }) + } +} + +func (e *EvaluatorTestSuite) TestProcess() { + now := time.Now().UTC() + data := []*res.Candidate{ + {PulledTime: daysAgo(now, 1)}, + {PulledTime: daysAgo(now, 2)}, + {PulledTime: daysAgo(now, 3)}, + {PulledTime: daysAgo(now, 4)}, + {PulledTime: daysAgo(now, 5)}, + {PulledTime: daysAgo(now, 10)}, + {PulledTime: daysAgo(now, 20)}, + {PulledTime: daysAgo(now, 30)}, + } + + tests := []struct { + n int + expected int + minPullTime int64 + }{ + {n: 0, expected: 0, minPullTime: 0}, + {n: 1, expected: 1, minPullTime: daysAgo(now, 1)}, + {n: 2, expected: 2, minPullTime: daysAgo(now, 2)}, + {n: 3, expected: 3, minPullTime: daysAgo(now, 3)}, + {n: 4, expected: 4, minPullTime: daysAgo(now, 4)}, + {n: 5, expected: 5, minPullTime: daysAgo(now, 5)}, + {n: 15, expected: 6, minPullTime: daysAgo(now, 10)}, + {n: 90, expected: 8, minPullTime: daysAgo(now, 30)}, + } + + for _, tt := range tests { + e.T().Run(strconv.Itoa(tt.n), func(t *testing.T) { + sut := New(map[string]rule.Parameter{ParameterN: tt.n}) + + result, err := sut.Process(data) + + require.NoError(t, err) + require.Len(t, result, tt.expected) + + for _, v := range result { + assert.False(t, v.PulledTime < tt.minPullTime) + } + }) + } +} + +func TestEvaluatorSuite(t *testing.T) { + suite.Run(t, &EvaluatorTestSuite{}) +} + +func daysAgo(from time.Time, n int) int64 { + return from.Add(time.Duration(-1*24*n) * time.Hour).Unix() +} diff --git a/src/pkg/retention/policy/rule/index/index.go b/src/pkg/retention/policy/rule/index/index.go index 0f482baee..40a4cccc0 100644 --- a/src/pkg/retention/policy/rule/index/index.go +++ b/src/pkg/retention/policy/rule/index/index.go @@ -15,17 +15,18 @@ package index import ( - "github.com/goharbor/harbor/src/pkg/retention/policy/rule/nothing" "sync" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/always" + "github.com/goharbor/harbor/src/pkg/retention/policy/rule/dayspl" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/daysps" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/lastx" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestpl" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps" + "github.com/goharbor/harbor/src/pkg/retention/policy/rule/nothing" "github.com/pkg/errors" ) @@ -134,6 +135,20 @@ func init() { Parameters: []*IndexedParam{}, }, always.New) + // Register dayspl + Register(&Metadata{ + TemplateID: dayspl.TemplateID, + Action: action.Retain, + Parameters: []*IndexedParam{ + { + Name: dayspl.ParameterN, + Type: "int", + Unit: "days", + Required: true, + }, + }, + }, dayspl.New) + // Register daysps Register(&Metadata{ TemplateID: daysps.TemplateID, diff --git a/src/pkg/retention/policy/rule/index/index_test.go b/src/pkg/retention/policy/rule/index/index_test.go index 230c9c9fa..b55d29f79 100644 --- a/src/pkg/retention/policy/rule/index/index_test.go +++ b/src/pkg/retention/policy/rule/index/index_test.go @@ -84,7 +84,7 @@ func (suite *IndexTestSuite) TestGet() { // TestIndex tests Index func (suite *IndexTestSuite) TestIndex() { metas := Index() - require.Equal(suite.T(), 7, len(metas)) + require.Equal(suite.T(), 9, len(metas)) assert.Condition(suite.T(), func() bool { for _, m := range metas { if m.TemplateID == "fakeEvaluator" &&