add retention job for deleting repository

Signed-off-by: Steven Zou <szou@vmware.com>

src/jobservice/job/known_jobs.go

@@ -32,4 +32,6 @@ const (
 	ReplicationScheduler = "IMAGE_REPLICATE"
 	// Retention : the name of the retention job
 	Retention = "RETENTION"
+	// RetentionDel is the name of retention deletion job
+	RetentionDel = "RETENTION_DEL"
 )

src/jobservice/runtime/bootstrap.go

@@ -245,6 +245,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(
 			job.Replication:          (*replication.Replication)(nil),
 			job.ReplicationScheduler: (*replication.Scheduler)(nil),
 			job.Retention:            (*retention.Job)(nil),
+			job.RetentionDel:         (*retention.DelRepoJob)(nil),
 		}); err != nil {
 		// exit
 		return nil, err

src/pkg/retention/dep/client.go

@@ -42,6 +42,15 @@ type Client interface {
 	//    error            : common error if any errors occurred
 	GetCandidates(repo *res.Repository) ([]*res.Candidate, error)
 
+	// Delete the given repository
+	//
+	//  Arguments:
+	//    repo *res.Repository : repository info
+	//
+	//  Returns:
+	//    error            : common error if any errors occurred
+	DeleteRepository(repo *res.Repository) error
+
 	// Delete the specified candidate
 	//
 	//  Arguments:
@@ -93,7 +102,7 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
 			return nil, err
 		}
 		for _, image := range images {
-			labels := []string{}
+			labels := make([]string, 0)
 			for _, label := range image.Labels {
 				labels = append(labels, label.Name)
 			}
@@ -115,7 +124,7 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
 			return nil, err
 		}
 		for _, chart := range charts {
-			labels := []string{}
+			labels := make([]string, 0)
 			for _, label := range chart.Labels {
 				labels = append(labels, label.Name)
 			}
@@ -126,8 +135,8 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
 				Tag:          chart.Name,
 				Labels:       labels,
 				CreationTime: chart.Created.Unix(),
-				PushedTime:   time.Now().Unix() - (int64)((rand.Int31n(5)+5)*3600),
-				PulledTime:   time.Now().Unix() - (int64)((rand.Int31n(4))*3600),
+				PushedTime:   time.Now().Unix() - (int64)((rand.Int31n(5)+5)*24*3600),
+				PulledTime:   time.Now().Unix() - (int64)((rand.Int31n(4))*24*3600),
 			}
 			candidates = append(candidates, candidate)
 		}
@@ -137,6 +146,11 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
 	return candidates, nil
 }
 
+// DeleteRepository deletes the specified repository
+func (bc *basicClient) DeleteRepository(repo *res.Repository) error {
+	return errors.New("not implemented")
+}
+
 // Deletes the specified candidate
 func (bc *basicClient) Delete(candidate *res.Candidate) error {
 	if candidate == nil {

src/pkg/retention/job.go

@@ -30,6 +30,12 @@ import (
 	"github.com/pkg/errors"
 )
 
+const (
+	actionMarkRetain   = "RETAIN"
+	actionMarkDeletion = "DEL"
+	actionMarkError    = "ERR"
+)
+
 // Job of running retention process
 type Job struct{}
 
@@ -44,20 +50,14 @@ func (pj *Job) ShouldRetry() bool {
 }
 
 // Validate the parameters
-func (pj *Job) Validate(params job.Parameters) error {
-	if _, err := getParamRepo(params); err != nil {
-		return err
+func (pj *Job) Validate(params job.Parameters) (err error) {
+	if _, err = getParamRepo(params); err == nil {
+		if _, err = getParamMeta(params); err == nil {
+			_, err = getParamDryRun(params)
+		}
 	}
 
-	if _, err := getParamMeta(params); err != nil {
-		return err
-	}
-
-	if _, err := getParamDryRun(params); err != nil {
-		return err
-	}
-
-	return nil
+	return
 }
 
 // Run the job
@@ -125,13 +125,13 @@ func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Re
 	op := func(art *res.Candidate) string {
 		if e, exists := hash[art.Hash()]; exists {
 			if e != nil {
-				return "ERR"
+				return actionMarkError
 			}
 
-			return "DEL"
+			return actionMarkDeletion
 		}
 
-		return "RETAIN"
+		return actionMarkRetain
 	}
 
 	var buf bytes.Buffer

src/pkg/retention/job_del_repo.go

@@ -0,0 +1,112 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package retention
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/jobservice/logger"
+	"github.com/goharbor/harbor/src/pkg/retention/dep"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/olekukonko/tablewriter"
+)
+
+// DelRepoJob tries to delete the whole given repository
+type DelRepoJob struct{}
+
+// MaxFails of the job
+func (drj *DelRepoJob) MaxFails() uint {
+	return 3
+}
+
+// ShouldRetry indicates job can be retried if failed
+func (drj *DelRepoJob) ShouldRetry() bool {
+	return true
+}
+
+// Validate the parameters
+func (drj *DelRepoJob) Validate(params job.Parameters) (err error) {
+	if _, err = getParamRepo(params); err == nil {
+		_, err = getParamDryRun(params)
+	}
+
+	return
+}
+
+// Run the job
+func (drj *DelRepoJob) Run(ctx job.Context, params job.Parameters) error {
+	// logger for logging
+	myLogger := ctx.GetLogger()
+
+	// Parameters have been validated, ignore error checking
+	repo, _ := getParamRepo(params)
+	isDryRun, _ := getParamDryRun(params)
+
+	// Log stage: start
+	repoPath := fmt.Sprintf("%s/%s", repo.Namespace, repo.Name)
+	myLogger.Infof("Run retention process.\n Repository: %s \n Dry Run: %v", repoPath, isDryRun)
+
+	// For printing retention log
+	allArtifacts, err := dep.DefaultClient.GetCandidates(repo)
+	if err != nil {
+		return err
+	}
+
+	// Stop check point:
+	if isStopped(ctx) {
+		logStop(myLogger)
+		return nil
+	}
+
+	// Delete the repository
+	if err := dep.DefaultClient.DeleteRepository(repo); err != nil {
+		return err
+	}
+
+	// Log deletions
+	logDeletions(myLogger, allArtifacts)
+
+	return nil
+}
+
+func logDeletions(logger logger.Interface, all []*res.Candidate) {
+	var buf bytes.Buffer
+
+	data := make([][]string, len(all))
+	for _, c := range all {
+		row := []string{
+			arn(c),
+			c.Kind,
+			strings.Join(c.Labels, ","),
+			t(c.PushedTime),
+			t(c.PulledTime),
+			t(c.CreationTime),
+			actionMarkDeletion,
+		}
+		data = append(data, row)
+	}
+
+	table := tablewriter.NewWriter(&buf)
+	table.SetHeader([]string{"Artifact", "Kind", "labels", "PushedTime", "PulledTime", "CreatedTime", "Retention"})
+	table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
+	table.SetCenterSeparator("|")
+	table.AppendBulk(data)
+	table.Render()
+
+	logger.Infof("\n%s", buf.String())
+}

src/pkg/retention/job_del_repo_test.go

@@ -0,0 +1,69 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package retention
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/pkg/retention/dep"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+// DelRepoJobSuite tests the del repository job
+type DelRepoJobSuite struct {
+	suite.Suite
+
+	oldClient dep.Client
+}
+
+// TestJob is entry of running JobTestSuite
+func TestDelRepoJob(t *testing.T) {
+	suite.Run(t, new(DelRepoJobSuite))
+}
+
+// SetupSuite ...
+func (suite *DelRepoJobSuite) SetupSuite() {
+	suite.oldClient = dep.DefaultClient
+	dep.DefaultClient = &fakeRetentionClient{}
+}
+
+// TearDownSuite ...
+func (suite *DelRepoJobSuite) TearDownSuite() {
+	dep.DefaultClient = suite.oldClient
+}
+
+// TestRun ...
+func (suite *DelRepoJobSuite) TestRun() {
+	params := make(job.Parameters)
+	params[ParamDryRun] = false
+	repository := &res.Repository{
+		Namespace: "library",
+		Name:      "harbor",
+		Kind:      res.Image,
+	}
+	repoJSON, err := repository.ToJSON()
+	require.NoError(suite.T(), err)
+	params[ParamRepo] = repoJSON
+
+	j := &DelRepoJob{}
+	err = j.Validate(params)
+	require.NoError(suite.T(), err)
+
+	err = j.Run(&fakeJobContext{}, params)
+	require.NoError(suite.T(), err)
+}

src/pkg/retention/job_test.go

@@ -16,7 +16,6 @@ package retention
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"testing"
 	"time"
@@ -149,8 +148,8 @@ func (frc *fakeRetentionClient) Delete(candidate *res.Candidate) error {
 }
 
 // SubmitTask ...
-func (frc *fakeRetentionClient) SubmitTask(taskID int64, repository *res.Repository, meta *lwp.Metadata) (string, error) {
-	return "", errors.New("not implemented")
+func (frc *fakeRetentionClient) DeleteRepository(repo *res.Repository) error {
+	return nil
 }
 
 type fakeLogger struct{}

src/pkg/retention/policy/action/index/index.go

@@ -18,7 +18,6 @@ import (
 	"sync"
 
 	"github.com/goharbor/harbor/src/pkg/retention/policy/action"
-
 	"github.com/pkg/errors"
 )
 

src/pkg/retention/policy/action/index/index_test.go

@@ -18,12 +18,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
-
-	"github.com/stretchr/testify/require"
-
 	"github.com/goharbor/harbor/src/pkg/retention/policy/action"
 	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 )
 

src/pkg/retention/policy/alg/index/index.go

@@ -18,9 +18,7 @@ import (
 	"sync"
 
 	"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
-
 	"github.com/goharbor/harbor/src/pkg/retention/policy/alg/or"
-
 	"github.com/pkg/errors"
 )
 

src/pkg/retention/policy/alg/or/processor_test.go

@@ -19,21 +19,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/goharbor/harbor/src/pkg/retention/policy/rule/always"
-
+	"github.com/goharbor/harbor/src/pkg/retention/dep"
 	"github.com/goharbor/harbor/src/pkg/retention/policy/action"
 	"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
+	"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
 	"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
+	"github.com/goharbor/harbor/src/pkg/retention/policy/rule/always"
 	"github.com/goharbor/harbor/src/pkg/retention/policy/rule/lastx"
 	"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
 	"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
 	"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
-
-	"github.com/goharbor/harbor/src/pkg/retention/dep"
-
-	"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
-
-	"github.com/goharbor/harbor/src/pkg/retention/res"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"

src/pkg/retention/policy/lwp/models.go

@@ -18,9 +18,8 @@ package lwp
 import (
 	"encoding/json"
 
-	"github.com/pkg/errors"
-
 	"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
+	"github.com/pkg/errors"
 )
 
 // Metadata contains partial metadata of policy
@@ -38,7 +37,7 @@ type Metadata struct {
 func (m *Metadata) ToJSON() (string, error) {
 	jsonData, err := json.Marshal(m)
 	if err != nil {
-		return "", errors.Wrap(err, "marshal reporitory")
+		return "", errors.Wrap(err, "marshal repository")
 	}
 
 	return string(jsonData), nil