Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix conflicts in runtime/bootstrap.go

This commit is contained in:
Steven Zou 2019-07-30 09:52:22 +08:00
parent a5c31d7160 3ca5116a53
commit 1e7f51615f
12 changed files with 227 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/jobservice/job/known_jobs.go
View File

@ -32,4 +32,6 @@ const (
ReplicationScheduler = "IMAGE_REPLICATE"
// Retention : the name of the retention job
Retention = "RETENTION"
// RetentionDel is the name of retention deletion job
RetentionDel = "RETENTION_DEL"
)

1
src/jobservice/runtime/bootstrap.go
View File

@ -247,6 +247,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(
job.Replication: (*replication.Replication)(nil),
job.ReplicationScheduler: (*replication.Scheduler)(nil),
job.Retention: (*retention.Job)(nil),
job.RetentionDel: (*retention.DelRepoJob)(nil),
scheduler.JobNameScheduler: (*scheduler.PeriodicJob)(nil),
}); err != nil {
// exit

22
src/pkg/retention/dep/client.go
View File

@ -42,6 +42,15 @@ type Client interface {
// error : common error if any errors occurred
GetCandidates(repo *res.Repository) ([]*res.Candidate, error)
// Delete the given repository
//
// Arguments:
// repo *res.Repository : repository info
//
// Returns:
// error : common error if any errors occurred
DeleteRepository(repo *res.Repository) error
// Delete the specified candidate
//
// Arguments:
@ -93,7 +102,7 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
return nil, err
}
for _, image := range images {
labels := []string{}
labels := make([]string, 0)
for _, label := range image.Labels {
labels = append(labels, label.Name)
}
@ -115,7 +124,7 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
return nil, err
}
for _, chart := range charts {
labels := []string{}
labels := make([]string, 0)
for _, label := range chart.Labels {
labels = append(labels, label.Name)
}
@ -126,8 +135,8 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
Tag: chart.Name,
Labels: labels,
CreationTime: chart.Created.Unix(),
PushedTime: time.Now().Unix() - (int64)((rand.Int31n(5)+5)*3600),
PulledTime: time.Now().Unix() - (int64)((rand.Int31n(4))*3600),
PushedTime: time.Now().Unix() - (int64)((rand.Int31n(5)+5)*24*3600),
PulledTime: time.Now().Unix() - (int64)((rand.Int31n(4))*24*3600),
}
candidates = append(candidates, candidate)
}
@ -137,6 +146,11 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
return candidates, nil
}
// DeleteRepository deletes the specified repository
func (bc *basicClient) DeleteRepository(repo *res.Repository) error {
return errors.New("not implemented")
}
// Deletes the specified candidate
func (bc *basicClient) Delete(candidate *res.Candidate) error {
if candidate == nil {

30
src/pkg/retention/job.go
View File

@ -30,6 +30,12 @@ import (
"github.com/pkg/errors"
)
const (
actionMarkRetain = "RETAIN"
actionMarkDeletion = "DEL"
actionMarkError = "ERR"
)
// Job of running retention process
type Job struct{}
@ -44,20 +50,14 @@ func (pj *Job) ShouldRetry() bool {
}
// Validate the parameters
func (pj *Job) Validate(params job.Parameters) error {
if _, err := getParamRepo(params); err != nil {
return err
func (pj *Job) Validate(params job.Parameters) (err error) {
if _, err = getParamRepo(params); err == nil {
if _, err = getParamMeta(params); err == nil {
_, err = getParamDryRun(params)
}
}
if _, err := getParamMeta(params); err != nil {
return err
}
if _, err := getParamDryRun(params); err != nil {
return err
}
return nil
return
}
// Run the job
@ -125,13 +125,13 @@ func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Re
op := func(art *res.Candidate) string {
if e, exists := hash[art.Hash()]; exists {
if e != nil {
return "ERR"
return actionMarkError
}
return "DEL"
return actionMarkDeletion
}
return "RETAIN"
return actionMarkRetain
}
var buf bytes.Buffer

112
src/pkg/retention/job_del_repo.go Normal file
View File

@ -0,0 +1,112 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package retention
import (
"bytes"
"fmt"
"strings"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/res"
"github.com/olekukonko/tablewriter"
)
// DelRepoJob tries to delete the whole given repository
type DelRepoJob struct{}
// MaxFails of the job
func (drj *DelRepoJob) MaxFails() uint {
return 3
}
// ShouldRetry indicates job can be retried if failed
func (drj *DelRepoJob) ShouldRetry() bool {
return true
}
// Validate the parameters
func (drj *DelRepoJob) Validate(params job.Parameters) (err error) {
if _, err = getParamRepo(params); err == nil {
_, err = getParamDryRun(params)
}
return
}
// Run the job
func (drj *DelRepoJob) Run(ctx job.Context, params job.Parameters) error {
// logger for logging
myLogger := ctx.GetLogger()
// Parameters have been validated, ignore error checking
repo, _ := getParamRepo(params)
isDryRun, _ := getParamDryRun(params)
// Log stage: start
repoPath := fmt.Sprintf("%s/%s", repo.Namespace, repo.Name)
myLogger.Infof("Run retention process.\n Repository: %s \n Dry Run: %v", repoPath, isDryRun)
// For printing retention log
allArtifacts, err := dep.DefaultClient.GetCandidates(repo)
if err != nil {
return err
}
// Stop check point:
if isStopped(ctx) {
logStop(myLogger)
return nil
}
// Delete the repository
if err := dep.DefaultClient.DeleteRepository(repo); err != nil {
return err
}
// Log deletions
logDeletions(myLogger, allArtifacts)
return nil
}
func logDeletions(logger logger.Interface, all []*res.Candidate) {
var buf bytes.Buffer
data := make([][]string, len(all))
for _, c := range all {
row := []string{
arn(c),
c.Kind,
strings.Join(c.Labels, ","),
t(c.PushedTime),
t(c.PulledTime),
t(c.CreationTime),
actionMarkDeletion,
}
data = append(data, row)
}
table := tablewriter.NewWriter(&buf)
table.SetHeader([]string{"Artifact", "Kind", "labels", "PushedTime", "PulledTime", "CreatedTime", "Retention"})
table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
table.SetCenterSeparator("|")
table.AppendBulk(data)
table.Render()
logger.Infof("\n%s", buf.String())
}

69
src/pkg/retention/job_del_repo_test.go Normal file
View File

@ -0,0 +1,69 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package retention
import (
"testing"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/res"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// DelRepoJobSuite tests the del repository job
type DelRepoJobSuite struct {
suite.Suite
oldClient dep.Client
}
// TestJob is entry of running JobTestSuite
func TestDelRepoJob(t *testing.T) {
suite.Run(t, new(DelRepoJobSuite))
}
// SetupSuite ...
func (suite *DelRepoJobSuite) SetupSuite() {
suite.oldClient = dep.DefaultClient
dep.DefaultClient = &fakeRetentionClient{}
}
// TearDownSuite ...
func (suite *DelRepoJobSuite) TearDownSuite() {
dep.DefaultClient = suite.oldClient
}
// TestRun ...
func (suite *DelRepoJobSuite) TestRun() {
params := make(job.Parameters)
params[ParamDryRun] = false
repository := &res.Repository{
Namespace: "library",
Name: "harbor",
Kind: res.Image,
}
repoJSON, err := repository.ToJSON()
require.NoError(suite.T(), err)
params[ParamRepo] = repoJSON
j := &DelRepoJob{}
err = j.Validate(params)
require.NoError(suite.T(), err)
err = j.Run(&fakeJobContext{}, params)
require.NoError(suite.T(), err)
}

5
src/pkg/retention/job_test.go
View File

@ -16,7 +16,6 @@ package retention
import (
"context"
"errors"
"fmt"
"testing"
"time"
@ -149,8 +148,8 @@ func (frc *fakeRetentionClient) Delete(candidate *res.Candidate) error {
}
// SubmitTask ...
func (frc *fakeRetentionClient) SubmitTask(taskID int64, repository *res.Repository, meta *lwp.Metadata) (string, error) {
return "", errors.New("not implemented")
func (frc *fakeRetentionClient) DeleteRepository(repo *res.Repository) error {
return nil
}
type fakeLogger struct{}

1
src/pkg/retention/policy/action/index/index.go
View File

@ -18,7 +18,6 @@ import (
"sync"
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
"github.com/pkg/errors"
)

6
src/pkg/retention/policy/action/index/index_test.go
View File

@ -18,12 +18,10 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
"github.com/goharbor/harbor/src/pkg/retention/res"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)

2
src/pkg/retention/policy/alg/index/index.go
View File

@ -18,9 +18,7 @@ import (
"sync"
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
"github.com/goharbor/harbor/src/pkg/retention/policy/alg/or"
"github.com/pkg/errors"
)

12
src/pkg/retention/policy/alg/or/processor_test.go
View File

@ -19,21 +19,17 @@ import (
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/always"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/always"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/lastx"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps"
"github.com/goharbor/harbor/src/pkg/retention/res"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
"github.com/goharbor/harbor/src/pkg/retention/res"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"

5
src/pkg/retention/policy/lwp/models.go
View File

@ -18,9 +18,8 @@ package lwp
import (
"encoding/json"
"github.com/pkg/errors"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/pkg/errors"
)
// Metadata contains partial metadata of policy
@ -38,7 +37,7 @@ type Metadata struct {
func (m *Metadata) ToJSON() (string, error) {
jsonData, err := json.Marshal(m)
if err != nil {
return "", errors.Wrap(err, "marshal reporitory")
return "", errors.Wrap(err, "marshal repository")
}
return string(jsonData), nil