Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-23 10:45:45 +01:00
Code Issues Projects Releases Wiki Activity

Create retention client with the configurations got from jobservice config

Browse Source 
As the retention client is only used in job which running in jobservice, the configurations that needed to init the client should be read from jobservice

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2019-07-24 13:10:10 +08:00
parent c4836727be
commit 1adf5c976e
10 changed files with 89 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/jobservice/config/config.go
View File

@ -24,7 +24,7 @@ import (
"strings"
"github.com/goharbor/harbor/src/jobservice/common/utils"
"gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v2"
)
const (
@ -37,6 +37,7 @@ const (
jobServiceRedisURL = "JOB_SERVICE_POOL_REDIS_URL"
jobServiceRedisNamespace = "JOB_SERVICE_POOL_REDIS_NAMESPACE"
jobServiceAuthSecret = "JOBSERVICE_SECRET"
coreURL = "CORE_URL"
// JobServiceProtocolHTTPS points to the 'https' protocol
JobServiceProtocolHTTPS = "https"
@ -163,6 +164,11 @@ func GetAuthSecret() string {
return utils.ReadEnv(jobServiceAuthSecret)
}
// GetCoreURL get the core url from the env
func GetCoreURL() string {
return utils.ReadEnv(coreURL)
}
// GetUIAuthSecret get the auth secret of UI side
func GetUIAuthSecret() string {
return utils.ReadEnv(uiAuthSecret)

7
src/jobservice/config/config_test.go
View File

@ -14,11 +14,12 @@
package config
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"os"
"testing"
)
// ConfigurationTestSuite tests the configuration loading
@ -84,6 +85,7 @@ func (suite *ConfigurationTestSuite) TestConfigLoadingWithEnv() {
)
assert.Equal(suite.T(), "js_secret", GetAuthSecret(), "expect auth secret 'js_secret' but got '%s'", GetAuthSecret())
assert.Equal(suite.T(), "core_secret", GetUIAuthSecret(), "expect auth secret 'core_secret' but got '%s'", GetUIAuthSecret())
assert.Equal(suite.T(), "core_url", GetCoreURL(), "expect core url 'core_url' but got '%s'", GetCoreURL())
}
// TestDefaultConfig ...
@ -134,6 +136,7 @@ func setENV() error {
err = os.Setenv("JOB_SERVICE_POOL_REDIS_NAMESPACE", "ut_namespace")
err = os.Setenv("JOBSERVICE_SECRET", "js_secret")
err = os.Setenv("CORE_SECRET", "core_secret")
err = os.Setenv("CORE_URL", "core_url")
return err
}

3
src/jobservice/main.go
View File

@ -19,7 +19,6 @@ import (
"errors"
"flag"
"fmt"
"os"
"github.com/goharbor/harbor/src/common"
comcfg "github.com/goharbor/harbor/src/common/config"
@ -64,7 +63,7 @@ func main() {
if utils.IsEmptyStr(secret) {
return nil, errors.New("empty auth secret")
}
coreURL := os.Getenv("CORE_URL")
coreURL := config.GetCoreURL()
configURL := coreURL + common.CoreConfigPath
cfgMgr := comcfg.NewRESTCfgManager(configURL, secret)
jobCtx := impl.NewContext(ctx, cfgMgr)

4
src/pkg/retention/boot.go
View File

@ -14,14 +14,10 @@
package retention
import "github.com/goharbor/harbor/src/pkg/retention/dep"
// TODO: Move to api.Init()
// Init the retention components
func Init() error {
// New default retention client
dep.DefaultClient = dep.NewClient()
return nil
}

64
src/pkg/retention/dep/client.go
View File

@ -17,29 +17,16 @@ package dep
import (
"errors"
"fmt"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
"net/http"
"github.com/goharbor/harbor/src/common/http/modifier/auth"
cjob "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/config"
"github.com/goharbor/harbor/src/pkg/clients/core"
"github.com/goharbor/harbor/src/pkg/retention/res"
)
const (
// ParamRepo ...
ParamRepo = "repository"
// ParamMeta ...
ParamMeta = "liteMeta"
)
// TODO: Move to api.Base
// DefaultClient for the retention
var DefaultClient Client
var DefaultClient = NewClient()
// Client is designed to access core service to get required infos
type Client interface {
@ -61,18 +48,6 @@ type Client interface {
// Returns:
// error : common error if any errors occurred
Delete(candidate *res.Candidate) error
// SubmitTask to jobservice
//
// Arguments:
// taskID : the ID of task
// repository *res.Repository : repository info
// meta *lwp.Metadata : policy lightweight metadata
//
// Returns:
// string : the job ID
// error : common error if any errors occurred
SubmitTask(taskID int64, repository *res.Repository, meta *lwp.Metadata) (string, error)
}
// NewClient new a basic client
@ -86,28 +61,21 @@ func NewClient(client ...*http.Client) Client {
}
// init core client
internalCoreURL := config.InternalCoreURL()
jobserviceSecret := config.JobserviceSecret()
internalCoreURL := config.GetCoreURL()
jobserviceSecret := config.GetAuthSecret()
authorizer := auth.NewSecretAuthorizer(jobserviceSecret)
coreClient := core.New(internalCoreURL, c, authorizer)
// init jobservice client
internalJobserviceURL := config.InternalJobServiceURL()
coreSecret := config.CoreSecret()
jobserviceClient := cjob.NewDefaultClient(internalJobserviceURL, coreSecret)
return &basicClient{
internalCoreURL: internalCoreURL,
coreClient: coreClient,
jobserviceClient: jobserviceClient,
internalCoreURL: internalCoreURL,
coreClient: coreClient,
}
}
// basicClient is a default
type basicClient struct {
internalCoreURL string
coreClient core.Client
jobserviceClient cjob.Client
internalCoreURL string
coreClient core.Client
}
// GetCandidates gets the tag candidates under the repository
@ -183,19 +151,3 @@ func (bc *basicClient) Delete(candidate *res.Candidate) error {
return fmt.Errorf("unsupported candidate kind: %s", candidate.Kind)
}
}
// SubmitTask to jobservice
func (bc *basicClient) SubmitTask(taskID int64, repository *res.Repository, meta *lwp.Metadata) (string, error) {
j := &models.JobData{
Metadata: &models.JobMetadata{
JobKind: job.KindGeneric,
},
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/retention/tasks/%d", bc.internalCoreURL, taskID),
}
j.Name = job.Retention
j.Parameters = map[string]interface{}{
ParamRepo: repository,
ParamMeta: meta,
}
return bc.jobserviceClient.SubmitJob(j)
}

8
src/pkg/retention/dep/client_test.go
View File

@ -129,14 +129,6 @@ func (c *clientTestSuite) TestDelete() {
require.NotNil(c.T(), err)
}
func (c *clientTestSuite) TestSubmitTask() {
client := &basicClient{}
client.jobserviceClient = &fakeJobserviceClient{}
jobID, err := client.SubmitTask(1, nil, nil)
require.Nil(c.T(), err)
assert.Equal(c.T(), "1", jobID)
}
func TestClientTestSuite(t *testing.T) {
suite.Run(t, new(clientTestSuite))
}

12
src/pkg/retention/job.go
View File

@ -205,28 +205,28 @@ func logError(logger logger.Interface, err error) error {
}
func getParamRepo(params job.Parameters) (*res.Repository, error) {
v, ok := params[dep.ParamRepo]
v, ok := params[ParamRepo]
if !ok {
return nil, errors.Errorf("missing parameter: %s", dep.ParamRepo)
return nil, errors.Errorf("missing parameter: %s", ParamRepo)
}
repo, ok := v.(*res.Repository)
if !ok {
return nil, errors.Errorf("invalid parameter: %s", dep.ParamRepo)
return nil, errors.Errorf("invalid parameter: %s", ParamRepo)
}
return repo, nil
}
func getParamMeta(params job.Parameters) (*lwp.Metadata, error) {
v, ok := params[dep.ParamMeta]
v, ok := params[ParamMeta]
if !ok {
return nil, errors.Errorf("missing parameter: %s", dep.ParamMeta)
return nil, errors.Errorf("missing parameter: %s", ParamMeta)
}
meta, ok := v.(*lwp.Metadata)
if !ok {
return nil, errors.Errorf("invalid parameter: %s", dep.ParamMeta)
return nil, errors.Errorf("invalid parameter: %s", ParamMeta)
}
return meta, nil

37
src/pkg/retention/job_test.go
View File

@ -21,32 +21,21 @@ import (
"testing"
"time"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
"github.com/goharbor/harbor/src/pkg/retention/policy/alg/or"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
"github.com/stretchr/testify/require"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk"
"github.com/goharbor/harbor/src/pkg/retention/res"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
@ -87,7 +76,7 @@ func (suite *JobTestSuite) TearDownSuite() {
func (suite *JobTestSuite) TestRunSuccess() {
params := make(job.Parameters)
params[dep.ParamRepo] = &res.Repository{
params[ParamRepo] = &res.Repository{
Namespace: "library",
Name: "harbor",
Kind: res.Image,
@ -103,7 +92,7 @@ func (suite *JobTestSuite) TestRunSuccess() {
ruleParams := make(rule.Parameters)
ruleParams[latestk.ParameterK] = 10
params[dep.ParamMeta] = &lwp.Metadata{
params[ParamMeta] = &lwp.Metadata{
Algorithm: policy.AlgorithmOR,
Rules: []*rule.Metadata{
{

48
src/pkg/retention/launcher.go
View File

@ -17,19 +17,28 @@ package retention
import (
"fmt"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
cjob "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
"github.com/goharbor/harbor/src/pkg/retention/res"
"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
"github.com/pkg/errors"
)
const (
// ParamRepo ...
ParamRepo = "repository"
// ParamMeta ...
ParamMeta = "liteMeta"
)
// Launcher provides function to launch the async jobs to run retentions based on the provided policy.
type Launcher interface {
// Launch async jobs for the retention policy
@ -47,20 +56,22 @@ type Launcher interface {
// NewLauncher returns an instance of Launcher
func NewLauncher(projectMgr project.Manager, repositoryMgr repository.Manager,
retentionMgr Manager, retentionClient dep.Client) Launcher {
retentionMgr Manager) Launcher {
return &launcher{
projectMgr: projectMgr,
repositoryMgr: repositoryMgr,
retentionMgr: retentionMgr,
retentionClient: retentionClient,
projectMgr: projectMgr,
repositoryMgr: repositoryMgr,
retentionMgr: retentionMgr,
jobserviceClient: cjob.GlobalClient,
internalCoreURL: config.InternalCoreURL(),
}
}
type launcher struct {
retentionMgr Manager
retentionClient dep.Client
projectMgr project.Manager
repositoryMgr repository.Manager
retentionMgr Manager
projectMgr project.Manager
repositoryMgr repository.Manager
jobserviceClient cjob.Client
internalCoreURL string
}
type jobData struct {
@ -174,7 +185,18 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64) (int64, error
allFailed := true
for _, jobData := range jobDatas {
_, err := l.retentionClient.SubmitTask(jobData.taskID, jobData.repository, jobData.policy)
j := &models.JobData{
Metadata: &models.JobMetadata{
JobKind: job.KindGeneric,
},
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/retention/tasks/%d", l.internalCoreURL, jobData.taskID),
}
j.Name = job.Retention
j.Parameters = map[string]interface{}{
ParamRepo: jobData.repository,
ParamMeta: jobData.policy,
}
_, err := l.jobserviceClient.SubmitJob(j)
if err != nil {
log.Error(launcherError(fmt.Errorf("failed to submit task %d: %v", jobData.taskID, err)))
continue

40
src/pkg/retention/launcher_test.go
View File

@ -16,20 +16,18 @@ package retention
import (
"fmt"
"strconv"
"testing"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/goharbor/harbor/src/pkg/retention/q"
"github.com/goharbor/harbor/src/pkg/retention/res"
_ "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
hjob "github.com/goharbor/harbor/src/testing/job"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -76,22 +74,6 @@ func (f *fakeRepositoryManager) ListChartRepositories(projectID int64) ([]*chart
return f.chartRepositories, nil
}
type fakeClient struct {
id int
}
func (f *fakeClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, error) {
return nil, nil
}
func (f *fakeClient) Delete(candidate *res.Candidate) error {
return nil
}
func (f *fakeClient) SubmitTask(taskID int64, repository *res.Repository, meta *lwp.Metadata) (string, error) {
f.id++
return strconv.Itoa(f.id), nil
}
type fakeRetentionManager struct{}
func (f *fakeRetentionManager) CreatePolicy(p *policy.Metadata) (int64, error) {
@ -139,10 +121,10 @@ func (f *fakeRetentionManager) ListHistories(executionID int64, query *q.Query)
type launchTestSuite struct {
suite.Suite
projectMgr project.Manager
repositoryMgr repository.Manager
retentionMgr Manager
retentionClient dep.Client
projectMgr project.Manager
repositoryMgr repository.Manager
retentionMgr Manager
jobserviceClient job.Client
}
func (l *launchTestSuite) SetupTest() {
@ -167,7 +149,7 @@ func (l *launchTestSuite) SetupTest() {
},
}
l.retentionMgr = &fakeRetentionManager{}
l.retentionClient = &fakeClient{}
l.jobserviceClient = &hjob.MockJobClient{}
}
func (l *launchTestSuite) TestGetProjects() {
@ -191,7 +173,13 @@ func (l *launchTestSuite) TestGetRepositories() {
}
func (l *launchTestSuite) TestLaunch() {
launcher := NewLauncher(l.projectMgr, l.repositoryMgr, l.retentionMgr, l.retentionClient)
launcher := &launcher{
projectMgr: l.projectMgr,
repositoryMgr: l.repositoryMgr,
retentionMgr: l.retentionMgr,
jobserviceClient: l.jobserviceClient,
}
var ply *policy.Metadata
// nil policy
n, err := launcher.Launch(ply, 1)