diff --git a/src/core/api/base.go b/src/core/api/base.go index 7e888d077..98f250e68 100644 --- a/src/core/api/base.go +++ b/src/core/api/base.go @@ -127,7 +127,7 @@ func Init() error { } return errors.New("bad retention callback param") } - err := scheduler.Register(retention.RetentionSchedulerCallback, callbackFun) + err := scheduler.Register(retention.SchedulerCallback, callbackFun) return err } diff --git a/src/core/main.go b/src/core/main.go index c8edce1b6..c8ac1997e 100644 --- a/src/core/main.go +++ b/src/core/main.go @@ -17,7 +17,6 @@ package main import ( "encoding/gob" "fmt" - "github.com/goharbor/harbor/src/pkg/retention" "os" "os/signal" "strconv" @@ -166,16 +165,11 @@ func main() { log.Infof("Because SYNC_REGISTRY set false , no need to sync registry \n") } - // Initialize retention - log.Info("Initialize retention") - if err := retention.Init(); err != nil { - log.Fatalf("Failed to initialize retention with error: %s", err) - } - log.Info("Init proxy") if err := middlewares.Init(); err != nil { log.Errorf("init proxy error, %v", err) } + // go proxy.StartProxy() beego.Run() } diff --git a/src/pkg/retention/boot.go b/src/pkg/retention/boot.go deleted file mode 100644 index 1f78d179d..000000000 --- a/src/pkg/retention/boot.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package retention - -// TODO: Move to api.Init() - -// Init the retention components -func Init() error { - - return nil -} diff --git a/src/pkg/retention/controller.go b/src/pkg/retention/controller.go index fcf7f84c1..53f0088b8 100644 --- a/src/pkg/retention/controller.go +++ b/src/pkg/retention/controller.go @@ -16,13 +16,14 @@ package retention import ( "fmt" + "time" + "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/repository" "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/retention/q" "github.com/goharbor/harbor/src/pkg/scheduler" - "time" ) // APIController to handle the requests related with retention @@ -66,8 +67,8 @@ type DefaultAPIController struct { } const ( - // RetentionSchedulerCallback ... - RetentionSchedulerCallback = "RetentionSchedulerCallback" + // SchedulerCallback ... + SchedulerCallback = "SchedulerCallback" ) // TriggerParam ... @@ -87,7 +88,7 @@ func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) error { if p.Trigger.Settings != nil { cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron] if ok { - jobid, err := r.scheduler.Schedule(cron.(string), RetentionSchedulerCallback, TriggerParam{ + jobid, err := r.scheduler.Schedule(cron.(string), SchedulerCallback, TriggerParam{ PolicyID: p.ID, Trigger: ExecutionTriggerSchedule, }) @@ -138,7 +139,7 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error { case "": default: - return fmt.Errorf("Not support Trigger %s", p.Trigger.Kind) + return fmt.Errorf("not support Trigger %s", p.Trigger.Kind) } } if needUn { @@ -148,7 +149,7 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error { } } if needSch { - jobid, err := r.scheduler.Schedule(p.Trigger.Settings[policy.TriggerSettingsCron].(string), RetentionSchedulerCallback, TriggerParam{ + jobid, err := r.scheduler.Schedule(p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{ PolicyID: p.ID, Trigger: ExecutionTriggerSchedule, }) @@ -192,8 +193,7 @@ func (r *DefaultAPIController) TriggerRetentionExec(policyID int64, trigger stri DryRun: dryRun, } id, err := r.manager.CreateExecution(exec) - // TODO launcher with DryRun param - num, err := r.launcher.Launch(p, id) + num, err := r.launcher.Launch(p, id, dryRun) if err != nil { return err } @@ -218,21 +218,21 @@ func (r *DefaultAPIController) OperateRetentionExec(eid int64, action string) er if err != nil { return err } - exec := &Execution{} + switch action { case "stop": if e.Status != ExecutionStatusInProgress { - return fmt.Errorf("Can't abort, current status is %s", e.Status) + return fmt.Errorf("cannot abort, current status is %s", e.Status) } - exec.ID = eid - exec.Status = ExecutionStatusStopped - exec.EndTime = time.Now() - // TODO stop the execution + + e.Status = ExecutionStatusStopped + e.EndTime = time.Now() + // TODO: STOP THE EXECUTION default: return fmt.Errorf("not support action %s", action) } - return r.manager.UpdateExecution(exec) + return r.manager.UpdateExecution(e) } // ListRetentionExecs List Retention Executions diff --git a/src/pkg/retention/dep/client.go b/src/pkg/retention/dep/client.go index 9d97f3451..2d1330027 100644 --- a/src/pkg/retention/dep/client.go +++ b/src/pkg/retention/dep/client.go @@ -17,7 +17,9 @@ package dep import ( "errors" "fmt" + "math/rand" "net/http" + "time" "github.com/goharbor/harbor/src/common/http/modifier/auth" "github.com/goharbor/harbor/src/jobservice/config" @@ -102,9 +104,8 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida Tag: image.Name, Labels: labels, CreationTime: image.Created.Unix(), - // TODO: populate the pull/push time - // PulledTime: , - // PushedTime:, + PulledTime: time.Now().Unix() - (int64)(rand.Int31n(4)*3600), + PushedTime: time.Now().Unix() - (int64)((rand.Int31n(5)+5)*3600), } candidates = append(candidates, candidate) } @@ -125,9 +126,8 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida Tag: chart.Name, Labels: labels, CreationTime: chart.Created.Unix(), - // TODO: populate the pull/push time - // PulledTime: , - // PushedTime:, + PushedTime: time.Now().Unix() - (int64)((rand.Int31n(5)+5)*3600), + PulledTime: time.Now().Unix() - (int64)((rand.Int31n(4))*3600), } candidates = append(candidates, candidate) } diff --git a/src/pkg/retention/job.go b/src/pkg/retention/job.go index d209507c5..3fb782749 100644 --- a/src/pkg/retention/job.go +++ b/src/pkg/retention/job.go @@ -16,14 +16,14 @@ package retention import ( "bytes" - "encoding/json" "fmt" "strings" "time" + "github.com/goharbor/harbor/src/pkg/retention/dep" + "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/logger" - "github.com/goharbor/harbor/src/pkg/retention/dep" "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/retention/policy/lwp" "github.com/goharbor/harbor/src/pkg/retention/res" @@ -32,11 +32,7 @@ import ( ) // Job of running retention process -type Job struct { - // client used to talk to core - // TODO: REFER THE GLOBAL CLIENT - client dep.Client -} +type Job struct{} // MaxFails of the job func (pj *Job) MaxFails() uint { @@ -58,6 +54,10 @@ func (pj *Job) Validate(params job.Parameters) error { return err } + if _, err := getParamDryRun(params); err != nil { + return err + } + return nil } @@ -69,6 +69,7 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error { // Parameters have been validated, ignore error checking repo, _ := getParamRepo(params) liteMeta, _ := getParamMeta(params) + isDryRun, _ := getParamDryRun(params) // Log stage: start repoPath := fmt.Sprintf("%s/%s", repo.Namespace, repo.Name) @@ -81,7 +82,7 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error { } // Retrieve all the candidates under the specified repository - allCandidates, err := pj.client.GetCandidates(repo) + allCandidates, err := dep.DefaultClient.GetCandidates(repo) if err != nil { return logError(myLogger, err) } @@ -91,7 +92,7 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error { // Build the processor builder := policy.NewBuilder(allCandidates) - processor, err := builder.Build(liteMeta) + processor, err := builder.Build(liteMeta, isDryRun) if err != nil { return logError(myLogger, err) } @@ -111,16 +112,6 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error { // Log stage: results with table view logResults(myLogger, allCandidates, results) - // Check in the results - bytes, err := json.Marshal(results) - if err != nil { - return logError(myLogger, err) - } - - if err := ctx.Checkin(string(bytes)); err != nil { - return logError(myLogger, err) - } - return nil } @@ -146,7 +137,7 @@ func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Re var buf bytes.Buffer - data := [][]string{} + data := make([][]string, len(all)) for _, c := range all { row := []string{ @@ -204,6 +195,20 @@ func logError(logger logger.Interface, err error) error { return wrappedErr } +func getParamDryRun(params job.Parameters) (bool, error) { + v, ok := params[ParamDryRun] + if !ok { + return false, errors.Errorf("missing parameter: %s", ParamDryRun) + } + + dryRun, ok := v.(bool) + if !ok { + return false, errors.Errorf("invalid parameter: %s", ParamDryRun) + } + + return dryRun, nil +} + func getParamRepo(params job.Parameters) (*res.Repository, error) { v, ok := params[ParamRepo] if !ok { diff --git a/src/pkg/retention/job_test.go b/src/pkg/retention/job_test.go index 78656330a..70a690687 100644 --- a/src/pkg/retention/job_test.go +++ b/src/pkg/retention/job_test.go @@ -76,6 +76,7 @@ func (suite *JobTestSuite) TearDownSuite() { func (suite *JobTestSuite) TestRunSuccess() { params := make(job.Parameters) + params[ParamDryRun] = false params[ParamRepo] = &res.Repository{ Namespace: "library", Name: "harbor", @@ -115,9 +116,7 @@ func (suite *JobTestSuite) TestRunSuccess() { }, } - j := &Job{ - client: &fakeRetentionClient{}, - } + j := &Job{} err := j.Validate(params) require.NoError(suite.T(), err) diff --git a/src/pkg/retention/launcher.go b/src/pkg/retention/launcher.go index fd9593607..2f931f4af 100644 --- a/src/pkg/retention/launcher.go +++ b/src/pkg/retention/launcher.go @@ -37,6 +37,8 @@ const ( ParamRepo = "repository" // ParamMeta ... ParamMeta = "liteMeta" + // ParamDryRun ... + ParamDryRun = "dryRun" ) // Launcher provides function to launch the async jobs to run retentions based on the provided policy. @@ -47,11 +49,12 @@ type Launcher interface { // Arguments: // policy *policy.Metadata: the policy info // executionID int64 : the execution ID + // isDryRun bool : indicate if it is a dry run // // Returns: // int64 : the count of tasks // error : common error if any errors occurred - Launch(policy *policy.Metadata, executionID int64) (int64, error) + Launch(policy *policy.Metadata, executionID int64, isDryRun bool) (int64, error) } // NewLauncher returns an instance of Launcher @@ -80,7 +83,7 @@ type jobData struct { taskID int64 } -func (l *launcher) Launch(ply *policy.Metadata, executionID int64) (int64, error) { +func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool) (int64, error) { if ply == nil { return 0, launcherError(fmt.Errorf("the policy is nil")) } @@ -193,8 +196,9 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64) (int64, error } j.Name = job.Retention j.Parameters = map[string]interface{}{ - ParamRepo: jobData.repository, - ParamMeta: jobData.policy, + ParamRepo: jobData.repository, + ParamMeta: jobData.policy, + ParamDryRun: isDryRun, } _, err := l.jobserviceClient.SubmitJob(j) if err != nil { diff --git a/src/pkg/retention/launcher_test.go b/src/pkg/retention/launcher_test.go index c5944ed78..b8f741dec 100644 --- a/src/pkg/retention/launcher_test.go +++ b/src/pkg/retention/launcher_test.go @@ -182,12 +182,12 @@ func (l *launchTestSuite) TestLaunch() { var ply *policy.Metadata // nil policy - n, err := launcher.Launch(ply, 1) + n, err := launcher.Launch(ply, 1, false) require.NotNil(l.T(), err) // nil rules ply = &policy.Metadata{} - n, err = launcher.Launch(ply, 1) + n, err = launcher.Launch(ply, 1, false) require.Nil(l.T(), err) assert.Equal(l.T(), int64(0), n) @@ -197,7 +197,7 @@ func (l *launchTestSuite) TestLaunch() { {}, }, } - _, err = launcher.Launch(ply, 1) + _, err = launcher.Launch(ply, 1, false) require.NotNil(l.T(), err) // system scope @@ -226,7 +226,7 @@ func (l *launchTestSuite) TestLaunch() { }, }, } - n, err = launcher.Launch(ply, 1) + n, err = launcher.Launch(ply, 1, false) require.Nil(l.T(), err) assert.Equal(l.T(), int64(2), n) } diff --git a/src/pkg/retention/policy/action/index.go b/src/pkg/retention/policy/action/index.go index 012d6f2bf..40bf8507f 100644 --- a/src/pkg/retention/policy/action/index.go +++ b/src/pkg/retention/policy/action/index.go @@ -15,8 +15,9 @@ package action import ( - "github.com/pkg/errors" "sync" + + "github.com/pkg/errors" ) // index for keeping the mapping action and its performer @@ -33,7 +34,7 @@ func Register(action string, factory PerformerFactory) { } // Get performer with the provided action -func Get(action string, params interface{}) (Performer, error) { +func Get(action string, params interface{}, isDryRun bool) (Performer, error) { if len(action) == 0 { return nil, errors.New("empty action") } @@ -48,5 +49,5 @@ func Get(action string, params interface{}) (Performer, error) { return nil, errors.Errorf("invalid action performer registered for action %s", action) } - return factory(params), nil + return factory(params, isDryRun), nil } diff --git a/src/pkg/retention/policy/action/index_test.go b/src/pkg/retention/policy/action/index_test.go index 6119ef05b..b1c8bf596 100644 --- a/src/pkg/retention/policy/action/index_test.go +++ b/src/pkg/retention/policy/action/index_test.go @@ -54,7 +54,7 @@ func (suite *IndexTestSuite) SetupSuite() { // TestRegister tests register func (suite *IndexTestSuite) TestGet() { - p, err := Get("fakeAction", nil) + p, err := Get("fakeAction", nil, false) require.NoError(suite.T(), err) require.NotNil(suite.T(), p) @@ -72,7 +72,10 @@ func (suite *IndexTestSuite) TestGet() { }) } -type fakePerformer struct{} +type fakePerformer struct { + parameters interface{} + isDryRun bool +} // Perform the artifacts func (p *fakePerformer) Perform(candidates []*res.Candidate) (results []*res.Result, err error) { @@ -85,6 +88,9 @@ func (p *fakePerformer) Perform(candidates []*res.Candidate) (results []*res.Res return } -func newFakePerformer(params interface{}) Performer { - return &fakePerformer{} +func newFakePerformer(params interface{}, isDryRun bool) Performer { + return &fakePerformer{ + parameters: params, + isDryRun: isDryRun, + } } diff --git a/src/pkg/retention/policy/action/performer.go b/src/pkg/retention/policy/action/performer.go index d16a0b2fe..b1ae8c13a 100644 --- a/src/pkg/retention/policy/action/performer.go +++ b/src/pkg/retention/policy/action/performer.go @@ -21,7 +21,7 @@ import ( const ( // Retain artifacts - Retain = "retain" + Retain = "Retain" ) // Performer performs the related actions targeting the candidates @@ -38,11 +38,13 @@ type Performer interface { } // PerformerFactory is factory method for creating Performer -type PerformerFactory func(params interface{}) Performer +type PerformerFactory func(params interface{}, isDryRun bool) Performer // retainAction make sure all the candidates will be retained and others will be cleared type retainAction struct { all []*res.Candidate + // Indicate if it is a dry run + isDryRun bool } // Perform the action @@ -60,8 +62,10 @@ func (ra *retainAction) Perform(candidates []*res.Candidate) (results []*res.Res Target: art, } - if err := dep.DefaultClient.Delete(art); err != nil { - result.Error = err + if !ra.isDryRun { + if err := dep.DefaultClient.Delete(art); err != nil { + result.Error = err + } } results = append(results, result) @@ -73,17 +77,19 @@ func (ra *retainAction) Perform(candidates []*res.Candidate) (results []*res.Res } // NewRetainAction is factory method for RetainAction -func NewRetainAction(params interface{}) Performer { +func NewRetainAction(params interface{}, isDryRun bool) Performer { if params != nil { if all, ok := params.([]*res.Candidate); ok { return &retainAction{ - all: all, + all: all, + isDryRun: isDryRun, } } } return &retainAction{ - all: make([]*res.Candidate, 0), + all: make([]*res.Candidate, 0), + isDryRun: isDryRun, } } diff --git a/src/pkg/retention/policy/alg/or/processor_test.go b/src/pkg/retention/policy/alg/or/processor_test.go index fa0bb63cc..d990b625a 100644 --- a/src/pkg/retention/policy/alg/or/processor_test.go +++ b/src/pkg/retention/policy/alg/or/processor_test.go @@ -74,7 +74,7 @@ func (suite *ProcessorTestSuite) SetupSuite() { params := make([]*alg.Parameter, 0) - perf := action.NewRetainAction(suite.all) + perf := action.NewRetainAction(suite.all, false) lastxParams := make(map[string]rule.Parameter) lastxParams[lastx.ParameterX] = 10 diff --git a/src/pkg/retention/policy/builder.go b/src/pkg/retention/policy/builder.go index cb84660d3..72639c5da 100644 --- a/src/pkg/retention/policy/builder.go +++ b/src/pkg/retention/policy/builder.go @@ -16,6 +16,7 @@ package policy import ( "fmt" + "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/alg" "github.com/goharbor/harbor/src/pkg/retention/policy/lwp" @@ -31,11 +32,12 @@ type Builder interface { // // Arguments: // policy *Metadata : the simple metadata of retention policy + // isDryRun bool : indicate if we need to build a processor for dry run // // Returns: // Processor : a processor implementation to process the candidates // error : common error object if any errors occurred - Build(policy *lwp.Metadata) (alg.Processor, error) + Build(policy *lwp.Metadata, isDryRun bool) (alg.Processor, error) } // NewBuilder news a basic builder @@ -51,7 +53,7 @@ type basicBuilder struct { } // Build policy processor from the raw policy -func (bb *basicBuilder) Build(policy *lwp.Metadata) (alg.Processor, error) { +func (bb *basicBuilder) Build(policy *lwp.Metadata, isDryRun bool) (alg.Processor, error) { if policy == nil { return nil, errors.New("nil policy to build processor") } @@ -64,7 +66,7 @@ func (bb *basicBuilder) Build(policy *lwp.Metadata) (alg.Processor, error) { return nil, err } - perf, err := action.Get(r.Action, bb.allCandidates) + perf, err := action.Get(r.Action, bb.allCandidates, isDryRun) if err != nil { return nil, errors.Wrap(err, "get action performer by metadata") } diff --git a/src/pkg/retention/policy/builder_test.go b/src/pkg/retention/policy/builder_test.go index 155fe5611..91fe0d655 100644 --- a/src/pkg/retention/policy/builder_test.go +++ b/src/pkg/retention/policy/builder_test.go @@ -137,7 +137,7 @@ func (suite *TestBuilderSuite) TestBuild() { }}, } - p, err := b.Build(lm) + p, err := b.Build(lm, false) require.NoError(suite.T(), err) require.NotNil(suite.T(), p) diff --git a/src/pkg/retention/policy/rule/latestpull/evaluator.go b/src/pkg/retention/policy/rule/latestpull/evaluator.go deleted file mode 100644 index 4ef924aaa..000000000 --- a/src/pkg/retention/policy/rule/latestpull/evaluator.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package latestk - -import ( - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/pkg/retention/policy/action" - "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" -) - -const ( - // TemplateID of latest pulled k rule - TemplateID = "latestPulledK" - // ParameterK ... - ParameterK = TemplateID - // DefaultK defines the default K - DefaultK = 10 -) - -// evaluator for evaluating latest pulled k images -type evaluator struct { - // latest k - k int -} - -// Process the candidates based on the rule definition -func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) { - // TODO: REPLACE SAMPLE CODE WITH REAL IMPLEMENTATION - return artifacts, nil -} - -// Specify what action is performed to the candidates processed by this evaluator -func (e *evaluator) Action() string { - return action.Retain -} - -// New a Evaluator -func New(params rule.Parameters) rule.Evaluator { - if params != nil { - if param, ok := params[ParameterK]; ok { - if v, ok := param.(int); ok { - return &evaluator{ - k: v, - } - } - } - } - - log.Debugf("default parameter %d used for rule %s", DefaultK, TemplateID) - - return &evaluator{ - k: DefaultK, - } -} - -func init() { - // Register itself - rule.Register(&rule.IndexMeta{ - TemplateID: TemplateID, - Action: action.Retain, - Parameters: []*rule.IndexedParam{ - { - Name: ParameterK, - Type: "int", - Unit: "count", - Required: true, - }, - }, - }, New) -} diff --git a/src/portal/lib/package.json b/src/portal/lib/package.json index 4804668db..9c49c4207 100644 --- a/src/portal/lib/package.json +++ b/src/portal/lib/package.json @@ -1,6 +1,6 @@ { "name": "@harbor/ui", - "version": "1.8.0-rc2", + "version": "1.9.0", "description": "Harbor shared UI components based on Clarity and Angular7", "author": "CNCF", "module": "index.js", diff --git a/src/portal/lib/src/log/recent-log.component.spec.ts b/src/portal/lib/src/log/recent-log.component.spec.ts index 84a033da3..39425c8b2 100644 --- a/src/portal/lib/src/log/recent-log.component.spec.ts +++ b/src/portal/lib/src/log/recent-log.component.spec.ts @@ -11,7 +11,7 @@ import { FilterComponent } from '../filter/filter.component'; import { click } from '../utils'; import { of } from 'rxjs'; -import { startWith, delay } from 'rxjs/operators'; +import { delay } from 'rxjs/operators'; describe('RecentLogComponent (inline template)', () => { let component: RecentLogComponent; @@ -127,7 +127,7 @@ describe('RecentLogComponent (inline template)', () => { })); // Will fail after upgrade to angular 6. todo: need to fix it. - xit('should support pagination', async(() => { + it('should support pagination', () => { fixture.detectChanges(); fixture.whenStable().then(() => { @@ -136,7 +136,7 @@ describe('RecentLogComponent (inline template)', () => { let el: HTMLButtonElement = fixture.nativeElement.querySelector('.pagination-next'); expect(el).toBeTruthy(); el.click(); - + jasmine.clock().tick(100); fixture.detectChanges(); fixture.whenStable().then(() => { @@ -147,7 +147,7 @@ describe('RecentLogComponent (inline template)', () => { expect(els.length).toEqual(4); }); }); - })); + }); it('should support filtering list by keywords', async(() => { fixture.detectChanges(); diff --git a/src/portal/package-lock.json b/src/portal/package-lock.json index 19af5ed29..31054d2c4 100644 --- a/src/portal/package-lock.json +++ b/src/portal/package-lock.json @@ -1,6 +1,6 @@ { "name": "harbor", - "version": "1.8.0", + "version": "1.9.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/portal/package.json b/src/portal/package.json index 9c9dd34eb..7112b41e4 100644 --- a/src/portal/package.json +++ b/src/portal/package.json @@ -1,6 +1,6 @@ { "name": "harbor", - "version": "1.8.0", + "version": "1.9.0", "description": "Harbor UI with Clarity", "angular-cli": {}, "scripts": { diff --git a/src/portal/src/app/app.module.ts b/src/portal/src/app/app.module.ts index 2cd122d1b..5d0cd92f8 100644 --- a/src/portal/src/app/app.module.ts +++ b/src/portal/src/app/app.module.ts @@ -45,7 +45,7 @@ registerLocaleData(localePt, 'pt-br'); export function initConfig(configService: AppConfigService, skinableService: SkinableConfig) { return () => { - skinableService.getCustomFile(); + skinableService.getCustomFile().subscribe(); configService.load().subscribe(); }; } diff --git a/src/portal/src/app/project/member/add-http-auth-group/add-http-auth-group.component.ts b/src/portal/src/app/project/member/add-http-auth-group/add-http-auth-group.component.ts index 65f4111fe..b59e077e9 100644 --- a/src/portal/src/app/project/member/add-http-auth-group/add-http-auth-group.component.ts +++ b/src/portal/src/app/project/member/add-http-auth-group/add-http-auth-group.component.ts @@ -109,6 +109,7 @@ export class AddHttpAuthGroupComponent implements OnInit { this.currentForm.reset(); this.addHttpAuthOpened = true; this.role_id = 1; + this.inlineAlert.close(); } diff --git a/src/replication/adapter/helmhub/adapter.go b/src/replication/adapter/helmhub/adapter.go new file mode 100644 index 000000000..45fb7a0a3 --- /dev/null +++ b/src/replication/adapter/helmhub/adapter.go @@ -0,0 +1,80 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helmhub + +import ( + "errors" + "github.com/goharbor/harbor/src/common/utils/log" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/model" +) + +func init() { + if err := adp.RegisterFactory(model.RegistryTypeHelmHub, func(registry *model.Registry) (adp.Adapter, error) { + return newAdapter(registry) + }); err != nil { + log.Errorf("failed to register factory for %s: %v", model.RegistryTypeHelmHub, err) + return + } + log.Infof("the factory for adapter %s registered", model.RegistryTypeHelmHub) +} + +type adapter struct { + registry *model.Registry + client *Client +} + +func newAdapter(registry *model.Registry) (*adapter, error) { + return &adapter{ + registry: registry, + client: NewClient(registry), + }, nil +} + +func (a *adapter) Info() (*model.RegistryInfo, error) { + return &model.RegistryInfo{ + Type: model.RegistryTypeHelmHub, + SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeChart, + }, + SupportedResourceFilters: []*model.FilterStyle{ + { + Type: model.FilterTypeName, + Style: model.FilterStyleTypeText, + }, + { + Type: model.FilterTypeTag, + Style: model.FilterStyleTypeText, + }, + }, + SupportedTriggers: []model.TriggerType{ + model.TriggerTypeManual, + model.TriggerTypeScheduled, + }, + }, nil +} + +func (a *adapter) PrepareForPush(resources []*model.Resource) error { + return errors.New("not supported") +} + +// HealthCheck checks health status of a registry +func (a *adapter) HealthCheck() (model.HealthStatus, error) { + err := a.client.checkHealthy() + if err == nil { + return model.Healthy, nil + } + return model.Unhealthy, err +} diff --git a/src/replication/adapter/helmhub/adapter_test.go b/src/replication/adapter/helmhub/adapter_test.go new file mode 100644 index 000000000..ee22fc6dd --- /dev/null +++ b/src/replication/adapter/helmhub/adapter_test.go @@ -0,0 +1,44 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helmhub + +import ( + "testing" + + "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInfo(t *testing.T) { + adapter := &adapter{} + info, err := adapter.Info() + require.Nil(t, err) + require.Equal(t, 1, len(info.SupportedResourceTypes)) + assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[0]) +} + +func TestPrepareForPush(t *testing.T) { + adapter := &adapter{} + err := adapter.PrepareForPush(nil) + require.NotNil(t, err) +} + +func TestHealthCheck(t *testing.T) { + adapter, _ := newAdapter(nil) + status, err := adapter.HealthCheck() + require.Equal(t, model.Healthy, string(status)) + require.Nil(t, err) +} diff --git a/src/replication/adapter/helmhub/chart.go b/src/replication/adapter/helmhub/chart.go new file mode 100644 index 000000000..5c46b9e64 --- /dev/null +++ b/src/replication/adapter/helmhub/chart.go @@ -0,0 +1,44 @@ +package helmhub + +type chart struct { + ID string `json:"id"` + Type string `json:"type"` +} + +type chartList struct { + Data []*chart `json:"data"` +} + +type chartAttributes struct { + Version string `json:"version"` + URLs []string `json:"urls"` +} + +type chartRepo struct { + Name string `json:"name"` + URL string `json:"url"` +} + +type chartData struct { + Name string `json:"name"` + Repo *chartRepo `json:"repo"` +} + +type chartInfo struct { + Data *chartData `json:"data"` +} + +type chartRelationships struct { + Chart *chartInfo `json:"chart"` +} + +type chartVersion struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes *chartAttributes `json:"attributes"` + Relationships *chartRelationships `json:"relationships"` +} + +type chartVersionList struct { + Data []*chartVersion `json:"data"` +} diff --git a/src/replication/adapter/helmhub/chart_registry.go b/src/replication/adapter/helmhub/chart_registry.go new file mode 100644 index 000000000..daba32952 --- /dev/null +++ b/src/replication/adapter/helmhub/chart_registry.go @@ -0,0 +1,146 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helmhub + +import ( + "fmt" + "io" + "net/http" + "strings" + + "github.com/goharbor/harbor/src/common/utils/log" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/model" + "github.com/pkg/errors" +) + +func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { + charts, err := a.client.fetchCharts() + if err != nil { + return nil, err + } + + resources := []*model.Resource{} + repositories := []*adp.Repository{} + for _, chart := range charts.Data { + repository := &adp.Repository{ + ResourceType: string(model.ResourceTypeChart), + Name: chart.ID, + } + repositories = append(repositories, repository) + } + + for _, filter := range filters { + if err = filter.DoFilter(&repositories); err != nil { + return nil, err + } + } + + for _, repository := range repositories { + versionList, err := a.client.fetchChartDetail(repository.Name) + if err != nil { + log.Errorf("fetch chart detail: %v", err) + return nil, err + } + + vTags := []*adp.VTag{} + for _, version := range versionList.Data { + vTags = append(vTags, &adp.VTag{ + Name: version.Attributes.Version, + ResourceType: string(model.ResourceTypeChart), + }) + } + + for _, filter := range filters { + if err = filter.DoFilter(&vTags); err != nil { + return nil, err + } + } + + for _, vTag := range vTags { + resources = append(resources, &model.Resource{ + Type: model.ResourceTypeChart, + Registry: a.registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: repository.Name, + }, + Vtags: []string{vTag.Name}, + }, + }) + } + } + return resources, nil +} + +func (a *adapter) ChartExist(name, version string) (bool, error) { + versionList, err := a.client.fetchChartDetail(name) + if err != nil { + if err == ErrHTTPNotFound { + return false, nil + } + return false, err + } + + for _, v := range versionList.Data { + if v.Attributes.Version == version { + return true, nil + } + } + return false, nil +} + +func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { + versionList, err := a.client.fetchChartDetail(name) + if err != nil { + return nil, err + } + + for _, v := range versionList.Data { + if v.Attributes.Version == version { + return a.download(v) + } + } + return nil, errors.New("chart not found") +} + +func (a *adapter) download(version *chartVersion) (io.ReadCloser, error) { + if len(version.Attributes.URLs) == 0 || len(version.Attributes.URLs[0]) == 0 { + return nil, fmt.Errorf("cannot got the download url for chart %s", version.ID) + } + + url := strings.ToLower(version.Attributes.URLs[0]) + if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { + url = fmt.Sprintf("%s/charts/%s", version.Relationships.Chart.Data.Repo.URL, url) + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := a.client.do(req) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +func (a *adapter) UploadChart(name, version string, chart io.Reader) error { + return errors.New("not supported") +} + +func (a *adapter) DeleteChart(name, version string) error { + return errors.New("not supported") +} diff --git a/src/replication/adapter/helmhub/chart_registry_test.go b/src/replication/adapter/helmhub/chart_registry_test.go new file mode 100644 index 000000000..504d14f20 --- /dev/null +++ b/src/replication/adapter/helmhub/chart_registry_test.go @@ -0,0 +1,94 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helmhub + +import ( + "testing" + + "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchCharts(t *testing.T) { + adapter, err := newAdapter(nil) + require.Nil(t, err) + // filter 1 + filters := []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "k*/*", + }, + } + resources, err := adapter.FetchCharts(filters) + require.Nil(t, err) + assert.NotZero(t, len(resources)) + assert.Equal(t, model.ResourceTypeChart, resources[0].Type) + assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) + assert.NotNil(t, resources[0].Metadata.Vtags[0]) + // filter 2 + filters = []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "harbor/*", + }, + } + resources, err = adapter.FetchCharts(filters) + require.Nil(t, err) + assert.NotZero(t, len(resources)) + assert.Equal(t, model.ResourceTypeChart, resources[0].Type) + assert.Equal(t, "harbor/harbor", resources[0].Metadata.Repository.Name) + assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) + assert.NotNil(t, resources[0].Metadata.Vtags[0]) +} + +func TestChartExist(t *testing.T) { + adapter, err := newAdapter(nil) + require.Nil(t, err) + exist, err := adapter.ChartExist("harbor/harbor", "1.0.0") + require.Nil(t, err) + require.True(t, exist) +} + +func TestChartExist2(t *testing.T) { + adapter, err := newAdapter(nil) + require.Nil(t, err) + exist, err := adapter.ChartExist("goharbor/harbor", "1.0.0") + require.Nil(t, err) + require.False(t, exist) + + exist, err = adapter.ChartExist("harbor/harbor", "1.0.100") + require.Nil(t, err) + require.False(t, exist) +} + +func TestDownloadChart(t *testing.T) { + adapter, err := newAdapter(nil) + require.Nil(t, err) + _, err = adapter.DownloadChart("harbor/harbor", "1.0.0") + require.Nil(t, err) +} + +func TestUploadChart(t *testing.T) { + adapter := &adapter{} + err := adapter.UploadChart("library/harbor", "1.0", nil) + require.NotNil(t, err) +} + +func TestDeleteChart(t *testing.T) { + adapter := &adapter{} + err := adapter.DeleteChart("library/harbor", "1.0") + require.NotNil(t, err) +} diff --git a/src/replication/adapter/helmhub/client.go b/src/replication/adapter/helmhub/client.go new file mode 100644 index 000000000..c69b0d7a7 --- /dev/null +++ b/src/replication/adapter/helmhub/client.go @@ -0,0 +1,116 @@ +package helmhub + +import ( + "encoding/json" + "fmt" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" + "github.com/pkg/errors" + "io/ioutil" + "net/http" +) + +// ErrHTTPNotFound defines the return error when receiving 404 response code +var ErrHTTPNotFound = errors.New("Not Found") + +// Client is a client to interact with HelmHub +type Client struct { + client *http.Client +} + +// NewClient creates a new HelmHub client. +func NewClient(registry *model.Registry) *Client { + return &Client{ + client: &http.Client{ + Transport: util.GetHTTPTransport(false), + }, + } +} + +// fetchCharts fetches the chart list from helm hub. +func (c *Client) fetchCharts() (*chartList, error) { + request, err := http.NewRequest(http.MethodGet, baseURL+listCharts, nil) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch chart list error %d: %s", resp.StatusCode, string(body)) + } + + list := &chartList{} + err = json.Unmarshal(body, list) + if err != nil { + return nil, fmt.Errorf("unmarshal chart list response error: %v", err) + } + + return list, nil +} + +// fetchChartDetail fetches the chart detail of a chart from helm hub. +func (c *Client) fetchChartDetail(chartName string) (*chartVersionList, error) { + request, err := http.NewRequest(http.MethodGet, baseURL+listVersions(chartName), nil) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusNotFound { + return nil, ErrHTTPNotFound + } else if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch chart detail error %d: %s", resp.StatusCode, string(body)) + } + + list := &chartVersionList{} + err = json.Unmarshal(body, list) + if err != nil { + return nil, fmt.Errorf("unmarshal chart detail response error: %v", err) + } + + return list, nil +} + +func (c *Client) checkHealthy() error { + request, err := http.NewRequest(http.MethodGet, baseURL, nil) + if err != nil { + return err + } + + resp, err := c.client.Do(request) + if err != nil { + return err + } + defer resp.Body.Close() + + ioutil.ReadAll(resp.Body) + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return nil + } + return errors.New("helm hub is unhealthy") +} + +// do work as a proxy of Do function from net.http +func (c *Client) do(req *http.Request) (*http.Response, error) { + return c.client.Do(req) +} diff --git a/src/replication/adapter/helmhub/consts.go b/src/replication/adapter/helmhub/consts.go new file mode 100644 index 000000000..dab17bfcf --- /dev/null +++ b/src/replication/adapter/helmhub/consts.go @@ -0,0 +1,12 @@ +package helmhub + +import "fmt" + +const ( + baseURL = "https://hub.helm.sh" + listCharts = "/api/chartsvc/v1/charts" +) + +func listVersions(chartName string) string { + return fmt.Sprintf("/api/chartsvc/v1/charts/%s/versions", chartName) +} diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index 4f2459fe9..f5af7e7e1 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -30,6 +30,8 @@ const ( RegistryTypeAwsEcr RegistryType = "aws-ecr" RegistryTypeAzureAcr RegistryType = "azure-acr" + RegistryTypeHelmHub RegistryType = "helm-hub" + FilterStyleTypeText = "input" FilterStyleTypeRadio = "radio" FilterStyleTypeList = "list" diff --git a/tests/resources/Docker-Util.robot b/tests/resources/Docker-Util.robot index bf2d989a6..f7c2e50c9 100644 --- a/tests/resources/Docker-Util.robot +++ b/tests/resources/Docker-Util.robot @@ -32,12 +32,14 @@ Pull image Should Not Contain ${output} No such image: Push image - [Arguments] ${ip} ${user} ${pwd} ${project} ${image} - Log To Console \nRunning docker push ${image}... - Wait Unitl Command Success docker pull ${image} + [Arguments] ${ip} ${user} ${pwd} ${project} ${image} ${sha256}=${null} ${tag}=${null} + ${full_image}= Set Variable If '${sha256}'=='${null}' ${image} ${image}@sha256:${sha256} + ${image_with_tag}= Set Variable If '${tag}'=='${null}' ${image} ${image}:${sha256} + Log To Console \nRunning docker push ${full_image}... + Wait Unitl Command Success docker pull ${full_image} Wait Unitl Command Success docker login -u ${user} -p ${pwd} ${ip} - Wait Unitl Command Success docker tag ${image} ${ip}/${project}/${image} - Wait Unitl Command Success docker push ${ip}/${project}/${image} + Wait Unitl Command Success docker tag ${full_image} ${ip}/${project}/${image_with_tag} + Wait Unitl Command Success docker push ${ip}/${project}/${image_with_tag} Wait Unitl Command Success docker logout ${ip} Push Image With Tag @@ -55,9 +57,10 @@ Cannot Docker Login Harbor Command Should be Failed docker login -u ${user} -p ${pwd} ${ip} Cannot Pull image - [Arguments] ${ip} ${user} ${pwd} ${project} ${image} + [Arguments] ${ip} ${user} ${pwd} ${project} ${image} ${tag}=${null} + ${image_with_tag}= Set Variable If '${tag}'=='${null}' ${image} ${image}:${tag} Wait Unitl Command Success docker login -u ${user} -p ${pwd} ${ip} - Command Should be Failed docker pull ${ip}/${project}/${image} + Command Should be Failed docker pull ${ip}/${project}/${image_with_tag} Cannot Pull Unsigned Image [Arguments] ${ip} ${user} ${pass} ${proj} ${imagewithtag} diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index ac49d8dd5..2be78f4ce 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -581,6 +581,7 @@ Test Case - Retag A Image Tag Test Case - Scan Image On Push Wait Unitl Vul Data Ready ${HARBOR_URL} 7200 30 Init Chrome Driver + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Go Into Project library Goto Project Config @@ -610,14 +611,16 @@ Test Case - Project Level Image Serverity Policy Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${d}= get current date result_format=%m%s + ${sha256}= Set Variable 68b49a280d2fbe9330c0031970ebb72015e1272dfa25f0ed7557514f9e5ad7b7 + ${image}= Set Variable postgres Create An New Project project${d} - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} haproxy + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} sha256=${sha256} tag=${sha256} Go Into Project project${d} - Go Into Repo haproxy - Scan Repo latest Succeed + Go Into Repo ${image} + Scan Repo ${sha256} Succeed Navigate To Projects Go Into Project project${d} Set Vulnerabilty Serverity 0 - Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} haproxy + Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} tag=${sha256} Close Browser