Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-27 04:35:16 +01:00
Code Issues Projects Releases Wiki Activity

merge with latest master code with quota branch

Browse Source 
Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
wang yan 2019-07-26 01:32:41 +08:00
commit 676b922c95
32 changed files with 660 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/core/api/base.go
View File

@ -127,7 +127,7 @@ func Init() error {
}
return errors.New("bad retention callback param")
}
err := scheduler.Register(retention.RetentionSchedulerCallback, callbackFun)
err := scheduler.Register(retention.SchedulerCallback, callbackFun)
return err
}

8
src/core/main.go
View File

@ -17,7 +17,6 @@ package main
import (
"encoding/gob"
"fmt"
"github.com/goharbor/harbor/src/pkg/retention"
"os"
"os/signal"
"strconv"
@ -166,16 +165,11 @@ func main() {
log.Infof("Because SYNC_REGISTRY set false , no need to sync registry \n")
}
// Initialize retention
log.Info("Initialize retention")
if err := retention.Init(); err != nil {
log.Fatalf("Failed to initialize retention with error: %s", err)
}
log.Info("Init proxy")
if err := middlewares.Init(); err != nil {
log.Errorf("init proxy error, %v", err)
}
// go proxy.StartProxy()
beego.Run()
}

23
src/pkg/retention/boot.go
View File

@ -1,23 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package retention
// TODO: Move to api.Init()
// Init the retention components
func Init() error {
return nil
}

30
src/pkg/retention/controller.go
View File

@ -16,13 +16,14 @@ package retention
import (
"fmt"
"time"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"time"
)
// APIController to handle the requests related with retention
@ -66,8 +67,8 @@ type DefaultAPIController struct {
}
const (
// RetentionSchedulerCallback ...
RetentionSchedulerCallback = "RetentionSchedulerCallback"
// SchedulerCallback ...
SchedulerCallback = "SchedulerCallback"
)
// TriggerParam ...
@ -87,7 +88,7 @@ func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) error {
if p.Trigger.Settings != nil {
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
if ok {
jobid, err := r.scheduler.Schedule(cron.(string), RetentionSchedulerCallback, TriggerParam{
jobid, err := r.scheduler.Schedule(cron.(string), SchedulerCallback, TriggerParam{
PolicyID: p.ID,
Trigger: ExecutionTriggerSchedule,
})
@ -138,7 +139,7 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
case "":
default:
return fmt.Errorf("Not support Trigger %s", p.Trigger.Kind)
return fmt.Errorf("not support Trigger %s", p.Trigger.Kind)
}
}
if needUn {
@ -148,7 +149,7 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
}
}
if needSch {
jobid, err := r.scheduler.Schedule(p.Trigger.Settings[policy.TriggerSettingsCron].(string), RetentionSchedulerCallback, TriggerParam{
jobid, err := r.scheduler.Schedule(p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{
PolicyID: p.ID,
Trigger: ExecutionTriggerSchedule,
})
@ -192,8 +193,7 @@ func (r *DefaultAPIController) TriggerRetentionExec(policyID int64, trigger stri
DryRun: dryRun,
}
id, err := r.manager.CreateExecution(exec)
// TODO launcher with DryRun param
num, err := r.launcher.Launch(p, id)
num, err := r.launcher.Launch(p, id, dryRun)
if err != nil {
return err
}
@ -218,21 +218,21 @@ func (r *DefaultAPIController) OperateRetentionExec(eid int64, action string) er
if err != nil {
return err
}
exec := &Execution{}
switch action {
case "stop":
if e.Status != ExecutionStatusInProgress {
return fmt.Errorf("Can't abort, current status is %s", e.Status)
return fmt.Errorf("cannot abort, current status is %s", e.Status)
}
exec.ID = eid
exec.Status = ExecutionStatusStopped
exec.EndTime = time.Now()
// TODO stop the execution
e.Status = ExecutionStatusStopped
e.EndTime = time.Now()
// TODO: STOP THE EXECUTION
default:
return fmt.Errorf("not support action %s", action)
}
return r.manager.UpdateExecution(exec)
return r.manager.UpdateExecution(e)
}
// ListRetentionExecs List Retention Executions

12
src/pkg/retention/dep/client.go
View File

@ -17,7 +17,9 @@ package dep
import (
"errors"
"fmt"
"math/rand"
"net/http"
"time"
"github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/jobservice/config"
@ -102,9 +104,8 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
Tag: image.Name,
Labels: labels,
CreationTime: image.Created.Unix(),
// TODO: populate the pull/push time
// PulledTime: ,
// PushedTime:,
PulledTime: time.Now().Unix() - (int64)(rand.Int31n(4)*3600),
PushedTime: time.Now().Unix() - (int64)((rand.Int31n(5)+5)*3600),
}
candidates = append(candidates, candidate)
}
@ -125,9 +126,8 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida
Tag: chart.Name,
Labels: labels,
CreationTime: chart.Created.Unix(),
// TODO: populate the pull/push time
// PulledTime: ,
// PushedTime:,
PushedTime: time.Now().Unix() - (int64)((rand.Int31n(5)+5)*3600),
PulledTime: time.Now().Unix() - (int64)((rand.Int31n(4))*3600),
}
candidates = append(candidates, candidate)
}

45
src/pkg/retention/job.go
View File

@ -16,14 +16,14 @@ package retention
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
"github.com/goharbor/harbor/src/pkg/retention/res"
@ -32,11 +32,7 @@ import (
)
// Job of running retention process
type Job struct {
// client used to talk to core
// TODO: REFER THE GLOBAL CLIENT
client dep.Client
}
type Job struct{}
// MaxFails of the job
func (pj *Job) MaxFails() uint {
@ -58,6 +54,10 @@ func (pj *Job) Validate(params job.Parameters) error {
return err
}
if _, err := getParamDryRun(params); err != nil {
return err
}
return nil
}
@ -69,6 +69,7 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error {
// Parameters have been validated, ignore error checking
repo, _ := getParamRepo(params)
liteMeta, _ := getParamMeta(params)
isDryRun, _ := getParamDryRun(params)
// Log stage: start
repoPath := fmt.Sprintf("%s/%s", repo.Namespace, repo.Name)
@ -81,7 +82,7 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error {
}
// Retrieve all the candidates under the specified repository
allCandidates, err := pj.client.GetCandidates(repo)
allCandidates, err := dep.DefaultClient.GetCandidates(repo)
if err != nil {
return logError(myLogger, err)
}
@ -91,7 +92,7 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error {
// Build the processor
builder := policy.NewBuilder(allCandidates)
processor, err := builder.Build(liteMeta)
processor, err := builder.Build(liteMeta, isDryRun)
if err != nil {
return logError(myLogger, err)
}
@ -111,16 +112,6 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error {
// Log stage: results with table view
logResults(myLogger, allCandidates, results)
// Check in the results
bytes, err := json.Marshal(results)
if err != nil {
return logError(myLogger, err)
}
if err := ctx.Checkin(string(bytes)); err != nil {
return logError(myLogger, err)
}
return nil
}
@ -146,7 +137,7 @@ func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Re
var buf bytes.Buffer
data := [][]string{}
data := make([][]string, len(all))
for _, c := range all {
row := []string{
@ -204,6 +195,20 @@ func logError(logger logger.Interface, err error) error {
return wrappedErr
}
func getParamDryRun(params job.Parameters) (bool, error) {
v, ok := params[ParamDryRun]
if !ok {
return false, errors.Errorf("missing parameter: %s", ParamDryRun)
}
dryRun, ok := v.(bool)
if !ok {
return false, errors.Errorf("invalid parameter: %s", ParamDryRun)
}
return dryRun, nil
}
func getParamRepo(params job.Parameters) (*res.Repository, error) {
v, ok := params[ParamRepo]
if !ok {

5
src/pkg/retention/job_test.go
View File

@ -76,6 +76,7 @@ func (suite *JobTestSuite) TearDownSuite() {
func (suite *JobTestSuite) TestRunSuccess() {
params := make(job.Parameters)
params[ParamDryRun] = false
params[ParamRepo] = &res.Repository{
Namespace: "library",
Name: "harbor",
@ -115,9 +116,7 @@ func (suite *JobTestSuite) TestRunSuccess() {
},
}
j := &Job{
client: &fakeRetentionClient{},
}
j := &Job{}
err := j.Validate(params)
require.NoError(suite.T(), err)

12
src/pkg/retention/launcher.go
View File

@ -37,6 +37,8 @@ const (
ParamRepo = "repository"
// ParamMeta ...
ParamMeta = "liteMeta"
// ParamDryRun ...
ParamDryRun = "dryRun"
)
// Launcher provides function to launch the async jobs to run retentions based on the provided policy.
@ -47,11 +49,12 @@ type Launcher interface {
// Arguments:
// policy *policy.Metadata: the policy info
// executionID int64 : the execution ID
// isDryRun bool : indicate if it is a dry run
//
// Returns:
// int64 : the count of tasks
// error : common error if any errors occurred
Launch(policy *policy.Metadata, executionID int64) (int64, error)
Launch(policy *policy.Metadata, executionID int64, isDryRun bool) (int64, error)
}
// NewLauncher returns an instance of Launcher
@ -80,7 +83,7 @@ type jobData struct {
taskID int64
}
func (l *launcher) Launch(ply *policy.Metadata, executionID int64) (int64, error) {
func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool) (int64, error) {
if ply == nil {
return 0, launcherError(fmt.Errorf("the policy is nil"))
}
@ -193,8 +196,9 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64) (int64, error
}
j.Name = job.Retention
j.Parameters = map[string]interface{}{
ParamRepo: jobData.repository,
ParamMeta: jobData.policy,
ParamRepo: jobData.repository,
ParamMeta: jobData.policy,
ParamDryRun: isDryRun,
}
_, err := l.jobserviceClient.SubmitJob(j)
if err != nil {

8
src/pkg/retention/launcher_test.go
View File

@ -182,12 +182,12 @@ func (l *launchTestSuite) TestLaunch() {
var ply *policy.Metadata
// nil policy
n, err := launcher.Launch(ply, 1)
n, err := launcher.Launch(ply, 1, false)
require.NotNil(l.T(), err)
// nil rules
ply = &policy.Metadata{}
n, err = launcher.Launch(ply, 1)
n, err = launcher.Launch(ply, 1, false)
require.Nil(l.T(), err)
assert.Equal(l.T(), int64(0), n)
@ -197,7 +197,7 @@ func (l *launchTestSuite) TestLaunch() {
{},
},
}
_, err = launcher.Launch(ply, 1)
_, err = launcher.Launch(ply, 1, false)
require.NotNil(l.T(), err)
// system scope
@ -226,7 +226,7 @@ func (l *launchTestSuite) TestLaunch() {
},
},
}
n, err = launcher.Launch(ply, 1)
n, err = launcher.Launch(ply, 1, false)
require.Nil(l.T(), err)
assert.Equal(l.T(), int64(2), n)
}

7
src/pkg/retention/policy/action/index.go
View File

@ -15,8 +15,9 @@
package action
import (
"github.com/pkg/errors"
"sync"
"github.com/pkg/errors"
)
// index for keeping the mapping action and its performer
@ -33,7 +34,7 @@ func Register(action string, factory PerformerFactory) {
}
// Get performer with the provided action
func Get(action string, params interface{}) (Performer, error) {
func Get(action string, params interface{}, isDryRun bool) (Performer, error) {
if len(action) == 0 {
return nil, errors.New("empty action")
}
@ -48,5 +49,5 @@ func Get(action string, params interface{}) (Performer, error) {
return nil, errors.Errorf("invalid action performer registered for action %s", action)
}
return factory(params), nil
return factory(params, isDryRun), nil
}

14
src/pkg/retention/policy/action/index_test.go
View File

@ -54,7 +54,7 @@ func (suite *IndexTestSuite) SetupSuite() {
// TestRegister tests register
func (suite *IndexTestSuite) TestGet() {
p, err := Get("fakeAction", nil)
p, err := Get("fakeAction", nil, false)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), p)
@ -72,7 +72,10 @@ func (suite *IndexTestSuite) TestGet() {
})
}
type fakePerformer struct{}
type fakePerformer struct {
parameters interface{}
isDryRun bool
}
// Perform the artifacts
func (p *fakePerformer) Perform(candidates []*res.Candidate) (results []*res.Result, err error) {
@ -85,6 +88,9 @@ func (p *fakePerformer) Perform(candidates []*res.Candidate) (results []*res.Res
return
}
func newFakePerformer(params interface{}) Performer {
return &fakePerformer{}
func newFakePerformer(params interface{}, isDryRun bool) Performer {
return &fakePerformer{
parameters: params,
isDryRun: isDryRun,
}
}

20
src/pkg/retention/policy/action/performer.go
View File

@ -21,7 +21,7 @@ import (
const (
// Retain artifacts
Retain = "retain"
Retain = "Retain"
)
// Performer performs the related actions targeting the candidates
@ -38,11 +38,13 @@ type Performer interface {
}
// PerformerFactory is factory method for creating Performer
type PerformerFactory func(params interface{}) Performer
type PerformerFactory func(params interface{}, isDryRun bool) Performer
// retainAction make sure all the candidates will be retained and others will be cleared
type retainAction struct {
all []*res.Candidate
// Indicate if it is a dry run
isDryRun bool
}
// Perform the action
@ -60,8 +62,10 @@ func (ra *retainAction) Perform(candidates []*res.Candidate) (results []*res.Res
Target: art,
}
if err := dep.DefaultClient.Delete(art); err != nil {
result.Error = err
if !ra.isDryRun {
if err := dep.DefaultClient.Delete(art); err != nil {
result.Error = err
}
}
results = append(results, result)
@ -73,17 +77,19 @@ func (ra *retainAction) Perform(candidates []*res.Candidate) (results []*res.Res
}
// NewRetainAction is factory method for RetainAction
func NewRetainAction(params interface{}) Performer {
func NewRetainAction(params interface{}, isDryRun bool) Performer {
if params != nil {
if all, ok := params.([]*res.Candidate); ok {
return &retainAction{
all: all,
all: all,
isDryRun: isDryRun,
}
}
}
return &retainAction{
all: make([]*res.Candidate, 0),
all: make([]*res.Candidate, 0),
isDryRun: isDryRun,
}
}

2
src/pkg/retention/policy/alg/or/processor_test.go
View File

@ -74,7 +74,7 @@ func (suite *ProcessorTestSuite) SetupSuite() {
params := make([]*alg.Parameter, 0)
perf := action.NewRetainAction(suite.all)
perf := action.NewRetainAction(suite.all, false)
lastxParams := make(map[string]rule.Parameter)
lastxParams[lastx.ParameterX] = 10

8
src/pkg/retention/policy/builder.go
View File

@ -16,6 +16,7 @@ package policy
import (
"fmt"
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
"github.com/goharbor/harbor/src/pkg/retention/policy/alg"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
@ -31,11 +32,12 @@ type Builder interface {
//
// Arguments:
// policy *Metadata : the simple metadata of retention policy
// isDryRun bool : indicate if we need to build a processor for dry run
//
// Returns:
// Processor : a processor implementation to process the candidates
// error : common error object if any errors occurred
Build(policy *lwp.Metadata) (alg.Processor, error)
Build(policy *lwp.Metadata, isDryRun bool) (alg.Processor, error)
}
// NewBuilder news a basic builder
@ -51,7 +53,7 @@ type basicBuilder struct {
}
// Build policy processor from the raw policy
func (bb *basicBuilder) Build(policy *lwp.Metadata) (alg.Processor, error) {
func (bb *basicBuilder) Build(policy *lwp.Metadata, isDryRun bool) (alg.Processor, error) {
if policy == nil {
return nil, errors.New("nil policy to build processor")
}
@ -64,7 +66,7 @@ func (bb *basicBuilder) Build(policy *lwp.Metadata) (alg.Processor, error) {
return nil, err
}
perf, err := action.Get(r.Action, bb.allCandidates)
perf, err := action.Get(r.Action, bb.allCandidates, isDryRun)
if err != nil {
return nil, errors.Wrap(err, "get action performer by metadata")
}

2
src/pkg/retention/policy/builder_test.go
View File

@ -137,7 +137,7 @@ func (suite *TestBuilderSuite) TestBuild() {
}},
}
p, err := b.Build(lm)
p, err := b.Build(lm, false)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), p)

83
src/pkg/retention/policy/rule/latestpull/evaluator.go
View File

@ -1,83 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package latestk
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/retention/policy/action"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/goharbor/harbor/src/pkg/retention/res"
)
const (
// TemplateID of latest pulled k rule
TemplateID = "latestPulledK"
// ParameterK ...
ParameterK = TemplateID
// DefaultK defines the default K
DefaultK = 10
)
// evaluator for evaluating latest pulled k images
type evaluator struct {
// latest k
k int
}
// Process the candidates based on the rule definition
func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) {
// TODO: REPLACE SAMPLE CODE WITH REAL IMPLEMENTATION
return artifacts, nil
}
// Specify what action is performed to the candidates processed by this evaluator
func (e *evaluator) Action() string {
return action.Retain
}
// New a Evaluator
func New(params rule.Parameters) rule.Evaluator {
if params != nil {
if param, ok := params[ParameterK]; ok {
if v, ok := param.(int); ok {
return &evaluator{
k: v,
}
}
}
}
log.Debugf("default parameter %d used for rule %s", DefaultK, TemplateID)
return &evaluator{
k: DefaultK,
}
}
func init() {
// Register itself
rule.Register(&rule.IndexMeta{
TemplateID: TemplateID,
Action: action.Retain,
Parameters: []*rule.IndexedParam{
{
Name: ParameterK,
Type: "int",
Unit: "count",
Required: true,
},
},
}, New)
}

2
src/portal/lib/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@harbor/ui",
"version": "1.8.0-rc2",
"version": "1.9.0",
"description": "Harbor shared UI components based on Clarity and Angular7",
"author": "CNCF",
"module": "index.js",

8
src/portal/lib/src/log/recent-log.component.spec.ts
View File

@ -11,7 +11,7 @@ import { FilterComponent } from '../filter/filter.component';
import { click } from '../utils';
import { of } from 'rxjs';
import { startWith, delay } from 'rxjs/operators';
import { delay } from 'rxjs/operators';
describe('RecentLogComponent (inline template)', () => {
let component: RecentLogComponent;
@ -127,7 +127,7 @@ describe('RecentLogComponent (inline template)', () => {
}));
// Will fail after upgrade to angular 6. todo: need to fix it.
xit('should support pagination', async(() => {
it('should support pagination', () => {
fixture.detectChanges();
fixture.whenStable().then(() => {
@ -136,7 +136,7 @@ describe('RecentLogComponent (inline template)', () => {
let el: HTMLButtonElement = fixture.nativeElement.querySelector('.pagination-next');
expect(el).toBeTruthy();
el.click();
jasmine.clock().tick(100);
fixture.detectChanges();
fixture.whenStable().then(() => {
@ -147,7 +147,7 @@ describe('RecentLogComponent (inline template)', () => {
expect(els.length).toEqual(4);
});
});
}));
});
it('should support filtering list by keywords', async(() => {
fixture.detectChanges();

2
src/portal/package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "harbor",
"version": "1.8.0",
"version": "1.9.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
src/portal/package.json
View File

@ -1,6 +1,6 @@
{
"name": "harbor",
"version": "1.8.0",
"version": "1.9.0",
"description": "Harbor UI with Clarity",
"angular-cli": {},
"scripts": {

2
src/portal/src/app/app.module.ts
View File

@ -45,7 +45,7 @@ registerLocaleData(localePt, 'pt-br');
export function initConfig(configService: AppConfigService, skinableService: SkinableConfig) {
return () => {
skinableService.getCustomFile();
skinableService.getCustomFile().subscribe();
configService.load().subscribe();
};
}

1
src/portal/src/app/project/member/add-http-auth-group/add-http-auth-group.component.ts
View File

@ -109,6 +109,7 @@ export class AddHttpAuthGroupComponent implements OnInit {
this.currentForm.reset();
this.addHttpAuthOpened = true;
this.role_id = 1;
this.inlineAlert.close();
}

80
src/replication/adapter/helmhub/adapter.go Normal file
View File

@ -0,0 +1,80 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helmhub
import (
"errors"
"github.com/goharbor/harbor/src/common/utils/log"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
)
func init() {
if err := adp.RegisterFactory(model.RegistryTypeHelmHub, func(registry *model.Registry) (adp.Adapter, error) {
return newAdapter(registry)
}); err != nil {
log.Errorf("failed to register factory for %s: %v", model.RegistryTypeHelmHub, err)
return
}
log.Infof("the factory for adapter %s registered", model.RegistryTypeHelmHub)
}
type adapter struct {
registry *model.Registry
client *Client
}
func newAdapter(registry *model.Registry) (*adapter, error) {
return &adapter{
registry: registry,
client: NewClient(registry),
}, nil
}
func (a *adapter) Info() (*model.RegistryInfo, error) {
return &model.RegistryInfo{
Type: model.RegistryTypeHelmHub,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeChart,
},
SupportedResourceFilters: []*model.FilterStyle{
{
Type: model.FilterTypeName,
Style: model.FilterStyleTypeText,
},
{
Type: model.FilterTypeTag,
Style: model.FilterStyleTypeText,
},
},
SupportedTriggers: []model.TriggerType{
model.TriggerTypeManual,
model.TriggerTypeScheduled,
},
}, nil
}
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
return errors.New("not supported")
}
// HealthCheck checks health status of a registry
func (a *adapter) HealthCheck() (model.HealthStatus, error) {
err := a.client.checkHealthy()
if err == nil {
return model.Healthy, nil
}
return model.Unhealthy, err
}

44
src/replication/adapter/helmhub/adapter_test.go Normal file
View File

@ -0,0 +1,44 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helmhub
import (
"testing"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestInfo(t *testing.T) {
adapter := &adapter{}
info, err := adapter.Info()
require.Nil(t, err)
require.Equal(t, 1, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[0])
}
func TestPrepareForPush(t *testing.T) {
adapter := &adapter{}
err := adapter.PrepareForPush(nil)
require.NotNil(t, err)
}
func TestHealthCheck(t *testing.T) {
adapter, _ := newAdapter(nil)
status, err := adapter.HealthCheck()
require.Equal(t, model.Healthy, string(status))
require.Nil(t, err)
}

44
src/replication/adapter/helmhub/chart.go Normal file
View File

@ -0,0 +1,44 @@
package helmhub
type chart struct {
ID string `json:"id"`
Type string `json:"type"`
}
type chartList struct {
Data []*chart `json:"data"`
}
type chartAttributes struct {
Version string `json:"version"`
URLs []string `json:"urls"`
}
type chartRepo struct {
Name string `json:"name"`
URL string `json:"url"`
}
type chartData struct {
Name string `json:"name"`
Repo *chartRepo `json:"repo"`
}
type chartInfo struct {
Data *chartData `json:"data"`
}
type chartRelationships struct {
Chart *chartInfo `json:"chart"`
}
type chartVersion struct {
ID string `json:"id"`
Type string `json:"type"`
Attributes *chartAttributes `json:"attributes"`
Relationships *chartRelationships `json:"relationships"`
}
type chartVersionList struct {
Data []*chartVersion `json:"data"`
}

146
src/replication/adapter/helmhub/chart_registry.go Normal file
View File

@ -0,0 +1,146 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helmhub
import (
"fmt"
"io"
"net/http"
"strings"
"github.com/goharbor/harbor/src/common/utils/log"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/pkg/errors"
)
func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
charts, err := a.client.fetchCharts()
if err != nil {
return nil, err
}
resources := []*model.Resource{}
repositories := []*adp.Repository{}
for _, chart := range charts.Data {
repository := &adp.Repository{
ResourceType: string(model.ResourceTypeChart),
Name: chart.ID,
}
repositories = append(repositories, repository)
}
for _, filter := range filters {
if err = filter.DoFilter(&repositories); err != nil {
return nil, err
}
}
for _, repository := range repositories {
versionList, err := a.client.fetchChartDetail(repository.Name)
if err != nil {
log.Errorf("fetch chart detail: %v", err)
return nil, err
}
vTags := []*adp.VTag{}
for _, version := range versionList.Data {
vTags = append(vTags, &adp.VTag{
Name: version.Attributes.Version,
ResourceType: string(model.ResourceTypeChart),
})
}
for _, filter := range filters {
if err = filter.DoFilter(&vTags); err != nil {
return nil, err
}
}
for _, vTag := range vTags {
resources = append(resources, &model.Resource{
Type: model.ResourceTypeChart,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repository.Name,
},
Vtags: []string{vTag.Name},
},
})
}
}
return resources, nil
}
func (a *adapter) ChartExist(name, version string) (bool, error) {
versionList, err := a.client.fetchChartDetail(name)
if err != nil {
if err == ErrHTTPNotFound {
return false, nil
}
return false, err
}
for _, v := range versionList.Data {
if v.Attributes.Version == version {
return true, nil
}
}
return false, nil
}
func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
versionList, err := a.client.fetchChartDetail(name)
if err != nil {
return nil, err
}
for _, v := range versionList.Data {
if v.Attributes.Version == version {
return a.download(v)
}
}
return nil, errors.New("chart not found")
}
func (a *adapter) download(version *chartVersion) (io.ReadCloser, error) {
if len(version.Attributes.URLs) == 0 || len(version.Attributes.URLs[0]) == 0 {
return nil, fmt.Errorf("cannot got the download url for chart %s", version.ID)
}
url := strings.ToLower(version.Attributes.URLs[0])
if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) {
url = fmt.Sprintf("%s/charts/%s", version.Relationships.Chart.Data.Repo.URL, url)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := a.client.do(req)
if err != nil {
return nil, err
}
return resp.Body, nil
}
func (a *adapter) UploadChart(name, version string, chart io.Reader) error {
return errors.New("not supported")
}
func (a *adapter) DeleteChart(name, version string) error {
return errors.New("not supported")
}

94
src/replication/adapter/helmhub/chart_registry_test.go Normal file
View File

@ -0,0 +1,94 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helmhub
import (
"testing"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFetchCharts(t *testing.T) {
adapter, err := newAdapter(nil)
require.Nil(t, err)
// filter 1
filters := []*model.Filter{
{
Type: model.FilterTypeName,
Value: "k*/*",
},
}
resources, err := adapter.FetchCharts(filters)
require.Nil(t, err)
assert.NotZero(t, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.NotNil(t, resources[0].Metadata.Vtags[0])
// filter 2
filters = []*model.Filter{
{
Type: model.FilterTypeName,
Value: "harbor/*",
},
}
resources, err = adapter.FetchCharts(filters)
require.Nil(t, err)
assert.NotZero(t, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, "harbor/harbor", resources[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.NotNil(t, resources[0].Metadata.Vtags[0])
}
func TestChartExist(t *testing.T) {
adapter, err := newAdapter(nil)
require.Nil(t, err)
exist, err := adapter.ChartExist("harbor/harbor", "1.0.0")
require.Nil(t, err)
require.True(t, exist)
}
func TestChartExist2(t *testing.T) {
adapter, err := newAdapter(nil)
require.Nil(t, err)
exist, err := adapter.ChartExist("goharbor/harbor", "1.0.0")
require.Nil(t, err)
require.False(t, exist)
exist, err = adapter.ChartExist("harbor/harbor", "1.0.100")
require.Nil(t, err)
require.False(t, exist)
}
func TestDownloadChart(t *testing.T) {
adapter, err := newAdapter(nil)
require.Nil(t, err)
_, err = adapter.DownloadChart("harbor/harbor", "1.0.0")
require.Nil(t, err)
}
func TestUploadChart(t *testing.T) {
adapter := &adapter{}
err := adapter.UploadChart("library/harbor", "1.0", nil)
require.NotNil(t, err)
}
func TestDeleteChart(t *testing.T) {
adapter := &adapter{}
err := adapter.DeleteChart("library/harbor", "1.0")
require.NotNil(t, err)
}

116
src/replication/adapter/helmhub/client.go Normal file
View File

@ -0,0 +1,116 @@
package helmhub
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"github.com/pkg/errors"
"io/ioutil"
"net/http"
)
// ErrHTTPNotFound defines the return error when receiving 404 response code
var ErrHTTPNotFound = errors.New("Not Found")
// Client is a client to interact with HelmHub
type Client struct {
client *http.Client
}
// NewClient creates a new HelmHub client.
func NewClient(registry *model.Registry) *Client {
return &Client{
client: &http.Client{
Transport: util.GetHTTPTransport(false),
},
}
}
// fetchCharts fetches the chart list from helm hub.
func (c *Client) fetchCharts() (*chartList, error) {
request, err := http.NewRequest(http.MethodGet, baseURL+listCharts, nil)
if err != nil {
return nil, err
}
resp, err := c.client.Do(request)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("fetch chart list error %d: %s", resp.StatusCode, string(body))
}
list := &chartList{}
err = json.Unmarshal(body, list)
if err != nil {
return nil, fmt.Errorf("unmarshal chart list response error: %v", err)
}
return list, nil
}
// fetchChartDetail fetches the chart detail of a chart from helm hub.
func (c *Client) fetchChartDetail(chartName string) (*chartVersionList, error) {
request, err := http.NewRequest(http.MethodGet, baseURL+listVersions(chartName), nil)
if err != nil {
return nil, err
}
resp, err := c.client.Do(request)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode == http.StatusNotFound {
return nil, ErrHTTPNotFound
} else if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("fetch chart detail error %d: %s", resp.StatusCode, string(body))
}
list := &chartVersionList{}
err = json.Unmarshal(body, list)
if err != nil {
return nil, fmt.Errorf("unmarshal chart detail response error: %v", err)
}
return list, nil
}
func (c *Client) checkHealthy() error {
request, err := http.NewRequest(http.MethodGet, baseURL, nil)
if err != nil {
return err
}
resp, err := c.client.Do(request)
if err != nil {
return err
}
defer resp.Body.Close()
ioutil.ReadAll(resp.Body)
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return nil
}
return errors.New("helm hub is unhealthy")
}
// do work as a proxy of Do function from net.http
func (c *Client) do(req *http.Request) (*http.Response, error) {
return c.client.Do(req)
}

12
src/replication/adapter/helmhub/consts.go Normal file
View File

@ -0,0 +1,12 @@
package helmhub
import "fmt"
const (
baseURL = "https://hub.helm.sh"
listCharts = "/api/chartsvc/v1/charts"
)
func listVersions(chartName string) string {
return fmt.Sprintf("/api/chartsvc/v1/charts/%s/versions", chartName)
}

2
src/replication/model/registry.go
View File

@ -30,6 +30,8 @@ const (
RegistryTypeAwsEcr RegistryType = "aws-ecr"
RegistryTypeAzureAcr RegistryType = "azure-acr"
RegistryTypeHelmHub RegistryType = "helm-hub"
FilterStyleTypeText = "input"
FilterStyleTypeRadio = "radio"
FilterStyleTypeList = "list"

17
tests/resources/Docker-Util.robot
View File

@ -32,12 +32,14 @@ Pull image
Should Not Contain ${output} No such image:
Push image
[Arguments] ${ip} ${user} ${pwd} ${project} ${image}
Log To Console \nRunning docker push ${image}...
Wait Unitl Command Success docker pull ${image}
[Arguments] ${ip} ${user} ${pwd} ${project} ${image} ${sha256}=${null} ${tag}=${null}
${full_image}= Set Variable If '${sha256}'=='${null}' ${image} ${image}@sha256:${sha256}
${image_with_tag}= Set Variable If '${tag}'=='${null}' ${image} ${image}:${sha256}
Log To Console \nRunning docker push ${full_image}...
Wait Unitl Command Success docker pull ${full_image}
Wait Unitl Command Success docker login -u ${user} -p ${pwd} ${ip}
Wait Unitl Command Success docker tag ${image} ${ip}/${project}/${image}
Wait Unitl Command Success docker push ${ip}/${project}/${image}
Wait Unitl Command Success docker tag ${full_image} ${ip}/${project}/${image_with_tag}
Wait Unitl Command Success docker push ${ip}/${project}/${image_with_tag}
Wait Unitl Command Success docker logout ${ip}
Push Image With Tag
@ -55,9 +57,10 @@ Cannot Docker Login Harbor
Command Should be Failed docker login -u ${user} -p ${pwd} ${ip}
Cannot Pull image
[Arguments] ${ip} ${user} ${pwd} ${project} ${image}
[Arguments] ${ip} ${user} ${pwd} ${project} ${image} ${tag}=${null}
${image_with_tag}= Set Variable If '${tag}'=='${null}' ${image} ${image}:${tag}
Wait Unitl Command Success docker login -u ${user} -p ${pwd} ${ip}
Command Should be Failed docker pull ${ip}/${project}/${image}
Command Should be Failed docker pull ${ip}/${project}/${image_with_tag}
Cannot Pull Unsigned Image
[Arguments] ${ip} ${user} ${pass} ${proj} ${imagewithtag}

11
tests/robot-cases/Group1-Nightly/Common.robot
View File

@ -581,6 +581,7 @@ Test Case - Retag A Image Tag
Test Case - Scan Image On Push
Wait Unitl Vul Data Ready ${HARBOR_URL} 7200 30
Init Chrome Driver
Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Go Into Project library
Goto Project Config
@ -610,14 +611,16 @@ Test Case - Project Level Image Serverity Policy
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
${d}= get current date result_format=%m%s
${sha256}= Set Variable 68b49a280d2fbe9330c0031970ebb72015e1272dfa25f0ed7557514f9e5ad7b7
${image}= Set Variable postgres
Create An New Project project${d}
Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} haproxy
Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} sha256=${sha256} tag=${sha256}
Go Into Project project${d}
Go Into Repo haproxy
Scan Repo latest Succeed
Go Into Repo ${image}
Scan Repo ${sha256} Succeed
Navigate To Projects
Go Into Project project${d}
Set Vulnerabilty Serverity 0
Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} haproxy
Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} tag=${sha256}
Close Browser