Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-19 07:07:42 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #12335 from kofj/p2p_preheat_api

Browse Source 
feat(preheat):add preheat api, controller and manager
This commit is contained in:
Steven Zou 2020-07-03 13:47:04 +08:00 committed by GitHub
commit f3fcb96570
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1495 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

262
api/v2.0/swagger.yaml
View File

@ -641,6 +641,179 @@ paths:
$ref: '#/responses/401' $ref: '#/responses/401'
'500': '500':
$ref: '#/responses/500' $ref: '#/responses/500'
/p2p/preheat/providers:
get:
summary: List P2P providers
description: List P2P providers
tags:
- preheat
operationId: ListProviders
parameters:
- $ref: '#/parameters/requestId'
responses:
'200':
description: Success
schema:
type: array
items:
$ref: '#/definitions/Metadata'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/p2p/preheat/instances:
get:
summary: List P2P provider instances
description: List P2P provider instances
tags:
- preheat
operationId: ListInstances
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/query'
responses:
'200':
description: Success
headers:
X-Total-Count:
description: The total count of preheating provider instances
type: integer
Link:
description: Link refers to the previous page and next page
type: string
schema:
type: array
items:
$ref: '#/definitions/Instance'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
post:
summary: Create p2p provider instances
description: Create p2p provider instances
tags:
- preheat
operationId: CreateInstance
parameters:
- $ref: '#/parameters/requestId'
- name: instance
in: body
description: The JSON object of instance.
required: true
schema:
$ref: '#/definitions/Instance'
responses:
'201':
description: Response to insatnce created
schema:
$ref: '#/definitions/InstanceCreatedResp'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'409':
$ref: '#/responses/409'
'500':
$ref: '#/responses/500'
/p2p/preheat/instances/{instance_id}:
get:
summary: Get a P2P provider instance
description: Get a P2P provider instance
tags:
- preheat
operationId: GetInstance
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/instanceId'
responses:
'200':
description: Success
schema:
$ref: '#/definitions/Instance'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
delete:
summary: Delete the specified P2P provider instance
description: Delete the specified P2P provider instance
tags:
- preheat
operationId: DeleteInstance
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/instanceId'
responses:
'200':
description: Instance ID deleted
schema:
$ref: '#/definitions/InstanceDeletedResp'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
put:
summary: Update the specified P2P provider instance
description: Update the specified P2P provider instance
tags:
- preheat
operationId: UpdateInstance
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/instanceId'
- name: propertySet
in: body
description: The property set to update
required: true
schema:
type: object
additionalProperties:
type: object
additionalProperties: true
responses:
'200':
description: Success
schema:
$ref: '#/definitions/InstanceUpdateResp'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
parameters: parameters:
query: query:
name: q name: q
@ -701,6 +874,12 @@ parameters:
required: false required: false
description: The size of per page description: The size of per page
default: 10 default: 10
instanceId:
name: instance_id
in: path
description: Instance ID
required: true
type: integer
responses: responses:
'200': '200':
description: Success description: Success
@ -1098,3 +1277,86 @@ definitions:
op_time: op_time:
type: string type: string
description: The time when this operation is triggered. description: The time when this operation is triggered.
Metadata:
type: object
properties:
id:
type: string
description: id
name:
type: string
description: name
icon:
type: string
description: icon
maintainers:
type: array
description: maintainers
items:
type: string
version:
type: string
description: version
source:
type: string
description: source
Instance:
type: object
properties:
id:
type: integer
description: Unique ID
name:
type: string
description: Instance name
description:
type: string
description: Description of instance
vendor:
type: string
description: Based on which driver, identified by ID
endpoint:
type: string
description: The service endpoint of this instance
auth_mode:
type: string
description: The authentication way supported
auth_info:
type: object
description: The auth credential data if exists
additionalProperties:
type: string
status:
type: string
description: The health status
enabled:
type: boolean
description: Whether the instance is activated or not
default:
type: boolean
description: Whether the instance is default or not
insecure:
type: boolean
description: Whether the instance endpoint is insecure or not
setup_timestamp:
type: integer
format: int64
description: The timestamp of instance setting up
InstanceUpdateResp:
type: object
properties:
updated:
type: integer
description: ID of instance updated
InstanceDeletedResp:
type: object
properties:
removed:
type: integer
description: ID of instance removed
InstanceCreatedResp:
type: object
properties:
id:
type: integer
description: ID of instance created

12
codecov.yml
View File

@ -27,14 +27,6 @@ comment:
require_changes: no require_changes: no
ignore: ignore:
- "**/*.md"
- "**/*.yml"
- "docs"
- "api"
- "make"
- "contrib"
- "tests"
- "tools"
- "src/vendor" - "src/vendor"
- "src/server/v2.0/models/**/*" - "src/github.com/goharbor/harbor/src/server/v2.0/restapi/**/*"
- "src/server/v2.0/restapi/**/*" - "src/github.com/goharbor/harbor/src/server/v2.0/models"

14
make/migrations/postgresql/0040_2.1.0_schema.up.sql
View File

@ -37,6 +37,20 @@ ALTER TABLE blob ADD COLUMN IF NOT EXISTS version BIGINT default 0;
CREATE INDEX IF NOT EXISTS idx_status ON blob (status); CREATE INDEX IF NOT EXISTS idx_status ON blob (status);
CREATE INDEX IF NOT EXISTS idx_version ON blob (version); CREATE INDEX IF NOT EXISTS idx_version ON blob (version);
CREATE TABLE p2p_preheat_instance (
id SERIAL PRIMARY KEY NOT NULL,
name varchar(255) NOT NULL,
description varchar(255),
vendor varchar(255) NOT NULL,
endpoint varchar(255) NOT NULL,
auth_mode varchar(255),
auth_data text,
enabled boolean,
is_default boolean,
insecure boolean,
setup_timestamp int
);
CREATE TABLE IF NOT EXISTS p2p_preheat_policy ( CREATE TABLE IF NOT EXISTS p2p_preheat_policy (
id SERIAL PRIMARY KEY NOT NULL, id SERIAL PRIMARY KEY NOT NULL,
name varchar(255) NOT NULL, name varchar(255) NOT NULL,

168
src/controller/p2p/preheat/controller.go Normal file
View File

@ -0,0 +1,168 @@
package preheat
import (
"context"
"errors"
"time"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/p2p/preheat/instance"
providerModels "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider"
"github.com/goharbor/harbor/src/pkg/p2p/preheat/provider"
)
var (
// Ctl is a global preheat controller instance
Ctl = NewController()
)
// ErrorConflict for handling conflicts
var ErrorConflict = errors.New("resource conflict")
// ErrorUnhealthy for unhealthy
var ErrorUnhealthy = errors.New("instance unhealthy")
// Controller defines related top interfaces to handle the workflow of
// the image distribution.
// TODO: Add health check API
type Controller interface {
// Get all the supported distribution providers
//
// If succeed, an metadata of provider list will be returned.
// Otherwise, a non nil error will be returned
//
GetAvailableProviders() ([]*provider.Metadata, error)
// CountInstance all the setup instances of distribution providers
//
// params *q.Query : parameters for querying
//
// If succeed, matched provider instance count will be returned.
// Otherwise, a non nil error will be returned
//
CountInstance(ctx context.Context, query *q.Query) (int64, error)
// ListInstance all the setup instances of distribution providers
//
// params *q.Query : parameters for querying
//
// If succeed, matched provider instance list will be returned.
// Otherwise, a non nil error will be returned
//
ListInstance(ctx context.Context, query *q.Query) ([]*providerModels.Instance, error)
// GetInstance the metadata of the specified instance
//
// id string : ID of the instance being deleted
//
// If succeed, the metadata with nil error are returned
// Otherwise, a non nil error is returned
//
GetInstance(ctx context.Context, id int64) (*providerModels.Instance, error)
// Create a new instance for the specified provider.
//
// If succeed, the ID of the instance will be returned.
// Any problems met, a non nil error will be returned.
//
CreateInstance(ctx context.Context, instance *providerModels.Instance) (int64, error)
// Delete the specified provider instance.
//
// id string : ID of the instance being deleted
//
// Any problems met, a non nil error will be returned.
//
DeleteInstance(ctx context.Context, id int64) error
// Update the instance with incremental way;
// Including update the enabled flag of the instance.
//
// id string : ID of the instance being updated
// properties ...string : The properties being updated
//
// Any problems met, a non nil error will be returned
//
UpdateInstance(ctx context.Context, instance *providerModels.Instance, properties ...string) error
}
var _ Controller = (*controller)(nil)
// controller is the default implementation of Controller interface.
//
type controller struct {
// For instance
iManager instance.Manager
}
// NewController is constructor of controller
func NewController() Controller {
return &controller{
iManager: instance.Mgr,
}
}
// GetAvailableProviders implements @Controller.GetAvailableProviders
func (cc *controller) GetAvailableProviders() ([]*provider.Metadata, error) {
return provider.ListProviders()
}
// CountInstance implements @Controller.CountInstance
func (cc *controller) CountInstance(ctx context.Context, query *q.Query) (int64, error) {
return cc.iManager.Count(ctx, query)
}
// List implements @Controller.ListInstance
func (cc *controller) ListInstance(ctx context.Context, query *q.Query) ([]*providerModels.Instance, error) {
return cc.iManager.List(ctx, query)
}
// CreateInstance implements @Controller.CreateInstance
func (cc *controller) CreateInstance(ctx context.Context, instance *providerModels.Instance) (int64, error) {
if instance == nil {
return 0, errors.New("nil instance object provided")
}
// Avoid duplicated endpoint
var query = &q.Query{
Keywords: map[string]interface{}{
"endpoint": instance.Endpoint,
},
}
num, err := cc.iManager.Count(ctx, query)
if err != nil {
return 0, err
}
if num > 0 {
return 0, ErrorConflict
}
// !WARN: Check healthy status at fronted.
if instance.Status != "healthy" {
return 0, ErrorUnhealthy
}
instance.SetupTimestamp = time.Now().Unix()
return cc.iManager.Save(ctx, instance)
}
// Delete implements @Controller.Delete
func (cc *controller) DeleteInstance(ctx context.Context, id int64) error {
return cc.iManager.Delete(ctx, id)
}
// Update implements @Controller.Update
func (cc *controller) UpdateInstance(ctx context.Context, instance *providerModels.Instance, properties ...string) error {
if len(properties) == 0 {
return errors.New("no properties provided to update")
}
return cc.iManager.Update(ctx, instance, properties...)
}
// Get implements @Controller.Get
func (cc *controller) GetInstance(ctx context.Context, id int64) (*providerModels.Instance, error) {
return cc.iManager.Get(ctx, id)
}

155
src/controller/p2p/preheat/controllor_test.go Normal file
View File

@ -0,0 +1,155 @@
package preheat
import (
"context"
"errors"
"testing"
"github.com/goharbor/harbor/src/core/config"
imocks "github.com/goharbor/harbor/src/pkg/p2p/preheat/instance/mocks"
providerModel "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider"
"github.com/goharbor/harbor/src/pkg/p2p/preheat/provider"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type preheatSuite struct {
suite.Suite
ctx context.Context
controller Controller
fackManager *imocks.Manager
}
func TestPreheatSuite(t *testing.T) {
t.Log("Start TestPreheatSuite")
fackManager := &imocks.Manager{}
var c = &controller{
iManager: fackManager,
}
assert.NotNil(t, c)
suite.Run(t, &preheatSuite{
ctx: context.Background(),
controller: c,
fackManager: fackManager,
})
}
func TestNewController(t *testing.T) {
c := NewController()
assert.NotNil(t, c)
}
func (s *preheatSuite) SetupSuite() {
config.Init()
s.fackManager.On("List", mock.Anything, mock.Anything).Return([]*providerModel.Instance{
{
ID: 1,
Vendor: "dragonfly",
Endpoint: "http://localhost",
Status: provider.DriverStatusHealthy,
Enabled: true,
},
}, nil)
s.fackManager.On("Save", mock.Anything, mock.Anything).Return(int64(1), nil)
s.fackManager.On("Count", mock.Anything, &providerModel.Instance{Endpoint: "http://localhost"}).Return(int64(1), nil)
s.fackManager.On("Count", mock.Anything, mock.Anything).Return(int64(0), nil)
s.fackManager.On("Delete", mock.Anything, int64(1)).Return(nil)
s.fackManager.On("Delete", mock.Anything, int64(0)).Return(errors.New("not found"))
s.fackManager.On("Get", mock.Anything, int64(1)).Return(&providerModel.Instance{
ID: 1,
Endpoint: "http://localhost",
}, nil)
s.fackManager.On("Get", mock.Anything, int64(0)).Return(nil, errors.New("not found"))
}
func (s *preheatSuite) TestGetAvailableProviders() {
providers, err := s.controller.GetAvailableProviders()
s.Equal(2, len(providers))
expectProviders := map[string]interface{}{}
expectProviders["dragonfly"] = nil
expectProviders["kraken"] = nil
_, ok := expectProviders[providers[0].ID]
s.True(ok)
_, ok = expectProviders[providers[1].ID]
s.True(ok)
s.NoError(err)
}
func (s *preheatSuite) TestListInstance() {
instances, err := s.controller.ListInstance(s.ctx, nil)
s.NoError(err)
s.Equal(1, len(instances))
s.Equal(int64(1), instances[0].ID)
}
func (s *preheatSuite) TestCreateInstance() {
// Case: nil instance, expect error.
id, err := s.controller.CreateInstance(s.ctx, nil)
s.Empty(id)
s.Error(err)
// Case: instance with already existed endpoint, expect conflict.
id, err = s.controller.CreateInstance(s.ctx, &providerModel.Instance{
Endpoint: "http://localhost",
})
s.Equal(ErrorUnhealthy, err)
s.Empty(id)
// Case: instance with invalid provider, expect error.
id, err = s.controller.CreateInstance(s.ctx, &providerModel.Instance{
Endpoint: "http://foo.bar",
Status: "healthy",
Vendor: "none",
})
s.NoError(err)
s.Equal(int64(1), id)
// Case: instance with valid provider, expect ok.
id, err = s.controller.CreateInstance(s.ctx, &providerModel.Instance{
Endpoint: "http://foo.bar",
Status: "healthy",
Vendor: "dragonfly",
})
s.NoError(err)
s.Equal(int64(1), id)
id, err = s.controller.CreateInstance(s.ctx, &providerModel.Instance{
Endpoint: "http://foo.bar2",
Status: "healthy",
Vendor: "kraken",
})
s.NoError(err)
s.Equal(int64(1), id)
}
func (s *preheatSuite) TestDeleteInstance() {
// err := s.controller.DeleteInstance(s.ctx, 0)
// s.Error(err)
err := s.controller.DeleteInstance(s.ctx, int64(1))
s.NoError(err)
}
func (s *preheatSuite) TestUpdateInstance() {
// TODO: test update more
s.fackManager.On("Update", s.ctx, nil).Return(errors.New("no properties provided to update"))
err := s.controller.UpdateInstance(s.ctx, nil)
s.Error(err)
err = s.controller.UpdateInstance(s.ctx, &providerModel.Instance{ID: 0})
s.Error(err)
s.fackManager.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(nil)
err = s.controller.UpdateInstance(s.ctx, &providerModel.Instance{ID: 1}, "enabled")
s.NoError(err)
}
func (s *preheatSuite) TestGetInstance() {
instance, err := s.controller.GetInstance(s.ctx, 1)
s.NoError(err)
s.NotNil(instance)
}

138
src/pkg/p2p/preheat/dao/instance/dao.go Normal file
View File

@ -0,0 +1,138 @@
package instance
import (
"context"
beego_orm "github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider"
)
// DAO for instance
type DAO interface {
Create(ctx context.Context, instance *provider.Instance) (int64, error)
Get(ctx context.Context, id int64) (*provider.Instance, error)
Update(ctx context.Context, instance *provider.Instance, props ...string) error
Delete(ctx context.Context, id int64) error
Count(ctx context.Context, query *q.Query) (total int64, err error)
List(ctx context.Context, query *q.Query) (ins []*provider.Instance, err error)
}
// New instance dao
func New() DAO {
return &dao{}
}
// ListInstanceQuery defines the query params of the instance record.
type ListInstanceQuery struct {
Page uint
PageSize uint
Keyword string
}
type dao struct{}
var _ DAO = (*dao)(nil)
// Create adds a new distribution instance.
func (d *dao) Create(ctx context.Context, instance *provider.Instance) (int64, error) {
var o, err = orm.FromContext(ctx)
if err != nil {
return 0, err
}
return o.Insert(instance)
}
// Get gets instance from db by id.
func (d *dao) Get(ctx context.Context, id int64) (*provider.Instance, error) {
var o, err = orm.FromContext(ctx)
if err != nil {
return nil, err
}
di := provider.Instance{ID: id}
err = o.Read(&di, "ID")
if err == beego_orm.ErrNoRows {
return nil, nil
}
return &di, err
}
// Update updates distribution instance.
func (d *dao) Update(ctx context.Context, instance *provider.Instance, props ...string) error {
var o, err = orm.FromContext(ctx)
if err != nil {
return err
}
err = o.Begin()
if err != nil {
return err
}
// check default instances first
for _, prop := range props {
if prop == "default" && instance.Default {
_, err = o.Raw("UPDATE ? SET default = false WHERE id != ?", instance.TableName(), instance.ID).Exec()
if err != nil {
if e := o.Rollback(); e != nil {
err = errors.Wrap(e, err.Error())
}
return err
}
break
}
}
_, err = o.Update(instance, props...)
if err != nil {
if e := o.Rollback(); e != nil {
err = errors.Wrap(e, err.Error())
}
} else {
err = o.Commit()
}
return err
}
// Delete deletes one distribution instance by id.
func (d *dao) Delete(ctx context.Context, id int64) error {
var o, err = orm.FromContext(ctx)
if err != nil {
return err
}
_, err = o.Delete(&provider.Instance{ID: id})
return err
}
// List count instances by query params.
func (d *dao) Count(ctx context.Context, query *q.Query) (total int64, err error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
qs, err := orm.QuerySetter(ctx, &provider.Instance{}, query)
if err != nil {
return 0, err
}
return qs.Count()
}
// List lists instances by query params.
func (d *dao) List(ctx context.Context, query *q.Query) (ins []*provider.Instance, err error) {
ins = []*provider.Instance{}
qs, err := orm.QuerySetter(ctx, &provider.Instance{}, query)
if err != nil {
return nil, err
}
if _, err = qs.All(&ins); err != nil {
return nil, err
}
return ins, nil
}

140
src/pkg/p2p/preheat/dao/instance/dao_test.go Normal file
View File

@ -0,0 +1,140 @@
package instance
import (
"context"
"testing"
beego_orm "github.com/astaxie/beego/orm"
common_dao "github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
models "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
var (
defaultInstance = &models.Instance{
ID: 1,
Name: "dragonfly-cn-1",
Description: "fake dragonfly server",
Vendor: "dragonfly",
Endpoint: "https://cn-1.dragonfly.com",
AuthMode: "basic",
AuthData: "{\"username\": \"admin\", \"password\": \"123456\"}",
Status: "healthy",
Enabled: true,
SetupTimestamp: 1582721396,
}
)
type instanceSuite struct {
suite.Suite
ctx context.Context
dao DAO
}
func (is *instanceSuite) SetupSuite() {
common_dao.PrepareTestForPostgresSQL()
is.ctx = orm.NewContext(nil, beego_orm.NewOrm())
is.dao = New()
}
func (is *instanceSuite) SetupTest() {
t := is.T()
_, err := is.dao.Create(is.ctx, defaultInstance)
assert.Nil(t, err)
}
func (is *instanceSuite) TearDownTest() {
t := is.T()
err := is.dao.Delete(is.ctx, defaultInstance.ID)
assert.Nil(t, err)
}
func (is *instanceSuite) TestGet() {
t := is.T()
i, err := is.dao.Get(is.ctx, defaultInstance.ID)
assert.Nil(t, err)
assert.Equal(t, defaultInstance.Name, i.Name)
// not exist
i, err = is.dao.Get(is.ctx, 0)
assert.Nil(t, i)
}
func (is *instanceSuite) TestUpdate() {
t := is.T()
i, err := is.dao.Get(is.ctx, defaultInstance.ID)
assert.Nil(t, err)
assert.NotNil(t, i)
i.Enabled = false
err = is.dao.Update(is.ctx, i, "enabled")
assert.Nil(t, err)
i.Default = true
err = is.dao.Update(is.ctx, i, "default")
assert.NotNil(t, err)
i, err = is.dao.Get(is.ctx, defaultInstance.ID)
assert.Nil(t, err)
assert.NotNil(t, i)
assert.False(t, i.Enabled)
}
func (is *instanceSuite) TestList() {
t := is.T()
// add more instances
testInstance1 := &models.Instance{
ID: 2,
Name: "kraken-us-1",
Description: "fake kraken server",
Vendor: "kraken",
Endpoint: "https://us-1.kraken.com",
AuthMode: "none",
AuthData: "",
Status: "success",
Enabled: true,
SetupTimestamp: 0,
}
_, err := is.dao.Create(is.ctx, testInstance1)
assert.Nilf(t, err, "Create %d", testInstance1.ID)
defer func() {
// clean data
err = is.dao.Delete(is.ctx, testInstance1.ID)
assert.Nilf(t, err, "delete instance %d", testInstance1.ID)
}()
total, err := is.dao.Count(is.ctx, nil)
assert.Nil(t, err)
assert.Equal(t, total, int64(2))
// limit 1
total, err = is.dao.Count(is.ctx, &q.Query{PageSize: 1, PageNumber: 1})
assert.Nil(t, err)
assert.Equal(t, total, int64(2))
// without limit should return all instances
instances, err := is.dao.List(is.ctx, nil)
assert.Nil(t, err)
assert.Len(t, instances, 2)
// limit 1
instances, err = is.dao.List(is.ctx, &q.Query{PageSize: 1, PageNumber: 1})
assert.Nil(t, err)
assert.Len(t, instances, 1, "instances number")
assert.Equal(t, defaultInstance.ID, instances[0].ID)
// keyword search
keywords := make(map[string]interface{})
keywords["name"] = "kraken-us-1"
instances, err = is.dao.List(is.ctx, &q.Query{Keywords: keywords})
assert.Nil(t, err)
assert.Len(t, instances, 1)
assert.Equal(t, testInstance1.Name, instances[0].Name)
}
func TestInstance(t *testing.T) {
suite.Run(t, &instanceSuite{})
}

52
src/pkg/p2p/preheat/helper/helper.go Normal file
View File

@ -0,0 +1,52 @@
package helper
import (
"strings"
)
// ImageRepository represents the image repository name
// e.g: library/ubuntu:latest
type ImageRepository string
// Valid checks if the repository name is valid
func (ir ImageRepository) Valid() bool {
if len(ir) == 0 {
return false
}
trimName := strings.TrimSpace(string(ir))
segments := strings.SplitN(trimName, "/", 2)
if len(segments) != 2 {
return false
}
nameAndTag := segments[1]
subSegments := strings.SplitN(nameAndTag, ":", 2)
if len(subSegments) != 2 {
return false
}
return true
}
// Name returns the name of the image repository
func (ir ImageRepository) Name() string {
// No check here, should call Valid() before calling name
segments := strings.SplitN(string(ir), ":", 2)
if len(segments) == 0 {
return ""
}
return segments[0]
}
// Tag returns the tag of the image repository
func (ir ImageRepository) Tag() string {
// No check here, should call Valid() before calling name
segments := strings.SplitN(string(ir), ":", 2)
if len(segments) < 2 {
return ""
}
return segments[1]
}

63
src/pkg/p2p/preheat/helper/helper_test.go Normal file
View File

@ -0,0 +1,63 @@
package helper
import "testing"
func TestImageRepository_Valid(t *testing.T) {
tests := []struct {
name string
ir ImageRepository
want bool
}{
{"empty", "", false},
{"invalid", "abc", false},
{"invalid", "abc/def", false},
{"valid", "abc/def:tag", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.ir.Valid(); got != tt.want {
t.Errorf("ImageRepository.Valid() = %v, want %v", got, tt.want)
}
})
}
}
func TestImageRepository_Name(t *testing.T) {
tests := []struct {
name string
ir ImageRepository
want string
}{
{"empty", "", ""},
{"invalid", "abc", "abc"},
{"invalid", "abc/def", "abc/def"},
{"valid", "abc/def:tag", "abc/def"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.ir.Name(); got != tt.want {
t.Errorf("ImageRepository.Name() = %v, want %v", got, tt.want)
}
})
}
}
func TestImageRepository_Tag(t *testing.T) {
tests := []struct {
name string
ir ImageRepository
want string
}{
{"empty", "", ""},
{"invalid", "abc", ""},
{"invalid", "abc/def", ""},
{"valid", "abc/def:tag", "tag"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.ir.Tag(); got != tt.want {
t.Errorf("ImageRepository.Tag() = %v, want %v", got, tt.want)
}
})
}
}

111
src/pkg/p2p/preheat/instance/manager.go Normal file
View File

@ -0,0 +1,111 @@
package instance
import (
"context"
"github.com/goharbor/harbor/src/lib/q"
dao "github.com/goharbor/harbor/src/pkg/p2p/preheat/dao/instance"
"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider"
)
// Mgr is the global instance manager instance
var Mgr = New()
// Manager is responsible for storing the instances
type Manager interface {
// Save the instance metadata to the backend store
//
// inst *Instance : a ptr of instance
//
// If succeed, the uuid of the saved instance is returned;
// otherwise, a non nil error is returned
//
Save(ctx context.Context, inst *provider.Instance) (int64, error)
// Delete the specified instance
//
// id int64 : the id of the instance
//
// If succeed, a nil error is returned;
// otherwise, a non nil error is returned
//
Delete(ctx context.Context, id int64) error
// Update the specified instance
//
// inst *Instance : a ptr of instance
//
// If succeed, a nil error is returned;
// otherwise, a non nil error is returned
//
Update(ctx context.Context, inst *provider.Instance, props ...string) error
// Get the instance with the ID
//
// id int64 : the id of the instance
//
// If succeed, a non nil Instance is returned;
// otherwise, a non nil error is returned
//
Get(ctx context.Context, id int64) (*provider.Instance, error)
// Count the instances by the param
//
// query *q.Query : the query params
Count(ctx context.Context, query *q.Query) (int64, error)
// Query the instances by the param
//
// query *q.Query : the query params
//
// If succeed, an instance list is returned;
// otherwise, a non nil error is returned
//
List(ctx context.Context, query *q.Query) ([]*provider.Instance, error)
}
// manager implement the Manager interface
type manager struct {
dao dao.DAO
}
// New returns an instance of DefaultManger
func New() Manager {
return &manager{
dao: dao.New(),
}
}
// Ensure *manager has implemented Manager interface.
var _ Manager = (*manager)(nil)
// Save implements @Manager.Save
func (dm *manager) Save(ctx context.Context, inst *provider.Instance) (int64, error) {
return dm.dao.Create(ctx, inst)
}
// Delete implements @Manager.Delete
func (dm *manager) Delete(ctx context.Context, id int64) error {
return dm.dao.Delete(ctx, id)
}
// Update implements @Manager.Update
func (dm *manager) Update(ctx context.Context, inst *provider.Instance, props ...string) error {
return dm.dao.Update(ctx, inst, props...)
}
// Get implements @Manager.Get
func (dm *manager) Get(ctx context.Context, id int64) (*provider.Instance, error) {
return dm.dao.Get(ctx, id)
}
// Count implements @Manager.Count
func (dm *manager) Count(ctx context.Context, query *q.Query) (int64, error) {
return dm.dao.Count(ctx, query)
}
// List implements @Manager.List
func (dm *manager) List(ctx context.Context, query *q.Query) ([]*provider.Instance, error) {
return dm.dao.List(ctx, query)
}

122
src/pkg/p2p/preheat/instance/manager_test.go Normal file
View File

@ -0,0 +1,122 @@
package instance
import (
"context"
"testing"
"github.com/goharbor/harbor/src/lib/q"
dao "github.com/goharbor/harbor/src/pkg/p2p/preheat/dao/instance"
"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider"
providerModel "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type fakeDao struct {
mock.Mock
}
var _ dao.DAO = (*fakeDao)(nil)
func (d *fakeDao) Create(ctx context.Context, instance *provider.Instance) (int64, error) {
var args = d.Called()
return int64(args.Int(0)), args.Error(1)
}
func (d *fakeDao) Get(ctx context.Context, id int64) (*provider.Instance, error) {
var args = d.Called()
var instance *provider.Instance
if args.Get(0) != nil {
instance = args.Get(0).(*provider.Instance)
}
return instance, args.Error(1)
}
func (d *fakeDao) Update(ctx context.Context, instance *provider.Instance, props ...string) error {
var args = d.Called()
return args.Error(0)
}
func (d *fakeDao) Delete(ctx context.Context, id int64) error {
var args = d.Called()
return args.Error(0)
}
func (d *fakeDao) Count(ctx context.Context, query *q.Query) (total int64, err error) {
var args = d.Called()
return int64(args.Int(0)), args.Error(1)
}
func (d *fakeDao) List(ctx context.Context, query *q.Query) (ins []*provider.Instance, err error) {
var args = d.Called()
var instances []*provider.Instance
if args.Get(0) != nil {
instances = args.Get(0).([]*provider.Instance)
}
return instances, args.Error(1)
}
type instanceManagerSuite struct {
suite.Suite
dao *fakeDao
ctx context.Context
manager Manager
}
func (im *instanceManagerSuite) SetupSuite() {
im.dao = &fakeDao{}
im.manager = &manager{dao: im.dao}
}
func (im *instanceManagerSuite) TestSave() {
im.dao.On("Create").Return(1, nil)
id, err := im.manager.Save(im.ctx, nil)
im.Require().Nil(err)
im.Require().Equal(int64(1), id)
}
func (im *instanceManagerSuite) TestDelete() {
im.dao.On("Delete").Return(nil)
err := im.manager.Delete(im.ctx, 1)
im.Require().Nil(err)
}
func (im *instanceManagerSuite) TestUpdate() {
im.dao.On("Update").Return(nil)
err := im.manager.Update(im.ctx, nil)
im.Require().Nil(err)
}
func (im *instanceManagerSuite) TestGet() {
ins := &providerModel.Instance{Name: "abc"}
im.dao.On("Get").Return(ins, nil)
res, err := im.manager.Get(im.ctx, 1)
im.Require().Nil(err)
im.Require().Equal(ins, res)
}
func (im *instanceManagerSuite) TestCount() {
im.dao.On("Count").Return(2, nil)
count, err := im.manager.Count(im.ctx, nil)
assert.Nil(im.T(), err)
assert.Equal(im.T(), int64(2), count)
}
func (im *instanceManagerSuite) TestList() {
lists := []*providerModel.Instance{
{Name: "abc"},
{Name: "def"},
}
im.dao.On("List").Return(lists, nil)
res, err := im.manager.List(im.ctx, nil)
assert.Nil(im.T(), err)
assert.Len(im.T(), res, 2)
assert.Equal(im.T(), lists, res)
}
func TestInstanceManager(t *testing.T) {
suite.Run(t, &instanceManagerSuite{})
}

138
src/pkg/p2p/preheat/instance/mocks/Manager.go Normal file
View File

@ -0,0 +1,138 @@
// Code generated by mockery v1.0.0. DO NOT EDIT.
package mocks
import (
context "context"
q "github.com/goharbor/harbor/src/lib/q"
provider "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider"
mock "github.com/stretchr/testify/mock"
)
// Manager is an autogenerated mock type for the Manager type
type Manager struct {
mock.Mock
}
// Count provides a mock function with given fields: ctx, query
func (_m *Manager) Count(ctx context.Context, query *q.Query) (int64, error) {
ret := _m.Called(ctx, query)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
r0 = rf(ctx, query)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
r1 = rf(ctx, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Delete provides a mock function with given fields: ctx, id
func (_m *Manager) Delete(ctx context.Context, id int64) error {
ret := _m.Called(ctx, id)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// Get provides a mock function with given fields: ctx, id
func (_m *Manager) Get(ctx context.Context, id int64) (*provider.Instance, error) {
ret := _m.Called(ctx, id)
var r0 *provider.Instance
if rf, ok := ret.Get(0).(func(context.Context, int64) *provider.Instance); ok {
r0 = rf(ctx, id)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*provider.Instance)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// List provides a mock function with given fields: ctx, query
func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*provider.Instance, error) {
ret := _m.Called(ctx, query)
var r0 []*provider.Instance
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*provider.Instance); ok {
r0 = rf(ctx, query)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*provider.Instance)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
r1 = rf(ctx, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Save provides a mock function with given fields: ctx, inst
func (_m *Manager) Save(ctx context.Context, inst *provider.Instance) (int64, error) {
ret := _m.Called(ctx, inst)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, *provider.Instance) int64); ok {
r0 = rf(ctx, inst)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *provider.Instance) error); ok {
r1 = rf(ctx, inst)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Update provides a mock function with given fields: ctx, inst, props
func (_m *Manager) Update(ctx context.Context, inst *provider.Instance, props ...string) error {
_va := make([]interface{}, len(props))
for _i := range props {
_va[_i] = props[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, inst)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, *provider.Instance, ...string) error); ok {
r0 = rf(ctx, inst, props...)
} else {
r0 = ret.Error(0)
}
return r0
}

14
src/pkg/p2p/preheat/models/provider/instance.go
View File

@ -17,6 +17,7 @@ package provider
import ( import (
"encoding/json" "encoding/json"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/errors"
) )
@ -33,6 +34,10 @@ const (
PreheatingStatusFail = "FAIL" PreheatingStatusFail = "FAIL"
) )
func init() {
orm.RegisterModel(&Instance{})
}
// Instance defines the properties of the preheating provider instance. // Instance defines the properties of the preheating provider instance.
type Instance struct { type Instance struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"` ID int64 `orm:"pk;auto;column(id)" json:"id"`
@ -42,11 +47,11 @@ type Instance struct {
Endpoint string `orm:"column(endpoint)" json:"endpoint"` Endpoint string `orm:"column(endpoint)" json:"endpoint"`
AuthMode string `orm:"column(auth_mode)" json:"auth_mode"` AuthMode string `orm:"column(auth_mode)" json:"auth_mode"`
// The auth credential data if exists // The auth credential data if exists
AuthInfo map[string]string `orm:"column(-)" json:"auth_info,omitempty"` AuthInfo map[string]string `orm:"-" json:"auth_info,omitempty"`
// Data format for "AuthInfo" // Data format for "AuthInfo"
AuthData string `orm:"column(auth_data)" json:"-"` AuthData string `orm:"column(auth_data)" json:"-"`
// Default 'Unknown', use separate API for client to retrieve // Default 'Unknown', use separate API for client to retrieve
Status string `orm:"column(-)" json:"status"` Status string `orm:"-" json:"status"`
Enabled bool `orm:"column(enabled)" json:"enabled"` Enabled bool `orm:"column(enabled)" json:"enabled"`
Default bool `orm:"column(is_default)" json:"default"` Default bool `orm:"column(is_default)" json:"default"`
Insecure bool `orm:"column(insecure)" json:"insecure"` Insecure bool `orm:"column(insecure)" json:"insecure"`
@ -75,3 +80,8 @@ func (ins *Instance) ToJSON() (string, error) {
return string(data), nil return string(data), nil
} }
// TableName ...
func (ins *Instance) TableName() string {
return "p2p_preheat_instance"
}

4
src/server/v2.0/handler/handler.go
View File

@ -15,10 +15,11 @@
package handler package handler
import ( import (
lib_http "github.com/goharbor/harbor/src/lib/http"
"log" "log"
"net/http" "net/http"
lib_http "github.com/goharbor/harbor/src/lib/http"
"github.com/goharbor/harbor/src/server/middleware" "github.com/goharbor/harbor/src/server/middleware"
"github.com/goharbor/harbor/src/server/middleware/blob" "github.com/goharbor/harbor/src/server/middleware/blob"
"github.com/goharbor/harbor/src/server/middleware/quota" "github.com/goharbor/harbor/src/server/middleware/quota"
@ -33,6 +34,7 @@ func New() http.Handler {
AuditlogAPI: newAuditLogAPI(), AuditlogAPI: newAuditLogAPI(),
ScanAPI: newScanAPI(), ScanAPI: newScanAPI(),
ProjectAPI: newProjectAPI(), ProjectAPI: newProjectAPI(),
PreheatAPI: newPreheatAPI(),
}) })
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)

82
src/server/v2.0/handler/preheat.go Normal file
View File

@ -0,0 +1,82 @@
package handler
import (
"context"
"github.com/go-openapi/runtime/middleware"
preheatCtl "github.com/goharbor/harbor/src/controller/p2p/preheat"
"github.com/goharbor/harbor/src/pkg/p2p/preheat/provider"
"github.com/goharbor/harbor/src/server/v2.0/models"
"github.com/goharbor/harbor/src/server/v2.0/restapi"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/preheat"
)
func newPreheatAPI() *preheatAPI {
return &preheatAPI{
preheatCtl: preheatCtl.Ctl,
}
}
var _ restapi.PreheatAPI = (*preheatAPI)(nil)
type preheatAPI struct {
BaseAPI
preheatCtl preheatCtl.Controller
}
func (api *preheatAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder {
return nil
}
func (api *preheatAPI) CreateInstance(ctx context.Context, params operation.CreateInstanceParams) middleware.Responder {
var payload *models.InstanceCreatedResp
return operation.NewCreateInstanceCreated().WithPayload(payload)
}
func (api *preheatAPI) DeleteInstance(ctx context.Context, params operation.DeleteInstanceParams) middleware.Responder {
var payload *models.InstanceDeletedResp
return operation.NewDeleteInstanceOK().WithPayload(payload)
}
func (api *preheatAPI) GetInstance(ctx context.Context, params operation.GetInstanceParams) middleware.Responder {
var payload *models.Instance
return operation.NewGetInstanceOK().WithPayload(payload)
}
// ListInstances is List p2p instances
func (api *preheatAPI) ListInstances(ctx context.Context, params operation.ListInstancesParams) middleware.Responder {
var payload []*models.Instance
return operation.NewListInstancesOK().WithPayload(payload)
}
func (api *preheatAPI) ListProviders(ctx context.Context, params operation.ListProvidersParams) middleware.Responder {
var providers, err = preheatCtl.Ctl.GetAvailableProviders()
if err != nil {
return operation.NewListProvidersInternalServerError()
}
var payload = convertProvidersToFrontend(providers)
return operation.NewListProvidersOK().WithPayload(payload)
}
// UpdateInstance is Update instance
func (api *preheatAPI) UpdateInstance(ctx context.Context, params operation.UpdateInstanceParams) middleware.Responder {
var payload *models.InstanceUpdateResp
return operation.NewUpdateInstanceOK().WithPayload(payload)
}
func convertProvidersToFrontend(backend []*provider.Metadata) (frontend []*models.Metadata) {
frontend = make([]*models.Metadata, 0)
for _, provider := range backend {
frontend = append(frontend, &models.Metadata{
ID: provider.ID,
Icon: provider.Icon,
Name: provider.Name,
Source: provider.Source,
Version: provider.Version,
Maintainers: provider.Maintainers,
})
}
return
}

33
src/server/v2.0/handler/preheat_test.go Normal file
View File

@ -0,0 +1,33 @@
package handler
import (
"reflect"
"testing"
"github.com/goharbor/harbor/src/pkg/p2p/preheat/provider"
"github.com/goharbor/harbor/src/server/v2.0/models"
)
func Test_convertProvidersToFrontend(t *testing.T) {
backend, _ := provider.ListProviders()
tests := []struct {
name string
backend []*provider.Metadata
wantFrontend []*models.Metadata
}{
{"",
backend,
[]*models.Metadata{
{ID: "dragonfly", Icon: "https://raw.githubusercontent.com/alibaba/Dragonfly/master/docs/images/logo.png", Maintainers: []string{"Jin Zhang/taiyun.zj@alibaba-inc.com"}, Name: "Dragonfly", Source: "https://github.com/alibaba/Dragonfly", Version: "0.10.1"},
{Icon: "https://github.com/uber/kraken/blob/master/assets/kraken-logo-color.svg", ID: "kraken", Maintainers: []string{"mmpei/peimingming@corp.netease.com"}, Name: "Kraken", Source: "https://github.com/uber/kraken", Version: "0.1.3"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotFrontend := convertProvidersToFrontend(tt.backend); !reflect.DeepEqual(gotFrontend, tt.wantFrontend) {
t.Errorf("convertProvidersToFrontend() = %#v, want %#v", gotFrontend, tt.wantFrontend)
}
})
}
}