Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

support acceleration service (#16682) 

1, support acceleration service endpoints manage.
2, support auto convert.
3, support nydus as a new kinds of accessory.

Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
Wang Yan 2022-04-11 11:46:59 +08:00 committed by GitHub
parent 955a857da0
commit 05d070865c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 1334 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

296
api/v2.0/swagger.yaml
View File

@ -3875,6 +3875,187 @@ paths:
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/accelerations:
post:
summary: Create a acceleration service
description: Create a acceleration service
tags:
- acceleration
operationId: createAccelerationService
parameters:
- $ref: '#/parameters/requestId'
- name: acceleration
in: body
description: The acceleration
required: true
schema:
$ref: '#/definitions/Acceleration'
responses:
'201':
$ref: '#/responses/201'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'409':
$ref: '#/responses/409'
'500':
$ref: '#/responses/500'
get:
summary: List the acceleration services
description: List the acceleration services
tags:
- acceleration
operationId: listAccelerationServices
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- name: name
in: query
type: string
required: false
description: Deprecated, use `q` instead.
responses:
'200':
description: Success
headers:
X-Total-Count:
description: The total count of the resources
type: integer
Link:
description: Link refers to the previous page and next page
type: string
schema:
type: array
items:
$ref: '#/definitions/Acceleration'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/accelerations/ping:
post:
summary: Check status of a acceleration service
description: Check status of a acceleration service
tags:
- acceleration
operationId: pingAccelerationService
parameters:
- $ref: '#/parameters/requestId'
- name: acceleration
in: body
description: The acceleration service
required: true
schema:
$ref: '#/definitions/AccelerationPing'
responses:
'200':
$ref: '#/responses/200'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/accelerations/{id}:
get:
summary: Get the specific acceleration service
description: Get the specific acceleration service
tags:
- acceleration
operationId: getAccelerationService
parameters:
- $ref: '#/parameters/requestId'
- name: id
in: path
type: integer
format: int64
required: true
description: acceleration ID
responses:
'200':
description: Success
schema:
$ref: '#/definitions/Acceleration'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
delete:
summary: Delete the specific acceleration service
description: Delete the specific acceleration service
tags:
- acceleration
operationId: deleteAccelerationService
parameters:
- $ref: '#/parameters/requestId'
- name: id
in: path
type: integer
format: int64
required: true
description: Acceleration ID
responses:
'200':
$ref: '#/responses/200'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'412':
$ref: '#/responses/412'
'500':
$ref: '#/responses/500'
put:
summary: Update the acceleration service
description: Update the acceleration service
tags:
- acceleration
operationId: updateAccelerationService
parameters:
- $ref: '#/parameters/requestId'
- name: id
in: path
type: integer
format: int64
required: true
description: The acceleration service ID
- name: acceleration
in: body
description: The acceleration service
required: true
schema:
$ref: '#/definitions/AccelerationUpdate'
responses:
'200':
$ref: '#/responses/200'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'409':
$ref: '#/responses/409'
'500':
$ref: '#/responses/500'
/scans/all/metrics:
get:
summary: Get the metrics of the latest scan all process
@ -6471,6 +6652,10 @@ definitions:
type: string
description: 'Whether scan images automatically when pushing. The valid values are "true", "false".'
x-nullable: true
auto_accelerate:
type: string
description: 'Whether acclerate images automatically when pushing. The valid values are "true", "false".'
x-nullable: true
reuse_sys_cve_allowlist:
type: string
description: 'Whether this project reuse the system level CVE allowlist as the allowlist of its own. The valid values are "true", "false".
@ -6826,6 +7011,117 @@ definitions:
value:
type: string
description: The endpoint value
AccelerationCredential:
type: object
properties:
type:
type: string
description: Credential type, such as 'basic', 'oauth'.
access_key:
type: string
description: Access key, e.g. user name when credential type is 'basic'.
access_secret:
type: string
description: Access secret, e.g. password when credential type is 'basic'.
Acceleration:
type: object
properties:
id:
type: integer
format: int64
description: The acceleration ID.
x-omitempty: false
url:
type: string
description: The acceleration URL string.
name:
type: string
description: The acceleration name.
type:
type: string
description: Type of the acceleration, e.g. 'nydus'.
credential:
$ref: '#/definitions/AccelerationCredential'
insecure:
type: boolean
description: Whether or not the certificate will be verified when Harbor tries to access the server.
description:
type: string
description: Description of the acceleration.
status:
type: string
description: Health status of the acceleration.
creation_time:
type: string
format: date-time
description: The create time of the policy.
update_time:
type: string
format: date-time
description: The update time of the policy.
AccelerationUpdate:
type: object
properties:
name:
type: string
description: The acceleration name.
x-nullable: true
description:
type: string
description: Description of the acceleration.
x-nullable: true
url:
type: string
description: The acceleration URL.
x-nullable: true
credential_type:
type: string
description: Credential type of the registry, e.g. 'basic'.
x-nullable: true
access_key:
type: string
description: The registry access key.
x-nullable: true
access_secret:
type: string
description: The registry access secret.
x-nullable: true
insecure:
type: boolean
description: Whether or not the certificate will be verified when Harbor tries to access the server.
x-nullable: true
AccelerationPing:
type: object
properties:
id:
type: integer
format: int64
description: The acceleration ID.
x-nullable: true
type:
type: string
description: Type of the acceleration, e.g. 'nudys'.
x-nullable: true
url:
type: string
description: The acceleration URL.
x-nullable: true
credential_type:
type: string
description: Credential type of the registry, e.g. 'basic'.
x-nullable: true
access_key:
type: string
description: The acceleration access key.
x-nullable: true
access_secret:
type: string
description: The acceleration access secret.
x-nullable: true
insecure:
type: boolean
description: Whether or not the certificate will be verified when Harbor tries to access the server.
x-nullable: true
FilterStyle:
type: object
description: The style of the resource filter

19
make/migrations/postgresql/0090_2.6.0_schema.up.sql
View File

@ -1,2 +1,19 @@
/* Correct project_metadata.public value, should only be true or false, other invaild value will be rewrite to false */
UPDATE project_metadata SET value='false' WHERE name='public' AND value NOT IN('true', 'false');
UPDATE project_metadata SET value='false' WHERE name='public' AND value NOT IN('true', 'false');
CREATE TABLE acceleration_registration
(
id SERIAL PRIMARY KEY NOT NULL,
name VARCHAR(128) UNIQUE NOT NULL,
url VARCHAR(256) NOT NULL,
access_key VARCHAR(255) NOT NULL,
access_secret VARCHAR(4096) NOT NULL,
insecure BOOLEAN NOT NULL DEFAULT FALSE,
creation_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
credential_type VARCHAR(16) NOT NULL,
type VARCHAR(128) NOT NULL,
description VARCHAR(1024) NULL,
health VARCHAR(16) NOT NULL
);

1
src/common/rbac/const.go
View File

@ -68,6 +68,7 @@ const (
ResourceUser = Resource("user")
ResourceUserGroup = Resource("user-group")
ResourceRegistry = Resource("registry")
ResourceAcceleration = Resource("acceleration")
ResourceReplication = Resource("replication")
ResourceDistribution = Resource("distribution")
ResourceGarbageCollection = Resource("garbage-collection")

1
src/controller/artifact/controller.go
View File

@ -173,6 +173,7 @@ func (c *controller) Ensure(ctx context.Context, repository, digest string, opti
e.Tag = option.Tags[0]
}
notification.AddEvent(ctx, e)
return created, artifact.ID, nil
}

6
src/controller/event/handler/internal/artifact.go
View File

@ -238,5 +238,11 @@ func (a *Handler) onPush(ctx context.Context, event *event.ArtifactEvent) error
}
}()
go func() {
if err := autoAcc(ctx, &artifact.Artifact{Artifact: *event.Artifact}, event.Tags...); err != nil {
log.Errorf("acc artifact %s@%s failed, error: %v", event.Artifact.RepositoryName, event.Artifact.Digest, err)
}
}()
return nil
}

38
src/controller/event/handler/internal/util.go
View File

@ -16,11 +16,15 @@ package internal
import (
"context"
"github.com/goharbor/harbor/src/lib/q"
accel "github.com/goharbor/harbor/src/pkg/acceleration"
accelModel "github.com/goharbor/harbor/src/pkg/acceleration/model"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/controller/scan"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/acceleration/adapter"
)
// autoScan scan artifact when the project of the artifact enable auto scan
@ -43,3 +47,37 @@ func autoScan(ctx context.Context, a *artifact.Artifact, tags ...string) error {
return scan.DefaultController.Scan(ctx, a, options...)
})(orm.SetTransactionOpNameToContext(ctx, "tx-auto-scan"))
}
// autoAcc accelerate artifact
func autoAcc(ctx context.Context, a *artifact.Artifact, tags ...string) error {
proj, err := project.Ctl.Get(ctx, a.ProjectID)
if err != nil {
return err
}
if !proj.AutoAcc() {
return nil
}
// convert
acceleration, err := adapter.GetFactory(accelModel.AccelerationTypeNydus)
if err != nil {
return err
}
accs, err := accel.Mgr.List(ctx, q.New(q.KeyWords{"type": accelModel.AccelerationTypeNydus}))
if err != nil {
return err
}
adapter, err := acceleration.Create(accs[0])
if err != nil {
return err
}
var tagName string
if len(tags) > 0 {
tagName = tags[0]
}
err = adapter.Convert(&a.Artifact, tagName)
if err != nil {
return err
}
return nil
}

48
src/pkg/acceleration/adapter/adpater.go Normal file
View File

@ -0,0 +1,48 @@
package adapter
import (
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/acceleration/model"
"github.com/goharbor/harbor/src/pkg/artifact"
)
var registry = map[string]Factory{}
// Factory creates a specific Adapter according to the params
type Factory interface {
Create(service *model.AccelerationService) (Adapter, error)
}
// Adapter interface defines the capabilities of AccelerationService
type Adapter interface {
// Convert ...
Convert(art *artifact.Artifact, tag string) error
// HealthCheck checks health status of registry
HealthCheck() (string, error)
}
// RegisterFactory registers one adapter factory to the registry
func RegisterFactory(t string, factory Factory) error {
if len(t) == 0 {
return errors.New("invalid registry type")
}
if factory == nil {
return errors.New("empty adapter factory")
}
if _, exist := registry[t]; exist {
return fmt.Errorf("adapter factory for %s already exists", t)
}
registry[t] = factory
return nil
}
// GetFactory gets the adapter factory by the specified name
func GetFactory(t string) (Factory, error) {
factory, exist := registry[t]
if !exist {
return nil, fmt.Errorf("adapter factory for %s not found", t)
}
return factory, nil
}

109
src/pkg/acceleration/adapter/nydus/nydus.go Normal file
View File

@ -0,0 +1,109 @@
package nydus
import (
"bytes"
"encoding/json"
"fmt"
event2 "github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/log"
adp "github.com/goharbor/harbor/src/pkg/acceleration/adapter"
"github.com/goharbor/harbor/src/pkg/acceleration/model"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/distribution"
notifyModel "github.com/goharbor/harbor/src/pkg/notifier/model"
"net/http"
"time"
)
func init() {
if err := adp.RegisterFactory(model.AccelerationTypeNydus, new(factory)); err != nil {
log.Errorf("failed to register factory for %s: %v", model.AccelerationTypeNydus, err)
return
}
log.Infof("the factory for adapter %s registered", model.AccelerationTypeNydus)
}
type factory struct {
}
// Create ...
func (f *factory) Create(r *model.AccelerationService) (adp.Adapter, error) {
return &adapter{
url: r.URL,
}, nil
}
var (
_ adp.Adapter = (*adapter)(nil)
)
type adapter struct {
url string
}
func (a *adapter) Convert(art *artifact.Artifact, tag string) error {
hc := &http.Client{}
addr := fmt.Sprintf("%s/api/v1/conversions", a.url)
payload, err := a.getPayload(art, tag)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, addr, bytes.NewReader(payload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := hc.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("nydus job(target: %s) response code is %d", addr, resp.StatusCode)
}
return nil
}
func (a *adapter) HealthCheck() (string, error) {
return model.Healthy, nil
}
func (a *adapter) getPayload(art *artifact.Artifact, tag string) ([]byte, error) {
url, err := BuildImageResourceURL(art.RepositoryName, tag)
if err != nil {
return []byte{}, err
}
payload := &notifyModel.Payload{
Type: event2.TopicPushArtifact,
Operator: "admin",
OccurAt: time.Now().Unix(),
EventData: &notifyModel.EventData{
Resources: []*notifyModel.Resource{
{
Digest: art.Digest,
Tag: tag,
ResourceURL: url,
},
},
},
}
return json.Marshal(payload)
}
// BuildImageResourceURL ...
func BuildImageResourceURL(repoName, reference string) (string, error) {
extURL, err := config.ExtURL()
if err != nil {
return "", fmt.Errorf("get external endpoint failed: %v", err)
}
if distribution.IsDigest(reference) {
return fmt.Sprintf("%s/%s@%s", extURL, repoName, reference), nil
}
return fmt.Sprintf("%s/%s:%s", extURL, repoName, reference), nil
}

129
src/pkg/acceleration/dao/dao.go Normal file
View File

@ -0,0 +1,129 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"context"
"time"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
)
// DAO defines the DAO operations of registry
type DAO interface {
// Create the registry
Create(ctx context.Context, as *AccelerationService) (id int64, err error)
// Count returns the count of registries according to the query
Count(ctx context.Context, query *q.Query) (count int64, err error)
// List the registries according to the query
List(ctx context.Context, query *q.Query) (ases []*AccelerationService, err error)
// Get the registry specified by ID
Get(ctx context.Context, id int64) (as *AccelerationService, err error)
// Update the specified registry
Update(ctx context.Context, as *AccelerationService, props ...string) (err error)
// Delete the registry specified by ID
Delete(ctx context.Context, id int64) (err error)
}
// NewDAO creates an instance of DAO
func NewDAO() DAO {
return &dao{}
}
type dao struct{}
func (d *dao) Create(ctx context.Context, as *AccelerationService) (int64, error) {
ormer, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
id, err := ormer.Insert(as)
if e := orm.AsConflictError(err, "AccelerationService %s already exists", as.Name); e != nil {
err = e
}
return id, err
}
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
qs, err := orm.QuerySetterForCount(ctx, &AccelerationService{}, query)
if err != nil {
return 0, err
}
return qs.Count()
}
func (d *dao) List(ctx context.Context, query *q.Query) ([]*AccelerationService, error) {
registries := []*AccelerationService{}
qs, err := orm.QuerySetter(ctx, &AccelerationService{}, query)
if err != nil {
return nil, err
}
if _, err = qs.All(&registries); err != nil {
return nil, err
}
return registries, nil
}
func (d *dao) Get(ctx context.Context, id int64) (*AccelerationService, error) {
registry := &AccelerationService{
ID: id,
}
ormer, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
if err := ormer.Read(registry); err != nil {
if e := orm.AsNotFoundError(err, "AccelerationService %d not found", id); e != nil {
err = e
}
return nil, err
}
return registry, nil
}
func (d *dao) Update(ctx context.Context, registry *AccelerationService, props ...string) error {
ormer, err := orm.FromContext(ctx)
if err != nil {
return err
}
registry.UpdateTime = time.Now()
n, err := ormer.Update(registry, props...)
if err != nil {
if e := orm.AsConflictError(err, "AccelerationService %s already exists", registry.Name); e != nil {
err = e
}
return err
}
if n == 0 {
return errors.NotFoundError(nil).WithMessage("registry %d not found", registry.ID)
}
return nil
}
func (d *dao) Delete(ctx context.Context, id int64) error {
ormer, err := orm.FromContext(ctx)
if err != nil {
return err
}
n, err := ormer.Delete(&AccelerationService{
ID: id,
})
if err != nil {
return err
}
if n == 0 {
return errors.NotFoundError(nil).WithMessage("AccelerationService %d not found", id)
}
return nil
}

45
src/pkg/acceleration/dao/model.go Normal file
View File

@ -0,0 +1,45 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"github.com/astaxie/beego/orm"
"time"
)
func init() {
orm.RegisterModel(&AccelerationService{})
}
// AccelerationService model in database
type AccelerationService struct {
ID int64 `orm:"pk;auto;column(id)"`
URL string `orm:"column(url)"`
Name string `orm:"column(name)"`
CredentialType string `orm:"column(credential_type);default(basic)"`
AccessKey string `orm:"column(access_key)"`
AccessSecret string `orm:"column(access_secret)"`
Type string `orm:"column(type)"`
Insecure bool `orm:"column(insecure)"`
Description string `orm:"column(description)"`
Status string `orm:"column(health)"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add"`
UpdateTime time.Time `orm:"column(update_time);auto_now"`
}
// TableName for artifact reference
func (a *AccelerationService) TableName() string {
return "acceleration_registration"
}

219
src/pkg/acceleration/manager.go Normal file
View File

@ -0,0 +1,219 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package acceleration
import (
"context"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/acceleration/adapter"
"github.com/goharbor/harbor/src/pkg/acceleration/dao"
"github.com/goharbor/harbor/src/pkg/acceleration/model"
_ "github.com/goharbor/harbor/src/pkg/acceleration/adapter/nydus"
)
var (
// Mgr is the global registry manager instance
Mgr = NewManager()
)
// Manager defines the registry related operations
type Manager interface {
// Create the registry
Create(ctx context.Context, registry *model.AccelerationService) (id int64, err error)
// Count returns the count of registries according to the query
Count(ctx context.Context, query *q.Query) (count int64, err error)
// List registries according to the query
List(ctx context.Context, query *q.Query) (registries []*model.AccelerationService, err error)
// Get the registry specified by ID
Get(ctx context.Context, id int64) (registry *model.AccelerationService, err error)
// Update the specified registry
Update(ctx context.Context, registry *model.AccelerationService, props ...string) (err error)
// Delete the registry specified by ID
Delete(ctx context.Context, id int64) (err error)
// CreateAdapter for the provided registry
CreateAdapter(ctx context.Context, registry *model.AccelerationService) (adapter adapter.Adapter, err error)
}
// NewManager creates an instance of registry manager
func NewManager() Manager {
return &manager{
dao: dao.NewDAO(),
}
}
type manager struct {
dao dao.DAO
}
func (m *manager) Create(ctx context.Context, registry *model.AccelerationService) (int64, error) {
reg, err := toDaoModel(registry)
if err != nil {
return 0, err
}
return m.dao.Create(ctx, reg)
}
func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) {
return m.dao.Count(ctx, query)
}
func (m *manager) List(ctx context.Context, query *q.Query) ([]*model.AccelerationService, error) {
registries, err := m.dao.List(ctx, query)
if err != nil {
return nil, err
}
var regs []*model.AccelerationService
for _, registry := range registries {
r, err := fromDaoModel(registry)
if err != nil {
return nil, err
}
regs = append(regs, r)
}
return regs, nil
}
func (m *manager) Get(ctx context.Context, id int64) (*model.AccelerationService, error) {
registry, err := m.dao.Get(ctx, id)
if err != nil {
return nil, err
}
return fromDaoModel(registry)
}
func (m *manager) Update(ctx context.Context, registry *model.AccelerationService, props ...string) error {
reg, err := toDaoModel(registry)
if err != nil {
return err
}
return m.dao.Update(ctx, reg, props...)
}
func (m *manager) Delete(ctx context.Context, id int64) error {
return m.dao.Delete(ctx, id)
}
func (m *manager) CreateAdapter(ctx context.Context, registry *model.AccelerationService) (adapter.Adapter, error) {
factory, err := adapter.GetFactory(registry.Type)
if err != nil {
return nil, err
}
return factory.Create(registry)
}
// decrypt checks whether access secret is set in the registry, if so, decrypt it.
func decrypt(secret string) (string, error) {
if len(secret) == 0 {
return "", nil
}
secretKey, err := config.SecretKey()
if err != nil {
return "", nil
}
decrypted, err := utils.ReversibleDecrypt(secret, secretKey)
if err != nil {
return "", err
}
return decrypted, nil
}
// encrypt checks whether access secret is set in the registry, if so, encrypt it.
func encrypt(secret string) (string, error) {
if len(secret) == 0 {
return secret, nil
}
secretKey, err := config.SecretKey()
if err != nil {
return "", nil
}
encrypted, err := utils.ReversibleEncrypt(secret, secretKey)
if err != nil {
return "", err
}
return encrypted, nil
}
// FromDaoModel converts DAO layer registry model to replication model.
// Also, if access secret is provided, decrypt it.
func fromDaoModel(registry *dao.AccelerationService) (*model.AccelerationService, error) {
r := &model.AccelerationService{
ID: registry.ID,
Name: registry.Name,
Description: registry.Description,
Type: registry.Type,
Credential: &model.Credential{},
URL: registry.URL,
Insecure: registry.Insecure,
Status: registry.Status,
CreationTime: registry.CreationTime,
UpdateTime: registry.UpdateTime,
}
if len(registry.AccessKey) != 0 {
credentialType := registry.CredentialType
if len(credentialType) == 0 {
credentialType = model.CredentialTypeBasic
}
decrypted, err := decrypt(registry.AccessSecret)
if err != nil {
return nil, err
}
r.Credential = &model.Credential{
Type: credentialType,
AccessKey: registry.AccessKey,
AccessSecret: decrypted,
}
}
return r, nil
}
// toDaoModel ...
func toDaoModel(registry *model.AccelerationService) (*dao.AccelerationService, error) {
m := &dao.AccelerationService{
ID: registry.ID,
URL: registry.URL,
Name: registry.Name,
Type: string(registry.Type),
Insecure: registry.Insecure,
Description: registry.Description,
Status: registry.Status,
CreationTime: registry.CreationTime,
UpdateTime: registry.UpdateTime,
}
if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 {
credentialType := registry.Credential.Type
if len(credentialType) == 0 {
credentialType = model.CredentialTypeBasic
}
encrypted, err := encrypt(registry.Credential.AccessSecret)
if err != nil {
return nil, err
}
m.CredentialType = string(credentialType)
m.AccessKey = registry.Credential.AccessKey
m.AccessSecret = encrypted
}
return m, nil
}

60
src/pkg/acceleration/model/model.go Normal file
View File

@ -0,0 +1,60 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"time"
)
const (
// AccelerationTypeNydu ...
AccelerationTypeNydus = "nydus"
// Healthy indicates registry is healthy
Healthy = "healthy"
// Unhealthy indicates registry is unhealthy
Unhealthy = "unhealthy"
// CredentialTypeBasic indicates credential by user name, password
CredentialTypeBasic = "basic"
// CredentialTypeOAuth indicates credential by OAuth token
CredentialTypeOAuth = "oauth"
// CredentialTypeSecret is only used by the communication of Harbor internal components
CredentialTypeSecret = "secret"
)
// Credential keeps the access key and/or secret for the related registry
type Credential struct {
// Type of the credential
Type string `json:"type"`
// The key of the access account, for OAuth token, it can be empty
AccessKey string `json:"access_key"`
// The secret or password for the key
AccessSecret string `json:"access_secret"`
}
// AccelerationService ...
type AccelerationService struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Type string `json:"type"`
URL string `json:"url"`
Credential *Credential `json:"credential"`
Insecure bool `json:"insecure"`
Status string `json:"status"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
}

1
src/pkg/accessory/manager.go
View File

@ -24,6 +24,7 @@ import (
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
)
var (

2
src/pkg/accessory/model/accessory.go
View File

@ -63,6 +63,8 @@ const (
TypeNone = "base"
// TypeCosignSignature ...
TypeCosignSignature = "signature.cosign"
// TypeAccelNydus ...
TypeAccelNydus = "acceleration.nydus"
)
// AccessoryData ...

46
src/pkg/accessory/model/nydus/nydus.go Normal file
View File

@ -0,0 +1,46 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nydus
import (
"github.com/goharbor/harbor/src/pkg/accessory/model"
"github.com/goharbor/harbor/src/pkg/accessory/model/base"
)
// Nydus signature model
type Nydus struct {
base.Default
}
// Kind gives the reference type of nydus.
func (n *Nydus) Kind() string {
return model.RefHard
}
// IsHard ...
func (n *Nydus) IsHard() bool {
return true
}
// New returns nydus
func New(data model.AccessoryData) model.Accessory {
return &Nydus{base.Default{
Data: data,
}}
}
func init() {
model.Register(model.TypeAccelNydus, New)
}

1
src/pkg/project/models/pro_meta.go
View File

@ -22,5 +22,6 @@ const (
ProMetaPreventVul = "prevent_vul" // prevent vulnerable images from being pulled
ProMetaSeverity = "severity"
ProMetaAutoScan = "auto_scan"
ProMetaAutoAcc = "auto_accelerate"
ProMetaReuseSysCVEAllowlist = "reuse_sys_cve_allowlist"
)

9
src/pkg/project/models/project.go
View File

@ -148,6 +148,15 @@ func (p *Project) AutoScan() bool {
return isTrue(auto)
}
// AutoAcc ...
func (p *Project) AutoAcc() bool {
auto, exist := p.GetMetadata(ProMetaAutoAcc)
if !exist {
return false
}
return isTrue(auto)
}
// FilterByPublic returns orm.QuerySeter with public filter
func (p *Project) FilterByPublic(ctx context.Context, qs orm.QuerySeter, key string, value interface{}) orm.QuerySeter {
subQuery := `SELECT project_id FROM project_metadata WHERE name = 'public' AND value = '%s'`

106
src/server/middleware/nydus/nydus.go Normal file
View File

@ -0,0 +1,106 @@
package nydus
import (
"context"
"encoding/json"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/accessory"
"github.com/goharbor/harbor/src/pkg/accessory/model"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"io/ioutil"
"net/http"
)
var (
// the media type of consign signature layer
mediaTypeNydusLayer = "application/vnd.oci.image.layer.nydus.blob.v1"
)
// NydusMiddleware middleware to record the linkage of artifact and its accessory
func NydusMiddleware() func(http.Handler) http.Handler {
return middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
if statusCode != http.StatusCreated {
return nil
}
ctx := r.Context()
logger := log.G(ctx).WithFields(log.Fields{"middleware": "nydus"})
none := lib.ArtifactInfo{}
info := lib.GetArtifactInfo(ctx)
if info == none {
return errors.New("artifactinfo middleware required before this middleware").WithCode(errors.NotFoundCode)
}
if info.Tag == "" {
return nil
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return err
}
contentType := r.Header.Get("Content-Type")
manifest, desc, err := distribution.UnmarshalManifest(contentType, body)
if err != nil {
logger.Errorf("unmarshal manifest failed, error: %v", err)
return err
}
var isNydus bool
for _, descriptor := range manifest.References() {
if descriptor.MediaType == mediaTypeNydusLayer {
isNydus = true
break
}
}
_, content, err := manifest.Payload()
if err != nil {
return err
}
// get manifest
mani := &v1.Manifest{}
if err := json.Unmarshal(content, mani); err != nil {
return err
}
if isNydus {
subjectArt, err := artifact.Ctl.GetByReference(ctx, info.Repository, mani.Annotations["io.goharbor.artifact.v1alpha1.acceleration.source.digest"], nil)
if err != nil {
logger.Errorf("failed to get subject artifact: %s, error: %v", info.Tag, err)
return err
}
art, err := artifact.Ctl.GetByReference(ctx, info.Repository, desc.Digest.String(), nil)
if err != nil {
logger.Errorf("failed to get cosign signature artifact: %s, error: %v", desc.Digest.String(), err)
return err
}
if err := orm.WithTransaction(func(ctx context.Context) error {
_, err := accessory.Mgr.Create(ctx, model.AccessoryData{
ArtifactID: art.ID,
SubArtifactID: subjectArt.ID,
Size: desc.Size,
Digest: desc.Digest.String(),
Type: model.TypeAccelNydus,
})
return err
})(orm.SetTransactionOpNameToContext(ctx, "tx-create-nydus-accessory")); err != nil {
if !errors.IsConflictErr(err) {
logger.Errorf("failed to create cosign signature artifact: %s, error: %v", desc.Digest.String(), err)
return err
}
}
}
return nil
})
}

3
src/server/middleware/requestid/requestid.go
View File

@ -15,12 +15,11 @@
package requestid
import (
"net/http"
tracelib "github.com/goharbor/harbor/src/lib/trace"
"github.com/goharbor/harbor/src/server/middleware"
"go.opentelemetry.io/otel/attribute"
oteltrace "go.opentelemetry.io/otel/trace"
"net/http"
"github.com/google/uuid"
)

2
src/server/registry/route.go
View File

@ -15,6 +15,7 @@
package registry
import (
"github.com/goharbor/harbor/src/server/middleware/nydus"
"net/http"
"github.com/goharbor/harbor/src/server/middleware/blob"
@ -79,6 +80,7 @@ func RegisterRoutes() {
Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()).
Middleware(immutable.Middleware()).
Middleware(quota.PutManifestMiddleware()).
Middleware(nydus.NydusMiddleware()).
Middleware(cosign.CosignSignatureMiddleware()).
Middleware(blob.PutManifestMiddleware()).
HandlerFunc(putManifest)

194
src/server/v2.0/handler/acceleration.go Normal file
View File

@ -0,0 +1,194 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package handler
import (
"context"
"fmt"
"github.com/go-openapi/strfmt"
"github.com/goharbor/harbor/src/pkg/acceleration"
"strings"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/acceleration/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/acceleration"
)
func newAccelerationAPI() *accelerationAPI {
return &accelerationAPI{
mgr: acceleration.Mgr,
}
}
type accelerationAPI struct {
BaseAPI
mgr acceleration.Manager
}
func (r *accelerationAPI) CreateAccelerationService(ctx context.Context, params operation.CreateAccelerationServiceParams) middleware.Responder {
if err := r.RequireSystemAccess(ctx, rbac.ActionCreate, rbac.ResourceAcceleration); err != nil {
return r.SendError(ctx, err)
}
accel := &model.AccelerationService{
Name: params.Acceleration.Name,
Description: params.Acceleration.Description,
Type: params.Acceleration.Type,
URL: params.Acceleration.URL,
Insecure: params.Acceleration.Insecure,
}
if params.Acceleration.Credential != nil {
accel.Credential = &model.Credential{
Type: params.Acceleration.Credential.Type,
AccessKey: params.Acceleration.Credential.AccessKey,
AccessSecret: params.Acceleration.Credential.AccessSecret,
}
}
id, err := r.mgr.Create(ctx, accel)
if err != nil {
return r.SendError(ctx, err)
}
location := fmt.Sprintf("%s/%d", strings.TrimSuffix(params.HTTPRequest.URL.Path, "/"), id)
return operation.NewCreateAccelerationServiceCreated().WithLocation(location)
}
func (r *accelerationAPI) GetAccelerationService(ctx context.Context, params operation.GetAccelerationServiceParams) middleware.Responder {
if err := r.RequireSystemAccess(ctx, rbac.ActionRead, rbac.ResourceAcceleration); err != nil {
return r.SendError(ctx, err)
}
accel, err := r.mgr.Get(ctx, params.ID)
if err != nil {
return r.SendError(ctx, err)
}
return operation.NewGetAccelerationServiceOK().WithPayload(convertAcceleration(accel))
}
func (r *accelerationAPI) ListAccelerationServices(ctx context.Context, params operation.ListAccelerationServicesParams) middleware.Responder {
if err := r.RequireSystemAccess(ctx, rbac.ActionList, rbac.ResourceAcceleration); err != nil {
return r.SendError(ctx, err)
}
query, err := r.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return r.SendError(ctx, err)
}
// keep backward compatibility for the "name" query
if params.Name != nil {
query.Keywords["Name"] = q.NewFuzzyMatchValue(*params.Name)
}
total, err := r.mgr.Count(ctx, query)
if err != nil {
return r.SendError(ctx, err)
}
accs, err := r.mgr.List(ctx, query)
if err != nil {
return r.SendError(ctx, err)
}
var accels []*models.Acceleration
for _, acc := range accs {
accels = append(accels, convertAcceleration(acc))
}
return operation.NewListAccelerationServicesOK().WithXTotalCount(total).
WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
WithPayload(accels)
}
func (r *accelerationAPI) DeleteAccelerationService(ctx context.Context, params operation.DeleteAccelerationServiceParams) middleware.Responder {
if err := r.RequireSystemAccess(ctx, rbac.ActionDelete, rbac.ResourceAcceleration); err != nil {
return r.SendError(ctx, err)
}
if err := r.mgr.Delete(ctx, params.ID); err != nil {
return r.SendError(ctx, err)
}
return operation.NewDeleteAccelerationServiceOK()
}
func (r *accelerationAPI) UpdateAccelerationService(ctx context.Context, params operation.UpdateAccelerationServiceParams) middleware.Responder {
if err := r.RequireSystemAccess(ctx, rbac.ActionUpdate, rbac.ResourceAcceleration); err != nil {
return r.SendError(ctx, err)
}
accel, err := r.mgr.Get(ctx, params.ID)
if err != nil {
return r.SendError(ctx, err)
}
if params.Acceleration != nil {
if params.Acceleration.Name != nil {
accel.Name = *params.Acceleration.Name
}
if params.Acceleration.Description != nil {
accel.Description = *params.Acceleration.Description
}
if params.Acceleration.URL != nil {
accel.URL = *params.Acceleration.URL
}
if params.Acceleration.Insecure != nil {
accel.Insecure = *params.Acceleration.Insecure
}
if accel.Credential == nil {
accel.Credential = &model.Credential{}
}
if params.Acceleration.CredentialType != nil {
accel.Credential.Type = *params.Acceleration.CredentialType
}
if params.Acceleration.AccessKey != nil {
accel.Credential.AccessKey = *params.Acceleration.AccessKey
}
if params.Acceleration.AccessSecret != nil {
accel.Credential.AccessSecret = *params.Acceleration.AccessSecret
}
}
if err := r.mgr.Update(ctx, accel); err != nil {
return r.SendError(ctx, err)
}
return operation.NewUpdateAccelerationServiceOK()
}
func (r *accelerationAPI) PingAccelerationService(ctx context.Context, params operation.PingAccelerationServiceParams) middleware.Responder {
if err := r.RequireSystemAccess(ctx, rbac.ActionRead, rbac.ResourceAcceleration); err != nil {
return r.SendError(ctx, err)
}
return operation.NewPingAccelerationServiceOK()
}
func convertAcceleration(registry *model.AccelerationService) *models.Acceleration {
r := &models.Acceleration{
CreationTime: strfmt.DateTime(registry.CreationTime),
Description: registry.Description,
ID: registry.ID,
Insecure: registry.Insecure,
Name: registry.Name,
Status: registry.Status,
Type: string(registry.Type),
UpdateTime: strfmt.DateTime(registry.UpdateTime),
URL: registry.URL,
}
if registry.Credential != nil {
credential := &models.AccelerationCredential{
AccessKey: registry.Credential.AccessKey,
Type: string(registry.Credential.Type),
}
if len(registry.Credential.AccessSecret) > 0 {
credential.AccessSecret = "*****"
}
r.Credential = credential
}
return r
}

1
src/server/v2.0/handler/handler.go
View File

@ -46,6 +46,7 @@ func New() http.Handler {
Robotv1API: newRobotV1API(),
ReplicationAPI: newReplicationAPI(),
RegistryAPI: newRegistryAPI(),
AccelerationAPI: newAccelerationAPI(),
SysteminfoAPI: newSystemInfoAPI(),
PingAPI: newPingAPI(),
LdapAPI: newLdapAPI(),

2
src/server/v2.0/handler/project_metadata.go
View File

@ -141,7 +141,7 @@ func (p *projectMetadataAPI) validate(metas map[string]string) (map[string]strin
switch key {
case proModels.ProMetaPublic, proModels.ProMetaEnableContentTrust, proModels.ProMetaEnableContentTrustCosign,
proModels.ProMetaPreventVul, proModels.ProMetaAutoScan:
proModels.ProMetaPreventVul, proModels.ProMetaAutoScan, proModels.ProMetaAutoAcc:
v, err := strconv.ParseBool(value)
if err != nil {
return nil, errors.New(nil).WithCode(errors.BadRequestCode).WithMessage("invalid value: %s", value)