Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-09-17 16:21:06 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7339 from ywk253100/190410_ns

Browse Source 
Update the adapter interface
This commit is contained in:
Wenkai Yin 2019-04-11 00:02:12 +08:00 committed by GitHub
commit c4c2577517
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 441 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/core/api/repository.go
View File

@ -333,9 +333,14 @@ func (ra *RepositoryAPI) Delete() {
Resource: &model.Resource{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Name: repoName,
Namespace: projectName,
Vtags: []string{tag},
Namespace: &model.Namespace{
Name: projectName,
// TODO filling the metadata
},
Repository: &model.Repository{
Name: strings.TrimPrefix(repoName, projectName+"/"),
},
Vtags: []string{tag},
},
Deleted: true,
},

11
src/core/service/notifications/registry/handler.go
View File

@ -120,9 +120,14 @@ func (n *NotificationHandler) Post() {
Resource: &model.Resource{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Name: repository,
Namespace: project,
Vtags: []string{tag},
Namespace: &model.Namespace{
Name: project,
// TODO filling the metadata
},
Repository: &model.Repository{
Name: strings.TrimPrefix(repository, project+"/"),
},
Vtags: []string{tag},
},
},
}

11
src/replication/ng/adapter/adapter.go
View File

@ -33,12 +33,15 @@ type Adapter interface {
// Lists the available namespaces under the specified registry with the
// provided credential/token
ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error)
// Create a new namespace
// This method should guarantee it's idempotent
// And returns nil if a namespace with the same name already exists
CreateNamespace(*model.Namespace) error
// ConvertResourceMetadata converts the namespace and repository part of the resource metadata
// to the one that the adapter can handle
ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error)
// PrepareForPush does the prepare work that needed for pushing/uploading the resource
// eg: create the namespace or repository
PrepareForPush(*model.Resource) error
// Get the namespace specified by the name, the returning value should
// contain the metadata about the namespace if it has
// TODO remove this method?
GetNamespace(string) (*model.Namespace, error)
// HealthCheck checks health status of registry
HealthCheck() (model.HealthStatus, error)

65
src/replication/ng/adapter/harbor/adapter.go
View File

@ -15,8 +15,10 @@
package harbor
import (
"errors"
"fmt"
"net/http"
"strings"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
@ -85,7 +87,8 @@ func newAdapter(registry *model.Registry) *adapter {
func (a *adapter) Info() (*model.RegistryInfo, error) {
info := &model.RegistryInfo{
Type: model.RegistryTypeHarbor,
Type: model.RegistryTypeHarbor,
SupportNamespace: true,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeRepository,
},
@ -126,13 +129,63 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
func (a *adapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
return nil, nil
}
func (a *adapter) CreateNamespace(namespace *model.Namespace) error {
func (a *adapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
if metadata == nil {
return nil, errors.New("the metadata cannot be null")
}
name := metadata.GetResourceName()
strs := strings.SplitN(name, "/", 2)
if len(strs) < 2 {
return nil, fmt.Errorf("unsupported resource name %s, at least contains one '/'", name)
}
meta := &model.ResourceMetadata{
Vtags: metadata.Vtags,
Labels: metadata.Labels,
}
meta.Namespace = &model.Namespace{
Name: strs[0],
}
if metadata.Namespace != nil {
meta.Namespace.Metadata = metadata.Namespace.Metadata
}
meta.Repository = &model.Repository{
Name: strs[1],
}
if metadata.Repository != nil {
meta.Repository.Metadata = metadata.Repository.Metadata
}
// replace the namespace if it is specified
if namespace == nil || len(namespace.Name) == 0 {
return meta, nil
}
if strings.Contains(namespace.Name, "/") {
return nil, fmt.Errorf("the namespace %s cannot contain '/'", namespace.Name)
}
meta.Namespace.Name = namespace.Name
if namespace.Metadata != nil {
meta.Namespace.Metadata = namespace.Metadata
}
return meta, nil
}
func (a *adapter) PrepareForPush(resource *model.Resource) error {
if resource == nil {
return errors.New("the resource cannot be null")
}
if resource.Metadata == nil {
return errors.New("the metadata of resource cannot be null")
}
if resource.Metadata.Namespace == nil {
return errors.New("the namespace of resource cannot be null")
}
if len(resource.Metadata.Namespace.Name) == 0 {
return errors.New("the name of the namespace cannot be null")
}
project := &struct {
Name string `json:"project_name"`
Metadata map[string]interface{} `json:"metadata"`
}{
Name: namespace.Name,
Metadata: namespace.Metadata,
Name: resource.Metadata.Namespace.Name,
Metadata: resource.Metadata.Namespace.Metadata,
}
// TODO
@ -160,11 +213,13 @@ func (a *adapter) CreateNamespace(namespace *model.Namespace) error {
err := a.client.Post(a.coreServiceURL+"/api/projects", project)
if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
log.Debugf("got 409 when trying to create project %s", namespace.Name)
log.Debugf("got 409 when trying to create project %s", resource.Metadata.Namespace.Name)
return nil
}
return err
}
// TODO remove this method
func (a *adapter) GetNamespace(namespace string) (*model.Namespace, error) {
project, err := a.getProject(namespace)
if err != nil {

53
src/replication/ng/adapter/harbor/adapter_test.go
View File

@ -18,12 +18,10 @@ import (
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication/ng/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestInfo(t *testing.T) {
@ -77,8 +75,7 @@ func TestListNamespaces(t *testing.T) {
// TODO
}
func TestCreateNamespace(t *testing.T) {
// project doesn't exist
func TestPrepareForPush(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodPost,
Pattern: "/api/projects",
@ -90,10 +87,41 @@ func TestCreateNamespace(t *testing.T) {
URL: server.URL,
}
adapter := newAdapter(registry)
err := adapter.CreateNamespace(&model.Namespace{
Name: "library",
// nil resource
err := adapter.PrepareForPush(nil)
require.NotNil(t, err)
// nil metadata
err = adapter.PrepareForPush(&model.Resource{})
require.NotNil(t, err)
// nil namespace
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{},
})
require.NotNil(t, err)
// nil namespace name
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{},
},
})
require.NotNil(t, err)
// nil namespace name
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{},
},
})
require.NotNil(t, err)
// project doesn't exist
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
},
})
require.Nil(t, err)
server.Close()
// project already exists
@ -108,11 +136,14 @@ func TestCreateNamespace(t *testing.T) {
URL: server.URL,
}
adapter = newAdapter(registry)
err = adapter.CreateNamespace(&model.Namespace{
Name: "library",
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
},
})
require.Nil(t, err)
server.Close()
}
func TestGetNamespace(t *testing.T) {

11
src/replication/ng/adapter/harbor/chart_registry.go
View File

@ -67,9 +67,14 @@ func (a *adapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*
Type: model.ResourceTypeChart,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Namespace: namespace,
Name: fmt.Sprintf("%s/%s", namespace, chart.Name),
Vtags: []string{version.Version},
Namespace: &model.Namespace{
Name: namespace,
// TODO filling the metadata
},
Repository: &model.Repository{
Name: chart.Name,
},
Vtags: []string{version.Version},
},
})
}

4
src/replication/ng/adapter/harbor/chart_registry_test.go
View File

@ -61,8 +61,8 @@ func TestFetchCharts(t *testing.T) {
require.Nil(t, err)
assert.Equal(t, 2, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, "library/harbor", resources[0].Metadata.Name)
assert.Equal(t, "library", resources[0].Metadata.Namespace)
assert.Equal(t, "harbor", resources[0].Metadata.Repository.Name)
assert.Equal(t, "library", resources[0].Metadata.Namespace.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
}

12
src/replication/ng/adapter/harbor/image_registry.go
View File

@ -16,6 +16,7 @@ package harbor
import (
"fmt"
"strings"
"github.com/goharbor/harbor/src/replication/ng/model"
)
@ -56,9 +57,14 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*
Type: model.ResourceTypeRepository,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Namespace: namespace,
Name: repository.Name,
Vtags: vtags,
Namespace: &model.Namespace{
Name: namespace,
// TODO filling the metadata
},
Repository: &model.Repository{
Name: strings.TrimPrefix(repository.Name, namespace+"/"),
},
Vtags: vtags,
},
})
}

4
src/replication/ng/adapter/harbor/image_registry_test.go
View File

@ -71,8 +71,8 @@ func TestFetchImages(t *testing.T) {
require.Nil(t, err)
assert.Equal(t, 1, len(resources))
assert.Equal(t, model.ResourceTypeRepository, resources[0].Type)
assert.Equal(t, "library/hello-world", resources[0].Metadata.Name)
assert.Equal(t, "library", resources[0].Metadata.Namespace)
assert.Equal(t, "hello-world", resources[0].Metadata.Repository.Name)
assert.Equal(t, "library", resources[0].Metadata.Namespace.Name)
assert.Equal(t, 2, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
assert.Equal(t, "2.0", resources[0].Metadata.Vtags[1])

4
src/replication/ng/event/handler.go
View File

@ -57,9 +57,9 @@ func (h *handler) Handle(event *Event) error {
var err error
switch event.Type {
case EventTypeImagePush, EventTypeChartUpload:
policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace)
policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name)
case EventTypeImageDelete, EventTypeChartDelete:
policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace, true)
policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name, true)
default:
return fmt.Errorf("unsupported event type %s", event.Type)
}

40
src/replication/ng/event/handler_test.go
View File

@ -161,9 +161,13 @@ func TestHandle(t *testing.T) {
err = handler.Handle(&Event{
Resource: &model.Resource{
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{},
},
},
Type: EventTypeImagePush,
@ -174,9 +178,13 @@ func TestHandle(t *testing.T) {
err = handler.Handle(&Event{
Resource: &model.Resource{
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
},
Type: "unsupported",
@ -187,9 +195,13 @@ func TestHandle(t *testing.T) {
err = handler.Handle(&Event{
Resource: &model.Resource{
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
},
Type: EventTypeImagePush,
@ -200,9 +212,13 @@ func TestHandle(t *testing.T) {
err = handler.Handle(&Event{
Resource: &model.Resource{
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
},
Type: EventTypeImageDelete,

1
src/replication/ng/model/registry.go
View File

@ -108,6 +108,7 @@ type FilterStyle struct {
type RegistryInfo struct {
Type RegistryType `json:"type"`
Description string `json:"description"`
SupportNamespace bool `json:"support_namespace"`
SupportedResourceTypes []ResourceType `json:"-"`
SupportedResourceFilters []*FilterStyle `json:"supported_resource_filters"`
SupportedTriggers []TriggerType `json:"supported_triggers"`

30
src/replication/ng/model/resource.go
View File

@ -30,10 +30,32 @@ func (r ResourceType) Valid() bool {
// ResourceMetadata of resource
type ResourceMetadata struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
Vtags []string `json:"v_tags"`
Labels []string `json:"labels"`
Namespace *Namespace `json:"namespace"`
Repository *Repository `json:"repository"`
Vtags []string `json:"v_tags"`
// TODO the labels should be put into tag and repository level?
Labels []string `json:"labels"`
}
// GetResourceName returns the name of the resource
func (r *ResourceMetadata) GetResourceName() string {
name := ""
if r.Namespace != nil && len(r.Namespace.Name) > 0 {
name += r.Namespace.Name
}
if r.Repository != nil && len(r.Repository.Name) > 0 {
if len(name) > 0 {
name += "/"
}
name += r.Repository.Name
}
return name
}
// Repository info of the resource
type Repository struct {
Name string `json:"name"`
Metadata map[string]interface{} `json:"metadata"`
}
// Resource represents the general replicating content

50
src/replication/ng/model/resource_test.go Normal file
View File

@ -0,0 +1,50 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetResourceName(t *testing.T) {
r := &ResourceMetadata{}
assert.Equal(t, "", r.GetResourceName())
r = &ResourceMetadata{
Namespace: &Namespace{
Name: "library",
},
}
assert.Equal(t, "library", r.GetResourceName())
r = &ResourceMetadata{
Repository: &Repository{
Name: "hello-world",
},
}
assert.Equal(t, "hello-world", r.GetResourceName())
r = &ResourceMetadata{
Namespace: &Namespace{
Name: "library",
},
Repository: &Repository{
Name: "hello-world",
},
}
assert.Equal(t, "library/hello-world", r.GetResourceName())
}

5
src/replication/ng/operation/controller.go
View File

@ -104,8 +104,9 @@ func (c *controller) createFlow(executionID int64, policy *model.Policy, resourc
Value: resource.Type,
},
{
Type: model.FilterTypeName,
Value: resource.Metadata.Name,
Type: model.FilterTypeName,
// TODO only filter the repo part?
Value: resource.Metadata.GetResourceName(),
},
{
Type: model.FilterTypeTag,

40
src/replication/ng/operation/controller_test.go
View File

@ -132,7 +132,18 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
return nil, nil
}
func (f *fakedAdapter) CreateNamespace(*model.Namespace) error {
func (f *fakedAdapter) ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error) {
return &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
}, nil
}
func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
return nil
}
func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
@ -155,9 +166,13 @@ func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter)
{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
Override: false,
},
@ -190,9 +205,13 @@ func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter)
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Name: "library/harbor",
Namespace: "library",
Vtags: []string{"0.2.0"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "harbor",
},
Vtags: []string{"0.2.0"},
},
},
}, nil
@ -232,7 +251,12 @@ func TestStartReplication(t *testing.T) {
resource := &model.Resource{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"1.0", "2.0"},
},
}

5
src/replication/ng/operation/flow/copy.go
View File

@ -57,14 +57,13 @@ func (c *copyFlow) Run(interface{}) (int, error) {
log.Infof("no resources need to be replicated for the execution %d, skip", c.executionID)
return 0, nil
}
dstNamespaces, err := assembleDestinationNamespaces(srcAdapter, srcResources, c.policy.DestNamespace)
dstResources, err := assembleDestinationResources(dstAdapter, srcResources, c.policy)
if err != nil {
return 0, err
}
if err = createNamespaces(dstAdapter, dstNamespaces); err != nil {
if err = prepareForPush(dstAdapter, dstResources); err != nil {
return 0, err
}
dstResources := assembleDestinationResources(srcResources, c.policy.DestRegistry, c.policy.DestNamespace, c.policy.Override)
items, err := preprocess(c.scheduler, srcResources, dstResources)
if err != nil {
return 0, err

9
src/replication/ng/operation/flow/deletion.go
View File

@ -43,6 +43,10 @@ func NewDeletionFlow(executionMgr execution.Manager, scheduler scheduler.Schedul
}
func (d *deletionFlow) Run(interface{}) (int, error) {
_, dstAdapter, err := initialize(d.policy)
if err != nil {
return 0, err
}
// filling the registry information
for _, resource := range d.resources {
resource.Registry = d.policy.SrcRegistry
@ -56,7 +60,10 @@ func (d *deletionFlow) Run(interface{}) (int, error) {
log.Infof("no resources need to be replicated for the execution %d, skip", d.executionID)
return 0, nil
}
dstResources := assembleDestinationResources(srcResources, d.policy.DestRegistry, d.policy.DestNamespace, d.policy.Override)
dstResources, err := assembleDestinationResources(dstAdapter, srcResources, d.policy)
if err != nil {
return 0, err
}
items, err := preprocess(d.scheduler, srcResources, dstResources)
if err != nil {
return 0, err

10
src/replication/ng/operation/flow/deletion_test.go
View File

@ -37,9 +37,13 @@ func TestRunOfDeletionFlow(t *testing.T) {
resources := []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
},
}

96
src/replication/ng/operation/flow/stage.go
View File

@ -134,7 +134,8 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m
match = false
break
}
m, err := util.Match(pattern, resource.Metadata.Name)
// TODO filter only the repository part?
m, err := util.Match(pattern, resource.Metadata.GetResourceName())
if err != nil {
return nil, err
}
@ -181,73 +182,46 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m
return res, nil
}
// Assemble the namespaces that need to be created on the destination registry:
// step 1: get the detail information for each of the source namespaces
// step 2: if the destination namespace isn't specified in the policy, then the
// same namespaces with the source will be returned. If it is specified, then
// returns the specified one with the merged metadatas of all source namespaces
func assembleDestinationNamespaces(srcAdapter adp.Adapter, srcResources []*model.Resource, dstNamespace string) ([]*model.Namespace, error) {
namespaces := []*model.Namespace{}
for _, srcResource := range srcResources {
namespace, err := srcAdapter.GetNamespace(srcResource.Metadata.Namespace)
if err != nil {
return nil, err
}
namespaces = append(namespaces, namespace)
}
if len(dstNamespace) != 0 {
namespaces = []*model.Namespace{
{
Name: dstNamespace,
// TODO merge the metadata
Metadata: map[string]interface{}{},
},
}
}
log.Debug("assemble the destination namespaces completed")
return namespaces, nil
}
// create the namespaces on the destination registry
func createNamespaces(adapter adp.Adapter, namespaces []*model.Namespace) error {
for _, namespace := range namespaces {
if err := adapter.CreateNamespace(namespace); err != nil {
return fmt.Errorf("failed to create the namespace %s on the destination registry: %v", namespace.Name, err)
}
log.Debugf("namespace %s created on the destination registry", namespace.Name)
}
return nil
}
// assemble the destination resources by filling the registry, namespace and override properties
func assembleDestinationResources(resources []*model.Resource,
registry *model.Registry, namespace string, override bool) []*model.Resource {
// assemble the destination resources by filling the metadata, registry and override properties
func assembleDestinationResources(adapter adp.Adapter, resources []*model.Resource,
policy *model.Policy) ([]*model.Resource, error) {
result := []*model.Resource{}
var namespace *model.Namespace
if len(policy.DestNamespace) > 0 {
namespace = &model.Namespace{
Name: policy.DestNamespace,
}
}
for _, resource := range resources {
metadata, err := adapter.ConvertResourceMetadata(resource.Metadata, namespace)
if err != nil {
return nil, fmt.Errorf("failed to convert the resource metadata of %s: %v", resource.Metadata.GetResourceName(), err)
}
res := &model.Resource{
Type: resource.Type,
Metadata: &model.ResourceMetadata{
Name: resource.Metadata.Name,
Namespace: resource.Metadata.Namespace,
Vtags: resource.Metadata.Vtags,
},
Registry: registry,
Type: resource.Type,
Metadata: metadata,
Registry: policy.DestRegistry,
ExtendedInfo: resource.ExtendedInfo,
Deleted: resource.Deleted,
Override: override,
}
// if the destination namespace is specified, use the specified one
if len(namespace) > 0 {
res.Metadata.Name = strings.Replace(resource.Metadata.Name,
resource.Metadata.Namespace, namespace, 1)
res.Metadata.Namespace = namespace
Override: policy.Override,
}
result = append(result, res)
}
log.Debug("assemble the destination resources completed")
return result
return result, nil
}
// do the prepare work for pushing/uploading the resources: create the namespace or repository
func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error {
// TODO need to consider how to handle that both contains public/private namespace
for _, resource := range resources {
name := resource.Metadata.GetResourceName()
if err := adapter.PrepareForPush(resource); err != nil {
return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err)
}
log.Debugf("the prepare work for pushing/uploading %s completed", name)
}
return nil
}
// preprocess
@ -341,7 +315,7 @@ func getResourceName(res *model.Resource) string {
return ""
}
if len(meta.Vtags) == 0 {
return meta.Name
return meta.GetResourceName()
}
return meta.Name + ":[" + strings.Join(meta.Vtags, ",") + "]"
return meta.GetResourceName() + ":[" + strings.Join(meta.Vtags, ",") + "]"
}

144
src/replication/ng/operation/flow/stage_test.go
View File

@ -49,7 +49,14 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
return nil, nil
}
func (f *fakedAdapter) CreateNamespace(*model.Namespace) error {
func (f *fakedAdapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
if namespace != nil {
metadata.Namespace = namespace
}
return metadata, nil
}
func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
return nil
}
func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
@ -72,9 +79,13 @@ func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter)
{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
Override: false,
},
@ -107,9 +118,13 @@ func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter)
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Name: "library/harbor",
Namespace: "library",
Vtags: []string{"0.2.0"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "harbor",
},
Vtags: []string{"0.2.0"},
},
},
}, nil
@ -226,9 +241,13 @@ func TestFilterResources(t *testing.T) {
{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
// TODO test labels
Labels: nil,
},
@ -238,9 +257,13 @@ func TestFilterResources(t *testing.T) {
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Name: "library/harbor",
Namespace: "library",
Vtags: []string{"0.2.0", "0.3.0"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "harbor",
},
Vtags: []string{"0.2.0", "0.3.0"},
// TODO test labels
Labels: nil,
},
@ -250,9 +273,13 @@ func TestFilterResources(t *testing.T) {
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Name: "library/mysql",
Namespace: "library",
Vtags: []string{"1.0"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "mysql",
},
Vtags: []string{"1.0"},
// TODO test labels
Labels: nil,
},
@ -281,67 +308,40 @@ func TestFilterResources(t *testing.T) {
res, err := filterResources(resources, filters)
require.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "library/harbor", res[0].Metadata.Name)
assert.Equal(t, "library", res[0].Metadata.Namespace.Name)
assert.Equal(t, "harbor", res[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(res[0].Metadata.Vtags))
assert.Equal(t, "0.2.0", res[0].Metadata.Vtags[0])
}
func TestAssembleDestinationNamespaces(t *testing.T) {
adapter := &fakedAdapter{}
resources := []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Namespace: "library",
},
},
}
namespace := ""
ns, err := assembleDestinationNamespaces(adapter, resources, namespace)
require.Nil(t, err)
assert.Equal(t, 1, len(ns))
assert.Equal(t, "library", ns[0].Name)
assert.Equal(t, true, ns[0].Metadata["public"].(bool))
namespace = "test"
ns, err = assembleDestinationNamespaces(adapter, resources, namespace)
require.Nil(t, err)
assert.Equal(t, 1, len(ns))
assert.Equal(t, "test", ns[0].Name)
// TODO add test for merged metadata
// assert.Equal(t, true, ns[0].Metadata["public"].(bool))
}
func TestCreateNamespaces(t *testing.T) {
adapter := &fakedAdapter{}
namespaces := []*model.Namespace{
{
Name: "library",
},
}
err := createNamespaces(adapter, namespaces)
require.Nil(t, err)
}
func TestAssembleDestinationResources(t *testing.T) {
adapter := &fakedAdapter{}
resources := []*model.Resource{
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
Override: false,
},
}
registry := &model.Registry{}
namespace := "test"
override := true
res := assembleDestinationResources(resources, registry, namespace, override)
policy := &model.Policy{
DestRegistry: &model.Registry{},
DestNamespace: "test",
Override: true,
}
res, err := assembleDestinationResources(adapter, resources, policy)
require.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, model.ResourceTypeChart, res[0].Type)
assert.Equal(t, "test/hello-world", res[0].Metadata.Name)
assert.Equal(t, namespace, res[0].Metadata.Namespace)
assert.Equal(t, "hello-world", res[0].Metadata.Repository.Name)
assert.Equal(t, "test", res[0].Metadata.Namespace.Name)
assert.Equal(t, 1, len(res[0].Metadata.Vtags))
assert.Equal(t, "latest", res[0].Metadata.Vtags[0])
}
@ -352,9 +352,13 @@ func TestPreprocess(t *testing.T) {
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Name: "library/hello-world",
Namespace: "library",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
Override: false,
},
@ -363,9 +367,13 @@ func TestPreprocess(t *testing.T) {
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Name: "test/hello-world",
Namespace: "test",
Vtags: []string{"latest"},
Namespace: &model.Namespace{
Name: "test",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
},
Override: false,
},

16
src/replication/ng/operation/scheduler/scheduler_test.go
View File

@ -50,9 +50,11 @@ func TestStop(t *testing.T) {
func generateData() ([]*ScheduleItem, error) {
srcResource := &model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: "namespace1",
Vtags: []string{"latest"},
Labels: []string{"latest"},
Namespace: &model.Namespace{
Name: "namespace1",
},
Vtags: []string{"latest"},
Labels: []string{"latest"},
},
Registry: &model.Registry{
Credential: &model.Credential{},
@ -60,9 +62,11 @@ func generateData() ([]*ScheduleItem, error) {
}
destResource := &model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: "namespace2",
Vtags: []string{"v1", "v2"},
Labels: []string{"latest"},
Namespace: &model.Namespace{
Name: "namespace2",
},
Vtags: []string{"v1", "v2"},
Labels: []string{"latest"},
},
Registry: &model.Registry{
Credential: &model.Credential{},

9
src/replication/ng/transfer/chart/transfer.go
View File

@ -17,10 +17,9 @@ package chart
import (
"errors"
"github.com/goharbor/harbor/src/replication/ng/adapter"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/jobservice/errs"
"github.com/goharbor/harbor/src/replication/ng/adapter"
"github.com/goharbor/harbor/src/replication/ng/model"
trans "github.com/goharbor/harbor/src/replication/ng/transfer"
)
@ -63,17 +62,17 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error {
// delete the chart on destination registry
if dst.Deleted {
return t.delete(&chart{
name: dst.Metadata.Name,
name: dst.Metadata.GetResourceName(),
version: dst.Metadata.Vtags[0],
})
}
srcChart := &chart{
name: src.Metadata.Name,
name: src.Metadata.GetResourceName(),
version: src.Metadata.Vtags[0],
}
dstChart := &chart{
name: dst.Metadata.Name,
name: dst.Metadata.GetResourceName(),
version: dst.Metadata.Vtags[0],
}
// copy the chart from source registry to the destination

10
src/replication/ng/transfer/chart/transfer_test.go
View File

@ -34,9 +34,13 @@ func (f *fakeRegistry) FetchCharts(namespaces []string, filters []*model.Filter)
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Name: "library/harbor",
Namespace: "library",
Vtags: []string{"0.2.0"},
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "harbor",
},
Vtags: []string{"0.2.0"},
},
},
}, nil

9
src/replication/ng/transfer/repository/transfer.go
View File

@ -18,13 +18,12 @@ import (
"errors"
"strings"
"github.com/goharbor/harbor/src/replication/ng/adapter"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/jobservice/errs"
"github.com/goharbor/harbor/src/replication/ng/adapter"
"github.com/goharbor/harbor/src/replication/ng/model"
trans "github.com/goharbor/harbor/src/replication/ng/transfer"
)
@ -67,17 +66,17 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error {
// delete the repository on destination registry
if dst.Deleted {
return t.delete(&repository{
repository: dst.Metadata.Name,
repository: dst.Metadata.GetResourceName(),
tags: dst.Metadata.Vtags,
})
}
srcRepo := &repository{
repository: src.Metadata.Name,
repository: src.Metadata.GetResourceName(),
tags: src.Metadata.Vtags,
}
dstRepo := &repository{
repository: dst.Metadata.Name,
repository: dst.Metadata.GetResourceName(),
tags: dst.Metadata.Vtags,
}
// copy the repository from source registry to the destination