Merge pull request #7946 from bitsf/replication_gcr_1.9

gcr driver for replication

src/common/utils/registry/repository.go

@@ -211,7 +211,7 @@ func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (
 
 	defer resp.Body.Close()
 
-	if resp.StatusCode == http.StatusCreated {
+	if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
 		digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
 		return
 	}

src/jobservice/job/impl/replication/replication.go

@@ -34,6 +34,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/native"
 	// register the Huawei adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/huawei"
+	// register the Google Gcr adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/googlegcr"
 	// register the AwsEcr adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/awsecr"
 )

src/replication/adapter/googlegcr/adapter.go

@@ -0,0 +1,106 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package googlegcr
+
+import (
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/common/utils/registry/auth"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
+	"net/http"
+)
+
+func init() {
+	if err := adp.RegisterFactory(model.RegistryTypeGoogleGcr, func(registry *model.Registry) (adp.Adapter, error) {
+		return newAdapter(registry)
+	}); err != nil {
+		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeGoogleGcr, err)
+		return
+	}
+	log.Infof("the factory for adapter %s registered", model.RegistryTypeGoogleGcr)
+}
+
+func newAdapter(registry *model.Registry) (*adapter, error) {
+	var credential auth.Credential
+	if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
+		credential = auth.NewBasicAuthCredential(
+			registry.Credential.AccessKey,
+			registry.Credential.AccessSecret)
+	}
+	authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
+		Transport: util.GetHTTPTransport(registry.Insecure),
+	}, credential)
+
+	reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	return &adapter{
+		registry:             registry,
+		DefaultImageRegistry: reg,
+	}, nil
+}
+
+type adapter struct {
+	*adp.DefaultImageRegistry
+	registry *model.Registry
+}
+
+var _ adp.Adapter = adapter{}
+
+func (adapter) Info() (info *model.RegistryInfo, err error) {
+	return &model.RegistryInfo{
+		Type: model.RegistryTypeGoogleGcr,
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeImage,
+		},
+		SupportedResourceFilters: []*model.FilterStyle{
+			{
+				Type:  model.FilterTypeName,
+				Style: model.FilterStyleTypeText,
+			},
+			{
+				Type:  model.FilterTypeTag,
+				Style: model.FilterStyleTypeText,
+			},
+		},
+		SupportedTriggers: []model.TriggerType{
+			model.TriggerTypeManual,
+			model.TriggerTypeScheduled,
+		},
+	}, nil
+}
+
+// HealthCheck checks health status of a registry
+func (a adapter) HealthCheck() (model.HealthStatus, error) {
+	var err error
+	if a.registry.Credential == nil ||
+		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
+		log.Errorf("no credential to ping registry %s", a.registry.URL)
+		return model.Unhealthy, nil
+	}
+	if err = a.PingGet(); err != nil {
+		log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
+		return model.Unhealthy, nil
+	}
+	return model.Healthy, nil
+}
+
+// PrepareForPush nothing need to do.
+func (a adapter) PrepareForPush(resources []*model.Resource) error {
+	return nil
+}

src/replication/adapter/googlegcr/adapter_test.go

@@ -0,0 +1,161 @@
+package googlegcr
+
+import (
+	"fmt"
+	"github.com/goharbor/harbor/src/common/utils/test"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Server) {
+	server := test.NewServer(
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/_catalog",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte(`
+		{
+			"repositories": [
+					"test1"
+			]
+		}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/{repo}/tags/list",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte(`
+			{
+			    "name": "test1",
+			    "tags": [
+			        "latest"
+			    ]
+			}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				fmt.Println(r.Method, r.URL)
+				if health {
+					w.WriteHeader(http.StatusOK)
+				} else {
+					w.WriteHeader(http.StatusBadRequest)
+				}
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				fmt.Println(r.Method, r.URL)
+				w.WriteHeader(http.StatusOK)
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodPost,
+			Pattern: "/",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				fmt.Println(r.Method, r.URL)
+				if buf, e := ioutil.ReadAll(&io.LimitedReader{R: r.Body, N: 80}); e == nil {
+					fmt.Println("\t", string(buf))
+				}
+				w.WriteHeader(http.StatusOK)
+			},
+		},
+	)
+	registry := &model.Registry{
+		Type: model.RegistryTypeGoogleGcr,
+		URL:  server.URL,
+	}
+	if hasCred {
+		registry.Credential = &model.Credential{
+			AccessKey:    "_json_key",
+			AccessSecret: "ppp",
+		}
+	}
+
+	factory, err := adp.GetFactory(model.RegistryTypeGoogleGcr)
+	assert.Nil(t, err)
+	assert.NotNil(t, factory)
+	a, err := factory(registry)
+
+	assert.Nil(t, err)
+	return a.(*adapter), server
+}
+
+func TestAdapter_Info(t *testing.T) {
+	a, s := getMockAdapter(t, true, true)
+	defer s.Close()
+	info, err := a.Info()
+	assert.Nil(t, err)
+	assert.NotNil(t, info)
+	assert.EqualValues(t, 1, len(info.SupportedResourceTypes))
+	assert.EqualValues(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
+}
+
+func TestAdapter_HealthCheck(t *testing.T) {
+	a, s := getMockAdapter(t, false, true)
+	defer s.Close()
+	status, err := a.HealthCheck()
+	assert.Nil(t, err)
+	assert.NotNil(t, status)
+	assert.EqualValues(t, model.Unhealthy, status)
+	a, s = getMockAdapter(t, true, false)
+	defer s.Close()
+	status, err = a.HealthCheck()
+	assert.Nil(t, err)
+	assert.NotNil(t, status)
+	assert.EqualValues(t, model.Unhealthy, status)
+	a, s = getMockAdapter(t, true, true)
+	defer s.Close()
+	status, err = a.HealthCheck()
+	assert.Nil(t, err)
+	assert.NotNil(t, status)
+	assert.EqualValues(t, model.Healthy, status)
+}
+
+func TestAdapter_PrepareForPush(t *testing.T) {
+	a, s := getMockAdapter(t, true, true)
+	defer s.Close()
+	resources := []*model.Resource{
+		{
+			Type: model.ResourceTypeImage,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: "busybox",
+				},
+			},
+		},
+	}
+	err := a.PrepareForPush(resources)
+	assert.Nil(t, err)
+}
+
+func TestAdapter_FetchImages(t *testing.T) {
+	a, s := getMockAdapter(t, true, true)
+	defer s.Close()
+	resources, err := a.FetchImages([]*model.Filter{
+		{
+			Type:  model.FilterTypeName,
+			Value: "*",
+		},
+		{
+			Type:  model.FilterTypeTag,
+			Value: "*",
+		},
+	})
+	assert.Nil(t, err)
+	assert.NotNil(t, resources)
+	assert.Equal(t, 1, len(resources))
+}

src/replication/adapter/googlegcr/image_registry.go

@@ -0,0 +1,113 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package googlegcr
+
+import (
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
+)
+
+var _ adp.ImageRegistry = adapter{}
+
+func (a adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
+	nameFilterPattern := ""
+	tagFilterPattern := ""
+	for _, filter := range filters {
+		switch filter.Type {
+		case model.FilterTypeName:
+			nameFilterPattern = filter.Value.(string)
+		case model.FilterTypeTag:
+			tagFilterPattern = filter.Value.(string)
+		}
+	}
+	repositories, err := a.filterRepositories(nameFilterPattern)
+	if err != nil {
+		return nil, err
+	}
+
+	var resources []*model.Resource
+	for _, repository := range repositories {
+		tags, err := a.filterTags(repository, tagFilterPattern)
+		if err != nil {
+			return nil, err
+		}
+		if len(tags) == 0 {
+			continue
+		}
+		resources = append(resources, &model.Resource{
+			Type:     model.ResourceTypeImage,
+			Registry: a.registry,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: repository,
+				},
+				Vtags: tags,
+			},
+		})
+	}
+
+	return resources, nil
+}
+
+func (a adapter) filterRepositories(pattern string) ([]string, error) {
+	// if the pattern is a specific repository name, just returns the parsed repositories
+	// and will check the existence later when filtering the tags
+	if repositories, ok := util.IsSpecificPath(pattern); ok {
+		return repositories, nil
+	}
+	// search repositories from catalog api
+	repositories, err := a.Catalog()
+	if err != nil {
+		return nil, err
+	}
+	// if the pattern is null, just return the result of catalog API
+	if len(pattern) == 0 {
+		return repositories, nil
+	}
+	result := []string{}
+	for _, repository := range repositories {
+		match, err := util.Match(pattern, repository)
+		if err != nil {
+			return nil, err
+		}
+		if match {
+			result = append(result, repository)
+		}
+	}
+	return result, nil
+}
+
+func (a adapter) filterTags(repository, pattern string) ([]string, error) {
+	tags, err := a.ListTag(repository)
+	if err != nil {
+		return nil, err
+	}
+	if len(pattern) == 0 {
+		return tags, nil
+	}
+
+	var result []string
+	for _, tag := range tags {
+		match, err := util.Match(pattern, tag)
+		if err != nil {
+			return nil, err
+		}
+		if match {
+			result = append(result, tag)
+		}
+	}
+	return result, nil
+}

src/replication/model/registry.go

@@ -26,6 +26,7 @@ const (
 	RegistryTypeDockerHub      RegistryType = "docker-hub"
 	RegistryTypeDockerRegistry RegistryType = "docker-registry"
 	RegistryTypeHuawei         RegistryType = "huawei-SWR"
+	RegistryTypeGoogleGcr      RegistryType = "google-gcr"
 	RegistryTypeAwsEcr         RegistryType = "aws-ecr"
 
 	FilterStyleTypeText  = "input"

src/replication/replication.go

@@ -35,6 +35,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/native"
 	// register the huawei adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/huawei"
+	// register the Google Gcr adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/googlegcr"
 	// register the AwsEcr adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/awsecr"
 )