Merge pull request #13694 from kofj/ghcr-provider

GHCR Provider

src/jobservice/job/impl/replication/replication.go

@@ -56,6 +56,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/artifacthub"
 	// register the TencentCloud TCR adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/tencentcr"
+	// register the Github Container Registry adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/githubcr"
 )
 
 // Replication implements the job interface

src/replication/adapter/githubcr/adapter.go

@@ -0,0 +1,199 @@
+package githubcr
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	common_http "github.com/goharbor/harbor/src/common/http"
+	"github.com/goharbor/harbor/src/common/http/modifier"
+	"github.com/goharbor/harbor/src/common/utils"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
+	"github.com/goharbor/harbor/src/replication/filter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
+)
+
+// !!!! Limits:
+// - GHCR not support `/v2/_catalog`, access this API will return 404 status code.
+// - NOT support DELETE manifest.
+
+func init() {
+	if err := adp.RegisterFactory(model.RegistryTypeGithubCR, new(factory)); err != nil {
+		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeGithubCR, err)
+		return
+	}
+	log.Infof("the factory for adapter %s registered", model.RegistryTypeGithubCR)
+}
+
+type factory struct{}
+
+// Create ...
+func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
+	return newAdapter(r), nil
+}
+
+// AdapterPattern ...
+func (f *factory) AdapterPattern() *model.AdapterPattern {
+	return getAdapterPattern()
+}
+
+func getAdapterPattern() *model.AdapterPattern {
+	return &model.AdapterPattern{
+		EndpointPattern: &model.EndpointPattern{
+			EndpointType: model.EndpointPatternTypeFix,
+			Endpoints: []*model.Endpoint{
+				{
+					Key:   "ghcr.io",
+					Value: "https://ghcr.io",
+				},
+			},
+		},
+	}
+}
+
+var (
+	_ adp.Adapter          = (*adapter)(nil)
+	_ adp.ArtifactRegistry = (*adapter)(nil)
+)
+
+// adapter for to github container registry
+type adapter struct {
+	client *common_http.Client
+	*native.Adapter
+	registry *model.Registry
+}
+
+var _ adp.Adapter = &adapter{}
+
+func newAdapter(registry *model.Registry) *adapter {
+	var authorizer modifier.Modifier
+	if registry.Credential != nil {
+		authorizer = basic.NewAuthorizer(
+			registry.Credential.AccessKey,
+			registry.Credential.AccessSecret)
+	}
+
+	var transport = util.GetHTTPTransport(registry.Insecure)
+
+	return &adapter{
+		Adapter:  native.NewAdapter(registry),
+		registry: registry,
+		client: common_http.NewClient(
+			&http.Client{
+				Transport: transport,
+			},
+			authorizer,
+		),
+	}
+}
+
+// Info ...
+func (a *adapter) Info() (info *model.RegistryInfo, err error) {
+	info = &model.RegistryInfo{
+		Type: model.RegistryTypeGithubCR,
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeImage,
+		},
+		SupportedResourceFilters: []*model.FilterStyle{
+			{
+				Type:  model.FilterTypeName,
+				Style: model.FilterStyleTypeText,
+			},
+			{
+				Type:  model.FilterTypeTag,
+				Style: model.FilterStyleTypeText,
+			},
+		},
+		SupportedTriggers: []model.TriggerType{
+			model.TriggerTypeManual,
+			model.TriggerTypeScheduled,
+		},
+	}
+	return
+}
+
+func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Resource, err error) {
+	pattern := ""
+	for _, filter := range filters {
+		if filter.Type == model.FilterTypeName {
+			pattern = filter.Value.(string)
+			break
+		}
+	}
+	var repositories []string
+	// if the pattern of repository name filter is a specific repository name, just returns
+	// the parsed repositories and will check the existence later when filtering the tags
+	if paths, ok := util.IsSpecificPath(pattern); ok {
+		repositories = paths
+	} else {
+		err = errors.New("Only support specific repository name")
+		return
+	}
+
+	if len(repositories) == 0 {
+		return nil, nil
+	}
+
+	var rawResources = make([]*model.Resource, len(repositories))
+	runner := utils.NewLimitedConcurrentRunner(adp.MaxConcurrency)
+
+	for i, r := range repositories {
+		index := i
+		repo := r
+		runner.AddTask(func() error {
+
+			artifacts, err := a.listArtifacts(repo, filters)
+			if err != nil {
+				return fmt.Errorf("failed to list artifacts of repository %s: %v", repo, err)
+			}
+			if len(artifacts) == 0 {
+				return nil
+			}
+			rawResources[index] = &model.Resource{
+				Type:     model.ResourceTypeImage,
+				Registry: a.registry,
+				Metadata: &model.ResourceMetadata{
+					Repository: &model.Repository{
+						Name: repo,
+					},
+					Artifacts: artifacts,
+				},
+			}
+
+			return nil
+		})
+	}
+	if err = runner.Wait(); err != nil {
+		return nil, fmt.Errorf("failed to fetch artifacts: %v", err)
+	}
+
+	for _, r := range rawResources {
+		if r != nil {
+			resources = append(resources, r)
+		}
+	}
+
+	return resources, nil
+}
+
+func (a *adapter) listArtifacts(repository string, filters []*model.Filter) ([]*model.Artifact, error) {
+	tags, err := a.ListTags(repository)
+	if err != nil {
+		return nil, err
+	}
+	var artifacts []*model.Artifact
+	for _, tag := range tags {
+		artifacts = append(artifacts, &model.Artifact{
+			Tags: []string{tag},
+		})
+	}
+	return filter.DoFilterArtifacts(artifacts, filters)
+}
+
+func (a *adapter) DeleteManifest(repository, reference string) error {
+	return nil
+}

src/replication/adapter/githubcr/adapter_test.go

@@ -0,0 +1,238 @@
+package githubcr
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/utils/test"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_native_Info(t *testing.T) {
+	var registry = &model.Registry{URL: "abc"}
+	adapter := newAdapter(registry)
+	assert.NotNil(t, adapter)
+
+	info, err := adapter.Info()
+	assert.Nil(t, err)
+	assert.NotNil(t, info)
+	assert.Equal(t, model.RegistryTypeGithubCR, info.Type)
+	assert.Equal(t, 1, len(info.SupportedResourceTypes))
+	assert.Equal(t, 2, len(info.SupportedResourceFilters))
+	assert.Equal(t, 2, len(info.SupportedTriggers))
+	assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
+}
+
+func Test_getAdapterPattern(t *testing.T) {
+	var pattern = getAdapterPattern()
+	assert.NotNil(t, pattern)
+	assert.Equal(t, model.EndpointPatternTypeFix, pattern.EndpointPattern.EndpointType)
+}
+
+func mockGHCR() (mock *httptest.Server) {
+	return test.NewServer(
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/_catalog",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				r.Response.StatusCode = http.StatusNotFound
+				w.Write([]byte(``))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/kofj/a1/tags/list",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte(`{"name":"kofj/a1","tags":["tag11"]}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/kofj/b2/tags/list",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte(`{"name":"kofj/b2","tags":["tag11","tag2","tag13"]}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/kofj/c3/l3/tags/list",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte(`{"name":"kofj/c3/l3","tags":["tag4"]}`))
+			},
+		},
+	)
+}
+
+func Test_native_FetchArtifacts(t *testing.T) {
+	var mock = mockGHCR()
+	defer mock.Close()
+	fmt.Println("mockGHCR URL: ", mock.URL)
+
+	var registry = &model.Registry{
+		Type:     model.RegistryTypeDockerRegistry,
+		URL:      mock.URL,
+		Insecure: true,
+	}
+	adapter := newAdapter(registry)
+	assert.NotNil(t, adapter)
+
+	tests := []struct {
+		name    string
+		filters []*model.Filter
+		want    []*model.Resource
+		wantErr bool
+	}{
+		{
+			name: "repository not exist",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "b1",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "tag not exist",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeTag,
+					Value: "this_tag_not_exist_in_the_mock_server",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name:    "no filters",
+			filters: []*model.Filter{},
+			want:    []*model.Resource{},
+			wantErr: true,
+		},
+
+		{
+			name: "only special repository",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "kofj/a1",
+				},
+			},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "kofj/a1"},
+						Artifacts: []*model.Artifact{
+							{
+								Tags: []string{"tag11"},
+							},
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+
+		{
+			name: "only special tag",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeTag,
+					Value: "tag11",
+				},
+			},
+			want:    []*model.Resource{},
+			wantErr: true,
+		},
+
+		{
+			name: "special repository and special tag",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "kofj/b2",
+				},
+				{
+					Type:  model.FilterTypeTag,
+					Value: "tag2",
+				},
+			},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "kofj/b2"},
+						Artifacts: []*model.Artifact{
+							{
+								Tags: []string{"tag2"},
+							},
+						},
+					},
+				},
+			},
+
+			wantErr: false,
+		},
+		{
+			name: "only wildcard repository",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "kofj/b*",
+				},
+			},
+			want:    []*model.Resource{},
+			wantErr: true,
+		},
+		{
+			name: "only wildcard tag",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeTag,
+					Value: "tag1*",
+				},
+			},
+			want:    []*model.Resource{},
+			wantErr: true,
+		},
+		{
+			name: "wildcard repository and wildcard tag",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "kofj/b*",
+				},
+				{
+					Type:  model.FilterTypeTag,
+					Value: "tag1*",
+				},
+			},
+			want:    []*model.Resource{},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var resources, err = adapter.FetchArtifacts(tt.filters)
+			if tt.wantErr {
+				require.Len(t, resources, 0)
+				require.NotNil(t, err)
+			} else {
+				if err != nil {
+					t.Logf("Name=%s, error: %v", t.Name(), err)
+				}
+				require.Equal(t, len(tt.want), len(resources))
+				for i, resource := range resources {
+					require.NotNil(t, resource.Metadata)
+					assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository)
+					assert.ElementsMatch(t, tt.want[i].Metadata.Artifacts, resource.Metadata.Artifacts)
+				}
+			}
+		})
+	}
+}
+
+// ***************************************

src/replication/model/registry.go

@@ -33,6 +33,7 @@ const (
 	RegistryTypeGitLab           RegistryType = "gitlab"
 	RegistryTypeDTR              RegistryType = "dtr"
 	RegistryTypeTencentTcr       RegistryType = "tencent-tcr"
+	RegistryTypeGithubCR         RegistryType = "github-ghcr"
 
 	RegistryTypeHelmHub     RegistryType = "helm-hub"
 	RegistryTypeArtifactHub RegistryType = "artifact-hub"

src/replication/replication.go

@@ -61,6 +61,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/artifacthub"
 	// register the TencentCloud TCR adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/tencentcr"
+	// register the Github Container Registry adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/githubcr"
 )
 
 var (