Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-23 18:55:18 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7498 from ywk253100/190423_docker_hub

Browse Source 
Support replicate public repositories from Docker Hub
This commit is contained in:
Wenkai Yin 2019-04-24 17:17:23 +08:00 committed by GitHub
commit 7160e411cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 430 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/common/utils/registry/auth/credential.go
View File

@ -38,12 +38,7 @@ func NewBasicAuthCredential(username, password string) Credential {
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
// only add the authentication info when the username isn't empty
// the logic is needed for requesting resources from docker hub's
// public repositories
if len(b.username) > 0 {
req.SetBasicAuth(b.username, b.password)
}
}
// implement github.com/goharbor/harbor/src/common/http/modifier.Modifier

85
src/replication/adapter/dockerhub/adapter.go
View File

@ -10,21 +10,23 @@ import (
"strings"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
func init() {
if err := adp.RegisterFactory(model.RegistryTypeDockerHub, func(registry *model.Registry) (adp.Adapter, error) {
registry.URL = baseURL
client, err := NewClient(registry)
if err != nil {
return nil, err
if err := adp.RegisterFactory(model.RegistryTypeDockerHub, factory); err != nil {
log.Errorf("Register adapter factory for %s error: %v", model.RegistryTypeDockerHub, err)
return
}
reg, err := adp.NewDefaultImageRegistry(&model.Registry{
Name: registry.Name,
URL: registryURL,
log.Infof("Factory for adapter %s registered", model.RegistryTypeDockerHub)
}
func factory(registry *model.Registry) (adp.Adapter, error) {
client, err := NewClient(&model.Registry{
URL: baseURL, // specify the URL of Docker Hub
Credential: registry.Credential,
Insecure: registry.Insecure,
})
@ -32,16 +34,34 @@ func init() {
return nil, err
}
// if the registry.Credentail isn't specified, the credential here is nil
// the client will request the token with no authentication
// this is needed for pulling images from public repositories
var credential auth.Credential
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
credential = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, credential)
reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(&model.Registry{
Name: registry.Name,
URL: registryURL, // specify the URL of Docker Hub registry service
Credential: registry.Credential,
Insecure: registry.Insecure,
}, authorizer)
if err != nil {
return nil, err
}
return &adapter{
client: client,
registry: registry,
DefaultImageRegistry: reg,
}, nil
}); err != nil {
log.Errorf("Register adapter factory for %s error: %v", model.RegistryTypeDockerHub, err)
return
}
log.Infof("Factory for adapter %s registered", model.RegistryTypeDockerHub)
}
type adapter struct {
@ -111,8 +131,7 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error {
return nil
}
// ListNamespaces lists namespaces from DockerHub with the provided query conditions.
func (a *adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) {
func (a *adapter) listNamespaces() ([]string, error) {
resp, err := a.client.Do(http.MethodGet, listNamespacePath, nil)
if err != nil {
return nil, err
@ -134,18 +153,8 @@ func (a *adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespac
if err != nil {
return nil, err
}
var result []*model.Namespace
for _, ns := range namespaces.Namespaces {
// If query set, skip the namespace that doesn't match the query.
if query != nil && len(query.Name) > 0 && strings.Index(ns, query.Name) != -1 {
continue
}
result = append(result, &model.Namespace{
Name: ns,
})
}
return result, nil
log.Debugf("got namespaces %v by calling the listing namespaces API", namespaces)
return namespaces.Namespaces, nil
}
// CreateNamespace creates a new namespace in DockerHub
@ -229,7 +238,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
return nil, err
}
namespaces, err := a.ListNamespaces(nil)
namespaces, err := a.listCandidateNamespaces(nameFilter)
if err != nil {
return nil, err
}
@ -239,7 +248,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
pageSize := 100
n := 0
for {
pageRepos, err := a.getRepos(ns.Name, "", page, pageSize)
pageRepos, err := a.getRepos(ns, "", page, pageSize)
if err != nil {
return nil, fmt.Errorf("get repos for namespace '%s' from DockerHub error: %v", ns, err)
}
@ -252,7 +261,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
page++
}
log.Debugf("got %d repositories for namespace %s", n, ns.Name)
log.Debugf("got %d repositories for namespace %s", n, ns)
}
var resources []*model.Resource
@ -319,6 +328,22 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
return resources, nil
}
func (a *adapter) listCandidateNamespaces(pattern string) ([]string, error) {
namespaces := []string{}
if len(pattern) > 0 {
substrings := strings.Split(pattern, "/")
namespacePattern := substrings[0]
if nms, ok := util.IsSpecificPathComponent(namespacePattern); ok {
namespaces = append(namespaces, nms...)
}
}
if len(namespaces) > 0 {
log.Debugf("parsed the namespaces %v from pattern %s", namespaces, pattern)
return namespaces, nil
}
return a.listNamespaces()
}
// DeleteManifest ...
// Note: DockerHub only supports delete by tag
func (a *adapter) DeleteManifest(repository, reference string) error {

6
src/replication/adapter/dockerhub/adapter_test.go
View File

@ -9,6 +9,8 @@ import (
"github.com/stretchr/testify/assert"
)
// TODO add more unit test
const (
testUser = ""
testPassword = ""
@ -42,9 +44,9 @@ func TestListNamespaces(t *testing.T) {
ad := getAdapter(t)
adapter := ad.(*adapter)
namespaces, err := adapter.ListNamespaces(nil)
namespaces, err := adapter.listNamespaces()
assert.Nil(err)
for _, ns := range namespaces {
fmt.Println(ns.Name)
fmt.Println(ns)
}
}

2
src/replication/adapter/harbor/adapter.go
View File

@ -231,5 +231,5 @@ func (a *adapter) getProject(name string) (*project, error) {
return p, nil
}
}
return nil, fmt.Errorf("project %s not found", name)
return nil, nil
}

3
src/replication/adapter/harbor/chart_registry.go
View File

@ -76,8 +76,7 @@ type chartVersionMetadata struct {
}
func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
// TODO optimize the performance
projects, err := a.getProjects("")
projects, err := a.listCandidateProjects(filters)
if err != nil {
return nil, err
}

43
src/replication/adapter/harbor/image_registry.go
View File

@ -16,8 +16,11 @@ package harbor
import (
"fmt"
"strings"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
type repository struct {
@ -55,8 +58,7 @@ func (t *tag) Match(filters []*model.Filter) (bool, error) {
}
func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
// TODO optimize the performance
projects, err := a.getProjects("")
projects, err := a.listCandidateProjects(filters)
if err != nil {
return nil, err
}
@ -105,6 +107,43 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
return resources, nil
}
func (a *adapter) listCandidateProjects(filters []*model.Filter) ([]*project, error) {
pattern := ""
for _, filter := range filters {
if filter.Type == model.FilterTypeName {
pattern = filter.Value.(string)
break
}
}
projects := []*project{}
if len(pattern) > 0 {
substrings := strings.Split(pattern, "/")
projectPattern := substrings[0]
names, ok := util.IsSpecificPathComponent(projectPattern)
if ok {
for _, name := range names {
project, err := a.getProject(name)
if err != nil {
return nil, err
}
if project == nil {
continue
}
projects = append(projects, project)
}
}
}
if len(projects) > 0 {
names := []string{}
for _, project := range projects {
names = append(names, project.Name)
}
log.Debugf("parsed the projects %v from pattern %s", names, pattern)
return projects, nil
}
return a.getProjects("")
}
// override the default implementation from the default image registry
// by calling Harbor API directly
func (a *adapter) DeleteManifest(repository, reference string) error {

23
src/replication/adapter/image_registry.go
View File

@ -62,12 +62,7 @@ type DefaultImageRegistry struct {
// NewDefaultImageRegistry returns an instance of DefaultImageRegistry
func NewDefaultImageRegistry(registry *model.Registry) (*DefaultImageRegistry, error) {
transport := util.GetHTTPTransport(registry.Insecure)
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: UserAgentReplication,
},
}
var authorizer modifier.Modifier
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
var cred modifier.Modifier
if registry.Credential.Type == model.CredentialTypeSecret {
@ -83,10 +78,22 @@ func NewDefaultImageRegistry(registry *model.Registry) (*DefaultImageRegistry, e
if len(registry.CoreURL) > 0 {
tokenServiceURL = fmt.Sprintf("%s/service/token", registry.CoreURL)
}
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: transport,
authorizer = auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, cred, tokenServiceURL)
}
return NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
}
// NewDefaultImageRegistryWithCustomizedAuthorizer returns an instance of DefaultImageRegistry with the customized authorizer
func NewDefaultImageRegistryWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*DefaultImageRegistry, error) {
transport := util.GetHTTPTransport(registry.Insecure)
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: UserAgentReplication,
},
}
if authorizer != nil {
modifiers = append(modifiers, authorizer)
}
client := &http.Client{

9
src/replication/adapter/native/image_registry.go
View File

@ -63,11 +63,10 @@ func (n native) FetchImages(filters []*model.Filter) ([]*model.Resource, error)
}
func (n native) filterRepositories(pattern string) ([]string, error) {
// if is a specific repository name
// just to make sure the repository exists
if len(pattern) > 0 && util.IsSpecificRepositoryName(pattern) {
// check is repository exist later at filterTags.
return []string{pattern}, nil
// if the pattern is a specific repository name, just returns the parsed repositories
// and will check the existence later when filtering the tags
if repositories, ok := util.IsSpecificPath(pattern); ok {
return repositories, nil
}
// search repositories from catalog api
repositories, err := n.Catalog()

121
src/replication/util/pattern.go Normal file
View File

@ -0,0 +1,121 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"fmt"
"strings"
"github.com/bmatcuk/doublestar"
)
// Match returns whether the str matches the pattern
func Match(pattern, str string) (bool, error) {
if len(pattern) == 0 {
return true, nil
}
return doublestar.Match(pattern, str)
}
// IsSpecificPath checks whether the input path is a specified string
// If it is, the function returns a string array that parsed from the input path
// A specified string means we can get a specific string array after parsing it
// "library/hello-world" is a specified string as it only matches "library/hello-world"
// "library/**" isn't a specified string as it can match all string that starts with "library/"
// "library/{test,busybox}" is a specified string as it only matches "library/hello-world" and "library/busybox"
func IsSpecificPath(path string) ([]string, bool) {
if len(path) == 0 {
return nil, false
}
components := [][]string{}
for _, component := range strings.Split(path, "/") {
strs, ok := IsSpecificPathComponent(component)
if !ok {
return nil, false
}
components = append(components, strs)
}
result := []string{}
for _, component := range components {
result = combinationPathComponents(result, component)
}
return result, true
}
func combinationPathComponents(components1, components2 []string) []string {
if len(components1) == 0 {
return components2
}
if len(components2) == 0 {
return components1
}
components := []string{}
for _, component1 := range components1 {
for _, component2 := range components2 {
components = append(components, fmt.Sprintf("%s/%s", component1, component2))
}
}
return components
}
// IsSpecificPathComponent checks whether the input path component is a specified string
// If it is, the function returns a string array that parsed from the input component
// A specified string means we can get a specific string array after parsing it
// "library" is a specified string as it only matches "library"
// "library*" isn't a specified string as it can match all string that starts with "library"
// "{library, test}" is a specified string as it only matches "library" and "test"
// Note: the function doesn't support the component that contains more than one "{"
// such as "a{b{c,d}e}f"
func IsSpecificPathComponent(component string) ([]string, bool) {
if len(component) == 0 {
return nil, false
}
// contains any of *?[\\]^
if strings.ContainsAny(component, "*?[\\]^/") {
return nil, false
}
// doesn't contain {},
if !strings.ContainsAny(component, "{},") {
return []string{component}, true
}
// support only one pair of {} currently
n := strings.Count(component, "{")
if n > 1 {
return nil, false
}
i := strings.Index(component, "{")
if i == -1 {
return nil, false
}
j := strings.LastIndex(component, "}")
if j == -1 {
return nil, false
}
if i > j {
return nil, false
}
prefix := component[:i]
suffix := ""
if j+1 < len(component) {
suffix = component[j+1:]
}
components := []string{}
strs := strings.Split(component[i+1:j], ",")
for _, str := range strs {
components = append(components, prefix+str+suffix)
}
return components, true
}

178
src/replication/util/pattern_test.go Normal file
View File

@ -0,0 +1,178 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMatch(t *testing.T) {
cases := []struct {
pattern string
str string
match bool
}{
{
pattern: "",
str: "library",
match: true,
},
{
pattern: "*",
str: "library",
match: true,
},
{
pattern: "*",
str: "library/hello-world",
match: false,
},
{
pattern: "**",
str: "library/hello-world",
match: true,
},
{
pattern: "{library,harbor}/**",
str: "library/hello-world",
match: true,
},
{
pattern: "{library,harbor}/**",
str: "harbor/hello-world",
match: true,
},
{
pattern: "1.?",
str: "1.0",
match: true,
},
{
pattern: "1.?",
str: "1.01",
match: false,
},
}
for _, c := range cases {
match, err := Match(c.pattern, c.str)
require.Nil(t, err)
assert.Equal(t, c.match, match)
}
}
func TestIsSpecificPathComponent(t *testing.T) {
cases := []struct {
component string
isSpecific bool
resultComponents []string
}{
{
component: "",
isSpecific: false,
resultComponents: []string{},
},
{
component: "library/hello-world",
isSpecific: false,
resultComponents: []string{},
},
{
component: "library",
isSpecific: true,
resultComponents: []string{"library"},
},
{
component: "lib*",
isSpecific: false,
resultComponents: []string{},
},
{
component: "{library}",
isSpecific: true,
resultComponents: []string{"library"},
},
{
component: "{library,test}",
isSpecific: true,
resultComponents: []string{"library", "test"},
},
{
component: "{library{a}c}",
isSpecific: false,
resultComponents: []string{},
},
}
for i, c := range cases {
fmt.Printf("running case %d ...\n", i)
components, ok := IsSpecificPathComponent(c.component)
require.Equal(t, c.isSpecific, ok)
require.Equal(t, len(c.resultComponents), len(components))
for i := range components {
assert.Equal(t, c.resultComponents[i], components[i])
}
}
}
func TestIsSpecificPath(t *testing.T) {
cases := []struct {
path string
isSpecific bool
resultPaths []string
}{
{
path: "",
isSpecific: false,
resultPaths: []string{},
},
{
path: "library",
isSpecific: true,
resultPaths: []string{"library"},
},
{
path: "library/hello-world",
isSpecific: true,
resultPaths: []string{"library/hello-world"},
},
{
path: "library/**",
isSpecific: false,
resultPaths: []string{},
},
{
path: "{library}",
isSpecific: true,
resultPaths: []string{"library"},
},
{
path: "library/{hello-world,busybox}",
isSpecific: true,
resultPaths: []string{"library/hello-world", "library/busybox"},
},
}
for i, c := range cases {
fmt.Printf("running case %d ...\n", i)
paths, ok := IsSpecificPath(c.path)
require.Equal(t, c.isSpecific, ok)
require.Equal(t, len(c.resultPaths), len(paths))
for i := range paths {
assert.Equal(t, c.resultPaths[i], paths[i])
}
}
}

22
src/replication/util/util.go
View File

@ -18,31 +18,9 @@ import (
"net/http"
"strings"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/bmatcuk/doublestar"
"github.com/goharbor/harbor/src/common/utils/registry"
)
// Match returns whether the str matches the pattern
func Match(pattern, str string) (bool, error) {
if len(pattern) == 0 {
return true, nil
}
match, err := doublestar.Match(pattern, str)
if err == doublestar.ErrBadPattern {
log.Warningf("failed to match the string %s against pattern %s: %v", str, pattern, err)
return false, nil
}
return match, err
}
// IsSpecificRepositoryName if the name not contains any char of "*?[{\\]}^,",
// it is a specific repository name.
func IsSpecificRepositoryName(name string) bool {
return !strings.ContainsAny(name, "*?[{\\]}^,")
}
// GetHTTPTransport can be used to share the common HTTP transport
func GetHTTPTransport(insecure bool) *http.Transport {
return registry.GetHTTPTransport(insecure)

127
src/replication/util/util_test.go
View File

@ -18,68 +18,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMatch(t *testing.T) {
cases := []struct {
pattern string
str string
match bool
}{
{
pattern: "",
str: "library",
match: true,
},
{
pattern: "*",
str: "library",
match: true,
},
{
pattern: "*",
str: "library/hello-world",
match: false,
},
{
pattern: "**",
str: "library/hello-world",
match: true,
},
{
pattern: "{library,harbor}/**",
str: "library/hello-world",
match: true,
},
{
pattern: "{library,harbor}/**",
str: "harbor/hello-world",
match: true,
},
{
pattern: "1.?",
str: "1.0",
match: true,
},
{
pattern: "1.?",
str: "1.01",
match: false,
},
{
pattern: "a[",
str: "aaa",
match: false,
},
}
for _, c := range cases {
match, err := Match(c.pattern, c.str)
require.Nil(t, err)
assert.Equal(t, c.match, match)
}
}
func TestGetHTTPTransport(t *testing.T) {
transport := GetHTTPTransport(true)
assert.True(t, transport.TLSClientConfig.InsecureSkipVerify)
@ -109,70 +49,3 @@ func TestParseRepository(t *testing.T) {
assert.Equal(t, "a/b", namespace)
assert.Equal(t, "c", rest)
}
func TestIsSpecificRepositoryName(t *testing.T) {
tests := []struct {
name string
input string
want bool
}{
{"Is Specific", "a", true},
{"Is Specific", "abc", true},
{"Is Specific", "a/b", true},
{"Is Specific", "a/b/c", true},
{"Not Specific", "*", false},
{"Not Specific", "?", false},
{"Not Specific", "*c", false},
{"Not Specific", "a*", false},
{"Not Specific", "a*b*c*d*e*", false},
{"Not Specific", "a*b?c*x", false},
{"Not Specific", "ab[c]", false},
{"Not Specific", "ab[b-d]", false},
{"Not Specific", "ab[e-g]", false},
{"Not Specific", "ab[^c]", false},
{"Not Specific", "ab[^b-d]", false},
{"Not Specific", "ab[^e-g]", false},
{"Not Specific", "a\\*b", false},
{"Not Specific", "a?b", false},
{"Not Specific", "a[^a]b", false},
{"Not Specific", "a???b", false},
{"Not Specific", "a[^a][^a][^a]b", false},
{"Not Specific", "[a-ζ]*", false},
{"Not Specific", "*[a-ζ]", false},
{"Not Specific", "a?b", false},
{"Not Specific", "a*b", false},
{"Not Specific", "[\\-]", false},
{"Not Specific", "[x\\-]", false},
{"Not Specific", "[x\\-]", false},
{"Not Specific", "[x\\-]", false},
{"Not Specific", "[\\-x]", false},
{"Not Specific", "[\\-x]", false},
{"Not Specific", "[\\-x]", false},
{"Not Specific", "[a-b-c]", false},
{"Not Specific", "*x", false},
{"Not Specific", "[abc]", false},
{"Not Specific", "**", false},
{"Not Specific", "ab{c,d}", false},
{"Not Specific", "ab{c,d,*}", false},
{"Not Specific", "abc**", false},
{"Not Specific", "[]a]", false},
{"Not Specific", "[-]", false},
{"Not Specific", "[x-]", false},
{"Not Specific", "[-x]", false},
{"Not Specific", "\\", false},
{"Not Specific", "[a-b-c]", false},
{"Not Specific", "[]", false},
{"Not Specific", "[", false},
{"Not Specific", "[^", false},
{"Not Specific", "^", false},
{"Not Specific", "]", false},
{"Not Specific", "[^bc", false},
{"Not Specific", "a[", false},
{"Not Specific", "ab{c,d}[", false},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
var got = IsSpecificRepositoryName(tt.input)
assert.Equal(t, tt.want, got)
})
}
}