Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-09-27 21:12:42 +02:00
Code Issues Projects Releases Wiki Activity

Reimplement the registry client

Browse Source 
This commit reimplements the registry client under directory src/pkg/registry and removes the useless code

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2020-02-21 09:29:28 +08:00
parent c2a77c2825
commit 528f598268
72 changed files with 2038 additions and 3147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/api/artifact/abstractor/blob/fetcher.go
View File

@ -18,12 +18,9 @@ import (
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/pkg/registry"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"io/ioutil"
"net/http"
)
var (
@ -39,6 +36,8 @@ var (
}
)
// TODO use the registry.Client directly? then the Fetcher can be deleted
// Fetcher fetches the content of blob
type Fetcher interface {
// FetchManifest the content of manifest under the repository
@ -49,49 +48,34 @@ type Fetcher interface {
// NewFetcher returns an instance of the default blob fetcher
func NewFetcher() Fetcher {
return &fetcher{}
return &fetcher{
client: registry.Cli,
}
}
type fetcher struct{}
type fetcher struct {
client registry.Client
}
// TODO re-implement it based on OCI registry driver
func (f *fetcher) FetchManifest(repository, digest string) (string, []byte, error) {
// TODO read from cache first
client, err := newRepositoryClient(repository)
manifest, _, err := f.client.PullManifest(repository, digest)
if err != nil {
return "", nil, err
}
mediaType, payload, err := manifest.Payload()
if err != nil {
return "", nil, err
}
_, mediaType, payload, err := client.PullManifest(digest, accept)
return mediaType, payload, err
}
// TODO re-implement it based on OCI registry driver
func (f *fetcher) FetchLayer(repository, digest string) ([]byte, error) {
// TODO read from cache first
client, err := newRepositoryClient(repository)
if err != nil {
return nil, err
}
_, reader, err := client.PullBlob(digest)
_, reader, err := f.client.PullBlob(repository, digest)
if err != nil {
return nil, err
}
defer reader.Close()
return ioutil.ReadAll(reader)
}
func newRepositoryClient(repository string) (*registry.Repository, error) {
uam := &auth.UserAgentModifier{
UserAgent: "harbor-registry-client",
}
authorizer := auth.DefaultBasicAuthorizer()
transport := registry.NewTransport(http.DefaultTransport, authorizer, uam)
client := &http.Client{
Transport: transport,
}
endpoint, err := config.RegistryURL()
if err != nil {
return nil, err
}
return registry.NewRepository(repository, endpoint, client)
}

17
src/common/http/client.go
View File

@ -19,14 +19,13 @@ import (
"crypto/tls"
"encoding/json"
"errors"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/internal"
"io"
"io/ioutil"
"net/http"
"net/url"
"reflect"
"strings"
"github.com/goharbor/harbor/src/common/http/modifier"
)
// Client is a util for common HTTP operations, such Get, Head, Post, Put and Delete.
@ -231,8 +230,8 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
if err != nil {
return err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return err
}
@ -250,12 +249,10 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
resources = reflect.AppendSlice(resources, reflect.Indirect(res))
endpoint = ""
link := resp.Header.Get("Link")
for _, str := range strings.Split(link, ",") {
if strings.HasSuffix(str, `rel="next"`) &&
strings.Index(str, "<") >= 0 &&
strings.Index(str, ">") >= 0 {
endpoint = url.Scheme + "://" + url.Host + str[strings.Index(str, "<")+1:strings.Index(str, ">")]
links := internal.ParseLinks(resp.Header.Get("Link"))
for _, link := range links {
if link.Rel == "next" {
endpoint = url.Scheme + "://" + url.Host + link.URL
break
}
}

40
src/common/utils/registry/auth/basicauthorizer.go
View File

@ -1,40 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/core/config"
"sync"
)
// NewBasicAuthorizer create an authorizer to add basic auth header as is set in the parameter
func NewBasicAuthorizer(u, p string) modifier.Modifier {
return NewBasicAuthCredential(u, p)
}
var (
defaultAuthorizer modifier.Modifier
once sync.Once
)
// DefaultBasicAuthorizer returns the basic authorizer that sets the basic auth as configured in env variables
func DefaultBasicAuthorizer() modifier.Modifier {
once.Do(func() {
u, p := config.RegistryCredential()
defaultAuthorizer = NewBasicAuthCredential(u, p)
})
return defaultAuthorizer
}

48
src/common/utils/registry/auth/credential.go
View File

@ -1,48 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"net/http"
"github.com/goharbor/harbor/src/common/http/modifier"
)
// Credential ...
type Credential modifier.Modifier
// Implements interface Credential
type basicAuthCredential struct {
username string
password string
}
// NewBasicAuthCredential ...
func NewBasicAuthCredential(username, password string) Credential {
return &basicAuthCredential{
username: username,
password: password,
}
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}
// implement github.com/goharbor/harbor/src/common/http/modifier.Modifier
func (b *basicAuthCredential) Modify(req *http.Request) error {
b.AddAuthorization(req)
return nil
}

58
src/common/utils/registry/auth/path.go
View File

@ -1,58 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"regexp"
"github.com/docker/distribution/reference"
"github.com/goharbor/harbor/src/common/utils/log"
)
var (
base = regexp.MustCompile("/v2")
catalog = regexp.MustCompile("/v2/_catalog")
tag = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/tags/list")
manifest = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/manifests/(" + reference.TagRegexp.String() + "|" + reference.DigestRegexp.String() + ")")
blob = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String())
blobUpload = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/uploads")
blobUploadChunk = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/uploads/[a-zA-Z0-9-_.=]+")
repoRegExps = []*regexp.Regexp{tag, manifest, blob, blobUploadChunk, blobUpload}
)
// parse the repository name from path, if the path doesn't match any
// regular expressions in repoRegExps, nil string will be returned
func parseRepository(path string) string {
for _, regExp := range repoRegExps {
subs := regExp.FindStringSubmatch(path)
// no match
if subs == nil {
continue
}
// match
// the subs should contain at least 2 matching texts, the first one matches
// the whole regular expression, and the second one matches the repository
// part
if len(subs) < 2 {
log.Warningf("unexpected length of sub matches: %d, should >= 2 ", len(subs))
continue
}
return subs[1]
}
return ""
}

43
src/common/utils/registry/auth/path_test.go
View File

@ -1,43 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseRepository(t *testing.T) {
cases := []struct {
input string
output string
}{
{"/v2", ""},
{"/v2/_catalog", ""},
{"/v2/library/tags/list", "library"},
{"/v2/tags/list", ""},
{"/v2/tags/list/tags/list", "tags/list"},
{"/v2/library/manifests/latest", "library"},
{"/v2/library/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", "library"},
{"/v2/library/blobs/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", "library"},
{"/v2/library/blobs/uploads", "library"},
{"/v2/library/blobs/uploads/1234567890", "library"},
}
for _, c := range cases {
assert.Equal(t, c.output, parseRepository(c.input))
}
}

354
src/common/utils/registry/auth/tokenauthorizer.go
View File

@ -1,354 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/docker/distribution/registry/auth/token"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
token_util "github.com/goharbor/harbor/src/core/service/token"
)
const (
latency int = 10 // second, the network latency when token is received
scheme = "bearer"
)
type tokenGenerator interface {
generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error)
}
// UserAgentModifier adds the "User-Agent" header to the request
type UserAgentModifier struct {
UserAgent string
}
// Modify adds user-agent header to the request
func (u *UserAgentModifier) Modify(req *http.Request) error {
req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.UserAgent)
return nil
}
// tokenAuthorizer implements registry.Modifier interface. It parses scopses
// from the request, generates authentication token and modifies the requset
// by adding the token
type tokenAuthorizer struct {
registryURL *url.URL // used to filter request
generator tokenGenerator
client *http.Client
cachedTokens map[string]*models.Token
sync.Mutex
}
// add token to the request
func (t *tokenAuthorizer) Modify(req *http.Request) error {
// only handle requests sent to registry
goon, err := t.filterReq(req)
if err != nil {
return err
}
if !goon {
log.Debugf("the request %s is not sent to registry, skip", req.URL.String())
return nil
}
// parse scopes from request
scopes, err := parseScopes(req)
if err != nil {
return err
}
var token *models.Token
// try to get token from cache if the request is for empty scope(login)
// or single scope
if len(scopes) <= 1 {
key := ""
if len(scopes) == 1 {
key = scopeString(scopes[0])
}
token = t.getCachedToken(key)
}
// request a new token if the token is null
if token == nil {
token, err = t.generator.generate(scopes, t.registryURL.String())
if err != nil {
return err
}
// if the token is null(this happens if the registry needs no authentication), return
// directly. Or the token will be cached
if token == nil {
return nil
}
// only cache the token for empty scope(login) or single scope request
if len(scopes) <= 1 {
key := ""
if len(scopes) == 1 {
key = scopeString(scopes[0])
}
t.updateCachedToken(key, token)
}
}
tk := token.GetToken()
if len(tk) == 0 {
return errors.New("empty token content")
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", tk))
return nil
}
func scopeString(scope *token.ResourceActions) string {
if scope == nil {
return ""
}
return fmt.Sprintf("%s:%s:%s", scope.Type, scope.Name, strings.Join(scope.Actions, ","))
}
// some requests are sent to backend storage, such as s3, this method filters
// the requests only sent to registry
func (t *tokenAuthorizer) filterReq(req *http.Request) (bool, error) {
// the registryURL is nil when the first request comes, init it with
// the scheme and host of the request which must be sent to the registry
if t.registryURL == nil {
u, err := url.Parse(buildPingURL(req.URL.Scheme + "://" + req.URL.Host))
if err != nil {
return false, err
}
t.registryURL = u
}
v2Index := strings.Index(req.URL.Path, "/v2/")
if v2Index == -1 {
return false, nil
}
if req.URL.Host != t.registryURL.Host || req.URL.Scheme != t.registryURL.Scheme ||
req.URL.Path[:v2Index+4] != t.registryURL.Path {
return false, nil
}
return true, nil
}
// parse scopes from the request according to its method, path and query string
func parseScopes(req *http.Request) ([]*token.ResourceActions, error) {
scopes := []*token.ResourceActions{}
from := req.URL.Query().Get("from")
if len(from) != 0 {
scopes = append(scopes, &token.ResourceActions{
Type: "repository",
Name: from,
Actions: []string{"pull"},
})
}
var scope *token.ResourceActions
path := strings.TrimRight(req.URL.Path, "/")
repository := parseRepository(path)
if len(repository) > 0 {
// pull, push, delete blob/manifest
scope = &token.ResourceActions{
Type: "repository",
Name: repository,
}
switch req.Method {
case http.MethodGet, http.MethodHead:
scope.Actions = []string{"pull"}
case http.MethodPost, http.MethodPut, http.MethodPatch:
scope.Actions = []string{"pull", "push"}
case http.MethodDelete:
scope.Actions = []string{"*"}
default:
scope = nil
log.Warningf("unsupported method: %s", req.Method)
}
} else if catalog.MatchString(path) {
// catalog
scope = &token.ResourceActions{
Type: "registry",
Name: "catalog",
Actions: []string{"*"},
}
} else if base.MatchString(path) {
// base
scope = nil
} else {
// unknown
return scopes, fmt.Errorf("can not parse scope from the request: %s %s", req.Method, req.URL.Path)
}
if scope != nil {
scopes = append(scopes, scope)
}
strs := []string{}
for _, s := range scopes {
strs = append(strs, scopeString(s))
}
log.Debugf("scopes parsed from request: %s", strings.Join(strs, " "))
return scopes, nil
}
func (t *tokenAuthorizer) getCachedToken(scope string) *models.Token {
t.Lock()
defer t.Unlock()
token := t.cachedTokens[scope]
if token == nil {
return nil
}
issueAt, err := time.Parse(time.RFC3339, token.IssuedAt)
if err != nil {
log.Errorf("failed parse %s: %v", token.IssuedAt, err)
delete(t.cachedTokens, scope)
return nil
}
if issueAt.Add(time.Duration(token.ExpiresIn-latency) * time.Second).Before(time.Now().UTC()) {
delete(t.cachedTokens, scope)
return nil
}
log.Debugf("get token for scope %s from cache", scope)
return token
}
func (t *tokenAuthorizer) updateCachedToken(scope string, token *models.Token) {
t.Lock()
defer t.Unlock()
t.cachedTokens[scope] = token
}
// ping returns the realm, service and error
func ping(client *http.Client, endpoint string) (string, string, error) {
resp, err := client.Get(endpoint)
if err != nil {
return "", "", err
}
defer resp.Body.Close()
challenges := ParseChallengeFromResponse(resp)
for _, challenge := range challenges {
if scheme == challenge.Scheme {
realm := challenge.Parameters["realm"]
service := challenge.Parameters["service"]
return realm, service, nil
}
}
log.Warningf("Schemas %v are unsupported", challenges)
return "", "", nil
}
// NewStandardTokenAuthorizer returns a standard token authorizer. The authorizer will request a token
// from token server and add it to the origin request
// If customizedTokenService is set, the token request will be sent to it instead of the server get from authorizer
func NewStandardTokenAuthorizer(client *http.Client, credential Credential,
customizedTokenService ...string) modifier.Modifier {
generator := &standardTokenGenerator{
credential: credential,
client: client,
}
// when the registry client is used inside Harbor, the token request
// can be posted to token service directly rather than going through nginx.
// If realm is set as the internal url of token service, this can resolve
// two problems:
// 1. performance issue
// 2. the realm field returned by registry is an IP which can not reachable
// inside Harbor
if len(customizedTokenService) > 0 && len(customizedTokenService[0]) > 0 {
generator.realm = customizedTokenService[0]
}
return &tokenAuthorizer{
cachedTokens: make(map[string]*models.Token),
generator: generator,
client: client,
}
}
// standardTokenGenerator implements interface tokenGenerator
type standardTokenGenerator struct {
realm string
service string
credential Credential
client *http.Client
}
// get token from token service
func (s *standardTokenGenerator) generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error) {
// ping first if the realm or service is null
if len(s.realm) == 0 || len(s.service) == 0 {
realm, service, err := ping(s.client, endpoint)
if err != nil {
return nil, err
}
if len(realm) == 0 {
log.Warning("empty realm, skip")
return nil, nil
}
if len(s.realm) == 0 {
s.realm = realm
}
s.service = service
}
return getToken(s.client, s.credential, s.realm, s.service, scopes)
}
// NewRawTokenAuthorizer returns a token authorizer which calls method to create
// token directly
func NewRawTokenAuthorizer(username, service string) modifier.Modifier {
generator := &rawTokenGenerator{
service: service,
username: username,
}
return &tokenAuthorizer{
cachedTokens: make(map[string]*models.Token),
generator: generator,
}
}
// rawTokenGenerator implements interface tokenGenerator
type rawTokenGenerator struct {
service string
username string
}
// generate token directly
func (r *rawTokenGenerator) generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error) {
return token_util.MakeToken(r.username, r.service, scopes)
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}

222
src/common/utils/registry/auth/tokenauthorizer_test.go
View File

@ -1,222 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"testing"
"time"
"github.com/docker/distribution/registry/auth/token"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFilterReq(t *testing.T) {
authorizer := tokenAuthorizer{}
// v2
req, err := http.NewRequest(http.MethodGet, "http://registry/v2/", nil)
require.Nil(t, err)
goon, err := authorizer.filterReq(req)
assert.Nil(t, err)
assert.True(t, goon)
// catalog
req, err = http.NewRequest(http.MethodGet, "http://registry/v2/_catalog?n=1000", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.True(t, goon)
// contains two v2 in path
req, err = http.NewRequest(http.MethodGet, "http://registry/v2/library/v2/tags/list", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.True(t, goon)
// different scheme
req, err = http.NewRequest(http.MethodGet, "https://registry/v2/library/golang/tags/list", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.False(t, goon)
// different host
req, err = http.NewRequest(http.MethodGet, "http://vmware.com/v2/library/golang/tags/list", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.False(t, goon)
// different path
req, err = http.NewRequest(http.MethodGet, "https://registry/s3/ssss", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.False(t, goon)
}
func TestParseScopes(t *testing.T) {
// contains from in query string
req, err := http.NewRequest(http.MethodGet, "http://registry/v2?from=library", nil)
require.Nil(t, err)
scopses, err := parseScopes(req)
assert.Nil(t, err)
assert.Equal(t, 1, len(scopses))
assert.EqualValues(t, &token.ResourceActions{
Type: "repository",
Name: "library",
Actions: []string{
"pull"},
}, scopses[0])
// v2
req, err = http.NewRequest(http.MethodGet, "http://registry/v2", nil)
require.Nil(t, err)
scopses, err = parseScopes(req)
assert.Nil(t, err)
assert.Equal(t, 0, len(scopses))
// catalog
req, err = http.NewRequest(http.MethodGet, "http://registry/v2/_catalog", nil)
require.Nil(t, err)
scopses, err = parseScopes(req)
assert.Nil(t, err)
assert.Equal(t, 1, len(scopses))
assert.EqualValues(t, &token.ResourceActions{
Type: "registry",
Name: "catalog",
Actions: []string{
"*"},
}, scopses[0])
// manifest
req, err = http.NewRequest(http.MethodPut, "http://registry/v2/library/mysql/5.6/manifests/1", nil)
require.Nil(t, err)
scopses, err = parseScopes(req)
assert.Nil(t, err)
assert.Equal(t, 1, len(scopses))
assert.EqualValues(t, &token.ResourceActions{
Type: "repository",
Name: "library/mysql/5.6",
Actions: []string{"pull", "push"},
}, scopses[0])
// invalid
req, err = http.NewRequest(http.MethodPut, "http://registry/other", nil)
require.Nil(t, err)
scopses, err = parseScopes(req)
assert.NotNil(t, err)
}
func TestGetAndUpdateCachedToken(t *testing.T) {
authorizer := &tokenAuthorizer{
cachedTokens: make(map[string]*models.Token),
}
// empty cache
token := authorizer.getCachedToken("")
assert.Nil(t, token)
// put a valid token into cache
token = &models.Token{
Token: "token",
ExpiresIn: 60,
IssuedAt: time.Now().Format(time.RFC3339),
}
authorizer.updateCachedToken("", token)
token2 := authorizer.getCachedToken("")
assert.EqualValues(t, token, token2)
// put a expired token into cache
token = &models.Token{
Token: "token",
ExpiresIn: 60,
IssuedAt: time.Now().Add(-time.Second * 120).Format("2006-01-02 15:04:05.999999999 -0700 MST"),
}
authorizer.updateCachedToken("", token)
token2 = authorizer.getCachedToken("")
assert.Nil(t, token2)
}
func TestModifyOfStandardTokenAuthorizer(t *testing.T) {
token := &models.Token{
Token: "token",
ExpiresIn: 3600,
IssuedAt: time.Now().String(),
}
data, err := json.Marshal(token)
require.Nil(t, err)
tokenHandler := test.Handler(&test.Response{
Body: data,
})
tokenServer := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/service/token",
Handler: tokenHandler,
})
defer tokenServer.Close()
header := fmt.Sprintf("Bearer realm=\"%s/service/token\",service=\"registry\"",
tokenServer.URL)
pingHandler := test.Handler(&test.Response{
StatusCode: http.StatusUnauthorized,
Headers: map[string]string{
"WWW-Authenticate": header,
},
})
registryServer := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2",
Handler: pingHandler,
})
defer registryServer.Close()
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/v2/", registryServer.URL), nil)
require.Nil(t, err)
authorizer := NewStandardTokenAuthorizer(http.DefaultClient, nil)
err = authorizer.Modify(req)
require.Nil(t, err)
tk := req.Header.Get("Authorization")
assert.Equal(t, strings.ToLower("Bearer "+token.Token), strings.ToLower(tk))
}
func TestUserAgentModifier(t *testing.T) {
agent := "harbor-registry-client"
modifier := &UserAgentModifier{
UserAgent: agent,
}
req, err := http.NewRequest(http.MethodGet, "http://registry/v2/", nil)
require.Nil(t, err)
modifier.Modify(req)
actual := req.Header.Get("User-Agent")
if actual != agent {
t.Errorf("expect request to have header User-Agent=%s, but got User-Agent=%s", agent, actual)
}
}

88
src/common/utils/registry/auth/util.go
View File

@ -1,88 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"github.com/docker/distribution/registry/auth/token"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/registry"
)
const (
service = "harbor-registry"
)
// GetToken requests a token against the endpoint using credential provided
func GetToken(endpoint string, insecure bool, credential Credential,
scopes []*token.ResourceActions) (*models.Token, error) {
client := &http.Client{
Transport: registry.GetHTTPTransport(insecure),
}
return getToken(client, credential, endpoint, service, scopes)
}
func getToken(client *http.Client, credential Credential, realm, service string,
scopes []*token.ResourceActions) (*models.Token, error) {
u, err := url.Parse(realm)
if err != nil {
return nil, err
}
query := u.Query()
query.Add("service", service)
for _, scope := range scopes {
query.Add("scope", scopeString(scope))
}
u.RawQuery = query.Encode()
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
if credential != nil {
credential.Modify(req)
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, &commonhttp.Error{
Code: resp.StatusCode,
Message: string(data),
}
}
token := &models.Token{}
if err = json.Unmarshal(data, token); err != nil {
return nil, err
}
return token, nil
}

60
src/common/utils/registry/manifest_test.go
View File

@ -1,60 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"testing"
"github.com/docker/distribution/manifest/schema2"
)
func TestUnMarshal(t *testing.T) {
b := []byte(`{
"schemaVersion":2,
"mediaType":"application/vnd.docker.distribution.manifest.v2+json",
"config":{
"mediaType":"application/vnd.docker.container.image.v1+json",
"size":1473,
"digest":"sha256:c54a2cc56cbb2f04003c1cd4507e118af7c0d340fe7e2720f70976c4b75237dc"
},
"layers":[
{
"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip",
"size":974,
"digest":"sha256:c04b14da8d1441880ed3fe6106fb2cc6fa1c9661846ac0266b8a5ec8edf37b7c"
}
]
}`)
manifest, _, err := UnMarshal(schema2.MediaTypeManifest, b)
if err != nil {
t.Fatalf("failed to parse manifest: %v", err)
}
refs := manifest.References()
if len(refs) != 2 {
t.Fatalf("unexpected length of reference: %d != %d", len(refs), 2)
}
digest := "sha256:c54a2cc56cbb2f04003c1cd4507e118af7c0d340fe7e2720f70976c4b75237dc"
if refs[0].Digest.String() != digest {
t.Errorf("unexpected digest: %s != %s", refs[0].Digest.String(), digest)
}
digest = "sha256:c04b14da8d1441880ed3fe6106fb2cc6fa1c9661846ac0266b8a5ec8edf37b7c"
if refs[1].Digest.String() != digest {
t.Errorf("unexpected digest: %s != %s", refs[1].Digest.String(), digest)
}
}

185
src/common/utils/registry/registry.go
View File

@ -1,185 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"crypto/tls"
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"strings"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils"
)
// Registry holds information of a registry entity
type Registry struct {
Endpoint *url.URL
client *http.Client
}
var defaultHTTPTransport, secureHTTPTransport, insecureHTTPTransport *http.Transport
func init() {
defaultHTTPTransport = &http.Transport{}
secureHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
},
}
insecureHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
}
// GetHTTPTransport returns HttpTransport based on insecure configuration
func GetHTTPTransport(insecure ...bool) *http.Transport {
if len(insecure) == 0 {
return defaultHTTPTransport
}
if insecure[0] {
return insecureHTTPTransport
}
return secureHTTPTransport
}
// NewRegistry returns an instance of registry
func NewRegistry(endpoint string, client *http.Client) (*Registry, error) {
u, err := utils.ParseEndpoint(endpoint)
if err != nil {
return nil, err
}
registry := &Registry{
Endpoint: u,
client: client,
}
return registry, nil
}
// Catalog ...
func (r *Registry) Catalog() ([]string, error) {
repos := []string{}
aurl := r.Endpoint.String() + "/v2/_catalog?n=1000"
for len(aurl) > 0 {
req, err := http.NewRequest("GET", aurl, nil)
if err != nil {
return repos, err
}
resp, err := r.client.Do(req)
if err != nil {
return nil, parseError(err)
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return repos, err
}
if resp.StatusCode == http.StatusOK {
catalogResp := struct {
Repositories []string `json:"repositories"`
}{}
if err := json.Unmarshal(b, &catalogResp); err != nil {
return repos, err
}
repos = append(repos, catalogResp.Repositories...)
// Link: </v2/_catalog?last=library%2Fhello-world-25&n=100>; rel="next"
// Link: <http://domain.com/v2/_catalog?last=library%2Fhello-world-25&n=100>; rel="next"
link := resp.Header.Get("Link")
if strings.HasSuffix(link, `rel="next"`) && strings.Index(link, "<") >= 0 && strings.Index(link, ">") >= 0 {
aurl = link[strings.Index(link, "<")+1 : strings.Index(link, ">")]
if strings.Index(aurl, ":") < 0 {
aurl = r.Endpoint.String() + aurl
}
} else {
aurl = ""
}
} else {
return repos, &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
}
return repos, nil
}
// Ping checks by Head method
func (r *Registry) Ping() error {
return r.ping(http.MethodHead)
}
// PingGet checks by Get method
func (r *Registry) PingGet() error {
return r.ping(http.MethodGet)
}
func (r *Registry) ping(method string) error {
req, err := http.NewRequest(method, buildPingURL(r.Endpoint.String()), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
// PingSimple checks whether the registry is available. It checks the connectivity and certificate (if TLS enabled)
// only, regardless of credential.
func (r *Registry) PingSimple() error {
err := r.Ping()
if err == nil {
return nil
}
httpErr, ok := err.(*commonhttp.Error)
if !ok {
return err
}
if httpErr.Code == http.StatusUnauthorized ||
httpErr.Code == http.StatusForbidden {
return nil
}
return httpErr
}

142
src/common/utils/registry/registry_test.go
View File

@ -1,142 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"testing"
"github.com/goharbor/harbor/src/common/utils/test"
)
func TestPing(t *testing.T) {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: http.MethodHead,
Pattern: "/v2/",
Handler: test.Handler(nil),
})
defer server.Close()
client, err := newRegistryClient(server.URL)
if err != nil {
t.Fatalf("failed to create client for registry: %v", err)
}
if err = client.Ping(); err != nil {
t.Errorf("failed to ping registry: %v", err)
}
}
func TestCatalog(t *testing.T) {
repositories := make([]string, 0, 1001)
for i := 0; i < 1001; i++ {
repositories = append(repositories, strconv.Itoa(i))
}
handler := func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
last := q.Get("last")
n, err := strconv.Atoi(q.Get("n"))
if err != nil || n <= 0 {
n = 1000
}
length := len(repositories)
begin := length
if len(last) == 0 {
begin = 0
} else {
for i, repository := range repositories {
if repository == last {
begin = i + 1
break
}
}
}
end := begin + n
if end > length {
end = length
}
w.Header().Set(http.CanonicalHeaderKey("Content-Type"), "application/json")
if end < length {
u, err := url.Parse("/v2/_catalog")
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
values := u.Query()
values.Add("last", repositories[end-1])
values.Add("n", strconv.Itoa(n))
u.RawQuery = values.Encode()
link := fmt.Sprintf("<%s>; rel=\"next\"", u.String())
w.Header().Set(http.CanonicalHeaderKey("link"), link)
}
repos := struct {
Repositories []string `json:"repositories"`
}{
Repositories: []string{},
}
if begin < length {
repos.Repositories = repositories[begin:end]
}
b, err := json.Marshal(repos)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Write(b)
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/_catalog",
Handler: handler,
})
defer server.Close()
client, err := newRegistryClient(server.URL)
if err != nil {
t.Fatalf("failed to create client for registry: %v", err)
}
repos, err := client.Catalog()
if err != nil {
t.Fatalf("failed to catalog repositories: %v", err)
}
if len(repos) != len(repositories) {
t.Errorf("unexpected length of repositories: %d != %d", len(repos), len(repositories))
}
}
func newRegistryClient(url string) (*Registry, error) {
return NewRegistry(url, &http.Client{})
}

535
src/common/utils/registry/repository.go
View File

@ -1,535 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils"
)
// Repository holds information of a repository entity
type Repository struct {
Name string
Endpoint *url.URL
client *http.Client
}
// NewRepository returns an instance of Repository
func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) {
name = strings.TrimSpace(name)
u, err := utils.ParseEndpoint(endpoint)
if err != nil {
return nil, err
}
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
return repository, nil
}
func parseError(err error) error {
if urlErr, ok := err.(*url.Error); ok {
if regErr, ok := urlErr.Err.(*commonhttp.Error); ok {
return regErr
}
}
return err
}
// ListTag ...
func (r *Repository) ListTag() ([]string, error) {
tags := []string{}
aurl := buildTagListURL(r.Endpoint.String(), r.Name)
for len(aurl) > 0 {
req, err := http.NewRequest("GET", aurl, nil)
if err != nil {
return tags, err
}
resp, err := r.client.Do(req)
if err != nil {
return nil, parseError(err)
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
if resp.StatusCode == http.StatusOK {
tagsResp := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
tags = append(tags, tagsResp.Tags...)
// Link: </v2/library/hello-world/tags/list?last=......>; rel="next"
// Link: <http://domain.com/v2/library/hello-world/tags/list?last=......>; rel="next"
link := resp.Header.Get("Link")
if strings.HasSuffix(link, `rel="next"`) && strings.Index(link, "<") >= 0 && strings.Index(link, ">") >= 0 {
aurl = link[strings.Index(link, "<")+1 : strings.Index(link, ">")]
if strings.Index(aurl, ":") < 0 {
aurl = r.Endpoint.String() + aurl
}
} else {
aurl = ""
}
} else if resp.StatusCode == http.StatusNotFound {
// TODO remove the logic if the bug of registry is fixed
// It's a workaround for a bug of registry: when listing tags of
// a repository which is being pushed, a "NAME_UNKNOWN" error will
// been returned, while the catalog API can list this repository.
return tags, nil
} else {
return tags, &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
}
sort.Strings(tags)
return tags, nil
}
// ManifestExist ...
func (r *Repository) ManifestExist(reference string) (digest string, exist bool, err error) {
req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema1.MediaTypeManifest)
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
exist = true
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
if resp.StatusCode == http.StatusNotFound {
return
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
// PullManifest ...
func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) {
req, err := http.NewRequest("GET", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
for _, mediaType := range acceptMediaTypes {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
payload = b
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
// PushManifest ...
func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (digest string, err error) {
req, err := http.NewRequest("PUT", buildManifestURL(r.Endpoint.String(), r.Name, reference),
bytes.NewReader(payload))
if err != nil {
return
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType)
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
// DeleteManifest ...
func (r *Repository) DeleteManifest(digest string) error {
req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusAccepted {
return nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
// MountBlob ...
func (r *Repository) MountBlob(digest, from string) error {
req, err := http.NewRequest("POST", buildMountBlobURL(r.Endpoint.String(), r.Name, digest, from), nil)
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode/100 != 2 {
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
return nil
}
// DeleteTag ...
func (r *Repository) DeleteTag(tag string) error {
digest, exist, err := r.ManifestExist(tag)
if err != nil {
return err
}
if !exist {
return &commonhttp.Error{
Code: http.StatusNotFound,
}
}
return r.DeleteManifest(digest)
}
// BlobExist ...
func (r *Repository) BlobExist(digest string) (bool, error) {
req, err := http.NewRequest("HEAD", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return false, err
}
resp, err := r.client.Do(req)
if err != nil {
return false, parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return true, nil
}
if resp.StatusCode == http.StatusNotFound {
return false, nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, err
}
return false, &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
// PullBlob : client must close data if it is not nil
func (r *Repository) PullBlob(digest string) (size int64, data io.ReadCloser, err error) {
req, err := http.NewRequest("GET", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return
}
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
if resp.StatusCode == http.StatusOK {
contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
size, err = strconv.ParseInt(contengLength, 10, 64)
if err != nil {
return
}
data = resp.Body
return
}
// can not close the connect if the status code is 200
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID string, err error) {
req, err := http.NewRequest("POST", buildInitiateBlobUploadURL(r.Endpoint.String(), r.Name), nil)
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusAccepted {
location = resp.Header.Get(http.CanonicalHeaderKey("Location"))
uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID"))
return
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) monolithicBlobUpload(location, digest string, size int64, data io.Reader) error {
url, err := buildMonolithicBlobUploadURL(r.Endpoint.String(), location, digest)
if err != nil {
return err
}
req, err := http.NewRequest("PUT", url, data)
if err != nil {
return err
}
req.ContentLength = size
resp, err := r.client.Do(req)
if err != nil {
return parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusCreated {
return nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
// PushBlob ...
func (r *Repository) PushBlob(digest string, size int64, data io.Reader) error {
location, _, err := r.initiateBlobUpload(r.Name)
if err != nil {
return err
}
return r.monolithicBlobUpload(location, digest, size, data)
}
// DeleteBlob ...
func (r *Repository) DeleteBlob(digest string) error {
req, err := http.NewRequest("DELETE", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusAccepted {
return nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}
func buildTagListURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName)
}
func buildManifestURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference)
}
func buildBlobURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repoName, reference)
}
func buildMountBlobURL(endpoint, repoName, digest, from string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/?mount=%s&from=%s", endpoint, repoName, digest, from)
}
func buildInitiateBlobUploadURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repoName)
}
func buildMonolithicBlobUploadURL(endpoint, location, digest string) (string, error) {
relative, err := isRelativeURL(location)
if err != nil {
return "", err
}
// when the registry enables "relativeurls", the location returned
// has no scheme and host part
if relative {
location = endpoint + location
}
query := ""
if strings.ContainsRune(location, '?') {
query = "&"
} else {
query = "?"
}
query += fmt.Sprintf("digest=%s", digest)
return fmt.Sprintf("%s%s", location, query), nil
}
func isRelativeURL(endpoint string) (bool, error) {
u, err := url.Parse(endpoint)
if err != nil {
return false, err
}
return !u.IsAbs(), nil
}

458
src/common/utils/registry/repository_test.go
View File

@ -1,458 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/docker/distribution/manifest/schema2"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/test"
)
var (
repository = "library/hello-world"
tag = "latest"
mediaType = schema2.MediaTypeManifest
manifest = []byte("manifest")
blob = []byte("blob")
uuid = "0663ff44-63bb-11e6-8b77-86f30ca893d3"
digest = "sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b"
)
func TestBlobExist(t *testing.T) {
handler := func(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
dgt := path[strings.LastIndex(path, "/")+1:]
if dgt == digest {
w.Header().Add(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(blob)))
w.Header().Add(http.CanonicalHeaderKey("Docker-Content-Digest"), digest)
w.Header().Add(http.CanonicalHeaderKey("Content-Type"), "application/octet-stream")
return
}
w.WriteHeader(http.StatusNotFound)
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: fmt.Sprintf("/v2/%s/blobs/", repository),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
err = parseError(err)
t.Fatalf("failed to create client for repository: %v", err)
}
exist, err := client.BlobExist(digest)
if err != nil {
t.Fatalf("failed to check the existence of blob: %v", err)
}
if !exist {
t.Errorf("blob should exist on registry, but it does not exist")
}
exist, err = client.BlobExist("invalid_digest")
if err != nil {
t.Fatalf("failed to check the existence of blob: %v", err)
}
if exist {
t.Errorf("blob should not exist on registry, but it exists")
}
}
func TestPullBlob(t *testing.T) {
handler := test.Handler(&test.Response{
Headers: map[string]string{
"Content-Length": strconv.Itoa(len(blob)),
"Docker-Content-Digest": digest,
"Content-Type": "application/octet-stream",
},
Body: blob,
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
size, reader, err := client.PullBlob(digest)
if err != nil {
t.Fatalf("failed to pull blob: %v", err)
}
if size != int64(len(blob)) {
t.Errorf("unexpected size of blob: %d != %d", size, len(blob))
}
b, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatalf("failed to read from reader: %v", err)
}
if bytes.Compare(b, blob) != 0 {
t.Errorf("unexpected blob: %s != %s", string(b), string(blob))
}
}
func TestPushBlob(t *testing.T) {
location := ""
initUploadHandler := func(w http.ResponseWriter, r *http.Request) {
w.Header().Add(http.CanonicalHeaderKey("Content-Length"), "0")
w.Header().Add(http.CanonicalHeaderKey("Location"), location)
w.Header().Add(http.CanonicalHeaderKey("Range"), "0-0")
w.Header().Add(http.CanonicalHeaderKey("Docker-Upload-UUID"), uuid)
w.WriteHeader(http.StatusAccepted)
}
monolithicUploadHandler := test.Handler(&test.Response{
StatusCode: http.StatusCreated,
Headers: map[string]string{
"Content-Length": "0",
"Location": fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
"Docker-Content-Digest": digest,
},
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "POST",
Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/", repository),
Handler: initUploadHandler,
},
&test.RequestHandlerMapping{
Method: "PUT",
Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/%s", repository, uuid),
Handler: monolithicUploadHandler,
})
defer server.Close()
location = fmt.Sprintf("%s/v2/%s/blobs/uploads/%s", server.URL, repository, uuid)
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
if err = client.PushBlob(digest, int64(len(blob)), bytes.NewReader(blob)); err != nil {
t.Fatalf("failed to push blob: %v", err)
}
}
func TestDeleteBlob(t *testing.T) {
handler := test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "DELETE",
Pattern: fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
if err = client.DeleteBlob(digest); err != nil {
t.Fatalf("failed to delete blob: %v", err)
}
}
func TestManifestExist(t *testing.T) {
handler := func(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
tg := path[strings.LastIndex(path, "/")+1:]
if tg == tag {
w.Header().Add(http.CanonicalHeaderKey("Docker-Content-Digest"), digest)
w.Header().Add(http.CanonicalHeaderKey("Content-Type"), mediaType)
return
}
w.WriteHeader(http.StatusNotFound)
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
d, exist, err := client.ManifestExist(tag)
if err != nil {
t.Fatalf("failed to check the existence of manifest: %v", err)
}
if !exist || d != digest {
t.Errorf("manifest should exist on registry, but it does not exist")
}
_, exist, err = client.ManifestExist("invalid_tag")
if err != nil {
t.Fatalf("failed to check the existence of manifest: %v", err)
}
if exist {
t.Errorf("manifest should not exist on registry, but it exists")
}
}
func TestPullManifest(t *testing.T) {
handler := test.Handler(&test.Response{
Headers: map[string]string{
"Docker-Content-Digest": digest,
"Content-Type": mediaType,
},
Body: manifest,
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
d, md, payload, err := client.PullManifest(tag, []string{mediaType})
if err != nil {
t.Fatalf("failed to pull manifest: %v", err)
}
if d != digest {
t.Errorf("unexpected digest of manifest: %s != %s", d, digest)
}
if md != mediaType {
t.Errorf("unexpected media type of manifest: %s != %s", md, mediaType)
}
if bytes.Compare(payload, manifest) != 0 {
t.Errorf("unexpected manifest: %s != %s", string(payload), string(manifest))
}
}
func TestPushManifest(t *testing.T) {
handler := test.Handler(&test.Response{
StatusCode: http.StatusCreated,
Headers: map[string]string{
"Content-Length": "0",
"Docker-Content-Digest": digest,
"Location": "",
},
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "PUT",
Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
d, err := client.PushManifest(tag, mediaType, manifest)
if err != nil {
t.Fatalf("failed to pull manifest: %v", err)
}
if d != digest {
t.Errorf("unexpected digest of manifest: %s != %s", d, digest)
}
}
func TestDeleteTag(t *testing.T) {
manifestExistHandler := test.Handler(&test.Response{
Headers: map[string]string{
"Docker-Content-Digest": digest,
"Content-Type": mediaType,
},
})
deleteManifestandler := test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: fmt.Sprintf("/v2/%s/manifests/", repository),
Handler: manifestExistHandler,
},
&test.RequestHandlerMapping{
Method: "DELETE",
Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, digest),
Handler: deleteManifestandler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
if err = client.DeleteTag(tag); err != nil {
t.Fatalf("failed to delete tag: %v", err)
}
}
func TestListTag(t *testing.T) {
handler := test.Handler(&test.Response{
Headers: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(fmt.Sprintf("{\"name\": \"%s\",\"tags\": [\"%s\"]}", repository, tag)),
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: fmt.Sprintf("/v2/%s/tags/list", repository),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
tags, err := client.ListTag()
if err != nil {
t.Fatalf("failed to list tags: %v", err)
}
if len(tags) != 1 {
t.Fatalf("unexpected length of tags: %d != %d", len(tags), 1)
}
if tags[0] != tag {
t.Errorf("unexpected tag: %s != %s", tags[0], tag)
}
}
func TestParseError(t *testing.T) {
err := &url.Error{
Err: &commonhttp.Error{},
}
e := parseError(err)
if _, ok := e.(*commonhttp.Error); !ok {
t.Errorf("error type does not match registry error")
}
}
func newRepository(endpoint string) (*Repository, error) {
return NewRepository(repository, endpoint, &http.Client{})
}
func TestBuildMonolithicBlobUploadURL(t *testing.T) {
endpoint := "http://192.169.0.1"
digest := "sha256:ef15416724f6e2d5d5b422dc5105add931c1f2a45959cd4993e75e47957b3b55"
// absolute URL
location := "http://192.169.0.1/v2/library/golang/blobs/uploads/c9f84fd7-0198-43e3-80a7-dd13771cd7f0?_state=GabyCujPu0dpxiY8yYZTq"
expected := location + "&digest=" + digest
url, err := buildMonolithicBlobUploadURL(endpoint, location, digest)
require.Nil(t, err)
assert.Equal(t, expected, url)
// relative URL
location = "/v2/library/golang/blobs/uploads/c9f84fd7-0198-43e3-80a7-dd13771cd7f0?_state=GabyCujPu0dpxiY8yYZTq"
expected = endpoint + location + "&digest=" + digest
url, err = buildMonolithicBlobUploadURL(endpoint, location, digest)
require.Nil(t, err)
assert.Equal(t, expected, url)
}
func TestBuildMountBlobURL(t *testing.T) {
endpoint := "http://192.169.0.1"
repoName := "library/hello-world"
digest := "sha256:ef15416724f6e2d5d5b422dc5105add931c1f2a45959cd4993e75e47957b3b55"
from := "library/hi-world"
expected := fmt.Sprintf("%s/v2/%s/blobs/uploads/?mount=%s&from=%s", endpoint, repoName, digest, from)
actual := buildMountBlobURL(endpoint, repoName, digest, from)
assert.Equal(t, expected, actual)
}
func TestMountBlob(t *testing.T) {
mountHandler := func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusAccepted)
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "POST",
Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/", repository),
Handler: mountHandler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
if err = client.MountBlob(digest, "library/hi-world"); err != nil {
t.Fatalf("failed to mount blob: %v", err)
}
}

43
src/core/api/quota/registry/registry.go
View File

@ -15,23 +15,19 @@
package registry
import (
"strings"
"sync"
"time"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/core/api"
quota "github.com/goharbor/harbor/src/core/api/quota"
"github.com/goharbor/harbor/src/core/promgr"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/registry"
"github.com/pkg/errors"
"strings"
"sync"
"time"
)
// Migrator ...
@ -60,7 +56,7 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
err error
)
reposInRegistry, err := api.Catalog()
reposInRegistry, err := registry.Cli.Catalog()
if err != nil {
return nil, err
}
@ -392,11 +388,7 @@ func infoOfProject(project string, repoList []string) (quota.ProjectInfo, error)
}
func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
repoClient, err := coreutils.NewRepositoryClientForUI("harbor-core", repo)
if err != nil {
return quota.RepoData{}, err
}
tags, err := repoClient.ListTag()
tags, err := registry.Cli.ListTags(repo)
if err != nil {
return quota.RepoData{}, err
}
@ -405,11 +397,7 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
var blobs []*models.Blob
for _, tag := range tags {
_, mediaType, payload, err := repoClient.PullManifest(tag, []string{
schema1.MediaTypeManifest,
schema1.MediaTypeSignedManifest,
schema2.MediaTypeManifest,
})
manifest, digest, err := registry.Cli.PullManifest(repo, tag)
if err != nil {
log.Error(err)
// To workaround issue: https://github.com/goharbor/harbor/issues/9299, just log the error and do not raise it.
@ -417,28 +405,27 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
// User still can view there images with size 0 in harbor.
continue
}
manifest, desc, err := registry.UnMarshal(mediaType, payload)
mediaType, payload, err := manifest.Payload()
if err != nil {
log.Error(err)
return quota.RepoData{}, err
}
// self
afnb := &models.ArtifactAndBlob{
DigestAF: desc.Digest.String(),
DigestBlob: desc.Digest.String(),
DigestAF: digest,
DigestBlob: digest,
}
afnbs = append(afnbs, afnb)
// add manifest as a blob.
blob := &models.Blob{
Digest: desc.Digest.String(),
ContentType: desc.MediaType,
Size: desc.Size,
Digest: digest,
ContentType: mediaType,
Size: int64(len(payload)),
CreationTime: time.Now(),
}
blobs = append(blobs, blob)
for _, layer := range manifest.References() {
afnb := &models.ArtifactAndBlob{
DigestAF: desc.Digest.String(),
DigestAF: digest,
DigestBlob: layer.Digest.String(),
}
afnbs = append(afnbs, afnb)
@ -454,7 +441,7 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
PID: pid,
Repo: repo,
Tag: tag,
Digest: desc.Digest.String(),
Digest: digest,
Kind: "Docker-Image",
CreationTime: time.Now(),
}

15
src/core/api/scan.go
View File

@ -15,6 +15,7 @@
package api
import (
"github.com/goharbor/harbor/src/pkg/registry"
"net/http"
"strconv"
@ -22,7 +23,6 @@ import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/errs"
"github.com/goharbor/harbor/src/pkg/scan/report"
@ -192,20 +192,15 @@ func (sa *ScanAPI) Log() {
// TODO: This can be removed if the registry access interface is ready.
type digestGetter func(repo, tag string, username string) (string, error)
// TODO this method should be reconsidered as the tags are stored in database
// TODO rather than in registry
func getDigest(repo, tag string, username string) (string, error) {
client, err := coreutils.NewRepositoryClientForUI(username, repo)
exist, digest, err := registry.Cli.ManifestExist(repo, tag)
if err != nil {
return "", err
}
digest, exists, err := client.ManifestExist(tag)
if err != nil {
return "", err
}
if !exists {
if !exist {
return "", errors.Errorf("tag %s does exist", tag)
}
return digest, nil
}

22
src/core/api/search.go
View File

@ -23,7 +23,6 @@ import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
coreutils "github.com/goharbor/harbor/src/core/utils"
"k8s.io/helm/cmd/helm/search"
)
@ -180,27 +179,10 @@ func filterRepositories(projects []*models.Project, keyword string) (
entry["project_public"] = project.IsPublic()
entry["pull_count"] = repository.PullCount
tags, err := getTags(repository.Name)
if err != nil {
return nil, err
}
entry["tags_count"] = len(tags)
// TODO populate artifact count
// entry["tags_count"] = len(tags)
result = append(result, entry)
}
return result, nil
}
func getTags(repository string) ([]string, error) {
client, err := coreutils.NewRepositoryClientForUI("harbor-core", repository)
if err != nil {
return nil, err
}
tags, err := client.ListTag()
if err != nil {
return nil, err
}
return tags, nil
}

63
src/core/api/utils.go
View File

@ -1,63 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"net/http"
"strings"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
)
// Catalog ...
func Catalog() ([]string, error) {
repositories := []string{}
rc, err := initRegistryClient()
if err != nil {
return repositories, err
}
repositories, err = rc.Catalog()
if err != nil {
return repositories, err
}
return repositories, nil
}
func initRegistryClient() (r *registry.Registry, err error) {
endpoint, err := config.RegistryURL()
if err != nil {
return nil, err
}
addr := endpoint
if strings.Contains(endpoint, "://") {
addr = strings.Split(endpoint, "://")[1]
}
if err := utils.TestTCPConn(addr, 60, 2); err != nil {
return nil, err
}
authorizer := auth.DefaultBasicAuthorizer()
return registry.NewRegistry(endpoint, &http.Client{
Transport: registry.NewTransport(registry.GetHTTPTransport(), authorizer),
})
}

45
src/core/utils/utils.go
View File

@ -1,45 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package utils contains methods to support security, cache, and webhook functions.
package utils
import (
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
"net/http"
)
// NewRepositoryClientForUI creates a repository client that can only be used to
// access the internal registry
func NewRepositoryClientForUI(username, repository string) (*registry.Repository, error) {
endpoint, err := config.RegistryURL()
if err != nil {
return nil, err
}
return newRepositoryClient(endpoint, username, repository)
}
func newRepositoryClient(endpoint, username, repository string) (*registry.Repository, error) {
uam := &auth.UserAgentModifier{
UserAgent: "harbor-registry-client",
}
authorizer := auth.DefaultBasicAuthorizer()
transport := registry.NewTransport(http.DefaultTransport, authorizer, uam)
client := &http.Client{
Transport: transport,
}
return registry.NewRepository(repository, endpoint, client)
}

12
src/common/utils/registry/auth/nullauthorizer.go → src/internal/authorizer.go
View File

@ -12,13 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
package internal
import "net/http"
import "github.com/goharbor/harbor/src/common/http/modifier"
type nullAuthorizer struct{}
func (n *nullAuthorizer) Modify(req *http.Request) error {
// do nothing
return nil
}
// Authorizer authorizes the request
type Authorizer modifier.Modifier

89
src/internal/link.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"fmt"
"strings"
)
// Link defines the model that describes the HTTP link header
type Link struct {
URL string
Rel string
Attrs map[string]string
}
// String returns the string representation of a link
func (l *Link) String() string {
s := fmt.Sprintf("<%s>", l.URL)
if len(l.Rel) > 0 {
s = fmt.Sprintf(`%s; rel="%s"`, s, l.Rel)
}
for key, value := range l.Attrs {
s = fmt.Sprintf(`%s; %s="%s"`, s, key, value)
}
return s
}
// Links is a link object array
type Links []*Link
// String returns the string representation of links
func (l Links) String() string {
var strs []string
for _, link := range l {
strs = append(strs, link.String())
}
return strings.Join(strs, " , ")
}
// ParseLinks parses the link header into Links
// e.g. <http://example.com/TheBook/chapter2>; rel="previous"; title="previous chapter" , <http://example.com/TheBook/chapter4>; rel="next"; title="next chapter"
func ParseLinks(str string) Links {
var links Links
for _, lk := range strings.Split(str, ",") {
link := &Link{
Attrs: map[string]string{},
}
for _, attr := range strings.Split(lk, ";") {
attr = strings.TrimSpace(attr)
if len(attr) == 0 {
continue
}
if attr[0] == '<' && attr[len(attr)-1] == '>' {
link.URL = attr[1 : len(attr)-1]
continue
}
parts := strings.SplitN(attr, "=", 2)
key := parts[0]
value := ""
if len(parts) == 2 {
value = strings.Trim(parts[1], `"`)
}
if key == "rel" {
link.Rel = value
} else {
link.Attrs[key] = value
}
}
if len(link.URL) == 0 {
continue
}
links = append(links, link)
}
return links
}

37
src/internal/link_test.go Normal file
View File

@ -0,0 +1,37 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
func TestMethodsOfLink(t *testing.T) {
str := `<http://example.com/TheBook/chapter2>; rel="previous"; title="previous chapter" , <http://example.com/TheBook/chapter4>; rel="next"; title="next chapter"`
links := ParseLinks(str)
require.Len(t, links, 2)
assert.Equal(t, "http://example.com/TheBook/chapter2", links[0].URL)
assert.Equal(t, "previous", links[0].Rel)
assert.Equal(t, "previous chapter", links[0].Attrs["title"])
assert.Equal(t, "http://example.com/TheBook/chapter4", links[1].URL)
assert.Equal(t, "next", links[1].Rel)
assert.Equal(t, "previous", links[0].Rel)
assert.Equal(t, "next chapter", links[1].Attrs["title"])
s := links.String()
assert.Equal(t, str, s)
}

42
src/common/utils/registry/auth/basicauthorizer_test.go → src/internal/transport.go
View File

@ -12,28 +12,32 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
package internal
import (
"github.com/stretchr/testify/assert"
"crypto/tls"
"net/http"
"os"
"testing"
)
func TestDefaultBasicAuthorizer(t *testing.T) {
os.Setenv("REGISTRY_CREDENTIAL_USERNAME", "testuser")
os.Setenv("REGISTRY_CREDENTIAL_PASSWORD", "testpassword")
defer func() {
os.Unsetenv("REGISTRY_CREDENTIAL_USERNAME")
os.Unsetenv("REGISTRY_CREDENTIAL_PASSWORD")
}()
req, _ := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
a := DefaultBasicAuthorizer()
err := a.Modify(req)
assert.Nil(t, err)
u, p, ok := req.BasicAuth()
assert.True(t, ok)
assert.Equal(t, "testuser", u)
assert.Equal(t, "testpassword", p)
var (
secureHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
},
}
insecureHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
)
// GetHTTPTransport returns the HTTP transport based on insecure configuration
func GetHTTPTransport(insecure ...bool) *http.Transport {
if len(insecure) > 0 && insecure[0] {
return insecureHTTPTransport
}
return secureHTTPTransport
}

4
src/jobservice/job/impl/replication/scheduler.go
View File

@ -16,12 +16,12 @@ package replication
import (
"fmt"
"github.com/goharbor/harbor/src/internal"
"net/http"
"os"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier/auth"
reg "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/replication/model"
)
@ -61,7 +61,7 @@ func (s *Scheduler) Run(ctx job.Context, params job.Parameters) error {
policyID := (int64)(params["policy_id"].(float64))
cred := auth.NewSecretAuthorizer(os.Getenv("JOBSERVICE_SECRET"))
client := common_http.NewClient(&http.Client{
Transport: reg.GetHTTPTransport(true),
Transport: internal.GetHTTPTransport(true),
}, cred)
if err := client.Post(url, struct {
PolicyID int64 `json:"policy_id"`

56
src/jobservice/job/impl/utils/utils.go
View File

@ -1,56 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"fmt"
"net/http"
"os"
"sync"
httpauth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils/registry"
)
var coreClient *http.Client
var mutex = &sync.Mutex{}
// UserAgentModifier adds the "User-Agent" header to the request
type UserAgentModifier struct {
UserAgent string
}
// Modify adds user-agent header to the request
func (u *UserAgentModifier) Modify(req *http.Request) error {
req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.UserAgent)
return nil
}
// GetClient returns the HTTP client that will attach jobservce secret to the request, which can be used for
// accessing Harbor's Core Service.
// This function returns error if the secret of Job service is not set.
func GetClient() (*http.Client, error) {
mutex.Lock()
defer mutex.Unlock()
if coreClient == nil {
secret := os.Getenv("JOBSERVICE_SECRET")
if len(secret) == 0 {
return nil, fmt.Errorf("unable to load secret for job service")
}
modifier := httpauth.NewSecretAuthorizer(secret)
coreClient = &http.Client{Transport: registry.NewTransport(&http.Transport{}, modifier)}
}
return coreClient, nil
}

45
src/jobservice/job/impl/utils/utils_test.go
View File

@ -1,45 +0,0 @@
// Copyright 2018 The Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/goharbor/harbor/src/common/secret"
"github.com/stretchr/testify/assert"
)
func TestGetClient(t *testing.T) {
assert := assert.New(t)
os.Setenv("", "")
_, err := GetClient()
assert.NotNil(err, "Error should be thrown if secret is not set")
os.Setenv("JOBSERVICE_SECRET", "thesecret")
c, err := GetClient()
assert.Nil(err)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
v := r.Header.Get("Authorization")
assert.Equal(secret.HeaderPrefix+"thesecret", v)
}))
defer ts.Close()
c.Get(ts.URL)
os.Setenv("", "")
_, err = GetClient()
assert.Nil(err, "Error should be nil once client is initialized")
}

45
src/common/utils/registry/auth/authorizer.go → src/pkg/registry/auth/authorizer.go
View File

@ -18,6 +18,10 @@ import (
"fmt"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/internal"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
"github.com/goharbor/harbor/src/pkg/registry/auth/null"
"net/http"
"net/url"
"strings"
@ -25,18 +29,14 @@ import (
)
// NewAuthorizer creates an authorizer that can handle different auth schemes
func NewAuthorizer(credential Credential, client ...*http.Client) modifier.Modifier {
authorizer := &authorizer{
credential: credential,
func NewAuthorizer(username, password string, insecure bool) internal.Authorizer {
return &authorizer{
username: username,
password: password,
client: &http.Client{
Transport: internal.GetHTTPTransport(insecure),
},
}
if len(client) > 0 {
authorizer.client = client[0]
}
if authorizer.client == nil {
authorizer.client = http.DefaultClient
}
return authorizer
}
// authorizer authorizes the request with the provided credential.
@ -44,15 +44,16 @@ func NewAuthorizer(credential Credential, client ...*http.Client) modifier.Modif
// different underlying authorizers to do the auth work
type authorizer struct {
sync.Mutex
username string
password string
client *http.Client
url *url.URL // registry URL
authorizer modifier.Modifier // the underlying authorizer
credential Credential
}
func (a *authorizer) Modify(req *http.Request) error {
// Nil URL means this is the first time the authorizer is called
// Try to connect to the registry and determine the auth method
// Try to connect to the registry and determine the auth scheme
if a.url == nil {
// to avoid concurrent issue
a.Lock()
@ -83,25 +84,25 @@ func (a *authorizer) initialize(u *url.URL) error {
if err != nil {
return err
}
challenges := ParseChallengeFromResponse(resp)
challenges := challenge.ResponseChallenges(resp)
// no challenge, mean no auth
if len(challenges) == 0 {
a.authorizer = &nullAuthorizer{}
a.authorizer = null.NewAuthorizer()
return nil
}
cm := map[string]challenge.Challenge{}
for _, challenge := range challenges {
cm[challenge.Scheme] = challenge
}
if _, exist := cm["basic"]; exist {
a.authorizer = a.credential
if challenge, exist := cm["bearer"]; exist {
a.authorizer = bearer.NewAuthorizer(challenge.Parameters["realm"],
challenge.Parameters["service"], basic.NewAuthorizer(a.username, a.password),
a.client.Transport.(*http.Transport))
return nil
}
if _, exist := cm["bearer"]; exist {
// TODO clean up the code of "StandardTokenAuthorizer"
// TODO Currently, the checking of auth scheme is done twice, this can be avoided
a.authorizer = NewStandardTokenAuthorizer(a.client, a.credential)
if _, exist := cm["basic"]; exist {
a.authorizer = basic.NewAuthorizer(a.username, a.password)
return nil
}
return fmt.Errorf("unspported auth scheme: %v", challenges)

41
src/common/utils/registry/auth/credential_test.go → src/pkg/registry/auth/basic/authorizer.go
View File

@ -12,32 +12,29 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
package basic
import (
"github.com/goharbor/harbor/src/internal"
"net/http"
"testing"
)
func TestAddAuthorizationOfBasicAuthCredential(t *testing.T) {
cred := NewBasicAuthCredential("usr", "pwd")
req, err := http.NewRequest("GET", "http://example.com", nil)
if err != nil {
t.Fatalf("failed to create request: %v", err)
}
cred.Modify(req)
usr, pwd, ok := req.BasicAuth()
if !ok {
t.Fatal("basic auth not found")
}
if usr != "usr" {
t.Errorf("unexpected username: %s != usr", usr)
}
if pwd != "pwd" {
t.Errorf("unexpected password: %s != pwd", pwd)
// NewAuthorizer return a basic authorizer
func NewAuthorizer(username, password string) internal.Authorizer {
return &authorizer{
username: username,
password: password,
}
}
type authorizer struct {
username string
password string
}
func (a *authorizer) Modify(req *http.Request) error {
if len(a.username) > 0 {
req.SetBasicAuth(a.username, a.password)
}
return nil
}

19
src/common/utils/registry/manifest.go → src/pkg/registry/auth/basic/authorizer_test.go
View File

@ -12,13 +12,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
package basic
import (
"github.com/docker/distribution"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"testing"
)
// UnMarshal converts []byte to be distribution.Manifest
func UnMarshal(mediaType string, data []byte) (distribution.Manifest, distribution.Descriptor, error) {
return distribution.UnmarshalManifest(mediaType, data)
func TestModify(t *testing.T) {
authorizer := NewAuthorizer("u", "p")
req, _ := http.NewRequest(http.MethodGet, "", nil)
err := authorizer.Modify(req)
require.Nil(t, err)
u, p, ok := req.BasicAuth()
require.True(t, ok)
assert.Equal(t, "u", u)
assert.Equal(t, "p", p)
}

146
src/pkg/registry/auth/bearer/authorizer.go Normal file
View File

@ -0,0 +1,146 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/internal"
ierror "github.com/goharbor/harbor/src/internal/error"
"io/ioutil"
"net/http"
"net/url"
)
const (
cacheCapacity = 100
)
// NewAuthorizer return a bearer token authorizer
// The parameter "a" is an authorizer used to fetch the token
func NewAuthorizer(realm, service string, a internal.Authorizer, transport ...*http.Transport) internal.Authorizer {
authorizer := &authorizer{
realm: realm,
service: service,
authorizer: a,
cache: newCache(cacheCapacity),
}
tp := http.DefaultTransport
if len(transport) > 0 && transport[0] != nil {
tp = transport[0]
}
authorizer.client = &http.Client{Transport: tp}
return authorizer
}
type authorizer struct {
realm string
service string
authorizer internal.Authorizer
cache *cache
client *http.Client
}
func (a *authorizer) Modify(req *http.Request) error {
// parse scopes from request
scopes := parseScopes(req)
// get token
token, err := a.getToken(scopes)
if err != nil {
return err
}
// set authorization header
if token != nil && len(token.Token) > 0 {
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token.Token))
}
return nil
}
func (a *authorizer) getToken(scopes []*scope) (*token, error) {
// get token from cache first
token := a.cache.get(scopes)
if token != nil {
return token, nil
}
// get no token from cache, fetch it from the token service
token, err := a.fetchToken(scopes)
if err != nil {
return nil, err
}
// set the token into the cache
a.cache.set(scopes, token)
return token, nil
}
type token struct {
Token string `json:"token"`
AccessToken string `json:"access_token"` // the token returned by azure container registry is called "access_token"
ExpiresIn int `json:"expires_in"`
IssuedAt string `json:"issued_at"`
}
func (a *authorizer) fetchToken(scopes []*scope) (*token, error) {
url, err := url.Parse(a.realm)
if err != nil {
return nil, err
}
query := url.Query()
query.Add("service", a.service)
for _, scope := range scopes {
query.Add("scope", scope.String())
}
url.RawQuery = query.Encode()
req, err := http.NewRequest(http.MethodGet, url.String(), nil)
if err != nil {
return nil, err
}
if a.authorizer != nil {
if err = a.authorizer.Modify(req); err != nil {
return nil, err
}
}
resp, err := a.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
message := fmt.Sprintf("http status code: %d, body: %s", resp.StatusCode, string(body))
code := ierror.GeneralCode
switch resp.StatusCode {
case http.StatusUnauthorized:
code = ierror.UnAuthorizedCode
case http.StatusForbidden:
code = ierror.ForbiddenCode
}
return nil, ierror.New(nil).WithCode(code).
WithMessage(message)
}
token := &token{}
if err = json.Unmarshal(body, token); err != nil {
return nil, err
}
return token, nil
}

55
src/pkg/registry/auth/bearer/authorizer_test.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"fmt"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"net/http/httptest"
"testing"
)
func TestModify(t *testing.T) {
token := "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IlBZWU86VEVXVTpWN0pIOjI2SlY6QVFUWjpMSkMzOlNYVko6WEdIQTozNEYyOjJMQVE6WlJNSzpaN1E2In0.eyJpc3MiOiJhdXRoLmRvY2tlci5jb20iLCJzdWIiOiJqbGhhd24iLCJhdWQiOiJyZWdpc3RyeS5kb2NrZXIuY29tIiwiZXhwIjoxNDE1Mzg3MzE1LCJuYmYiOjE0MTUzODcwMTUsImlhdCI6MTQxNTM4NzAxNSwianRpIjoidFlKQ08xYzZjbnl5N2tBbjBjN3JLUGdiVjFIMWJGd3MiLCJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InNhbWFsYmEvbXktYXBwIiwiYWN0aW9ucyI6WyJwdXNoIl19XX0.QhflHPfbd6eVF4lM9bwYpFZIV0PfikbyXuLx959ykRTBpe3CYnzs6YBK8FToVb5R47920PVLrh8zuLzdCr9t3w"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
u, p, ok := r.BasicAuth()
if !ok || u != "username" || p != "password" {
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Write([]byte(fmt.Sprintf(`{"token": "%s", "expires_in": 3600,"issued_at": "2009-11-10T23:00:00Z"}`, token)))
}))
defer server.Close()
// invalid credential
a := basic.NewAuthorizer("username", "invalid_password")
authorizer := NewAuthorizer(server.URL, "service", a)
req, _ := http.NewRequest(http.MethodGet, server.URL, nil)
err := authorizer.Modify(req)
require.NotNil(t, err)
assert.True(t, ierror.IsErr(err, ierror.UnAuthorizedCode))
// valid credential
a = basic.NewAuthorizer("username", "password")
authorizer = NewAuthorizer(server.URL, "service", a)
req, _ = http.NewRequest(http.MethodGet, server.URL, nil)
err = authorizer.Modify(req)
require.Nil(t, err)
assert.Equal(t, fmt.Sprintf("Bearer %s", token), req.Header.Get("Authorization"))
}

93
src/pkg/registry/auth/bearer/cache.go Normal file
View File

@ -0,0 +1,93 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"github.com/goharbor/harbor/src/common/utils/log"
"strings"
"sync"
"time"
)
func newCache(capacity int) *cache {
return &cache{
latency: 10,
capacity: capacity,
cache: map[string]*token{},
}
}
type cache struct {
sync.RWMutex
latency int // second, the network latency in case that when the token is checked it doesn't expire but it does when used
capacity int // the capacity of the cache map
cache map[string]*token
}
func (c *cache) get(scopes []*scope) *token {
c.RLock()
defer c.RUnlock()
return c.cache[c.key(scopes)]
}
func (c *cache) set(scopes []*scope, token *token) {
c.Lock()
defer c.Unlock()
// exceed the capacity, empty some elements: all expired token will be removed,
// if no expired token, move the earliest one
if len(c.cache) >= c.capacity {
now := time.Now().UTC()
var candidates []string
var earliestKey string
var earliestExpireTime time.Time
for key, value := range c.cache {
// parse error
issueAt, err := time.Parse(time.RFC3339, value.IssuedAt)
if err != nil {
log.Errorf("failed to parse the issued at time of token %s: %v", token.IssuedAt, err)
candidates = append(candidates, key)
continue
}
expireAt := issueAt.Add(time.Duration(value.ExpiresIn-c.latency) * time.Second)
// expired
if expireAt.Before(now) {
candidates = append(candidates, key)
continue
}
// doesn't expired
if len(earliestKey) == 0 || expireAt.Before(earliestExpireTime) {
earliestKey = key
earliestExpireTime = expireAt
continue
}
}
if len(candidates) == 0 {
candidates = append(candidates, earliestKey)
}
for _, candidate := range candidates {
delete(c.cache, candidate)
}
}
c.cache[c.key(scopes)] = token
}
func (c *cache) key(scopes []*scope) string {
var strs []string
for _, scope := range scopes {
strs = append(strs, scope.String())
}
return strings.Join(strs, "#")
}

153
src/pkg/registry/auth/bearer/cache_test.go Normal file
View File

@ -0,0 +1,153 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"github.com/stretchr/testify/suite"
"testing"
"time"
)
type cacheTestSuite struct {
suite.Suite
cache *cache
}
func (c *cacheTestSuite) SetupTest() {
c.cache = newCache(2)
}
func (c *cacheTestSuite) TestKey() {
// nil scopes
var scopes []*scope
key := c.cache.key(scopes)
c.Equal("", key)
// single one scope
scopes = []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world",
Actions: []string{scopeActionPull, scopeActionPush},
},
}
key = c.cache.key(scopes)
c.Equal("repository:library/hello-world:pull,push", key)
// multiple scopes
scopes = []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world",
Actions: []string{scopeActionPull, scopeActionPush},
},
{
Type: scopeTypeRepository,
Name: "library/alpine",
Actions: []string{scopeActionPull},
},
}
key = c.cache.key(scopes)
c.Equal("repository:library/hello-world:pull,push#repository:library/alpine:pull", key)
}
func (c *cacheTestSuite) TestGet() {
token := &token{
Token: "token",
}
c.cache.set(nil, token)
tk := c.cache.get(nil)
c.Require().NotNil(tk)
c.Equal(token.Token, tk.Token)
}
func (c *cacheTestSuite) TestSet() {
now := time.Now()
// set the first token
scope1 := []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world01",
Actions: []string{scopeActionPull},
},
}
token1 := &token{
Token: "token1",
ExpiresIn: 10,
IssuedAt: now.Format(time.RFC3339),
}
c.cache.set(scope1, token1)
c.Len(c.cache.cache, 1)
// set the second token
scope2 := []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world02",
Actions: []string{scopeActionPull},
},
}
token2 := &token{
Token: "token2",
ExpiresIn: 15,
IssuedAt: now.Format(time.RFC3339),
}
c.cache.set(scope2, token2)
c.Len(c.cache.cache, 2)
// set the third token
// as the capacity is 2 and token1 is expired, token1 should be replaced by token3
scope3 := []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world03",
Actions: []string{scopeActionPull},
},
}
token3 := &token{
Token: "token3",
ExpiresIn: 15,
IssuedAt: now.Format(time.RFC3339),
}
c.cache.set(scope3, token3)
c.Require().Len(c.cache.cache, 2)
c.Require().NotNil(c.cache.get(scope2))
c.Require().NotNil(c.cache.get(scope3))
// sleep 5 seconds to make sure all tokens expire
time.Sleep(5 * time.Second)
// set the fourth token
// as the capacity is 2 and both token2 and token3 are expired, token2 and token3 should be removed
scope4 := []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world04",
Actions: []string{scopeActionPull},
},
}
token4 := &token{
Token: "token4",
ExpiresIn: 20,
IssuedAt: now.Format(time.RFC3339),
}
c.cache.set(scope4, token4)
c.Require().Len(c.cache.cache, 1)
c.Require().NotNil(c.cache.get(scope4))
}
func TestCacheTestSuite(t *testing.T) {
suite.Run(t, &cacheTestSuite{})
}

107
src/pkg/registry/auth/bearer/scope.go Normal file
View File

@ -0,0 +1,107 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"fmt"
"github.com/docker/distribution/reference"
"net/http"
"regexp"
"strings"
)
const (
scopeTypeRegistry = "registry"
scopeTypeRepository = "repository"
scopeActionPull = "pull"
scopeActionPush = "push"
scopeActionAll = "*"
)
var (
catalog = regexp.MustCompile("/v2/_catalog$")
tag = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/tags/list")
manifest = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/manifests/(" + reference.TagRegexp.String() + "|" + reference.DigestRegexp.String() + ")")
blob = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String())
blobUpload = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/uploads")
)
type scope struct {
Type string
Name string
Actions []string
}
func (s *scope) String() string {
return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
}
func parseScopes(req *http.Request) []*scope {
path := strings.TrimRight(req.URL.Path, "/")
var scopes []*scope
repository := ""
// manifest
if subs := manifest.FindStringSubmatch(path); len(subs) >= 2 {
// manifest
repository = subs[1]
} else if subs := blob.FindStringSubmatch(path); len(subs) >= 2 {
// blob
repository = subs[1]
} else if subs := blobUpload.FindStringSubmatch(path); len(subs) >= 2 {
// blob upload
repository = subs[1]
// blob mount
from := req.URL.Query().Get("from")
if len(from) > 0 {
scopes = append(scopes, &scope{
Type: scopeTypeRepository,
Name: from,
Actions: []string{scopeActionPull},
})
}
} else if subs := tag.FindStringSubmatch(path); len(subs) >= 2 {
// tag
repository = subs[1]
}
if len(repository) > 0 {
scp := &scope{
Type: scopeTypeRepository,
Name: repository,
}
switch req.Method {
case http.MethodGet, http.MethodHead:
scp.Actions = []string{scopeActionPull}
case http.MethodPost, http.MethodPut, http.MethodPatch:
scp.Actions = []string{scopeActionPull, scopeActionPush}
case http.MethodDelete:
scp.Actions = []string{scopeActionAll}
}
scopes = append(scopes, scp)
return scopes
}
// catalog
if catalog.MatchString(path) {
return []*scope{
{
Type: scopeTypeRegistry,
Name: "catalog",
Actions: []string{scopeActionAll},
}}
}
// base or no match, return nil
return nil
}

112
src/pkg/registry/auth/bearer/scope_test.go Normal file
View File

@ -0,0 +1,112 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"testing"
)
func TestStringOfScope(t *testing.T) {
scope := &scope{
Type: scopeTypeRepository,
Name: "library/hello-world",
Actions: []string{scopeActionPull, scopeActionPush},
}
assert.Equal(t, "repository:library/hello-world:pull,push", scope.String())
}
func TestParseScopes(t *testing.T) {
// base
req, _ := http.NewRequest(http.MethodGet, "/v2/", nil)
scopes := parseScopes(req)
require.Nil(t, scopes)
// catalog
req, _ = http.NewRequest(http.MethodGet, "/v2/_catalog", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRegistry, scopes[0].Type)
assert.Equal(t, "catalog", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionAll, scopes[0].Actions[0])
// list tags
req, _ = http.NewRequest(http.MethodGet, "/v2/library/hello-world/tags/list", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionPull, scopes[0].Actions[0])
// get manifest by tag
req, _ = http.NewRequest(http.MethodGet, "/v2/library/hello-world/manifests/latest", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionPull, scopes[0].Actions[0])
// get manifest by digest
req, _ = http.NewRequest(http.MethodGet, "/v2/library/hello-world/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionPull, scopes[0].Actions[0])
// push manifest
req, _ = http.NewRequest(http.MethodPut, "/v2/library/hello-world/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 2)
assert.Equal(t, scopeActionPull, scopes[0].Actions[0])
assert.Equal(t, scopeActionPush, scopes[0].Actions[1])
// delete manifest
req, _ = http.NewRequest(http.MethodDelete, "/v2/library/hello-world/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionAll, scopes[0].Actions[0])
// mount blob
req, _ = http.NewRequest(http.MethodPost, "/v2/library/hello-world/blobs/uploads/?mount=sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae&from=library/alpine", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 2)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/alpine", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionPull, scopes[1].Actions[0])
assert.Equal(t, scopeTypeRepository, scopes[1].Type)
assert.Equal(t, "library/hello-world", scopes[1].Name)
require.Len(t, scopes[1].Actions, 2)
assert.Equal(t, scopeActionPull, scopes[1].Actions[0])
assert.Equal(t, scopeActionPush, scopes[1].Actions[1])
// no match
req, _ = http.NewRequest(http.MethodPost, "/api/others", nil)
scopes = parseScopes(req)
require.Nil(t, scopes)
}

19
src/common/utils/registry/auth/challenge.go → src/pkg/registry/auth/null/authorizer.go
View File

@ -12,17 +12,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
package null
import (
"github.com/goharbor/harbor/src/internal"
"net/http"
"github.com/docker/distribution/registry/client/auth/challenge"
)
// ParseChallengeFromResponse ...
func ParseChallengeFromResponse(resp *http.Response) []challenge.Challenge {
challenges := challenge.ResponseChallenges(resp)
// NewAuthorizer returns a null authorizer
func NewAuthorizer() internal.Authorizer {
return &authorizer{}
return challenges
}
type authorizer struct{}
func (a *authorizer) Modify(req *http.Request) error {
// do nothing
return nil
}

497
src/pkg/registry/client.go
View File

@ -15,26 +15,35 @@
package registry
import (
"bytes"
"encoding/json"
"fmt"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/manifestlist"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
// register oci manifest unmarshal function
_ "github.com/docker/distribution/manifest/ocischema"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/internal"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/replication/util"
"github.com/goharbor/harbor/src/pkg/registry/auth"
"github.com/opencontainers/go-digest"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"net/http"
)
// TODO we'll merge all registry related code into this package before releasing 2.0
var (
// Cli is the global registry client instance, it targets to the backend docker registry
Cli = func() Client {
url, _ := config.RegistryURL()
username, password := config.RegistryCredential()
return NewClient(url, true, username, password)
return NewClient(url, username, password, true)
}()
accepts = []string{
@ -48,54 +57,378 @@ var (
// Client defines the methods that a registry client should implements
type Client interface {
// Ping the base API endpoint "/v2/"
Ping() (err error)
// Catalog the repositories
Catalog() (repositories []string, err error)
// ListTags lists the tags under the specified repository
ListTags(repository string) (tags []string, err error)
// ManifestExist checks the existence of the manifest
ManifestExist(repository, reference string) (exist bool, digest string, err error)
// PullManifest pulls the specified manifest
PullManifest(repository, reference string, acceptedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error)
// PushManifest pushes the specified manifest
PushManifest(repository, reference, mediaType string, payload []byte) (digest string, err error)
// DeleteManifest deletes the specified manifest. The "reference" can be "tag" or "digest"
DeleteManifest(repository, reference string) (err error)
// BlobExist checks the existence of the specified blob
BlobExist(repository, digest string) (exist bool, err error)
// PullBlob pulls the specified blob. The caller must close the returned "blob"
PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error)
// PushBlob pushes the specified blob
PushBlob(repository, digest string, size int64, blob io.Reader) error
// MountBlob mounts the blob from the source repository
MountBlob(srcRepository, digest, dstRepository string) (err error)
// DeleteBlob deletes the specified blob
DeleteBlob(repository, digest string) (err error)
// Copy the artifact from source repository to the destination. The "override"
// is used to specify whether the destination artifact will be overridden if
// its name is same with source but digest isn't
Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) (err error)
// TODO defines other methods
Copy(srcRepository, srcReference, dstRepository, dstReference string, override bool) (err error)
}
// NewClient creates a registry client based on the provided information
// TODO support HTTPS
func NewClient(url string, insecure bool, username, password string) Client {
transport := util.GetHTTPTransport(insecure)
authorizer := auth.NewAuthorizer(auth.NewBasicAuthCredential(username, password),
&http.Client{
Transport: transport,
})
// TODO TODO support HTTPS
// NewClient creates a registry client with the default authorizer which determines the auth scheme
// of the registry automatically and calls the corresponding underlying authorizers(basic/bearer) to
// do the auth work. If a customized authorizer is needed, use "NewClientWithAuthorizer" instead
func NewClient(url, username, password string, insecure bool) Client {
return &client{
url: url,
url: url,
authorizer: auth.NewAuthorizer(username, password, insecure),
client: &http.Client{
Transport: registry.NewTransport(transport, authorizer),
Transport: internal.GetHTTPTransport(insecure),
},
}
}
// NewClientWithAuthorizer creates a registry client with the provided authorizer
func NewClientWithAuthorizer(url string, authorizer internal.Authorizer, insecure bool) Client {
return &client{
url: url,
authorizer: authorizer,
client: &http.Client{
Transport: internal.GetHTTPTransport(insecure),
},
}
}
type client struct {
url string
client *http.Client
url string
authorizer internal.Authorizer
client *http.Client
}
func (c *client) Ping() error {
req, err := http.NewRequest(http.MethodGet, buildPingURL(c.url), nil)
if err != nil {
return err
}
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *client) Catalog() ([]string, error) {
var repositories []string
url := buildCatalogURL(c.url)
for {
repos, next, err := c.catalog(url)
if err != nil {
return nil, err
}
repositories = append(repositories, repos...)
url = next
// no next page, end the loop
if len(url) == 0 {
break
}
// relative URL
if !strings.Contains(url, "://") {
url = c.url + url
}
}
return repositories, nil
}
func (c *client) catalog(url string) ([]string, string, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, "", err
}
resp, err := c.do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, "", err
}
repositories := struct {
Repositories []string `json:"repositories"`
}{}
if err := json.Unmarshal(body, &repositories); err != nil {
return nil, "", err
}
return repositories.Repositories, next(resp.Header.Get("Link")), nil
}
func (c *client) ListTags(repository string) ([]string, error) {
var tags []string
url := buildTagListURL(c.url, repository)
for {
tgs, next, err := c.listTags(url)
if err != nil {
return nil, err
}
tags = append(tags, tgs...)
url = next
// no next page, end the loop
if len(url) == 0 {
break
}
// relative URL
if !strings.Contains(url, "://") {
url = c.url + url
}
}
return tags, nil
}
func (c *client) listTags(url string) ([]string, string, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, "", err
}
resp, err := c.do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, "", err
}
tgs := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(body, &tgs); err != nil {
return nil, "", err
}
return tgs.Tags, next(resp.Header.Get("Link")), nil
}
func (c *client) ManifestExist(repository, reference string) (bool, string, error) {
req, err := http.NewRequest(http.MethodHead, buildManifestURL(c.url, repository, reference), nil)
if err != nil {
return false, "", err
}
for _, mediaType := range accepts {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := c.do(req)
if err != nil {
if ierror.IsErr(err, ierror.NotFoundCode) {
return false, "", nil
}
return false, "", err
}
defer resp.Body.Close()
return true, resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")), nil
}
func (c *client) PullManifest(repository, reference string, acceptedMediaTypes ...string) (
distribution.Manifest, string, error) {
req, err := http.NewRequest(http.MethodGet, buildManifestURL(c.url, repository, reference), nil)
if err != nil {
return nil, "", err
}
if len(acceptedMediaTypes) == 0 {
acceptedMediaTypes = accepts
}
for _, mediaType := range acceptedMediaTypes {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := c.do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
payload, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, "", err
}
mediaType := resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
manifest, _, err := distribution.UnmarshalManifest(mediaType, payload)
if err != nil {
return nil, "", err
}
digest := resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return manifest, digest, nil
}
func (c *client) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
req, err := http.NewRequest(http.MethodPut, buildManifestURL(c.url, repository, reference),
bytes.NewReader(payload))
if err != nil {
return "", err
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType)
resp, err := c.do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
return resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")), nil
}
func (c *client) DeleteManifest(repository, reference string) error {
_, err := digest.Parse(reference)
if err != nil {
// the reference is tag, get the digest first
exist, digest, err := c.ManifestExist(repository, reference)
if err != nil {
return err
}
if !exist {
return ierror.New(nil).WithCode(ierror.NotFoundCode).
WithMessage("%s:%s not found", repository, reference)
}
reference = digest
}
req, err := http.NewRequest(http.MethodDelete, buildManifestURL(c.url, repository, reference), nil)
if err != nil {
return err
}
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *client) BlobExist(repository, digest string) (bool, error) {
req, err := http.NewRequest(http.MethodHead, buildBlobURL(c.url, repository, digest), nil)
if err != nil {
return false, err
}
resp, err := c.do(req)
if err != nil {
if ierror.IsErr(err, ierror.NotFoundCode) {
return false, nil
}
return false, err
}
defer resp.Body.Close()
return true, nil
}
func (c *client) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
req, err := http.NewRequest(http.MethodGet, buildBlobURL(c.url, repository, digest), nil)
if err != nil {
return 0, nil, err
}
resp, err := c.do(req)
if err != nil {
return 0, nil, err
}
n := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
size, err := strconv.ParseInt(n, 10, 64)
if err != nil {
defer resp.Body.Close()
return 0, nil, err
}
return size, resp.Body, nil
}
func (c *client) PushBlob(repository, digest string, size int64, blob io.Reader) error {
location, _, err := c.initiateBlobUpload(repository)
if err != nil {
return err
}
return c.monolithicBlobUpload(location, digest, size, blob)
}
func (c *client) initiateBlobUpload(repository string) (string, string, error) {
req, err := http.NewRequest(http.MethodPost, buildInitiateBlobUploadURL(c.url, repository), nil)
if err != nil {
return "", "", err
}
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := c.do(req)
if err != nil {
return "", "", err
}
defer resp.Body.Close()
return resp.Header.Get(http.CanonicalHeaderKey("Location")),
resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID")), nil
}
func (c *client) monolithicBlobUpload(location, digest string, size int64, data io.Reader) error {
url, err := buildMonolithicBlobUploadURL(c.url, location, digest)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPut, url, data)
if err != nil {
return err
}
req.ContentLength = size
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *client) MountBlob(srcRepository, digest, dstRepository string) error {
req, err := http.NewRequest(http.MethodPost, buildMountBlobURL(c.url, dstRepository, digest, srcRepository), nil)
if err != nil {
return err
}
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *client) DeleteBlob(repository, digest string) error {
req, err := http.NewRequest(http.MethodDelete, buildBlobURL(c.url, repository, digest), nil)
if err != nil {
return err
}
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
// TODO extend this method to support copy artifacts between different registries when merging codes
// TODO this can be used in replication to replace the existing implementation
// TODO add unit test case
func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
src, err := registry.NewRepository(srcRepo, c.url, c.client)
if err != nil {
return err
}
dst, err := registry.NewRepository(dstRepo, c.url, c.client)
if err != nil {
return err
}
// pull the manifest from the source repository
srcDgt, mediaType, payload, err := src.PullManifest(srcRef, accepts)
manifest, srcDgt, err := c.PullManifest(srcRepo, srcRef)
if err != nil {
return err
}
// check the existence of the artifact on the destination repository
dstDgt, exist, err := dst.ManifestExist(dstRef)
exist, dstDgt, err := c.ManifestExist(dstRepo, dstRef)
if err != nil {
return err
}
@ -111,10 +444,6 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
}
}
manifest, _, err := registry.UnMarshal(mediaType, payload)
if err != nil {
return err
}
for _, descriptor := range manifest.References() {
digest := descriptor.Digest.String()
switch descriptor.MediaType {
@ -130,7 +459,7 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
}
// common layer
default:
exist, err := dst.BlobExist(digest)
exist, err := c.BlobExist(dstRepo, digest)
if err != nil {
return err
}
@ -139,7 +468,7 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
continue
}
// when the copy happens inside the same registry, use mount
if err = dst.MountBlob(digest, srcRepo); err != nil {
if err = c.MountBlob(srcRepo, digest, dstRepo); err != nil {
return err
}
/*
@ -156,10 +485,100 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
}
}
mediaType, payload, err := manifest.Payload()
if err != nil {
return err
}
// push manifest to the destination repository
if _, err = dst.PushManifest(dstRef, mediaType, payload); err != nil {
if _, err = c.PushManifest(dstRepo, dstRef, mediaType, payload); err != nil {
return err
}
return nil
}
func (c *client) do(req *http.Request) (*http.Response, error) {
if c.authorizer != nil {
if err := c.authorizer.Modify(req); err != nil {
return nil, err
}
}
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
message := fmt.Sprintf("http status code: %d, body: %s", resp.StatusCode, string(body))
code := ierror.GeneralCode
switch resp.StatusCode {
case http.StatusUnauthorized:
code = ierror.UnAuthorizedCode
case http.StatusForbidden:
code = ierror.ForbiddenCode
case http.StatusNotFound:
code = ierror.NotFoundCode
}
return nil, ierror.New(nil).WithCode(code).
WithMessage(message)
}
return resp, nil
}
// parse the next page link from the link header
func next(link string) string {
links := internal.ParseLinks(link)
for _, lk := range links {
if lk.Rel == "next" {
return lk.URL
}
}
return ""
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}
func buildCatalogURL(endpoint string) string {
return fmt.Sprintf("%s/v2/_catalog?n=1000", endpoint)
}
func buildTagListURL(endpoint, repository string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repository)
}
func buildManifestURL(endpoint, repository, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repository, reference)
}
func buildBlobURL(endpoint, repository, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, reference)
}
func buildMountBlobURL(endpoint, repository, digest, from string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/?mount=%s&from=%s", endpoint, repository, digest, from)
}
func buildInitiateBlobUploadURL(endpoint, repository string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repository)
}
func buildMonolithicBlobUploadURL(endpoint, location, digest string) (string, error) {
url, err := url.Parse(location)
if err != nil {
return "", err
}
q := url.Query()
q.Set("digest", digest)
url.RawQuery = q.Encode()
if url.IsAbs() {
return url.String(), nil
}
// the "relativeurls" is enabled in registry
return endpoint + url.String(), nil
}

388
src/pkg/registry/client_test.go Normal file
View File

@ -0,0 +1,388 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"encoding/json"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/internal"
"github.com/stretchr/testify/suite"
"io/ioutil"
"net/http"
"strconv"
"testing"
)
type clientTestSuite struct {
suite.Suite
client Client
}
func (c *clientTestSuite) TestPing() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/",
Handler: test.Handler(nil),
})
defer server.Close()
err := NewClient(server.URL, "", "", true).Ping()
c.Require().Nil(err)
}
func (c *clientTestSuite) TestCatalog() {
type repositories struct {
Repositories []string `json:"repositories"`
}
isFirstRequest := true
handler := func(w http.ResponseWriter, r *http.Request) {
if isFirstRequest {
isFirstRequest = false
repos := &repositories{
Repositories: []string{"library/alpine"},
}
link := internal.Link{
URL: `/v2/_catalog?last=library/alpine`,
Rel: "next",
}
w.Header().Set(http.CanonicalHeaderKey("link"), link.String())
encoder := json.NewEncoder(w)
err := encoder.Encode(repos)
c.Require().Nil(err)
return
}
if r.URL.String() != "/v2/_catalog?last=library/alpine" {
w.WriteHeader(http.StatusBadRequest)
return
}
repos := &repositories{
Repositories: []string{"library/hello-world"},
}
encoder := json.NewEncoder(w)
err := encoder.Encode(repos)
c.Require().Nil(err)
return
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/_catalog",
Handler: handler,
})
defer server.Close()
repos, err := NewClient(server.URL, "", "", true).Catalog()
c.Require().Nil(err)
c.Len(repos, 2)
c.EqualValues([]string{"library/alpine", "library/hello-world"}, repos)
}
func (c *clientTestSuite) TestListTags() {
type tags struct {
Tags []string `json:"tags"`
}
isFirstRequest := true
handler := func(w http.ResponseWriter, r *http.Request) {
if isFirstRequest {
isFirstRequest = false
tgs := &tags{
Tags: []string{"1.0"},
}
link := internal.Link{
URL: `/v2/library/hello-world/tags/list?last=1.0`,
Rel: "next",
}
w.Header().Set(http.CanonicalHeaderKey("link"), link.String())
encoder := json.NewEncoder(w)
err := encoder.Encode(tgs)
c.Require().Nil(err)
return
}
if r.URL.String() != "/v2/library/hello-world/tags/list?last=1.0" {
w.WriteHeader(http.StatusBadRequest)
return
}
tgs := &tags{
Tags: []string{"2.0"},
}
encoder := json.NewEncoder(w)
err := encoder.Encode(tgs)
c.Require().Nil(err)
return
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/library/hello-world/tags/list",
Handler: handler,
})
defer server.Close()
repos, err := NewClient(server.URL, "", "", true).ListTags("library/hello-world")
c.Require().Nil(err)
c.Len(repos, 2)
c.EqualValues([]string{"1.0", "2.0"}, repos)
}
func (c *clientTestSuite) TestManifestExist() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/alpine/manifests/latest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusNotFound,
}),
},
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/hello-world/manifests/latest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusOK,
Headers: map[string]string{
"Docker-Content-Digest": "digest",
},
}),
},
)
defer server.Close()
client := NewClient(server.URL, "", "", true)
// doesn't exist
exist, digest, err := client.ManifestExist("library/alpine", "latest")
c.Require().Nil(err)
c.False(exist)
// exist
exist, digest, err = client.ManifestExist("library/hello-world", "latest")
c.Require().Nil(err)
c.True(exist)
c.Equal("digest", digest)
}
func (c *clientTestSuite) TestPullManifest() {
data := `{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": 1510,
"digest": "sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e"
},
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 977,
"digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
}
]
}`
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/library/hello-world/manifests/latest",
Handler: test.Handler(&test.Response{
Headers: map[string]string{
"Docker-Content-Digest": "digest",
"Content-Type": schema2.MediaTypeManifest,
},
Body: []byte(data),
}),
})
defer server.Close()
manifest, digest, err := NewClient(server.URL, "", "", true).PullManifest("library/hello-world", "latest")
c.Require().Nil(err)
c.Equal("digest", digest)
mediaType, payload, err := manifest.Payload()
c.Require().Nil(err)
c.Equal(schema2.MediaTypeManifest, mediaType)
c.Equal(data, string(payload))
}
func (c *clientTestSuite) TestPushManifest() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "PUT",
Pattern: "/v2/library/hello-world/manifests/latest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusCreated,
Headers: map[string]string{
"Docker-Content-Digest": "digest",
},
}),
})
defer server.Close()
digest, err := NewClient(server.URL, "", "", true).PushManifest("library/hello-world", "latest", "", nil)
c.Require().Nil(err)
c.Equal("digest", digest)
}
func (c *clientTestSuite) TestDeleteManifest() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/hello-world/manifests/latest",
Handler: test.Handler(&test.Response{
Headers: map[string]string{
"Docker-Content-Digest": "digest",
},
}),
},
&test.RequestHandlerMapping{
Method: "DELETE",
Pattern: "/v2/library/hello-world/manifests/digest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
}),
})
defer server.Close()
err := NewClient(server.URL, "", "", true).DeleteManifest("library/hello-world", "latest")
c.Require().Nil(err)
}
func (c *clientTestSuite) TestBlobExist() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/hello-world/blobs/digest1",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusNotFound,
}),
},
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/hello-world/blobs/digest2",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusOK,
}),
},
)
defer server.Close()
// doesn't exist
client := NewClient(server.URL, "", "", true)
exist, err := client.BlobExist("library/hello-world", "digest1")
c.Require().Nil(err)
c.False(exist)
// exist
exist, err = client.BlobExist("library/hello-world", "digest2")
c.Require().Nil(err)
c.True(exist)
}
func (c *clientTestSuite) TestPullBlob() {
data := []byte{'a'}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/library/hello-world/blobs/digest",
Handler: test.Handler(&test.Response{
Headers: map[string]string{
"Content-Length": strconv.Itoa(len(data)),
},
Body: data,
}),
})
defer server.Close()
size, blob, err := NewClient(server.URL, "", "", true).PullBlob("library/hello-world", "digest")
c.Require().Nil(err)
c.Equal(int64(len(data)), size)
b, err := ioutil.ReadAll(blob)
c.Require().Nil(err)
c.EqualValues(data, b)
}
func (c *clientTestSuite) TestPushBlob() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "POST",
Pattern: "/v2/library/hello-world/blobs/uploads/",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
Headers: map[string]string{
"Location": "/v2/library/hello-world/blobs/uploads/uuid",
},
}),
},
&test.RequestHandlerMapping{
Method: "PUT",
Pattern: "/v2/library/hello-world/blobs/uploads/uuid",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusCreated,
}),
})
defer server.Close()
err := NewClient(server.URL, "", "", true).PushBlob("library/hello-world", "digest", 0, nil)
c.Require().Nil(err)
}
func (c *clientTestSuite) TestDeleteBlob() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "DELETE",
Pattern: "/v2/library/hello-world/blobs/digest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
}),
})
defer server.Close()
err := NewClient(server.URL, "", "", true).DeleteBlob("library/hello-world", "digest")
c.Require().Nil(err)
}
func (c *clientTestSuite) TestMountBlob() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "POST",
Pattern: "/v2/library/hello-world/blobs/uploads/",
Handler: func(w http.ResponseWriter, r *http.Request) {
mount := r.URL.Query().Get("mount")
from := r.URL.Query().Get("from")
if mount != "digest" || from != "library/alpine" {
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusAccepted)
},
})
defer server.Close()
err := NewClient(server.URL, "", "", true).MountBlob("library/alpine", "digest", "library/hello-world")
c.Require().Nil(err)
}
func TestClientTestSuite(t *testing.T) {
suite.Run(t, &clientTestSuite{})
}

4
src/pkg/signature/notary/helper.go
View File

@ -17,6 +17,7 @@ package notary
import (
"encoding/hex"
"fmt"
"github.com/goharbor/harbor/src/internal"
model2 "github.com/goharbor/harbor/src/pkg/signature/notary/model"
"net/http"
"os"
@ -25,7 +26,6 @@ import (
"github.com/docker/distribution/registry/auth/token"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/core/config"
tokenutil "github.com/goharbor/harbor/src/core/service/token"
"github.com/theupdateframework/notary"
@ -82,7 +82,7 @@ func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]model2
authorizer := &notaryAuthorizer{
token: t.Token,
}
tr := registry.NewTransport(registry.GetHTTPTransport(), authorizer)
tr := NewTransport(internal.GetHTTPTransport(), authorizer)
gun := data.GUN(fqRepo)
notaryRepo, err := client.NewFileCachedRepository(notaryCachePath, gun, notaryEndpoint, tr, mockRetriever, trustPin)
if err != nil {

2
src/common/utils/registry/transport.go → src/pkg/signature/notary/transport.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
package notary
import (
"net/http"

2
src/common/utils/registry/transport_test.go → src/pkg/signature/notary/transport_test.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
package notary
import (
"fmt"

4
src/replication/adapter/adapter.go
View File

@ -54,8 +54,8 @@ type Adapter interface {
type ImageRegistry interface {
FetchImages(filters []*model.Filter) ([]*model.Resource, error)
ManifestExist(repository, reference string) (exist bool, digest string, err error)
PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error)
PushManifest(repository, reference, mediaType string, payload []byte) error
PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error)
PushManifest(repository, reference, mediaType string, payload []byte) (string, error)
// the "reference" can be "tag" or "digest", the function needs to handle both
DeleteManifest(repository, reference string) error
BlobExist(repository, digest string) (exist bool, err error)

34
src/replication/adapter/aliacr/adapter.go
View File

@ -4,6 +4,9 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/goharbor/harbor/src/internal"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
"net/http"
"path/filepath"
"regexp"
@ -11,7 +14,6 @@ import (
"github.com/aliyun/alibaba-cloud-sdk-go/services/cr"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
@ -50,24 +52,38 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
}
// fix url (allow user input cr service url)
registry.URL = fmt.Sprintf(registryEndpointTpl, region)
credential := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret)
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, credential)
nativeRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
realm, service, err := ping(registry)
if err != nil {
return nil, err
}
credential := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret)
authorizer := bearer.NewAuthorizer(realm, service, credential, util.GetHTTPTransport(registry.Insecure))
return &adapter{
region: region,
registry: registry,
domain: fmt.Sprintf(endpointTpl, region),
Adapter: nativeRegistry,
Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
}, nil
}
func ping(registry *model.Registry) (string, string, error) {
client := &http.Client{
Transport: internal.GetHTTPTransport(registry.Insecure),
}
resp, err := client.Get(registry.URL + "/v2/")
if err != nil {
return "", "", err
}
defer resp.Body.Close()
challenges := challenge.ResponseChallenges(resp)
for _, challenge := range challenges {
if challenge.Scheme == "bearer" {
return challenge.Parameters["realm"], challenge.Parameters["service"], nil
}
}
return "", "", fmt.Errorf("bearer auth scheme isn't supported: %v", challenges)
}
type factory struct {
}

42
src/replication/adapter/aliacr/adapter_test.go
View File

@ -10,47 +10,11 @@ import (
"time"
"github.com/goharbor/harbor/src/common/utils/test"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
)
func TestAdapter_NewAdapter(t *testing.T) {
factory, err := adp.GetFactory("BadName")
assert.Nil(t, factory)
assert.NotNil(t, err)
factory, err = adp.GetFactory(model.RegistryTypeAliAcr)
assert.Nil(t, err)
assert.NotNil(t, factory)
// test case for URL is registry.
adapter, err := newAdapter(&model.Registry{
Type: model.RegistryTypeAliAcr,
Credential: &model.Credential{
AccessKey: "MockAccessKey",
AccessSecret: "MockAccessSecret",
},
URL: "https://registry.test-region.aliyuncs.com",
})
assert.Nil(t, err)
assert.NotNil(t, adapter)
// test case for URL is cr service.
adapter, err = newAdapter(&model.Registry{
Type: model.RegistryTypeAliAcr,
Credential: &model.Credential{
AccessKey: "MockAccessKey",
AccessSecret: "MockAccessSecret",
},
URL: "https://cr.test-region.aliyuncs.com",
})
assert.Nil(t, err)
assert.NotNil(t, adapter)
}
func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Server) {
server := test.NewServer(
&test.RequestHandlerMapping{
@ -96,12 +60,8 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
AccessSecret: "MockAccessSecret",
}
}
nativeRegistry, err := native.NewAdapter(registry)
if err != nil {
panic(err)
}
return &adapter{
Adapter: nativeRegistry,
Adapter: native.NewAdapter(registry),
region: "test-region",
domain: server.URL,
registry: registry,

14
src/replication/adapter/awsecr/adapter.go
View File

@ -16,6 +16,7 @@ package awsecr
import (
"errors"
"github.com/goharbor/harbor/src/internal"
"net/http"
"regexp"
@ -25,7 +26,6 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
@ -53,13 +53,9 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
return nil, err
}
authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure)
dockerRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{
registry: registry,
Adapter: dockerRegistry,
Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
region: region,
}, nil
}
@ -201,7 +197,7 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) {
log.Errorf("no credential to ping registry %s", a.registry.URL)
return model.Unhealthy, nil
}
if err := a.PingGet(); err != nil {
if err := a.Ping(); err != nil {
log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
return model.Unhealthy, nil
}
@ -248,7 +244,7 @@ func (a *adapter) createRepository(repository string) error {
Credentials: cred,
Region: &a.region,
HTTPClient: &http.Client{
Transport: registry.GetHTTPTransport(a.registry.Insecure),
Transport: internal.GetHTTPTransport(a.registry.Insecure),
},
}
if a.forceEndpoint != nil {
@ -290,7 +286,7 @@ func (a *adapter) DeleteManifest(repository, reference string) error {
Credentials: cred,
Region: &a.region,
HTTPClient: &http.Client{
Transport: registry.GetHTTPTransport(a.registry.Insecure),
Transport: internal.GetHTTPTransport(a.registry.Insecure),
},
}
if a.forceEndpoint != nil {

6
src/replication/adapter/awsecr/adapter_test.go
View File

@ -133,13 +133,9 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
AccessSecret: "ppp",
}
}
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
panic(err)
}
return &adapter{
registry: registry,
Adapter: dockerRegistryAdapter,
Adapter: native.NewAdapter(registry),
region: "test-region",
forceEndpoint: &server.URL,
}, server

4
src/replication/adapter/awsecr/auth.go
View File

@ -25,7 +25,7 @@ import (
awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/internal"
"net/http"
"net/url"
"strings"
@ -100,7 +100,7 @@ func (a *awsAuthCredential) getAuthorization() (string, string, string, *time.Ti
Credentials: cred,
Region: &a.region,
HTTPClient: &http.Client{
Transport: registry.GetHTTPTransport(a.insecure),
Transport: internal.GetHTTPTransport(a.insecure),
},
}
if a.forceEndpoint != nil {

6
src/replication/adapter/azurecr/adapter.go
View File

@ -16,12 +16,8 @@ func init() {
}
func newAdapter(registry *model.Registry) (adp.Adapter, error) {
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
return &adapter{
Adapter: dockerRegistryAdapter,
Adapter: native.NewAdapter(registry),
}, nil
}

15
src/replication/adapter/dockerhub/adapter.go
View File

@ -31,19 +31,14 @@ func newAdapter(registry *model.Registry) (adp.Adapter, error) {
return nil, err
}
dockerRegistryAdapter, err := native.NewAdapter(&model.Registry{
URL: registryURL,
Credential: registry.Credential,
Insecure: registry.Insecure,
})
if err != nil {
return nil, err
}
return &adapter{
client: client,
registry: registry,
Adapter: dockerRegistryAdapter,
Adapter: native.NewAdapter(&model.Registry{
URL: registryURL,
Credential: registry.Credential,
Insecure: registry.Insecure,
}),
}, nil
}

30
src/replication/adapter/gitlab/adapter.go
View File

@ -2,12 +2,10 @@ package gitlab
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"net/http"
"strings"
)
@ -24,7 +22,7 @@ type factory struct {
// Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
return newAdapter(r)
return newAdapter(r), nil
}
// AdapterPattern ...
@ -41,33 +39,13 @@ type adapter struct {
clientGitlabAPI *Client
}
func newAdapter(registry *model.Registry) (*adapter, error) {
var credential auth.Credential
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
credential = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, credential)
dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(&model.Registry{
Name: registry.Name,
URL: registry.URL,
Credential: registry.Credential,
Insecure: registry.Insecure,
}, authorizer)
if err != nil {
return nil, err
}
func newAdapter(registry *model.Registry) *adapter {
return &adapter{
registry: registry,
url: registry.URL,
clientGitlabAPI: NewClient(registry),
Adapter: dockerRegistryAdapter,
}, nil
Adapter: native.NewAdapter(registry),
}
}
func (a *adapter) Info() (info *model.RegistryInfo, err error) {

4
src/replication/adapter/gitlab/client.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"io"
@ -65,7 +65,7 @@ func ping(client *http.Client, endpoint string) (string, string, error) {
}
defer resp.Body.Close()
challenges := auth.ParseChallengeFromResponse(resp)
challenges := challenge.ResponseChallenges(resp)
for _, challenge := range challenges {
if scheme == challenge.Scheme {
realm := challenge.Parameters["realm"]

15
src/replication/adapter/googlegcr/adapter.go
View File

@ -29,16 +29,11 @@ func init() {
log.Infof("the factory for adapter %s registered", model.RegistryTypeGoogleGcr)
}
func newAdapter(registry *model.Registry) (*adapter, error) {
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
func newAdapter(registry *model.Registry) *adapter {
return &adapter{
registry: registry,
Adapter: dockerRegistryAdapter,
}, nil
Adapter: native.NewAdapter(registry),
}
}
type factory struct {
@ -46,7 +41,7 @@ type factory struct {
// Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
return newAdapter(r)
return newAdapter(r), nil
}
// AdapterPattern ...
@ -125,7 +120,7 @@ func (a adapter) HealthCheck() (model.HealthStatus, error) {
log.Errorf("no credential to ping registry %s", a.registry.URL)
return model.Unhealthy, nil
}
if err = a.PingGet(); err != nil {
if err = a.Ping(); err != nil {
log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
return model.Unhealthy, nil
}

5
src/replication/adapter/googlegcr/adapter_test.go
View File

@ -88,10 +88,7 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
factory, err := adp.GetFactory(model.RegistryTypeGoogleGcr)
assert.Nil(t, err)
assert.NotNil(t, factory)
a, err := newAdapter(registry)
assert.Nil(t, err)
return a, server
return newAdapter(registry), server
}
func TestAdapter_Info(t *testing.T) {

58
src/replication/adapter/harbor/adapter.go
View File

@ -17,19 +17,18 @@ package harbor
import (
"errors"
"fmt"
"net/http"
"strconv"
"strings"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"net/http"
"strconv"
"strings"
)
func init() {
@ -62,26 +61,25 @@ type adapter struct {
func newAdapter(registry *model.Registry) (*adapter, error) {
transport := util.GetHTTPTransport(registry.Insecure)
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
},
}
if registry.Credential != nil {
var authorizer modifier.Modifier
if registry.Credential.Type == model.CredentialTypeSecret {
authorizer = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
} else {
authorizer = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
modifiers = append(modifiers, authorizer)
// local Harbor instance
if registry.Credential != nil && registry.Credential.Type == model.CredentialTypeSecret {
authorizer := common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
return &adapter{
registry: registry,
url: registry.URL,
client: common_http.NewClient(
&http.Client{
Transport: transport,
}, authorizer),
Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
}, nil
}
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
var authorizers []modifier.Modifier
if registry.Credential != nil {
authorizers = append(authorizers, basic.NewAuthorizer(
registry.Credential.AccessKey,
registry.Credential.AccessSecret))
}
return &adapter{
registry: registry,
@ -89,8 +87,8 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
client: common_http.NewClient(
&http.Client{
Transport: transport,
}, modifiers...),
Adapter: dockerRegistryAdapter,
}, authorizers...),
Adapter: native.NewAdapter(registry),
}, nil
}
@ -119,7 +117,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
sys := &struct {
ChartRegistryEnabled bool `json:"with_chartmuseum"`
}{}
if err := a.client.Get(a.getURL()+"/api/systeminfo", sys); err != nil {
if err := a.client.Get(a.getURL()+"/api/v2.0/systeminfo", sys); err != nil {
return nil, err
}
if sys.ChartRegistryEnabled {
@ -129,7 +127,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
Name string `json:"name"`
}{}
// label isn't supported in some previous version of Harbor
if err := a.client.Get(a.getURL()+"/api/labels?scope=g", &labels); err != nil {
if err := a.client.Get(a.getURL()+"/api/v2.0/labels?scope=g", &labels); err != nil {
if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound {
return nil, err
}
@ -185,7 +183,7 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error {
Name: project.Name,
Metadata: project.Metadata,
}
err := a.client.Post(a.getURL()+"/api/projects", pro)
err := a.client.Post(a.getURL()+"/api/v2.0/projects", pro)
if err != nil {
if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
log.Debugf("got 409 when trying to create project %s", project.Name)
@ -251,7 +249,7 @@ type project struct {
func (a *adapter) getProjects(name string) ([]*project, error) {
projects := []*project{}
url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=500", a.getURL(), name)
url := fmt.Sprintf("%s/api/v2.0/projects?name=%s&page=1&page_size=500", a.getURL(), name)
if err := a.client.GetAndIteratePagination(url, &projects); err != nil {
return nil, err
}
@ -286,7 +284,7 @@ func (a *adapter) getProject(name string) (*project, error) {
func (a *adapter) getRepositories(projectID int64) ([]*adp.Repository, error) {
repositories := []*adp.Repository{}
url := fmt.Sprintf("%s/api/repositories?project_id=%d&page=1&page_size=500", a.getURL(), projectID)
url := fmt.Sprintf("%s/api/v2.0/repositories?project_id=%d&page=1&page_size=500", a.getURL(), projectID)
if err := a.client.GetAndIteratePagination(url, &repositories); err != nil {
return nil, err
}

8
src/replication/adapter/harbor/adapter_test.go
View File

@ -29,7 +29,7 @@ func TestInfo(t *testing.T) {
// chart museum enabled
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/api/systeminfo",
Pattern: "/api/v2.0/systeminfo",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `{"with_chartmuseum":true}`
w.Write([]byte(data))
@ -53,7 +53,7 @@ func TestInfo(t *testing.T) {
// chart museum disabled
server = test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/api/systeminfo",
Pattern: "/api/v2.0/systeminfo",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `{"with_chartmuseum":false}`
w.Write([]byte(data))
@ -77,7 +77,7 @@ func TestInfo(t *testing.T) {
func TestPrepareForPush(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodPost,
Pattern: "/api/projects",
Pattern: "/api/v2.0/projects",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
},
@ -131,7 +131,7 @@ func TestPrepareForPush(t *testing.T) {
// project already exists
server = test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodPost,
Pattern: "/api/projects",
Pattern: "/api/v2.0/projects",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusConflict)
},

10
src/replication/adapter/harbor/chart_registry.go
View File

@ -52,7 +52,7 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
}
resources := []*model.Resource{}
for _, project := range projects {
url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.getURL(), project.Name)
url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts", a.getURL(), project.Name)
repositories := []*adp.Repository{}
if err := a.client.Get(url, &repositories); err != nil {
return nil, err
@ -71,7 +71,7 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
}
for _, repository := range repositories {
name := strings.SplitN(repository.Name, "/", 2)[1]
url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.getURL(), project.Name, name)
url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts/%s", a.getURL(), project.Name, name)
versions := []*chartVersion{}
if err := a.client.Get(url, &versions); err != nil {
return nil, err
@ -131,7 +131,7 @@ func (a *adapter) getChartInfo(name, version string) (*chartVersionDetail, error
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s/%s", a.url, project, name, version)
url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts/%s/%s", a.url, project, name, version)
info := &chartVersionDetail{}
if err = a.client.Get(url, info); err != nil {
return nil, err
@ -191,7 +191,7 @@ func (a *adapter) UploadChart(name, version string, chart io.Reader) error {
}
w.Close()
url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.url, project)
url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts", a.url, project)
req, err := http.NewRequest(http.MethodPost, url, buf)
if err != nil {
@ -222,7 +222,7 @@ func (a *adapter) DeleteChart(name, version string) error {
if err != nil {
return err
}
url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s/%s", a.url, project, name, version)
url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts/%s/%s", a.url, project, name, version)
return a.client.Delete(url)
}

14
src/replication/adapter/harbor/chart_registry_test.go
View File

@ -29,7 +29,7 @@ func TestFetchCharts(t *testing.T) {
server := test.NewServer([]*test.RequestHandlerMapping{
{
Method: http.MethodGet,
Pattern: "/api/projects",
Pattern: "/api/v2.0/projects",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "library",
@ -40,7 +40,7 @@ func TestFetchCharts(t *testing.T) {
},
{
Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts/harbor",
Pattern: "/api/v2.0/chartrepo/library/charts/harbor",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "harbor",
@ -54,7 +54,7 @@ func TestFetchCharts(t *testing.T) {
},
{
Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts",
Pattern: "/api/v2.0/chartrepo/library/charts",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "harbor"
@ -100,7 +100,7 @@ func TestFetchCharts(t *testing.T) {
func TestChartExist(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts/harbor/1.0",
Pattern: "/api/v2.0/chartrepo/library/charts/harbor/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `{
"metadata": {
@ -125,7 +125,7 @@ func TestDownloadChart(t *testing.T) {
server := test.NewServer([]*test.RequestHandlerMapping{
{
Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts/harbor/1.0",
Pattern: "/api/v2.0/chartrepo/library/charts/harbor/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `{
"metadata": {
@ -156,7 +156,7 @@ func TestDownloadChart(t *testing.T) {
func TestUploadChart(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodPost,
Pattern: "/api/chartrepo/library/charts",
Pattern: "/api/v2.0/chartrepo/library/charts",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
},
@ -174,7 +174,7 @@ func TestUploadChart(t *testing.T) {
func TestDeleteChart(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodDelete,
Pattern: "/api/chartrepo/library/charts/harbor/1.0",
Pattern: "/api/v2.0/chartrepo/library/charts/harbor/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
},

4
src/replication/adapter/harbor/image_registry.go
View File

@ -146,12 +146,12 @@ func (a *adapter) listCandidateProjects(filters []*model.Filter) ([]*project, er
// override the default implementation from the default image registry
// by calling Harbor API directly
func (a *adapter) DeleteManifest(repository, reference string) error {
url := fmt.Sprintf("%s/api/repositories/%s/tags/%s", a.url, repository, reference)
url := fmt.Sprintf("%s/api/v2.0/repositories/%s/tags/%s", a.url, repository, reference)
return a.client.Delete(url)
}
func (a *adapter) getTags(repository string) ([]*adp.VTag, error) {
url := fmt.Sprintf("%s/api/repositories/%s/tags", a.getURL(), repository)
url := fmt.Sprintf("%s/api/v2.0/repositories/%s/tags", a.getURL(), repository)
tags := []*struct {
Name string `json:"name"`
Labels []*struct {

8
src/replication/adapter/harbor/image_registry_test.go
View File

@ -28,7 +28,7 @@ func TestFetchImages(t *testing.T) {
server := test.NewServer([]*test.RequestHandlerMapping{
{
Method: http.MethodGet,
Pattern: "/api/projects",
Pattern: "/api/v2.0/projects",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "library",
@ -39,7 +39,7 @@ func TestFetchImages(t *testing.T) {
},
{
Method: http.MethodGet,
Pattern: "/api/repositories/library/hello-world/tags",
Pattern: "/api/v2.0/repositories/library/hello-world/tags",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "1.0"
@ -51,7 +51,7 @@ func TestFetchImages(t *testing.T) {
},
{
Method: http.MethodGet,
Pattern: "/api/repositories",
Pattern: "/api/v2.0/repositories",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "library/hello-world"
@ -98,7 +98,7 @@ func TestFetchImages(t *testing.T) {
func TestDeleteManifest(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodDelete,
Pattern: "/api/repositories/library/hello-world/tags/1.0",
Pattern: "/api/v2.0/repositories/library/hello-world/tags/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}})

16
src/replication/adapter/huawei/huawei_adapter.go
View File

@ -3,6 +3,7 @@ package huawei
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"io/ioutil"
"net/http"
"regexp"
@ -11,7 +12,6 @@ import (
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
@ -228,20 +228,12 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) {
}
func newAdapter(registry *model.Registry) (adp.Adapter, error) {
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
var (
modifiers = []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
}}
modifiers = []modifier.Modifier{}
authorizer modifier.Modifier
)
if registry.Credential != nil {
authorizer = auth.NewBasicAuthCredential(
authorizer = basic.NewAuthorizer(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
modifiers = append(modifiers, authorizer)
@ -249,7 +241,7 @@ func newAdapter(registry *model.Registry) (adp.Adapter, error) {
transport := util.GetHTTPTransport(registry.Insecure)
return &adapter{
Adapter: dockerRegistryAdapter,
Adapter: native.NewAdapter(registry),
registry: registry,
client: common_http.NewClient(
&http.Client{

16
src/replication/adapter/jfrog/adapter.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"io"
"io/ioutil"
"net/http"
@ -14,7 +15,6 @@ import (
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
@ -78,25 +78,17 @@ func (a *adapter) Info() (info *model.RegistryInfo, err error) {
}
func newAdapter(registry *model.Registry) (adp.Adapter, error) {
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
var (
modifiers = []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
}}
modifiers = []modifier.Modifier{}
)
if registry.Credential != nil {
modifiers = append(modifiers, auth.NewBasicAuthCredential(
modifiers = append(modifiers, basic.NewAuthorizer(
registry.Credential.AccessKey,
registry.Credential.AccessSecret))
}
return &adapter{
Adapter: dockerRegistryAdapter,
Adapter: native.NewAdapter(registry),
registry: registry,
client: common_http.NewClient(
&http.Client{

213
src/replication/adapter/native/adapter.go
View File

@ -16,22 +16,15 @@ package native
import (
"fmt"
"io"
"net/http"
"strings"
"sync"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/goharbor/harbor/src/common/http/modifier"
common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
registry_pkg "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/internal"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/registry"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"sync"
)
func init() {
@ -49,7 +42,7 @@ type factory struct {
// Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
return NewAdapter(r)
return NewAdapter(r), nil
}
// AdapterPattern ...
@ -61,60 +54,30 @@ func (f *factory) AdapterPattern() *model.AdapterPattern {
// that implement the registry V2 API
type Adapter struct {
sync.RWMutex
*registry_pkg.Registry
registry *model.Registry
client *http.Client
clients map[string]*registry_pkg.Repository // client for repositories
registry.Client
}
// NewAdapter returns an instance of the Adapter
func NewAdapter(registry *model.Registry) (*Adapter, error) {
var cred modifier.Modifier
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
if registry.Credential.Type == model.CredentialTypeSecret {
cred = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
} else {
cred = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
func NewAdapter(reg *model.Registry) *Adapter {
adapter := &Adapter{
registry: reg,
}
authorizer := auth.NewAuthorizer(cred, &http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
})
/*
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, cred, registry.TokenServiceURL)
*/
return NewAdapterWithCustomizedAuthorizer(registry, authorizer)
username, password := "", ""
if reg.Credential != nil {
username = reg.Credential.AccessKey
password = reg.Credential.AccessSecret
}
adapter.Client = registry.NewClient(reg.URL, username, password, reg.Insecure)
return adapter
}
// NewAdapterWithCustomizedAuthorizer returns an instance of the Adapter with the customized authorizer
func NewAdapterWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*Adapter, error) {
transport := util.GetHTTPTransport(registry.Insecure)
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
},
}
if authorizer != nil {
modifiers = append(modifiers, authorizer)
}
client := &http.Client{
Transport: registry_pkg.NewTransport(transport, modifiers...),
}
reg, err := registry_pkg.NewRegistry(registry.URL, client)
if err != nil {
return nil, err
}
// NewAdapterWithAuthorizer returns an instance of the Adapter with provided authorizer
func NewAdapterWithAuthorizer(reg *model.Registry, authorizer internal.Authorizer) *Adapter {
return &Adapter{
Registry: reg,
registry: registry,
client: client,
clients: map[string]*registry_pkg.Repository{},
}, nil
registry: reg,
Client: registry.NewClientWithAuthorizer(reg.URL, authorizer, reg.Insecure),
}
}
// Info returns the basic information about the adapter
@ -267,7 +230,7 @@ func (a *Adapter) getRepositories(filters []*model.Filter) ([]*adp.Repository, e
}
func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) {
tags, err := a.ListTag(repository)
tags, err := a.ListTags(repository)
if err != nil {
return nil, err
}
@ -281,131 +244,15 @@ func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) {
return result, nil
}
// ManifestExist ...
func (a *Adapter) ManifestExist(repository, reference string) (bool, string, error) {
client, err := a.getClient(repository)
if err != nil {
return false, "", err
// PingSimple checks whether the registry is available. It checks the connectivity and certificate (if TLS enabled)
// only, regardless of 401/403 error.
func (a *Adapter) PingSimple() error {
err := a.Ping()
if err == nil {
return nil
}
digest, exist, err := client.ManifestExist(reference)
return exist, digest, err
}
// PullManifest ...
func (a *Adapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) {
client, err := a.getClient(repository)
if err != nil {
return nil, "", err
if ierror.IsErr(err, ierror.UnAuthorizedCode) || ierror.IsErr(err, ierror.ForbiddenCode) {
return nil
}
digest, mediaType, payload, err := client.PullManifest(reference, accepttedMediaTypes)
if err != nil {
return nil, "", err
}
if strings.Contains(mediaType, "application/json") {
mediaType = schema1.MediaTypeManifest
}
manifest, _, err := registry_pkg.UnMarshal(mediaType, payload)
if err != nil {
return nil, "", err
}
return manifest, digest, nil
}
// PushManifest ...
func (a *Adapter) PushManifest(repository, reference, mediaType string, payload []byte) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
_, err = client.PushManifest(reference, mediaType, payload)
return err
}
// DeleteManifest ...
func (a *Adapter) DeleteManifest(repository, reference string) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
digest := reference
if !isDigest(digest) {
dgt, exist, err := client.ManifestExist(reference)
if err != nil {
return err
}
if !exist {
log.Debugf("the manifest of %s:%s doesn't exist", repository, reference)
return nil
}
digest = dgt
}
return client.DeleteManifest(digest)
}
// BlobExist ...
func (a *Adapter) BlobExist(repository, digest string) (bool, error) {
client, err := a.getClient(repository)
if err != nil {
return false, err
}
return client.BlobExist(digest)
}
// PullBlob ...
func (a *Adapter) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
client, err := a.getClient(repository)
if err != nil {
return 0, nil, err
}
return client.PullBlob(digest)
}
// PushBlob ...
func (a *Adapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
return client.PushBlob(digest, size, blob)
}
func isDigest(str string) bool {
return strings.Contains(str, ":")
}
// ListTag ...
func (a *Adapter) ListTag(repository string) ([]string, error) {
client, err := a.getClient(repository)
if err != nil {
return []string{}, err
}
return client.ListTag()
}
func (a *Adapter) getClient(repository string) (*registry_pkg.Repository, error) {
a.RLock()
client, exist := a.clients[repository]
a.RUnlock()
if exist {
return client, nil
}
return a.create(repository)
}
func (a *Adapter) create(repository string) (*registry_pkg.Repository, error) {
a.Lock()
defer a.Unlock()
// double check
client, exist := a.clients[repository]
if exist {
return client, nil
}
client, err := registry_pkg.NewRepository(repository, a.registry.URL, a.client)
if err != nil {
return nil, err
}
a.clients[repository] = client
return client, nil
}

57
src/replication/adapter/native/adapter_test.go
View File

@ -26,33 +26,9 @@ import (
"github.com/stretchr/testify/require"
)
func Test_newAdapter(t *testing.T) {
tests := []struct {
name string
registry *model.Registry
wantErr bool
}{
{name: "Nil Registry URL", registry: &model.Registry{}, wantErr: true},
{name: "Right", registry: &model.Registry{URL: "abc"}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NewAdapter(tt.registry)
if tt.wantErr {
assert.NotNil(t, err)
assert.Nil(t, got)
} else {
assert.Nil(t, err)
assert.NotNil(t, got)
}
})
}
}
func Test_native_Info(t *testing.T) {
var registry = &model.Registry{URL: "abc"}
adapter, err := NewAdapter(registry)
require.Nil(t, err)
adapter := NewAdapter(registry)
assert.NotNil(t, adapter)
info, err := adapter.Info()
@ -67,11 +43,10 @@ func Test_native_Info(t *testing.T) {
func Test_native_PrepareForPush(t *testing.T) {
var registry = &model.Registry{URL: "abc"}
adapter, err := NewAdapter(registry)
require.Nil(t, err)
adapter := NewAdapter(registry)
assert.NotNil(t, adapter)
err = adapter.PrepareForPush(nil)
err := adapter.PrepareForPush(nil)
assert.Nil(t, err)
}
@ -117,8 +92,7 @@ func Test_native_FetchImages(t *testing.T) {
URL: mock.URL,
Insecure: true,
}
adapter, err := NewAdapter(registry)
assert.Nil(t, err)
adapter := NewAdapter(registry)
assert.NotNil(t, adapter)
tests := []struct {
@ -320,26 +294,3 @@ func Test_native_FetchImages(t *testing.T) {
})
}
}
func TestIsDigest(t *testing.T) {
cases := []struct {
str string
isDigest bool
}{
{
str: "",
isDigest: false,
},
{
str: "latest",
isDigest: false,
},
{
str: "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
isDigest: true,
},
}
for _, c := range cases {
assert.Equal(t, c.isDigest, isDigest(c.str))
}
}

18
src/replication/adapter/quayio/adapter.go
View File

@ -12,7 +12,6 @@ import (
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
@ -35,27 +34,16 @@ func init() {
}
func newAdapter(registry *model.Registry) (*adapter, error) {
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
},
}
modifiers := []modifier.Modifier{}
var authorizer modifier.Modifier
if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 {
authorizer = auth.NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), auth.APIKeyInHeader)
authorizer = NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), APIKeyInHeader)
}
if authorizer != nil {
modifiers = append(modifiers, authorizer)
}
nativeRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{
Adapter: nativeRegistryAdapter,
Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
registry: registry,
client: common_http.NewClient(
&http.Client{

2
src/replication/adapter/quayio/adapter_test.go
View File

@ -41,7 +41,7 @@ func TestAdapter_Info(t *testing.T) {
func TestAdapter_PullManifests(t *testing.T) {
quayAdapter := getMockAdapter(t)
registry, _, err := quayAdapter.(*adapter).PullManifest("quay/busybox", "latest", []string{})
registry, _, err := quayAdapter.(*adapter).PullManifest("quay/busybox", "latest")
assert.Nil(t, err)
assert.NotNil(t, registry)
t.Log(registry)

2
src/common/utils/registry/auth/apikey.go → src/replication/adapter/quayio/apikey.go
View File

@ -1,4 +1,4 @@
package auth
package quayio
import (
"fmt"

2
src/common/utils/registry/auth/apikey_test.go → src/replication/adapter/quayio/apikey_test.go
View File

@ -1,4 +1,4 @@
package auth
package quayio
import (
"net/http"

6
src/replication/operation/flow/stage_test.go
View File

@ -77,11 +77,11 @@ func (f *fakedAdapter) FetchImages(filters []*model.Filter) ([]*model.Resource,
func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) {
return false, "", nil
}
func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) {
func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error) {
return nil, "", nil
}
func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) error {
return nil
func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
return "", nil
}
func (f *fakedAdapter) DeleteManifest(repository, digest string) error {
return nil

16
src/replication/transfer/image/transfer.go
View File

@ -16,12 +16,10 @@ package image
import (
"errors"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"strings"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/adapter"
@ -207,7 +205,7 @@ func (t *transfer) copyContent(content distribution.Descriptor, srcRepo, dstRepo
switch content.MediaType {
// when the media type of pulled manifest is manifest list,
// the contents it contains are a few manifests
case schema2.MediaTypeManifest:
case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
// as using digest as the reference, so set the override to true directly
return t.copyImage(srcRepo, digest, dstRepo, digest, true)
// handle foreign layer
@ -258,13 +256,7 @@ func (t *transfer) pullManifest(repository, reference string) (
return nil, "", nil
}
t.logger.Infof("pulling the manifest of image %s:%s ...", repository, reference)
// TODO add OCI media types
manifest, digest, err := t.src.PullManifest(repository, reference, []string{
schema1.MediaTypeManifest,
schema1.MediaTypeSignedManifest,
schema2.MediaTypeManifest,
manifestlist.MediaTypeManifestList,
})
manifest, digest, err := t.src.PullManifest(repository, reference)
if err != nil {
t.logger.Errorf("failed to pull the manifest of image %s:%s: %v", repository, reference, err)
return nil, "", err
@ -295,7 +287,7 @@ func (t *transfer) pushManifest(manifest distribution.Manifest, repository, tag
repository, tag, err)
return err
}
if err := t.dst.PushManifest(repository, tag, mediaType, payload); err != nil {
if _, err := t.dst.PushManifest(repository, tag, mediaType, payload); err != nil {
t.logger.Errorf("failed to push manifest of image %s:%s: %v",
repository, tag, err)
return err

9
src/replication/transfer/image/transfer_test.go
View File

@ -23,7 +23,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/log"
pkg_registry "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/replication/model"
trans "github.com/goharbor/harbor/src/replication/transfer"
"github.com/stretchr/testify/assert"
@ -42,7 +41,7 @@ func (f *fakeRegistry) ManifestExist(repository, reference string) (bool, string
}
return false, "sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", nil
}
func (f *fakeRegistry) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) {
func (f *fakeRegistry) PullManifest(repository, reference string, accepttedMediaTypes ...string) (distribution.Manifest, string, error) {
manifest := `{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
@ -71,14 +70,14 @@ func (f *fakeRegistry) PullManifest(repository, reference string, accepttedMedia
}`
mediaType := schema2.MediaTypeManifest
payload := []byte(manifest)
mani, _, err := pkg_registry.UnMarshal(mediaType, payload)
mani, _, err := distribution.UnmarshalManifest(mediaType, payload)
if err != nil {
return nil, "", err
}
return mani, "sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", nil
}
func (f *fakeRegistry) PushManifest(repository, reference, mediaType string, payload []byte) error {
return nil
func (f *fakeRegistry) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
return "", nil
}
func (f *fakeRegistry) DeleteManifest(repository, reference string) error {
return nil

5
src/replication/util/util.go
View File

@ -15,15 +15,14 @@
package util
import (
"github.com/goharbor/harbor/src/internal"
"net/http"
"strings"
"github.com/goharbor/harbor/src/common/utils/registry"
)
// GetHTTPTransport can be used to share the common HTTP transport
func GetHTTPTransport(insecure bool) *http.Transport {
return registry.GetHTTPTransport(insecure)
return internal.GetHTTPTransport(insecure)
}
// ParseRepository parses the "repository" provided into two parts: namespace and the rest

90
src/testing/pkg/registry/client.go
View File

@ -15,7 +15,9 @@
package registry
import (
"github.com/docker/distribution"
"github.com/stretchr/testify/mock"
"io"
)
// FakeClient is a fake registry client that implement src/pkg/registry.Client interface
@ -23,6 +25,94 @@ type FakeClient struct {
mock.Mock
}
// Ping ...
func (f *FakeClient) Ping() (err error) {
args := f.Called()
return args.Error(0)
}
// Catalog ...
func (f *FakeClient) Catalog() ([]string, error) {
args := f.Called()
var repositories []string
if args[0] != nil {
repositories = args[0].([]string)
}
return repositories, args.Error(1)
}
// ListTags ...
func (f *FakeClient) ListTags(repository string) ([]string, error) {
args := f.Called()
var tags []string
if args[0] != nil {
tags = args[0].([]string)
}
return tags, args.Error(1)
}
// ManifestExist ...
func (f *FakeClient) ManifestExist(repository, reference string) (bool, string, error) {
args := f.Called()
return args.Bool(0), args.String(1), args.Error(2)
}
// PullManifest ...
func (f *FakeClient) PullManifest(repository, reference string, acceptedMediaTypes ...string) (distribution.Manifest, string, error) {
args := f.Called()
var manifest distribution.Manifest
if args[0] != nil {
manifest = args[0].(distribution.Manifest)
}
return manifest, args.String(1), args.Error(2)
}
// PushManifest ...
func (f *FakeClient) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
args := f.Called()
return args.String(0), args.Error(1)
}
// DeleteManifest ...
func (f *FakeClient) DeleteManifest(repository, reference string) error {
args := f.Called()
return args.Error(0)
}
// BlobExist ...
func (f *FakeClient) BlobExist(repository, digest string) (bool, error) {
args := f.Called()
return args.Bool(0), args.Error(1)
}
// PullBlob ...
func (f *FakeClient) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
args := f.Called()
var blob io.ReadCloser
if args[0] != nil {
blob = args[0].(io.ReadCloser)
}
return int64(args.Int(0)), blob, args.Error(2)
}
// PushBlob ...
func (f *FakeClient) PushBlob(repository, digest string, size int64, blob io.Reader) error {
args := f.Called()
return args.Error(0)
}
// MountBlob ...
func (f *FakeClient) MountBlob(srcRepository, digest, dstRepository string) (err error) {
args := f.Called()
return args.Error(0)
}
// DeleteBlob ...
func (f *FakeClient) DeleteBlob(repository, digest string) (err error) {
args := f.Called()
return args.Error(0)
}
// Copy ...
func (f *FakeClient) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
args := f.Called()