Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-19 15:17:43 +01:00
Code Issues Projects Releases Wiki Activity

Reimplement the registry client

Browse Source 
This commit reimplements the registry client under directory src/pkg/registry and removes the useless code

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2020-02-21 09:29:28 +08:00
parent c2a77c2825
commit 528f598268
72 changed files with 2038 additions and 3147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/api/artifact/abstractor/blob/fetcher.go
View File

@ -18,12 +18,9 @@ import (
"github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/registry" "github.com/goharbor/harbor/src/pkg/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
v1 "github.com/opencontainers/image-spec/specs-go/v1" v1 "github.com/opencontainers/image-spec/specs-go/v1"
"io/ioutil" "io/ioutil"
"net/http"
) )
var ( var (
@ -39,6 +36,8 @@ var (
} }
) )
// TODO use the registry.Client directly? then the Fetcher can be deleted
// Fetcher fetches the content of blob // Fetcher fetches the content of blob
type Fetcher interface { type Fetcher interface {
// FetchManifest the content of manifest under the repository // FetchManifest the content of manifest under the repository
@ -49,49 +48,34 @@ type Fetcher interface {
// NewFetcher returns an instance of the default blob fetcher // NewFetcher returns an instance of the default blob fetcher
func NewFetcher() Fetcher { func NewFetcher() Fetcher {
return &fetcher{} return &fetcher{
client: registry.Cli,
}
} }
type fetcher struct{} type fetcher struct {
client registry.Client
}
// TODO re-implement it based on OCI registry driver
func (f *fetcher) FetchManifest(repository, digest string) (string, []byte, error) { func (f *fetcher) FetchManifest(repository, digest string) (string, []byte, error) {
// TODO read from cache first // TODO read from cache first
client, err := newRepositoryClient(repository) manifest, _, err := f.client.PullManifest(repository, digest)
if err != nil {
return "", nil, err
}
mediaType, payload, err := manifest.Payload()
if err != nil { if err != nil {
return "", nil, err return "", nil, err
} }
_, mediaType, payload, err := client.PullManifest(digest, accept)
return mediaType, payload, err return mediaType, payload, err
} }
// TODO re-implement it based on OCI registry driver
func (f *fetcher) FetchLayer(repository, digest string) ([]byte, error) { func (f *fetcher) FetchLayer(repository, digest string) ([]byte, error) {
// TODO read from cache first // TODO read from cache first
client, err := newRepositoryClient(repository) _, reader, err := f.client.PullBlob(repository, digest)
if err != nil {
return nil, err
}
_, reader, err := client.PullBlob(digest)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer reader.Close() defer reader.Close()
return ioutil.ReadAll(reader) return ioutil.ReadAll(reader)
} }
func newRepositoryClient(repository string) (*registry.Repository, error) {
uam := &auth.UserAgentModifier{
UserAgent: "harbor-registry-client",
}
authorizer := auth.DefaultBasicAuthorizer()
transport := registry.NewTransport(http.DefaultTransport, authorizer, uam)
client := &http.Client{
Transport: transport,
}
endpoint, err := config.RegistryURL()
if err != nil {
return nil, err
}
return registry.NewRepository(repository, endpoint, client)
}

17
src/common/http/client.go
View File

@ -19,14 +19,13 @@ import (
"crypto/tls" "crypto/tls"
"encoding/json" "encoding/json"
"errors" "errors"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/internal"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"reflect" "reflect"
"strings"
"github.com/goharbor/harbor/src/common/http/modifier"
) )
// Client is a util for common HTTP operations, such Get, Head, Post, Put and Delete. // Client is a util for common HTTP operations, such Get, Head, Post, Put and Delete.
@ -231,8 +230,8 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
if err != nil { if err != nil {
return err return err
} }
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body) data, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil { if err != nil {
return err return err
} }
@ -250,12 +249,10 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
resources = reflect.AppendSlice(resources, reflect.Indirect(res)) resources = reflect.AppendSlice(resources, reflect.Indirect(res))
endpoint = "" endpoint = ""
link := resp.Header.Get("Link") links := internal.ParseLinks(resp.Header.Get("Link"))
for _, str := range strings.Split(link, ",") { for _, link := range links {
if strings.HasSuffix(str, `rel="next"`) && if link.Rel == "next" {
strings.Index(str, "<") >= 0 && endpoint = url.Scheme + "://" + url.Host + link.URL
strings.Index(str, ">") >= 0 {
endpoint = url.Scheme + "://" + url.Host + str[strings.Index(str, "<")+1:strings.Index(str, ">")]
break break
} }
} }

40
src/common/utils/registry/auth/basicauthorizer.go
View File

@ -1,40 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/core/config"
"sync"
)
// NewBasicAuthorizer create an authorizer to add basic auth header as is set in the parameter
func NewBasicAuthorizer(u, p string) modifier.Modifier {
return NewBasicAuthCredential(u, p)
}
var (
defaultAuthorizer modifier.Modifier
once sync.Once
)
// DefaultBasicAuthorizer returns the basic authorizer that sets the basic auth as configured in env variables
func DefaultBasicAuthorizer() modifier.Modifier {
once.Do(func() {
u, p := config.RegistryCredential()
defaultAuthorizer = NewBasicAuthCredential(u, p)
})
return defaultAuthorizer
}

48
src/common/utils/registry/auth/credential.go
View File

@ -1,48 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"net/http"
"github.com/goharbor/harbor/src/common/http/modifier"
)
// Credential ...
type Credential modifier.Modifier
// Implements interface Credential
type basicAuthCredential struct {
username string
password string
}
// NewBasicAuthCredential ...
func NewBasicAuthCredential(username, password string) Credential {
return &basicAuthCredential{
username: username,
password: password,
}
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}
// implement github.com/goharbor/harbor/src/common/http/modifier.Modifier
func (b *basicAuthCredential) Modify(req *http.Request) error {
b.AddAuthorization(req)
return nil
}

58
src/common/utils/registry/auth/path.go
View File

@ -1,58 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"regexp"
"github.com/docker/distribution/reference"
"github.com/goharbor/harbor/src/common/utils/log"
)
var (
base = regexp.MustCompile("/v2")
catalog = regexp.MustCompile("/v2/_catalog")
tag = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/tags/list")
manifest = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/manifests/(" + reference.TagRegexp.String() + "|" + reference.DigestRegexp.String() + ")")
blob = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String())
blobUpload = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/uploads")
blobUploadChunk = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/uploads/[a-zA-Z0-9-_.=]+")
repoRegExps = []*regexp.Regexp{tag, manifest, blob, blobUploadChunk, blobUpload}
)
// parse the repository name from path, if the path doesn't match any
// regular expressions in repoRegExps, nil string will be returned
func parseRepository(path string) string {
for _, regExp := range repoRegExps {
subs := regExp.FindStringSubmatch(path)
// no match
if subs == nil {
continue
}
// match
// the subs should contain at least 2 matching texts, the first one matches
// the whole regular expression, and the second one matches the repository
// part
if len(subs) < 2 {
log.Warningf("unexpected length of sub matches: %d, should >= 2 ", len(subs))
continue
}
return subs[1]
}
return ""
}

43
src/common/utils/registry/auth/path_test.go
View File

@ -1,43 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseRepository(t *testing.T) {
cases := []struct {
input string
output string
}{
{"/v2", ""},
{"/v2/_catalog", ""},
{"/v2/library/tags/list", "library"},
{"/v2/tags/list", ""},
{"/v2/tags/list/tags/list", "tags/list"},
{"/v2/library/manifests/latest", "library"},
{"/v2/library/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", "library"},
{"/v2/library/blobs/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", "library"},
{"/v2/library/blobs/uploads", "library"},
{"/v2/library/blobs/uploads/1234567890", "library"},
}
for _, c := range cases {
assert.Equal(t, c.output, parseRepository(c.input))
}
}

354
src/common/utils/registry/auth/tokenauthorizer.go
View File

@ -1,354 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/docker/distribution/registry/auth/token"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
token_util "github.com/goharbor/harbor/src/core/service/token"
)
const (
latency int = 10 // second, the network latency when token is received
scheme = "bearer"
)
type tokenGenerator interface {
generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error)
}
// UserAgentModifier adds the "User-Agent" header to the request
type UserAgentModifier struct {
UserAgent string
}
// Modify adds user-agent header to the request
func (u *UserAgentModifier) Modify(req *http.Request) error {
req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.UserAgent)
return nil
}
// tokenAuthorizer implements registry.Modifier interface. It parses scopses
// from the request, generates authentication token and modifies the requset
// by adding the token
type tokenAuthorizer struct {
registryURL *url.URL // used to filter request
generator tokenGenerator
client *http.Client
cachedTokens map[string]*models.Token
sync.Mutex
}
// add token to the request
func (t *tokenAuthorizer) Modify(req *http.Request) error {
// only handle requests sent to registry
goon, err := t.filterReq(req)
if err != nil {
return err
}
if !goon {
log.Debugf("the request %s is not sent to registry, skip", req.URL.String())
return nil
}
// parse scopes from request
scopes, err := parseScopes(req)
if err != nil {
return err
}
var token *models.Token
// try to get token from cache if the request is for empty scope(login)
// or single scope
if len(scopes) <= 1 {
key := ""
if len(scopes) == 1 {
key = scopeString(scopes[0])
}
token = t.getCachedToken(key)
}
// request a new token if the token is null
if token == nil {
token, err = t.generator.generate(scopes, t.registryURL.String())
if err != nil {
return err
}
// if the token is null(this happens if the registry needs no authentication), return
// directly. Or the token will be cached
if token == nil {
return nil
}
// only cache the token for empty scope(login) or single scope request
if len(scopes) <= 1 {
key := ""
if len(scopes) == 1 {
key = scopeString(scopes[0])
}
t.updateCachedToken(key, token)
}
}
tk := token.GetToken()
if len(tk) == 0 {
return errors.New("empty token content")
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", tk))
return nil
}
func scopeString(scope *token.ResourceActions) string {
if scope == nil {
return ""
}
return fmt.Sprintf("%s:%s:%s", scope.Type, scope.Name, strings.Join(scope.Actions, ","))
}
// some requests are sent to backend storage, such as s3, this method filters
// the requests only sent to registry
func (t *tokenAuthorizer) filterReq(req *http.Request) (bool, error) {
// the registryURL is nil when the first request comes, init it with
// the scheme and host of the request which must be sent to the registry
if t.registryURL == nil {
u, err := url.Parse(buildPingURL(req.URL.Scheme + "://" + req.URL.Host))
if err != nil {
return false, err
}
t.registryURL = u
}
v2Index := strings.Index(req.URL.Path, "/v2/")
if v2Index == -1 {
return false, nil
}
if req.URL.Host != t.registryURL.Host || req.URL.Scheme != t.registryURL.Scheme ||
req.URL.Path[:v2Index+4] != t.registryURL.Path {
return false, nil
}
return true, nil
}
// parse scopes from the request according to its method, path and query string
func parseScopes(req *http.Request) ([]*token.ResourceActions, error) {
scopes := []*token.ResourceActions{}
from := req.URL.Query().Get("from")
if len(from) != 0 {
scopes = append(scopes, &token.ResourceActions{
Type: "repository",
Name: from,
Actions: []string{"pull"},
})
}
var scope *token.ResourceActions
path := strings.TrimRight(req.URL.Path, "/")
repository := parseRepository(path)
if len(repository) > 0 {
// pull, push, delete blob/manifest
scope = &token.ResourceActions{
Type: "repository",
Name: repository,
}
switch req.Method {
case http.MethodGet, http.MethodHead:
scope.Actions = []string{"pull"}
case http.MethodPost, http.MethodPut, http.MethodPatch:
scope.Actions = []string{"pull", "push"}
case http.MethodDelete:
scope.Actions = []string{"*"}
default:
scope = nil
log.Warningf("unsupported method: %s", req.Method)
}
} else if catalog.MatchString(path) {
// catalog
scope = &token.ResourceActions{
Type: "registry",
Name: "catalog",
Actions: []string{"*"},
}
} else if base.MatchString(path) {
// base
scope = nil
} else {
// unknown
return scopes, fmt.Errorf("can not parse scope from the request: %s %s", req.Method, req.URL.Path)
}
if scope != nil {
scopes = append(scopes, scope)
}
strs := []string{}
for _, s := range scopes {
strs = append(strs, scopeString(s))
}
log.Debugf("scopes parsed from request: %s", strings.Join(strs, " "))
return scopes, nil
}
func (t *tokenAuthorizer) getCachedToken(scope string) *models.Token {
t.Lock()
defer t.Unlock()
token := t.cachedTokens[scope]
if token == nil {
return nil
}
issueAt, err := time.Parse(time.RFC3339, token.IssuedAt)
if err != nil {
log.Errorf("failed parse %s: %v", token.IssuedAt, err)
delete(t.cachedTokens, scope)
return nil
}
if issueAt.Add(time.Duration(token.ExpiresIn-latency) * time.Second).Before(time.Now().UTC()) {
delete(t.cachedTokens, scope)
return nil
}
log.Debugf("get token for scope %s from cache", scope)
return token
}
func (t *tokenAuthorizer) updateCachedToken(scope string, token *models.Token) {
t.Lock()
defer t.Unlock()
t.cachedTokens[scope] = token
}
// ping returns the realm, service and error
func ping(client *http.Client, endpoint string) (string, string, error) {
resp, err := client.Get(endpoint)
if err != nil {
return "", "", err
}
defer resp.Body.Close()
challenges := ParseChallengeFromResponse(resp)
for _, challenge := range challenges {
if scheme == challenge.Scheme {
realm := challenge.Parameters["realm"]
service := challenge.Parameters["service"]
return realm, service, nil
}
}
log.Warningf("Schemas %v are unsupported", challenges)
return "", "", nil
}
// NewStandardTokenAuthorizer returns a standard token authorizer. The authorizer will request a token
// from token server and add it to the origin request
// If customizedTokenService is set, the token request will be sent to it instead of the server get from authorizer
func NewStandardTokenAuthorizer(client *http.Client, credential Credential,
customizedTokenService ...string) modifier.Modifier {
generator := &standardTokenGenerator{
credential: credential,
client: client,
}
// when the registry client is used inside Harbor, the token request
// can be posted to token service directly rather than going through nginx.
// If realm is set as the internal url of token service, this can resolve
// two problems:
// 1. performance issue
// 2. the realm field returned by registry is an IP which can not reachable
// inside Harbor
if len(customizedTokenService) > 0 && len(customizedTokenService[0]) > 0 {
generator.realm = customizedTokenService[0]
}
return &tokenAuthorizer{
cachedTokens: make(map[string]*models.Token),
generator: generator,
client: client,
}
}
// standardTokenGenerator implements interface tokenGenerator
type standardTokenGenerator struct {
realm string
service string
credential Credential
client *http.Client
}
// get token from token service
func (s *standardTokenGenerator) generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error) {
// ping first if the realm or service is null
if len(s.realm) == 0 || len(s.service) == 0 {
realm, service, err := ping(s.client, endpoint)
if err != nil {
return nil, err
}
if len(realm) == 0 {
log.Warning("empty realm, skip")
return nil, nil
}
if len(s.realm) == 0 {
s.realm = realm
}
s.service = service
}
return getToken(s.client, s.credential, s.realm, s.service, scopes)
}
// NewRawTokenAuthorizer returns a token authorizer which calls method to create
// token directly
func NewRawTokenAuthorizer(username, service string) modifier.Modifier {
generator := &rawTokenGenerator{
service: service,
username: username,
}
return &tokenAuthorizer{
cachedTokens: make(map[string]*models.Token),
generator: generator,
}
}
// rawTokenGenerator implements interface tokenGenerator
type rawTokenGenerator struct {
service string
username string
}
// generate token directly
func (r *rawTokenGenerator) generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error) {
return token_util.MakeToken(r.username, r.service, scopes)
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}

222
src/common/utils/registry/auth/tokenauthorizer_test.go
View File

@ -1,222 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"testing"
"time"
"github.com/docker/distribution/registry/auth/token"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFilterReq(t *testing.T) {
authorizer := tokenAuthorizer{}
// v2
req, err := http.NewRequest(http.MethodGet, "http://registry/v2/", nil)
require.Nil(t, err)
goon, err := authorizer.filterReq(req)
assert.Nil(t, err)
assert.True(t, goon)
// catalog
req, err = http.NewRequest(http.MethodGet, "http://registry/v2/_catalog?n=1000", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.True(t, goon)
// contains two v2 in path
req, err = http.NewRequest(http.MethodGet, "http://registry/v2/library/v2/tags/list", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.True(t, goon)
// different scheme
req, err = http.NewRequest(http.MethodGet, "https://registry/v2/library/golang/tags/list", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.False(t, goon)
// different host
req, err = http.NewRequest(http.MethodGet, "http://vmware.com/v2/library/golang/tags/list", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.False(t, goon)
// different path
req, err = http.NewRequest(http.MethodGet, "https://registry/s3/ssss", nil)
require.Nil(t, err)
goon, err = authorizer.filterReq(req)
assert.Nil(t, err)
assert.False(t, goon)
}
func TestParseScopes(t *testing.T) {
// contains from in query string
req, err := http.NewRequest(http.MethodGet, "http://registry/v2?from=library", nil)
require.Nil(t, err)
scopses, err := parseScopes(req)
assert.Nil(t, err)
assert.Equal(t, 1, len(scopses))
assert.EqualValues(t, &token.ResourceActions{
Type: "repository",
Name: "library",
Actions: []string{
"pull"},
}, scopses[0])
// v2
req, err = http.NewRequest(http.MethodGet, "http://registry/v2", nil)
require.Nil(t, err)
scopses, err = parseScopes(req)
assert.Nil(t, err)
assert.Equal(t, 0, len(scopses))
// catalog
req, err = http.NewRequest(http.MethodGet, "http://registry/v2/_catalog", nil)
require.Nil(t, err)
scopses, err = parseScopes(req)
assert.Nil(t, err)
assert.Equal(t, 1, len(scopses))
assert.EqualValues(t, &token.ResourceActions{
Type: "registry",
Name: "catalog",
Actions: []string{
"*"},
}, scopses[0])
// manifest
req, err = http.NewRequest(http.MethodPut, "http://registry/v2/library/mysql/5.6/manifests/1", nil)
require.Nil(t, err)
scopses, err = parseScopes(req)
assert.Nil(t, err)
assert.Equal(t, 1, len(scopses))
assert.EqualValues(t, &token.ResourceActions{
Type: "repository",
Name: "library/mysql/5.6",
Actions: []string{"pull", "push"},
}, scopses[0])
// invalid
req, err = http.NewRequest(http.MethodPut, "http://registry/other", nil)
require.Nil(t, err)
scopses, err = parseScopes(req)
assert.NotNil(t, err)
}
func TestGetAndUpdateCachedToken(t *testing.T) {
authorizer := &tokenAuthorizer{
cachedTokens: make(map[string]*models.Token),
}
// empty cache
token := authorizer.getCachedToken("")
assert.Nil(t, token)
// put a valid token into cache
token = &models.Token{
Token: "token",
ExpiresIn: 60,
IssuedAt: time.Now().Format(time.RFC3339),
}
authorizer.updateCachedToken("", token)
token2 := authorizer.getCachedToken("")
assert.EqualValues(t, token, token2)
// put a expired token into cache
token = &models.Token{
Token: "token",
ExpiresIn: 60,
IssuedAt: time.Now().Add(-time.Second * 120).Format("2006-01-02 15:04:05.999999999 -0700 MST"),
}
authorizer.updateCachedToken("", token)
token2 = authorizer.getCachedToken("")
assert.Nil(t, token2)
}
func TestModifyOfStandardTokenAuthorizer(t *testing.T) {
token := &models.Token{
Token: "token",
ExpiresIn: 3600,
IssuedAt: time.Now().String(),
}
data, err := json.Marshal(token)
require.Nil(t, err)
tokenHandler := test.Handler(&test.Response{
Body: data,
})
tokenServer := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/service/token",
Handler: tokenHandler,
})
defer tokenServer.Close()
header := fmt.Sprintf("Bearer realm=\"%s/service/token\",service=\"registry\"",
tokenServer.URL)
pingHandler := test.Handler(&test.Response{
StatusCode: http.StatusUnauthorized,
Headers: map[string]string{
"WWW-Authenticate": header,
},
})
registryServer := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2",
Handler: pingHandler,
})
defer registryServer.Close()
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/v2/", registryServer.URL), nil)
require.Nil(t, err)
authorizer := NewStandardTokenAuthorizer(http.DefaultClient, nil)
err = authorizer.Modify(req)
require.Nil(t, err)
tk := req.Header.Get("Authorization")
assert.Equal(t, strings.ToLower("Bearer "+token.Token), strings.ToLower(tk))
}
func TestUserAgentModifier(t *testing.T) {
agent := "harbor-registry-client"
modifier := &UserAgentModifier{
UserAgent: agent,
}
req, err := http.NewRequest(http.MethodGet, "http://registry/v2/", nil)
require.Nil(t, err)
modifier.Modify(req)
actual := req.Header.Get("User-Agent")
if actual != agent {
t.Errorf("expect request to have header User-Agent=%s, but got User-Agent=%s", agent, actual)
}
}

88
src/common/utils/registry/auth/util.go
View File

@ -1,88 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"github.com/docker/distribution/registry/auth/token"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/registry"
)
const (
service = "harbor-registry"
)
// GetToken requests a token against the endpoint using credential provided
func GetToken(endpoint string, insecure bool, credential Credential,
scopes []*token.ResourceActions) (*models.Token, error) {
client := &http.Client{
Transport: registry.GetHTTPTransport(insecure),
}
return getToken(client, credential, endpoint, service, scopes)
}
func getToken(client *http.Client, credential Credential, realm, service string,
scopes []*token.ResourceActions) (*models.Token, error) {
u, err := url.Parse(realm)
if err != nil {
return nil, err
}
query := u.Query()
query.Add("service", service)
for _, scope := range scopes {
query.Add("scope", scopeString(scope))
}
u.RawQuery = query.Encode()
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
if credential != nil {
credential.Modify(req)
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, &commonhttp.Error{
Code: resp.StatusCode,
Message: string(data),
}
}
token := &models.Token{}
if err = json.Unmarshal(data, token); err != nil {
return nil, err
}
return token, nil
}

60
src/common/utils/registry/manifest_test.go
View File

@ -1,60 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"testing"
"github.com/docker/distribution/manifest/schema2"
)
func TestUnMarshal(t *testing.T) {
b := []byte(`{
"schemaVersion":2,
"mediaType":"application/vnd.docker.distribution.manifest.v2+json",
"config":{
"mediaType":"application/vnd.docker.container.image.v1+json",
"size":1473,
"digest":"sha256:c54a2cc56cbb2f04003c1cd4507e118af7c0d340fe7e2720f70976c4b75237dc"
},
"layers":[
{
"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip",
"size":974,
"digest":"sha256:c04b14da8d1441880ed3fe6106fb2cc6fa1c9661846ac0266b8a5ec8edf37b7c"
}
]
}`)
manifest, _, err := UnMarshal(schema2.MediaTypeManifest, b)
if err != nil {
t.Fatalf("failed to parse manifest: %v", err)
}
refs := manifest.References()
if len(refs) != 2 {
t.Fatalf("unexpected length of reference: %d != %d", len(refs), 2)
}
digest := "sha256:c54a2cc56cbb2f04003c1cd4507e118af7c0d340fe7e2720f70976c4b75237dc"
if refs[0].Digest.String() != digest {
t.Errorf("unexpected digest: %s != %s", refs[0].Digest.String(), digest)
}
digest = "sha256:c04b14da8d1441880ed3fe6106fb2cc6fa1c9661846ac0266b8a5ec8edf37b7c"
if refs[1].Digest.String() != digest {
t.Errorf("unexpected digest: %s != %s", refs[1].Digest.String(), digest)
}
}

185
src/common/utils/registry/registry.go
View File

@ -1,185 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"crypto/tls"
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"strings"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils"
)
// Registry holds information of a registry entity
type Registry struct {
Endpoint *url.URL
client *http.Client
}
var defaultHTTPTransport, secureHTTPTransport, insecureHTTPTransport *http.Transport
func init() {
defaultHTTPTransport = &http.Transport{}
secureHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
},
}
insecureHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
}
// GetHTTPTransport returns HttpTransport based on insecure configuration
func GetHTTPTransport(insecure ...bool) *http.Transport {
if len(insecure) == 0 {
return defaultHTTPTransport
}
if insecure[0] {
return insecureHTTPTransport
}
return secureHTTPTransport
}
// NewRegistry returns an instance of registry
func NewRegistry(endpoint string, client *http.Client) (*Registry, error) {
u, err := utils.ParseEndpoint(endpoint)
if err != nil {
return nil, err
}
registry := &Registry{
Endpoint: u,
client: client,
}
return registry, nil
}
// Catalog ...
func (r *Registry) Catalog() ([]string, error) {
repos := []string{}
aurl := r.Endpoint.String() + "/v2/_catalog?n=1000"
for len(aurl) > 0 {
req, err := http.NewRequest("GET", aurl, nil)
if err != nil {
return repos, err
}
resp, err := r.client.Do(req)
if err != nil {
return nil, parseError(err)
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return repos, err
}
if resp.StatusCode == http.StatusOK {
catalogResp := struct {
Repositories []string `json:"repositories"`
}{}
if err := json.Unmarshal(b, &catalogResp); err != nil {
return repos, err
}
repos = append(repos, catalogResp.Repositories...)
// Link: </v2/_catalog?last=library%2Fhello-world-25&n=100>; rel="next"
// Link: <http://domain.com/v2/_catalog?last=library%2Fhello-world-25&n=100>; rel="next"
link := resp.Header.Get("Link")
if strings.HasSuffix(link, `rel="next"`) && strings.Index(link, "<") >= 0 && strings.Index(link, ">") >= 0 {
aurl = link[strings.Index(link, "<")+1 : strings.Index(link, ">")]
if strings.Index(aurl, ":") < 0 {
aurl = r.Endpoint.String() + aurl
}
} else {
aurl = ""
}
} else {
return repos, &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
}
return repos, nil
}
// Ping checks by Head method
func (r *Registry) Ping() error {
return r.ping(http.MethodHead)
}
// PingGet checks by Get method
func (r *Registry) PingGet() error {
return r.ping(http.MethodGet)
}
func (r *Registry) ping(method string) error {
req, err := http.NewRequest(method, buildPingURL(r.Endpoint.String()), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
// PingSimple checks whether the registry is available. It checks the connectivity and certificate (if TLS enabled)
// only, regardless of credential.
func (r *Registry) PingSimple() error {
err := r.Ping()
if err == nil {
return nil
}
httpErr, ok := err.(*commonhttp.Error)
if !ok {
return err
}
if httpErr.Code == http.StatusUnauthorized ||
httpErr.Code == http.StatusForbidden {
return nil
}
return httpErr
}

142
src/common/utils/registry/registry_test.go
View File

@ -1,142 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"testing"
"github.com/goharbor/harbor/src/common/utils/test"
)
func TestPing(t *testing.T) {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: http.MethodHead,
Pattern: "/v2/",
Handler: test.Handler(nil),
})
defer server.Close()
client, err := newRegistryClient(server.URL)
if err != nil {
t.Fatalf("failed to create client for registry: %v", err)
}
if err = client.Ping(); err != nil {
t.Errorf("failed to ping registry: %v", err)
}
}
func TestCatalog(t *testing.T) {
repositories := make([]string, 0, 1001)
for i := 0; i < 1001; i++ {
repositories = append(repositories, strconv.Itoa(i))
}
handler := func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
last := q.Get("last")
n, err := strconv.Atoi(q.Get("n"))
if err != nil || n <= 0 {
n = 1000
}
length := len(repositories)
begin := length
if len(last) == 0 {
begin = 0
} else {
for i, repository := range repositories {
if repository == last {
begin = i + 1
break
}
}
}
end := begin + n
if end > length {
end = length
}
w.Header().Set(http.CanonicalHeaderKey("Content-Type"), "application/json")
if end < length {
u, err := url.Parse("/v2/_catalog")
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
values := u.Query()
values.Add("last", repositories[end-1])
values.Add("n", strconv.Itoa(n))
u.RawQuery = values.Encode()
link := fmt.Sprintf("<%s>; rel=\"next\"", u.String())
w.Header().Set(http.CanonicalHeaderKey("link"), link)
}
repos := struct {
Repositories []string `json:"repositories"`
}{
Repositories: []string{},
}
if begin < length {
repos.Repositories = repositories[begin:end]
}
b, err := json.Marshal(repos)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Write(b)
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/_catalog",
Handler: handler,
})
defer server.Close()
client, err := newRegistryClient(server.URL)
if err != nil {
t.Fatalf("failed to create client for registry: %v", err)
}
repos, err := client.Catalog()
if err != nil {
t.Fatalf("failed to catalog repositories: %v", err)
}
if len(repos) != len(repositories) {
t.Errorf("unexpected length of repositories: %d != %d", len(repos), len(repositories))
}
}
func newRegistryClient(url string) (*Registry, error) {
return NewRegistry(url, &http.Client{})
}

535
src/common/utils/registry/repository.go
View File

@ -1,535 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils"
)
// Repository holds information of a repository entity
type Repository struct {
Name string
Endpoint *url.URL
client *http.Client
}
// NewRepository returns an instance of Repository
func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) {
name = strings.TrimSpace(name)
u, err := utils.ParseEndpoint(endpoint)
if err != nil {
return nil, err
}
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
return repository, nil
}
func parseError(err error) error {
if urlErr, ok := err.(*url.Error); ok {
if regErr, ok := urlErr.Err.(*commonhttp.Error); ok {
return regErr
}
}
return err
}
// ListTag ...
func (r *Repository) ListTag() ([]string, error) {
tags := []string{}
aurl := buildTagListURL(r.Endpoint.String(), r.Name)
for len(aurl) > 0 {
req, err := http.NewRequest("GET", aurl, nil)
if err != nil {
return tags, err
}
resp, err := r.client.Do(req)
if err != nil {
return nil, parseError(err)
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
if resp.StatusCode == http.StatusOK {
tagsResp := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
tags = append(tags, tagsResp.Tags...)
// Link: </v2/library/hello-world/tags/list?last=......>; rel="next"
// Link: <http://domain.com/v2/library/hello-world/tags/list?last=......>; rel="next"
link := resp.Header.Get("Link")
if strings.HasSuffix(link, `rel="next"`) && strings.Index(link, "<") >= 0 && strings.Index(link, ">") >= 0 {
aurl = link[strings.Index(link, "<")+1 : strings.Index(link, ">")]
if strings.Index(aurl, ":") < 0 {
aurl = r.Endpoint.String() + aurl
}
} else {
aurl = ""
}
} else if resp.StatusCode == http.StatusNotFound {
// TODO remove the logic if the bug of registry is fixed
// It's a workaround for a bug of registry: when listing tags of
// a repository which is being pushed, a "NAME_UNKNOWN" error will
// been returned, while the catalog API can list this repository.
return tags, nil
} else {
return tags, &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
}
sort.Strings(tags)
return tags, nil
}
// ManifestExist ...
func (r *Repository) ManifestExist(reference string) (digest string, exist bool, err error) {
req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema1.MediaTypeManifest)
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
exist = true
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
if resp.StatusCode == http.StatusNotFound {
return
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
// PullManifest ...
func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) {
req, err := http.NewRequest("GET", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
for _, mediaType := range acceptMediaTypes {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
payload = b
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
// PushManifest ...
func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (digest string, err error) {
req, err := http.NewRequest("PUT", buildManifestURL(r.Endpoint.String(), r.Name, reference),
bytes.NewReader(payload))
if err != nil {
return
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType)
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
// DeleteManifest ...
func (r *Repository) DeleteManifest(digest string) error {
req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusAccepted {
return nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
// MountBlob ...
func (r *Repository) MountBlob(digest, from string) error {
req, err := http.NewRequest("POST", buildMountBlobURL(r.Endpoint.String(), r.Name, digest, from), nil)
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode/100 != 2 {
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
return nil
}
// DeleteTag ...
func (r *Repository) DeleteTag(tag string) error {
digest, exist, err := r.ManifestExist(tag)
if err != nil {
return err
}
if !exist {
return &commonhttp.Error{
Code: http.StatusNotFound,
}
}
return r.DeleteManifest(digest)
}
// BlobExist ...
func (r *Repository) BlobExist(digest string) (bool, error) {
req, err := http.NewRequest("HEAD", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return false, err
}
resp, err := r.client.Do(req)
if err != nil {
return false, parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return true, nil
}
if resp.StatusCode == http.StatusNotFound {
return false, nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, err
}
return false, &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
// PullBlob : client must close data if it is not nil
func (r *Repository) PullBlob(digest string) (size int64, data io.ReadCloser, err error) {
req, err := http.NewRequest("GET", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return
}
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
if resp.StatusCode == http.StatusOK {
contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
size, err = strconv.ParseInt(contengLength, 10, 64)
if err != nil {
return
}
data = resp.Body
return
}
// can not close the connect if the status code is 200
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID string, err error) {
req, err := http.NewRequest("POST", buildInitiateBlobUploadURL(r.Endpoint.String(), r.Name), nil)
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := r.client.Do(req)
if err != nil {
err = parseError(err)
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusAccepted {
location = resp.Header.Get(http.CanonicalHeaderKey("Location"))
uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID"))
return
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) monolithicBlobUpload(location, digest string, size int64, data io.Reader) error {
url, err := buildMonolithicBlobUploadURL(r.Endpoint.String(), location, digest)
if err != nil {
return err
}
req, err := http.NewRequest("PUT", url, data)
if err != nil {
return err
}
req.ContentLength = size
resp, err := r.client.Do(req)
if err != nil {
return parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusCreated {
return nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
// PushBlob ...
func (r *Repository) PushBlob(digest string, size int64, data io.Reader) error {
location, _, err := r.initiateBlobUpload(r.Name)
if err != nil {
return err
}
return r.monolithicBlobUpload(location, digest, size, data)
}
// DeleteBlob ...
func (r *Repository) DeleteBlob(digest string) error {
req, err := http.NewRequest("DELETE", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return parseError(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusAccepted {
return nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return &commonhttp.Error{
Code: resp.StatusCode,
Message: string(b),
}
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}
func buildTagListURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName)
}
func buildManifestURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference)
}
func buildBlobURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repoName, reference)
}
func buildMountBlobURL(endpoint, repoName, digest, from string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/?mount=%s&from=%s", endpoint, repoName, digest, from)
}
func buildInitiateBlobUploadURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repoName)
}
func buildMonolithicBlobUploadURL(endpoint, location, digest string) (string, error) {
relative, err := isRelativeURL(location)
if err != nil {
return "", err
}
// when the registry enables "relativeurls", the location returned
// has no scheme and host part
if relative {
location = endpoint + location
}
query := ""
if strings.ContainsRune(location, '?') {
query = "&"
} else {
query = "?"
}
query += fmt.Sprintf("digest=%s", digest)
return fmt.Sprintf("%s%s", location, query), nil
}
func isRelativeURL(endpoint string) (bool, error) {
u, err := url.Parse(endpoint)
if err != nil {
return false, err
}
return !u.IsAbs(), nil
}

458
src/common/utils/registry/repository_test.go
View File

@ -1,458 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/docker/distribution/manifest/schema2"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/test"
)
var (
repository = "library/hello-world"
tag = "latest"
mediaType = schema2.MediaTypeManifest
manifest = []byte("manifest")
blob = []byte("blob")
uuid = "0663ff44-63bb-11e6-8b77-86f30ca893d3"
digest = "sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b"
)
func TestBlobExist(t *testing.T) {
handler := func(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
dgt := path[strings.LastIndex(path, "/")+1:]
if dgt == digest {
w.Header().Add(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(blob)))
w.Header().Add(http.CanonicalHeaderKey("Docker-Content-Digest"), digest)
w.Header().Add(http.CanonicalHeaderKey("Content-Type"), "application/octet-stream")
return
}
w.WriteHeader(http.StatusNotFound)
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: fmt.Sprintf("/v2/%s/blobs/", repository),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
err = parseError(err)
t.Fatalf("failed to create client for repository: %v", err)
}
exist, err := client.BlobExist(digest)
if err != nil {
t.Fatalf("failed to check the existence of blob: %v", err)
}
if !exist {
t.Errorf("blob should exist on registry, but it does not exist")
}
exist, err = client.BlobExist("invalid_digest")
if err != nil {
t.Fatalf("failed to check the existence of blob: %v", err)
}
if exist {
t.Errorf("blob should not exist on registry, but it exists")
}
}
func TestPullBlob(t *testing.T) {
handler := test.Handler(&test.Response{
Headers: map[string]string{
"Content-Length": strconv.Itoa(len(blob)),
"Docker-Content-Digest": digest,
"Content-Type": "application/octet-stream",
},
Body: blob,
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
size, reader, err := client.PullBlob(digest)
if err != nil {
t.Fatalf("failed to pull blob: %v", err)
}
if size != int64(len(blob)) {
t.Errorf("unexpected size of blob: %d != %d", size, len(blob))
}
b, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatalf("failed to read from reader: %v", err)
}
if bytes.Compare(b, blob) != 0 {
t.Errorf("unexpected blob: %s != %s", string(b), string(blob))
}
}
func TestPushBlob(t *testing.T) {
location := ""
initUploadHandler := func(w http.ResponseWriter, r *http.Request) {
w.Header().Add(http.CanonicalHeaderKey("Content-Length"), "0")
w.Header().Add(http.CanonicalHeaderKey("Location"), location)
w.Header().Add(http.CanonicalHeaderKey("Range"), "0-0")
w.Header().Add(http.CanonicalHeaderKey("Docker-Upload-UUID"), uuid)
w.WriteHeader(http.StatusAccepted)
}
monolithicUploadHandler := test.Handler(&test.Response{
StatusCode: http.StatusCreated,
Headers: map[string]string{
"Content-Length": "0",
"Location": fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
"Docker-Content-Digest": digest,
},
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "POST",
Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/", repository),
Handler: initUploadHandler,
},
&test.RequestHandlerMapping{
Method: "PUT",
Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/%s", repository, uuid),
Handler: monolithicUploadHandler,
})
defer server.Close()
location = fmt.Sprintf("%s/v2/%s/blobs/uploads/%s", server.URL, repository, uuid)
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
if err = client.PushBlob(digest, int64(len(blob)), bytes.NewReader(blob)); err != nil {
t.Fatalf("failed to push blob: %v", err)
}
}
func TestDeleteBlob(t *testing.T) {
handler := test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "DELETE",
Pattern: fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
if err = client.DeleteBlob(digest); err != nil {
t.Fatalf("failed to delete blob: %v", err)
}
}
func TestManifestExist(t *testing.T) {
handler := func(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
tg := path[strings.LastIndex(path, "/")+1:]
if tg == tag {
w.Header().Add(http.CanonicalHeaderKey("Docker-Content-Digest"), digest)
w.Header().Add(http.CanonicalHeaderKey("Content-Type"), mediaType)
return
}
w.WriteHeader(http.StatusNotFound)
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
d, exist, err := client.ManifestExist(tag)
if err != nil {
t.Fatalf("failed to check the existence of manifest: %v", err)
}
if !exist || d != digest {
t.Errorf("manifest should exist on registry, but it does not exist")
}
_, exist, err = client.ManifestExist("invalid_tag")
if err != nil {
t.Fatalf("failed to check the existence of manifest: %v", err)
}
if exist {
t.Errorf("manifest should not exist on registry, but it exists")
}
}
func TestPullManifest(t *testing.T) {
handler := test.Handler(&test.Response{
Headers: map[string]string{
"Docker-Content-Digest": digest,
"Content-Type": mediaType,
},
Body: manifest,
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
d, md, payload, err := client.PullManifest(tag, []string{mediaType})
if err != nil {
t.Fatalf("failed to pull manifest: %v", err)
}
if d != digest {
t.Errorf("unexpected digest of manifest: %s != %s", d, digest)
}
if md != mediaType {
t.Errorf("unexpected media type of manifest: %s != %s", md, mediaType)
}
if bytes.Compare(payload, manifest) != 0 {
t.Errorf("unexpected manifest: %s != %s", string(payload), string(manifest))
}
}
func TestPushManifest(t *testing.T) {
handler := test.Handler(&test.Response{
StatusCode: http.StatusCreated,
Headers: map[string]string{
"Content-Length": "0",
"Docker-Content-Digest": digest,
"Location": "",
},
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "PUT",
Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
d, err := client.PushManifest(tag, mediaType, manifest)
if err != nil {
t.Fatalf("failed to pull manifest: %v", err)
}
if d != digest {
t.Errorf("unexpected digest of manifest: %s != %s", d, digest)
}
}
func TestDeleteTag(t *testing.T) {
manifestExistHandler := test.Handler(&test.Response{
Headers: map[string]string{
"Docker-Content-Digest": digest,
"Content-Type": mediaType,
},
})
deleteManifestandler := test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: fmt.Sprintf("/v2/%s/manifests/", repository),
Handler: manifestExistHandler,
},
&test.RequestHandlerMapping{
Method: "DELETE",
Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, digest),
Handler: deleteManifestandler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
if err = client.DeleteTag(tag); err != nil {
t.Fatalf("failed to delete tag: %v", err)
}
}
func TestListTag(t *testing.T) {
handler := test.Handler(&test.Response{
Headers: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(fmt.Sprintf("{\"name\": \"%s\",\"tags\": [\"%s\"]}", repository, tag)),
})
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: fmt.Sprintf("/v2/%s/tags/list", repository),
Handler: handler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
tags, err := client.ListTag()
if err != nil {
t.Fatalf("failed to list tags: %v", err)
}
if len(tags) != 1 {
t.Fatalf("unexpected length of tags: %d != %d", len(tags), 1)
}
if tags[0] != tag {
t.Errorf("unexpected tag: %s != %s", tags[0], tag)
}
}
func TestParseError(t *testing.T) {
err := &url.Error{
Err: &commonhttp.Error{},
}
e := parseError(err)
if _, ok := e.(*commonhttp.Error); !ok {
t.Errorf("error type does not match registry error")
}
}
func newRepository(endpoint string) (*Repository, error) {
return NewRepository(repository, endpoint, &http.Client{})
}
func TestBuildMonolithicBlobUploadURL(t *testing.T) {
endpoint := "http://192.169.0.1"
digest := "sha256:ef15416724f6e2d5d5b422dc5105add931c1f2a45959cd4993e75e47957b3b55"
// absolute URL
location := "http://192.169.0.1/v2/library/golang/blobs/uploads/c9f84fd7-0198-43e3-80a7-dd13771cd7f0?_state=GabyCujPu0dpxiY8yYZTq"
expected := location + "&digest=" + digest
url, err := buildMonolithicBlobUploadURL(endpoint, location, digest)
require.Nil(t, err)
assert.Equal(t, expected, url)
// relative URL
location = "/v2/library/golang/blobs/uploads/c9f84fd7-0198-43e3-80a7-dd13771cd7f0?_state=GabyCujPu0dpxiY8yYZTq"
expected = endpoint + location + "&digest=" + digest
url, err = buildMonolithicBlobUploadURL(endpoint, location, digest)
require.Nil(t, err)
assert.Equal(t, expected, url)
}
func TestBuildMountBlobURL(t *testing.T) {
endpoint := "http://192.169.0.1"
repoName := "library/hello-world"
digest := "sha256:ef15416724f6e2d5d5b422dc5105add931c1f2a45959cd4993e75e47957b3b55"
from := "library/hi-world"
expected := fmt.Sprintf("%s/v2/%s/blobs/uploads/?mount=%s&from=%s", endpoint, repoName, digest, from)
actual := buildMountBlobURL(endpoint, repoName, digest, from)
assert.Equal(t, expected, actual)
}
func TestMountBlob(t *testing.T) {
mountHandler := func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusAccepted)
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "POST",
Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/", repository),
Handler: mountHandler,
})
defer server.Close()
client, err := newRepository(server.URL)
if err != nil {
t.Fatalf("failed to create client for repository: %v", err)
}
if err = client.MountBlob(digest, "library/hi-world"); err != nil {
t.Fatalf("failed to mount blob: %v", err)
}
}

43
src/core/api/quota/registry/registry.go
View File

@ -15,23 +15,19 @@
package registry package registry
import ( import (
"strings"
"sync"
"time"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota" common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/core/api" "github.com/goharbor/harbor/src/core/api"
quota "github.com/goharbor/harbor/src/core/api/quota" quota "github.com/goharbor/harbor/src/core/api/quota"
"github.com/goharbor/harbor/src/core/promgr" "github.com/goharbor/harbor/src/core/promgr"
coreutils "github.com/goharbor/harbor/src/core/utils" "github.com/goharbor/harbor/src/pkg/registry"
"github.com/pkg/errors" "github.com/pkg/errors"
"strings"
"sync"
"time"
) )
// Migrator ... // Migrator ...
@ -60,7 +56,7 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
err error err error
) )
reposInRegistry, err := api.Catalog() reposInRegistry, err := registry.Cli.Catalog()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -392,11 +388,7 @@ func infoOfProject(project string, repoList []string) (quota.ProjectInfo, error)
} }
func infoOfRepo(pid int64, repo string) (quota.RepoData, error) { func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
repoClient, err := coreutils.NewRepositoryClientForUI("harbor-core", repo) tags, err := registry.Cli.ListTags(repo)
if err != nil {
return quota.RepoData{}, err
}
tags, err := repoClient.ListTag()
if err != nil { if err != nil {
return quota.RepoData{}, err return quota.RepoData{}, err
} }
@ -405,11 +397,7 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
var blobs []*models.Blob var blobs []*models.Blob
for _, tag := range tags { for _, tag := range tags {
_, mediaType, payload, err := repoClient.PullManifest(tag, []string{ manifest, digest, err := registry.Cli.PullManifest(repo, tag)
schema1.MediaTypeManifest,
schema1.MediaTypeSignedManifest,
schema2.MediaTypeManifest,
})
if err != nil { if err != nil {
log.Error(err) log.Error(err)
// To workaround issue: https://github.com/goharbor/harbor/issues/9299, just log the error and do not raise it. // To workaround issue: https://github.com/goharbor/harbor/issues/9299, just log the error and do not raise it.
@ -417,28 +405,27 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
// User still can view there images with size 0 in harbor. // User still can view there images with size 0 in harbor.
continue continue
} }
manifest, desc, err := registry.UnMarshal(mediaType, payload) mediaType, payload, err := manifest.Payload()
if err != nil { if err != nil {
log.Error(err)
return quota.RepoData{}, err return quota.RepoData{}, err
} }
// self // self
afnb := &models.ArtifactAndBlob{ afnb := &models.ArtifactAndBlob{
DigestAF: desc.Digest.String(), DigestAF: digest,
DigestBlob: desc.Digest.String(), DigestBlob: digest,
} }
afnbs = append(afnbs, afnb) afnbs = append(afnbs, afnb)
// add manifest as a blob. // add manifest as a blob.
blob := &models.Blob{ blob := &models.Blob{
Digest: desc.Digest.String(), Digest: digest,
ContentType: desc.MediaType, ContentType: mediaType,
Size: desc.Size, Size: int64(len(payload)),
CreationTime: time.Now(), CreationTime: time.Now(),
} }
blobs = append(blobs, blob) blobs = append(blobs, blob)
for _, layer := range manifest.References() { for _, layer := range manifest.References() {
afnb := &models.ArtifactAndBlob{ afnb := &models.ArtifactAndBlob{
DigestAF: desc.Digest.String(), DigestAF: digest,
DigestBlob: layer.Digest.String(), DigestBlob: layer.Digest.String(),
} }
afnbs = append(afnbs, afnb) afnbs = append(afnbs, afnb)
@ -454,7 +441,7 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
PID: pid, PID: pid,
Repo: repo, Repo: repo,
Tag: tag, Tag: tag,
Digest: desc.Digest.String(), Digest: digest,
Kind: "Docker-Image", Kind: "Docker-Image",
CreationTime: time.Now(), CreationTime: time.Now(),
} }

15
src/core/api/scan.go
View File

@ -15,6 +15,7 @@
package api package api
import ( import (
"github.com/goharbor/harbor/src/pkg/registry"
"net/http" "net/http"
"strconv" "strconv"
@ -22,7 +23,6 @@ import (
"github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/errs" "github.com/goharbor/harbor/src/pkg/scan/errs"
"github.com/goharbor/harbor/src/pkg/scan/report" "github.com/goharbor/harbor/src/pkg/scan/report"
@ -192,20 +192,15 @@ func (sa *ScanAPI) Log() {
// TODO: This can be removed if the registry access interface is ready. // TODO: This can be removed if the registry access interface is ready.
type digestGetter func(repo, tag string, username string) (string, error) type digestGetter func(repo, tag string, username string) (string, error)
// TODO this method should be reconsidered as the tags are stored in database
// TODO rather than in registry
func getDigest(repo, tag string, username string) (string, error) { func getDigest(repo, tag string, username string) (string, error) {
client, err := coreutils.NewRepositoryClientForUI(username, repo) exist, digest, err := registry.Cli.ManifestExist(repo, tag)
if err != nil { if err != nil {
return "", err return "", err
} }
if !exist {
digest, exists, err := client.ManifestExist(tag)
if err != nil {
return "", err
}
if !exists {
return "", errors.Errorf("tag %s does exist", tag) return "", errors.Errorf("tag %s does exist", tag)
} }
return digest, nil return digest, nil
} }

22
src/core/api/search.go
View File

@ -23,7 +23,6 @@ import (
"github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/config"
coreutils "github.com/goharbor/harbor/src/core/utils"
"k8s.io/helm/cmd/helm/search" "k8s.io/helm/cmd/helm/search"
) )
@ -180,27 +179,10 @@ func filterRepositories(projects []*models.Project, keyword string) (
entry["project_public"] = project.IsPublic() entry["project_public"] = project.IsPublic()
entry["pull_count"] = repository.PullCount entry["pull_count"] = repository.PullCount
tags, err := getTags(repository.Name) // TODO populate artifact count
if err != nil { // entry["tags_count"] = len(tags)
return nil, err
}
entry["tags_count"] = len(tags)
result = append(result, entry) result = append(result, entry)
} }
return result, nil return result, nil
} }
func getTags(repository string) ([]string, error) {
client, err := coreutils.NewRepositoryClientForUI("harbor-core", repository)
if err != nil {
return nil, err
}
tags, err := client.ListTag()
if err != nil {
return nil, err
}
return tags, nil
}

63
src/core/api/utils.go
View File

@ -1,63 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"net/http"
"strings"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
)
// Catalog ...
func Catalog() ([]string, error) {
repositories := []string{}
rc, err := initRegistryClient()
if err != nil {
return repositories, err
}
repositories, err = rc.Catalog()
if err != nil {
return repositories, err
}
return repositories, nil
}
func initRegistryClient() (r *registry.Registry, err error) {
endpoint, err := config.RegistryURL()
if err != nil {
return nil, err
}
addr := endpoint
if strings.Contains(endpoint, "://") {
addr = strings.Split(endpoint, "://")[1]
}
if err := utils.TestTCPConn(addr, 60, 2); err != nil {
return nil, err
}
authorizer := auth.DefaultBasicAuthorizer()
return registry.NewRegistry(endpoint, &http.Client{
Transport: registry.NewTransport(registry.GetHTTPTransport(), authorizer),
})
}

45
src/core/utils/utils.go
View File

@ -1,45 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package utils contains methods to support security, cache, and webhook functions.
package utils
import (
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
"net/http"
)
// NewRepositoryClientForUI creates a repository client that can only be used to
// access the internal registry
func NewRepositoryClientForUI(username, repository string) (*registry.Repository, error) {
endpoint, err := config.RegistryURL()
if err != nil {
return nil, err
}
return newRepositoryClient(endpoint, username, repository)
}
func newRepositoryClient(endpoint, username, repository string) (*registry.Repository, error) {
uam := &auth.UserAgentModifier{
UserAgent: "harbor-registry-client",
}
authorizer := auth.DefaultBasicAuthorizer()
transport := registry.NewTransport(http.DefaultTransport, authorizer, uam)
client := &http.Client{
Transport: transport,
}
return registry.NewRepository(repository, endpoint, client)
}

12
src/common/utils/registry/auth/nullauthorizer.go → src/internal/authorizer.go
View File

@ -12,13 +12,9 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package auth package internal
import "net/http" import "github.com/goharbor/harbor/src/common/http/modifier"
type nullAuthorizer struct{} // Authorizer authorizes the request
type Authorizer modifier.Modifier
func (n *nullAuthorizer) Modify(req *http.Request) error {
// do nothing
return nil
}

89
src/internal/link.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"fmt"
"strings"
)
// Link defines the model that describes the HTTP link header
type Link struct {
URL string
Rel string
Attrs map[string]string
}
// String returns the string representation of a link
func (l *Link) String() string {
s := fmt.Sprintf("<%s>", l.URL)
if len(l.Rel) > 0 {
s = fmt.Sprintf(`%s; rel="%s"`, s, l.Rel)
}
for key, value := range l.Attrs {
s = fmt.Sprintf(`%s; %s="%s"`, s, key, value)
}
return s
}
// Links is a link object array
type Links []*Link
// String returns the string representation of links
func (l Links) String() string {
var strs []string
for _, link := range l {
strs = append(strs, link.String())
}
return strings.Join(strs, " , ")
}
// ParseLinks parses the link header into Links
// e.g. <http://example.com/TheBook/chapter2>; rel="previous"; title="previous chapter" , <http://example.com/TheBook/chapter4>; rel="next"; title="next chapter"
func ParseLinks(str string) Links {
var links Links
for _, lk := range strings.Split(str, ",") {
link := &Link{
Attrs: map[string]string{},
}
for _, attr := range strings.Split(lk, ";") {
attr = strings.TrimSpace(attr)
if len(attr) == 0 {
continue
}
if attr[0] == '<' && attr[len(attr)-1] == '>' {
link.URL = attr[1 : len(attr)-1]
continue
}
parts := strings.SplitN(attr, "=", 2)
key := parts[0]
value := ""
if len(parts) == 2 {
value = strings.Trim(parts[1], `"`)
}
if key == "rel" {
link.Rel = value
} else {
link.Attrs[key] = value
}
}
if len(link.URL) == 0 {
continue
}
links = append(links, link)
}
return links
}

37
src/internal/link_test.go Normal file
View File

@ -0,0 +1,37 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
func TestMethodsOfLink(t *testing.T) {
str := `<http://example.com/TheBook/chapter2>; rel="previous"; title="previous chapter" , <http://example.com/TheBook/chapter4>; rel="next"; title="next chapter"`
links := ParseLinks(str)
require.Len(t, links, 2)
assert.Equal(t, "http://example.com/TheBook/chapter2", links[0].URL)
assert.Equal(t, "previous", links[0].Rel)
assert.Equal(t, "previous chapter", links[0].Attrs["title"])
assert.Equal(t, "http://example.com/TheBook/chapter4", links[1].URL)
assert.Equal(t, "next", links[1].Rel)
assert.Equal(t, "previous", links[0].Rel)
assert.Equal(t, "next chapter", links[1].Attrs["title"])
s := links.String()
assert.Equal(t, str, s)
}

42
src/common/utils/registry/auth/basicauthorizer_test.go → src/internal/transport.go
View File

@ -12,28 +12,32 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package auth package internal
import ( import (
"github.com/stretchr/testify/assert" "crypto/tls"
"net/http" "net/http"
"os"
"testing"
) )
func TestDefaultBasicAuthorizer(t *testing.T) { var (
os.Setenv("REGISTRY_CREDENTIAL_USERNAME", "testuser") secureHTTPTransport = &http.Transport{
os.Setenv("REGISTRY_CREDENTIAL_PASSWORD", "testpassword") Proxy: http.ProxyFromEnvironment,
defer func() { TLSClientConfig: &tls.Config{
os.Unsetenv("REGISTRY_CREDENTIAL_USERNAME") InsecureSkipVerify: false,
os.Unsetenv("REGISTRY_CREDENTIAL_PASSWORD") },
}() }
req, _ := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil) insecureHTTPTransport = &http.Transport{
a := DefaultBasicAuthorizer() Proxy: http.ProxyFromEnvironment,
err := a.Modify(req) TLSClientConfig: &tls.Config{
assert.Nil(t, err) InsecureSkipVerify: true,
u, p, ok := req.BasicAuth() },
assert.True(t, ok) }
assert.Equal(t, "testuser", u) )
assert.Equal(t, "testpassword", p)
// GetHTTPTransport returns the HTTP transport based on insecure configuration
func GetHTTPTransport(insecure ...bool) *http.Transport {
if len(insecure) > 0 && insecure[0] {
return insecureHTTPTransport
}
return secureHTTPTransport
} }

4
src/jobservice/job/impl/replication/scheduler.go
View File

@ -16,12 +16,12 @@ package replication
import ( import (
"fmt" "fmt"
"github.com/goharbor/harbor/src/internal"
"net/http" "net/http"
"os" "os"
common_http "github.com/goharbor/harbor/src/common/http" common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier/auth" "github.com/goharbor/harbor/src/common/http/modifier/auth"
reg "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
) )
@ -61,7 +61,7 @@ func (s *Scheduler) Run(ctx job.Context, params job.Parameters) error {
policyID := (int64)(params["policy_id"].(float64)) policyID := (int64)(params["policy_id"].(float64))
cred := auth.NewSecretAuthorizer(os.Getenv("JOBSERVICE_SECRET")) cred := auth.NewSecretAuthorizer(os.Getenv("JOBSERVICE_SECRET"))
client := common_http.NewClient(&http.Client{ client := common_http.NewClient(&http.Client{
Transport: reg.GetHTTPTransport(true), Transport: internal.GetHTTPTransport(true),
}, cred) }, cred)
if err := client.Post(url, struct { if err := client.Post(url, struct {
PolicyID int64 `json:"policy_id"` PolicyID int64 `json:"policy_id"`

56
src/jobservice/job/impl/utils/utils.go
View File

@ -1,56 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"fmt"
"net/http"
"os"
"sync"
httpauth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils/registry"
)
var coreClient *http.Client
var mutex = &sync.Mutex{}
// UserAgentModifier adds the "User-Agent" header to the request
type UserAgentModifier struct {
UserAgent string
}
// Modify adds user-agent header to the request
func (u *UserAgentModifier) Modify(req *http.Request) error {
req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.UserAgent)
return nil
}
// GetClient returns the HTTP client that will attach jobservce secret to the request, which can be used for
// accessing Harbor's Core Service.
// This function returns error if the secret of Job service is not set.
func GetClient() (*http.Client, error) {
mutex.Lock()
defer mutex.Unlock()
if coreClient == nil {
secret := os.Getenv("JOBSERVICE_SECRET")
if len(secret) == 0 {
return nil, fmt.Errorf("unable to load secret for job service")
}
modifier := httpauth.NewSecretAuthorizer(secret)
coreClient = &http.Client{Transport: registry.NewTransport(&http.Transport{}, modifier)}
}
return coreClient, nil
}

45
src/jobservice/job/impl/utils/utils_test.go
View File

@ -1,45 +0,0 @@
// Copyright 2018 The Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/goharbor/harbor/src/common/secret"
"github.com/stretchr/testify/assert"
)
func TestGetClient(t *testing.T) {
assert := assert.New(t)
os.Setenv("", "")
_, err := GetClient()
assert.NotNil(err, "Error should be thrown if secret is not set")
os.Setenv("JOBSERVICE_SECRET", "thesecret")
c, err := GetClient()
assert.Nil(err)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
v := r.Header.Get("Authorization")
assert.Equal(secret.HeaderPrefix+"thesecret", v)
}))
defer ts.Close()
c.Get(ts.URL)
os.Setenv("", "")
_, err = GetClient()
assert.Nil(err, "Error should be nil once client is initialized")
}

45
src/common/utils/registry/auth/authorizer.go → src/pkg/registry/auth/authorizer.go
View File

@ -18,6 +18,10 @@ import (
"fmt" "fmt"
"github.com/docker/distribution/registry/client/auth/challenge" "github.com/docker/distribution/registry/client/auth/challenge"
"github.com/goharbor/harbor/src/common/http/modifier" "github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/internal"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
"github.com/goharbor/harbor/src/pkg/registry/auth/null"
"net/http" "net/http"
"net/url" "net/url"
"strings" "strings"
@ -25,18 +29,14 @@ import (
) )
// NewAuthorizer creates an authorizer that can handle different auth schemes // NewAuthorizer creates an authorizer that can handle different auth schemes
func NewAuthorizer(credential Credential, client ...*http.Client) modifier.Modifier { func NewAuthorizer(username, password string, insecure bool) internal.Authorizer {
authorizer := &authorizer{ return &authorizer{
credential: credential, username: username,
password: password,
client: &http.Client{
Transport: internal.GetHTTPTransport(insecure),
},
} }
if len(client) > 0 {
authorizer.client = client[0]
}
if authorizer.client == nil {
authorizer.client = http.DefaultClient
}
return authorizer
} }
// authorizer authorizes the request with the provided credential. // authorizer authorizes the request with the provided credential.
@ -44,15 +44,16 @@ func NewAuthorizer(credential Credential, client ...*http.Client) modifier.Modif
// different underlying authorizers to do the auth work // different underlying authorizers to do the auth work
type authorizer struct { type authorizer struct {
sync.Mutex sync.Mutex
username string
password string
client *http.Client client *http.Client
url *url.URL // registry URL url *url.URL // registry URL
authorizer modifier.Modifier // the underlying authorizer authorizer modifier.Modifier // the underlying authorizer
credential Credential
} }
func (a *authorizer) Modify(req *http.Request) error { func (a *authorizer) Modify(req *http.Request) error {
// Nil URL means this is the first time the authorizer is called // Nil URL means this is the first time the authorizer is called
// Try to connect to the registry and determine the auth method // Try to connect to the registry and determine the auth scheme
if a.url == nil { if a.url == nil {
// to avoid concurrent issue // to avoid concurrent issue
a.Lock() a.Lock()
@ -83,25 +84,25 @@ func (a *authorizer) initialize(u *url.URL) error {
if err != nil { if err != nil {
return err return err
} }
challenges := ParseChallengeFromResponse(resp)
challenges := challenge.ResponseChallenges(resp)
// no challenge, mean no auth // no challenge, mean no auth
if len(challenges) == 0 { if len(challenges) == 0 {
a.authorizer = &nullAuthorizer{} a.authorizer = null.NewAuthorizer()
return nil return nil
} }
cm := map[string]challenge.Challenge{} cm := map[string]challenge.Challenge{}
for _, challenge := range challenges { for _, challenge := range challenges {
cm[challenge.Scheme] = challenge cm[challenge.Scheme] = challenge
} }
if _, exist := cm["basic"]; exist { if challenge, exist := cm["bearer"]; exist {
a.authorizer = a.credential a.authorizer = bearer.NewAuthorizer(challenge.Parameters["realm"],
challenge.Parameters["service"], basic.NewAuthorizer(a.username, a.password),
a.client.Transport.(*http.Transport))
return nil return nil
} }
if _, exist := cm["basic"]; exist {
if _, exist := cm["bearer"]; exist { a.authorizer = basic.NewAuthorizer(a.username, a.password)
// TODO clean up the code of "StandardTokenAuthorizer"
// TODO Currently, the checking of auth scheme is done twice, this can be avoided
a.authorizer = NewStandardTokenAuthorizer(a.client, a.credential)
return nil return nil
} }
return fmt.Errorf("unspported auth scheme: %v", challenges) return fmt.Errorf("unspported auth scheme: %v", challenges)

41
src/common/utils/registry/auth/credential_test.go → src/pkg/registry/auth/basic/authorizer.go
View File

@ -12,32 +12,29 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package auth package basic
import ( import (
"github.com/goharbor/harbor/src/internal"
"net/http" "net/http"
"testing"
) )
func TestAddAuthorizationOfBasicAuthCredential(t *testing.T) { // NewAuthorizer return a basic authorizer
cred := NewBasicAuthCredential("usr", "pwd") func NewAuthorizer(username, password string) internal.Authorizer {
req, err := http.NewRequest("GET", "http://example.com", nil) return &authorizer{
if err != nil { username: username,
t.Fatalf("failed to create request: %v", err) password: password,
}
cred.Modify(req)
usr, pwd, ok := req.BasicAuth()
if !ok {
t.Fatal("basic auth not found")
}
if usr != "usr" {
t.Errorf("unexpected username: %s != usr", usr)
}
if pwd != "pwd" {
t.Errorf("unexpected password: %s != pwd", pwd)
} }
} }
type authorizer struct {
username string
password string
}
func (a *authorizer) Modify(req *http.Request) error {
if len(a.username) > 0 {
req.SetBasicAuth(a.username, a.password)
}
return nil
}

19
src/common/utils/registry/manifest.go → src/pkg/registry/auth/basic/authorizer_test.go
View File

@ -12,13 +12,22 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package registry package basic
import ( import (
"github.com/docker/distribution" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"testing"
) )
// UnMarshal converts []byte to be distribution.Manifest func TestModify(t *testing.T) {
func UnMarshal(mediaType string, data []byte) (distribution.Manifest, distribution.Descriptor, error) { authorizer := NewAuthorizer("u", "p")
return distribution.UnmarshalManifest(mediaType, data) req, _ := http.NewRequest(http.MethodGet, "", nil)
err := authorizer.Modify(req)
require.Nil(t, err)
u, p, ok := req.BasicAuth()
require.True(t, ok)
assert.Equal(t, "u", u)
assert.Equal(t, "p", p)
} }

146
src/pkg/registry/auth/bearer/authorizer.go Normal file
View File

@ -0,0 +1,146 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/internal"
ierror "github.com/goharbor/harbor/src/internal/error"
"io/ioutil"
"net/http"
"net/url"
)
const (
cacheCapacity = 100
)
// NewAuthorizer return a bearer token authorizer
// The parameter "a" is an authorizer used to fetch the token
func NewAuthorizer(realm, service string, a internal.Authorizer, transport ...*http.Transport) internal.Authorizer {
authorizer := &authorizer{
realm: realm,
service: service,
authorizer: a,
cache: newCache(cacheCapacity),
}
tp := http.DefaultTransport
if len(transport) > 0 && transport[0] != nil {
tp = transport[0]
}
authorizer.client = &http.Client{Transport: tp}
return authorizer
}
type authorizer struct {
realm string
service string
authorizer internal.Authorizer
cache *cache
client *http.Client
}
func (a *authorizer) Modify(req *http.Request) error {
// parse scopes from request
scopes := parseScopes(req)
// get token
token, err := a.getToken(scopes)
if err != nil {
return err
}
// set authorization header
if token != nil && len(token.Token) > 0 {
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token.Token))
}
return nil
}
func (a *authorizer) getToken(scopes []*scope) (*token, error) {
// get token from cache first
token := a.cache.get(scopes)
if token != nil {
return token, nil
}
// get no token from cache, fetch it from the token service
token, err := a.fetchToken(scopes)
if err != nil {
return nil, err
}
// set the token into the cache
a.cache.set(scopes, token)
return token, nil
}
type token struct {
Token string `json:"token"`
AccessToken string `json:"access_token"` // the token returned by azure container registry is called "access_token"
ExpiresIn int `json:"expires_in"`
IssuedAt string `json:"issued_at"`
}
func (a *authorizer) fetchToken(scopes []*scope) (*token, error) {
url, err := url.Parse(a.realm)
if err != nil {
return nil, err
}
query := url.Query()
query.Add("service", a.service)
for _, scope := range scopes {
query.Add("scope", scope.String())
}
url.RawQuery = query.Encode()
req, err := http.NewRequest(http.MethodGet, url.String(), nil)
if err != nil {
return nil, err
}
if a.authorizer != nil {
if err = a.authorizer.Modify(req); err != nil {
return nil, err
}
}
resp, err := a.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
message := fmt.Sprintf("http status code: %d, body: %s", resp.StatusCode, string(body))
code := ierror.GeneralCode
switch resp.StatusCode {
case http.StatusUnauthorized:
code = ierror.UnAuthorizedCode
case http.StatusForbidden:
code = ierror.ForbiddenCode
}
return nil, ierror.New(nil).WithCode(code).
WithMessage(message)
}
token := &token{}
if err = json.Unmarshal(body, token); err != nil {
return nil, err
}
return token, nil
}

55
src/pkg/registry/auth/bearer/authorizer_test.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"fmt"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"net/http/httptest"
"testing"
)
func TestModify(t *testing.T) {
token := "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IlBZWU86VEVXVTpWN0pIOjI2SlY6QVFUWjpMSkMzOlNYVko6WEdIQTozNEYyOjJMQVE6WlJNSzpaN1E2In0.eyJpc3MiOiJhdXRoLmRvY2tlci5jb20iLCJzdWIiOiJqbGhhd24iLCJhdWQiOiJyZWdpc3RyeS5kb2NrZXIuY29tIiwiZXhwIjoxNDE1Mzg3MzE1LCJuYmYiOjE0MTUzODcwMTUsImlhdCI6MTQxNTM4NzAxNSwianRpIjoidFlKQ08xYzZjbnl5N2tBbjBjN3JLUGdiVjFIMWJGd3MiLCJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InNhbWFsYmEvbXktYXBwIiwiYWN0aW9ucyI6WyJwdXNoIl19XX0.QhflHPfbd6eVF4lM9bwYpFZIV0PfikbyXuLx959ykRTBpe3CYnzs6YBK8FToVb5R47920PVLrh8zuLzdCr9t3w"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
u, p, ok := r.BasicAuth()
if !ok || u != "username" || p != "password" {
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Write([]byte(fmt.Sprintf(`{"token": "%s", "expires_in": 3600,"issued_at": "2009-11-10T23:00:00Z"}`, token)))
}))
defer server.Close()
// invalid credential
a := basic.NewAuthorizer("username", "invalid_password")
authorizer := NewAuthorizer(server.URL, "service", a)
req, _ := http.NewRequest(http.MethodGet, server.URL, nil)
err := authorizer.Modify(req)
require.NotNil(t, err)
assert.True(t, ierror.IsErr(err, ierror.UnAuthorizedCode))
// valid credential
a = basic.NewAuthorizer("username", "password")
authorizer = NewAuthorizer(server.URL, "service", a)
req, _ = http.NewRequest(http.MethodGet, server.URL, nil)
err = authorizer.Modify(req)
require.Nil(t, err)
assert.Equal(t, fmt.Sprintf("Bearer %s", token), req.Header.Get("Authorization"))
}

93
src/pkg/registry/auth/bearer/cache.go Normal file
View File

@ -0,0 +1,93 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"github.com/goharbor/harbor/src/common/utils/log"
"strings"
"sync"
"time"
)
func newCache(capacity int) *cache {
return &cache{
latency: 10,
capacity: capacity,
cache: map[string]*token{},
}
}
type cache struct {
sync.RWMutex
latency int // second, the network latency in case that when the token is checked it doesn't expire but it does when used
capacity int // the capacity of the cache map
cache map[string]*token
}
func (c *cache) get(scopes []*scope) *token {
c.RLock()
defer c.RUnlock()
return c.cache[c.key(scopes)]
}
func (c *cache) set(scopes []*scope, token *token) {
c.Lock()
defer c.Unlock()
// exceed the capacity, empty some elements: all expired token will be removed,
// if no expired token, move the earliest one
if len(c.cache) >= c.capacity {
now := time.Now().UTC()
var candidates []string
var earliestKey string
var earliestExpireTime time.Time
for key, value := range c.cache {
// parse error
issueAt, err := time.Parse(time.RFC3339, value.IssuedAt)
if err != nil {
log.Errorf("failed to parse the issued at time of token %s: %v", token.IssuedAt, err)
candidates = append(candidates, key)
continue
}
expireAt := issueAt.Add(time.Duration(value.ExpiresIn-c.latency) * time.Second)
// expired
if expireAt.Before(now) {
candidates = append(candidates, key)
continue
}
// doesn't expired
if len(earliestKey) == 0 || expireAt.Before(earliestExpireTime) {
earliestKey = key
earliestExpireTime = expireAt
continue
}
}
if len(candidates) == 0 {
candidates = append(candidates, earliestKey)
}
for _, candidate := range candidates {
delete(c.cache, candidate)
}
}
c.cache[c.key(scopes)] = token
}
func (c *cache) key(scopes []*scope) string {
var strs []string
for _, scope := range scopes {
strs = append(strs, scope.String())
}
return strings.Join(strs, "#")
}

153
src/pkg/registry/auth/bearer/cache_test.go Normal file
View File

@ -0,0 +1,153 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"github.com/stretchr/testify/suite"
"testing"
"time"
)
type cacheTestSuite struct {
suite.Suite
cache *cache
}
func (c *cacheTestSuite) SetupTest() {
c.cache = newCache(2)
}
func (c *cacheTestSuite) TestKey() {
// nil scopes
var scopes []*scope
key := c.cache.key(scopes)
c.Equal("", key)
// single one scope
scopes = []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world",
Actions: []string{scopeActionPull, scopeActionPush},
},
}
key = c.cache.key(scopes)
c.Equal("repository:library/hello-world:pull,push", key)
// multiple scopes
scopes = []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world",
Actions: []string{scopeActionPull, scopeActionPush},
},
{
Type: scopeTypeRepository,
Name: "library/alpine",
Actions: []string{scopeActionPull},
},
}
key = c.cache.key(scopes)
c.Equal("repository:library/hello-world:pull,push#repository:library/alpine:pull", key)
}
func (c *cacheTestSuite) TestGet() {
token := &token{
Token: "token",
}
c.cache.set(nil, token)
tk := c.cache.get(nil)
c.Require().NotNil(tk)
c.Equal(token.Token, tk.Token)
}
func (c *cacheTestSuite) TestSet() {
now := time.Now()
// set the first token
scope1 := []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world01",
Actions: []string{scopeActionPull},
},
}
token1 := &token{
Token: "token1",
ExpiresIn: 10,
IssuedAt: now.Format(time.RFC3339),
}
c.cache.set(scope1, token1)
c.Len(c.cache.cache, 1)
// set the second token
scope2 := []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world02",
Actions: []string{scopeActionPull},
},
}
token2 := &token{
Token: "token2",
ExpiresIn: 15,
IssuedAt: now.Format(time.RFC3339),
}
c.cache.set(scope2, token2)
c.Len(c.cache.cache, 2)
// set the third token
// as the capacity is 2 and token1 is expired, token1 should be replaced by token3
scope3 := []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world03",
Actions: []string{scopeActionPull},
},
}
token3 := &token{
Token: "token3",
ExpiresIn: 15,
IssuedAt: now.Format(time.RFC3339),
}
c.cache.set(scope3, token3)
c.Require().Len(c.cache.cache, 2)
c.Require().NotNil(c.cache.get(scope2))
c.Require().NotNil(c.cache.get(scope3))
// sleep 5 seconds to make sure all tokens expire
time.Sleep(5 * time.Second)
// set the fourth token
// as the capacity is 2 and both token2 and token3 are expired, token2 and token3 should be removed
scope4 := []*scope{
{
Type: scopeTypeRepository,
Name: "library/hello-world04",
Actions: []string{scopeActionPull},
},
}
token4 := &token{
Token: "token4",
ExpiresIn: 20,
IssuedAt: now.Format(time.RFC3339),
}
c.cache.set(scope4, token4)
c.Require().Len(c.cache.cache, 1)
c.Require().NotNil(c.cache.get(scope4))
}
func TestCacheTestSuite(t *testing.T) {
suite.Run(t, &cacheTestSuite{})
}

107
src/pkg/registry/auth/bearer/scope.go Normal file
View File

@ -0,0 +1,107 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"fmt"
"github.com/docker/distribution/reference"
"net/http"
"regexp"
"strings"
)
const (
scopeTypeRegistry = "registry"
scopeTypeRepository = "repository"
scopeActionPull = "pull"
scopeActionPush = "push"
scopeActionAll = "*"
)
var (
catalog = regexp.MustCompile("/v2/_catalog$")
tag = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/tags/list")
manifest = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/manifests/(" + reference.TagRegexp.String() + "|" + reference.DigestRegexp.String() + ")")
blob = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String())
blobUpload = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/uploads")
)
type scope struct {
Type string
Name string
Actions []string
}
func (s *scope) String() string {
return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
}
func parseScopes(req *http.Request) []*scope {
path := strings.TrimRight(req.URL.Path, "/")
var scopes []*scope
repository := ""
// manifest
if subs := manifest.FindStringSubmatch(path); len(subs) >= 2 {
// manifest
repository = subs[1]
} else if subs := blob.FindStringSubmatch(path); len(subs) >= 2 {
// blob
repository = subs[1]
} else if subs := blobUpload.FindStringSubmatch(path); len(subs) >= 2 {
// blob upload
repository = subs[1]
// blob mount
from := req.URL.Query().Get("from")
if len(from) > 0 {
scopes = append(scopes, &scope{
Type: scopeTypeRepository,
Name: from,
Actions: []string{scopeActionPull},
})
}
} else if subs := tag.FindStringSubmatch(path); len(subs) >= 2 {
// tag
repository = subs[1]
}
if len(repository) > 0 {
scp := &scope{
Type: scopeTypeRepository,
Name: repository,
}
switch req.Method {
case http.MethodGet, http.MethodHead:
scp.Actions = []string{scopeActionPull}
case http.MethodPost, http.MethodPut, http.MethodPatch:
scp.Actions = []string{scopeActionPull, scopeActionPush}
case http.MethodDelete:
scp.Actions = []string{scopeActionAll}
}
scopes = append(scopes, scp)
return scopes
}
// catalog
if catalog.MatchString(path) {
return []*scope{
{
Type: scopeTypeRegistry,
Name: "catalog",
Actions: []string{scopeActionAll},
}}
}
// base or no match, return nil
return nil
}

112
src/pkg/registry/auth/bearer/scope_test.go Normal file
View File

@ -0,0 +1,112 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bearer
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"testing"
)
func TestStringOfScope(t *testing.T) {
scope := &scope{
Type: scopeTypeRepository,
Name: "library/hello-world",
Actions: []string{scopeActionPull, scopeActionPush},
}
assert.Equal(t, "repository:library/hello-world:pull,push", scope.String())
}
func TestParseScopes(t *testing.T) {
// base
req, _ := http.NewRequest(http.MethodGet, "/v2/", nil)
scopes := parseScopes(req)
require.Nil(t, scopes)
// catalog
req, _ = http.NewRequest(http.MethodGet, "/v2/_catalog", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRegistry, scopes[0].Type)
assert.Equal(t, "catalog", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionAll, scopes[0].Actions[0])
// list tags
req, _ = http.NewRequest(http.MethodGet, "/v2/library/hello-world/tags/list", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionPull, scopes[0].Actions[0])
// get manifest by tag
req, _ = http.NewRequest(http.MethodGet, "/v2/library/hello-world/manifests/latest", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionPull, scopes[0].Actions[0])
// get manifest by digest
req, _ = http.NewRequest(http.MethodGet, "/v2/library/hello-world/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionPull, scopes[0].Actions[0])
// push manifest
req, _ = http.NewRequest(http.MethodPut, "/v2/library/hello-world/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 2)
assert.Equal(t, scopeActionPull, scopes[0].Actions[0])
assert.Equal(t, scopeActionPush, scopes[0].Actions[1])
// delete manifest
req, _ = http.NewRequest(http.MethodDelete, "/v2/library/hello-world/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 1)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/hello-world", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionAll, scopes[0].Actions[0])
// mount blob
req, _ = http.NewRequest(http.MethodPost, "/v2/library/hello-world/blobs/uploads/?mount=sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae&from=library/alpine", nil)
scopes = parseScopes(req)
require.Len(t, scopes, 2)
assert.Equal(t, scopeTypeRepository, scopes[0].Type)
assert.Equal(t, "library/alpine", scopes[0].Name)
require.Len(t, scopes[0].Actions, 1)
assert.Equal(t, scopeActionPull, scopes[1].Actions[0])
assert.Equal(t, scopeTypeRepository, scopes[1].Type)
assert.Equal(t, "library/hello-world", scopes[1].Name)
require.Len(t, scopes[1].Actions, 2)
assert.Equal(t, scopeActionPull, scopes[1].Actions[0])
assert.Equal(t, scopeActionPush, scopes[1].Actions[1])
// no match
req, _ = http.NewRequest(http.MethodPost, "/api/others", nil)
scopes = parseScopes(req)
require.Nil(t, scopes)
}

19
src/common/utils/registry/auth/challenge.go → src/pkg/registry/auth/null/authorizer.go
View File

@ -12,17 +12,22 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package auth package null
import ( import (
"github.com/goharbor/harbor/src/internal"
"net/http" "net/http"
"github.com/docker/distribution/registry/client/auth/challenge"
) )
// ParseChallengeFromResponse ... // NewAuthorizer returns a null authorizer
func ParseChallengeFromResponse(resp *http.Response) []challenge.Challenge { func NewAuthorizer() internal.Authorizer {
challenges := challenge.ResponseChallenges(resp) return &authorizer{}
return challenges }
type authorizer struct{}
func (a *authorizer) Modify(req *http.Request) error {
// do nothing
return nil
} }

497
src/pkg/registry/client.go
View File

@ -15,26 +15,35 @@
package registry package registry
import ( import (
"bytes"
"encoding/json"
"fmt"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/manifestlist"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
// register oci manifest unmarshal function
_ "github.com/docker/distribution/manifest/ocischema"
"github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/internal"
ierror "github.com/goharbor/harbor/src/internal/error" ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/replication/util" "github.com/goharbor/harbor/src/pkg/registry/auth"
"github.com/opencontainers/go-digest"
v1 "github.com/opencontainers/image-spec/specs-go/v1" v1 "github.com/opencontainers/image-spec/specs-go/v1"
"net/http"
) )
// TODO we'll merge all registry related code into this package before releasing 2.0
var ( var (
// Cli is the global registry client instance, it targets to the backend docker registry // Cli is the global registry client instance, it targets to the backend docker registry
Cli = func() Client { Cli = func() Client {
url, _ := config.RegistryURL() url, _ := config.RegistryURL()
username, password := config.RegistryCredential() username, password := config.RegistryCredential()
return NewClient(url, true, username, password) return NewClient(url, username, password, true)
}() }()
accepts = []string{ accepts = []string{
@ -48,54 +57,378 @@ var (
// Client defines the methods that a registry client should implements // Client defines the methods that a registry client should implements
type Client interface { type Client interface {
// Ping the base API endpoint "/v2/"
Ping() (err error)
// Catalog the repositories
Catalog() (repositories []string, err error)
// ListTags lists the tags under the specified repository
ListTags(repository string) (tags []string, err error)
// ManifestExist checks the existence of the manifest
ManifestExist(repository, reference string) (exist bool, digest string, err error)
// PullManifest pulls the specified manifest
PullManifest(repository, reference string, acceptedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error)
// PushManifest pushes the specified manifest
PushManifest(repository, reference, mediaType string, payload []byte) (digest string, err error)
// DeleteManifest deletes the specified manifest. The "reference" can be "tag" or "digest"
DeleteManifest(repository, reference string) (err error)
// BlobExist checks the existence of the specified blob
BlobExist(repository, digest string) (exist bool, err error)
// PullBlob pulls the specified blob. The caller must close the returned "blob"
PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error)
// PushBlob pushes the specified blob
PushBlob(repository, digest string, size int64, blob io.Reader) error
// MountBlob mounts the blob from the source repository
MountBlob(srcRepository, digest, dstRepository string) (err error)
// DeleteBlob deletes the specified blob
DeleteBlob(repository, digest string) (err error)
// Copy the artifact from source repository to the destination. The "override" // Copy the artifact from source repository to the destination. The "override"
// is used to specify whether the destination artifact will be overridden if // is used to specify whether the destination artifact will be overridden if
// its name is same with source but digest isn't // its name is same with source but digest isn't
Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) (err error) Copy(srcRepository, srcReference, dstRepository, dstReference string, override bool) (err error)
// TODO defines other methods
} }
// NewClient creates a registry client based on the provided information // TODO TODO support HTTPS
// TODO support HTTPS
func NewClient(url string, insecure bool, username, password string) Client { // NewClient creates a registry client with the default authorizer which determines the auth scheme
transport := util.GetHTTPTransport(insecure) // of the registry automatically and calls the corresponding underlying authorizers(basic/bearer) to
authorizer := auth.NewAuthorizer(auth.NewBasicAuthCredential(username, password), // do the auth work. If a customized authorizer is needed, use "NewClientWithAuthorizer" instead
&http.Client{ func NewClient(url, username, password string, insecure bool) Client {
Transport: transport,
})
return &client{ return &client{
url: url, url: url,
authorizer: auth.NewAuthorizer(username, password, insecure),
client: &http.Client{ client: &http.Client{
Transport: registry.NewTransport(transport, authorizer), Transport: internal.GetHTTPTransport(insecure),
},
}
}
// NewClientWithAuthorizer creates a registry client with the provided authorizer
func NewClientWithAuthorizer(url string, authorizer internal.Authorizer, insecure bool) Client {
return &client{
url: url,
authorizer: authorizer,
client: &http.Client{
Transport: internal.GetHTTPTransport(insecure),
}, },
} }
} }
type client struct { type client struct {
url string url string
client *http.Client authorizer internal.Authorizer
client *http.Client
}
func (c *client) Ping() error {
req, err := http.NewRequest(http.MethodGet, buildPingURL(c.url), nil)
if err != nil {
return err
}
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *client) Catalog() ([]string, error) {
var repositories []string
url := buildCatalogURL(c.url)
for {
repos, next, err := c.catalog(url)
if err != nil {
return nil, err
}
repositories = append(repositories, repos...)
url = next
// no next page, end the loop
if len(url) == 0 {
break
}
// relative URL
if !strings.Contains(url, "://") {
url = c.url + url
}
}
return repositories, nil
}
func (c *client) catalog(url string) ([]string, string, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, "", err
}
resp, err := c.do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, "", err
}
repositories := struct {
Repositories []string `json:"repositories"`
}{}
if err := json.Unmarshal(body, &repositories); err != nil {
return nil, "", err
}
return repositories.Repositories, next(resp.Header.Get("Link")), nil
}
func (c *client) ListTags(repository string) ([]string, error) {
var tags []string
url := buildTagListURL(c.url, repository)
for {
tgs, next, err := c.listTags(url)
if err != nil {
return nil, err
}
tags = append(tags, tgs...)
url = next
// no next page, end the loop
if len(url) == 0 {
break
}
// relative URL
if !strings.Contains(url, "://") {
url = c.url + url
}
}
return tags, nil
}
func (c *client) listTags(url string) ([]string, string, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, "", err
}
resp, err := c.do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, "", err
}
tgs := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(body, &tgs); err != nil {
return nil, "", err
}
return tgs.Tags, next(resp.Header.Get("Link")), nil
}
func (c *client) ManifestExist(repository, reference string) (bool, string, error) {
req, err := http.NewRequest(http.MethodHead, buildManifestURL(c.url, repository, reference), nil)
if err != nil {
return false, "", err
}
for _, mediaType := range accepts {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := c.do(req)
if err != nil {
if ierror.IsErr(err, ierror.NotFoundCode) {
return false, "", nil
}
return false, "", err
}
defer resp.Body.Close()
return true, resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")), nil
}
func (c *client) PullManifest(repository, reference string, acceptedMediaTypes ...string) (
distribution.Manifest, string, error) {
req, err := http.NewRequest(http.MethodGet, buildManifestURL(c.url, repository, reference), nil)
if err != nil {
return nil, "", err
}
if len(acceptedMediaTypes) == 0 {
acceptedMediaTypes = accepts
}
for _, mediaType := range acceptedMediaTypes {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := c.do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
payload, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, "", err
}
mediaType := resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
manifest, _, err := distribution.UnmarshalManifest(mediaType, payload)
if err != nil {
return nil, "", err
}
digest := resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return manifest, digest, nil
}
func (c *client) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
req, err := http.NewRequest(http.MethodPut, buildManifestURL(c.url, repository, reference),
bytes.NewReader(payload))
if err != nil {
return "", err
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType)
resp, err := c.do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
return resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")), nil
}
func (c *client) DeleteManifest(repository, reference string) error {
_, err := digest.Parse(reference)
if err != nil {
// the reference is tag, get the digest first
exist, digest, err := c.ManifestExist(repository, reference)
if err != nil {
return err
}
if !exist {
return ierror.New(nil).WithCode(ierror.NotFoundCode).
WithMessage("%s:%s not found", repository, reference)
}
reference = digest
}
req, err := http.NewRequest(http.MethodDelete, buildManifestURL(c.url, repository, reference), nil)
if err != nil {
return err
}
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *client) BlobExist(repository, digest string) (bool, error) {
req, err := http.NewRequest(http.MethodHead, buildBlobURL(c.url, repository, digest), nil)
if err != nil {
return false, err
}
resp, err := c.do(req)
if err != nil {
if ierror.IsErr(err, ierror.NotFoundCode) {
return false, nil
}
return false, err
}
defer resp.Body.Close()
return true, nil
}
func (c *client) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
req, err := http.NewRequest(http.MethodGet, buildBlobURL(c.url, repository, digest), nil)
if err != nil {
return 0, nil, err
}
resp, err := c.do(req)
if err != nil {
return 0, nil, err
}
n := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
size, err := strconv.ParseInt(n, 10, 64)
if err != nil {
defer resp.Body.Close()
return 0, nil, err
}
return size, resp.Body, nil
}
func (c *client) PushBlob(repository, digest string, size int64, blob io.Reader) error {
location, _, err := c.initiateBlobUpload(repository)
if err != nil {
return err
}
return c.monolithicBlobUpload(location, digest, size, blob)
}
func (c *client) initiateBlobUpload(repository string) (string, string, error) {
req, err := http.NewRequest(http.MethodPost, buildInitiateBlobUploadURL(c.url, repository), nil)
if err != nil {
return "", "", err
}
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := c.do(req)
if err != nil {
return "", "", err
}
defer resp.Body.Close()
return resp.Header.Get(http.CanonicalHeaderKey("Location")),
resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID")), nil
}
func (c *client) monolithicBlobUpload(location, digest string, size int64, data io.Reader) error {
url, err := buildMonolithicBlobUploadURL(c.url, location, digest)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPut, url, data)
if err != nil {
return err
}
req.ContentLength = size
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *client) MountBlob(srcRepository, digest, dstRepository string) error {
req, err := http.NewRequest(http.MethodPost, buildMountBlobURL(c.url, dstRepository, digest, srcRepository), nil)
if err != nil {
return err
}
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *client) DeleteBlob(repository, digest string) error {
req, err := http.NewRequest(http.MethodDelete, buildBlobURL(c.url, repository, digest), nil)
if err != nil {
return err
}
resp, err := c.do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
} }
// TODO extend this method to support copy artifacts between different registries when merging codes // TODO extend this method to support copy artifacts between different registries when merging codes
// TODO this can be used in replication to replace the existing implementation // TODO this can be used in replication to replace the existing implementation
// TODO add unit test case
func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error { func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
src, err := registry.NewRepository(srcRepo, c.url, c.client)
if err != nil {
return err
}
dst, err := registry.NewRepository(dstRepo, c.url, c.client)
if err != nil {
return err
}
// pull the manifest from the source repository // pull the manifest from the source repository
srcDgt, mediaType, payload, err := src.PullManifest(srcRef, accepts) manifest, srcDgt, err := c.PullManifest(srcRepo, srcRef)
if err != nil { if err != nil {
return err return err
} }
// check the existence of the artifact on the destination repository // check the existence of the artifact on the destination repository
dstDgt, exist, err := dst.ManifestExist(dstRef) exist, dstDgt, err := c.ManifestExist(dstRepo, dstRef)
if err != nil { if err != nil {
return err return err
} }
@ -111,10 +444,6 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
} }
} }
manifest, _, err := registry.UnMarshal(mediaType, payload)
if err != nil {
return err
}
for _, descriptor := range manifest.References() { for _, descriptor := range manifest.References() {
digest := descriptor.Digest.String() digest := descriptor.Digest.String()
switch descriptor.MediaType { switch descriptor.MediaType {
@ -130,7 +459,7 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
} }
// common layer // common layer
default: default:
exist, err := dst.BlobExist(digest) exist, err := c.BlobExist(dstRepo, digest)
if err != nil { if err != nil {
return err return err
} }
@ -139,7 +468,7 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
continue continue
} }
// when the copy happens inside the same registry, use mount // when the copy happens inside the same registry, use mount
if err = dst.MountBlob(digest, srcRepo); err != nil { if err = c.MountBlob(srcRepo, digest, dstRepo); err != nil {
return err return err
} }
/* /*
@ -156,10 +485,100 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
} }
} }
mediaType, payload, err := manifest.Payload()
if err != nil {
return err
}
// push manifest to the destination repository // push manifest to the destination repository
if _, err = dst.PushManifest(dstRef, mediaType, payload); err != nil { if _, err = c.PushManifest(dstRepo, dstRef, mediaType, payload); err != nil {
return err return err
} }
return nil return nil
} }
func (c *client) do(req *http.Request) (*http.Response, error) {
if c.authorizer != nil {
if err := c.authorizer.Modify(req); err != nil {
return nil, err
}
}
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
message := fmt.Sprintf("http status code: %d, body: %s", resp.StatusCode, string(body))
code := ierror.GeneralCode
switch resp.StatusCode {
case http.StatusUnauthorized:
code = ierror.UnAuthorizedCode
case http.StatusForbidden:
code = ierror.ForbiddenCode
case http.StatusNotFound:
code = ierror.NotFoundCode
}
return nil, ierror.New(nil).WithCode(code).
WithMessage(message)
}
return resp, nil
}
// parse the next page link from the link header
func next(link string) string {
links := internal.ParseLinks(link)
for _, lk := range links {
if lk.Rel == "next" {
return lk.URL
}
}
return ""
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}
func buildCatalogURL(endpoint string) string {
return fmt.Sprintf("%s/v2/_catalog?n=1000", endpoint)
}
func buildTagListURL(endpoint, repository string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repository)
}
func buildManifestURL(endpoint, repository, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repository, reference)
}
func buildBlobURL(endpoint, repository, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, reference)
}
func buildMountBlobURL(endpoint, repository, digest, from string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/?mount=%s&from=%s", endpoint, repository, digest, from)
}
func buildInitiateBlobUploadURL(endpoint, repository string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repository)
}
func buildMonolithicBlobUploadURL(endpoint, location, digest string) (string, error) {
url, err := url.Parse(location)
if err != nil {
return "", err
}
q := url.Query()
q.Set("digest", digest)
url.RawQuery = q.Encode()
if url.IsAbs() {
return url.String(), nil
}
// the "relativeurls" is enabled in registry
return endpoint + url.String(), nil
}

388
src/pkg/registry/client_test.go Normal file
View File

@ -0,0 +1,388 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"encoding/json"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/internal"
"github.com/stretchr/testify/suite"
"io/ioutil"
"net/http"
"strconv"
"testing"
)
type clientTestSuite struct {
suite.Suite
client Client
}
func (c *clientTestSuite) TestPing() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/",
Handler: test.Handler(nil),
})
defer server.Close()
err := NewClient(server.URL, "", "", true).Ping()
c.Require().Nil(err)
}
func (c *clientTestSuite) TestCatalog() {
type repositories struct {
Repositories []string `json:"repositories"`
}
isFirstRequest := true
handler := func(w http.ResponseWriter, r *http.Request) {
if isFirstRequest {
isFirstRequest = false
repos := &repositories{
Repositories: []string{"library/alpine"},
}
link := internal.Link{
URL: `/v2/_catalog?last=library/alpine`,
Rel: "next",
}
w.Header().Set(http.CanonicalHeaderKey("link"), link.String())
encoder := json.NewEncoder(w)
err := encoder.Encode(repos)
c.Require().Nil(err)
return
}
if r.URL.String() != "/v2/_catalog?last=library/alpine" {
w.WriteHeader(http.StatusBadRequest)
return
}
repos := &repositories{
Repositories: []string{"library/hello-world"},
}
encoder := json.NewEncoder(w)
err := encoder.Encode(repos)
c.Require().Nil(err)
return
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/_catalog",
Handler: handler,
})
defer server.Close()
repos, err := NewClient(server.URL, "", "", true).Catalog()
c.Require().Nil(err)
c.Len(repos, 2)
c.EqualValues([]string{"library/alpine", "library/hello-world"}, repos)
}
func (c *clientTestSuite) TestListTags() {
type tags struct {
Tags []string `json:"tags"`
}
isFirstRequest := true
handler := func(w http.ResponseWriter, r *http.Request) {
if isFirstRequest {
isFirstRequest = false
tgs := &tags{
Tags: []string{"1.0"},
}
link := internal.Link{
URL: `/v2/library/hello-world/tags/list?last=1.0`,
Rel: "next",
}
w.Header().Set(http.CanonicalHeaderKey("link"), link.String())
encoder := json.NewEncoder(w)
err := encoder.Encode(tgs)
c.Require().Nil(err)
return
}
if r.URL.String() != "/v2/library/hello-world/tags/list?last=1.0" {
w.WriteHeader(http.StatusBadRequest)
return
}
tgs := &tags{
Tags: []string{"2.0"},
}
encoder := json.NewEncoder(w)
err := encoder.Encode(tgs)
c.Require().Nil(err)
return
}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/library/hello-world/tags/list",
Handler: handler,
})
defer server.Close()
repos, err := NewClient(server.URL, "", "", true).ListTags("library/hello-world")
c.Require().Nil(err)
c.Len(repos, 2)
c.EqualValues([]string{"1.0", "2.0"}, repos)
}
func (c *clientTestSuite) TestManifestExist() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/alpine/manifests/latest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusNotFound,
}),
},
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/hello-world/manifests/latest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusOK,
Headers: map[string]string{
"Docker-Content-Digest": "digest",
},
}),
},
)
defer server.Close()
client := NewClient(server.URL, "", "", true)
// doesn't exist
exist, digest, err := client.ManifestExist("library/alpine", "latest")
c.Require().Nil(err)
c.False(exist)
// exist
exist, digest, err = client.ManifestExist("library/hello-world", "latest")
c.Require().Nil(err)
c.True(exist)
c.Equal("digest", digest)
}
func (c *clientTestSuite) TestPullManifest() {
data := `{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": 1510,
"digest": "sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e"
},
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 977,
"digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
}
]
}`
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/library/hello-world/manifests/latest",
Handler: test.Handler(&test.Response{
Headers: map[string]string{
"Docker-Content-Digest": "digest",
"Content-Type": schema2.MediaTypeManifest,
},
Body: []byte(data),
}),
})
defer server.Close()
manifest, digest, err := NewClient(server.URL, "", "", true).PullManifest("library/hello-world", "latest")
c.Require().Nil(err)
c.Equal("digest", digest)
mediaType, payload, err := manifest.Payload()
c.Require().Nil(err)
c.Equal(schema2.MediaTypeManifest, mediaType)
c.Equal(data, string(payload))
}
func (c *clientTestSuite) TestPushManifest() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "PUT",
Pattern: "/v2/library/hello-world/manifests/latest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusCreated,
Headers: map[string]string{
"Docker-Content-Digest": "digest",
},
}),
})
defer server.Close()
digest, err := NewClient(server.URL, "", "", true).PushManifest("library/hello-world", "latest", "", nil)
c.Require().Nil(err)
c.Equal("digest", digest)
}
func (c *clientTestSuite) TestDeleteManifest() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/hello-world/manifests/latest",
Handler: test.Handler(&test.Response{
Headers: map[string]string{
"Docker-Content-Digest": "digest",
},
}),
},
&test.RequestHandlerMapping{
Method: "DELETE",
Pattern: "/v2/library/hello-world/manifests/digest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
}),
})
defer server.Close()
err := NewClient(server.URL, "", "", true).DeleteManifest("library/hello-world", "latest")
c.Require().Nil(err)
}
func (c *clientTestSuite) TestBlobExist() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/hello-world/blobs/digest1",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusNotFound,
}),
},
&test.RequestHandlerMapping{
Method: "HEAD",
Pattern: "/v2/library/hello-world/blobs/digest2",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusOK,
}),
},
)
defer server.Close()
// doesn't exist
client := NewClient(server.URL, "", "", true)
exist, err := client.BlobExist("library/hello-world", "digest1")
c.Require().Nil(err)
c.False(exist)
// exist
exist, err = client.BlobExist("library/hello-world", "digest2")
c.Require().Nil(err)
c.True(exist)
}
func (c *clientTestSuite) TestPullBlob() {
data := []byte{'a'}
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "GET",
Pattern: "/v2/library/hello-world/blobs/digest",
Handler: test.Handler(&test.Response{
Headers: map[string]string{
"Content-Length": strconv.Itoa(len(data)),
},
Body: data,
}),
})
defer server.Close()
size, blob, err := NewClient(server.URL, "", "", true).PullBlob("library/hello-world", "digest")
c.Require().Nil(err)
c.Equal(int64(len(data)), size)
b, err := ioutil.ReadAll(blob)
c.Require().Nil(err)
c.EqualValues(data, b)
}
func (c *clientTestSuite) TestPushBlob() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "POST",
Pattern: "/v2/library/hello-world/blobs/uploads/",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
Headers: map[string]string{
"Location": "/v2/library/hello-world/blobs/uploads/uuid",
},
}),
},
&test.RequestHandlerMapping{
Method: "PUT",
Pattern: "/v2/library/hello-world/blobs/uploads/uuid",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusCreated,
}),
})
defer server.Close()
err := NewClient(server.URL, "", "", true).PushBlob("library/hello-world", "digest", 0, nil)
c.Require().Nil(err)
}
func (c *clientTestSuite) TestDeleteBlob() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "DELETE",
Pattern: "/v2/library/hello-world/blobs/digest",
Handler: test.Handler(&test.Response{
StatusCode: http.StatusAccepted,
}),
})
defer server.Close()
err := NewClient(server.URL, "", "", true).DeleteBlob("library/hello-world", "digest")
c.Require().Nil(err)
}
func (c *clientTestSuite) TestMountBlob() {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: "POST",
Pattern: "/v2/library/hello-world/blobs/uploads/",
Handler: func(w http.ResponseWriter, r *http.Request) {
mount := r.URL.Query().Get("mount")
from := r.URL.Query().Get("from")
if mount != "digest" || from != "library/alpine" {
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusAccepted)
},
})
defer server.Close()
err := NewClient(server.URL, "", "", true).MountBlob("library/alpine", "digest", "library/hello-world")
c.Require().Nil(err)
}
func TestClientTestSuite(t *testing.T) {
suite.Run(t, &clientTestSuite{})
}

4
src/pkg/signature/notary/helper.go
View File

@ -17,6 +17,7 @@ package notary
import ( import (
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"github.com/goharbor/harbor/src/internal"
model2 "github.com/goharbor/harbor/src/pkg/signature/notary/model" model2 "github.com/goharbor/harbor/src/pkg/signature/notary/model"
"net/http" "net/http"
"os" "os"
@ -25,7 +26,6 @@ import (
"github.com/docker/distribution/registry/auth/token" "github.com/docker/distribution/registry/auth/token"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/config"
tokenutil "github.com/goharbor/harbor/src/core/service/token" tokenutil "github.com/goharbor/harbor/src/core/service/token"
"github.com/theupdateframework/notary" "github.com/theupdateframework/notary"
@ -82,7 +82,7 @@ func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]model2
authorizer := &notaryAuthorizer{ authorizer := &notaryAuthorizer{
token: t.Token, token: t.Token,
} }
tr := registry.NewTransport(registry.GetHTTPTransport(), authorizer) tr := NewTransport(internal.GetHTTPTransport(), authorizer)
gun := data.GUN(fqRepo) gun := data.GUN(fqRepo)
notaryRepo, err := client.NewFileCachedRepository(notaryCachePath, gun, notaryEndpoint, tr, mockRetriever, trustPin) notaryRepo, err := client.NewFileCachedRepository(notaryCachePath, gun, notaryEndpoint, tr, mockRetriever, trustPin)
if err != nil { if err != nil {

2
src/common/utils/registry/transport.go → src/pkg/signature/notary/transport.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package registry package notary
import ( import (
"net/http" "net/http"

2
src/common/utils/registry/transport_test.go → src/pkg/signature/notary/transport_test.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package registry package notary
import ( import (
"fmt" "fmt"

4
src/replication/adapter/adapter.go
View File

@ -54,8 +54,8 @@ type Adapter interface {
type ImageRegistry interface { type ImageRegistry interface {
FetchImages(filters []*model.Filter) ([]*model.Resource, error) FetchImages(filters []*model.Filter) ([]*model.Resource, error)
ManifestExist(repository, reference string) (exist bool, digest string, err error) ManifestExist(repository, reference string) (exist bool, digest string, err error)
PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error)
PushManifest(repository, reference, mediaType string, payload []byte) error PushManifest(repository, reference, mediaType string, payload []byte) (string, error)
// the "reference" can be "tag" or "digest", the function needs to handle both // the "reference" can be "tag" or "digest", the function needs to handle both
DeleteManifest(repository, reference string) error DeleteManifest(repository, reference string) error
BlobExist(repository, digest string) (exist bool, err error) BlobExist(repository, digest string) (exist bool, err error)

34
src/replication/adapter/aliacr/adapter.go
View File

@ -4,6 +4,9 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/goharbor/harbor/src/internal"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
"net/http" "net/http"
"path/filepath" "path/filepath"
"regexp" "regexp"
@ -11,7 +14,6 @@ import (
"github.com/aliyun/alibaba-cloud-sdk-go/services/cr" "github.com/aliyun/alibaba-cloud-sdk-go/services/cr"
"github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
@ -50,24 +52,38 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
} }
// fix url (allow user input cr service url) // fix url (allow user input cr service url)
registry.URL = fmt.Sprintf(registryEndpointTpl, region) registry.URL = fmt.Sprintf(registryEndpointTpl, region)
realm, service, err := ping(registry)
credential := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret)
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, credential)
nativeRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil { if err != nil {
return nil, err return nil, err
} }
credential := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret)
authorizer := bearer.NewAuthorizer(realm, service, credential, util.GetHTTPTransport(registry.Insecure))
return &adapter{ return &adapter{
region: region, region: region,
registry: registry, registry: registry,
domain: fmt.Sprintf(endpointTpl, region), domain: fmt.Sprintf(endpointTpl, region),
Adapter: nativeRegistry, Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
}, nil }, nil
} }
func ping(registry *model.Registry) (string, string, error) {
client := &http.Client{
Transport: internal.GetHTTPTransport(registry.Insecure),
}
resp, err := client.Get(registry.URL + "/v2/")
if err != nil {
return "", "", err
}
defer resp.Body.Close()
challenges := challenge.ResponseChallenges(resp)
for _, challenge := range challenges {
if challenge.Scheme == "bearer" {
return challenge.Parameters["realm"], challenge.Parameters["service"], nil
}
}
return "", "", fmt.Errorf("bearer auth scheme isn't supported: %v", challenges)
}
type factory struct { type factory struct {
} }

42
src/replication/adapter/aliacr/adapter_test.go
View File

@ -10,47 +10,11 @@ import (
"time" "time"
"github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/common/utils/test"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func TestAdapter_NewAdapter(t *testing.T) {
factory, err := adp.GetFactory("BadName")
assert.Nil(t, factory)
assert.NotNil(t, err)
factory, err = adp.GetFactory(model.RegistryTypeAliAcr)
assert.Nil(t, err)
assert.NotNil(t, factory)
// test case for URL is registry.
adapter, err := newAdapter(&model.Registry{
Type: model.RegistryTypeAliAcr,
Credential: &model.Credential{
AccessKey: "MockAccessKey",
AccessSecret: "MockAccessSecret",
},
URL: "https://registry.test-region.aliyuncs.com",
})
assert.Nil(t, err)
assert.NotNil(t, adapter)
// test case for URL is cr service.
adapter, err = newAdapter(&model.Registry{
Type: model.RegistryTypeAliAcr,
Credential: &model.Credential{
AccessKey: "MockAccessKey",
AccessSecret: "MockAccessSecret",
},
URL: "https://cr.test-region.aliyuncs.com",
})
assert.Nil(t, err)
assert.NotNil(t, adapter)
}
func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Server) { func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Server) {
server := test.NewServer( server := test.NewServer(
&test.RequestHandlerMapping{ &test.RequestHandlerMapping{
@ -96,12 +60,8 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
AccessSecret: "MockAccessSecret", AccessSecret: "MockAccessSecret",
} }
} }
nativeRegistry, err := native.NewAdapter(registry)
if err != nil {
panic(err)
}
return &adapter{ return &adapter{
Adapter: nativeRegistry, Adapter: native.NewAdapter(registry),
region: "test-region", region: "test-region",
domain: server.URL, domain: server.URL,
registry: registry, registry: registry,

14
src/replication/adapter/awsecr/adapter.go
View File

@ -16,6 +16,7 @@ package awsecr
import ( import (
"errors" "errors"
"github.com/goharbor/harbor/src/internal"
"net/http" "net/http"
"regexp" "regexp"
@ -25,7 +26,6 @@ import (
"github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/aws/session"
awsecrapi "github.com/aws/aws-sdk-go/service/ecr" awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
@ -53,13 +53,9 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
return nil, err return nil, err
} }
authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure) authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure)
dockerRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{ return &adapter{
registry: registry, registry: registry,
Adapter: dockerRegistry, Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
region: region, region: region,
}, nil }, nil
} }
@ -201,7 +197,7 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) {
log.Errorf("no credential to ping registry %s", a.registry.URL) log.Errorf("no credential to ping registry %s", a.registry.URL)
return model.Unhealthy, nil return model.Unhealthy, nil
} }
if err := a.PingGet(); err != nil { if err := a.Ping(); err != nil {
log.Errorf("failed to ping registry %s: %v", a.registry.URL, err) log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
return model.Unhealthy, nil return model.Unhealthy, nil
} }
@ -248,7 +244,7 @@ func (a *adapter) createRepository(repository string) error {
Credentials: cred, Credentials: cred,
Region: &a.region, Region: &a.region,
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Transport: registry.GetHTTPTransport(a.registry.Insecure), Transport: internal.GetHTTPTransport(a.registry.Insecure),
}, },
} }
if a.forceEndpoint != nil { if a.forceEndpoint != nil {
@ -290,7 +286,7 @@ func (a *adapter) DeleteManifest(repository, reference string) error {
Credentials: cred, Credentials: cred,
Region: &a.region, Region: &a.region,
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Transport: registry.GetHTTPTransport(a.registry.Insecure), Transport: internal.GetHTTPTransport(a.registry.Insecure),
}, },
} }
if a.forceEndpoint != nil { if a.forceEndpoint != nil {

6
src/replication/adapter/awsecr/adapter_test.go
View File

@ -133,13 +133,9 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
AccessSecret: "ppp", AccessSecret: "ppp",
} }
} }
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
panic(err)
}
return &adapter{ return &adapter{
registry: registry, registry: registry,
Adapter: dockerRegistryAdapter, Adapter: native.NewAdapter(registry),
region: "test-region", region: "test-region",
forceEndpoint: &server.URL, forceEndpoint: &server.URL,
}, server }, server

4
src/replication/adapter/awsecr/auth.go
View File

@ -25,7 +25,7 @@ import (
awsecrapi "github.com/aws/aws-sdk-go/service/ecr" awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
"github.com/goharbor/harbor/src/common/http/modifier" "github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry" "github.com/goharbor/harbor/src/internal"
"net/http" "net/http"
"net/url" "net/url"
"strings" "strings"
@ -100,7 +100,7 @@ func (a *awsAuthCredential) getAuthorization() (string, string, string, *time.Ti
Credentials: cred, Credentials: cred,
Region: &a.region, Region: &a.region,
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Transport: registry.GetHTTPTransport(a.insecure), Transport: internal.GetHTTPTransport(a.insecure),
}, },
} }
if a.forceEndpoint != nil { if a.forceEndpoint != nil {

6
src/replication/adapter/azurecr/adapter.go
View File

@ -16,12 +16,8 @@ func init() {
} }
func newAdapter(registry *model.Registry) (adp.Adapter, error) { func newAdapter(registry *model.Registry) (adp.Adapter, error) {
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
return &adapter{ return &adapter{
Adapter: dockerRegistryAdapter, Adapter: native.NewAdapter(registry),
}, nil }, nil
} }

15
src/replication/adapter/dockerhub/adapter.go
View File

@ -31,19 +31,14 @@ func newAdapter(registry *model.Registry) (adp.Adapter, error) {
return nil, err return nil, err
} }
dockerRegistryAdapter, err := native.NewAdapter(&model.Registry{
URL: registryURL,
Credential: registry.Credential,
Insecure: registry.Insecure,
})
if err != nil {
return nil, err
}
return &adapter{ return &adapter{
client: client, client: client,
registry: registry, registry: registry,
Adapter: dockerRegistryAdapter, Adapter: native.NewAdapter(&model.Registry{
URL: registryURL,
Credential: registry.Credential,
Insecure: registry.Insecure,
}),
}, nil }, nil
} }

30
src/replication/adapter/gitlab/adapter.go
View File

@ -2,12 +2,10 @@ package gitlab
import ( import (
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util" "github.com/goharbor/harbor/src/replication/util"
"net/http"
"strings" "strings"
) )
@ -24,7 +22,7 @@ type factory struct {
// Create ... // Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) { func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
return newAdapter(r) return newAdapter(r), nil
} }
// AdapterPattern ... // AdapterPattern ...
@ -41,33 +39,13 @@ type adapter struct {
clientGitlabAPI *Client clientGitlabAPI *Client
} }
func newAdapter(registry *model.Registry) (*adapter, error) { func newAdapter(registry *model.Registry) *adapter {
var credential auth.Credential
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
credential = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, credential)
dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(&model.Registry{
Name: registry.Name,
URL: registry.URL,
Credential: registry.Credential,
Insecure: registry.Insecure,
}, authorizer)
if err != nil {
return nil, err
}
return &adapter{ return &adapter{
registry: registry, registry: registry,
url: registry.URL, url: registry.URL,
clientGitlabAPI: NewClient(registry), clientGitlabAPI: NewClient(registry),
Adapter: dockerRegistryAdapter, Adapter: native.NewAdapter(registry),
}, nil }
} }
func (a *adapter) Info() (info *model.RegistryInfo, err error) { func (a *adapter) Info() (info *model.RegistryInfo, err error) {

4
src/replication/adapter/gitlab/client.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util" "github.com/goharbor/harbor/src/replication/util"
"io" "io"
@ -65,7 +65,7 @@ func ping(client *http.Client, endpoint string) (string, string, error) {
} }
defer resp.Body.Close() defer resp.Body.Close()
challenges := auth.ParseChallengeFromResponse(resp) challenges := challenge.ResponseChallenges(resp)
for _, challenge := range challenges { for _, challenge := range challenges {
if scheme == challenge.Scheme { if scheme == challenge.Scheme {
realm := challenge.Parameters["realm"] realm := challenge.Parameters["realm"]

15
src/replication/adapter/googlegcr/adapter.go
View File

@ -29,16 +29,11 @@ func init() {
log.Infof("the factory for adapter %s registered", model.RegistryTypeGoogleGcr) log.Infof("the factory for adapter %s registered", model.RegistryTypeGoogleGcr)
} }
func newAdapter(registry *model.Registry) (*adapter, error) { func newAdapter(registry *model.Registry) *adapter {
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
return &adapter{ return &adapter{
registry: registry, registry: registry,
Adapter: dockerRegistryAdapter, Adapter: native.NewAdapter(registry),
}, nil }
} }
type factory struct { type factory struct {
@ -46,7 +41,7 @@ type factory struct {
// Create ... // Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) { func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
return newAdapter(r) return newAdapter(r), nil
} }
// AdapterPattern ... // AdapterPattern ...
@ -125,7 +120,7 @@ func (a adapter) HealthCheck() (model.HealthStatus, error) {
log.Errorf("no credential to ping registry %s", a.registry.URL) log.Errorf("no credential to ping registry %s", a.registry.URL)
return model.Unhealthy, nil return model.Unhealthy, nil
} }
if err = a.PingGet(); err != nil { if err = a.Ping(); err != nil {
log.Errorf("failed to ping registry %s: %v", a.registry.URL, err) log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
return model.Unhealthy, nil return model.Unhealthy, nil
} }

5
src/replication/adapter/googlegcr/adapter_test.go
View File

@ -88,10 +88,7 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
factory, err := adp.GetFactory(model.RegistryTypeGoogleGcr) factory, err := adp.GetFactory(model.RegistryTypeGoogleGcr)
assert.Nil(t, err) assert.Nil(t, err)
assert.NotNil(t, factory) assert.NotNil(t, factory)
a, err := newAdapter(registry) return newAdapter(registry), server
assert.Nil(t, err)
return a, server
} }
func TestAdapter_Info(t *testing.T) { func TestAdapter_Info(t *testing.T) {

58
src/replication/adapter/harbor/adapter.go
View File

@ -17,19 +17,18 @@ package harbor
import ( import (
"errors" "errors"
"fmt" "fmt"
"net/http"
"strconv"
"strings"
common_http "github.com/goharbor/harbor/src/common/http" common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier" "github.com/goharbor/harbor/src/common/http/modifier"
common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth" common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth" "github.com/goharbor/harbor/src/pkg/registry/auth/basic"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util" "github.com/goharbor/harbor/src/replication/util"
"net/http"
"strconv"
"strings"
) )
func init() { func init() {
@ -62,26 +61,25 @@ type adapter struct {
func newAdapter(registry *model.Registry) (*adapter, error) { func newAdapter(registry *model.Registry) (*adapter, error) {
transport := util.GetHTTPTransport(registry.Insecure) transport := util.GetHTTPTransport(registry.Insecure)
modifiers := []modifier.Modifier{ // local Harbor instance
&auth.UserAgentModifier{ if registry.Credential != nil && registry.Credential.Type == model.CredentialTypeSecret {
UserAgent: adp.UserAgentReplication, authorizer := common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
}, return &adapter{
} registry: registry,
if registry.Credential != nil { url: registry.URL,
var authorizer modifier.Modifier client: common_http.NewClient(
if registry.Credential.Type == model.CredentialTypeSecret { &http.Client{
authorizer = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret) Transport: transport,
} else { }, authorizer),
authorizer = auth.NewBasicAuthCredential( Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
registry.Credential.AccessKey, }, nil
registry.Credential.AccessSecret)
}
modifiers = append(modifiers, authorizer)
} }
dockerRegistryAdapter, err := native.NewAdapter(registry) var authorizers []modifier.Modifier
if err != nil { if registry.Credential != nil {
return nil, err authorizers = append(authorizers, basic.NewAuthorizer(
registry.Credential.AccessKey,
registry.Credential.AccessSecret))
} }
return &adapter{ return &adapter{
registry: registry, registry: registry,
@ -89,8 +87,8 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
client: common_http.NewClient( client: common_http.NewClient(
&http.Client{ &http.Client{
Transport: transport, Transport: transport,
}, modifiers...), }, authorizers...),
Adapter: dockerRegistryAdapter, Adapter: native.NewAdapter(registry),
}, nil }, nil
} }
@ -119,7 +117,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
sys := &struct { sys := &struct {
ChartRegistryEnabled bool `json:"with_chartmuseum"` ChartRegistryEnabled bool `json:"with_chartmuseum"`
}{} }{}
if err := a.client.Get(a.getURL()+"/api/systeminfo", sys); err != nil { if err := a.client.Get(a.getURL()+"/api/v2.0/systeminfo", sys); err != nil {
return nil, err return nil, err
} }
if sys.ChartRegistryEnabled { if sys.ChartRegistryEnabled {
@ -129,7 +127,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
Name string `json:"name"` Name string `json:"name"`
}{} }{}
// label isn't supported in some previous version of Harbor // label isn't supported in some previous version of Harbor
if err := a.client.Get(a.getURL()+"/api/labels?scope=g", &labels); err != nil { if err := a.client.Get(a.getURL()+"/api/v2.0/labels?scope=g", &labels); err != nil {
if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound { if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound {
return nil, err return nil, err
} }
@ -185,7 +183,7 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error {
Name: project.Name, Name: project.Name,
Metadata: project.Metadata, Metadata: project.Metadata,
} }
err := a.client.Post(a.getURL()+"/api/projects", pro) err := a.client.Post(a.getURL()+"/api/v2.0/projects", pro)
if err != nil { if err != nil {
if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict { if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
log.Debugf("got 409 when trying to create project %s", project.Name) log.Debugf("got 409 when trying to create project %s", project.Name)
@ -251,7 +249,7 @@ type project struct {
func (a *adapter) getProjects(name string) ([]*project, error) { func (a *adapter) getProjects(name string) ([]*project, error) {
projects := []*project{} projects := []*project{}
url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=500", a.getURL(), name) url := fmt.Sprintf("%s/api/v2.0/projects?name=%s&page=1&page_size=500", a.getURL(), name)
if err := a.client.GetAndIteratePagination(url, &projects); err != nil { if err := a.client.GetAndIteratePagination(url, &projects); err != nil {
return nil, err return nil, err
} }
@ -286,7 +284,7 @@ func (a *adapter) getProject(name string) (*project, error) {
func (a *adapter) getRepositories(projectID int64) ([]*adp.Repository, error) { func (a *adapter) getRepositories(projectID int64) ([]*adp.Repository, error) {
repositories := []*adp.Repository{} repositories := []*adp.Repository{}
url := fmt.Sprintf("%s/api/repositories?project_id=%d&page=1&page_size=500", a.getURL(), projectID) url := fmt.Sprintf("%s/api/v2.0/repositories?project_id=%d&page=1&page_size=500", a.getURL(), projectID)
if err := a.client.GetAndIteratePagination(url, &repositories); err != nil { if err := a.client.GetAndIteratePagination(url, &repositories); err != nil {
return nil, err return nil, err
} }

8
src/replication/adapter/harbor/adapter_test.go
View File

@ -29,7 +29,7 @@ func TestInfo(t *testing.T) {
// chart museum enabled // chart museum enabled
server := test.NewServer(&test.RequestHandlerMapping{ server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/systeminfo", Pattern: "/api/v2.0/systeminfo",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `{"with_chartmuseum":true}` data := `{"with_chartmuseum":true}`
w.Write([]byte(data)) w.Write([]byte(data))
@ -53,7 +53,7 @@ func TestInfo(t *testing.T) {
// chart museum disabled // chart museum disabled
server = test.NewServer(&test.RequestHandlerMapping{ server = test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/systeminfo", Pattern: "/api/v2.0/systeminfo",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `{"with_chartmuseum":false}` data := `{"with_chartmuseum":false}`
w.Write([]byte(data)) w.Write([]byte(data))
@ -77,7 +77,7 @@ func TestInfo(t *testing.T) {
func TestPrepareForPush(t *testing.T) { func TestPrepareForPush(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{ server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodPost, Method: http.MethodPost,
Pattern: "/api/projects", Pattern: "/api/v2.0/projects",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated) w.WriteHeader(http.StatusCreated)
}, },
@ -131,7 +131,7 @@ func TestPrepareForPush(t *testing.T) {
// project already exists // project already exists
server = test.NewServer(&test.RequestHandlerMapping{ server = test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodPost, Method: http.MethodPost,
Pattern: "/api/projects", Pattern: "/api/v2.0/projects",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusConflict) w.WriteHeader(http.StatusConflict)
}, },

10
src/replication/adapter/harbor/chart_registry.go
View File

@ -52,7 +52,7 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
} }
resources := []*model.Resource{} resources := []*model.Resource{}
for _, project := range projects { for _, project := range projects {
url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.getURL(), project.Name) url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts", a.getURL(), project.Name)
repositories := []*adp.Repository{} repositories := []*adp.Repository{}
if err := a.client.Get(url, &repositories); err != nil { if err := a.client.Get(url, &repositories); err != nil {
return nil, err return nil, err
@ -71,7 +71,7 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
} }
for _, repository := range repositories { for _, repository := range repositories {
name := strings.SplitN(repository.Name, "/", 2)[1] name := strings.SplitN(repository.Name, "/", 2)[1]
url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.getURL(), project.Name, name) url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts/%s", a.getURL(), project.Name, name)
versions := []*chartVersion{} versions := []*chartVersion{}
if err := a.client.Get(url, &versions); err != nil { if err := a.client.Get(url, &versions); err != nil {
return nil, err return nil, err
@ -131,7 +131,7 @@ func (a *adapter) getChartInfo(name, version string) (*chartVersionDetail, error
if err != nil { if err != nil {
return nil, err return nil, err
} }
url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s/%s", a.url, project, name, version) url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts/%s/%s", a.url, project, name, version)
info := &chartVersionDetail{} info := &chartVersionDetail{}
if err = a.client.Get(url, info); err != nil { if err = a.client.Get(url, info); err != nil {
return nil, err return nil, err
@ -191,7 +191,7 @@ func (a *adapter) UploadChart(name, version string, chart io.Reader) error {
} }
w.Close() w.Close()
url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.url, project) url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts", a.url, project)
req, err := http.NewRequest(http.MethodPost, url, buf) req, err := http.NewRequest(http.MethodPost, url, buf)
if err != nil { if err != nil {
@ -222,7 +222,7 @@ func (a *adapter) DeleteChart(name, version string) error {
if err != nil { if err != nil {
return err return err
} }
url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s/%s", a.url, project, name, version) url := fmt.Sprintf("%s/api/v2.0/chartrepo/%s/charts/%s/%s", a.url, project, name, version)
return a.client.Delete(url) return a.client.Delete(url)
} }

14
src/replication/adapter/harbor/chart_registry_test.go
View File

@ -29,7 +29,7 @@ func TestFetchCharts(t *testing.T) {
server := test.NewServer([]*test.RequestHandlerMapping{ server := test.NewServer([]*test.RequestHandlerMapping{
{ {
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/projects", Pattern: "/api/v2.0/projects",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{ data := `[{
"name": "library", "name": "library",
@ -40,7 +40,7 @@ func TestFetchCharts(t *testing.T) {
}, },
{ {
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts/harbor", Pattern: "/api/v2.0/chartrepo/library/charts/harbor",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{ data := `[{
"name": "harbor", "name": "harbor",
@ -54,7 +54,7 @@ func TestFetchCharts(t *testing.T) {
}, },
{ {
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts", Pattern: "/api/v2.0/chartrepo/library/charts",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{ data := `[{
"name": "harbor" "name": "harbor"
@ -100,7 +100,7 @@ func TestFetchCharts(t *testing.T) {
func TestChartExist(t *testing.T) { func TestChartExist(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{ server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts/harbor/1.0", Pattern: "/api/v2.0/chartrepo/library/charts/harbor/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `{ data := `{
"metadata": { "metadata": {
@ -125,7 +125,7 @@ func TestDownloadChart(t *testing.T) {
server := test.NewServer([]*test.RequestHandlerMapping{ server := test.NewServer([]*test.RequestHandlerMapping{
{ {
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts/harbor/1.0", Pattern: "/api/v2.0/chartrepo/library/charts/harbor/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `{ data := `{
"metadata": { "metadata": {
@ -156,7 +156,7 @@ func TestDownloadChart(t *testing.T) {
func TestUploadChart(t *testing.T) { func TestUploadChart(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{ server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodPost, Method: http.MethodPost,
Pattern: "/api/chartrepo/library/charts", Pattern: "/api/v2.0/chartrepo/library/charts",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
}, },
@ -174,7 +174,7 @@ func TestUploadChart(t *testing.T) {
func TestDeleteChart(t *testing.T) { func TestDeleteChart(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{ server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodDelete, Method: http.MethodDelete,
Pattern: "/api/chartrepo/library/charts/harbor/1.0", Pattern: "/api/v2.0/chartrepo/library/charts/harbor/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
}, },

4
src/replication/adapter/harbor/image_registry.go
View File

@ -146,12 +146,12 @@ func (a *adapter) listCandidateProjects(filters []*model.Filter) ([]*project, er
// override the default implementation from the default image registry // override the default implementation from the default image registry
// by calling Harbor API directly // by calling Harbor API directly
func (a *adapter) DeleteManifest(repository, reference string) error { func (a *adapter) DeleteManifest(repository, reference string) error {
url := fmt.Sprintf("%s/api/repositories/%s/tags/%s", a.url, repository, reference) url := fmt.Sprintf("%s/api/v2.0/repositories/%s/tags/%s", a.url, repository, reference)
return a.client.Delete(url) return a.client.Delete(url)
} }
func (a *adapter) getTags(repository string) ([]*adp.VTag, error) { func (a *adapter) getTags(repository string) ([]*adp.VTag, error) {
url := fmt.Sprintf("%s/api/repositories/%s/tags", a.getURL(), repository) url := fmt.Sprintf("%s/api/v2.0/repositories/%s/tags", a.getURL(), repository)
tags := []*struct { tags := []*struct {
Name string `json:"name"` Name string `json:"name"`
Labels []*struct { Labels []*struct {

8
src/replication/adapter/harbor/image_registry_test.go
View File

@ -28,7 +28,7 @@ func TestFetchImages(t *testing.T) {
server := test.NewServer([]*test.RequestHandlerMapping{ server := test.NewServer([]*test.RequestHandlerMapping{
{ {
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/projects", Pattern: "/api/v2.0/projects",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{ data := `[{
"name": "library", "name": "library",
@ -39,7 +39,7 @@ func TestFetchImages(t *testing.T) {
}, },
{ {
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/repositories/library/hello-world/tags", Pattern: "/api/v2.0/repositories/library/hello-world/tags",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{ data := `[{
"name": "1.0" "name": "1.0"
@ -51,7 +51,7 @@ func TestFetchImages(t *testing.T) {
}, },
{ {
Method: http.MethodGet, Method: http.MethodGet,
Pattern: "/api/repositories", Pattern: "/api/v2.0/repositories",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{ data := `[{
"name": "library/hello-world" "name": "library/hello-world"
@ -98,7 +98,7 @@ func TestFetchImages(t *testing.T) {
func TestDeleteManifest(t *testing.T) { func TestDeleteManifest(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{ server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodDelete, Method: http.MethodDelete,
Pattern: "/api/repositories/library/hello-world/tags/1.0", Pattern: "/api/v2.0/repositories/library/hello-world/tags/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
}}) }})

16
src/replication/adapter/huawei/huawei_adapter.go
View File

@ -3,6 +3,7 @@ package huawei
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"regexp" "regexp"
@ -11,7 +12,6 @@ import (
common_http "github.com/goharbor/harbor/src/common/http" common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier" "github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
@ -228,20 +228,12 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) {
} }
func newAdapter(registry *model.Registry) (adp.Adapter, error) { func newAdapter(registry *model.Registry) (adp.Adapter, error) {
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
var ( var (
modifiers = []modifier.Modifier{ modifiers = []modifier.Modifier{}
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
}}
authorizer modifier.Modifier authorizer modifier.Modifier
) )
if registry.Credential != nil { if registry.Credential != nil {
authorizer = auth.NewBasicAuthCredential( authorizer = basic.NewAuthorizer(
registry.Credential.AccessKey, registry.Credential.AccessKey,
registry.Credential.AccessSecret) registry.Credential.AccessSecret)
modifiers = append(modifiers, authorizer) modifiers = append(modifiers, authorizer)
@ -249,7 +241,7 @@ func newAdapter(registry *model.Registry) (adp.Adapter, error) {
transport := util.GetHTTPTransport(registry.Insecure) transport := util.GetHTTPTransport(registry.Insecure)
return &adapter{ return &adapter{
Adapter: dockerRegistryAdapter, Adapter: native.NewAdapter(registry),
registry: registry, registry: registry,
client: common_http.NewClient( client: common_http.NewClient(
&http.Client{ &http.Client{

16
src/replication/adapter/jfrog/adapter.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -14,7 +15,6 @@ import (
common_http "github.com/goharbor/harbor/src/common/http" common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier" "github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
@ -78,25 +78,17 @@ func (a *adapter) Info() (info *model.RegistryInfo, err error) {
} }
func newAdapter(registry *model.Registry) (adp.Adapter, error) { func newAdapter(registry *model.Registry) (adp.Adapter, error) {
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
var ( var (
modifiers = []modifier.Modifier{ modifiers = []modifier.Modifier{}
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
}}
) )
if registry.Credential != nil { if registry.Credential != nil {
modifiers = append(modifiers, auth.NewBasicAuthCredential( modifiers = append(modifiers, basic.NewAuthorizer(
registry.Credential.AccessKey, registry.Credential.AccessKey,
registry.Credential.AccessSecret)) registry.Credential.AccessSecret))
} }
return &adapter{ return &adapter{
Adapter: dockerRegistryAdapter, Adapter: native.NewAdapter(registry),
registry: registry, registry: registry,
client: common_http.NewClient( client: common_http.NewClient(
&http.Client{ &http.Client{

213
src/replication/adapter/native/adapter.go
View File

@ -16,22 +16,15 @@ package native
import ( import (
"fmt" "fmt"
"io"
"net/http"
"strings"
"sync"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/goharbor/harbor/src/common/http/modifier"
common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
registry_pkg "github.com/goharbor/harbor/src/common/utils/registry" "github.com/goharbor/harbor/src/internal"
"github.com/goharbor/harbor/src/common/utils/registry/auth" ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/registry"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util" "github.com/goharbor/harbor/src/replication/util"
"sync"
) )
func init() { func init() {
@ -49,7 +42,7 @@ type factory struct {
// Create ... // Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) { func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
return NewAdapter(r) return NewAdapter(r), nil
} }
// AdapterPattern ... // AdapterPattern ...
@ -61,60 +54,30 @@ func (f *factory) AdapterPattern() *model.AdapterPattern {
// that implement the registry V2 API // that implement the registry V2 API
type Adapter struct { type Adapter struct {
sync.RWMutex sync.RWMutex
*registry_pkg.Registry
registry *model.Registry registry *model.Registry
client *http.Client registry.Client
clients map[string]*registry_pkg.Repository // client for repositories
} }
// NewAdapter returns an instance of the Adapter // NewAdapter returns an instance of the Adapter
func NewAdapter(registry *model.Registry) (*Adapter, error) { func NewAdapter(reg *model.Registry) *Adapter {
var cred modifier.Modifier adapter := &Adapter{
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 { registry: reg,
if registry.Credential.Type == model.CredentialTypeSecret {
cred = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
} else {
cred = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
} }
authorizer := auth.NewAuthorizer(cred, &http.Client{ username, password := "", ""
Transport: util.GetHTTPTransport(registry.Insecure), if reg.Credential != nil {
}) username = reg.Credential.AccessKey
/* password = reg.Credential.AccessSecret
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{ }
Transport: util.GetHTTPTransport(registry.Insecure), adapter.Client = registry.NewClient(reg.URL, username, password, reg.Insecure)
}, cred, registry.TokenServiceURL) return adapter
*/
return NewAdapterWithCustomizedAuthorizer(registry, authorizer)
} }
// NewAdapterWithCustomizedAuthorizer returns an instance of the Adapter with the customized authorizer // NewAdapterWithAuthorizer returns an instance of the Adapter with provided authorizer
func NewAdapterWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*Adapter, error) { func NewAdapterWithAuthorizer(reg *model.Registry, authorizer internal.Authorizer) *Adapter {
transport := util.GetHTTPTransport(registry.Insecure)
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
},
}
if authorizer != nil {
modifiers = append(modifiers, authorizer)
}
client := &http.Client{
Transport: registry_pkg.NewTransport(transport, modifiers...),
}
reg, err := registry_pkg.NewRegistry(registry.URL, client)
if err != nil {
return nil, err
}
return &Adapter{ return &Adapter{
Registry: reg, registry: reg,
registry: registry, Client: registry.NewClientWithAuthorizer(reg.URL, authorizer, reg.Insecure),
client: client, }
clients: map[string]*registry_pkg.Repository{},
}, nil
} }
// Info returns the basic information about the adapter // Info returns the basic information about the adapter
@ -267,7 +230,7 @@ func (a *Adapter) getRepositories(filters []*model.Filter) ([]*adp.Repository, e
} }
func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) { func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) {
tags, err := a.ListTag(repository) tags, err := a.ListTags(repository)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -281,131 +244,15 @@ func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) {
return result, nil return result, nil
} }
// ManifestExist ... // PingSimple checks whether the registry is available. It checks the connectivity and certificate (if TLS enabled)
func (a *Adapter) ManifestExist(repository, reference string) (bool, string, error) { // only, regardless of 401/403 error.
client, err := a.getClient(repository) func (a *Adapter) PingSimple() error {
if err != nil { err := a.Ping()
return false, "", err if err == nil {
return nil
} }
digest, exist, err := client.ManifestExist(reference) if ierror.IsErr(err, ierror.UnAuthorizedCode) || ierror.IsErr(err, ierror.ForbiddenCode) {
return exist, digest, err return nil
}
// PullManifest ...
func (a *Adapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) {
client, err := a.getClient(repository)
if err != nil {
return nil, "", err
} }
digest, mediaType, payload, err := client.PullManifest(reference, accepttedMediaTypes)
if err != nil {
return nil, "", err
}
if strings.Contains(mediaType, "application/json") {
mediaType = schema1.MediaTypeManifest
}
manifest, _, err := registry_pkg.UnMarshal(mediaType, payload)
if err != nil {
return nil, "", err
}
return manifest, digest, nil
}
// PushManifest ...
func (a *Adapter) PushManifest(repository, reference, mediaType string, payload []byte) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
_, err = client.PushManifest(reference, mediaType, payload)
return err return err
} }
// DeleteManifest ...
func (a *Adapter) DeleteManifest(repository, reference string) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
digest := reference
if !isDigest(digest) {
dgt, exist, err := client.ManifestExist(reference)
if err != nil {
return err
}
if !exist {
log.Debugf("the manifest of %s:%s doesn't exist", repository, reference)
return nil
}
digest = dgt
}
return client.DeleteManifest(digest)
}
// BlobExist ...
func (a *Adapter) BlobExist(repository, digest string) (bool, error) {
client, err := a.getClient(repository)
if err != nil {
return false, err
}
return client.BlobExist(digest)
}
// PullBlob ...
func (a *Adapter) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
client, err := a.getClient(repository)
if err != nil {
return 0, nil, err
}
return client.PullBlob(digest)
}
// PushBlob ...
func (a *Adapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
return client.PushBlob(digest, size, blob)
}
func isDigest(str string) bool {
return strings.Contains(str, ":")
}
// ListTag ...
func (a *Adapter) ListTag(repository string) ([]string, error) {
client, err := a.getClient(repository)
if err != nil {
return []string{}, err
}
return client.ListTag()
}
func (a *Adapter) getClient(repository string) (*registry_pkg.Repository, error) {
a.RLock()
client, exist := a.clients[repository]
a.RUnlock()
if exist {
return client, nil
}
return a.create(repository)
}
func (a *Adapter) create(repository string) (*registry_pkg.Repository, error) {
a.Lock()
defer a.Unlock()
// double check
client, exist := a.clients[repository]
if exist {
return client, nil
}
client, err := registry_pkg.NewRepository(repository, a.registry.URL, a.client)
if err != nil {
return nil, err
}
a.clients[repository] = client
return client, nil
}

57
src/replication/adapter/native/adapter_test.go
View File

@ -26,33 +26,9 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func Test_newAdapter(t *testing.T) {
tests := []struct {
name string
registry *model.Registry
wantErr bool
}{
{name: "Nil Registry URL", registry: &model.Registry{}, wantErr: true},
{name: "Right", registry: &model.Registry{URL: "abc"}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NewAdapter(tt.registry)
if tt.wantErr {
assert.NotNil(t, err)
assert.Nil(t, got)
} else {
assert.Nil(t, err)
assert.NotNil(t, got)
}
})
}
}
func Test_native_Info(t *testing.T) { func Test_native_Info(t *testing.T) {
var registry = &model.Registry{URL: "abc"} var registry = &model.Registry{URL: "abc"}
adapter, err := NewAdapter(registry) adapter := NewAdapter(registry)
require.Nil(t, err)
assert.NotNil(t, adapter) assert.NotNil(t, adapter)
info, err := adapter.Info() info, err := adapter.Info()
@ -67,11 +43,10 @@ func Test_native_Info(t *testing.T) {
func Test_native_PrepareForPush(t *testing.T) { func Test_native_PrepareForPush(t *testing.T) {
var registry = &model.Registry{URL: "abc"} var registry = &model.Registry{URL: "abc"}
adapter, err := NewAdapter(registry) adapter := NewAdapter(registry)
require.Nil(t, err)
assert.NotNil(t, adapter) assert.NotNil(t, adapter)
err = adapter.PrepareForPush(nil) err := adapter.PrepareForPush(nil)
assert.Nil(t, err) assert.Nil(t, err)
} }
@ -117,8 +92,7 @@ func Test_native_FetchImages(t *testing.T) {
URL: mock.URL, URL: mock.URL,
Insecure: true, Insecure: true,
} }
adapter, err := NewAdapter(registry) adapter := NewAdapter(registry)
assert.Nil(t, err)
assert.NotNil(t, adapter) assert.NotNil(t, adapter)
tests := []struct { tests := []struct {
@ -320,26 +294,3 @@ func Test_native_FetchImages(t *testing.T) {
}) })
} }
} }
func TestIsDigest(t *testing.T) {
cases := []struct {
str string
isDigest bool
}{
{
str: "",
isDigest: false,
},
{
str: "latest",
isDigest: false,
},
{
str: "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
isDigest: true,
},
}
for _, c := range cases {
assert.Equal(t, c.isDigest, isDigest(c.str))
}
}

18
src/replication/adapter/quayio/adapter.go
View File

@ -12,7 +12,6 @@ import (
common_http "github.com/goharbor/harbor/src/common/http" common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier" "github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
@ -35,27 +34,16 @@ func init() {
} }
func newAdapter(registry *model.Registry) (*adapter, error) { func newAdapter(registry *model.Registry) (*adapter, error) {
modifiers := []modifier.Modifier{ modifiers := []modifier.Modifier{}
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
},
}
var authorizer modifier.Modifier var authorizer modifier.Modifier
if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 { if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 {
authorizer = auth.NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), auth.APIKeyInHeader) authorizer = NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), APIKeyInHeader)
} }
if authorizer != nil { if authorizer != nil {
modifiers = append(modifiers, authorizer) modifiers = append(modifiers, authorizer)
} }
nativeRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{ return &adapter{
Adapter: nativeRegistryAdapter, Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
registry: registry, registry: registry,
client: common_http.NewClient( client: common_http.NewClient(
&http.Client{ &http.Client{

2
src/replication/adapter/quayio/adapter_test.go
View File

@ -41,7 +41,7 @@ func TestAdapter_Info(t *testing.T) {
func TestAdapter_PullManifests(t *testing.T) { func TestAdapter_PullManifests(t *testing.T) {
quayAdapter := getMockAdapter(t) quayAdapter := getMockAdapter(t)
registry, _, err := quayAdapter.(*adapter).PullManifest("quay/busybox", "latest", []string{}) registry, _, err := quayAdapter.(*adapter).PullManifest("quay/busybox", "latest")
assert.Nil(t, err) assert.Nil(t, err)
assert.NotNil(t, registry) assert.NotNil(t, registry)
t.Log(registry) t.Log(registry)

2
src/common/utils/registry/auth/apikey.go → src/replication/adapter/quayio/apikey.go
View File

@ -1,4 +1,4 @@
package auth package quayio
import ( import (
"fmt" "fmt"

2
src/common/utils/registry/auth/apikey_test.go → src/replication/adapter/quayio/apikey_test.go
View File

@ -1,4 +1,4 @@
package auth package quayio
import ( import (
"net/http" "net/http"

6
src/replication/operation/flow/stage_test.go
View File

@ -77,11 +77,11 @@ func (f *fakedAdapter) FetchImages(filters []*model.Filter) ([]*model.Resource,
func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) { func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) {
return false, "", nil return false, "", nil
} }
func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) { func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error) {
return nil, "", nil return nil, "", nil
} }
func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) error { func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
return nil return "", nil
} }
func (f *fakedAdapter) DeleteManifest(repository, digest string) error { func (f *fakedAdapter) DeleteManifest(repository, digest string) error {
return nil return nil

16
src/replication/transfer/image/transfer.go
View File

@ -16,12 +16,10 @@ package image
import ( import (
"errors" "errors"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"strings" "strings"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution" "github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/adapter" "github.com/goharbor/harbor/src/replication/adapter"
@ -207,7 +205,7 @@ func (t *transfer) copyContent(content distribution.Descriptor, srcRepo, dstRepo
switch content.MediaType { switch content.MediaType {
// when the media type of pulled manifest is manifest list, // when the media type of pulled manifest is manifest list,
// the contents it contains are a few manifests // the contents it contains are a few manifests
case schema2.MediaTypeManifest: case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
// as using digest as the reference, so set the override to true directly // as using digest as the reference, so set the override to true directly
return t.copyImage(srcRepo, digest, dstRepo, digest, true) return t.copyImage(srcRepo, digest, dstRepo, digest, true)
// handle foreign layer // handle foreign layer
@ -258,13 +256,7 @@ func (t *transfer) pullManifest(repository, reference string) (
return nil, "", nil return nil, "", nil
} }
t.logger.Infof("pulling the manifest of image %s:%s ...", repository, reference) t.logger.Infof("pulling the manifest of image %s:%s ...", repository, reference)
// TODO add OCI media types manifest, digest, err := t.src.PullManifest(repository, reference)
manifest, digest, err := t.src.PullManifest(repository, reference, []string{
schema1.MediaTypeManifest,
schema1.MediaTypeSignedManifest,
schema2.MediaTypeManifest,
manifestlist.MediaTypeManifestList,
})
if err != nil { if err != nil {
t.logger.Errorf("failed to pull the manifest of image %s:%s: %v", repository, reference, err) t.logger.Errorf("failed to pull the manifest of image %s:%s: %v", repository, reference, err)
return nil, "", err return nil, "", err
@ -295,7 +287,7 @@ func (t *transfer) pushManifest(manifest distribution.Manifest, repository, tag
repository, tag, err) repository, tag, err)
return err return err
} }
if err := t.dst.PushManifest(repository, tag, mediaType, payload); err != nil { if _, err := t.dst.PushManifest(repository, tag, mediaType, payload); err != nil {
t.logger.Errorf("failed to push manifest of image %s:%s: %v", t.logger.Errorf("failed to push manifest of image %s:%s: %v",
repository, tag, err) repository, tag, err)
return err return err

9
src/replication/transfer/image/transfer_test.go
View File

@ -23,7 +23,6 @@ import (
"github.com/docker/distribution" "github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/log"
pkg_registry "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
trans "github.com/goharbor/harbor/src/replication/transfer" trans "github.com/goharbor/harbor/src/replication/transfer"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -42,7 +41,7 @@ func (f *fakeRegistry) ManifestExist(repository, reference string) (bool, string
} }
return false, "sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", nil return false, "sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", nil
} }
func (f *fakeRegistry) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) { func (f *fakeRegistry) PullManifest(repository, reference string, accepttedMediaTypes ...string) (distribution.Manifest, string, error) {
manifest := `{ manifest := `{
"schemaVersion": 2, "schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json", "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
@ -71,14 +70,14 @@ func (f *fakeRegistry) PullManifest(repository, reference string, accepttedMedia
}` }`
mediaType := schema2.MediaTypeManifest mediaType := schema2.MediaTypeManifest
payload := []byte(manifest) payload := []byte(manifest)
mani, _, err := pkg_registry.UnMarshal(mediaType, payload) mani, _, err := distribution.UnmarshalManifest(mediaType, payload)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
return mani, "sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", nil return mani, "sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", nil
} }
func (f *fakeRegistry) PushManifest(repository, reference, mediaType string, payload []byte) error { func (f *fakeRegistry) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
return nil return "", nil
} }
func (f *fakeRegistry) DeleteManifest(repository, reference string) error { func (f *fakeRegistry) DeleteManifest(repository, reference string) error {
return nil return nil

5
src/replication/util/util.go
View File

@ -15,15 +15,14 @@
package util package util
import ( import (
"github.com/goharbor/harbor/src/internal"
"net/http" "net/http"
"strings" "strings"
"github.com/goharbor/harbor/src/common/utils/registry"
) )
// GetHTTPTransport can be used to share the common HTTP transport // GetHTTPTransport can be used to share the common HTTP transport
func GetHTTPTransport(insecure bool) *http.Transport { func GetHTTPTransport(insecure bool) *http.Transport {
return registry.GetHTTPTransport(insecure) return internal.GetHTTPTransport(insecure)
} }
// ParseRepository parses the "repository" provided into two parts: namespace and the rest // ParseRepository parses the "repository" provided into two parts: namespace and the rest

90
src/testing/pkg/registry/client.go
View File

@ -15,7 +15,9 @@
package registry package registry
import ( import (
"github.com/docker/distribution"
"github.com/stretchr/testify/mock" "github.com/stretchr/testify/mock"
"io"
) )
// FakeClient is a fake registry client that implement src/pkg/registry.Client interface // FakeClient is a fake registry client that implement src/pkg/registry.Client interface
@ -23,6 +25,94 @@ type FakeClient struct {
mock.Mock mock.Mock
} }
// Ping ...
func (f *FakeClient) Ping() (err error) {
args := f.Called()
return args.Error(0)
}
// Catalog ...
func (f *FakeClient) Catalog() ([]string, error) {
args := f.Called()
var repositories []string
if args[0] != nil {
repositories = args[0].([]string)
}
return repositories, args.Error(1)
}
// ListTags ...
func (f *FakeClient) ListTags(repository string) ([]string, error) {
args := f.Called()
var tags []string
if args[0] != nil {
tags = args[0].([]string)
}
return tags, args.Error(1)
}
// ManifestExist ...
func (f *FakeClient) ManifestExist(repository, reference string) (bool, string, error) {
args := f.Called()
return args.Bool(0), args.String(1), args.Error(2)
}
// PullManifest ...
func (f *FakeClient) PullManifest(repository, reference string, acceptedMediaTypes ...string) (distribution.Manifest, string, error) {
args := f.Called()
var manifest distribution.Manifest
if args[0] != nil {
manifest = args[0].(distribution.Manifest)
}
return manifest, args.String(1), args.Error(2)
}
// PushManifest ...
func (f *FakeClient) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
args := f.Called()
return args.String(0), args.Error(1)
}
// DeleteManifest ...
func (f *FakeClient) DeleteManifest(repository, reference string) error {
args := f.Called()
return args.Error(0)
}
// BlobExist ...
func (f *FakeClient) BlobExist(repository, digest string) (bool, error) {
args := f.Called()
return args.Bool(0), args.Error(1)
}
// PullBlob ...
func (f *FakeClient) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
args := f.Called()
var blob io.ReadCloser
if args[0] != nil {
blob = args[0].(io.ReadCloser)
}
return int64(args.Int(0)), blob, args.Error(2)
}
// PushBlob ...
func (f *FakeClient) PushBlob(repository, digest string, size int64, blob io.Reader) error {
args := f.Called()
return args.Error(0)
}
// MountBlob ...
func (f *FakeClient) MountBlob(srcRepository, digest, dstRepository string) (err error) {
args := f.Called()
return args.Error(0)
}
// DeleteBlob ...
func (f *FakeClient) DeleteBlob(repository, digest string) (err error) {
args := f.Called()
return args.Error(0)
}
// Copy ... // Copy ...
func (f *FakeClient) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error { func (f *FakeClient) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
args := f.Called() args := f.Called()