Merge pull request #8206 from wy65701436/project-quota-yan-linux

Refactor interceptors code with chain
This commit is contained in:
Wang Yan 2019-07-09 13:29:18 +08:00 committed by GitHub
commit 5660e5512b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1509 additions and 517 deletions

View File

@ -32,7 +32,7 @@ import (
"github.com/goharbor/harbor/src/common/models"
utilstest "github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/proxy"
"github.com/goharbor/harbor/src/core/middlewares"
"github.com/stretchr/testify/assert"
)
@ -102,8 +102,9 @@ func TestRedirectForOIDC(t *testing.T) {
// TestMain is a sample to run an endpoint test
func TestAll(t *testing.T) {
config.InitWithSettings(utilstest.GetUnitTestConfig())
proxy.Init()
assert := assert.New(t)
err := middlewares.Init()
assert.Nil(err)
r, _ := http.NewRequest("POST", "/c/login", nil)
w := httptest.NewRecorder()

View File

@ -2,7 +2,7 @@ package controllers
import (
"github.com/astaxie/beego"
"github.com/goharbor/harbor/src/core/proxy"
"github.com/goharbor/harbor/src/core/middlewares"
)
// RegistryProxy is the endpoint on UI for a reverse proxy pointing to registry
@ -14,7 +14,7 @@ type RegistryProxy struct {
func (p *RegistryProxy) Handle() {
req := p.Ctx.Request
rw := p.Ctx.ResponseWriter
proxy.Handle(rw, req)
middlewares.Handle(rw, req)
}
// Render ...

View File

@ -35,7 +35,7 @@ import (
_ "github.com/goharbor/harbor/src/core/auth/uaa"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/src/core/proxy"
"github.com/goharbor/harbor/src/core/middlewares"
"github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/replication"
)
@ -158,7 +158,9 @@ func main() {
}
log.Info("Init proxy")
proxy.Init()
if err := middlewares.Init(); err != nil {
log.Errorf("init proxy error, %v", err)
}
// go proxy.StartProxy()
beego.Run()
}

View File

@ -0,0 +1,120 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blobquota
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"net/http"
"time"
)
type blobQuotaHandler struct {
next http.Handler
}
// New ...
func New(next http.Handler) http.Handler {
return &blobQuotaHandler{
next: next,
}
}
// ServeHTTP ...
func (bqh blobQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if req.Method == http.MethodPut {
match, _ := util.MatchPutBlobURL(req)
if match {
dgstStr := req.FormValue("digest")
if dgstStr == "" {
http.Error(rw, util.MarshalError("InternalServerError", "blob digest missing"), http.StatusInternalServerError)
return
}
dgst, err := digest.Parse(dgstStr)
if err != nil {
http.Error(rw, util.MarshalError("InternalServerError", "blob digest parsing failed"), http.StatusInternalServerError)
return
}
// ToDo lock digest with redis
// ToDo read placeholder from config
state, err := hmacKey("placeholder").unpackUploadState(req.FormValue("_state"))
if err != nil {
http.Error(rw, util.MarshalError("InternalServerError", "failed to decode state"), http.StatusInternalServerError)
return
}
log.Infof("we need to insert blob data into DB.")
log.Infof("blob digest, %v", dgst)
log.Infof("blob size, %v", state.Offset)
}
}
bqh.next.ServeHTTP(rw, req)
}
// blobUploadState captures the state serializable state of the blob upload.
type blobUploadState struct {
// name is the primary repository under which the blob will be linked.
Name string
// UUID identifies the upload.
UUID string
// offset contains the current progress of the upload.
Offset int64
// StartedAt is the original start time of the upload.
StartedAt time.Time
}
type hmacKey string
var errInvalidSecret = errors.New("invalid secret")
// unpackUploadState unpacks and validates the blob upload state from the
// token, using the hmacKey secret.
func (secret hmacKey) unpackUploadState(token string) (blobUploadState, error) {
var state blobUploadState
tokenBytes, err := base64.URLEncoding.DecodeString(token)
if err != nil {
return state, err
}
mac := hmac.New(sha256.New, []byte(secret))
if len(tokenBytes) < mac.Size() {
return state, errInvalidSecret
}
macBytes := tokenBytes[:mac.Size()]
messageBytes := tokenBytes[mac.Size():]
mac.Write(messageBytes)
if !hmac.Equal(mac.Sum(nil), macBytes) {
return state, errInvalidSecret
}
if err := json.Unmarshal(messageBytes, &state); err != nil {
return state, err
}
return state, nil
}

View File

@ -0,0 +1,72 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package middlewares
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/blobquota"
"github.com/goharbor/harbor/src/core/middlewares/contenttrust"
"github.com/goharbor/harbor/src/core/middlewares/listrepo"
"github.com/goharbor/harbor/src/core/middlewares/multiplmanifest"
"github.com/goharbor/harbor/src/core/middlewares/readonly"
"github.com/goharbor/harbor/src/core/middlewares/regquota"
"github.com/goharbor/harbor/src/core/middlewares/url"
"github.com/goharbor/harbor/src/core/middlewares/vulnerable"
"github.com/justinas/alice"
"net/http"
)
// DefaultCreator ...
type DefaultCreator struct {
middlewares []string
}
// New ...
func New(middlewares []string) *DefaultCreator {
return &DefaultCreator{
middlewares: middlewares,
}
}
// Create creates a middleware chain ...
func (b *DefaultCreator) Create() *alice.Chain {
chain := alice.New()
for _, mName := range b.middlewares {
middlewareName := mName
chain = chain.Append(func(next http.Handler) http.Handler {
constructor := b.geMiddleware(middlewareName)
if constructor == nil {
log.Errorf("cannot init middle %s", middlewareName)
return nil
}
return constructor(next)
})
}
return &chain
}
func (b *DefaultCreator) geMiddleware(mName string) alice.Constructor {
middlewares := map[string]alice.Constructor{
READONLY: func(next http.Handler) http.Handler { return readonly.New(next) },
URL: func(next http.Handler) http.Handler { return url.New(next) },
MUITIPLEMANIFEST: func(next http.Handler) http.Handler { return multiplmanifest.New(next) },
LISTREPO: func(next http.Handler) http.Handler { return listrepo.New(next) },
CONTENTTRUST: func(next http.Handler) http.Handler { return contenttrust.New(next) },
VULNERABLE: func(next http.Handler) http.Handler { return vulnerable.New(next) },
REGQUOTA: func(next http.Handler) http.Handler { return regquota.New(next) },
BLOBQUOTA: func(next http.Handler) http.Handler { return blobquota.New(next) },
}
return middlewares[mName]
}

View File

@ -0,0 +1,30 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package middlewares
// const variables
const (
READONLY = "readonly"
URL = "url"
MUITIPLEMANIFEST = "manifest"
LISTREPO = "listrepo"
CONTENTTRUST = "contenttrust"
VULNERABLE = "vulnerable"
REGQUOTA = "regquota"
BLOBQUOTA = "blobquota"
)
// Middlewares with sequential organization
var Middlewares = []string{READONLY, URL, MUITIPLEMANIFEST, LISTREPO, CONTENTTRUST, VULNERABLE, BLOBQUOTA, REGQUOTA}

View File

@ -0,0 +1,106 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package contenttrust
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/notary"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
"strings"
)
// NotaryEndpoint ...
var NotaryEndpoint = ""
type contentTrustHandler struct {
next http.Handler
}
// New ...
func New(next http.Handler) http.Handler {
return &contentTrustHandler{
next: next,
}
}
// ServeHTTP ...
func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(util.ImageInfoCtxKey)
if imgRaw == nil || !config.WithNotary() {
cth.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo)
if img.Digest == "" {
cth.next.ServeHTTP(rw, req)
return
}
if !util.GetPolicyChecker().ContentTrustEnabled(img.ProjectName) {
cth.next.ServeHTTP(rw, req)
return
}
match, err := matchNotaryDigest(img)
if err != nil {
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError)
return
}
if !match {
log.Debugf("digest mismatch, failing the response.")
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed)
return
}
cth.next.ServeHTTP(rw, req)
}
func matchNotaryDigest(img util.ImageInfo) (bool, error) {
if NotaryEndpoint == "" {
NotaryEndpoint = config.InternalNotaryEndpoint()
}
targets, err := notary.GetInternalTargets(NotaryEndpoint, util.TokenUsername, img.Repository)
if err != nil {
return false, err
}
for _, t := range targets {
if isDigest(img.Reference) {
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if img.Digest == d {
return true, nil
}
} else {
if t.Tag == img.Reference {
log.Debugf("found reference: %s in notary, try to match digest.", img.Reference)
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if img.Digest == d {
return true, nil
}
}
}
}
log.Debugf("image: %#v, not found in notary", img)
return false, nil
}
// A sha256 is a string with 64 characters.
func isDigest(ref string) bool {
return strings.HasPrefix(ref, "sha256:") && len(ref) == 71
}

View File

@ -0,0 +1,69 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package contenttrust
import (
"github.com/goharbor/harbor/src/common"
notarytest "github.com/goharbor/harbor/src/common/utils/notary/test"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"github.com/stretchr/testify/assert"
"net/http/httptest"
"os"
"testing"
)
var endpoint = "10.117.4.142"
var notaryServer *httptest.Server
var admiralEndpoint = "http://127.0.0.1:8282"
var token = ""
func TestMain(m *testing.M) {
notaryServer = notarytest.NewNotaryServer(endpoint)
defer notaryServer.Close()
NotaryEndpoint = notaryServer.URL
var defaultConfig = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
common.TokenExpiration: 30,
}
config.InitWithSettings(defaultConfig)
result := m.Run()
if result != 0 {
os.Exit(result)
}
}
func TestMatchNotaryDigest(t *testing.T) {
assert := assert.New(t)
// The data from common/utils/notary/helper_test.go
img1 := util.ImageInfo{Repository: "notary-demo/busybox", Reference: "1.0", ProjectName: "notary-demo", Digest: "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
img2 := util.ImageInfo{Repository: "notary-demo/busybox", Reference: "2.0", ProjectName: "notary-demo", Digest: "sha256:12345678"}
res1, err := matchNotaryDigest(img1)
assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)
assert.True(res1)
res2, err := matchNotaryDigest(img2)
assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2)
assert.False(res2)
}
func TestIsDigest(t *testing.T) {
assert := assert.New(t)
assert.False(isDigest("latest"))
assert.True(isDigest("sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"))
}

View File

@ -0,0 +1,39 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package middlewares
import (
"errors"
"github.com/goharbor/harbor/src/core/middlewares/registryproxy"
"net/http"
)
var head http.Handler
// Init initialize the Proxy instance and handler chain.
func Init() error {
ph := registryproxy.New()
if ph == nil {
return errors.New("get nil when to create proxy")
}
handlerChain := New(Middlewares).Create()
head = handlerChain.Then(ph)
return nil
}
// Handle handles the request.
func Handle(rw http.ResponseWriter, req *http.Request) {
head.ServeHTTP(rw, req)
}

View File

@ -0,0 +1,22 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package middlewares
import "github.com/justinas/alice"
// ChainCreator ...
type ChainCreator interface {
Create(middlewares []string) *alice.Chain
}

View File

@ -0,0 +1,104 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package listrepo
import (
"encoding/json"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
"net/http/httptest"
"regexp"
"strconv"
)
const (
catalogURLPattern = `/v2/_catalog`
)
type listReposHandler struct {
next http.Handler
}
// New ...
func New(next http.Handler) http.Handler {
return &listReposHandler{
next: next,
}
}
// ServeHTTP ...
func (lrh listReposHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
var rec *httptest.ResponseRecorder
listReposFlag := matchListRepos(req)
if listReposFlag {
rec = httptest.NewRecorder()
lrh.next.ServeHTTP(rec, req)
if rec.Result().StatusCode != http.StatusOK {
util.CopyResp(rec, rw)
return
}
var ctlg struct {
Repositories []string `json:"repositories"`
}
decoder := json.NewDecoder(rec.Body)
if err := decoder.Decode(&ctlg); err != nil {
log.Errorf("Decode repositories error: %v", err)
util.CopyResp(rec, rw)
return
}
var entries []string
for repo := range ctlg.Repositories {
log.Debugf("the repo in the response %s", ctlg.Repositories[repo])
exist := dao.RepositoryExists(ctlg.Repositories[repo])
if exist {
entries = append(entries, ctlg.Repositories[repo])
}
}
type Repos struct {
Repositories []string `json:"repositories"`
}
resp := &Repos{Repositories: entries}
respJSON, err := json.Marshal(resp)
if err != nil {
log.Errorf("Encode repositories error: %v", err)
util.CopyResp(rec, rw)
return
}
for k, v := range rec.Header() {
rw.Header()[k] = v
}
clen := len(respJSON)
rw.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(clen))
rw.Write(respJSON)
return
}
lrh.next.ServeHTTP(rw, req)
}
// matchListRepos checks if the request looks like a request to list repositories.
func matchListRepos(req *http.Request) bool {
if req.Method != http.MethodGet {
return false
}
re := regexp.MustCompile(catalogURLPattern)
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 1 {
return true
}
return false
}

View File

@ -0,0 +1,37 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package listrepo
import (
"github.com/stretchr/testify/assert"
"net/http"
"testing"
)
func TestMatchListRepos(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil)
res1 := matchListRepos(req1)
assert.False(res1, "%s %v is not a request to list repos", req1.Method, req1.URL)
req2, _ := http.NewRequest("GET", "http://127.0.0.1:5000/v2/_catalog", nil)
res2 := matchListRepos(req2)
assert.True(res2, "%s %v is a request to list repos", req2.Method, req2.URL)
req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/_catalog", nil)
res3 := matchListRepos(req3)
assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL)
}

View File

@ -0,0 +1,48 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package multiplmanifest
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
"strings"
)
type multipleManifestHandler struct {
next http.Handler
}
// New ...
func New(next http.Handler) http.Handler {
return &multipleManifestHandler{
next: next,
}
}
// ServeHTTP The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor.
func (mh multipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
match, _, _ := util.MatchPushManifest(req)
if match {
contentType := req.Header.Get("Content-type")
// application/vnd.docker.distribution.manifest.list.v2+json
if strings.Contains(contentType, "manifest.list.v2") {
log.Debugf("Content-type: %s is not supported, failing the response.", contentType)
http.Error(rw, util.MarshalError("UNSUPPORTED_MEDIA_TYPE", "Manifest.list is not supported."), http.StatusUnsupportedMediaType)
return
}
}
mh.next.ServeHTTP(rw, req)
}

View File

@ -0,0 +1,45 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package readonly
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
)
type readonlyHandler struct {
next http.Handler
}
// New ...
func New(next http.Handler) http.Handler {
return &readonlyHandler{
next: next,
}
}
// ServeHTTP ...
func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if config.ReadOnly() {
if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch || req.Method == http.MethodPut {
log.Warningf("The request is prohibited in readonly mode, url is: %s", req.URL.Path)
http.Error(rw, util.MarshalError("DENIED", "The system is in read only mode. Any modification is prohibited."), http.StatusForbidden)
return
}
}
rh.next.ServeHTTP(rw, req)
}

View File

@ -0,0 +1,123 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registryproxy
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
"net/http/httputil"
"net/url"
"strings"
)
type proxyHandler struct {
handler http.Handler
}
// New ...
func New(urls ...string) http.Handler {
var registryURL string
var err error
if len(urls) > 1 {
log.Errorf("the parm, urls should have only 0 or 1 elements")
return nil
}
if len(urls) == 0 {
registryURL, err = config.RegistryURL()
if err != nil {
log.Error(err)
return nil
}
} else {
registryURL = urls[0]
}
targetURL, err := url.Parse(registryURL)
if err != nil {
log.Error(err)
return nil
}
return &proxyHandler{
handler: &httputil.ReverseProxy{
Director: func(req *http.Request) {
director(targetURL, req)
},
ModifyResponse: modifyResponse,
},
}
}
// Overwrite the http requests
func director(target *url.URL, req *http.Request) {
targetQuery := target.RawQuery
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
} else {
req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
}
if _, ok := req.Header["User-Agent"]; !ok {
// explicitly disable User-Agent so it's not set to default value
req.Header.Set("User-Agent", "")
}
}
// Modify the http response
func modifyResponse(res *http.Response) error {
if res.Request.Method == http.MethodPut {
// PUT manifest
matchMF, _, _ := util.MatchManifestURL(res.Request)
if matchMF {
if res.StatusCode == http.StatusCreated {
log.Infof("we need to insert data here ... ")
} else if res.StatusCode >= 202 || res.StatusCode <= 511 {
log.Infof("we need to roll back data here ... ")
}
}
// PUT blob
matchBB, _ := util.MatchPutBlobURL(res.Request)
if matchBB {
if res.StatusCode != http.StatusCreated {
log.Infof("we need to rollback DB and unlock digest ... ")
}
}
}
return nil
}
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
// ServeHTTP ...
func (ph proxyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
ph.handler.ServeHTTP(rw, req)
}

View File

@ -0,0 +1,74 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package regquota
import (
"bytes"
"fmt"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"io/ioutil"
"net/http"
)
type regQuotaHandler struct {
next http.Handler
}
// New ...
func New(next http.Handler) http.Handler {
return &regQuotaHandler{
next: next,
}
}
// ServeHTTP PATCH manifest ...
func (rqh regQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
match, _, _ := util.MatchManifestURL(req)
if match {
var mfSize int64
var mfDigest string
mediaType := req.Header.Get("Content-Type")
if req.Method == http.MethodPut {
if mediaType == schema1.MediaTypeManifest ||
mediaType == schema1.MediaTypeSignedManifest ||
mediaType == schema2.MediaTypeManifest {
data, err := ioutil.ReadAll(req.Body)
if err != nil {
log.Warningf("Error occurred when to copy manifest body %v", err)
http.Error(rw, util.MarshalError("InternalServerError", fmt.Sprintf("Error occurred when to decode manifest body %v", err)), http.StatusInternalServerError)
return
}
req.Body = ioutil.NopCloser(bytes.NewBuffer(data))
_, desc, err := distribution.UnmarshalManifest(mediaType, data)
if err != nil {
log.Warningf("Error occurred when to Unmarshal Manifest %v", err)
http.Error(rw, util.MarshalError("InternalServerError", fmt.Sprintf("Error occurred when to Unmarshal Manifest %v", err)), http.StatusInternalServerError)
return
}
mfDigest = desc.Digest.String()
mfSize = desc.Size
log.Infof("manifest digest... %s", mfDigest)
log.Infof("manifest size... %v", mfSize)
}
}
}
rqh.next.ServeHTTP(rw, req)
}

View File

@ -0,0 +1,74 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package url
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
coreutils "github.com/goharbor/harbor/src/core/utils"
"net/http"
"strings"
)
type urlHandler struct {
next http.Handler
}
// New ...
func New(next http.Handler) http.Handler {
return &urlHandler{
next: next,
}
}
// ServeHTTP ...
func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
log.Debugf("in url handler, path: %s", req.URL.Path)
flag, repository, reference := util.MatchPullManifest(req)
if flag {
components := strings.SplitN(repository, "/", 2)
if len(components) < 2 {
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest)
return
}
client, err := coreutils.NewRepositoryClientForUI(util.TokenUsername, repository)
if err != nil {
log.Errorf("Error creating repository Client: %v", err)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
return
}
digest, _, err := client.ManifestExist(reference)
if err != nil {
log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
return
}
img := util.ImageInfo{
Repository: repository,
Reference: reference,
ProjectName: components[0],
Digest: digest,
}
log.Debugf("image info of the request: %#v", img)
ctx := context.WithValue(req.Context(), util.ImageInfoCtxKey, img)
req = req.WithContext(ctx)
}
uh.next.ServeHTTP(rw, req)
}

View File

@ -0,0 +1,176 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"encoding/json"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
"net/http"
"net/http/httptest"
"regexp"
"strings"
)
type contextKey string
const (
manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`
blobURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)blobs/uploads/`
// ImageInfoCtxKey the context key for image information
ImageInfoCtxKey = contextKey("ImageInfo")
// TokenUsername ...
// TODO: temp solution, remove after vmware/harbor#2242 is resolved.
TokenUsername = "harbor-core"
)
// ImageInfo ...
type ImageInfo struct {
Repository string
Reference string
ProjectName string
Digest string
}
// JSONError wraps a concrete Code and Message, it's readable for docker deamon.
type JSONError struct {
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
Detail string `json:"detail,omitempty"`
}
// MarshalError ...
func MarshalError(code, msg string) string {
var tmpErrs struct {
Errors []JSONError `json:"errors,omitempty"`
}
tmpErrs.Errors = append(tmpErrs.Errors, JSONError{
Code: code,
Message: msg,
Detail: msg,
})
str, err := json.Marshal(tmpErrs)
if err != nil {
log.Debugf("failed to marshal json error, %v", err)
return msg
}
return string(str)
}
// MatchManifestURL ...
func MatchManifestURL(req *http.Request) (bool, string, string) {
re, err := regexp.Compile(manifestURLPattern)
if err != nil {
log.Errorf("error to match manifest url, %v", err)
return false, "", ""
}
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 3 {
s[1] = strings.TrimSuffix(s[1], "/")
return true, s[1], s[2]
}
return false, "", ""
}
// MatchPutBlobURL ...
func MatchPutBlobURL(req *http.Request) (bool, string) {
if req.Method != http.MethodPut {
return false, ""
}
re, err := regexp.Compile(blobURLPattern)
if err != nil {
log.Errorf("error to match put blob url, %v", err)
return false, ""
}
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 2 {
s[1] = strings.TrimSuffix(s[1], "/")
return true, s[1]
}
return false, ""
}
// MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPullManifest(req *http.Request) (bool, string, string) {
if req.Method != http.MethodGet {
return false, "", ""
}
return MatchManifestURL(req)
}
// MatchPushManifest checks if the request looks like a request to push manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPushManifest(req *http.Request) (bool, string, string) {
if req.Method != http.MethodPut {
return false, "", ""
}
return MatchManifestURL(req)
}
// CopyResp ...
func CopyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
for k, v := range rec.Header() {
rw.Header()[k] = v
}
rw.WriteHeader(rec.Result().StatusCode)
rw.Write(rec.Body.Bytes())
}
// PolicyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project.
type PolicyChecker interface {
// contentTrustEnabled returns whether a project has enabled content trust.
ContentTrustEnabled(name string) bool
// vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity.
VulnerablePolicy(name string) (bool, models.Severity)
}
// PmsPolicyChecker ...
type PmsPolicyChecker struct {
pm promgr.ProjectManager
}
// ContentTrustEnabled ...
func (pc PmsPolicyChecker) ContentTrustEnabled(name string) bool {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true
}
return project.ContentTrustEnabled()
}
// VulnerablePolicy ...
func (pc PmsPolicyChecker) VulnerablePolicy(name string) (bool, models.Severity) {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true, models.SevUnknown
}
return project.VulPrevented(), clair.ParseClairSev(project.Severity())
}
// NewPMSPolicyChecker returns an instance of an pmsPolicyChecker
func NewPMSPolicyChecker(pm promgr.ProjectManager) PolicyChecker {
return &PmsPolicyChecker{
pm: pm,
}
}
// GetPolicyChecker ...
func GetPolicyChecker() PolicyChecker {
return NewPMSPolicyChecker(config.GlobalProjectMgr)
}

View File

@ -1,4 +1,18 @@
package proxy
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"github.com/goharbor/harbor/src/common"
@ -24,7 +38,6 @@ var token = ""
func TestMain(m *testing.M) {
notaryServer = notarytest.NewNotaryServer(endpoint)
defer notaryServer.Close()
NotaryEndpoint = notaryServer.URL
var defaultConfig = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
@ -78,6 +91,22 @@ func TestMatchPullManifest(t *testing.T) {
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7)
}
func TestMatchPutBlob(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil)
res1, repo1 := MatchPutBlobURL(req1)
assert.True(res1, "%s %v is not a request to put blob", req1.Method, req1.URL)
assert.Equal("library/ubuntu", repo1)
req2, _ := http.NewRequest("PATCH", "http://127.0.0.1:5000/v2/library/blobs/uploads/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil)
res2, _ := MatchPutBlobURL(req2)
assert.False(res2, "%s %v is a request to put blob", req2.Method, req2.URL)
req3, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/manifest/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil)
res3, _ := MatchPutBlobURL(req3)
assert.False(res3, "%s %v is not a request to put blob", req3.Method, req3.URL)
}
func TestMatchPushManifest(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil)
@ -125,22 +154,6 @@ func TestMatchPushManifest(t *testing.T) {
assert.Equal("14.04", tag8)
}
func TestMatchListRepos(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil)
res1 := MatchListRepos(req1)
assert.False(res1, "%s %v is not a request to list repos", req1.Method, req1.URL)
req2, _ := http.NewRequest("GET", "http://127.0.0.1:5000/v2/_catalog", nil)
res2 := MatchListRepos(req2)
assert.True(res2, "%s %v is a request to list repos", req2.Method, req2.URL)
req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/_catalog", nil)
res3 := MatchListRepos(req3)
assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL)
}
func TestPMSPolicyChecker(t *testing.T) {
var defaultConfigAdmiral = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
@ -178,49 +191,28 @@ func TestPMSPolicyChecker(t *testing.T) {
}
}(id)
contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_sev_low")
contentTrustFlag := GetPolicyChecker().ContentTrustEnabled("project_for_test_get_sev_low")
assert.True(t, contentTrustFlag)
projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy("project_for_test_get_sev_low")
projectVulnerableEnabled, projectVulnerableSeverity := GetPolicyChecker().VulnerablePolicy("project_for_test_get_sev_low")
assert.True(t, projectVulnerableEnabled)
assert.Equal(t, projectVulnerableSeverity, models.SevLow)
}
func TestMatchNotaryDigest(t *testing.T) {
assert := assert.New(t)
// The data from common/utils/notary/helper_test.go
img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo", "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo", "sha256:12345678"}
res1, err := matchNotaryDigest(img1)
assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)
assert.True(res1)
res2, err := matchNotaryDigest(img2)
assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2)
assert.False(res2)
}
func TestCopyResp(t *testing.T) {
assert := assert.New(t)
rec1 := httptest.NewRecorder()
rec2 := httptest.NewRecorder()
rec1.Header().Set("X-Test", "mytest")
rec1.WriteHeader(418)
copyResp(rec1, rec2)
CopyResp(rec1, rec2)
assert.Equal(418, rec2.Result().StatusCode)
assert.Equal("mytest", rec2.Header().Get("X-Test"))
}
func TestMarshalError(t *testing.T) {
assert := assert.New(t)
js1 := marshalError("PROJECT_POLICY_VIOLATION", "Not Found")
js1 := MarshalError("PROJECT_POLICY_VIOLATION", "Not Found")
assert.Equal("{\"errors\":[{\"code\":\"PROJECT_POLICY_VIOLATION\",\"message\":\"Not Found\",\"detail\":\"Not Found\"}]}", js1)
js2 := marshalError("DENIED", "The action is denied")
js2 := MarshalError("DENIED", "The action is denied")
assert.Equal("{\"errors\":[{\"code\":\"DENIED\",\"message\":\"The action is denied\",\"detail\":\"The action is denied\"}]}", js2)
}
func TestIsDigest(t *testing.T) {
assert := assert.New(t)
assert.False(isDigest("latest"))
assert.True(isDigest("sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"))
}

View File

@ -0,0 +1,73 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vulnerable
import (
"fmt"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
)
type vulnerableHandler struct {
next http.Handler
}
// New ...
func New(next http.Handler) http.Handler {
return &vulnerableHandler{
next: next,
}
}
// ServeHTTP ...
func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(util.ImageInfoCtxKey)
if imgRaw == nil || !config.WithClair() {
vh.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo)
if img.Digest == "" {
vh.next.ServeHTTP(rw, req)
return
}
projectVulnerableEnabled, projectVulnerableSeverity := util.GetPolicyChecker().VulnerablePolicy(img.ProjectName)
if !projectVulnerableEnabled {
vh.next.ServeHTTP(rw, req)
return
}
overview, err := dao.GetImgScanOverview(img.Digest)
if err != nil {
log.Errorf("failed to get ImgScanOverview with repo: %s, reference: %s, digest: %s. Error: %v", img.Repository, img.Reference, img.Digest, err)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Failed to get ImgScanOverview."), http.StatusPreconditionFailed)
return
}
// severity is 0 means that the image fails to scan or not scanned successfully.
if overview == nil || overview.Sev == 0 {
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Cannot get the image severity."), http.StatusPreconditionFailed)
return
}
imageSev := overview.Sev
if imageSev >= int(projectVulnerableSeverity) {
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", models.Severity(imageSev), projectVulnerableSeverity)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed)
return
}
vh.next.ServeHTTP(rw, req)
}

View File

@ -1,411 +0,0 @@
package proxy
import (
"encoding/json"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/notary"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan"
"context"
"fmt"
"net/http"
"net/http/httptest"
"regexp"
"strconv"
"strings"
)
type contextKey string
const (
manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`
catalogURLPattern = `/v2/_catalog`
imageInfoCtxKey = contextKey("ImageInfo")
// TODO: temp solution, remove after vmware/harbor#2242 is resolved.
tokenUsername = "harbor-core"
)
// Record the docker deamon raw response.
var rec *httptest.ResponseRecorder
// NotaryEndpoint , exported for testing.
var NotaryEndpoint = ""
// MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPullManifest(req *http.Request) (bool, string, string) {
// TODO: add user agent check.
if req.Method != http.MethodGet {
return false, "", ""
}
return matchManifestURL(req)
}
// MatchPushManifest checks if the request looks like a request to push manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPushManifest(req *http.Request) (bool, string, string) {
if req.Method != http.MethodPut {
return false, "", ""
}
return matchManifestURL(req)
}
func matchManifestURL(req *http.Request) (bool, string, string) {
re := regexp.MustCompile(manifestURLPattern)
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 3 {
s[1] = strings.TrimSuffix(s[1], "/")
return true, s[1], s[2]
}
return false, "", ""
}
// MatchListRepos checks if the request looks like a request to list repositories.
func MatchListRepos(req *http.Request) bool {
if req.Method != http.MethodGet {
return false
}
re := regexp.MustCompile(catalogURLPattern)
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 1 {
return true
}
return false
}
// policyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project.
type policyChecker interface {
// contentTrustEnabled returns whether a project has enabled content trust.
contentTrustEnabled(name string) bool
// vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity.
vulnerablePolicy(name string) (bool, models.Severity)
}
type pmsPolicyChecker struct {
pm promgr.ProjectManager
}
func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true
}
return project.ContentTrustEnabled()
}
func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true, models.SevUnknown
}
return project.VulPrevented(), clair.ParseClairSev(project.Severity())
}
// newPMSPolicyChecker returns an instance of an pmsPolicyChecker
func newPMSPolicyChecker(pm promgr.ProjectManager) policyChecker {
return &pmsPolicyChecker{
pm: pm,
}
}
func getPolicyChecker() policyChecker {
return newPMSPolicyChecker(config.GlobalProjectMgr)
}
type imageInfo struct {
repository string
reference string
projectName string
digest string
}
type urlHandler struct {
next http.Handler
}
func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
log.Debugf("in url handler, path: %s", req.URL.Path)
flag, repository, reference := MatchPullManifest(req)
if flag {
components := strings.SplitN(repository, "/", 2)
if len(components) < 2 {
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest)
return
}
client, err := coreutils.NewRepositoryClientForUI(tokenUsername, repository)
if err != nil {
log.Errorf("Error creating repository Client: %v", err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
return
}
digest, _, err := client.ManifestExist(reference)
if err != nil {
log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
return
}
img := imageInfo{
repository: repository,
reference: reference,
projectName: components[0],
digest: digest,
}
log.Debugf("image info of the request: %#v", img)
ctx := context.WithValue(req.Context(), imageInfoCtxKey, img)
req = req.WithContext(ctx)
}
uh.next.ServeHTTP(rw, req)
}
type readonlyHandler struct {
next http.Handler
}
func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if config.ReadOnly() {
if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch || req.Method == http.MethodPut {
log.Warningf("The request is prohibited in readonly mode, url is: %s", req.URL.Path)
http.Error(rw, marshalError("DENIED", "The system is in read only mode. Any modification is prohibited."), http.StatusForbidden)
return
}
}
rh.next.ServeHTTP(rw, req)
}
type multipleManifestHandler struct {
next http.Handler
}
// The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor.
func (mh multipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
match, _, _ := MatchPushManifest(req)
if match {
contentType := req.Header.Get("Content-type")
// application/vnd.docker.distribution.manifest.list.v2+json
if strings.Contains(contentType, "manifest.list.v2") {
log.Debugf("Content-type: %s is not supported, failing the response.", contentType)
http.Error(rw, marshalError("UNSUPPORTED_MEDIA_TYPE", "Manifest.list is not supported."), http.StatusUnsupportedMediaType)
return
}
}
mh.next.ServeHTTP(rw, req)
}
type listReposHandler struct {
next http.Handler
}
func (lrh listReposHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
listReposFlag := MatchListRepos(req)
if listReposFlag {
rec = httptest.NewRecorder()
lrh.next.ServeHTTP(rec, req)
if rec.Result().StatusCode != http.StatusOK {
copyResp(rec, rw)
return
}
var ctlg struct {
Repositories []string `json:"repositories"`
}
decoder := json.NewDecoder(rec.Body)
if err := decoder.Decode(&ctlg); err != nil {
log.Errorf("Decode repositories error: %v", err)
copyResp(rec, rw)
return
}
var entries []string
for repo := range ctlg.Repositories {
log.Debugf("the repo in the response %s", ctlg.Repositories[repo])
exist := dao.RepositoryExists(ctlg.Repositories[repo])
if exist {
entries = append(entries, ctlg.Repositories[repo])
}
}
type Repos struct {
Repositories []string `json:"repositories"`
}
resp := &Repos{Repositories: entries}
respJSON, err := json.Marshal(resp)
if err != nil {
log.Errorf("Encode repositories error: %v", err)
copyResp(rec, rw)
return
}
for k, v := range rec.Header() {
rw.Header()[k] = v
}
clen := len(respJSON)
rw.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(clen))
rw.Write(respJSON)
return
}
lrh.next.ServeHTTP(rw, req)
}
type contentTrustHandler struct {
next http.Handler
}
func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(imageInfoCtxKey)
if imgRaw == nil || !config.WithNotary() {
cth.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
if img.digest == "" {
cth.next.ServeHTTP(rw, req)
return
}
if !getPolicyChecker().contentTrustEnabled(img.projectName) {
cth.next.ServeHTTP(rw, req)
return
}
match, err := matchNotaryDigest(img)
if err != nil {
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError)
return
}
if !match {
log.Debugf("digest mismatch, failing the response.")
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed)
return
}
cth.next.ServeHTTP(rw, req)
}
type vulnerableHandler struct {
next http.Handler
}
func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(imageInfoCtxKey)
if imgRaw == nil || !config.WithClair() {
vh.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
if img.digest == "" {
vh.next.ServeHTTP(rw, req)
return
}
projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy(img.projectName)
if !projectVulnerableEnabled {
vh.next.ServeHTTP(rw, req)
return
}
// TODO: Get whitelist based on project setting
wl, err := dao.GetSysCVEWhitelist()
if err != nil {
log.Errorf("Failed to get the whitelist, error: %v", err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get CVE whitelist."), http.StatusPreconditionFailed)
return
}
vl, err := scan.VulnListByDigest(img.digest)
if err != nil {
log.Errorf("Failed to get the vulnerability list, error: %v", err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get vulnerabilities."), http.StatusPreconditionFailed)
return
}
filtered := vl.ApplyWhitelist(*wl)
msg := vh.filterMsg(img, filtered)
log.Info(msg)
if int(vl.Severity()) >= int(projectVulnerableSeverity) {
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", vl.Severity(), projectVulnerableSeverity)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", vl.Severity(), projectVulnerableSeverity)), http.StatusPreconditionFailed)
return
}
vh.next.ServeHTTP(rw, req)
}
func (vh vulnerableHandler) filterMsg(img imageInfo, filtered scan.VulnerabilityList) string {
filterMsg := fmt.Sprintf("Image: %s/%s:%s, digest: %s, vulnerabilities fitered by whitelist:", img.projectName, img.repository, img.reference, img.digest)
if len(filtered) == 0 {
filterMsg = fmt.Sprintf("%s none.", filterMsg)
}
for _, v := range filtered {
filterMsg = fmt.Sprintf("%s ID: %s, severity: %s;", filterMsg, v.ID, v.Severity)
}
return filterMsg
}
func matchNotaryDigest(img imageInfo) (bool, error) {
if NotaryEndpoint == "" {
NotaryEndpoint = config.InternalNotaryEndpoint()
}
targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository)
if err != nil {
return false, err
}
for _, t := range targets {
if isDigest(img.reference) {
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if img.digest == d {
return true, nil
}
} else {
if t.Tag == img.reference {
log.Debugf("found reference: %s in notary, try to match digest.", img.reference)
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if img.digest == d {
return true, nil
}
}
}
}
log.Debugf("image: %#v, not found in notary", img)
return false, nil
}
// A sha256 is a string with 64 characters.
func isDigest(ref string) bool {
return strings.HasPrefix(ref, "sha256:") && len(ref) == 71
}
func copyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
for k, v := range rec.Header() {
rw.Header()[k] = v
}
rw.WriteHeader(rec.Result().StatusCode)
rw.Write(rec.Body.Bytes())
}
func marshalError(code, msg string) string {
var tmpErrs struct {
Errors []JSONError `json:"errors,omitempty"`
}
tmpErrs.Errors = append(tmpErrs.Errors, JSONError{
Code: code,
Message: msg,
Detail: msg,
})
str, err := json.Marshal(tmpErrs)
if err != nil {
log.Debugf("failed to marshal json error, %v", err)
return msg
}
return string(str)
}
// JSONError wraps a concrete Code and Message, it's readable for docker deamon.
type JSONError struct {
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
Detail string `json:"detail,omitempty"`
}

View File

@ -1,56 +0,0 @@
package proxy
import (
"github.com/goharbor/harbor/src/core/config"
"fmt"
"net/http"
"net/http/httputil"
"net/url"
)
// Proxy is the instance of the reverse proxy in this package.
var Proxy *httputil.ReverseProxy
var handlers handlerChain
type handlerChain struct {
head http.Handler
}
// Init initialize the Proxy instance and handler chain.
func Init(urls ...string) error {
var err error
var registryURL string
if len(urls) > 1 {
return fmt.Errorf("the parm, urls should have only 0 or 1 elements")
}
if len(urls) == 0 {
registryURL, err = config.RegistryURL()
if err != nil {
return err
}
} else {
registryURL = urls[0]
}
targetURL, err := url.Parse(registryURL)
if err != nil {
return err
}
Proxy = httputil.NewSingleHostReverseProxy(targetURL)
handlers = handlerChain{
head: readonlyHandler{
next: urlHandler{
next: multipleManifestHandler{
next: listReposHandler{
next: contentTrustHandler{
next: vulnerableHandler{
next: Proxy,
}}}}}}}
return nil
}
// Handle handles the request.
func Handle(rw http.ResponseWriter, req *http.Request) {
handlers.head.ServeHTTP(rw, req)
}

View File

@ -46,6 +46,7 @@ require (
github.com/gorilla/mux v1.6.2
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/jinzhu/gorm v1.9.8 // indirect
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/lib/pq v1.1.0

View File

@ -169,6 +169,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da h1:5y58+OCjoHCYB8182mpf/dEsq0vwTKPOo4zGfH0xW9A=
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da/go.mod h1:oLH0CmIaxCGXD67VKGR5AacGXZSMznlmeqM8RzPrcY8=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

17
src/vendor/github.com/justinas/alice/.travis.yml generated vendored Normal file
View File

@ -0,0 +1,17 @@
language: go
matrix:
include:
- go: 1.0.x
- go: 1.1.x
- go: 1.2.x
- go: 1.3.x
- go: 1.4.x
- go: 1.5.x
- go: 1.6.x
- go: 1.7.x
- go: 1.8.x
- go: 1.9.x
- go: tip
allow_failures:
- go: tip

20
src/vendor/github.com/justinas/alice/LICENSE generated vendored Normal file
View File

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2014 Justinas Stankevicius
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

98
src/vendor/github.com/justinas/alice/README.md generated vendored Normal file
View File

@ -0,0 +1,98 @@
# Alice
[![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](http://godoc.org/github.com/justinas/alice)
[![Build Status](https://travis-ci.org/justinas/alice.svg?branch=master)](https://travis-ci.org/justinas/alice)
[![Coverage](http://gocover.io/_badge/github.com/justinas/alice)](http://gocover.io/github.com/justinas/alice)
Alice provides a convenient way to chain
your HTTP middleware functions and the app handler.
In short, it transforms
```go
Middleware1(Middleware2(Middleware3(App)))
```
to
```go
alice.New(Middleware1, Middleware2, Middleware3).Then(App)
```
### Why?
None of the other middleware chaining solutions
behaves exactly like Alice.
Alice is as minimal as it gets:
in essence, it's just a for loop that does the wrapping for you.
Check out [this blog post](http://justinas.org/alice-painless-middleware-chaining-for-go/)
for explanation how Alice is different from other chaining solutions.
### Usage
Your middleware constructors should have the form of
```go
func (http.Handler) http.Handler
```
Some middleware provide this out of the box.
For ones that don't, it's trivial to write one yourself.
```go
func myStripPrefix(h http.Handler) http.Handler {
return http.StripPrefix("/old", h)
}
```
This complete example shows the full power of Alice.
```go
package main
import (
"net/http"
"time"
"github.com/throttled/throttled"
"github.com/justinas/alice"
"github.com/justinas/nosurf"
)
func timeoutHandler(h http.Handler) http.Handler {
return http.TimeoutHandler(h, 1*time.Second, "timed out")
}
func myApp(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hello world!"))
}
func main() {
th := throttled.Interval(throttled.PerSec(10), 1, &throttled.VaryBy{Path: true}, 50)
myHandler := http.HandlerFunc(myApp)
chain := alice.New(th.Throttle, timeoutHandler, nosurf.NewPure).Then(myHandler)
http.ListenAndServe(":8000", chain)
}
```
Here, the request will pass [throttled](https://github.com/PuerkitoBio/throttled) first,
then an http.TimeoutHandler we've set up,
then [nosurf](https://github.com/justinas/nosurf)
and will finally reach our handler.
Note that Alice makes **no guarantees** for
how one or another piece of middleware will behave.
Once it passes the execution to the outer layer of middleware,
it has no saying in whether middleware will execute the inner handlers.
This is intentional behavior.
Alice works with Go 1.0 and higher.
### Contributing
0. Find an issue that bugs you / open a new one.
1. Discuss.
2. Branch off, commit, test.
3. Make a pull request / attach the commits to the issue.

112
src/vendor/github.com/justinas/alice/chain.go generated vendored Normal file
View File

@ -0,0 +1,112 @@
// Package alice provides a convenient way to chain http handlers.
package alice
import "net/http"
// A constructor for a piece of middleware.
// Some middleware use this constructor out of the box,
// so in most cases you can just pass somepackage.New
type Constructor func(http.Handler) http.Handler
// Chain acts as a list of http.Handler constructors.
// Chain is effectively immutable:
// once created, it will always hold
// the same set of constructors in the same order.
type Chain struct {
constructors []Constructor
}
// New creates a new chain,
// memorizing the given list of middleware constructors.
// New serves no other function,
// constructors are only called upon a call to Then().
func New(constructors ...Constructor) Chain {
return Chain{append(([]Constructor)(nil), constructors...)}
}
// Then chains the middleware and returns the final http.Handler.
// New(m1, m2, m3).Then(h)
// is equivalent to:
// m1(m2(m3(h)))
// When the request comes in, it will be passed to m1, then m2, then m3
// and finally, the given handler
// (assuming every middleware calls the following one).
//
// A chain can be safely reused by calling Then() several times.
// stdStack := alice.New(ratelimitHandler, csrfHandler)
// indexPipe = stdStack.Then(indexHandler)
// authPipe = stdStack.Then(authHandler)
// Note that constructors are called on every call to Then()
// and thus several instances of the same middleware will be created
// when a chain is reused in this way.
// For proper middleware, this should cause no problems.
//
// Then() treats nil as http.DefaultServeMux.
func (c Chain) Then(h http.Handler) http.Handler {
if h == nil {
h = http.DefaultServeMux
}
for i := range c.constructors {
h = c.constructors[len(c.constructors)-1-i](h)
}
return h
}
// ThenFunc works identically to Then, but takes
// a HandlerFunc instead of a Handler.
//
// The following two statements are equivalent:
// c.Then(http.HandlerFunc(fn))
// c.ThenFunc(fn)
//
// ThenFunc provides all the guarantees of Then.
func (c Chain) ThenFunc(fn http.HandlerFunc) http.Handler {
if fn == nil {
return c.Then(nil)
}
return c.Then(fn)
}
// Append extends a chain, adding the specified constructors
// as the last ones in the request flow.
//
// Append returns a new chain, leaving the original one untouched.
//
// stdChain := alice.New(m1, m2)
// extChain := stdChain.Append(m3, m4)
// // requests in stdChain go m1 -> m2
// // requests in extChain go m1 -> m2 -> m3 -> m4
func (c Chain) Append(constructors ...Constructor) Chain {
newCons := make([]Constructor, 0, len(c.constructors)+len(constructors))
newCons = append(newCons, c.constructors...)
newCons = append(newCons, constructors...)
return Chain{newCons}
}
// Extend extends a chain by adding the specified chain
// as the last one in the request flow.
//
// Extend returns a new chain, leaving the original one untouched.
//
// stdChain := alice.New(m1, m2)
// ext1Chain := alice.New(m3, m4)
// ext2Chain := stdChain.Extend(ext1Chain)
// // requests in stdChain go m1 -> m2
// // requests in ext1Chain go m3 -> m4
// // requests in ext2Chain go m1 -> m2 -> m3 -> m4
//
// Another example:
// aHtmlAfterNosurf := alice.New(m2)
// aHtml := alice.New(m1, func(h http.Handler) http.Handler {
// csrf := nosurf.New(h)
// csrf.SetFailureHandler(aHtmlAfterNosurf.ThenFunc(csrfFail))
// return csrf
// }).Extend(aHtmlAfterNosurf)
// // requests to aHtml hitting nosurfs success handler go m1 -> nosurf -> m2 -> target-handler
// // requests to aHtml hitting nosurfs failure handler go m1 -> nosurf -> m2 -> csrfFail
func (c Chain) Extend(chain Chain) Chain {
return c.Append(chain.constructors...)
}

View File

@ -114,6 +114,8 @@ github.com/gorilla/handlers
github.com/gorilla/mux
# github.com/json-iterator/go v1.1.6
github.com/json-iterator/go
# github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da
github.com/justinas/alice
# github.com/konsorten/go-windows-terminal-sequences v1.0.2
github.com/konsorten/go-windows-terminal-sequences
# github.com/lib/pq v1.1.0