Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-23 18:55:18 +01:00
Code Issues Projects Releases Wiki Activity

Refactor interceptors code with chain

Browse Source 
1, add a blob inteceptors for quota usage
2, add a manifest inteceptors for quota usage

Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
wang yan 2019-06-20 04:33:23 -07:00
parent 0f28fe42fd
commit 57821b1b4c
27 changed files with 1180 additions and 698 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/core/controllers/controllers_test.go
View File

@ -32,7 +32,7 @@ import (
"github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/models"
utilstest "github.com/goharbor/harbor/src/common/utils/test" utilstest "github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/proxy" "github.com/goharbor/review/harbor/src/core/proxy"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )

5
src/core/controllers/proxy.go
View File

@ -2,7 +2,7 @@ package controllers
import ( import (
"github.com/astaxie/beego" "github.com/astaxie/beego"
"github.com/goharbor/harbor/src/core/proxy" "github.com/goharbor/harbor/src/core/middlewares"
) )
// RegistryProxy is the endpoint on UI for a reverse proxy pointing to registry // RegistryProxy is the endpoint on UI for a reverse proxy pointing to registry
@ -14,10 +14,11 @@ type RegistryProxy struct {
func (p *RegistryProxy) Handle() { func (p *RegistryProxy) Handle() {
req := p.Ctx.Request req := p.Ctx.Request
rw := p.Ctx.ResponseWriter rw := p.Ctx.ResponseWriter
proxy.Handle(rw, req) middlewares.Handle(rw, req)
} }
// Render ... // Render ...
func (p *RegistryProxy) Render() error { func (p *RegistryProxy) Render() error {
return nil return nil
} }

6
src/core/main.go
View File

@ -35,7 +35,7 @@ import (
_ "github.com/goharbor/harbor/src/core/auth/uaa" _ "github.com/goharbor/harbor/src/core/auth/uaa"
"github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/filter" "github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/src/core/proxy" "github.com/goharbor/harbor/src/core/middlewares"
"github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication"
) )
@ -158,7 +158,9 @@ func main() {
} }
log.Info("Init proxy") log.Info("Init proxy")
proxy.Init() if err := middlewares.Init(); err != nil {
log.Errorf("init proxy error, %v", err)
}
// go proxy.StartProxy() // go proxy.StartProxy()
beego.Run() beego.Run()
} }

104
src/core/middlewares/blobquota/handler.go Normal file
View File

@ -0,0 +1,104 @@
package blobquota
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"net/http"
"time"
)
type blobQuotaHandler struct {
next http.Handler
}
func New(next http.Handler) http.Handler {
return &blobQuotaHandler{
next: next,
}
}
func (bqh blobQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if req.Method == http.MethodPut {
match, _ := util.MatchPutBlobURL(req)
if match {
dgstStr := req.FormValue("digest")
if dgstStr == "" {
http.Error(rw, util.MarshalError("InternalServerError", "blob digest missing"), http.StatusInternalServerError)
return
}
dgst, err := digest.Parse(dgstStr)
if err != nil {
http.Error(rw, util.MarshalError("InternalServerError", "blob digest parsing failed"), http.StatusInternalServerError)
return
}
// ToDo lock digest with redis
// ToDo read placeholder from config
state, err := hmacKey("placeholder").unpackUploadState(req.FormValue("_state"))
if err != nil {
http.Error(rw, util.MarshalError("InternalServerError", "failed to decode state"), http.StatusInternalServerError)
return
}
log.Infof("we need to insert blob data into DB.")
log.Infof("blob digest, %v", dgst)
log.Infof("blob size, %v", state.Offset)
}
}
bqh.next.ServeHTTP(rw, req)
}
// blobUploadState captures the state serializable state of the blob upload.
type blobUploadState struct {
// name is the primary repository under which the blob will be linked.
Name string
// UUID identifies the upload.
UUID string
// offset contains the current progress of the upload.
Offset int64
// StartedAt is the original start time of the upload.
StartedAt time.Time
}
type hmacKey string
var errInvalidSecret = errors.New("invalid secret")
// unpackUploadState unpacks and validates the blob upload state from the
// token, using the hmacKey secret.
func (secret hmacKey) unpackUploadState(token string) (blobUploadState, error) {
var state blobUploadState
tokenBytes, err := base64.URLEncoding.DecodeString(token)
if err != nil {
return state, err
}
mac := hmac.New(sha256.New, []byte(secret))
if len(tokenBytes) < mac.Size() {
return state, errInvalidSecret
}
macBytes := tokenBytes[:mac.Size()]
messageBytes := tokenBytes[mac.Size():]
mac.Write(messageBytes)
if !hmac.Equal(mac.Sum(nil), macBytes) {
return state, errInvalidSecret
}
if err := json.Unmarshal(messageBytes, &state); err != nil {
return state, err
}
return state, nil
}

110
src/core/middlewares/chain.go Normal file
View File

@ -0,0 +1,110 @@
package middlewares
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/blobquota"
"github.com/goharbor/harbor/src/core/middlewares/contenttrust"
"github.com/goharbor/harbor/src/core/middlewares/listrepo"
"github.com/goharbor/harbor/src/core/middlewares/multiplmanifest"
"github.com/goharbor/harbor/src/core/middlewares/readonly"
"github.com/goharbor/harbor/src/core/middlewares/regquota"
"github.com/goharbor/harbor/src/core/middlewares/url"
"github.com/goharbor/harbor/src/core/middlewares/vulnerable"
"github.com/justinas/alice"
"net/http"
)
type DefaultCreator struct {
middlewares []string
}
func New(middlewares []string) *DefaultCreator {
return &DefaultCreator{
middlewares: middlewares,
}
}
// CreateChain ...
func (b *DefaultCreator) Create() *alice.Chain {
chain := alice.New()
for _, mName := range b.middlewares {
middlewareName := mName
chain = chain.Append(func(next http.Handler) http.Handler {
constructor := b.getMiddleware(middlewareName)
if constructor == nil {
log.Errorf("cannot init middle %s", middlewareName)
return nil
}
return constructor(next)
})
}
return &chain
}
func (b *DefaultCreator) getMiddleware(mName string) alice.Constructor {
var middleware alice.Constructor
if mName == READONLY {
middleware = func(next http.Handler) http.Handler {
return readonly.New(next)
}
}
if mName == URL {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return url.New(next)
}
}
if mName == MUITIPLEMANIFEST {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return multiplmanifest.New(next)
}
}
if mName == LISTREPO {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return listrepo.New(next)
}
}
if mName == CONTENTTRUST {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return contenttrust.New(next)
}
}
if mName == VULNERABLE {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return vulnerable.New(next)
}
}
if mName == REGQUOTA {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return regquota.New(next)
}
}
if mName == BLOBQUOTA {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return blobquota.New(next)
}
}
return middleware
}

16
src/core/middlewares/config.go Normal file
View File

@ -0,0 +1,16 @@
package middlewares
// const variables
const (
READONLY = "readonly"
URL = "url"
MUITIPLEMANIFEST = "manifest"
LISTREPO = "listrepo"
CONTENTTRUST = "contenttrust"
VULNERABLE = "vulnerable"
REGQUOTA = "regquota"
BLOBQUOTA = "blobquota"
)
// sequential organization
var Middlewares = []string{READONLY, URL, MUITIPLEMANIFEST, LISTREPO, CONTENTTRUST, VULNERABLE, BLOBQUOTA, REGQUOTA}

89
src/core/middlewares/contenttrust/handler.go Normal file
View File

@ -0,0 +1,89 @@
package contenttrust
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/notary"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
"strings"
)
var NotaryEndpoint = ""
type contentTrustHandler struct {
next http.Handler
}
func New(next http.Handler) http.Handler {
return &contentTrustHandler{
next: next,
}
}
func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(util.ImageInfoCtxKey)
if imgRaw == nil || !config.WithNotary() {
cth.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo)
if img.Digest == "" {
cth.next.ServeHTTP(rw, req)
return
}
if !util.GetPolicyChecker().ContentTrustEnabled(img.ProjectName) {
cth.next.ServeHTTP(rw, req)
return
}
match, err := matchNotaryDigest(img)
if err != nil {
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError)
return
}
if !match {
log.Debugf("digest mismatch, failing the response.")
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed)
return
}
cth.next.ServeHTTP(rw, req)
}
func matchNotaryDigest(img util.ImageInfo) (bool, error) {
if NotaryEndpoint == "" {
NotaryEndpoint = config.InternalNotaryEndpoint()
}
targets, err := notary.GetInternalTargets(NotaryEndpoint, util.TokenUsername, img.Repository)
if err != nil {
return false, err
}
for _, t := range targets {
if isDigest(img.Reference) {
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if img.Digest == d {
return true, nil
}
} else {
if t.Tag == img.Reference {
log.Debugf("found reference: %s in notary, try to match digest.", img.Reference)
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if img.Digest == d {
return true, nil
}
}
}
}
log.Debugf("image: %#v, not found in notary", img)
return false, nil
}
// A sha256 is a string with 64 characters.
func isDigest(ref string) bool {
return strings.HasPrefix(ref, "sha256:") && len(ref) == 71
}

25
src/core/middlewares/inlet.go Normal file
View File

@ -0,0 +1,25 @@
package middlewares
import (
"errors"
"github.com/goharbor/harbor/src/core/middlewares/registryproxy"
"net/http"
)
var head http.Handler
// Init initialize the Proxy instance and handler chain.
func Init() error {
ph := registryproxy.New()
if ph == nil {
return errors.New("get nil when to create proxy")
}
handlerChain := New(Middlewares).Create()
head = handlerChain.Then(ph)
return nil
}
// Handle handles the request.
func Handle(rw http.ResponseWriter, req *http.Request) {
head.ServeHTTP(rw, req)
}

7
src/core/middlewares/interface.go Normal file
View File

@ -0,0 +1,7 @@
package middlewares
import "github.com/justinas/alice"
type ChainCreator interface {
Create(middlewares []string) *alice.Chain
}

88
src/core/middlewares/listrepo/handler.go Normal file
View File

@ -0,0 +1,88 @@
package listrepo
import (
"encoding/json"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
"net/http/httptest"
"regexp"
"strconv"
)
const (
catalogURLPattern = `/v2/_catalog`
)
type listReposHandler struct {
next http.Handler
}
func New(next http.Handler) http.Handler {
return &listReposHandler{
next: next,
}
}
func (lrh listReposHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
var rec *httptest.ResponseRecorder
listReposFlag := matchListRepos(req)
if listReposFlag {
rec = httptest.NewRecorder()
lrh.next.ServeHTTP(rec, req)
if rec.Result().StatusCode != http.StatusOK {
util.CopyResp(rec, rw)
return
}
var ctlg struct {
Repositories []string `json:"repositories"`
}
decoder := json.NewDecoder(rec.Body)
if err := decoder.Decode(&ctlg); err != nil {
log.Errorf("Decode repositories error: %v", err)
util.CopyResp(rec, rw)
return
}
var entries []string
for repo := range ctlg.Repositories {
log.Debugf("the repo in the response %s", ctlg.Repositories[repo])
exist := dao.RepositoryExists(ctlg.Repositories[repo])
if exist {
entries = append(entries, ctlg.Repositories[repo])
}
}
type Repos struct {
Repositories []string `json:"repositories"`
}
resp := &Repos{Repositories: entries}
respJSON, err := json.Marshal(resp)
if err != nil {
log.Errorf("Encode repositories error: %v", err)
util.CopyResp(rec, rw)
return
}
for k, v := range rec.Header() {
rw.Header()[k] = v
}
clen := len(respJSON)
rw.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(clen))
rw.Write(respJSON)
return
}
lrh.next.ServeHTTP(rw, req)
}
// matchListRepos checks if the request looks like a request to list repositories.
func matchListRepos(req *http.Request) bool {
if req.Method != http.MethodGet {
return false
}
re := regexp.MustCompile(catalogURLPattern)
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 1 {
return true
}
return false
}

33
src/core/middlewares/multiplmanifest/handler.go Normal file
View File

@ -0,0 +1,33 @@
package multiplmanifest
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
"strings"
)
type MultipleManifestHandler struct {
next http.Handler
}
func New(next http.Handler) http.Handler {
return &MultipleManifestHandler{
next: next,
}
}
// The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor.
func (mh MultipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
match, _, _ := util.MatchManifestURL(req)
if match {
contentType := req.Header.Get("Content-type")
// application/vnd.docker.distribution.manifest.list.v2+json
if strings.Contains(contentType, "manifest.list.v2") {
log.Debugf("Content-type: %s is not supported, failing the response.", contentType)
http.Error(rw, util.MarshalError("UNSUPPORTED_MEDIA_TYPE", "Manifest.list is not supported."), http.StatusUnsupportedMediaType)
return
}
}
mh.next.ServeHTTP(rw, req)
}

29
src/core/middlewares/readonly/hanlder.go Normal file
View File

@ -0,0 +1,29 @@
package readonly
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
)
type readonlyHandler struct {
next http.Handler
}
func New(next http.Handler) http.Handler {
return &readonlyHandler{
next: next,
}
}
func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if config.ReadOnly() {
if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch || req.Method == http.MethodPut {
log.Warningf("The request is prohibited in readonly mode, url is: %s", req.URL.Path)
http.Error(rw, util.MarshalError("DENIED", "The system is in read only mode. Any modification is prohibited."), http.StatusForbidden)
return
}
}
rh.next.ServeHTTP(rw, req)
}

107
src/core/middlewares/registryproxy/handler.go Normal file
View File

@ -0,0 +1,107 @@
package registryproxy
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
"net/http/httputil"
"net/url"
"strings"
)
type proxyHandler struct {
handler http.Handler
}
func New(urls ...string) http.Handler {
var registryURL string
var err error
if len(urls) > 1 {
log.Errorf("the parm, urls should have only 0 or 1 elements")
return nil
}
if len(urls) == 0 {
registryURL, err = config.RegistryURL()
if err != nil {
log.Error(err)
return nil
}
} else {
registryURL = urls[0]
}
targetURL, err := url.Parse(registryURL)
if err != nil {
log.Error(err)
return nil
}
return &proxyHandler{
handler: &httputil.ReverseProxy{
Director: func(req *http.Request) {
director(targetURL, req)
},
ModifyResponse: modifyResponse,
},
}
}
// Overwrite the http requests
func director(target *url.URL, req *http.Request) {
targetQuery := target.RawQuery
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
} else {
req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
}
if _, ok := req.Header["User-Agent"]; !ok {
// explicitly disable User-Agent so it's not set to default value
req.Header.Set("User-Agent", "")
}
}
// Modify the http response
func modifyResponse(res *http.Response) error {
if res.Request.Method == http.MethodPut {
// PUT manifest
matchMF, _, _ := util.MatchManifestURL(res.Request)
if matchMF {
if res.StatusCode == http.StatusCreated {
log.Infof("we need to insert data here ... ")
} else if res.StatusCode >= 202 || res.StatusCode <= 511 {
log.Infof("we need to roll back data here ... ")
}
}
// PUT blob
matchBB, _ := util.MatchPutBlobURL(res.Request)
if matchBB {
if res.StatusCode != http.StatusCreated {
log.Infof("we need to rollback DB and unlock digest ... ")
}
}
}
return nil
}
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
func (ph proxyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
ph.handler.ServeHTTP(rw, req)
}

53
src/core/middlewares/regquota/handler.go Normal file
View File

@ -0,0 +1,53 @@
package regquota
import (
"bytes"
"fmt"
"github.com/docker/distribution"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"io/ioutil"
"net/http"
)
type regQuotaHandler struct {
next http.Handler
}
func New(next http.Handler) http.Handler {
return &regQuotaHandler{
next: next,
}
}
//PATCH manifest ...
func (rqh regQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
match, _, _ := util.MatchManifestURL(req)
if match {
var mfSize int64
var mfDigest string
mediaType := req.Header.Get("Content-Type")
if req.Method == http.MethodPut && mediaType == "application/vnd.docker.distribution.manifest.v2+json" {
data, err := ioutil.ReadAll(req.Body)
if err != nil {
log.Warningf("Error occured when to copy manifest body %v", err)
http.Error(rw, util.MarshalError("InternalServerError", fmt.Sprintf("Error occured when to decode manifest body %v", err)), http.StatusInternalServerError)
return
}
req.Body = ioutil.NopCloser(bytes.NewBuffer(data))
_, desc, err := distribution.UnmarshalManifest(mediaType, data)
if err != nil {
log.Warningf("Error occured when to Unmarshal Manifest %v", err)
http.Error(rw, util.MarshalError("InternalServerError", fmt.Sprintf("Error occured when to Unmarshal Manifest %v", err)), http.StatusInternalServerError)
return
}
mfDigest = desc.Digest.String()
mfSize = desc.Size
log.Infof("manifest digest... %s", mfDigest)
log.Infof("manifest size... %v", mfSize)
}
}
rqh.next.ServeHTTP(rw, req)
}

58
src/core/middlewares/url/handler.go Normal file
View File

@ -0,0 +1,58 @@
package url
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
coreutils "github.com/goharbor/harbor/src/core/utils"
"net/http"
"strings"
)
type urlHandler struct {
next http.Handler
}
func New(next http.Handler) http.Handler {
return &urlHandler{
next: next,
}
}
func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
log.Debugf("in url handler, path: %s", req.URL.Path)
flag, repository, reference := util.MatchPullManifest(req)
if flag {
components := strings.SplitN(repository, "/", 2)
if len(components) < 2 {
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest)
return
}
client, err := coreutils.NewRepositoryClientForUI(util.TokenUsername, repository)
if err != nil {
log.Errorf("Error creating repository Client: %v", err)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
return
}
digest, _, err := client.ManifestExist(reference)
if err != nil {
log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
return
}
img := util.ImageInfo{
Repository: repository,
Reference: reference,
ProjectName: components[0],
Digest: digest,
}
log.Debugf("image info of the request: %#v", img)
ctx := context.WithValue(req.Context(), util.ImageInfoCtxKey, img)
req = req.WithContext(ctx)
}
uh.next.ServeHTTP(rw, req)
}

144
src/core/middlewares/util/util.go Normal file
View File

@ -0,0 +1,144 @@
package util
import (
"encoding/json"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
"net/http"
"net/http/httptest"
"regexp"
"strings"
)
type contextKey string
const (
manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`
blobURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)blobs/uploads/`
ImageInfoCtxKey = contextKey("ImageInfo")
// TODO: temp solution, remove after vmware/harbor#2242 is resolved.
TokenUsername = "harbor-core"
)
// ImageInfo
type ImageInfo struct {
Repository string
Reference string
ProjectName string
Digest string
}
// JSONError wraps a concrete Code and Message, it's readable for docker deamon.
type JSONError struct {
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
Detail string `json:"detail,omitempty"`
}
// MarshalError ...
func MarshalError(code, msg string) string {
var tmpErrs struct {
Errors []JSONError `json:"errors,omitempty"`
}
tmpErrs.Errors = append(tmpErrs.Errors, JSONError{
Code: code,
Message: msg,
Detail: msg,
})
str, err := json.Marshal(tmpErrs)
if err != nil {
log.Debugf("failed to marshal json error, %v", err)
return msg
}
return string(str)
}
// MatchManifestURL ...
func MatchManifestURL(req *http.Request) (bool, string, string) {
re := regexp.MustCompile(manifestURLPattern)
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 3 {
s[1] = strings.TrimSuffix(s[1], "/")
return true, s[1], s[2]
}
return false, "", ""
}
// MatchPutBlobURL ...
func MatchPutBlobURL(req *http.Request) (bool, string) {
if req.Method != http.MethodPut {
return false, ""
}
re := regexp.MustCompile(blobURLPattern)
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 2 {
s[1] = strings.TrimSuffix(s[1], "/")
return true, s[1]
}
return false, ""
}
// MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPullManifest(req *http.Request) (bool, string, string) {
if req.Method != http.MethodGet {
return false, "", ""
}
return MatchManifestURL(req)
}
// CopyResp ...
func CopyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
for k, v := range rec.Header() {
rw.Header()[k] = v
}
rw.WriteHeader(rec.Result().StatusCode)
rw.Write(rec.Body.Bytes())
}
// PolicyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project.
type PolicyChecker interface {
// contentTrustEnabled returns whether a project has enabled content trust.
ContentTrustEnabled(name string) bool
// vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity.
VulnerablePolicy(name string) (bool, models.Severity)
}
// PmsPolicyChecker ...
type PmsPolicyChecker struct {
pm promgr.ProjectManager
}
// ContentTrustEnabled ...
func (pc PmsPolicyChecker) ContentTrustEnabled(name string) bool {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true
}
return project.ContentTrustEnabled()
}
// VulnerablePolicy ...
func (pc PmsPolicyChecker) VulnerablePolicy(name string) (bool, models.Severity) {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true, models.SevUnknown
}
return project.VulPrevented(), clair.ParseClairSev(project.Severity())
}
// NewPMSPolicyChecker returns an instance of an pmsPolicyChecker
func NewPMSPolicyChecker(pm promgr.ProjectManager) PolicyChecker {
return &PmsPolicyChecker{
pm: pm,
}
}
// GetPolicyChecker ...
func GetPolicyChecker() PolicyChecker {
return NewPMSPolicyChecker(config.GlobalProjectMgr)
}

57
src/core/middlewares/vulnerable/handler.go Normal file
View File

@ -0,0 +1,57 @@
package vulnerable
import (
"fmt"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"net/http"
)
type vulnerableHandler struct {
next http.Handler
}
func New(next http.Handler) http.Handler {
return &vulnerableHandler{
next: next,
}
}
func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(util.ImageInfoCtxKey)
if imgRaw == nil || !config.WithClair() {
vh.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo)
if img.Digest == "" {
vh.next.ServeHTTP(rw, req)
return
}
projectVulnerableEnabled, projectVulnerableSeverity := util.GetPolicyChecker().VulnerablePolicy(img.ProjectName)
if !projectVulnerableEnabled {
vh.next.ServeHTTP(rw, req)
return
}
overview, err := dao.GetImgScanOverview(img.Digest)
if err != nil {
log.Errorf("failed to get ImgScanOverview with repo: %s, reference: %s, digest: %s. Error: %v", img.Repository, img.Reference, img.Digest, err)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Failed to get ImgScanOverview."), http.StatusPreconditionFailed)
return
}
// severity is 0 means that the image fails to scan or not scanned successfully.
if overview == nil || overview.Sev == 0 {
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Cannot get the image severity."), http.StatusPreconditionFailed)
return
}
imageSev := overview.Sev
if imageSev >= int(projectVulnerableSeverity) {
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", models.Severity(imageSev), projectVulnerableSeverity)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed)
return
}
vh.next.ServeHTTP(rw, req)
}

226
src/core/proxy/interceptor_test.go
View File

@ -1,226 +0,0 @@
package proxy
import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/models"
notarytest "github.com/goharbor/harbor/src/common/utils/notary/test"
testutils "github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/core/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"net/http/httptest"
"os"
"testing"
)
var endpoint = "10.117.4.142"
var notaryServer *httptest.Server
var admiralEndpoint = "http://127.0.0.1:8282"
var token = ""
func TestMain(m *testing.M) {
notaryServer = notarytest.NewNotaryServer(endpoint)
defer notaryServer.Close()
NotaryEndpoint = notaryServer.URL
var defaultConfig = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
common.TokenExpiration: 30,
}
config.InitWithSettings(defaultConfig)
result := m.Run()
if result != 0 {
os.Exit(result)
}
}
func TestMatchPullManifest(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil)
res1, _, _ := MatchPullManifest(req1)
assert.False(res1, "%s %v is not a request to pull manifest", req1.Method, req1.URL)
req2, _ := http.NewRequest("GET", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil)
res2, repo2, tag2 := MatchPullManifest(req2)
assert.True(res2, "%s %v is a request to pull manifest", req2.Method, req2.URL)
assert.Equal("library/ubuntu", repo2)
assert.Equal("14.04", tag2)
req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/library/ubuntu/manifests/14.04", nil)
res3, _, _ := MatchPullManifest(req3)
assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL)
req4, _ := http.NewRequest("GET", "https://192.168.0.5/v2/library/ubuntu/manifests/14.04", nil)
res4, repo4, tag4 := MatchPullManifest(req4)
assert.True(res4, "%s %v is a request to pull manifest", req4.Method, req4.URL)
assert.Equal("library/ubuntu", repo4)
assert.Equal("14.04", tag4)
req5, _ := http.NewRequest("GET", "https://myregistry.com/v2/path1/path2/golang/manifests/1.6.2", nil)
res5, repo5, tag5 := MatchPullManifest(req5)
assert.True(res5, "%s %v is a request to pull manifest", req5.Method, req5.URL)
assert.Equal("path1/path2/golang", repo5)
assert.Equal("1.6.2", tag5)
req6, _ := http.NewRequest("GET", "https://myregistry.com/v2/myproject/registry/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
res6, repo6, tag6 := MatchPullManifest(req6)
assert.True(res6, "%s %v is a request to pull manifest", req6.Method, req6.URL)
assert.Equal("myproject/registry", repo6)
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag6)
req7, _ := http.NewRequest("GET", "https://myregistry.com/v2/myproject/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
res7, repo7, tag7 := MatchPullManifest(req7)
assert.True(res7, "%s %v is a request to pull manifest", req7.Method, req7.URL)
assert.Equal("myproject", repo7)
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7)
}
func TestMatchPushManifest(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil)
res1, _, _ := MatchPushManifest(req1)
assert.False(res1, "%s %v is not a request to push manifest", req1.Method, req1.URL)
req2, _ := http.NewRequest("PUT", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil)
res2, repo2, tag2 := MatchPushManifest(req2)
assert.True(res2, "%s %v is a request to push manifest", req2.Method, req2.URL)
assert.Equal("library/ubuntu", repo2)
assert.Equal("14.04", tag2)
req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/library/ubuntu/manifests/14.04", nil)
res3, _, _ := MatchPushManifest(req3)
assert.False(res3, "%s %v is not a request to push manifest", req3.Method, req3.URL)
req4, _ := http.NewRequest("PUT", "https://192.168.0.5/v2/library/ubuntu/manifests/14.04", nil)
res4, repo4, tag4 := MatchPushManifest(req4)
assert.True(res4, "%s %v is a request to push manifest", req4.Method, req4.URL)
assert.Equal("library/ubuntu", repo4)
assert.Equal("14.04", tag4)
req5, _ := http.NewRequest("PUT", "https://myregistry.com/v2/path1/path2/golang/manifests/1.6.2", nil)
res5, repo5, tag5 := MatchPushManifest(req5)
assert.True(res5, "%s %v is a request to push manifest", req5.Method, req5.URL)
assert.Equal("path1/path2/golang", repo5)
assert.Equal("1.6.2", tag5)
req6, _ := http.NewRequest("PUT", "https://myregistry.com/v2/myproject/registry/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
res6, repo6, tag6 := MatchPushManifest(req6)
assert.True(res6, "%s %v is a request to push manifest", req6.Method, req6.URL)
assert.Equal("myproject/registry", repo6)
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag6)
req7, _ := http.NewRequest("PUT", "https://myregistry.com/v2/myproject/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
res7, repo7, tag7 := MatchPushManifest(req7)
assert.True(res7, "%s %v is a request to push manifest", req7.Method, req7.URL)
assert.Equal("myproject", repo7)
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7)
req8, _ := http.NewRequest("PUT", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil)
res8, repo8, tag8 := MatchPushManifest(req8)
assert.True(res8, "%s %v is a request to push manifest", req8.Method, req8.URL)
assert.Equal("library/ubuntu", repo8)
assert.Equal("14.04", tag8)
}
func TestMatchListRepos(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil)
res1 := MatchListRepos(req1)
assert.False(res1, "%s %v is not a request to list repos", req1.Method, req1.URL)
req2, _ := http.NewRequest("GET", "http://127.0.0.1:5000/v2/_catalog", nil)
res2 := MatchListRepos(req2)
assert.True(res2, "%s %v is a request to list repos", req2.Method, req2.URL)
req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/_catalog", nil)
res3 := MatchListRepos(req3)
assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL)
}
func TestPMSPolicyChecker(t *testing.T) {
var defaultConfigAdmiral = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
common.TokenExpiration: 30,
common.DatabaseType: "postgresql",
common.PostGreSQLHOST: "127.0.0.1",
common.PostGreSQLPort: 5432,
common.PostGreSQLUsername: "postgres",
common.PostGreSQLPassword: "root123",
common.PostGreSQLDatabase: "registry",
}
if err := config.Init(); err != nil {
panic(err)
}
testutils.InitDatabaseFromEnv()
config.Upload(defaultConfigAdmiral)
name := "project_for_test_get_sev_low"
id, err := config.GlobalProjectMgr.Create(&models.Project{
Name: name,
OwnerID: 1,
Metadata: map[string]string{
models.ProMetaEnableContentTrust: "true",
models.ProMetaPreventVul: "true",
models.ProMetaSeverity: "low",
},
})
require.Nil(t, err)
defer func(id int64) {
if err := config.GlobalProjectMgr.Delete(id); err != nil {
t.Logf("failed to delete project %d: %v", id, err)
}
}(id)
contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_sev_low")
assert.True(t, contentTrustFlag)
projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy("project_for_test_get_sev_low")
assert.True(t, projectVulnerableEnabled)
assert.Equal(t, projectVulnerableSeverity, models.SevLow)
}
func TestMatchNotaryDigest(t *testing.T) {
assert := assert.New(t)
// The data from common/utils/notary/helper_test.go
img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo", "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo", "sha256:12345678"}
res1, err := matchNotaryDigest(img1)
assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)
assert.True(res1)
res2, err := matchNotaryDigest(img2)
assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2)
assert.False(res2)
}
func TestCopyResp(t *testing.T) {
assert := assert.New(t)
rec1 := httptest.NewRecorder()
rec2 := httptest.NewRecorder()
rec1.Header().Set("X-Test", "mytest")
rec1.WriteHeader(418)
copyResp(rec1, rec2)
assert.Equal(418, rec2.Result().StatusCode)
assert.Equal("mytest", rec2.Header().Get("X-Test"))
}
func TestMarshalError(t *testing.T) {
assert := assert.New(t)
js1 := marshalError("PROJECT_POLICY_VIOLATION", "Not Found")
assert.Equal("{\"errors\":[{\"code\":\"PROJECT_POLICY_VIOLATION\",\"message\":\"Not Found\",\"detail\":\"Not Found\"}]}", js1)
js2 := marshalError("DENIED", "The action is denied")
assert.Equal("{\"errors\":[{\"code\":\"DENIED\",\"message\":\"The action is denied\",\"detail\":\"The action is denied\"}]}", js2)
}
func TestIsDigest(t *testing.T) {
assert := assert.New(t)
assert.False(isDigest("latest"))
assert.True(isDigest("sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"))
}

411
src/core/proxy/interceptors.go
View File

@ -1,411 +0,0 @@
package proxy
import (
"encoding/json"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/notary"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan"
"context"
"fmt"
"net/http"
"net/http/httptest"
"regexp"
"strconv"
"strings"
)
type contextKey string
const (
manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`
catalogURLPattern = `/v2/_catalog`
imageInfoCtxKey = contextKey("ImageInfo")
// TODO: temp solution, remove after vmware/harbor#2242 is resolved.
tokenUsername = "harbor-core"
)
// Record the docker deamon raw response.
var rec *httptest.ResponseRecorder
// NotaryEndpoint , exported for testing.
var NotaryEndpoint = ""
// MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPullManifest(req *http.Request) (bool, string, string) {
// TODO: add user agent check.
if req.Method != http.MethodGet {
return false, "", ""
}
return matchManifestURL(req)
}
// MatchPushManifest checks if the request looks like a request to push manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPushManifest(req *http.Request) (bool, string, string) {
if req.Method != http.MethodPut {
return false, "", ""
}
return matchManifestURL(req)
}
func matchManifestURL(req *http.Request) (bool, string, string) {
re := regexp.MustCompile(manifestURLPattern)
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 3 {
s[1] = strings.TrimSuffix(s[1], "/")
return true, s[1], s[2]
}
return false, "", ""
}
// MatchListRepos checks if the request looks like a request to list repositories.
func MatchListRepos(req *http.Request) bool {
if req.Method != http.MethodGet {
return false
}
re := regexp.MustCompile(catalogURLPattern)
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 1 {
return true
}
return false
}
// policyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project.
type policyChecker interface {
// contentTrustEnabled returns whether a project has enabled content trust.
contentTrustEnabled(name string) bool
// vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity.
vulnerablePolicy(name string) (bool, models.Severity)
}
type pmsPolicyChecker struct {
pm promgr.ProjectManager
}
func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true
}
return project.ContentTrustEnabled()
}
func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true, models.SevUnknown
}
return project.VulPrevented(), clair.ParseClairSev(project.Severity())
}
// newPMSPolicyChecker returns an instance of an pmsPolicyChecker
func newPMSPolicyChecker(pm promgr.ProjectManager) policyChecker {
return &pmsPolicyChecker{
pm: pm,
}
}
func getPolicyChecker() policyChecker {
return newPMSPolicyChecker(config.GlobalProjectMgr)
}
type imageInfo struct {
repository string
reference string
projectName string
digest string
}
type urlHandler struct {
next http.Handler
}
func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
log.Debugf("in url handler, path: %s", req.URL.Path)
flag, repository, reference := MatchPullManifest(req)
if flag {
components := strings.SplitN(repository, "/", 2)
if len(components) < 2 {
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest)
return
}
client, err := coreutils.NewRepositoryClientForUI(tokenUsername, repository)
if err != nil {
log.Errorf("Error creating repository Client: %v", err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
return
}
digest, _, err := client.ManifestExist(reference)
if err != nil {
log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
return
}
img := imageInfo{
repository: repository,
reference: reference,
projectName: components[0],
digest: digest,
}
log.Debugf("image info of the request: %#v", img)
ctx := context.WithValue(req.Context(), imageInfoCtxKey, img)
req = req.WithContext(ctx)
}
uh.next.ServeHTTP(rw, req)
}
type readonlyHandler struct {
next http.Handler
}
func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if config.ReadOnly() {
if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch || req.Method == http.MethodPut {
log.Warningf("The request is prohibited in readonly mode, url is: %s", req.URL.Path)
http.Error(rw, marshalError("DENIED", "The system is in read only mode. Any modification is prohibited."), http.StatusForbidden)
return
}
}
rh.next.ServeHTTP(rw, req)
}
type multipleManifestHandler struct {
next http.Handler
}
// The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor.
func (mh multipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
match, _, _ := MatchPushManifest(req)
if match {
contentType := req.Header.Get("Content-type")
// application/vnd.docker.distribution.manifest.list.v2+json
if strings.Contains(contentType, "manifest.list.v2") {
log.Debugf("Content-type: %s is not supported, failing the response.", contentType)
http.Error(rw, marshalError("UNSUPPORTED_MEDIA_TYPE", "Manifest.list is not supported."), http.StatusUnsupportedMediaType)
return
}
}
mh.next.ServeHTTP(rw, req)
}
type listReposHandler struct {
next http.Handler
}
func (lrh listReposHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
listReposFlag := MatchListRepos(req)
if listReposFlag {
rec = httptest.NewRecorder()
lrh.next.ServeHTTP(rec, req)
if rec.Result().StatusCode != http.StatusOK {
copyResp(rec, rw)
return
}
var ctlg struct {
Repositories []string `json:"repositories"`
}
decoder := json.NewDecoder(rec.Body)
if err := decoder.Decode(&ctlg); err != nil {
log.Errorf("Decode repositories error: %v", err)
copyResp(rec, rw)
return
}
var entries []string
for repo := range ctlg.Repositories {
log.Debugf("the repo in the response %s", ctlg.Repositories[repo])
exist := dao.RepositoryExists(ctlg.Repositories[repo])
if exist {
entries = append(entries, ctlg.Repositories[repo])
}
}
type Repos struct {
Repositories []string `json:"repositories"`
}
resp := &Repos{Repositories: entries}
respJSON, err := json.Marshal(resp)
if err != nil {
log.Errorf("Encode repositories error: %v", err)
copyResp(rec, rw)
return
}
for k, v := range rec.Header() {
rw.Header()[k] = v
}
clen := len(respJSON)
rw.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(clen))
rw.Write(respJSON)
return
}
lrh.next.ServeHTTP(rw, req)
}
type contentTrustHandler struct {
next http.Handler
}
func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(imageInfoCtxKey)
if imgRaw == nil || !config.WithNotary() {
cth.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
if img.digest == "" {
cth.next.ServeHTTP(rw, req)
return
}
if !getPolicyChecker().contentTrustEnabled(img.projectName) {
cth.next.ServeHTTP(rw, req)
return
}
match, err := matchNotaryDigest(img)
if err != nil {
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError)
return
}
if !match {
log.Debugf("digest mismatch, failing the response.")
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed)
return
}
cth.next.ServeHTTP(rw, req)
}
type vulnerableHandler struct {
next http.Handler
}
func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(imageInfoCtxKey)
if imgRaw == nil || !config.WithClair() {
vh.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
if img.digest == "" {
vh.next.ServeHTTP(rw, req)
return
}
projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy(img.projectName)
if !projectVulnerableEnabled {
vh.next.ServeHTTP(rw, req)
return
}
// TODO: Get whitelist based on project setting
wl, err := dao.GetSysCVEWhitelist()
if err != nil {
log.Errorf("Failed to get the whitelist, error: %v", err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get CVE whitelist."), http.StatusPreconditionFailed)
return
}
vl, err := scan.VulnListByDigest(img.digest)
if err != nil {
log.Errorf("Failed to get the vulnerability list, error: %v", err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get vulnerabilities."), http.StatusPreconditionFailed)
return
}
filtered := vl.ApplyWhitelist(*wl)
msg := vh.filterMsg(img, filtered)
log.Info(msg)
if int(vl.Severity()) >= int(projectVulnerableSeverity) {
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", vl.Severity(), projectVulnerableSeverity)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", vl.Severity(), projectVulnerableSeverity)), http.StatusPreconditionFailed)
return
}
vh.next.ServeHTTP(rw, req)
}
func (vh vulnerableHandler) filterMsg(img imageInfo, filtered scan.VulnerabilityList) string {
filterMsg := fmt.Sprintf("Image: %s/%s:%s, digest: %s, vulnerabilities fitered by whitelist:", img.projectName, img.repository, img.reference, img.digest)
if len(filtered) == 0 {
filterMsg = fmt.Sprintf("%s none.", filterMsg)
}
for _, v := range filtered {
filterMsg = fmt.Sprintf("%s ID: %s, severity: %s;", filterMsg, v.ID, v.Severity)
}
return filterMsg
}
func matchNotaryDigest(img imageInfo) (bool, error) {
if NotaryEndpoint == "" {
NotaryEndpoint = config.InternalNotaryEndpoint()
}
targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository)
if err != nil {
return false, err
}
for _, t := range targets {
if isDigest(img.reference) {
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if img.digest == d {
return true, nil
}
} else {
if t.Tag == img.reference {
log.Debugf("found reference: %s in notary, try to match digest.", img.reference)
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if img.digest == d {
return true, nil
}
}
}
}
log.Debugf("image: %#v, not found in notary", img)
return false, nil
}
// A sha256 is a string with 64 characters.
func isDigest(ref string) bool {
return strings.HasPrefix(ref, "sha256:") && len(ref) == 71
}
func copyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
for k, v := range rec.Header() {
rw.Header()[k] = v
}
rw.WriteHeader(rec.Result().StatusCode)
rw.Write(rec.Body.Bytes())
}
func marshalError(code, msg string) string {
var tmpErrs struct {
Errors []JSONError `json:"errors,omitempty"`
}
tmpErrs.Errors = append(tmpErrs.Errors, JSONError{
Code: code,
Message: msg,
Detail: msg,
})
str, err := json.Marshal(tmpErrs)
if err != nil {
log.Debugf("failed to marshal json error, %v", err)
return msg
}
return string(str)
}
// JSONError wraps a concrete Code and Message, it's readable for docker deamon.
type JSONError struct {
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
Detail string `json:"detail,omitempty"`
}

56
src/core/proxy/proxy.go
View File

@ -1,56 +0,0 @@
package proxy
import (
"github.com/goharbor/harbor/src/core/config"
"fmt"
"net/http"
"net/http/httputil"
"net/url"
)
// Proxy is the instance of the reverse proxy in this package.
var Proxy *httputil.ReverseProxy
var handlers handlerChain
type handlerChain struct {
head http.Handler
}
// Init initialize the Proxy instance and handler chain.
func Init(urls ...string) error {
var err error
var registryURL string
if len(urls) > 1 {
return fmt.Errorf("the parm, urls should have only 0 or 1 elements")
}
if len(urls) == 0 {
registryURL, err = config.RegistryURL()
if err != nil {
return err
}
} else {
registryURL = urls[0]
}
targetURL, err := url.Parse(registryURL)
if err != nil {
return err
}
Proxy = httputil.NewSingleHostReverseProxy(targetURL)
handlers = handlerChain{
head: readonlyHandler{
next: urlHandler{
next: multipleManifestHandler{
next: listReposHandler{
next: contentTrustHandler{
next: vulnerableHandler{
next: Proxy,
}}}}}}}
return nil
}
// Handle handles the request.
func Handle(rw http.ResponseWriter, req *http.Request) {
handlers.head.ServeHTTP(rw, req)
}

1
src/go.mod
View File

@ -46,6 +46,7 @@ require (
github.com/gorilla/mux v1.6.2 github.com/gorilla/mux v1.6.2
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/jinzhu/gorm v1.9.8 // indirect github.com/jinzhu/gorm v1.9.8 // indirect
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/lib/pq v1.1.0 github.com/lib/pq v1.1.0

2
src/go.sum
View File

@ -169,6 +169,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da h1:5y58+OCjoHCYB8182mpf/dEsq0vwTKPOo4zGfH0xW9A=
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da/go.mod h1:oLH0CmIaxCGXD67VKGR5AacGXZSMznlmeqM8RzPrcY8=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

17
src/vendor/github.com/justinas/alice/.travis.yml generated vendored Normal file
View File

@ -0,0 +1,17 @@
language: go
matrix:
include:
- go: 1.0.x
- go: 1.1.x
- go: 1.2.x
- go: 1.3.x
- go: 1.4.x
- go: 1.5.x
- go: 1.6.x
- go: 1.7.x
- go: 1.8.x
- go: 1.9.x
- go: tip
allow_failures:
- go: tip

20
src/vendor/github.com/justinas/alice/LICENSE generated vendored Normal file
View File

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2014 Justinas Stankevicius
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

98
src/vendor/github.com/justinas/alice/README.md generated vendored Normal file
View File

@ -0,0 +1,98 @@
# Alice
[![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](http://godoc.org/github.com/justinas/alice)
[![Build Status](https://travis-ci.org/justinas/alice.svg?branch=master)](https://travis-ci.org/justinas/alice)
[![Coverage](http://gocover.io/_badge/github.com/justinas/alice)](http://gocover.io/github.com/justinas/alice)
Alice provides a convenient way to chain
your HTTP middleware functions and the app handler.
In short, it transforms
```go
Middleware1(Middleware2(Middleware3(App)))
```
to
```go
alice.New(Middleware1, Middleware2, Middleware3).Then(App)
```
### Why?
None of the other middleware chaining solutions
behaves exactly like Alice.
Alice is as minimal as it gets:
in essence, it's just a for loop that does the wrapping for you.
Check out [this blog post](http://justinas.org/alice-painless-middleware-chaining-for-go/)
for explanation how Alice is different from other chaining solutions.
### Usage
Your middleware constructors should have the form of
```go
func (http.Handler) http.Handler
```
Some middleware provide this out of the box.
For ones that don't, it's trivial to write one yourself.
```go
func myStripPrefix(h http.Handler) http.Handler {
return http.StripPrefix("/old", h)
}
```
This complete example shows the full power of Alice.
```go
package main
import (
"net/http"
"time"
"github.com/throttled/throttled"
"github.com/justinas/alice"
"github.com/justinas/nosurf"
)
func timeoutHandler(h http.Handler) http.Handler {
return http.TimeoutHandler(h, 1*time.Second, "timed out")
}
func myApp(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hello world!"))
}
func main() {
th := throttled.Interval(throttled.PerSec(10), 1, &throttled.VaryBy{Path: true}, 50)
myHandler := http.HandlerFunc(myApp)
chain := alice.New(th.Throttle, timeoutHandler, nosurf.NewPure).Then(myHandler)
http.ListenAndServe(":8000", chain)
}
```
Here, the request will pass [throttled](https://github.com/PuerkitoBio/throttled) first,
then an http.TimeoutHandler we've set up,
then [nosurf](https://github.com/justinas/nosurf)
and will finally reach our handler.
Note that Alice makes **no guarantees** for
how one or another piece of middleware will behave.
Once it passes the execution to the outer layer of middleware,
it has no saying in whether middleware will execute the inner handlers.
This is intentional behavior.
Alice works with Go 1.0 and higher.
### Contributing
0. Find an issue that bugs you / open a new one.
1. Discuss.
2. Branch off, commit, test.
3. Make a pull request / attach the commits to the issue.

112
src/vendor/github.com/justinas/alice/chain.go generated vendored Normal file
View File

@ -0,0 +1,112 @@
// Package alice provides a convenient way to chain http handlers.
package alice
import "net/http"
// A constructor for a piece of middleware.
// Some middleware use this constructor out of the box,
// so in most cases you can just pass somepackage.New
type Constructor func(http.Handler) http.Handler
// Chain acts as a list of http.Handler constructors.
// Chain is effectively immutable:
// once created, it will always hold
// the same set of constructors in the same order.
type Chain struct {
constructors []Constructor
}
// New creates a new chain,
// memorizing the given list of middleware constructors.
// New serves no other function,
// constructors are only called upon a call to Then().
func New(constructors ...Constructor) Chain {
return Chain{append(([]Constructor)(nil), constructors...)}
}
// Then chains the middleware and returns the final http.Handler.
// New(m1, m2, m3).Then(h)
// is equivalent to:
// m1(m2(m3(h)))
// When the request comes in, it will be passed to m1, then m2, then m3
// and finally, the given handler
// (assuming every middleware calls the following one).
//
// A chain can be safely reused by calling Then() several times.
// stdStack := alice.New(ratelimitHandler, csrfHandler)
// indexPipe = stdStack.Then(indexHandler)
// authPipe = stdStack.Then(authHandler)
// Note that constructors are called on every call to Then()
// and thus several instances of the same middleware will be created
// when a chain is reused in this way.
// For proper middleware, this should cause no problems.
//
// Then() treats nil as http.DefaultServeMux.
func (c Chain) Then(h http.Handler) http.Handler {
if h == nil {
h = http.DefaultServeMux
}
for i := range c.constructors {
h = c.constructors[len(c.constructors)-1-i](h)
}
return h
}
// ThenFunc works identically to Then, but takes
// a HandlerFunc instead of a Handler.
//
// The following two statements are equivalent:
// c.Then(http.HandlerFunc(fn))
// c.ThenFunc(fn)
//
// ThenFunc provides all the guarantees of Then.
func (c Chain) ThenFunc(fn http.HandlerFunc) http.Handler {
if fn == nil {
return c.Then(nil)
}
return c.Then(fn)
}
// Append extends a chain, adding the specified constructors
// as the last ones in the request flow.
//
// Append returns a new chain, leaving the original one untouched.
//
// stdChain := alice.New(m1, m2)
// extChain := stdChain.Append(m3, m4)
// // requests in stdChain go m1 -> m2
// // requests in extChain go m1 -> m2 -> m3 -> m4
func (c Chain) Append(constructors ...Constructor) Chain {
newCons := make([]Constructor, 0, len(c.constructors)+len(constructors))
newCons = append(newCons, c.constructors...)
newCons = append(newCons, constructors...)
return Chain{newCons}
}
// Extend extends a chain by adding the specified chain
// as the last one in the request flow.
//
// Extend returns a new chain, leaving the original one untouched.
//
// stdChain := alice.New(m1, m2)
// ext1Chain := alice.New(m3, m4)
// ext2Chain := stdChain.Extend(ext1Chain)
// // requests in stdChain go m1 -> m2
// // requests in ext1Chain go m3 -> m4
// // requests in ext2Chain go m1 -> m2 -> m3 -> m4
//
// Another example:
// aHtmlAfterNosurf := alice.New(m2)
// aHtml := alice.New(m1, func(h http.Handler) http.Handler {
// csrf := nosurf.New(h)
// csrf.SetFailureHandler(aHtmlAfterNosurf.ThenFunc(csrfFail))
// return csrf
// }).Extend(aHtmlAfterNosurf)
// // requests to aHtml hitting nosurfs success handler go m1 -> nosurf -> m2 -> target-handler
// // requests to aHtml hitting nosurfs failure handler go m1 -> nosurf -> m2 -> csrfFail
func (c Chain) Extend(chain Chain) Chain {
return c.Append(chain.constructors...)
}

2
src/vendor/modules.txt vendored
View File

@ -114,6 +114,8 @@ github.com/gorilla/handlers
github.com/gorilla/mux github.com/gorilla/mux
# github.com/json-iterator/go v1.1.6 # github.com/json-iterator/go v1.1.6
github.com/json-iterator/go github.com/json-iterator/go
# github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da
github.com/justinas/alice
# github.com/konsorten/go-windows-terminal-sequences v1.0.2 # github.com/konsorten/go-windows-terminal-sequences v1.0.2
github.com/konsorten/go-windows-terminal-sequences github.com/konsorten/go-windows-terminal-sequences
# github.com/lib/pq v1.1.0 # github.com/lib/pq v1.1.0