1
0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-25 18:18:04 +01:00

perf(db): skip tx for get blob, patch/put blob upload apis

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2020-07-20 16:41:00 +00:00
parent ee35e1ecc6
commit 6db1a1cb91
4 changed files with 58 additions and 64 deletions
src
core/middlewares
pkg/distribution
server/middleware/blob

View File

@ -15,8 +15,11 @@
package middlewares package middlewares
import ( import (
"net/http"
"regexp"
"github.com/astaxie/beego" "github.com/astaxie/beego"
"github.com/docker/distribution/reference" "github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware" "github.com/goharbor/harbor/src/server/middleware"
"github.com/goharbor/harbor/src/server/middleware/csrf" "github.com/goharbor/harbor/src/server/middleware/csrf"
"github.com/goharbor/harbor/src/server/middleware/log" "github.com/goharbor/harbor/src/server/middleware/log"
@ -27,19 +30,22 @@ import (
"github.com/goharbor/harbor/src/server/middleware/security" "github.com/goharbor/harbor/src/server/middleware/security"
"github.com/goharbor/harbor/src/server/middleware/session" "github.com/goharbor/harbor/src/server/middleware/session"
"github.com/goharbor/harbor/src/server/middleware/transaction" "github.com/goharbor/harbor/src/server/middleware/transaction"
"net/http"
"regexp"
) )
var ( var (
match = regexp.MustCompile match = regexp.MustCompile
numericRegexp = match(`[0-9]+`) numericRegexp = match(`[0-9]+`)
blobURLRe = match("^/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String()) // dbTxSkippers skip the transaction middleware for GET Blob, PATCH Blob Upload and PUT Blob Upload APIs
// because the APIs may take a long time to run, enable the transaction middleware in them will hold the database connections
// fetchBlobAPISkipper skip transaction middleware for fetch blob API // until the API finished, this behavior may eat all the database connections.
// because transaction use the ResponseBuffer for the response which will degrade the performance for fetch blob // There are no database writing operations in the GET Blob and PATCH Blob APIs, so skip the transaction middleware is all ok.
fetchBlobAPISkipper = middleware.MethodAndPathSkipper(http.MethodGet, blobURLRe) // For the PUT Blob Upload API, we will make a transaction manually to write blob info to the database when put blob upload successfully.
dbTxSkippers = []middleware.Skipper{
middleware.MethodAndPathSkipper(http.MethodGet, distribution.BlobURLRegexp),
middleware.MethodAndPathSkipper(http.MethodPatch, distribution.BlobUploadURLRegexp),
middleware.MethodAndPathSkipper(http.MethodPut, distribution.BlobUploadURLRegexp),
}
// readonlySkippers skip the post request when harbor sets to readonly. // readonlySkippers skip the post request when harbor sets to readonly.
readonlySkippers = []middleware.Skipper{ readonlySkippers = []middleware.Skipper{
@ -65,11 +71,10 @@ func MiddleWares() []beego.MiddleWare {
log.Middleware(), log.Middleware(),
session.Middleware(), session.Middleware(),
csrf.Middleware(), csrf.Middleware(),
orm.Middleware(),
notification.Middleware(), // notification must ahead of transaction ensure the DB transaction execution complete
transaction.Middleware(dbTxSkippers...),
security.Middleware(), security.Middleware(),
readonly.Middleware(readonlySkippers...), readonly.Middleware(readonlySkippers...),
orm.Middleware(),
// notification must ahead of transaction ensure the DB transaction execution complete
notification.Middleware(),
transaction.Middleware(fetchBlobAPISkipper),
} }
} }

View File

@ -20,28 +20,6 @@ import (
"testing" "testing"
) )
func Test_fetchBlobAPISkipper(t *testing.T) {
type args struct {
r *http.Request
}
tests := []struct {
name string
args args
want bool
}{
{"fetch blob", args{httptest.NewRequest(http.MethodGet, "/v2/library/photon/blobs/sha256:6e0447537050cf871f9ab6a3fec5715f9c6fff5212f6666993f1fc46b1f717a3", nil)}, true},
{"delete blob", args{httptest.NewRequest(http.MethodDelete, "/v2/library/photon/blobs/sha256:6e0447537050cf871f9ab6a3fec5715f9c6fff5212f6666993f1fc46b1f717a3", nil)}, false},
{"get manifest", args{httptest.NewRequest(http.MethodDelete, "/v2/library/photon/manifests/sha256:6e0447537050cf871f9ab6a3fec5715f9c6fff5212f6666993f1fc46b1f717a3", nil)}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := fetchBlobAPISkipper(tt.args.r); got != tt.want {
t.Errorf("fetchBlobAPISkipper() = %v, want %v", got, tt.want)
}
})
}
}
func Test_readonlySkipper(t *testing.T) { func Test_readonlySkipper(t *testing.T) {
type args struct { type args struct {
r *http.Request r *http.Request

View File

@ -47,8 +47,12 @@ var (
var ( var (
name = fmt.Sprintf("(?P<name>%s)", ref.NameRegexp) name = fmt.Sprintf("(?P<name>%s)", ref.NameRegexp)
reference = fmt.Sprintf("(?P<reference>((%s)|(%s)))", ref.DigestRegexp, ref.TagRegexp) reference = fmt.Sprintf("(?P<reference>((%s)|(%s)))", ref.DigestRegexp, ref.TagRegexp)
dgt = fmt.Sprintf("(?P<digest>%s)", ref.DigestRegexp)
sessionID = "(?P<session_id>[a-zA-Z0-9-_.=]+)" sessionID = "(?P<session_id>[a-zA-Z0-9-_.=]+)"
// BlobURLRegexp regexp which match blob url
BlobURLRegexp = regexp.MustCompile(`^/v2/` + name + `/blobs/` + dgt)
// BlobUploadURLRegexp regexp which match blob upload url // BlobUploadURLRegexp regexp which match blob upload url
BlobUploadURLRegexp = regexp.MustCompile(`^/v2/` + name + `/blobs/uploads/` + sessionID) BlobUploadURLRegexp = regexp.MustCompile(`^/v2/` + name + `/blobs/uploads/` + sessionID)

View File

@ -15,11 +15,14 @@
package blob package blob
import ( import (
"github.com/goharbor/harbor/src/lib/log" "context"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
"net/http" "net/http"
"strconv" "strconv"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
) )
// PutBlobUploadMiddleware middleware is to update the blob status according to the different situation before the request passed into proxy(distribution). // PutBlobUploadMiddleware middleware is to update the blob status according to the different situation before the request passed into proxy(distribution).
@ -42,6 +45,7 @@ func PutBlobUploadMiddleware() func(http.Handler) http.Handler {
ctx := r.Context() ctx := r.Context()
h := func(ctx context.Context) error {
logger := log.G(ctx).WithFields(log.Fields{"middleware": "blob"}) logger := log.G(ctx).WithFields(log.Fields{"middleware": "blob"})
size, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) size, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
@ -72,6 +76,9 @@ func PutBlobUploadMiddleware() func(http.Handler) http.Handler {
} }
return nil return nil
}
return orm.WithTransaction(h)(ctx)
}) })
return middleware.Chain(before, after) return middleware.Chain(before, after)