Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-18 22:57:38 +01:00
Code Issues Projects Releases Wiki Activity

perf(db): skip tx for get blob, patch/put blob upload apis

Browse Source 
Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2020-07-20 16:41:00 +00:00
parent ee35e1ecc6
commit 6db1a1cb91
4 changed files with 58 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/core/middlewares/middlewares.go
View File

@ -15,8 +15,11 @@
package middlewares
import (
"net/http"
"regexp"
"github.com/astaxie/beego"
"github.com/docker/distribution/reference"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
"github.com/goharbor/harbor/src/server/middleware/csrf"
"github.com/goharbor/harbor/src/server/middleware/log"
@ -27,19 +30,22 @@ import (
"github.com/goharbor/harbor/src/server/middleware/security"
"github.com/goharbor/harbor/src/server/middleware/session"
"github.com/goharbor/harbor/src/server/middleware/transaction"
"net/http"
"regexp"
)
var (
match = regexp.MustCompile
numericRegexp = match(`[0-9]+`)
blobURLRe = match("^/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String())
// fetchBlobAPISkipper skip transaction middleware for fetch blob API
// because transaction use the ResponseBuffer for the response which will degrade the performance for fetch blob
fetchBlobAPISkipper = middleware.MethodAndPathSkipper(http.MethodGet, blobURLRe)
// dbTxSkippers skip the transaction middleware for GET Blob, PATCH Blob Upload and PUT Blob Upload APIs
// because the APIs may take a long time to run, enable the transaction middleware in them will hold the database connections
// until the API finished, this behavior may eat all the database connections.
// There are no database writing operations in the GET Blob and PATCH Blob APIs, so skip the transaction middleware is all ok.
// For the PUT Blob Upload API, we will make a transaction manually to write blob info to the database when put blob upload successfully.
dbTxSkippers = []middleware.Skipper{
middleware.MethodAndPathSkipper(http.MethodGet, distribution.BlobURLRegexp),
middleware.MethodAndPathSkipper(http.MethodPatch, distribution.BlobUploadURLRegexp),
middleware.MethodAndPathSkipper(http.MethodPut, distribution.BlobUploadURLRegexp),
}
// readonlySkippers skip the post request when harbor sets to readonly.
readonlySkippers = []middleware.Skipper{
@ -65,11 +71,10 @@ func MiddleWares() []beego.MiddleWare {
log.Middleware(),
session.Middleware(),
csrf.Middleware(),
orm.Middleware(),
notification.Middleware(), // notification must ahead of transaction ensure the DB transaction execution complete
transaction.Middleware(dbTxSkippers...),
security.Middleware(),
readonly.Middleware(readonlySkippers...),
orm.Middleware(),
// notification must ahead of transaction ensure the DB transaction execution complete
notification.Middleware(),
transaction.Middleware(fetchBlobAPISkipper),
}
}

22
src/core/middlewares/middlewares_test.go
View File

@ -20,28 +20,6 @@ import (
"testing"
)
func Test_fetchBlobAPISkipper(t *testing.T) {
type args struct {
r *http.Request
}
tests := []struct {
name string
args args
want bool
}{
{"fetch blob", args{httptest.NewRequest(http.MethodGet, "/v2/library/photon/blobs/sha256:6e0447537050cf871f9ab6a3fec5715f9c6fff5212f6666993f1fc46b1f717a3", nil)}, true},
{"delete blob", args{httptest.NewRequest(http.MethodDelete, "/v2/library/photon/blobs/sha256:6e0447537050cf871f9ab6a3fec5715f9c6fff5212f6666993f1fc46b1f717a3", nil)}, false},
{"get manifest", args{httptest.NewRequest(http.MethodDelete, "/v2/library/photon/manifests/sha256:6e0447537050cf871f9ab6a3fec5715f9c6fff5212f6666993f1fc46b1f717a3", nil)}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := fetchBlobAPISkipper(tt.args.r); got != tt.want {
t.Errorf("fetchBlobAPISkipper() = %v, want %v", got, tt.want)
}
})
}
}
func Test_readonlySkipper(t *testing.T) {
type args struct {
r *http.Request

4
src/pkg/distribution/distribution.go
View File

@ -47,8 +47,12 @@ var (
var (
name = fmt.Sprintf("(?P<name>%s)", ref.NameRegexp)
reference = fmt.Sprintf("(?P<reference>((%s)|(%s)))", ref.DigestRegexp, ref.TagRegexp)
dgt = fmt.Sprintf("(?P<digest>%s)", ref.DigestRegexp)
sessionID = "(?P<session_id>[a-zA-Z0-9-_.=]+)"
// BlobURLRegexp regexp which match blob url
BlobURLRegexp = regexp.MustCompile(`^/v2/` + name + `/blobs/` + dgt)
// BlobUploadURLRegexp regexp which match blob upload url
BlobUploadURLRegexp = regexp.MustCompile(`^/v2/` + name + `/blobs/uploads/` + sessionID)

13
src/server/middleware/blob/put_blob_upload.go
View File

@ -15,11 +15,14 @@
package blob
import (
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
"context"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
)
// PutBlobUploadMiddleware middleware is to update the blob status according to the different situation before the request passed into proxy(distribution).
@ -42,6 +45,7 @@ func PutBlobUploadMiddleware() func(http.Handler) http.Handler {
ctx := r.Context()
h := func(ctx context.Context) error {
logger := log.G(ctx).WithFields(log.Fields{"middleware": "blob"})
size, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
@ -72,6 +76,9 @@ func PutBlobUploadMiddleware() func(http.Handler) http.Handler {
}
return nil
}
return orm.WithTransaction(h)(ctx)
})
return middleware.Chain(before, after)