diff --git a/src/go.mod b/src/go.mod index 0ab764d60..bfe65f019 100644 --- a/src/go.mod +++ b/src/go.mod @@ -48,7 +48,6 @@ require ( github.com/gorilla/mux v1.7.4 github.com/graph-gophers/dataloader v5.0.0+incompatible github.com/jinzhu/gorm v1.9.8 // indirect - github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da github.com/lib/pq v1.3.0 github.com/mattn/go-runewidth v0.0.4 // indirect github.com/miekg/pkcs11 v0.0.0-20170220202408-7283ca79f35e // indirect @@ -57,6 +56,7 @@ require ( github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/pkg/errors v0.9.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect github.com/robfig/cron v1.0.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect diff --git a/src/go.sum b/src/go.sum index 88b6f893a..44d375ad1 100644 --- a/src/go.sum +++ b/src/go.sum @@ -499,8 +499,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da h1:5y58+OCjoHCYB8182mpf/dEsq0vwTKPOo4zGfH0xW9A= -github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da/go.mod h1:oLH0CmIaxCGXD67VKGR5AacGXZSMznlmeqM8RzPrcY8= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= diff --git a/src/server/middleware/blob/put_manifest.go b/src/server/middleware/blob/put_manifest.go index 4076d3ed1..fe516a10e 100644 --- a/src/server/middleware/blob/put_manifest.go +++ b/src/server/middleware/blob/put_manifest.go @@ -15,21 +15,60 @@ package blob import ( + "fmt" + "github.com/goharbor/harbor/src/lib/errors" + blob_models "github.com/goharbor/harbor/src/pkg/blob/models" + "github.com/goharbor/harbor/src/server/middleware/requestid" "io/ioutil" "net/http" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/distribution" "github.com/goharbor/harbor/src/server/middleware" - "github.com/justinas/alice" ) -// PutManifestMiddleware middleware which create Blobs for the foreign layers and associate them with the project, -// update the content type of the Blobs which already exist, +// PutManifestMiddleware middleware middleware is to update the manifest status according to the different situation before the request passed into proxy(distribution). +// and it creates Blobs for the foreign layers and associate them with the project, updates the content type of the Blobs which already exist, // create Blob for the manifest, associate all Blobs with the manifest after PUT /v2//manifests/ success. func PutManifestMiddleware() func(http.Handler) http.Handler { before := middleware.BeforeRequest(func(r *http.Request) error { - // Do nothing, only make the request nopclose + ctx := r.Context() + logger := log.G(ctx) + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + + contentType := r.Header.Get("Content-Type") + _, descriptor, err := distribution.UnmarshalManifest(contentType, body) + if err != nil { + logger.Errorf("unmarshal manifest failed, error: %v", err) + return errors.Wrapf(err, "unmarshal manifest failed").WithCode(errors.MANIFESTINVALID) + } + + // bb here is the actually a manifest, which is also stored as a blob in DB and storage. + bb, err := blobController.Get(r.Context(), descriptor.Digest.String()) + if err != nil { + if errors.IsNotFoundErr(err) { + return nil + } + return err + } + + switch bb.Status { + case blob_models.StatusNone, blob_models.StatusDelete, blob_models.StatusDeleteFailed: + err := blobController.Touch(r.Context(), bb) + if err != nil { + logger.Errorf("failed to update manifest: %s status to StatusNone, error:%v", bb.Digest, err) + return errors.Wrapf(err, fmt.Sprintf("the request id is: %s", r.Header.Get(requestid.HeaderXRequestID))) + } + case blob_models.StatusDeleting: + logger.Warningf(fmt.Sprintf("the asking manifest is in GC, mark it as non existing, request id: %s", r.Header.Get(requestid.HeaderXRequestID))) + return errors.New(nil).WithMessage(fmt.Sprintf("the asking manifest is in GC, mark it as non existing, request id: %s", r.Header.Get(requestid.HeaderXRequestID))).WithCode(errors.NotFoundCode) + default: + return nil + } return nil }) @@ -99,7 +138,5 @@ func PutManifestMiddleware() func(http.Handler) http.Handler { return nil }) - return func(next http.Handler) http.Handler { - return alice.New(before, after).Then(next) - } + return middleware.Chain(before, after) } diff --git a/src/server/middleware/blob/put_manifest_test.go b/src/server/middleware/blob/put_manifest_test.go index 06facac21..cbd50f788 100644 --- a/src/server/middleware/blob/put_manifest_test.go +++ b/src/server/middleware/blob/put_manifest_test.go @@ -16,12 +16,15 @@ package blob import ( "fmt" + "github.com/goharbor/harbor/src/controller/blob" + "github.com/goharbor/harbor/src/lib" + pkg_blob "github.com/goharbor/harbor/src/pkg/blob" + blob_models "github.com/goharbor/harbor/src/pkg/blob/models" "net/http" "net/http/httptest" "strings" "testing" - "github.com/goharbor/harbor/src/controller/blob" "github.com/goharbor/harbor/src/pkg/distribution" htesting "github.com/goharbor/harbor/src/testing" "github.com/google/uuid" @@ -47,15 +50,15 @@ func (suite *PutManifestMiddlewareTestSuite) pushBlob(name string, digest string suite.Equal(res.Code, http.StatusCreated) } -func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { - body := ` +func (suite *PutManifestMiddlewareTestSuite) prepare(name string) (distribution.Manifest, distribution.Descriptor, *http.Request) { + body := fmt.Sprintf(` { "schemaVersion": 2, "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "config": { "mediaType": "application/vnd.docker.container.image.v1+json", "size": 6868, - "digest": "sha256:9b188f5fb1e6e1c7b10045585cb386892b2b4e1d31d62e3688c6fa8bf9fd32b5" + "digest": "%s" }, "layers": [ { @@ -89,15 +92,27 @@ func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { "digest": "sha256:727f8da63ac248054cb7dda635ee16da76e553ec99be565a54180c83d04025a8" } ] - }` + }`, suite.DigestString()) + manifest, descriptor, err := distribution.UnmarshalManifest("application/vnd.docker.distribution.manifest.v2+json", []byte(body)) suite.Nil(err) + req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/manifests/%s", name, descriptor.Digest.String()), strings.NewReader(body)) + req.Header.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json") + info := lib.ArtifactInfo{ + Repository: name, + Reference: "latest", + Tag: "latest", + Digest: descriptor.Digest.String(), + } + return manifest, descriptor, req.WithContext(lib.WithArtifactInfo(req.Context(), info)) +} + +func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { suite.WithProject(func(projectID int64, projectName string) { name := fmt.Sprintf("%s/redis", projectName) - req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/manifests/%s", name, descriptor.Digest.String()), strings.NewReader(body)) - req.Header.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json") + manifest, descriptor, req := suite.prepare(name) res := httptest.NewRecorder() next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()}) @@ -132,6 +147,60 @@ func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { }) } +func (suite *PutManifestMiddlewareTestSuite) TestMFInDeleting() { + suite.WithProject(func(projectID int64, projectName string) { + name := fmt.Sprintf("%s/photon", projectName) + _, descriptor, req := suite.prepare(name) + res := httptest.NewRecorder() + + id, err := blob.Ctl.Ensure(suite.Context(), descriptor.Digest.String(), "application/vnd.docker.distribution.manifest.v2+json", 512) + suite.Nil(err) + + // status-none -> status-delete -> status-deleting + _, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), &blob_models.Blob{ID: id, Status: blob_models.StatusDelete}) + suite.Nil(err) + _, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), &blob_models.Blob{ID: id, Status: blob_models.StatusDeleting, Version: 1}) + suite.Nil(err) + + next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()}) + PutManifestMiddleware()(next).ServeHTTP(res, req) + suite.Equal(http.StatusNotFound, res.Code) + }) +} + +func (suite *PutManifestMiddlewareTestSuite) TestMFInDelete() { + suite.WithProject(func(projectID int64, projectName string) { + name := fmt.Sprintf("%s/photon", projectName) + manifest, descriptor, req := suite.prepare(name) + res := httptest.NewRecorder() + + id, err := blob.Ctl.Ensure(suite.Context(), descriptor.Digest.String(), "application/vnd.docker.distribution.manifest.v2+json", 512) + suite.Nil(err) + + // status-none -> status-delete -> status-deleting + _, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), &blob_models.Blob{ID: id, Status: blob_models.StatusDelete}) + suite.Nil(err) + + next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()}) + PutManifestMiddleware()(next).ServeHTTP(res, req) + suite.Equal(http.StatusCreated, res.Code) + + for _, reference := range manifest.References() { + opts := []blob.Option{ + blob.IsAssociatedWithArtifact(descriptor.Digest.String()), + blob.IsAssociatedWithProject(projectID), + } + + b, err := blob.Ctl.Get(suite.Context(), reference.Digest.String(), opts...) + if suite.Nil(err) { + suite.Equal(reference.MediaType, b.ContentType) + suite.Equal(reference.Size, b.Size) + suite.Equal(blob_models.StatusNone, b.Status) + } + } + }) +} + func TestPutManifestMiddlewareTestSuite(t *testing.T) { suite.Run(t, &PutManifestMiddlewareTestSuite{}) } diff --git a/src/vendor/github.com/justinas/alice/.travis.yml b/src/vendor/github.com/justinas/alice/.travis.yml deleted file mode 100644 index dc6bea671..000000000 --- a/src/vendor/github.com/justinas/alice/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: go - -matrix: - include: - - go: 1.0.x - - go: 1.1.x - - go: 1.2.x - - go: 1.3.x - - go: 1.4.x - - go: 1.5.x - - go: 1.6.x - - go: 1.7.x - - go: 1.8.x - - go: 1.9.x - - go: tip - allow_failures: - - go: tip diff --git a/src/vendor/github.com/justinas/alice/LICENSE b/src/vendor/github.com/justinas/alice/LICENSE deleted file mode 100644 index 0d0d352ec..000000000 --- a/src/vendor/github.com/justinas/alice/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Justinas Stankevicius - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/vendor/github.com/justinas/alice/README.md b/src/vendor/github.com/justinas/alice/README.md deleted file mode 100644 index e4f9157c0..000000000 --- a/src/vendor/github.com/justinas/alice/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# Alice - -[![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](http://godoc.org/github.com/justinas/alice) -[![Build Status](https://travis-ci.org/justinas/alice.svg?branch=master)](https://travis-ci.org/justinas/alice) -[![Coverage](http://gocover.io/_badge/github.com/justinas/alice)](http://gocover.io/github.com/justinas/alice) - -Alice provides a convenient way to chain -your HTTP middleware functions and the app handler. - -In short, it transforms - -```go -Middleware1(Middleware2(Middleware3(App))) -``` - -to - -```go -alice.New(Middleware1, Middleware2, Middleware3).Then(App) -``` - -### Why? - -None of the other middleware chaining solutions -behaves exactly like Alice. -Alice is as minimal as it gets: -in essence, it's just a for loop that does the wrapping for you. - -Check out [this blog post](http://justinas.org/alice-painless-middleware-chaining-for-go/) -for explanation how Alice is different from other chaining solutions. - -### Usage - -Your middleware constructors should have the form of - -```go -func (http.Handler) http.Handler -``` - -Some middleware provide this out of the box. -For ones that don't, it's trivial to write one yourself. - -```go -func myStripPrefix(h http.Handler) http.Handler { - return http.StripPrefix("/old", h) -} -``` - -This complete example shows the full power of Alice. - -```go -package main - -import ( - "net/http" - "time" - - "github.com/throttled/throttled" - "github.com/justinas/alice" - "github.com/justinas/nosurf" -) - -func timeoutHandler(h http.Handler) http.Handler { - return http.TimeoutHandler(h, 1*time.Second, "timed out") -} - -func myApp(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("Hello world!")) -} - -func main() { - th := throttled.Interval(throttled.PerSec(10), 1, &throttled.VaryBy{Path: true}, 50) - myHandler := http.HandlerFunc(myApp) - - chain := alice.New(th.Throttle, timeoutHandler, nosurf.NewPure).Then(myHandler) - http.ListenAndServe(":8000", chain) -} -``` - -Here, the request will pass [throttled](https://github.com/PuerkitoBio/throttled) first, -then an http.TimeoutHandler we've set up, -then [nosurf](https://github.com/justinas/nosurf) -and will finally reach our handler. - -Note that Alice makes **no guarantees** for -how one or another piece of middleware will behave. -Once it passes the execution to the outer layer of middleware, -it has no saying in whether middleware will execute the inner handlers. -This is intentional behavior. - -Alice works with Go 1.0 and higher. - -### Contributing - -0. Find an issue that bugs you / open a new one. -1. Discuss. -2. Branch off, commit, test. -3. Make a pull request / attach the commits to the issue. diff --git a/src/vendor/github.com/justinas/alice/chain.go b/src/vendor/github.com/justinas/alice/chain.go deleted file mode 100644 index da0e2b580..000000000 --- a/src/vendor/github.com/justinas/alice/chain.go +++ /dev/null @@ -1,112 +0,0 @@ -// Package alice provides a convenient way to chain http handlers. -package alice - -import "net/http" - -// A constructor for a piece of middleware. -// Some middleware use this constructor out of the box, -// so in most cases you can just pass somepackage.New -type Constructor func(http.Handler) http.Handler - -// Chain acts as a list of http.Handler constructors. -// Chain is effectively immutable: -// once created, it will always hold -// the same set of constructors in the same order. -type Chain struct { - constructors []Constructor -} - -// New creates a new chain, -// memorizing the given list of middleware constructors. -// New serves no other function, -// constructors are only called upon a call to Then(). -func New(constructors ...Constructor) Chain { - return Chain{append(([]Constructor)(nil), constructors...)} -} - -// Then chains the middleware and returns the final http.Handler. -// New(m1, m2, m3).Then(h) -// is equivalent to: -// m1(m2(m3(h))) -// When the request comes in, it will be passed to m1, then m2, then m3 -// and finally, the given handler -// (assuming every middleware calls the following one). -// -// A chain can be safely reused by calling Then() several times. -// stdStack := alice.New(ratelimitHandler, csrfHandler) -// indexPipe = stdStack.Then(indexHandler) -// authPipe = stdStack.Then(authHandler) -// Note that constructors are called on every call to Then() -// and thus several instances of the same middleware will be created -// when a chain is reused in this way. -// For proper middleware, this should cause no problems. -// -// Then() treats nil as http.DefaultServeMux. -func (c Chain) Then(h http.Handler) http.Handler { - if h == nil { - h = http.DefaultServeMux - } - - for i := range c.constructors { - h = c.constructors[len(c.constructors)-1-i](h) - } - - return h -} - -// ThenFunc works identically to Then, but takes -// a HandlerFunc instead of a Handler. -// -// The following two statements are equivalent: -// c.Then(http.HandlerFunc(fn)) -// c.ThenFunc(fn) -// -// ThenFunc provides all the guarantees of Then. -func (c Chain) ThenFunc(fn http.HandlerFunc) http.Handler { - if fn == nil { - return c.Then(nil) - } - return c.Then(fn) -} - -// Append extends a chain, adding the specified constructors -// as the last ones in the request flow. -// -// Append returns a new chain, leaving the original one untouched. -// -// stdChain := alice.New(m1, m2) -// extChain := stdChain.Append(m3, m4) -// // requests in stdChain go m1 -> m2 -// // requests in extChain go m1 -> m2 -> m3 -> m4 -func (c Chain) Append(constructors ...Constructor) Chain { - newCons := make([]Constructor, 0, len(c.constructors)+len(constructors)) - newCons = append(newCons, c.constructors...) - newCons = append(newCons, constructors...) - - return Chain{newCons} -} - -// Extend extends a chain by adding the specified chain -// as the last one in the request flow. -// -// Extend returns a new chain, leaving the original one untouched. -// -// stdChain := alice.New(m1, m2) -// ext1Chain := alice.New(m3, m4) -// ext2Chain := stdChain.Extend(ext1Chain) -// // requests in stdChain go m1 -> m2 -// // requests in ext1Chain go m3 -> m4 -// // requests in ext2Chain go m1 -> m2 -> m3 -> m4 -// -// Another example: -// aHtmlAfterNosurf := alice.New(m2) -// aHtml := alice.New(m1, func(h http.Handler) http.Handler { -// csrf := nosurf.New(h) -// csrf.SetFailureHandler(aHtmlAfterNosurf.ThenFunc(csrfFail)) -// return csrf -// }).Extend(aHtmlAfterNosurf) -// // requests to aHtml hitting nosurfs success handler go m1 -> nosurf -> m2 -> target-handler -// // requests to aHtml hitting nosurfs failure handler go m1 -> nosurf -> m2 -> csrfFail -func (c Chain) Extend(chain Chain) Chain { - return c.Append(chain.constructors...) -} diff --git a/src/vendor/modules.txt b/src/vendor/modules.txt index cbf232b25..80004b3a6 100644 --- a/src/vendor/modules.txt +++ b/src/vendor/modules.txt @@ -269,8 +269,6 @@ github.com/hashicorp/go-multierror github.com/jmespath/go-jmespath # github.com/json-iterator/go v1.1.8 github.com/json-iterator/go -# github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da -github.com/justinas/alice # github.com/konsorten/go-windows-terminal-sequences v1.0.2 github.com/konsorten/go-windows-terminal-sequences # github.com/lib/pq v1.3.0