diff --git a/src/controller/proxy/controller.go b/src/controller/proxy/controller.go index e63687abf..1b7dc7ef5 100644 --- a/src/controller/proxy/controller.go +++ b/src/controller/proxy/controller.go @@ -166,6 +166,9 @@ func (c *controller) UseLocalManifest(ctx context.Context, art lib.ArtifactInfo, remoteRepo := getRemoteRepo(art) exist, desc, err := remote.ManifestExist(remoteRepo, getReference(art)) // HEAD if err != nil { + if errors.IsRateLimitError(err) && a != nil { // if rate limit, use local if it exists, otherwise return error + return true, nil, nil + } return false, nil, err } if !exist || desc == nil { diff --git a/src/controller/proxy/controller_test.go b/src/controller/proxy/controller_test.go index 50bcad22d..f141e1bc6 100644 --- a/src/controller/proxy/controller_test.go +++ b/src/controller/proxy/controller_test.go @@ -122,6 +122,30 @@ func (p *proxyControllerTestSuite) TestUseLocalManifest_False() { p.Assert().False(result) } +func (p *proxyControllerTestSuite) TestUseLocalManifest_429() { + ctx := context.Background() + dig := "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" + desc := &distribution.Descriptor{Digest: digest.Digest(dig)} + art := lib.ArtifactInfo{Repository: "library/hello-world", Digest: dig} + p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(false, desc, errors.New("too many requests").WithCode(errors.RateLimitCode)) + p.local.On("GetManifest", mock.Anything, mock.Anything).Return(nil, nil) + _, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote) + p.Assert().NotNil(err) + errors.IsRateLimitError(err) +} + +func (p *proxyControllerTestSuite) TestUseLocalManifest_429ToLocal() { + ctx := context.Background() + dig := "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" + desc := &distribution.Descriptor{Digest: digest.Digest(dig)} + art := lib.ArtifactInfo{Repository: "library/hello-world", Digest: dig} + p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(false, desc, errors.New("too many requests").WithCode(errors.RateLimitCode)) + p.local.On("GetManifest", mock.Anything, mock.Anything).Return(&artifact.Artifact{}, nil) + result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote) + p.Assert().Nil(err) + p.Assert().True(result) +} + func (p *proxyControllerTestSuite) TestUseLocalManifestWithTag_False() { ctx := context.Background() art := lib.ArtifactInfo{Repository: "library/hello-world", Tag: "latest"} diff --git a/src/lib/errors/const.go b/src/lib/errors/const.go index d8cb28df1..1dbee8cb7 100644 --- a/src/lib/errors/const.go +++ b/src/lib/errors/const.go @@ -13,6 +13,8 @@ const ( ForbiddenCode = "FORBIDDEN" // MethodNotAllowedCode ... MethodNotAllowedCode = "METHOD_NOT_ALLOWED" + // RateLimitCode + RateLimitCode = "TOO_MANY_REQUEST" // PreconditionCode ... PreconditionCode = "PRECONDITION" // GeneralCode ... @@ -91,3 +93,8 @@ func IsConflictErr(err error) bool { func IsChallengesUnsupportedErr(err error) bool { return IsErr(err, ChallengesUnsupportedCode) } + +// IsRateLimitError checks whether the err chains contains rate limit error +func IsRateLimitError(err error) bool { + return IsErr(err, RateLimitCode) +} diff --git a/src/lib/http/error.go b/src/lib/http/error.go index 382339755..c3abd94ba 100644 --- a/src/lib/http/error.go +++ b/src/lib/http/error.go @@ -37,6 +37,7 @@ var ( errors.MethodNotAllowedCode: http.StatusMethodNotAllowed, errors.DENIED: http.StatusForbidden, errors.NotFoundCode: http.StatusNotFound, + errors.RateLimitCode: http.StatusTooManyRequests, errors.ConflictCode: http.StatusConflict, errors.PreconditionCode: http.StatusPreconditionFailed, errors.ViolateForeignKeyConstraintCode: http.StatusPreconditionFailed, diff --git a/src/pkg/registry/client.go b/src/pkg/registry/client.go index 5fe9399e5..3c0b86ed8 100644 --- a/src/pkg/registry/client.go +++ b/src/pkg/registry/client.go @@ -678,6 +678,8 @@ func (c *client) do(req *http.Request) (*http.Response, error) { code = errors.ForbiddenCode case http.StatusNotFound: code = errors.NotFoundCode + case http.StatusTooManyRequests: + code = errors.RateLimitCode } return nil, errors.New(nil).WithCode(code). WithMessage(message) diff --git a/src/server/middleware/repoproxy/proxy.go b/src/server/middleware/repoproxy/proxy.go index 1263ddaee..26155bb8f 100644 --- a/src/server/middleware/repoproxy/proxy.go +++ b/src/server/middleware/repoproxy/proxy.go @@ -46,6 +46,8 @@ const ( ensureTagMaxRetry = 60 ) +var tooManyRequestsError = errors.New("too many requests to upstream registry").WithCode(errors.RateLimitCode) + // BlobGetMiddleware handle get blob request func BlobGetMiddleware() func(http.Handler) http.Handler { return middleware.New(func(w http.ResponseWriter, r *http.Request, next http.Handler) { @@ -112,6 +114,10 @@ func ManifestMiddleware() func(http.Handler) http.Handler { httpLib.SendError(w, err) return } + if errors.IsRateLimitError(err) { + httpLib.SendError(w, tooManyRequestsError) + return + } log.Errorf("failed to proxy manifest, fallback to local, request uri: %v, error: %v", r.RequestURI, err) next.ServeHTTP(w, r) } @@ -202,7 +208,7 @@ func handleManifest(w http.ResponseWriter, r *http.Request, next http.Handler) e err = proxyManifestGet(ctx, w, proxyCtl, p, art, remote) } if err != nil { - if errors.IsNotFoundErr(err) { + if errors.IsNotFoundErr(err) || errors.IsRateLimitError(err) { return err } log.Warningf("Proxy to remote failed, fallback to local repo, error: %v", err)