diff --git a/src/common/secret/store.go b/src/common/secret/store.go
index 8e4f30d0e..42e0babd4 100644
--- a/src/common/secret/store.go
+++ b/src/common/secret/store.go
@@ -17,8 +17,6 @@ package secret
 const (
 	// JobserviceUser is the name of jobservice user
 	JobserviceUser = "harbor-jobservice"
-	// ProxyserviceUser is the name of proxyservice user
-	ProxyserviceUser = "harbor-proxyservice"
 	// CoreUser is the name of ui user
 	CoreUser = "harbor-core"
 )
diff --git a/src/common/security/proxycachesecret/context.go b/src/common/security/proxycachesecret/context.go
new file mode 100644
index 000000000..6324d87f0
--- /dev/null
+++ b/src/common/security/proxycachesecret/context.go
@@ -0,0 +1,96 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proxycachesecret
+
+import (
+	"github.com/goharbor/harbor/src/common/rbac"
+	"github.com/goharbor/harbor/src/common/utils"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/pkg/permission/types"
+	"github.com/goharbor/harbor/src/pkg/project"
+)
+
+// const definition
+const (
+	// contains "#" to avoid the conflict with normal user
+	ProxyCacheService = "harbor#proxy-cache-service"
+)
+
+// SecurityContext is the security context for proxy cache secret
+type SecurityContext struct {
+	repository string
+	mgr        project.Manager
+}
+
+// NewSecurityContext returns an instance of the proxy cache secret security context
+func NewSecurityContext(repository string) *SecurityContext {
+	return &SecurityContext{
+		repository: repository,
+		mgr:        project.Mgr,
+	}
+}
+
+// Name returns the name of the security context
+func (s *SecurityContext) Name() string {
+	return "proxy_cache_secret"
+}
+
+// IsAuthenticated always returns true
+func (s *SecurityContext) IsAuthenticated() bool {
+	return true
+}
+
+// GetUsername returns the name of proxy cache service
+func (s *SecurityContext) GetUsername() string {
+	return ProxyCacheService
+}
+
+// IsSysAdmin always returns false
+func (s *SecurityContext) IsSysAdmin() bool {
+	return false
+}
+
+// IsSolutionUser always returns false
+func (s *SecurityContext) IsSolutionUser() bool {
+	return false
+}
+
+// Can returns true only when requesting pull/push operation against the specific project
+func (s *SecurityContext) Can(action types.Action, resource types.Resource) bool {
+	if !(action == rbac.ActionPull || action == rbac.ActionPush) {
+		log.Debugf("unauthorized for action %s", action)
+		return false
+	}
+	namespace, ok := rbac.ProjectNamespaceParse(resource)
+	if !ok {
+		log.Debugf("got no namespace from the resource %s", resource)
+		return false
+	}
+	project, err := s.mgr.Get(namespace.Identity())
+	if err != nil {
+		log.Errorf("failed to get project %v: %v", namespace.Identity(), err)
+		return false
+	}
+	if project == nil {
+		log.Debugf("project not found %v", namespace.Identity())
+		return false
+	}
+	pro, _ := utils.ParseRepository(s.repository)
+	if project.Name != pro {
+		log.Debugf("unauthorized for project %s", project.Name)
+		return false
+	}
+	return true
+}
diff --git a/src/common/security/proxycachesecret/context_test.go b/src/common/security/proxycachesecret/context_test.go
new file mode 100644
index 000000000..182ba79ee
--- /dev/null
+++ b/src/common/security/proxycachesecret/context_test.go
@@ -0,0 +1,108 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proxycachesecret
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/rbac"
+	"github.com/goharbor/harbor/src/testing/mock"
+	"github.com/goharbor/harbor/src/testing/pkg/project"
+	"github.com/stretchr/testify/suite"
+)
+
+type proxyCacheSecretTestSuite struct {
+	suite.Suite
+	sc  *SecurityContext
+	mgr *project.FakeManager
+}
+
+func (p *proxyCacheSecretTestSuite) SetupTest() {
+	p.mgr = &project.FakeManager{}
+	p.sc = &SecurityContext{
+		repository: "library/hello-world",
+		mgr:        p.mgr,
+	}
+}
+
+func (p *proxyCacheSecretTestSuite) TestName() {
+	p.Equal("proxy_cache_secret", p.sc.Name())
+}
+
+func (p *proxyCacheSecretTestSuite) TestIsAuthenticated() {
+	p.True(p.sc.IsAuthenticated())
+}
+
+func (p *proxyCacheSecretTestSuite) TestGetUsername() {
+	p.Equal(ProxyCacheService, p.sc.GetUsername())
+}
+
+func (p *proxyCacheSecretTestSuite) TestIsSysAdmin() {
+	p.False(p.sc.IsSysAdmin())
+}
+
+func (p *proxyCacheSecretTestSuite) TestIsSolutionUser() {
+	p.False(p.sc.IsSolutionUser())
+}
+
+func (p *proxyCacheSecretTestSuite) TestCan() {
+	// the action isn't pull/push
+	action := rbac.ActionDelete
+	resource := rbac.NewProjectNamespace(1).Resource(rbac.ResourceRepository)
+	p.False(p.sc.Can(action, resource))
+
+	// the resource isn't repository
+	action = rbac.ActionPull
+	resource = rbac.ResourceConfiguration
+	p.False(p.sc.Can(action, resource))
+
+	// the requested project not found
+	action = rbac.ActionPull
+	resource = rbac.NewProjectNamespace(2).Resource(rbac.ResourceRepository)
+	p.mgr.On("Get", mock.Anything).Return(nil, nil)
+	p.False(p.sc.Can(action, resource))
+	p.mgr.AssertExpectations(p.T())
+
+	// reset the mock
+	p.SetupTest()
+
+	// pass for action pull
+	action = rbac.ActionPull
+	resource = rbac.NewProjectNamespace(1).Resource(rbac.ResourceRepository)
+	p.mgr.On("Get", mock.Anything).Return(&models.Project{
+		ProjectID: 1,
+		Name:      "library",
+	}, nil)
+	p.True(p.sc.Can(action, resource))
+	p.mgr.AssertExpectations(p.T())
+
+	// reset the mock
+	p.SetupTest()
+
+	// pass for action push
+	action = rbac.ActionPush
+	resource = rbac.NewProjectNamespace(1).Resource(rbac.ResourceRepository)
+	p.mgr.On("Get", mock.Anything).Return(&models.Project{
+		ProjectID: 1,
+		Name:      "library",
+	}, nil)
+	p.True(p.sc.Can(action, resource))
+	p.mgr.AssertExpectations(p.T())
+}
+
+func TestProxyCacheSecretTestSuite(t *testing.T) {
+	suite.Run(t, &proxyCacheSecretTestSuite{})
+}
diff --git a/src/common/security/secret/context.go b/src/common/security/secret/context.go
index 4476ab606..bb6f72561 100644
--- a/src/common/security/secret/context.go
+++ b/src/common/security/secret/context.go
@@ -80,6 +80,5 @@ func (s *SecurityContext) Can(action types.Action, resource types.Resource) bool
 		return false
 	}
 	return s.store.GetUsername(s.secret) == secret.JobserviceUser ||
-		s.store.GetUsername(s.secret) == secret.CoreUser ||
-		s.store.GetUsername(s.secret) == secret.ProxyserviceUser
+		s.store.GetUsername(s.secret) == secret.CoreUser
 }
diff --git a/src/controller/proxy/local.go b/src/controller/proxy/local.go
index 4ca570860..76660e75e 100644
--- a/src/controller/proxy/local.go
+++ b/src/controller/proxy/local.go
@@ -20,11 +20,11 @@ import (
 	"fmt"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
-	comHttpAuth "github.com/goharbor/harbor/src/common/http/modifier/auth"
 	"github.com/goharbor/harbor/src/controller/artifact"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/lib"
 	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/pkg/proxy/secret"
 	"github.com/goharbor/harbor/src/pkg/registry"
 	"io"
 	"time"
@@ -85,8 +85,7 @@ func (l *localHelper) init() {
 	log.Debugf("core url:%s, local core url: %v", config.GetCoreURL(), config.LocalCoreURL())
 	// the traffic is internal only
 	registryURL := config.LocalCoreURL()
-	authorizer := comHttpAuth.NewSecretAuthorizer(config.ProxyServiceSecret)
-	l.registry = registry.NewClientWithAuthorizer(registryURL, authorizer, true)
+	l.registry = registry.NewClientWithAuthorizer(registryURL, secret.NewAuthorizer(), true)
 }
 
 func (l *localHelper) PushBlob(localRepo string, desc distribution.Descriptor, bReader io.ReadCloser) error {
diff --git a/src/core/config/config.go b/src/core/config/config.go
index a54525cf3..0a410f452 100755
--- a/src/core/config/config.go
+++ b/src/core/config/config.go
@@ -30,8 +30,6 @@ import (
 	"github.com/goharbor/harbor/src/core/promgr"
 	"github.com/goharbor/harbor/src/core/promgr/pmsdriver/local"
 	"github.com/goharbor/harbor/src/lib/log"
-
-	"github.com/goharbor/harbor/src/common/utils"
 )
 
 const (
@@ -51,8 +49,6 @@ var (
 	// defined as a var for testing.
 	defaultCACertPath = "/etc/core/ca/ca.crt"
 	cfgMgr            *comcfg.CfgManager
-	// ProxyServiceSecret is the secret used by proxy service
-	ProxyServiceSecret = utils.GenerateRandomStringWithLen(16)
 )
 
 // Init configurations
@@ -93,7 +89,6 @@ func initKeyProvider() {
 func initSecretStore() {
 	m := map[string]string{}
 	m[JobserviceSecret()] = secret.JobserviceUser
-	m[ProxyServiceSecret] = secret.ProxyserviceUser
 	SecretStore = secret.NewStore(m)
 }
 
diff --git a/src/core/middlewares/middlewares.go b/src/core/middlewares/middlewares.go
index 6edaf986e..e029d8c76 100644
--- a/src/core/middlewares/middlewares.go
+++ b/src/core/middlewares/middlewares.go
@@ -21,6 +21,7 @@ import (
 	"github.com/astaxie/beego"
 	"github.com/goharbor/harbor/src/pkg/distribution"
 	"github.com/goharbor/harbor/src/server/middleware"
+	"github.com/goharbor/harbor/src/server/middleware/artifactinfo"
 	"github.com/goharbor/harbor/src/server/middleware/csrf"
 	"github.com/goharbor/harbor/src/server/middleware/log"
 	"github.com/goharbor/harbor/src/server/middleware/notification"
@@ -74,6 +75,7 @@ func MiddleWares() []beego.MiddleWare {
 		orm.Middleware(),
 		notification.Middleware(), // notification must ahead of transaction ensure the DB transaction execution complete
 		transaction.Middleware(dbTxSkippers...),
+		artifactinfo.Middleware(),
 		security.Middleware(),
 		readonly.Middleware(readonlySkippers...),
 	}
diff --git a/src/server/middleware/patterns.go b/src/lib/patterns.go
similarity index 62%
rename from src/server/middleware/patterns.go
rename to src/lib/patterns.go
index c9558c1f5..1e210fbb8 100644
--- a/src/server/middleware/patterns.go
+++ b/src/lib/patterns.go
@@ -1,4 +1,4 @@
-package middleware
+package lib
 
 import (
 	"fmt"
@@ -29,3 +29,33 @@ var (
 	// V2CatalogURLRe is the regular expression for mathing the request to v2 handler to list catalog
 	V2CatalogURLRe = regexp.MustCompile(`^/v2/_catalog$`)
 )
+
+// MatchManifestURLPattern checks whether the provided path matches the manifest URL pattern,
+// if does, returns the repository and reference as well
+func MatchManifestURLPattern(path string) (repository, reference string, match bool) {
+	strs := V2ManifestURLRe.FindStringSubmatch(path)
+	if len(strs) < 3 {
+		return "", "", false
+	}
+	return strs[1], strs[2], true
+}
+
+// MatchBlobURLPattern checks whether the provided path matches the blob URL pattern,
+// if does, returns the repository and reference as well
+func MatchBlobURLPattern(path string) (repository, digest string, match bool) {
+	strs := V2BlobURLRe.FindStringSubmatch(path)
+	if len(strs) < 3 {
+		return "", "", false
+	}
+	return strs[1], strs[2], true
+}
+
+// MatchBlobUploadURLPattern checks whether the provided path matches the blob upload URL pattern,
+// if does, returns the repository as well
+func MatchBlobUploadURLPattern(path string) (repository string, match bool) {
+	strs := V2BlobUploadURLRe.FindStringSubmatch(path)
+	if len(strs) < 2 {
+		return "", false
+	}
+	return strs[1], true
+}
diff --git a/src/lib/patterns_test.go b/src/lib/patterns_test.go
new file mode 100644
index 000000000..5060f2307
--- /dev/null
+++ b/src/lib/patterns_test.go
@@ -0,0 +1,68 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lib
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMatchManifestURLPattern(t *testing.T) {
+	_, _, ok := MatchManifestURLPattern("")
+	assert.False(t, ok)
+
+	_, _, ok = MatchManifestURLPattern("/v2/")
+	assert.False(t, ok)
+
+	repository, reference, ok := MatchManifestURLPattern("/v2/library/hello-world/manifests/latest")
+	assert.True(t, ok)
+	assert.Equal(t, "library/hello-world", repository)
+	assert.Equal(t, "latest", reference)
+
+	repository, reference, ok = MatchManifestURLPattern("/v2/library/hello-world/manifests/sha256:e5785cb0c62cebbed4965129bae371f0589cadd6d84798fb58c2c5f9e237efd9")
+	assert.True(t, ok)
+	assert.Equal(t, "library/hello-world", repository)
+	assert.Equal(t, "sha256:e5785cb0c62cebbed4965129bae371f0589cadd6d84798fb58c2c5f9e237efd9", reference)
+}
+
+func TestMatchBlobURLPattern(t *testing.T) {
+	_, _, ok := MatchBlobURLPattern("")
+	assert.False(t, ok)
+
+	_, _, ok = MatchBlobURLPattern("/v2/")
+	assert.False(t, ok)
+
+	repository, digest, ok := MatchBlobURLPattern("/v2/library/hello-world/blobs/sha256:e5785cb0c62cebbed4965129bae371f0589cadd6d84798fb58c2c5f9e237efd9")
+	assert.True(t, ok)
+	assert.Equal(t, "library/hello-world", repository)
+	assert.Equal(t, "sha256:e5785cb0c62cebbed4965129bae371f0589cadd6d84798fb58c2c5f9e237efd9", digest)
+}
+
+func TestMatchBlobUploadURLPattern(t *testing.T) {
+	_, ok := MatchBlobUploadURLPattern("")
+	assert.False(t, ok)
+
+	_, ok = MatchBlobUploadURLPattern("/v2/")
+	assert.False(t, ok)
+
+	repository, ok := MatchBlobUploadURLPattern("/v2/library/hello-world/blobs/uploads/")
+	assert.True(t, ok)
+	assert.Equal(t, "library/hello-world", repository)
+
+	repository, ok = MatchBlobUploadURLPattern("/v2/library/hello-world/blobs/uploads/uuid")
+	assert.True(t, ok)
+	assert.Equal(t, "library/hello-world", repository)
+}
diff --git a/src/pkg/proxy/secret/authorizer.go b/src/pkg/proxy/secret/authorizer.go
new file mode 100644
index 000000000..f783cb479
--- /dev/null
+++ b/src/pkg/proxy/secret/authorizer.go
@@ -0,0 +1,63 @@
+//  Copyright Project Harbor Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package secret
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/goharbor/harbor/src/lib"
+)
+
+const (
+	secretPrefix = "Proxy-Cache-Secret"
+)
+
+// NewAuthorizer returns an instance of the authorizer
+func NewAuthorizer() lib.Authorizer {
+	return &authorizer{}
+}
+
+type authorizer struct{}
+
+func (s *authorizer) Modify(req *http.Request) error {
+	if req == nil {
+		return errors.New("the request is null")
+	}
+	repository, _, ok := lib.MatchManifestURLPattern(req.URL.Path)
+	if !ok {
+		repository, _, ok = lib.MatchBlobURLPattern(req.URL.Path)
+		if !ok {
+			repository, ok = lib.MatchBlobUploadURLPattern(req.URL.Path)
+			if !ok {
+				return nil
+			}
+		}
+	}
+	secret := GetManager().Generate(repository)
+	req.Header.Set("Authorization", fmt.Sprintf("%s %s", secretPrefix, secret))
+	return nil
+}
+
+// GetSecret gets the secret from the request authorization header
+func GetSecret(req *http.Request) string {
+	auth := req.Header.Get("Authorization")
+	if !strings.HasPrefix(auth, secretPrefix) {
+		return ""
+	}
+	return strings.TrimSpace(strings.TrimPrefix(auth, secretPrefix))
+}
diff --git a/src/pkg/proxy/secret/authorizer_test.go b/src/pkg/proxy/secret/authorizer_test.go
new file mode 100644
index 000000000..453b50bf4
--- /dev/null
+++ b/src/pkg/proxy/secret/authorizer_test.go
@@ -0,0 +1,50 @@
+//  Copyright Project Harbor Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package secret
+
+import (
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"net/http"
+	"testing"
+)
+
+func TestAuthorizer(t *testing.T) {
+	authorizer := &authorizer{}
+
+	// not manifest/blob requests
+	req, _ := http.NewRequest(http.MethodGet, "http://127.0.0.1/v2/_catalog", nil)
+	err := authorizer.Modify(req)
+	require.Nil(t, err)
+	assert.Empty(t, GetSecret(req))
+
+	// pass, manifest URL
+	req, _ = http.NewRequest(http.MethodGet, "http://127.0.0.1/v2/library/hello-world/manifests/latest", nil)
+	err = authorizer.Modify(req)
+	require.Nil(t, err)
+	assert.NotEmpty(t, GetSecret(req))
+
+	// pass, blob URL
+	req, _ = http.NewRequest(http.MethodGet, "http://127.0.0.1/v2/library/hello-world/blobs/sha256:e5785cb0c62cebbed4965129bae371f0589cadd6d84798fb58c2c5f9e237efd9", nil)
+	err = authorizer.Modify(req)
+	require.Nil(t, err)
+	assert.NotEmpty(t, GetSecret(req))
+
+	// pass, blob upload URL
+	req, _ = http.NewRequest(http.MethodGet, "http://127.0.0.1/v2/library/hello-world/blobs/uploads/uuid", nil)
+	err = authorizer.Modify(req)
+	require.Nil(t, err)
+	assert.NotEmpty(t, GetSecret(req))
+}
diff --git a/src/server/middleware/artifactinfo/artifact_info.go b/src/server/middleware/artifactinfo/artifact_info.go
index d4c6bae9a..704ba41bf 100644
--- a/src/server/middleware/artifactinfo/artifact_info.go
+++ b/src/server/middleware/artifactinfo/artifact_info.go
@@ -25,7 +25,6 @@ import (
 	"github.com/goharbor/harbor/src/lib"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/server/middleware"
 	"github.com/opencontainers/go-digest"
 )
 
@@ -39,10 +38,10 @@ const (
 
 var (
 	urlPatterns = map[string]*regexp.Regexp{
-		"manifest":    middleware.V2ManifestURLRe,
-		"tag_list":    middleware.V2TagListURLRe,
-		"blob_upload": middleware.V2BlobUploadURLRe,
-		"blob":        middleware.V2BlobURLRe,
+		"manifest":    lib.V2ManifestURLRe,
+		"tag_list":    lib.V2TagListURLRe,
+		"blob_upload": lib.V2BlobUploadURLRe,
+		"blob":        lib.V2BlobURLRe,
 	}
 )
 
@@ -56,7 +55,7 @@ func Middleware() func(http.Handler) http.Handler {
 				next.ServeHTTP(rw, req)
 				return
 			}
-			repo := m[middleware.RepositorySubexp]
+			repo := m[lib.RepositorySubexp]
 			pn, err := projectNameFromRepo(repo)
 			if err != nil {
 				lib_http.SendError(rw, errors.BadRequestError(err))
@@ -66,10 +65,10 @@ func Middleware() func(http.Handler) http.Handler {
 				Repository:  repo,
 				ProjectName: pn,
 			}
-			if d, ok := m[middleware.DigestSubexp]; ok {
+			if d, ok := m[lib.DigestSubexp]; ok {
 				art.Digest = d
 			}
-			if ref, ok := m[middleware.ReferenceSubexp]; ok {
+			if ref, ok := m[lib.ReferenceSubexp]; ok {
 				art.Reference = ref
 			}
 			if t, ok := m[tag]; ok {
@@ -120,9 +119,9 @@ func parse(url *url.URL) (map[string]string, bool) {
 			break
 		}
 	}
-	if digest.DigestRegexp.MatchString(m[middleware.ReferenceSubexp]) {
-		m[middleware.DigestSubexp] = m[middleware.ReferenceSubexp]
-	} else if ref, ok := m[middleware.ReferenceSubexp]; ok {
+	if digest.DigestRegexp.MatchString(m[lib.ReferenceSubexp]) {
+		m[lib.DigestSubexp] = m[lib.ReferenceSubexp]
+	} else if ref, ok := m[lib.ReferenceSubexp]; ok {
 		m[tag] = ref
 	}
 	return m, match
diff --git a/src/server/middleware/artifactinfo/artifact_info_test.go b/src/server/middleware/artifactinfo/artifact_info_test.go
index d5de72c9b..602801dfe 100644
--- a/src/server/middleware/artifactinfo/artifact_info_test.go
+++ b/src/server/middleware/artifactinfo/artifact_info_test.go
@@ -22,7 +22,6 @@ import (
 	"testing"
 
 	"github.com/goharbor/harbor/src/lib"
-	"github.com/goharbor/harbor/src/server/middleware"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -45,16 +44,16 @@ func TestParseURL(t *testing.T) {
 		{
 			input: "/v2/no-project-repo/tags/list",
 			expect: map[string]string{
-				middleware.RepositorySubexp: "no-project-repo",
+				lib.RepositorySubexp: "no-project-repo",
 			},
 			match: true,
 		},
 		{
 			input: "/v2/development/golang/manifests/sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
 			expect: map[string]string{
-				middleware.RepositorySubexp: "development/golang",
-				middleware.ReferenceSubexp:  "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
-				middleware.DigestSubexp:     "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+				lib.RepositorySubexp: "development/golang",
+				lib.ReferenceSubexp:  "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+				lib.DigestSubexp:     "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
 			},
 			match: true,
 		},
@@ -67,8 +66,8 @@ func TestParseURL(t *testing.T) {
 		{
 			input: "/v2/multi/sector/repository/blobs/sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
 			expect: map[string]string{
-				middleware.RepositorySubexp: "multi/sector/repository",
-				middleware.DigestSubexp:     "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+				lib.RepositorySubexp: "multi/sector/repository",
+				lib.DigestSubexp:     "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
 			},
 			match: true,
 		},
@@ -80,23 +79,23 @@ func TestParseURL(t *testing.T) {
 		{
 			input: "/v2/library/ubuntu/blobs/uploads",
 			expect: map[string]string{
-				middleware.RepositorySubexp: "library/ubuntu",
+				lib.RepositorySubexp: "library/ubuntu",
 			},
 			match: true,
 		},
 		{
 			input: "/v2/library/ubuntu/blobs/uploads/?mount=sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f&from=old/ubuntu",
 			expect: map[string]string{
-				middleware.RepositorySubexp: "library/ubuntu",
-				blobMountDigest:             "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
-				blobMountRepo:               "old/ubuntu",
+				lib.RepositorySubexp: "library/ubuntu",
+				blobMountDigest:      "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+				blobMountRepo:        "old/ubuntu",
 			},
 			match: true,
 		},
 		{
 			input: "/v2/library/centos/blobs/uploads/u-12345",
 			expect: map[string]string{
-				middleware.RepositorySubexp: "library/centos",
+				lib.RepositorySubexp: "library/centos",
 			},
 			match: true,
 		},
diff --git a/src/server/middleware/repoproxy/proxy.go b/src/server/middleware/repoproxy/proxy.go
index 565195b5d..1c8fe9a78 100644
--- a/src/server/middleware/repoproxy/proxy.go
+++ b/src/server/middleware/repoproxy/proxy.go
@@ -17,8 +17,8 @@ package repoproxy
 import (
 	"context"
 	"fmt"
-	"github.com/goharbor/harbor/src/common/secret"
 	"github.com/goharbor/harbor/src/common/security"
+	"github.com/goharbor/harbor/src/common/security/proxycachesecret"
 	"github.com/goharbor/harbor/src/lib/errors"
 	httpLib "github.com/goharbor/harbor/src/lib/http"
 	"github.com/goharbor/harbor/src/replication/model"
@@ -158,7 +158,7 @@ func isProxySession(ctx context.Context) bool {
 		log.Error("Failed to get security context")
 		return false
 	}
-	if sc.IsSolutionUser() && sc.GetUsername() == secret.ProxyserviceUser {
+	if sc.GetUsername() == proxycachesecret.ProxyCacheService {
 		return true
 	}
 	return false
diff --git a/src/server/middleware/repoproxy/proxy_test.go b/src/server/middleware/repoproxy/proxy_test.go
index ab0b44db6..fd2923d67 100644
--- a/src/server/middleware/repoproxy/proxy_test.go
+++ b/src/server/middleware/repoproxy/proxy_test.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/security"
+	"github.com/goharbor/harbor/src/common/security/proxycachesecret"
 	securitySecret "github.com/goharbor/harbor/src/common/security/secret"
 	"github.com/goharbor/harbor/src/core/config"
 	"testing"
@@ -59,7 +60,7 @@ func TestIsProxySession(t *testing.T) {
 	sc1 := securitySecret.NewSecurityContext("123456789", config.SecretStore)
 	otherCtx := security.NewContext(context.Background(), sc1)
 
-	sc2 := securitySecret.NewSecurityContext(config.ProxyServiceSecret, config.SecretStore)
+	sc2 := proxycachesecret.NewSecurityContext("library/hello-world")
 	proxyCtx := security.NewContext(context.Background(), sc2)
 	cases := []struct {
 		name string
diff --git a/src/server/middleware/security/proxy_cache_secret.go b/src/server/middleware/security/proxy_cache_secret.go
new file mode 100644
index 000000000..2045d3229
--- /dev/null
+++ b/src/server/middleware/security/proxy_cache_secret.go
@@ -0,0 +1,42 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package security
+
+import (
+	"net/http"
+
+	"github.com/goharbor/harbor/src/common/security"
+	"github.com/goharbor/harbor/src/common/security/proxycachesecret"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/goharbor/harbor/src/lib/log"
+	ps "github.com/goharbor/harbor/src/pkg/proxy/secret"
+)
+
+type proxyCacheSecret struct{}
+
+func (p *proxyCacheSecret) Generate(req *http.Request) security.Context {
+	log := log.G(req.Context())
+
+	artifact := lib.GetArtifactInfo(req.Context())
+	if artifact == (lib.ArtifactInfo{}) {
+		return nil
+	}
+	secret := ps.GetSecret(req)
+	if !ps.GetManager().Verify(secret, artifact.Repository) {
+		return nil
+	}
+	log.Debugf("a proxy cache secret security context generated for request %s %s", req.Method, req.URL.Path)
+	return proxycachesecret.NewSecurityContext(artifact.Repository)
+}
diff --git a/src/server/middleware/security/proxy_cache_secret_test.go b/src/server/middleware/security/proxy_cache_secret_test.go
new file mode 100644
index 000000000..c68b73ffd
--- /dev/null
+++ b/src/server/middleware/security/proxy_cache_secret_test.go
@@ -0,0 +1,49 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package security
+
+import (
+	"fmt"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"testing"
+
+	ps "github.com/goharbor/harbor/src/pkg/proxy/secret"
+)
+
+func TestProxyCacheSecret(t *testing.T) {
+	psc := &proxyCacheSecret{}
+
+	// request without artifact info in context
+	req, _ := http.NewRequest(http.MethodGet, "http://127.0.0.1/v2/library/hello-world/manifests/latest", nil)
+	sc := psc.Generate(req)
+	assert.Nil(t, sc)
+
+	// request with invalid secret
+	ctx := lib.WithArtifactInfo(req.Context(), lib.ArtifactInfo{
+		Repository: "library/hello-world",
+	})
+	req = req.WithContext(ctx)
+	req.Header.Set("Authorization", fmt.Sprintf("Proxy-Cache-Secret %s", "invalid-secret"))
+	sc = psc.Generate(req)
+	assert.Nil(t, sc)
+
+	// pass
+	secret := ps.GetManager().Generate("library/hello-world")
+	req.Header.Set("Authorization", fmt.Sprintf("Proxy-Cache-Secret %s", secret))
+	sc = psc.Generate(req)
+	assert.NotNil(t, sc)
+}
diff --git a/src/server/middleware/security/security.go b/src/server/middleware/security/security.go
index b9764f72a..4c900dd04 100644
--- a/src/server/middleware/security/security.go
+++ b/src/server/middleware/security/security.go
@@ -33,6 +33,7 @@ var (
 		&robot{},
 		&basicAuth{},
 		&session{},
+		&proxyCacheSecret{},
 		&unauthorized{},
 	}
 )
diff --git a/src/server/middleware/v2auth/access.go b/src/server/middleware/v2auth/access.go
index 8dff98a0d..b460ae0b7 100644
--- a/src/server/middleware/v2auth/access.go
+++ b/src/server/middleware/v2auth/access.go
@@ -8,7 +8,6 @@ import (
 	"github.com/goharbor/harbor/src/common/rbac"
 	"github.com/goharbor/harbor/src/lib"
 	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/server/middleware"
 )
 
 type target int
@@ -72,7 +71,7 @@ func accessList(req *http.Request) []access {
 		})
 		return l
 	}
-	if len(middleware.V2CatalogURLRe.FindStringSubmatch(req.URL.Path)) == 1 {
+	if len(lib.V2CatalogURLRe.FindStringSubmatch(req.URL.Path)) == 1 {
 		l = append(l, access{
 			target: catalog,
 		})
diff --git a/src/server/registry/route.go b/src/server/registry/route.go
index 636f3a644..ec2a7c0be 100644
--- a/src/server/registry/route.go
+++ b/src/server/registry/route.go
@@ -17,7 +17,6 @@ package registry
 import (
 	"net/http"
 
-	"github.com/goharbor/harbor/src/server/middleware/artifactinfo"
 	"github.com/goharbor/harbor/src/server/middleware/blob"
 	"github.com/goharbor/harbor/src/server/middleware/contenttrust"
 	"github.com/goharbor/harbor/src/server/middleware/immutable"
@@ -32,7 +31,6 @@ import (
 func RegisterRoutes() {
 	root := router.NewRoute().
 		Path("/v2").
-		Middleware(artifactinfo.Middleware()).
 		Middleware(v2auth.Middleware())
 	// catalog
 	root.NewRoute().