Merge branch 'master' into redis-idle-timeout

.github/workflows/CI.yml

@@ -39,7 +39,6 @@ jobs:
           docker_channel: stable
       - uses: actions/checkout@v1
         with:
-         fetch-depth: 2
          path: src/github.com/goharbor/harbor
       - name: setup env
         run: |
@@ -100,7 +99,6 @@ jobs:
           docker_channel: stable
       - uses: actions/checkout@v1
         with:
-          fetch-depth: 2
           path: src/github.com/goharbor/harbor
       - name: setup env
         run: |
@@ -154,7 +152,6 @@ jobs:
           docker_channel: stable
       - uses: actions/checkout@v1
         with:
-          fetch-depth: 2
           path: src/github.com/goharbor/harbor
       - name: setup env
         run: |
@@ -208,7 +205,6 @@ jobs:
           docker_channel: stable
       - uses: actions/checkout@v1
         with:
-          fetch-depth: 2
           path: src/github.com/goharbor/harbor
       - name: setup env
         run: |
@@ -252,7 +248,6 @@ jobs:
           node-version: '10.16.2'
       - uses: actions/checkout@v1
         with:
-          fetch-depth: 2
           path: src/github.com/goharbor/harbor
       - name: setup env
         run: |

Makefile

@@ -374,6 +374,12 @@ build_base_docker:
 		$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) goharbor/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) ; \
 	done
 
+pull_base_docker:
+	@for name in chartserver clair clair-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
+		echo $$name ; \
+		$(DOCKERPULL) goharbor/harbor-$$name-base:$(BASEIMAGETAG) ; \
+	done
+
 install: compile build prepare start
 
 package_online: update_prepare_version
@@ -498,6 +504,7 @@ swagger_client:
 	rm -rf harborclient
 	mkdir  -p harborclient/harbor_swagger_client
 	mkdir  -p harborclient/harbor_v2_swagger_client
+	sed -i "/type: basic/ a\\security:\n  - basicAuth: []" api/v2.0/swagger.yaml
 	java -jar swagger-codegen-cli.jar generate -i api/v2.0/legacy_swagger.yaml -l python -o harborclient/harbor_swagger_client -DpackageName=swagger_client
 	java -jar swagger-codegen-cli.jar generate -i api/v2.0/swagger.yaml -l python -o harborclient/harbor_v2_swagger_client -DpackageName=v2_swagger_client
 	cd harborclient/harbor_swagger_client; python ./setup.py install

api/v2.0/legacy_swagger.yaml

@@ -1066,31 +1066,6 @@ paths:
         '500':
           description: Unexpected internal errors.
   '/repositories/{repo_name}':
-    delete:
-      summary: Delete a repository.
-      description: |
-        This endpoint let user delete a repository with name.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: The name of repository which will be deleted.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Delete successfully.
-        '400':
-          description: Invalid repo_name.
-        '401':
-          description: Unauthorized.
-        '403':
-          description: Forbidden.
-        '404':
-          description: Repository not found.
-        '412':
-          description: Precondition Failed.
     put:
       summary: Update description of the repository.
       description: |
@@ -1118,382 +1093,6 @@ paths:
           description: Forbidden.
         '404':
           description: Repository not found.
-  '/repositories/{repo_name}/labels':
-    get:
-      summary: Get labels of a repository.
-      description: |
-        Get labels of a repository specified by the repo_name.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: The name of repository.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Successfully.
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/Label'
-        '401':
-          description: Unauthorized.
-        '403':
-          description: Forbidden. User should have read permisson for the repository to perform the action.
-        '404':
-          description: Repository not found.
-    post:
-      summary: Add a label to the repository.
-      description: |
-        Add a label to the repository.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: The name of repository.
-        - name: label
-          in: body
-          description: Only the ID property is required.
-          required: true
-          schema:
-            $ref: '#/definitions/Label'
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Successfully.
-        '401':
-          description: Unauthorized.
-        '403':
-          description: Forbidden. User should have write permisson for the repository to perform the action.
-        '404':
-          description: Resource not found.
-  '/repositories/{repo_name}/labels/{label_id}':
-    delete:
-      summary: Delete label from the repository.
-      description: |
-        Delete the label from the repository specified by the repo_name.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: The name of repository.
-        - name: label_id
-          in: path
-          type: integer
-          required: true
-          description: The ID of label.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Successfully.
-        '401':
-          description: Unauthorized.
-        '403':
-          description: Forbidden. User should have write permisson for the repository to perform the action.
-        '404':
-          description: Resource not found.
-  '/repositories/{repo_name}/tags/{tag}':
-    get:
-      summary: Get the tag of the repository.
-      description: |
-        This endpoint aims to retrieve the tag of the repository. If deployed with Notary, the signature property of response represents whether the image is singed or not. If the property is null, the image is unsigned.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: Relevant repository name.
-        - name: tag
-          in: path
-          type: string
-          required: true
-          description: Tag of the repository.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Get tag successfully.
-          schema:
-            $ref: '#/definitions/DetailedTag'
-        '500':
-          description: Unexpected internal errors.
-    delete:
-      summary: Delete a tag in a repository.
-      description: |
-        This endpoint let user delete tags with repo name and tag.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: The name of repository which will be deleted.
-        - name: tag
-          in: path
-          type: string
-          required: true
-          description: Tag of a repository.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Delete tag successfully.
-        '400':
-          description: Invalid repo_name.
-        '401':
-          description: Unauthorized.
-        '403':
-          description: Forbidden.
-        '404':
-          description: Repository or tag not found.
-  '/repositories/{repo_name}/tags':
-    get:
-      summary: Get tags of a relevant repository.
-      description: |
-        This endpoint aims to retrieve tags from a relevant repository. If deployed with Notary, the signature property of response represents whether the image is singed or not. If the property is null, the image is unsigned.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: Relevant repository name.
-        - name: label_id
-          in: query
-          type: string
-          required: false
-          description: A label ID.
-        - name: detail
-          in: query
-          type: boolean
-          required: false
-          description: Bool value indicating whether return detailed information of the tag, such as vulnerability scan info, if set to false, only tag name is returned.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Get tags successfully.
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/DetailedTag'
-        '500':
-          description: Unexpected internal errors.
-    post:
-      summary: Retag an image
-      description: >
-        This endpoint tags an existing image with another tag in this repo, source images
-        can be in different repos or projects.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: Relevant repository name.
-        - name: request
-          in: body
-          description: Request to give source image and target tag.
-          required: true
-          schema:
-            $ref: '#/definitions/RetagReq'
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Image retag successfully.
-        '400':
-          description: Invalid image values provided.
-        '401':
-          description: User has no permission to the source project or destination project.
-        '403':
-          description: Forbiden as quota exceeded.
-        '404':
-          description: Project or repository not found.
-        '409':
-          description: Target tag already exists.
-        '500':
-          description: Unexpected internal errors.
-  '/repositories/{repo_name}/tags/{tag}/labels':
-    get:
-      summary: Get labels of an image.
-      description: |
-        Get labels of an image specified by the repo_name and tag.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: The name of repository.
-        - name: tag
-          in: path
-          type: string
-          required: true
-          description: The tag of the image.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Successfully.
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/Label'
-        '401':
-          description: Unauthorized.
-        '403':
-          description: Forbidden. User should have read permisson for the image to perform the action.
-        '404':
-          description: Resource not found.
-    post:
-      summary: Add a label to image.
-      description: |
-        Add a label to the image.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: The name of repository.
-        - name: tag
-          in: path
-          type: string
-          required: true
-          description: The tag of the image.
-        - name: label
-          in: body
-          description: Only the ID property is required.
-          required: true
-          schema:
-            $ref: '#/definitions/Label'
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Successfully.
-        '401':
-          description: Unauthorized.
-        '403':
-          description: Forbidden. User should have write permisson for the image to perform the action.
-        '404':
-          description: Resource not found.
-  '/repositories/{repo_name}/tags/{tag}/labels/{label_id}':
-    delete:
-      summary: Delete label from the image.
-      description: |
-        Delete the label from the image specified by the repo_name and tag.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: The name of repository.
-        - name: tag
-          in: path
-          type: string
-          required: true
-          description: The tag of the image.
-        - name: label_id
-          in: path
-          type: integer
-          required: true
-          description: The ID of label.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Successfully.
-        '401':
-          description: Unauthorized.
-        '403':
-          description: Forbidden. User should have write permisson for the image to perform the action.
-        '404':
-          description: Resource not found.
-  '/repositories/{repo_name}/tags/{tag}/manifest':
-    get:
-      summary: Get manifests of a relevant repository.
-      description: |
-        This endpoint aims to retreive manifests from a relevant repository.
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: Repository name
-        - name: tag
-          in: path
-          type: string
-          required: true
-          description: Tag name
-        - name: version
-          in: query
-          type: string
-          required: false
-          description: 'The version of manifest, valid value are "v1" and "v2", default is "v2"'
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Retrieved manifests from a relevant repository successfully.
-          schema:
-            $ref: '#/definitions/Manifest'
-        '404':
-          description: Retrieved manifests from a relevant repository not found.
-        '500':
-          description: Unexpected internal errors.
-  '/repositories/{repo_name}/signatures':
-    get:
-      summary: Get signature information of a repository
-      description: |
-        This endpoint aims to retrieve signature information of a repository, the data is
-        from the nested notary instance of Harbor.
-        If the repository does not have any signature information in notary, this API will
-        return an empty list with response code 200, instead of 404
-      parameters:
-        - name: repo_name
-          in: path
-          type: string
-          required: true
-          description: repository name.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Retrieved signatures.
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/RepoSignature'
-        '500':
-          description: Server side error.
-  /repositories/top:
-    get:
-      summary: Get public repositories which are accessed most.
-      description: |
-        This endpoint aims to let users see the most popular public repositories
-      parameters:
-        - name: count
-          in: query
-          type: integer
-          format: int32
-          required: false
-          description: 'The number of the requested public repositories, default is 10 if not provided.'
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Get popular repositories successfully.
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/Repository'
-        '400':
-          description: Bad request because of invalid count.
-        '500':
-          description: Unexpected internal errors.
   /logs:
     get:
       summary: Get recent logs of the projects which the user is a member of
@@ -6077,6 +5676,9 @@ definitions:
       job_kind:
         type: string
         description: the job kind of gc job.
+      job_parameters:
+        type: string
+        description: the job parameters of gc job.
       schedule:
         $ref: '#/definitions/AdminJobScheduleObj'
       job_status:
@@ -6096,6 +5698,9 @@ definitions:
     properties:
       schedule:
         $ref: '#/definitions/AdminJobScheduleObj'
+      parameters:
+        type: string
+        description: The parameters of admin job
   AdminJobScheduleObj:
     type: object
     properties:

api/v2.0/swagger.yaml

@@ -58,6 +58,31 @@ paths:
         '500':
           $ref: '#/responses/500'
   /projects/{project_name}/repositories/{repository_name}:
+    get:
+      summary: Get repository
+      description: Get the repository specified by name
+      tags:
+        - repository
+      operationId: getRepository
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/projectName'
+        - $ref: '#/parameters/repositoryName'
+      responses:
+        '200':
+          description: Success
+          schema:
+            $ref: '#/definitions/Repository'
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '404':
+          $ref: '#/responses/404'
+        '500':
+          $ref: '#/responses/500'
     put:
       summary: Update repository
       description: Update the repository specified by name
@@ -299,6 +324,29 @@ paths:
           $ref: '#/responses/404'
         '500':
           $ref: '#/responses/500'
+  /projects/{project_name}/repositories/{repository_name}/artifacts/{reference}/scan:
+    post:
+      summary: Scan the artifact
+      description: Scan the specified artifact
+      tags:
+        - artifact
+      operationId: scanArtifact
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/projectName'
+        - $ref: '#/parameters/repositoryName'
+        - $ref: '#/parameters/reference'
+      responses:
+        '202':
+          $ref: '#/responses/202'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '404':
+          $ref: '#/responses/404'
+        '500':
+          $ref: '#/responses/500'
   /projects/{project_name}/repositories/{repository_name}/artifacts/{reference}/tags:
     post:
       summary: Create tag
@@ -512,6 +560,12 @@ responses:
       X-Request-Id:
         description: The ID of the corresponding request for the response
         type: string
+  '202':
+    description: Accepted
+    headers:
+      X-Request-Id:
+        description: The ID of the corresponding request for the response
+        type: string
   '400':
     description: Bad request
     headers:

make/migrations/postgresql/0030_2.0.0_schema.up.sql

@@ -1,3 +1,4 @@
+ALTER TABLE admin_job ADD COLUMN job_parameters varchar(255) Default '';
 ALTER TABLE artifact ADD COLUMN repository_id int;
 ALTER TABLE artifact ADD COLUMN media_type varchar(255);
 ALTER TABLE artifact ADD COLUMN manifest_media_type varchar(255);
@@ -55,9 +56,6 @@ WHERE ordered_art.seq=1;
 
 ALTER TABLE artifact DROP COLUMN tag;
 
-/*TODO: remove this after insert the repository_name when create artifact*/
-ALTER TABLE artifact ALTER COLUMN repository_name DROP NOT NULL;
-
 /*remove the duplicate artifact rows*/
 DELETE FROM artifact
 WHERE id NOT IN (

make/photon/core/Dockerfile

@@ -1,7 +1,7 @@
 ARG harbor_base_image_version
 FROM goharbor/harbor-core-base:${harbor_base_image_version}
 
-HEALTHCHECK CMD curl --fail -s http://127.0.0.1:8080/api/ping || exit 1
+HEALTHCHECK CMD curl --fail -s http://127.0.0.1:8080/api/v2.0/ping || exit 1
 COPY ./make/photon/core/harbor_core /harbor/
 COPY ./src/core/views /harbor/views
 COPY ./make/migrations /harbor/migrations

make/photon/prepare/Dockerfile.base

@@ -1,6 +1,4 @@
 FROM photon:2.0
 
-RUN tdnf install -y python3 python3-pip httpd
+RUN tdnf install -y python3 python3-pip httpd && tdnf clean all
-RUN pip3 install pipenv==2018.11.26
+RUN pip3 install setuptools && pip3 install pipenv==2018.11.26
-
-

make/photon/prepare/templates/core/env.jinja

@@ -1,7 +1,6 @@
 CONFIG_PATH=/etc/core/app.conf
 UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
 _REDIS_URL={{redis_host}}:{{redis_port}},100,{{redis_password}},0,{{redis_idle_timeout_seconds}}
-SYNC_REGISTRY=false
 SYNC_QUOTA=true
 CHART_CACHE_DRIVER={{chart_cache_driver}}
 _REDIS_URL_REG={{redis_url_reg}}

make/photon/prepare/templates/registry/config.yml.jinja

@@ -31,24 +31,6 @@ auth:
     path: /etc/registry/passwd
 validation:
   disabled: true
-notifications:
-  endpoints:
-  - name: harbor
-    disabled: false
-    url: {{core_url}}/service/notifications
-    timeout: 3000ms
-    threshold: 5
-    backoff: 1s
-    ignoredmediatypes:
-        - application/vnd.docker.image.rootfs.diff.tar.gzip
-        - application/vnd.docker.image.rootfs.foreign.diff.tar.gzip
-        - application/vnd.oci.image.layer.v1.tar
-        - application/vnd.oci.image.layer.v1.tar+gzip
-        - application/vnd.oci.image.layer.v1.tar+zstd
-        - application/vnd.oci.image.layer.nondistributable.v1.tar
-        - application/vnd.oci.image.layer.nondistributable.v1.tar+gzip
-        - application/vnd.oci.image.layer.nondistributable.v1.tar+zstd
-        - application/octet-stream
 compatibility:
   schema1:
     enabled: true

make/photon/redis/Dockerfile.base

@@ -1,3 +1,3 @@
 FROM photon:2.0
 
-RUN tdnf install -y redis sudo
+RUN tdnf install -y redis sudo && tdnf clean all

src/api/artifact/abstractor/abstractor.go

@@ -25,17 +25,9 @@ import (
 	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/artifact"
-	"github.com/goharbor/harbor/src/pkg/repository"
 	"github.com/opencontainers/image-spec/specs-go/v1"
-	"regexp"
-	"strings"
 )
 
-// ArtifactTypeUnknown defines the type for the unknown artifacts
-const ArtifactTypeUnknown = "UNKNOWN"
-
-var artifactTypeRegExp = regexp.MustCompile(`^application/vnd\.[^.]*\.(.*)\.config\.[^.]*\+json$`)
-
 // Abstractor abstracts the specific information for different types of artifacts
 type Abstractor interface {
 	// AbstractMetadata abstracts the metadata for the specific artifact type into the artifact model,
@@ -50,26 +42,18 @@ type Abstractor interface {
 // NewAbstractor returns an instance of the default abstractor
 func NewAbstractor() Abstractor {
 	return &abstractor{
-		repoMgr:     repository.Mgr,
 		blobFetcher: blob.Fcher,
 	}
 }
 
 type abstractor struct {
-	repoMgr     repository.Manager
 	blobFetcher blob.Fetcher
 }
 
-// TODO try CNAB, how to forbid CNAB
-
 // TODO add white list for supported artifact type
 func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Artifact) error {
-	repository, err := a.repoMgr.Get(ctx, artifact.RepositoryID)
-	if err != nil {
-		return err
-	}
 	// read manifest content
-	manifestMediaType, content, err := a.blobFetcher.FetchManifest(repository.Name, artifact.Digest)
+	manifestMediaType, content, err := a.blobFetcher.FetchManifest(artifact.RepositoryName, artifact.Digest)
 	if err != nil {
 		return err
 	}
@@ -101,9 +85,8 @@ func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Ar
 		artifact.Annotations = manifest.Annotations
 	// OCI index/docker manifest list
 	case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList:
-		// the identity of index is still in progress, only handle image index for now
+		// the identity of index is still in progress, we use the manifest mediaType
-		// and use the manifestMediaType as the media type of artifact
+		// as the media type of artifact
-		// If we want to support CNAB, we should get the media type from annotation
 		artifact.MediaType = artifact.ManifestMediaType
 
 		index := &v1.Index{}
@@ -115,7 +98,15 @@ func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Ar
 
 		// set annotations
 		artifact.Annotations = index.Annotations
-		// TODO handle references in resolvers
+
+		// Currently, CNAB put its media type inside the annotations
+		// try to parse the artifact media type from the annotations
+		if artifact.Annotations != nil {
+			mediaType := artifact.Annotations["org.opencontainers.artifactType"]
+			if len(mediaType) > 0 {
+				artifact.MediaType = mediaType
+			}
+		}
 	default:
 		return fmt.Errorf("unsupported manifest media type: %s", artifact.ManifestMediaType)
 	}
@@ -125,8 +116,6 @@ func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Ar
 		return resolver.ResolveMetadata(ctx, content, artifact)
 	}
 
-	// if got no resolver, try to parse the artifact type based on the media type
-	artifact.Type = parseArtifactType(artifact.MediaType)
 	return nil
 }
 
@@ -138,12 +127,3 @@ func (a *abstractor) AbstractAddition(ctx context.Context, artifact *artifact.Ar
 	}
 	return resolver.ResolveAddition(ctx, artifact, addition)
 }
-
-func parseArtifactType(mediaType string) string {
-	strs := artifactTypeRegExp.FindStringSubmatch(mediaType)
-	if len(strs) == 2 {
-		return strings.ToUpper(strs[1])
-	}
-	// can not get the artifact type from the media type, return unknown
-	return ArtifactTypeUnknown
-}

src/api/artifact/abstractor/abstractor_test.go

@@ -18,12 +18,10 @@ import (
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
-	"github.com/goharbor/harbor/src/common/models"
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob"
 	tresolver "github.com/goharbor/harbor/src/testing/api/artifact/abstractor/resolver"
-	"github.com/goharbor/harbor/src/testing/pkg/repository"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/stretchr/testify/suite"
 	"testing"
@@ -203,16 +201,13 @@ type abstractorTestSuite struct {
 	suite.Suite
 	abstractor Abstractor
 	fetcher    *blob.FakeFetcher
-	repoMgr    *repository.FakeManager
 	resolver   *tresolver.FakeResolver
 }
 
 func (a *abstractorTestSuite) SetupTest() {
 	a.fetcher = &blob.FakeFetcher{}
-	a.repoMgr = &repository.FakeManager{}
 	a.resolver = &tresolver.FakeResolver{}
 	a.abstractor = &abstractor{
-		repoMgr:     a.repoMgr,
 		blobFetcher: a.fetcher,
 	}
 }
@@ -220,7 +215,6 @@ func (a *abstractorTestSuite) SetupTest() {
 // docker manifest v1
 func (a *abstractorTestSuite) TestAbstractMetadataOfV1Manifest() {
 	resolver.Register(a.resolver, schema1.MediaTypeSignedManifest)
-	a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
 	a.fetcher.On("FetchManifest").Return(schema1.MediaTypeSignedManifest, []byte(v1Manifest), nil)
 	a.resolver.On("ArtifactType").Return(fakeArtifactType)
 	a.resolver.On("ResolveMetadata").Return(nil)
@@ -238,7 +232,6 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfV1Manifest() {
 // docker manifest v2
 func (a *abstractorTestSuite) TestAbstractMetadataOfV2Manifest() {
 	resolver.Register(a.resolver, schema2.MediaTypeImageConfig)
-	a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
 	a.fetcher.On("FetchManifest").Return(schema2.MediaTypeManifest, []byte(v2Manifest), nil)
 	a.resolver.On("ArtifactType").Return(fakeArtifactType)
 	a.resolver.On("ResolveMetadata").Return(nil)
@@ -257,7 +250,6 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfV2Manifest() {
 // OCI index
 func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() {
 	resolver.Register(a.resolver, v1.MediaTypeImageIndex)
-	a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
 	a.fetcher.On("FetchManifest").Return(v1.MediaTypeImageIndex, []byte(index), nil)
 	a.resolver.On("ArtifactType").Return(fakeArtifactType)
 	a.resolver.On("ResolveMetadata").Return(nil)
@@ -275,7 +267,6 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() {
 
 // OCI index
 func (a *abstractorTestSuite) TestAbstractMetadataOfUnsupported() {
-	a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
 	a.fetcher.On("FetchManifest").Return("unsupported-manifest", []byte{}, nil)
 	artifact := &artifact.Artifact{
 		ID: 1,
@@ -284,28 +275,6 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfUnsupported() {
 	a.Require().NotNil(err)
 }
 
-func (a *abstractorTestSuite) TestParseArtifactType() {
-	mediaType := ""
-	typee := parseArtifactType(mediaType)
-	a.Equal(ArtifactTypeUnknown, typee)
-
-	mediaType = "unknown"
-	typee = parseArtifactType(mediaType)
-	a.Equal(ArtifactTypeUnknown, typee)
-
-	mediaType = "application/vnd.oci.image.config.v1+json"
-	typee = parseArtifactType(mediaType)
-	a.Equal("IMAGE", typee)
-
-	mediaType = "application/vnd.cncf.helm.chart.config.v1+json"
-	typee = parseArtifactType(mediaType)
-	a.Equal("HELM.CHART", typee)
-
-	mediaType = "application/vnd.sylabs.sif.config.v1+json"
-	typee = parseArtifactType(mediaType)
-	a.Equal("SIF", typee)
-}
-
 func (a *abstractorTestSuite) TestAbstractAddition() {
 	resolver.Register(a.resolver, v1.MediaTypeImageConfig)
 	// cannot get the resolver

src/api/artifact/abstractor/blob/fetcher.go

@@ -18,12 +18,9 @@ import (
 	"github.com/docker/distribution/manifest/manifestlist"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
-	"github.com/goharbor/harbor/src/common/utils/registry"
+	"github.com/goharbor/harbor/src/pkg/registry"
-	"github.com/goharbor/harbor/src/common/utils/registry/auth"
-	"github.com/goharbor/harbor/src/core/config"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"io/ioutil"
-	"net/http"
 )
 
 var (
@@ -39,6 +36,8 @@ var (
 	}
 )
 
+// TODO use the registry.Client directly? then the Fetcher can be deleted
+
 // Fetcher fetches the content of blob
 type Fetcher interface {
 	// FetchManifest the content of manifest under the repository
@@ -49,49 +48,34 @@ type Fetcher interface {
 
 // NewFetcher returns an instance of the default blob fetcher
 func NewFetcher() Fetcher {
-	return &fetcher{}
+	return &fetcher{
+		client: registry.Cli,
+	}
 }
 
-type fetcher struct{}
+type fetcher struct {
+	client registry.Client
+}
 
-// TODO re-implement it based on OCI registry driver
 func (f *fetcher) FetchManifest(repository, digest string) (string, []byte, error) {
 	// TODO read from cache first
-	client, err := newRepositoryClient(repository)
+	manifest, _, err := f.client.PullManifest(repository, digest)
+	if err != nil {
+		return "", nil, err
+	}
+	mediaType, payload, err := manifest.Payload()
 	if err != nil {
 		return "", nil, err
 	}
-	_, mediaType, payload, err := client.PullManifest(digest, accept)
 	return mediaType, payload, err
 }
 
-// TODO re-implement it based on OCI registry driver
 func (f *fetcher) FetchLayer(repository, digest string) ([]byte, error) {
 	// TODO read from cache first
-	client, err := newRepositoryClient(repository)
+	_, reader, err := f.client.PullBlob(repository, digest)
-	if err != nil {
-		return nil, err
-	}
-	_, reader, err := client.PullBlob(digest)
 	if err != nil {
 		return nil, err
 	}
 	defer reader.Close()
 	return ioutil.ReadAll(reader)
 }
-
-func newRepositoryClient(repository string) (*registry.Repository, error) {
-	uam := &auth.UserAgentModifier{
-		UserAgent: "harbor-registry-client",
-	}
-	authorizer := auth.DefaultBasicAuthorizer()
-	transport := registry.NewTransport(http.DefaultTransport, authorizer, uam)
-	client := &http.Client{
-		Transport: transport,
-	}
-	endpoint, err := config.RegistryURL()
-	if err != nil {
-		return nil, err
-	}
-	return registry.NewRepository(repository, endpoint, client)
-}

src/api/artifact/abstractor/resolver/chart/chart.go

@@ -24,7 +24,6 @@ import (
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/pkg/chart"
-	"github.com/goharbor/harbor/src/pkg/repository"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -41,7 +40,6 @@ const (
 
 func init() {
 	resolver := &resolver{
-		repoMgr:       repository.Mgr,
 		blobFetcher:   blob.Fcher,
 		chartOperator: chart.Optr,
 	}
@@ -56,22 +54,17 @@ func init() {
 }
 
 type resolver struct {
-	repoMgr       repository.Manager
 	blobFetcher   blob.Fetcher
 	chartOperator chart.Operator
 }
 
 func (r *resolver) ResolveMetadata(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
-	repository, err := r.repoMgr.Get(ctx, artifact.RepositoryID)
-	if err != nil {
-		return err
-	}
 	m := &v1.Manifest{}
 	if err := json.Unmarshal(manifest, m); err != nil {
 		return err
 	}
 	digest := m.Config.Digest.String()
-	layer, err := r.blobFetcher.FetchLayer(repository.Name, digest)
+	layer, err := r.blobFetcher.FetchLayer(artifact.RepositoryName, digest)
 	if err != nil {
 		return err
 	}
@@ -95,11 +88,7 @@ func (r *resolver) ResolveAddition(ctx context.Context, artifact *artifact.Artif
 			WithMessage("addition %s isn't supported for %s", addition, ArtifactTypeChart)
 	}
 
-	repository, err := r.repoMgr.Get(ctx, artifact.RepositoryID)
+	_, content, err := r.blobFetcher.FetchManifest(artifact.RepositoryName, artifact.Digest)
-	if err != nil {
-		return nil, err
-	}
-	_, content, err := r.blobFetcher.FetchManifest(repository.Name, artifact.Digest)
 	if err != nil {
 		return nil, err
 	}
@@ -112,7 +101,7 @@ func (r *resolver) ResolveAddition(ctx context.Context, artifact *artifact.Artif
 		// chart do have two layers, one is config, we should resolve the other one.
 		layerDgst := layer.Digest.String()
 		if layerDgst != manifest.Config.Digest.String() {
-			content, err = r.blobFetcher.FetchLayer(repository.Name, layerDgst)
+			content, err = r.blobFetcher.FetchLayer(artifact.RepositoryName, layerDgst)
 			if err != nil {
 				return nil, err
 			}

src/api/artifact/abstractor/resolver/chart/chart_test.go

@@ -15,13 +15,11 @@
 package chart
 
 import (
-	"github.com/goharbor/harbor/src/common/models"
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	chartserver "github.com/goharbor/harbor/src/pkg/chart"
 	"github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob"
 	"github.com/goharbor/harbor/src/testing/pkg/chart"
-	"github.com/goharbor/harbor/src/testing/pkg/repository"
 	"github.com/stretchr/testify/suite"
 	"k8s.io/helm/pkg/chartutil"
 	"testing"
@@ -30,17 +28,14 @@ import (
 type resolverTestSuite struct {
 	suite.Suite
 	resolver    *resolver
-	repoMgr     *repository.FakeManager
 	blobFetcher *blob.FakeFetcher
 	chartOptr   *chart.FakeOpertaor
 }
 
 func (r *resolverTestSuite) SetupTest() {
-	r.repoMgr = &repository.FakeManager{}
 	r.blobFetcher = &blob.FakeFetcher{}
 	r.chartOptr = &chart.FakeOpertaor{}
 	r.resolver = &resolver{
-		repoMgr:       r.repoMgr,
 		blobFetcher:   r.blobFetcher,
 		chartOperator: r.chartOptr,
 	}
@@ -92,11 +87,9 @@ func (r *resolverTestSuite) TestResolveMetadata() {
   "appVersion": "1.8.2"
 }`
 	artifact := &artifact.Artifact{}
-	r.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
 	r.blobFetcher.On("FetchLayer").Return([]byte(config), nil)
 	err := r.resolver.ResolveMetadata(nil, []byte(content), artifact)
 	r.Require().Nil(err)
-	r.repoMgr.AssertExpectations(r.T())
 	r.blobFetcher.AssertExpectations(r.T())
 	r.Assert().Equal("1.1.2", artifact.ExtraAttrs["version"].(string))
 	r.Assert().Equal("1.8.2", artifact.ExtraAttrs["appVersion"].(string))
@@ -158,7 +151,6 @@ func (r *resolverTestSuite) TestResolveAddition() {
 	}
 
 	artifact := &artifact.Artifact{}
-	r.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
 	r.blobFetcher.On("FetchManifest").Return("", []byte(chartManifest), nil)
 	r.blobFetcher.On("FetchLayer").Return([]byte(chartYaml), nil)
 	r.chartOptr.On("GetDetails").Return(chartDetails, nil)

src/api/artifact/abstractor/resolver/cnab/cnab.go

@@ -0,0 +1,123 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cnab
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor/blob"
+	resolv "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
+	"github.com/goharbor/harbor/src/api/artifact/descriptor"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	ierror "github.com/goharbor/harbor/src/internal/error"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// const definitions
+const (
+	ArtifactTypeCNAB = "CNAB"
+	mediaType        = "application/vnd.cnab.manifest.v1"
+)
+
+func init() {
+	resolver := &resolver{
+		argMgr:      artifact.Mgr,
+		blobFetcher: blob.Fcher,
+	}
+	if err := resolv.Register(resolver, mediaType); err != nil {
+		log.Errorf("failed to register resolver for media type %s: %v", mediaType, err)
+		return
+	}
+	if err := descriptor.Register(resolver, mediaType); err != nil {
+		log.Errorf("failed to register descriptor for media type %s: %v", mediaType, err)
+		return
+	}
+}
+
+type resolver struct {
+	argMgr      artifact.Manager
+	blobFetcher blob.Fetcher
+}
+
+func (r *resolver) ResolveMetadata(ctx context.Context, manifest []byte, art *artifact.Artifact) error {
+	index := &v1.Index{}
+	if err := json.Unmarshal(manifest, index); err != nil {
+		return err
+	}
+	cfgManiDgt := ""
+	// populate the referenced artifacts
+	for _, mani := range index.Manifests {
+		digest := mani.Digest.String()
+		// make sure the child artifact exist
+		ar, err := r.argMgr.GetByDigest(ctx, art.RepositoryName, digest)
+		if err != nil {
+			return err
+		}
+		art.References = append(art.References, &artifact.Reference{
+			ChildID:  ar.ID,
+			Platform: mani.Platform,
+		})
+		// try to get the digest of the manifest that the config layer is referenced by
+		if mani.Annotations != nil &&
+			mani.Annotations["io.cnab.manifest.type"] == "config" {
+			cfgManiDgt = mani.Digest.String()
+		}
+	}
+	if len(cfgManiDgt) == 0 {
+		return nil
+	}
+
+	// resolve the config of CNAB
+	// get the manifest that the config layer is referenced by
+	_, cfgMani, err := r.blobFetcher.FetchManifest(art.RepositoryName, cfgManiDgt)
+	if err != nil {
+		return err
+	}
+	m := &v1.Manifest{}
+	if err := json.Unmarshal(cfgMani, m); err != nil {
+		return err
+	}
+	cfgDgt := m.Config.Digest.String()
+	// get the config layer
+	cfg, err := r.blobFetcher.FetchLayer(art.RepositoryName, cfgDgt)
+	if err != nil {
+		return err
+	}
+	metadata := map[string]interface{}{}
+	if err := json.Unmarshal(cfg, &metadata); err != nil {
+		return err
+	}
+	if art.ExtraAttrs == nil {
+		art.ExtraAttrs = map[string]interface{}{}
+	}
+	for k, v := range metadata {
+		art.ExtraAttrs[k] = v
+	}
+	return nil
+}
+
+func (r *resolver) ResolveAddition(ctx context.Context, artifact *artifact.Artifact, addition string) (*resolv.Addition, error) {
+	return nil, ierror.New(nil).WithCode(ierror.BadRequestCode).
+		WithMessage("addition %s isn't supported for %s", addition, ArtifactTypeCNAB)
+}
+
+func (r *resolver) GetArtifactType() string {
+	return ArtifactTypeCNAB
+}
+
+func (r *resolver) ListAdditionTypes() []string {
+	return nil
+}

src/api/artifact/abstractor/resolver/cnab/cnab_test.go

@@ -0,0 +1,138 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cnab
+
+import (
+	ierror "github.com/goharbor/harbor/src/internal/error"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	"github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob"
+	testingartifact "github.com/goharbor/harbor/src/testing/pkg/artifact"
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+type resolverTestSuite struct {
+	suite.Suite
+	resolver    *resolver
+	artMgr      *testingartifact.FakeManager
+	blobFetcher *blob.FakeFetcher
+}
+
+func (r *resolverTestSuite) SetupTest() {
+	r.artMgr = &testingartifact.FakeManager{}
+	r.blobFetcher = &blob.FakeFetcher{}
+	r.resolver = &resolver{
+		argMgr:      r.artMgr,
+		blobFetcher: r.blobFetcher,
+	}
+
+}
+
+func (r *resolverTestSuite) TestResolveMetadata() {
+	index := `{
+  "schemaVersion": 2,
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "digest": "sha256:b9616da7500f8c7c9a5e8d915714cd02d11bcc71ff5b4fd190bb77b1355c8549",
+      "size": 193,
+      "annotations": {
+        "io.cnab.manifest.type": "config"
+      }
+    },
+    {
+      "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+      "digest": "sha256:a59a4e74d9cc89e4e75dfb2cc7ea5c108e4236ba6231b53081a9e2506d1197b6",
+      "size": 942,
+      "annotations": {
+        "io.cnab.manifest.type": "invocation"
+      }
+    }
+  ],
+  "annotations": {
+    "io.cnab.keywords": "[\"helloworld\",\"cnab\",\"tutorial\"]",
+    "io.cnab.runtime_version": "v1.0.0",
+    "org.opencontainers.artifactType": "application/vnd.cnab.manifest.v1",
+    "org.opencontainers.image.authors": "[{\"name\":\"Jane Doe\",\"email\":\"jane.doe@example.com\",\"url\":\"https://example.com\"}]",
+    "org.opencontainers.image.description": "A short description of your bundle",
+    "org.opencontainers.image.title": "helloworld",
+    "org.opencontainers.image.version": "0.1.1"
+  }
+}`
+
+	manifest := `{
+  "schemaVersion": 2,
+  "config": {
+    "mediaType": "application/vnd.oci.image.config.v1+json",
+    "digest": "sha256:e91b9dfcbbb3b88bac94726f276b89de46e4460b55f6e6d6f876e666b150ec5b",
+    "size": 498
+  },
+  "layers": null
+}`
+	config := `{
+  "description": "A short description of your bundle",
+  "invocationImages": [
+    {
+      "contentDigest": "sha256:a59a4e74d9cc89e4e75dfb2cc7ea5c108e4236ba6231b53081a9e2506d1197b6",
+      "image": "cnab/helloworld:0.1.1",
+      "imageType": "docker",
+      "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+      "size": 942
+    }
+  ],
+  "keywords": [
+    "helloworld",
+    "cnab",
+    "tutorial"
+  ],
+  "maintainers": [
+    {
+      "email": "jane.doe@example.com",
+      "name": "Jane Doe",
+      "url": "https://example.com"
+    }
+  ],
+  "name": "helloworld",
+  "schemaVersion": "v1.0.0",
+  "version": "0.1.1"
+}`
+	art := &artifact.Artifact{}
+	r.artMgr.On("GetByDigest").Return(&artifact.Artifact{ID: 1}, nil)
+	r.blobFetcher.On("FetchManifest").Return("", []byte(manifest), nil)
+	r.blobFetcher.On("FetchLayer").Return([]byte(config), nil)
+	err := r.resolver.ResolveMetadata(nil, []byte(index), art)
+	r.Require().Nil(err)
+	r.Len(art.References, 2)
+	r.Equal("0.1.1", art.ExtraAttrs["version"].(string))
+	r.Equal("helloworld", art.ExtraAttrs["name"].(string))
+}
+
+func (r *resolverTestSuite) TestResolveAddition() {
+	_, err := r.resolver.ResolveAddition(nil, nil, "")
+	r.Require().NotNil(err)
+	r.True(ierror.IsErr(err, ierror.BadRequestCode))
+}
+
+func (r *resolverTestSuite) TestGetArtifactType() {
+	r.Assert().Equal(ArtifactTypeCNAB, r.resolver.GetArtifactType())
+}
+
+func (r *resolverTestSuite) TestListAdditionTypes() {
+	r.Nil(r.resolver.ListAdditionTypes())
+}
+
+func TestResolverTestSuite(t *testing.T) {
+	suite.Run(t, &resolverTestSuite{})
+}

src/api/artifact/abstractor/resolver/image/index.go

@@ -58,7 +58,7 @@ func (i *indexResolver) ResolveMetadata(ctx context.Context, manifest []byte, ar
 	for _, mani := range index.Manifests {
 		digest := mani.Digest.String()
 		// make sure the child artifact exist
-		ar, err := i.artMgr.GetByDigest(ctx, art.RepositoryID, digest)
+		ar, err := i.artMgr.GetByDigest(ctx, art.RepositoryName, digest)
 		if err != nil {
 			return err
 		}

src/api/artifact/abstractor/resolver/image/manifest_v2.go

@@ -24,7 +24,6 @@ import (
 	"github.com/goharbor/harbor/src/common/utils/log"
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/artifact"
-	"github.com/goharbor/harbor/src/pkg/repository"
 	"github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -37,7 +36,6 @@ const (
 
 func init() {
 	rslver := &manifestV2Resolver{
-		repoMgr:     repository.Mgr,
 		blobFetcher: blob.Fcher,
 	}
 	mediaTypes := []string{
@@ -56,21 +54,16 @@ func init() {
 
 // manifestV2Resolver resolve artifact with OCI manifest and docker v2 manifest
 type manifestV2Resolver struct {
-	repoMgr     repository.Manager
 	blobFetcher blob.Fetcher
 }
 
 func (m *manifestV2Resolver) ResolveMetadata(ctx context.Context, content []byte, artifact *artifact.Artifact) error {
-	repository, err := m.repoMgr.Get(ctx, artifact.RepositoryID)
-	if err != nil {
-		return err
-	}
 	manifest := &v1.Manifest{}
 	if err := json.Unmarshal(content, manifest); err != nil {
 		return err
 	}
 	digest := manifest.Config.Digest.String()
-	layer, err := m.blobFetcher.FetchLayer(repository.Name, digest)
+	layer, err := m.blobFetcher.FetchLayer(artifact.RepositoryName, digest)
 	if err != nil {
 		return err
 	}
@@ -93,11 +86,7 @@ func (m *manifestV2Resolver) ResolveAddition(ctx context.Context, artifact *arti
 		return nil, ierror.New(nil).WithCode(ierror.BadRequestCode).
 			WithMessage("addition %s isn't supported for %s(manifest version 2)", addition, ArtifactTypeImage)
 	}
-	repository, err := m.repoMgr.Get(ctx, artifact.RepositoryID)
+	_, content, err := m.blobFetcher.FetchManifest(artifact.RepositoryName, artifact.Digest)
-	if err != nil {
-		return nil, err
-	}
-	_, content, err := m.blobFetcher.FetchManifest(repository.Name, artifact.Digest)
 	if err != nil {
 		return nil, err
 	}
@@ -105,7 +94,7 @@ func (m *manifestV2Resolver) ResolveAddition(ctx context.Context, artifact *arti
 	if err := json.Unmarshal(content, manifest); err != nil {
 		return nil, err
 	}
-	content, err = m.blobFetcher.FetchLayer(repository.Name, manifest.Config.Digest.String())
+	content, err = m.blobFetcher.FetchLayer(artifact.RepositoryName, manifest.Config.Digest.String())
 	if err != nil {
 		return nil, err
 	}

src/api/artifact/abstractor/resolver/image/manifest_v2_test.go

@@ -15,11 +15,9 @@
 package image
 
 import (
-	"github.com/goharbor/harbor/src/common/models"
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob"
-	"github.com/goharbor/harbor/src/testing/pkg/repository"
 	"github.com/stretchr/testify/suite"
 	"testing"
 )
@@ -123,15 +121,12 @@ var (
 type manifestV2ResolverTestSuite struct {
 	suite.Suite
 	resolver    *manifestV2Resolver
-	repoMgr     *repository.FakeManager
 	blobFetcher *blob.FakeFetcher
 }
 
 func (m *manifestV2ResolverTestSuite) SetupTest() {
-	m.repoMgr = &repository.FakeManager{}
 	m.blobFetcher = &blob.FakeFetcher{}
 	m.resolver = &manifestV2Resolver{
-		repoMgr:     m.repoMgr,
 		blobFetcher: m.blobFetcher,
 	}
 
@@ -139,11 +134,9 @@ func (m *manifestV2ResolverTestSuite) SetupTest() {
 
 func (m *manifestV2ResolverTestSuite) TestResolveMetadata() {
 	artifact := &artifact.Artifact{}
-	m.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
 	m.blobFetcher.On("FetchLayer").Return([]byte(config), nil)
 	err := m.resolver.ResolveMetadata(nil, []byte(manifest), artifact)
 	m.Require().Nil(err)
-	m.repoMgr.AssertExpectations(m.T())
 	m.blobFetcher.AssertExpectations(m.T())
 	m.Assert().Equal("amd64", artifact.ExtraAttrs["architecture"].(string))
 	m.Assert().Equal("linux", artifact.ExtraAttrs["os"].(string))
@@ -156,7 +149,6 @@ func (m *manifestV2ResolverTestSuite) TestResolveAddition() {
 
 	// build history
 	artifact := &artifact.Artifact{}
-	m.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
 	m.blobFetcher.On("FetchManifest").Return("", []byte(manifest), nil)
 	m.blobFetcher.On("FetchLayer").Return([]byte(config), nil)
 	addition, err := m.resolver.ResolveAddition(nil, artifact, AdditionTypeBuildHistory)

src/api/artifact/controller.go

@@ -17,13 +17,9 @@ package artifact
 import (
 	"context"
 	"fmt"
-	"strings"
-	"time"
-
 	"github.com/goharbor/harbor/src/api/artifact/abstractor"
 	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
 	"github.com/goharbor/harbor/src/api/artifact/descriptor"
-	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/internal"
 	"github.com/goharbor/harbor/src/pkg/art"
@@ -36,11 +32,14 @@ import (
 	"github.com/goharbor/harbor/src/pkg/registry"
 	"github.com/goharbor/harbor/src/pkg/signature"
 	"github.com/opencontainers/go-digest"
-
+	"strings"
+	"time"
 	// registry image resolvers
 	_ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/image"
 	// register chart resolver
 	_ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/chart"
+	// register CNAB resolver
+	_ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/cnab"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/artifact"
@@ -61,7 +60,7 @@ type Controller interface {
 	// creates it if it doesn't exist. If tags are provided, ensure they exist
 	// and are attached to the artifact. If the tags don't exist, create them first.
 	// The "created" will be set as true when the artifact is created
-	Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (created bool, id int64, err error)
+	Ensure(ctx context.Context, repository, digest string, tags ...string) (created bool, id int64, err error)
 	// Count returns the total count of artifacts according to the query.
 	// The artifacts that referenced by others and without tags are not counted
 	Count(ctx context.Context, query *q.Query) (total int64, err error)
@@ -75,8 +74,8 @@ type Controller interface {
 	GetByReference(ctx context.Context, repository, reference string, option *Option) (artifact *Artifact, err error)
 	// Delete the artifact specified by ID. All tags attached to the artifact are deleted as well
 	Delete(ctx context.Context, id int64) (err error)
-	// Copy the artifact whose ID is specified by "srcArtID" into the repository specified by "dstRepoID"
+	// Copy the artifact specified by "srcRepo" and "reference" into the repository specified by "dstRepo"
-	Copy(ctx context.Context, srcArtID, dstRepoID int64) (id int64, err error)
+	Copy(ctx context.Context, srcRepo, reference, dstRepo string) (id int64, err error)
 	// ListTags lists the tags according to the query, specify the properties returned with option
 	ListTags(ctx context.Context, query *q.Query, option *TagOption) (tags []*Tag, err error)
 	// CreateTag creates a tag
@@ -127,69 +126,67 @@ type controller struct {
 	regCli       registry.Client
 }
 
-func (c *controller) Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (bool, int64, error) {
+func (c *controller) Ensure(ctx context.Context, repository, digest string, tags ...string) (bool, int64, error) {
-	created, id, err := c.ensureArtifact(ctx, repositoryID, digest)
+	created, artifact, err := c.ensureArtifact(ctx, repository, digest)
 	if err != nil {
 		return false, 0, err
 	}
 	for _, tag := range tags {
-		if err = c.ensureTag(ctx, repositoryID, id, tag); err != nil {
+		if err = c.ensureTag(ctx, artifact.RepositoryID, artifact.ID, tag); err != nil {
 			return false, 0, err
 		}
 	}
-	return created, id, nil
+	return created, artifact.ID, nil
 }
 
 // ensure the artifact exists under the repository, create it if doesn't exist.
-func (c *controller) ensureArtifact(ctx context.Context, repositoryID int64, digest string) (bool, int64, error) {
+func (c *controller) ensureArtifact(ctx context.Context, repository, digest string) (bool, *artifact.Artifact, error) {
-	art, err := c.artMgr.GetByDigest(ctx, repositoryID, digest)
+	art, err := c.artMgr.GetByDigest(ctx, repository, digest)
 	// the artifact already exists under the repository, return directly
 	if err == nil {
-		return false, art.ID, nil
+		return false, art, nil
 	}
 
 	// got other error when get the artifact, return the error
 	if !ierror.IsErr(err, ierror.NotFoundCode) {
-		return false, 0, err
+		return false, nil, err
 	}
 
 	// the artifact doesn't exist under the repository, create it first
-	repository, err := c.repoMgr.Get(ctx, repositoryID)
+	repo, err := c.repoMgr.GetByName(ctx, repository)
 	if err != nil {
-		return false, 0, err
+		return false, nil, err
 	}
 	artifact := &artifact.Artifact{
-		ProjectID:    repository.ProjectID,
+		ProjectID:      repo.ProjectID,
-		RepositoryID: repositoryID,
+		RepositoryID:   repo.RepositoryID,
-		Digest:       digest,
+		RepositoryName: repository,
-		PushTime:     time.Now(),
+		Digest:         digest,
+		PushTime:       time.Now(),
 	}
 	// abstract the metadata for the artifact
 	if err = c.abstractor.AbstractMetadata(ctx, artifact); err != nil {
-		return false, 0, err
+		return false, nil, err
 	}
 
 	// populate the artifact type
-	typee, err := descriptor.GetArtifactType(artifact.MediaType)
+	artifact.Type = descriptor.GetArtifactType(artifact.MediaType)
-	if err != nil {
-		return false, 0, err
-	}
-	artifact.Type = typee
 
 	// create it
 	id, err := c.artMgr.Create(ctx, artifact)
 	if err != nil {
 		// if got conflict error, try to get the artifact again
 		if ierror.IsConflictErr(err) {
-			art, err = c.artMgr.GetByDigest(ctx, repositoryID, digest)
+			art, err = c.artMgr.GetByDigest(ctx, repository, digest)
 			if err == nil {
-				return false, art.ID, nil
+				return false, art, nil
 			}
-			return false, 0, err
+			return false, nil, err
 		}
-		return false, 0, err
+		return false, nil, err
 	}
-	return true, id, nil
+	artifact.ID = id
+	return true, artifact, nil
 }
 
 func (c *controller) ensureTag(ctx context.Context, repositoryID, artifactID int64, name string) error {
@@ -240,10 +237,6 @@ func (c *controller) List(ctx context.Context, query *q.Query, option *Option) (
 		return nil, err
 	}
 
-	if err := c.populateRepositoryName(ctx, arts...); err != nil {
-		return nil, err
-	}
-
 	var artifacts []*Artifact
 	for _, art := range arts {
 		artifacts = append(artifacts, c.assembleArtifact(ctx, art, option))
@@ -269,11 +262,7 @@ func (c *controller) GetByReference(ctx context.Context, repository, reference s
 }
 
 func (c *controller) getByDigest(ctx context.Context, repository, digest string, option *Option) (*Artifact, error) {
-	repo, err := c.repoMgr.GetByName(ctx, repository)
+	art, err := c.artMgr.GetByDigest(ctx, repository, digest)
-	if err != nil {
-		return nil, err
-	}
-	art, err := c.artMgr.GetByDigest(ctx, repo.RepositoryID, digest)
 	if err != nil {
 		return nil, err
 	}
@@ -381,14 +370,10 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot bool) er
 		return err
 	}
 
-	repo, err := c.repoMgr.Get(ctx, art.RepositoryID)
-	if err != nil && !ierror.IsErr(err, ierror.NotFoundCode) {
-		return err
-	}
 	_, err = c.artrashMgr.Create(ctx, &model.ArtifactTrash{
 		MediaType:         art.MediaType,
 		ManifestMediaType: art.ManifestMediaType,
-		RepositoryName:    repo.Name,
+		RepositoryName:    art.RepositoryName,
 		Digest:            art.Digest,
 	})
 	if err != nil && !ierror.IsErr(err, ierror.ConflictCode) {
@@ -399,61 +384,63 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot bool) er
 	return nil
 }
 
-func (c *controller) Copy(ctx context.Context, srcArtID, dstRepoID int64) (int64, error) {
+func (c *controller) Copy(ctx context.Context, srcRepo, reference, dstRepo string) (int64, error) {
-	srcArt, err := c.Get(ctx, srcArtID, &Option{WithTag: true})
+	return c.copyDeeply(ctx, srcRepo, reference, dstRepo, true)
-	if err != nil {
+}
-		return 0, err
+
+// as we call the docker registry APIs in the registry client directly,
+// this bypass our own logic(ensure, fire event, etc.) inside the registry handlers,
+// these logic must be covered explicitly here.
+// "copyDeeply" iterates the child artifacts and copy them first
+func (c *controller) copyDeeply(ctx context.Context, srcRepo, reference, dstRepo string, isRoot bool) (int64, error) {
+	var option *Option
+	// only get the tags of the root parent
+	if isRoot {
+		option = &Option{WithTag: true}
 	}
-	srcRepo, err := c.repoMgr.Get(ctx, srcArt.RepositoryID)
+
-	if err != nil {
+	srcArt, err := c.GetByReference(ctx, srcRepo, reference, option)
-		return 0, err
-	}
-	dstRepo, err := c.repoMgr.Get(ctx, dstRepoID)
 	if err != nil {
 		return 0, err
 	}
 
-	_, err = c.artMgr.GetByDigest(ctx, dstRepoID, srcArt.Digest)
+	digest := srcArt.Digest
-	// the artifact already exists in the destination repository
+
+	// check the existence of artifact in the destination repository
+	dstArt, err := c.GetByReference(ctx, dstRepo, digest, option)
 	if err == nil {
-		return 0, ierror.New(nil).WithCode(ierror.ConflictCode).
+		// return conflict error if the root parent artifact already exists under the destination repository
-			WithMessage("the artifact %s already exists under the repository %s",
+		if isRoot {
-				srcArt.Digest, dstRepo.Name)
+			return 0, ierror.New(nil).WithCode(ierror.ConflictCode).
+				WithMessage("the artifact %s@%s already exists", dstRepo, digest)
+		}
+		// the child artifact already under the destination repository, skip
+		return dstArt.ID, nil
 	}
 	if !ierror.IsErr(err, ierror.NotFoundCode) {
 		return 0, err
 	}
 
+	// the artifact doesn't exist under the destination repository, continue to copy
+	// copy child artifacts if contains any
+	for _, reference := range srcArt.References {
+		if _, err = c.copyDeeply(ctx, srcRepo, reference.ChildDigest, dstRepo, false); err != nil {
+			return 0, err
+		}
+	}
+
+	// copy the parent artifact into the backend docker registry
+	if err := c.regCli.Copy(srcRepo, digest, dstRepo, digest, false); err != nil {
+		return 0, err
+	}
+
 	// only copy the tags of outermost artifact
 	var tags []string
 	for _, tag := range srcArt.Tags {
 		tags = append(tags, tag.Name)
 	}
-	return c.copyDeeply(ctx, srcRepo, srcArt, dstRepo, tags...)
+	// ensure the parent artifact exist in the database
-}
+	_, id, err := c.Ensure(ctx, dstRepo, digest, tags...)
-
-// as we call the docker registry APIs in the registry client directly,
-// this bypass our own logic(ensure, fire event, etc.) inside the registry handlers,
-// these logic must be covered explicitly here.
-// "copyDeeply" iterates the child artifacts and copy them first
-func (c *controller) copyDeeply(ctx context.Context, srcRepo *models.RepoRecord, srcArt *Artifact,
-	dstRepo *models.RepoRecord, tags ...string) (int64, error) {
-	// copy child artifacts if contains any
-	for _, reference := range srcArt.References {
-		childArt, err := c.Get(ctx, reference.ChildID, nil)
-		if err != nil {
-			return 0, err
-		}
-		if _, err = c.copyDeeply(ctx, srcRepo, childArt, dstRepo); err != nil {
-			return 0, err
-		}
-	}
-	// copy the parent artifact
-	if err := c.regCli.Copy(srcRepo.Name, srcArt.Digest,
-		dstRepo.Name, srcArt.Digest, false); err != nil {
-		return 0, err
-	}
-	_, id, err := c.Ensure(ctx, dstRepo.RepositoryID, srcArt.Digest, tags...)
 	if err != nil {
 		return 0, err
 	}
@@ -472,7 +459,11 @@ func (c *controller) ListTags(ctx context.Context, query *q.Query, option *TagOp
 	}
 	var tags []*Tag
 	for _, tg := range tgs {
-		tags = append(tags, c.assembleTag(ctx, tg, option))
+		art, err := c.artMgr.Get(ctx, tg.ArtifactID)
+		if err != nil {
+			return nil, err
+		}
+		tags = append(tags, c.assembleTag(ctx, art, tg, option))
 	}
 	return tags, nil
 }
@@ -521,19 +512,9 @@ func (c *controller) assembleArtifact(ctx context.Context, art *artifact.Artifac
 	artifact := &Artifact{
 		Artifact: *art,
 	}
-
-	if artifact.RepositoryName == "" {
-		repo, err := c.repoMgr.Get(ctx, artifact.RepositoryID)
-		if err != nil {
-			log.Errorf("get repository %d failed, error: %v", artifact.RepositoryID, err)
-			return artifact
-		}
-
-		artifact.RepositoryName = repo.Name
-	}
-
 	// populate addition links
 	c.populateAdditionLinks(ctx, artifact)
+
 	if option == nil {
 		return artifact
 	}
@@ -543,39 +524,9 @@ func (c *controller) assembleArtifact(ctx context.Context, art *artifact.Artifac
 	if option.WithLabel {
 		c.populateLabels(ctx, artifact)
 	}
-	// populate addition links
-	c.populateAdditionLinks(ctx, artifact)
 	return artifact
 }
 
-func (c *controller) populateRepositoryName(ctx context.Context, artifacts ...*artifact.Artifact) error {
-	var ids []int64
-	for _, artifact := range artifacts {
-		ids = append(ids, artifact.RepositoryID)
-	}
-
-	repositories, err := c.repoMgr.List(ctx, &q.Query{Keywords: map[string]interface{}{"repository_id__in": ids}})
-	if err != nil {
-		return err
-	}
-
-	mp := make(map[int64]string, len(repositories))
-	for _, repository := range repositories {
-		mp[repository.RepositoryID] = repository.Name
-	}
-
-	for _, artifact := range artifacts {
-		repositoryName, ok := mp[artifact.RepositoryID]
-		if !ok {
-			return ierror.NotFoundError(nil).WithMessage("repository %d not found", artifact.RepositoryID)
-		}
-
-		artifact.RepositoryName = repositoryName
-	}
-
-	return nil
-}
-
 func (c *controller) populateTags(ctx context.Context, art *Artifact, option *TagOption) {
 	tags, err := c.tagMgr.List(ctx, &q.Query{
 		Keywords: map[string]interface{}{
@@ -587,46 +538,51 @@ func (c *controller) populateTags(ctx context.Context, art *Artifact, option *Ta
 		return
 	}
 	for _, tag := range tags {
-		art.Tags = append(art.Tags, c.assembleTag(ctx, tag, option))
+		art.Tags = append(art.Tags, c.assembleTag(ctx, &art.Artifact, tag, option))
 	}
 }
 
 // assemble several part into a single tag
-func (c *controller) assembleTag(ctx context.Context, tag *tm.Tag, option *TagOption) *Tag {
+func (c *controller) assembleTag(ctx context.Context, art *artifact.Artifact, tag *tm.Tag, option *TagOption) *Tag {
 	t := &Tag{
 		Tag: *tag,
 	}
 	if option == nil {
 		return t
 	}
-	repo, err := c.repoMgr.Get(ctx, tag.RepositoryID)
-	if err != nil {
-		log.Errorf("Failed to get repo for tag: %s, error: %v", tag.Name, err)
-		return t
-	}
 	if option.WithImmutableStatus {
-		c.populateImmutableStatus(ctx, t)
+		c.populateImmutableStatus(ctx, art, t)
 	}
 	if option.WithSignature {
-		if a, err := c.artMgr.Get(ctx, t.ArtifactID); err != nil {
+		c.populateTagSignature(ctx, art, t, option)
-			log.Errorf("Failed to get artifact for tag: %s, error: %v, skip populating signature", t.Name, err)
-		} else {
-			c.populateTagSignature(ctx, repo.Name, t, a.Digest, option)
-		}
 	}
 	return t
 }
 
-func (c *controller) populateTagSignature(ctx context.Context, repo string, tag *Tag, digest string, option *TagOption) {
+func (c *controller) populateImmutableStatus(ctx context.Context, artifact *artifact.Artifact, tag *Tag) {
+	_, repoName := utils.ParseRepository(artifact.RepositoryName)
+	matched, err := c.immutableMtr.Match(artifact.ProjectID, art.Candidate{
+		Repository:  repoName,
+		Tags:        []string{tag.Name},
+		NamespaceID: artifact.ProjectID,
+	})
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	tag.Immutable = matched
+}
+
+func (c *controller) populateTagSignature(ctx context.Context, artifact *artifact.Artifact, tag *Tag, option *TagOption) {
 	if option.SignatureChecker == nil {
-		chk, err := signature.GetManager().GetCheckerByRepo(ctx, repo)
+		chk, err := signature.GetManager().GetCheckerByRepo(ctx, artifact.RepositoryName)
 		if err != nil {
 			log.Error(err)
 			return
 		}
 		option.SignatureChecker = chk
 	}
-	tag.Signed = option.SignatureChecker.IsTagSigned(tag.Name, digest)
+	tag.Signed = option.SignatureChecker.IsTagSigned(tag.Name, artifact.Digest)
 }
 
 func (c *controller) populateLabels(ctx context.Context, art *Artifact) {
@@ -638,32 +594,8 @@ func (c *controller) populateLabels(ctx context.Context, art *Artifact) {
 	art.Labels = labels
 }
 
-func (c *controller) populateImmutableStatus(ctx context.Context, tag *Tag) {
-	repo, err := c.repoMgr.Get(ctx, tag.RepositoryID)
-	if err != nil {
-		log.Error(err)
-		return
-	}
-	_, repoName := utils.ParseRepository(repo.Name)
-	matched, err := c.immutableMtr.Match(repo.ProjectID, art.Candidate{
-		Repository:  repoName,
-		Tags:        []string{tag.Name},
-		NamespaceID: repo.ProjectID,
-	})
-	if err != nil {
-		log.Error(err)
-		return
-	}
-	tag.Immutable = matched
-}
-
 func (c *controller) populateAdditionLinks(ctx context.Context, artifact *Artifact) {
-	types, err := descriptor.ListAdditionTypes(artifact.MediaType)
+	types := descriptor.ListAdditionTypes(artifact.MediaType)
-	if err != nil {
-		log.Error(err.Error())
-		return
-	}
-
 	if len(types) > 0 {
 		version := internal.GetAPIVersion(ctx)
 		for _, t := range types {

src/api/artifact/controller_test.go

@@ -39,6 +39,8 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
+// TODO find another way to test artifact controller, it's hard to maintain currently
+
 type fakeAbstractor struct {
 	mock.Mock
 }
@@ -107,6 +109,13 @@ func (c *controllerTestSuite) SetupTest() {
 }
 
 func (c *controllerTestSuite) TestAssembleTag() {
+	art := &artifact.Artifact{
+		ID:             1,
+		ProjectID:      1,
+		RepositoryID:   1,
+		RepositoryName: "library/hello-world",
+		Digest:         "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
+	}
 	tg := &tag.Tag{
 		ID:           1,
 		RepositoryID: 1,
@@ -119,13 +128,8 @@ func (c *controllerTestSuite) TestAssembleTag() {
 		WithImmutableStatus: true,
 	}
 
-	c.repoMgr.On("Get").Return(&models.RepoRecord{
-		ProjectID: 1,
-		Name:      "hello-world",
-	}, nil)
-
 	c.immutableMtr.On("Match").Return(true, nil)
-	tag := c.ctl.assembleTag(nil, tg, option)
+	tag := c.ctl.assembleTag(nil, art, tg, option)
 	c.Require().NotNil(tag)
 	c.Equal(tag.ID, tg.ID)
 	c.Equal(true, tag.Immutable)
@@ -154,9 +158,6 @@ func (c *controllerTestSuite) TestAssembleArtifact() {
 		PullTime:     time.Now(),
 	}
 	c.tagMgr.On("List").Return([]*tag.Tag{tg}, nil)
-	c.repoMgr.On("Get").Return(&models.RepoRecord{
-		Name: "library/hello-world",
-	}, nil)
 	ctx := internal.SetAPIVersion(nil, "2.0")
 	lb := &models.Label{
 		ID:   1,
@@ -185,25 +186,25 @@ func (c *controllerTestSuite) TestEnsureArtifact() {
 	c.artMgr.On("GetByDigest").Return(&artifact.Artifact{
 		ID: 1,
 	}, nil)
-	created, id, err := c.ctl.ensureArtifact(nil, 1, digest)
+	created, art, err := c.ctl.ensureArtifact(nil, "library/hello-world", digest)
 	c.Require().Nil(err)
 	c.False(created)
-	c.Equal(int64(1), id)
+	c.Equal(int64(1), art.ID)
 
 	// reset the mock
 	c.SetupTest()
 
 	// the artifact doesn't exist
-	c.repoMgr.On("Get").Return(&models.RepoRecord{
+	c.repoMgr.On("GetByName").Return(&models.RepoRecord{
 		ProjectID: 1,
 	}, nil)
 	c.artMgr.On("GetByDigest").Return(nil, ierror.NotFoundError(nil))
 	c.artMgr.On("Create").Return(1, nil)
 	c.abstractor.On("AbstractMetadata").Return(nil)
-	created, id, err = c.ctl.ensureArtifact(nil, 1, digest)
+	created, art, err = c.ctl.ensureArtifact(nil, "library/hello-world", digest)
 	c.Require().Nil(err)
 	c.True(created)
-	c.Equal(int64(1), id)
+	c.Equal(int64(1), art.ID)
 }
 
 func (c *controllerTestSuite) TestEnsureTag() {
@@ -252,7 +253,7 @@ func (c *controllerTestSuite) TestEnsure() {
 	digest := "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180"
 
 	// both the artifact and the tag don't exist
-	c.repoMgr.On("Get").Return(&models.RepoRecord{
+	c.repoMgr.On("GetByName").Return(&models.RepoRecord{
 		ProjectID: 1,
 	}, nil)
 	c.artMgr.On("GetByDigest").Return(nil, ierror.NotFoundError(nil))
@@ -260,7 +261,7 @@ func (c *controllerTestSuite) TestEnsure() {
 	c.tagMgr.On("List").Return([]*tag.Tag{}, nil)
 	c.tagMgr.On("Create").Return(1, nil)
 	c.abstractor.On("AbstractMetadata").Return(nil)
-	_, id, err := c.ctl.Ensure(nil, 1, digest, "latest")
+	_, id, err := c.ctl.Ensure(nil, "library/hello-world", digest, "latest")
 	c.Require().Nil(err)
 	c.repoMgr.AssertExpectations(c.T())
 	c.artMgr.AssertExpectations(c.T())
@@ -508,7 +509,12 @@ func (c *controllerTestSuite) TestDeleteDeeply() {
 
 func (c *controllerTestSuite) TestCopy() {
 	c.artMgr.On("Get").Return(&artifact.Artifact{
-		ID: 1,
+		ID:     1,
+		Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
+	}, nil)
+	c.repoMgr.On("GetByName").Return(&models.RepoRecord{
+		RepositoryID: 1,
+		Name:         "library/hello-world",
 	}, nil)
 	c.artMgr.On("GetByDigest").Return(nil, ierror.NotFoundError(nil))
 	c.tagMgr.On("List").Return([]*tag.Tag{
@@ -525,7 +531,7 @@ func (c *controllerTestSuite) TestCopy() {
 	c.abstractor.On("AbstractMetadata").Return(nil)
 	c.artMgr.On("Create").Return(1, nil)
 	c.regCli.On("Copy").Return(nil)
-	_, err := c.ctl.Copy(nil, 1, 1)
+	_, err := c.ctl.Copy(nil, "library/hello-world", "latest", "library/hello-world2")
 	c.Require().Nil(err)
 }
 
@@ -538,6 +544,7 @@ func (c *controllerTestSuite) TestListTags() {
 			ArtifactID:   1,
 		},
 	}, nil)
+	c.artMgr.On("Get").Return(&artifact.Artifact{}, nil)
 	tags, err := c.ctl.ListTags(nil, nil, nil)
 	c.Require().Nil(err)
 	c.Len(tags, 1)

src/api/artifact/descriptor/descriptor.go

@@ -17,10 +17,16 @@ package descriptor
 import (
 	"fmt"
 	"github.com/goharbor/harbor/src/common/utils/log"
+	"regexp"
+	"strings"
 )
 
+// ArtifactTypeUnknown defines the type for the unknown artifacts
+const ArtifactTypeUnknown = "UNKNOWN"
+
 var (
-	registry = map[string]Descriptor{}
+	registry           = map[string]Descriptor{}
+	artifactTypeRegExp = regexp.MustCompile(`^application/vnd\.[^.]*\.(.*)\.config\.[^.]*\+json$`)
 )
 
 // Descriptor describes the static information for one kind of media type
@@ -45,28 +51,34 @@ func Register(descriptor Descriptor, mediaTypes ...string) error {
 }
 
 // Get the descriptor according to the media type
-func Get(mediaType string) (Descriptor, error) {
+func Get(mediaType string) Descriptor {
-	descriptor := registry[mediaType]
+	return registry[mediaType]
-	if descriptor == nil {
-		return nil, fmt.Errorf("descriptor for media type %s not found", mediaType)
-	}
-	return descriptor, nil
 }
 
 // GetArtifactType gets the artifact type according to the media type
-func GetArtifactType(mediaType string) (string, error) {
+func GetArtifactType(mediaType string) string {
-	descriptor, err := Get(mediaType)
+	descriptor := Get(mediaType)
-	if err != nil {
+	if descriptor != nil {
-		return "", err
+		return descriptor.GetArtifactType()
 	}
-	return descriptor.GetArtifactType(), nil
+	// if got no descriptor, try to parse the artifact type based on the media type
+	return parseArtifactType(mediaType)
 }
 
 // ListAdditionTypes lists the supported addition types according to the media type
-func ListAdditionTypes(mediaType string) ([]string, error) {
+func ListAdditionTypes(mediaType string) []string {
-	descriptor, err := Get(mediaType)
+	descriptor := Get(mediaType)
-	if err != nil {
+	if descriptor != nil {
-		return nil, err
+		return descriptor.ListAdditionTypes()
 	}
-	return descriptor.ListAdditionTypes(), nil
+	return nil
+}
+
+func parseArtifactType(mediaType string) string {
+	strs := artifactTypeRegExp.FindStringSubmatch(mediaType)
+	if len(strs) == 2 {
+		return strings.ToUpper(strs[1])
+	}
+	// can not get the artifact type from the media type, return unknown
+	return ArtifactTypeUnknown
 }

src/api/artifact/descriptor/descriptor_test.go

@@ -0,0 +1,50 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package descriptor
+
+import (
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+type descriptorTestSuite struct {
+	suite.Suite
+}
+
+func (d *descriptorTestSuite) TestParseArtifactType() {
+	mediaType := ""
+	typee := parseArtifactType(mediaType)
+	d.Equal(ArtifactTypeUnknown, typee)
+
+	mediaType = "unknown"
+	typee = parseArtifactType(mediaType)
+	d.Equal(ArtifactTypeUnknown, typee)
+
+	mediaType = "application/vnd.oci.image.config.v1+json"
+	typee = parseArtifactType(mediaType)
+	d.Equal("IMAGE", typee)
+
+	mediaType = "application/vnd.cncf.helm.chart.config.v1+json"
+	typee = parseArtifactType(mediaType)
+	d.Equal("HELM.CHART", typee)
+
+	mediaType = "application/vnd.sylabs.sif.config.v1+json"
+	typee = parseArtifactType(mediaType)
+	d.Equal("SIF", typee)
+}
+
+func TestDescriptorTestSuite(t *testing.T) {
+	suite.Run(t, &descriptorTestSuite{})
+}

src/chartserver/handler_manipulation.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/ghodss/yaml"
+	"github.com/goharbor/harbor/src/common/api"
 	commonhttp "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/replication"
@@ -69,7 +70,7 @@ func (c *Controller) DeleteChartVersion(namespace, chartName, version string) er
 		return errors.New("invalid chart for deleting")
 	}
 
-	url := fmt.Sprintf("/api/chartrepo/%s/charts/%s/%s", namespace, chartName, version)
+	url := fmt.Sprintf("/api/%s/chartrepo/%s/charts/%s/%s", api.APIVersion, namespace, chartName, version)
 	req, _ := http.NewRequest(http.MethodDelete, url, nil)
 	w := httptest.NewRecorder()
 

src/chartserver/handler_proxy_traffic_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/ghodss/yaml"
+	"github.com/goharbor/harbor/src/common/api"
 	htesting "github.com/goharbor/harbor/src/testing"
 	helm_repo "k8s.io/helm/pkg/repo"
 )
@@ -36,7 +37,7 @@ func TestStartMockServers(t *testing.T) {
 
 // Test /health
 func TestGetHealthOfBaseHandler(t *testing.T) {
-	content, err := httpClient.GetContent(fmt.Sprintf("%s/api/chartrepo/health", frontServer.URL))
+	content, err := httpClient.GetContent(fmt.Sprintf("%s/api/%s/chartrepo/health", frontServer.URL, api.APIVersion))
 	if err != nil {
 		t.Fatal(err)
 	}

src/chartserver/reverse_proxy.go

@@ -12,24 +12,29 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/goharbor/harbor/src/common"
+	"github.com/goharbor/harbor/src/common/api"
 	hlog "github.com/goharbor/harbor/src/common/utils/log"
 	n_event "github.com/goharbor/harbor/src/pkg/notifier/event"
 	"github.com/goharbor/harbor/src/replication"
 	rep_event "github.com/goharbor/harbor/src/replication/event"
 	"github.com/justinas/alice"
-	"time"
 )
 
 const (
 	agentHarbor         = "HARBOR"
 	contentLengthHeader = "Content-Length"
+)
 
-	defaultRepo             = "library"
+var (
-	rootUploadingEndpoint   = "/api/chartrepo/charts"
+	defaultRepo = "library"
-	rootIndexEndpoint       = "/chartrepo/index.yaml"
+
-	chartRepoHealthEndpoint = "/api/chartrepo/health"
+	chartRepoAPIPrefix      = fmt.Sprintf("/api/%s/chartrepo", api.APIVersion)
+	rootUploadingEndpoint   = fmt.Sprintf("/api/%s/chartrepo/charts", api.APIVersion)
+	chartRepoHealthEndpoint = fmt.Sprintf("/api/%s/chartrepo/health", api.APIVersion)
+	chartRepoPrefix         = "/chartrepo"
 )
 
 // ProxyEngine is used to proxy the related traffics
@@ -220,19 +225,19 @@ func rewriteURLPath(req *http.Request) {
 
 	// Root uploading endpoint
 	if incomingURLPath == rootUploadingEndpoint {
-		req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
+		req.URL.Path = strings.Replace(incomingURLPath, fmt.Sprintf("%s/chartrepo", api.APIVersion), defaultRepo, 1)
 		return
 	}
 
 	// Repository endpoints
-	if strings.HasPrefix(incomingURLPath, "/chartrepo") {
+	if strings.HasPrefix(incomingURLPath, chartRepoPrefix) {
 		req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
 		return
 	}
 
 	// API endpoints
-	if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
+	if strings.HasPrefix(incomingURLPath, chartRepoAPIPrefix) {
-		req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
+		req.URL.Path = strings.Replace(incomingURLPath, fmt.Sprintf("/%s/chartrepo", api.APIVersion), "", 1)
 		return
 	}
 }

src/chartserver/reverse_proxy_test.go

@@ -1,13 +1,16 @@
 package chartserver
 
 import (
+	"fmt"
 	"net/http"
 	"testing"
+
+	"github.com/goharbor/harbor/src/common/api"
 )
 
 // Test the URL rewrite function
 func TestURLRewrite(t *testing.T) {
-	req, err := createRequest(http.MethodGet, "/api/chartrepo/health")
+	req, err := createRequest(http.MethodGet, fmt.Sprintf("/api/%s/chartrepo/health", api.APIVersion))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -16,7 +19,7 @@ func TestURLRewrite(t *testing.T) {
 		t.Fatalf("Expect url format %s but got %s", "/health", req.URL.Path)
 	}
 
-	req, err = createRequest(http.MethodGet, "/api/chartrepo/library/charts")
+	req, err = createRequest(http.MethodGet, fmt.Sprintf("/api/%s/chartrepo/library/charts", api.APIVersion))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -25,7 +28,7 @@ func TestURLRewrite(t *testing.T) {
 		t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
 	}
 
-	req, err = createRequest(http.MethodPost, "/api/chartrepo/charts")
+	req, err = createRequest(http.MethodPost, fmt.Sprintf("/api/%s/chartrepo/charts", api.APIVersion))
 	if err != nil {
 		t.Fatal(err)
 	}

src/common/api/base.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 
 	"errors"
+
 	"github.com/astaxie/beego"
 	"github.com/astaxie/beego/validation"
 	commonhttp "github.com/goharbor/harbor/src/common/http"
@@ -31,6 +32,9 @@ import (
 const (
 	defaultPageSize int64 = 500
 	maxPageSize     int64 = 500
+
+	// APIVersion is the current core api version
+	APIVersion = "v2.0"
 )
 
 // BaseAPI wraps common methods for controllers to host API

src/common/dao/admin_job.go

@@ -28,10 +28,10 @@ func AddAdminJob(job *models.AdminJob) (int64, error) {
 	if len(job.Status) == 0 {
 		job.Status = models.JobPending
 	}
-	sql := "insert into admin_job (job_name, job_kind, status, job_uuid, cron_str, creation_time, update_time) values (?, ?, ?, ?, ?, ?, ?) RETURNING id"
+	sql := "insert into admin_job (job_name, job_parameters, job_kind, status, job_uuid, cron_str, creation_time, update_time) values (?, ?, ?, ?, ?, ?, ?, ?) RETURNING id"
 	var id int64
 	now := time.Now()
-	err := o.Raw(sql, job.Name, job.Kind, job.Status, job.UUID, job.Cron, now, now).QueryRow(&id)
+	err := o.Raw(sql, job.Name, job.Parameters, job.Kind, job.Status, job.UUID, job.Cron, now, now).QueryRow(&id)
 	if err != nil {
 		return 0, err
 	}

src/common/dao/admin_job_test.go

@@ -44,8 +44,9 @@ func (suite *AdminJobSuite) SetupSuite() {
 	}
 
 	job0 := &models.AdminJob{
-		Name: "GC",
+		Name:       "GC",
-		Kind: "testKind",
+		Kind:       "testKind",
+		Parameters: "{test:test}",
 	}
 
 	suite.ids = make([]int64, 0)
@@ -77,6 +78,7 @@ func (suite *AdminJobSuite) TestAdminJobBase() {
 	require.Nil(suite.T(), err)
 	suite.Equal(job1.ID, suite.job0.ID)
 	suite.Equal(job1.Name, suite.job0.Name)
+	suite.Equal(job1.Parameters, suite.job0.Parameters)
 
 	// set uuid
 	err = SetAdminJobUUID(suite.job0.ID, "f5ef34f4cb3588d663176132")

src/common/dao/dao_test.go

@@ -751,29 +751,6 @@ func TestGetRepositoryByName(t *testing.T) {
 	}
 }
 
-func TestIncreasePullCount(t *testing.T) {
-	if err := IncreasePullCount(currentRepository.Name); err != nil {
-		log.Errorf("Error happens when increasing pull count: %v", currentRepository.Name)
-	}
-
-	repository, err := GetRepositoryByName(currentRepository.Name)
-	if err != nil {
-		t.Errorf("Error occurred in GetRepositoryByName: %v", err)
-	}
-
-	if repository.PullCount != 1 {
-		t.Errorf("repository pull count is not 1 after IncreasePullCount, expected: 1, actual: %d", repository.PullCount)
-	}
-}
-
-func TestRepositoryExists(t *testing.T) {
-	var exists bool
-	exists = RepositoryExists(currentRepository.Name)
-	if !exists {
-		t.Errorf("The repository with name: %s, does not exist", currentRepository.Name)
-	}
-}
-
 func TestDeleteRepository(t *testing.T) {
 	err := DeleteRepository(currentRepository.Name)
 	if err != nil {

src/common/dao/repository.go

@@ -69,54 +69,12 @@ func DeleteRepository(name string) error {
 	return err
 }
 
-// UpdateRepository ...
-func UpdateRepository(repo models.RepoRecord) error {
-	o := GetOrmer()
-	repo.UpdateTime = time.Now()
-	_, err := o.Update(&repo)
-	return err
-}
-
-// IncreasePullCount ...
-func IncreasePullCount(name string) (err error) {
-	o := GetOrmer()
-	num, err := o.QueryTable("repository").Filter("name", name).Update(
-		orm.Params{
-			"pull_count":  orm.ColValue(orm.ColAdd, 1),
-			"update_time": time.Now(),
-		})
-	if err != nil {
-		return err
-	}
-	if num == 0 {
-		return fmt.Errorf("Failed to increase repository pull count with name: %s", name)
-	}
-	return nil
-}
-
 // RepositoryExists returns whether the repository exists according to its name.
 func RepositoryExists(name string) bool {
 	o := GetOrmer()
 	return o.QueryTable("repository").Filter("name", name).Exist()
 }
 
-// GetTopRepos returns the most popular repositories whose project ID is
-// in projectIDs
-func GetTopRepos(projectIDs []int64, n int) ([]*models.RepoRecord, error) {
-	repositories := []*models.RepoRecord{}
-	if len(projectIDs) == 0 {
-		return repositories, nil
-	}
-
-	_, err := GetOrmer().QueryTable(&models.RepoRecord{}).
-		Filter("project_id__in", projectIDs).
-		OrderBy("-pull_count").
-		Limit(n).
-		All(&repositories)
-
-	return repositories, err
-}
-
 // GetTotalOfRepositories ...
 func GetTotalOfRepositories(query ...*models.RepositoryQuery) (int64, error) {
 	sql, params := repositoryQueryConditions(query...)

src/common/dao/repository_test.go

@@ -15,7 +15,6 @@
 package dao
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/goharbor/harbor/src/common"
@@ -122,66 +121,6 @@ func TestGetRepositories(t *testing.T) {
 	assert.Equal(t, name, repositories[0].Name)
 }
 
-func TestGetTopRepos(t *testing.T) {
-	var err error
-	require := require.New(t)
-
-	require.NoError(GetOrmer().Begin())
-	defer func() {
-		require.NoError(GetOrmer().Rollback())
-	}()
-
-	projectIDs := []int64{}
-
-	project1 := models.Project{
-		OwnerID: 1,
-		Name:    "project1",
-	}
-	project1.ProjectID, err = AddProject(project1)
-	require.NoError(err)
-	projectIDs = append(projectIDs, project1.ProjectID)
-
-	project2 := models.Project{
-		OwnerID: 1,
-		Name:    "project2",
-	}
-	project2.ProjectID, err = AddProject(project2)
-	require.NoError(err)
-	projectIDs = append(projectIDs, project2.ProjectID)
-
-	repository1 := &models.RepoRecord{
-		Name:      fmt.Sprintf("%v/repository1", project1.Name),
-		ProjectID: project1.ProjectID,
-	}
-	err = AddRepository(*repository1)
-	require.NoError(err)
-	require.NoError(IncreasePullCount(repository1.Name))
-
-	repository2 := &models.RepoRecord{
-		Name:      fmt.Sprintf("%v/repository2", project1.Name),
-		ProjectID: project1.ProjectID,
-	}
-	err = AddRepository(*repository2)
-	require.NoError(err)
-	require.NoError(IncreasePullCount(repository2.Name))
-	require.NoError(IncreasePullCount(repository2.Name))
-
-	repository3 := &models.RepoRecord{
-		Name:      fmt.Sprintf("%v/repository3", project2.Name),
-		ProjectID: project2.ProjectID,
-	}
-	err = AddRepository(*repository3)
-	require.NoError(err)
-	require.NoError(IncreasePullCount(repository3.Name))
-	require.NoError(IncreasePullCount(repository3.Name))
-	require.NoError(IncreasePullCount(repository3.Name))
-
-	topRepos, err := GetTopRepos(projectIDs, 100)
-	require.NoError(err)
-	require.Len(topRepos, 3)
-	require.Equal(topRepos[0].Name, repository3.Name)
-}
-
 func addRepository(repository *models.RepoRecord) error {
 	return AddRepository(*repository)
 }

src/common/http/client.go

@@ -19,14 +19,13 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"errors"
+	"github.com/goharbor/harbor/src/common/http/modifier"
+	"github.com/goharbor/harbor/src/internal"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"reflect"
-	"strings"
-
-	"github.com/goharbor/harbor/src/common/http/modifier"
 )
 
 // Client is a util for common HTTP operations, such Get, Head, Post, Put and Delete.
@@ -231,8 +230,8 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
 		if err != nil {
 			return err
 		}
-		defer resp.Body.Close()
 		data, err := ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
 		if err != nil {
 			return err
 		}
@@ -250,12 +249,10 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
 		resources = reflect.AppendSlice(resources, reflect.Indirect(res))
 
 		endpoint = ""
-		link := resp.Header.Get("Link")
+		links := internal.ParseLinks(resp.Header.Get("Link"))
-		for _, str := range strings.Split(link, ",") {
+		for _, link := range links {
-			if strings.HasSuffix(str, `rel="next"`) &&
+			if link.Rel == "next" {
-				strings.Index(str, "<") >= 0 &&
+				endpoint = url.Scheme + "://" + url.Host + link.URL
-				strings.Index(str, ">") >= 0 {
-				endpoint = url.Scheme + "://" + url.Host + str[strings.Index(str, "<")+1:strings.Index(str, ">")]
 				break
 			}
 		}

src/common/models/adminjob.go

@@ -29,6 +29,7 @@ type AdminJob struct {
 	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
 	Name         string    `orm:"column(job_name)"  json:"job_name"`
 	Kind         string    `orm:"column(job_kind)"  json:"job_kind"`
+	Parameters   string    `orm:"column(job_parameters)"  json:"job_parameters"`
 	Cron         string    `orm:"column(cron_str)"  json:"cron_str"`
 	Status       string    `orm:"column(status)"  json:"job_status"`
 	UUID         string    `orm:"column(job_uuid)" json:"-"`

src/common/models/retag.go

@@ -1,52 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package models
-
-import (
-	"fmt"
-	"strings"
-)
-
-// RetagRequest gives the source image and target image of retag
-type RetagRequest struct {
-	Tag      string `json:"tag"`       // The new tag
-	SrcImage string `json:"src_image"` // Source images in format <project>/<repo>:<reference>
-	Override bool   `json:"override"`  // If target tag exists, whether override it
-}
-
-// Image holds each part (project, repo, tag) of an image name
-type Image struct {
-	Project string
-	Repo    string
-	Tag     string
-}
-
-// ParseImage parses an image name such as 'library/app:v1.0' to a structure with
-// project, repo, and tag fields
-func ParseImage(image string) (*Image, error) {
-	repo := strings.SplitN(image, "/", 2)
-	if len(repo) < 2 {
-		return nil, fmt.Errorf("unable to parse image from string: %s", image)
-	}
-	i := strings.SplitN(repo[1], ":", 2)
-	res := &Image{
-		Project: repo[0],
-		Repo:    i[0],
-	}
-	if len(i) == 2 {
-		res.Tag = i[1]
-	}
-	return res, nil
-}

src/common/models/retag_test.go

@@ -1,75 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package models
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestParseImage(t *testing.T) {
-	cases := []struct {
-		Input    string
-		Expected *Image
-		Valid    bool
-	}{
-		{
-			Input: "library/busybox",
-			Expected: &Image{
-				Project: "library",
-				Repo:    "busybox",
-				Tag:     "",
-			},
-			Valid: true,
-		},
-		{
-			Input: "library/busybox:v1.0",
-			Expected: &Image{
-				Project: "library",
-				Repo:    "busybox",
-				Tag:     "v1.0",
-			},
-			Valid: true,
-		},
-		{
-			Input: "library/busybox:sha256:9e2c9d5f44efbb6ee83aecd17a120c513047d289d142ec5738c9f02f9b24ad07",
-			Expected: &Image{
-				Project: "library",
-				Repo:    "busybox",
-				Tag:     "sha256:9e2c9d5f44efbb6ee83aecd17a120c513047d289d142ec5738c9f02f9b24ad07",
-			},
-			Valid: true,
-		},
-		{
-			Input: "busybox/v1.0",
-			Valid: false,
-		},
-	}
-
-	for _, c := range cases {
-		output, err := ParseImage(c.Input)
-		if c.Valid {
-			if !reflect.DeepEqual(output, c.Expected) {
-				assert.Equal(t, c.Expected, output)
-			}
-		} else {
-			if err != nil {
-				t.Errorf("failed to parse image %s", c.Input)
-			}
-		}
-	}
-}

src/common/utils/registry/auth/basicauthorizer.go

@@ -1,40 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"github.com/goharbor/harbor/src/common/http/modifier"
-	"github.com/goharbor/harbor/src/core/config"
-	"sync"
-)
-
-// NewBasicAuthorizer create an authorizer to add basic auth header as is set in the parameter
-func NewBasicAuthorizer(u, p string) modifier.Modifier {
-	return NewBasicAuthCredential(u, p)
-}
-
-var (
-	defaultAuthorizer modifier.Modifier
-	once              sync.Once
-)
-
-// DefaultBasicAuthorizer returns the basic authorizer that sets the basic auth as configured in env variables
-func DefaultBasicAuthorizer() modifier.Modifier {
-	once.Do(func() {
-		u, p := config.RegistryCredential()
-		defaultAuthorizer = NewBasicAuthCredential(u, p)
-	})
-	return defaultAuthorizer
-}

src/common/utils/registry/auth/basicauthorizer_test.go

@@ -1,39 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"github.com/stretchr/testify/assert"
-	"net/http"
-	"os"
-	"testing"
-)
-
-func TestDefaultBasicAuthorizer(t *testing.T) {
-	os.Setenv("REGISTRY_CREDENTIAL_USERNAME", "testuser")
-	os.Setenv("REGISTRY_CREDENTIAL_PASSWORD", "testpassword")
-	defer func() {
-		os.Unsetenv("REGISTRY_CREDENTIAL_USERNAME")
-		os.Unsetenv("REGISTRY_CREDENTIAL_PASSWORD")
-	}()
-	req, _ := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
-	a := DefaultBasicAuthorizer()
-	err := a.Modify(req)
-	assert.Nil(t, err)
-	u, p, ok := req.BasicAuth()
-	assert.True(t, ok)
-	assert.Equal(t, "testuser", u)
-	assert.Equal(t, "testpassword", p)
-}

src/common/utils/registry/auth/credential.go

@@ -1,48 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"net/http"
-
-	"github.com/goharbor/harbor/src/common/http/modifier"
-)
-
-// Credential ...
-type Credential modifier.Modifier
-
-// Implements interface Credential
-type basicAuthCredential struct {
-	username string
-	password string
-}
-
-// NewBasicAuthCredential ...
-func NewBasicAuthCredential(username, password string) Credential {
-	return &basicAuthCredential{
-		username: username,
-		password: password,
-	}
-}
-
-func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
-	req.SetBasicAuth(b.username, b.password)
-}
-
-// implement github.com/goharbor/harbor/src/common/http/modifier.Modifier
-func (b *basicAuthCredential) Modify(req *http.Request) error {
-	b.AddAuthorization(req)
-	return nil
-}

src/common/utils/registry/auth/path.go

@@ -1,58 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"regexp"
-
-	"github.com/docker/distribution/reference"
-	"github.com/goharbor/harbor/src/common/utils/log"
-)
-
-var (
-	base            = regexp.MustCompile("/v2")
-	catalog         = regexp.MustCompile("/v2/_catalog")
-	tag             = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/tags/list")
-	manifest        = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/manifests/(" + reference.TagRegexp.String() + "|" + reference.DigestRegexp.String() + ")")
-	blob            = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String())
-	blobUpload      = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/uploads")
-	blobUploadChunk = regexp.MustCompile("/v2/(" + reference.NameRegexp.String() + ")/blobs/uploads/[a-zA-Z0-9-_.=]+")
-
-	repoRegExps = []*regexp.Regexp{tag, manifest, blob, blobUploadChunk, blobUpload}
-)
-
-// parse the repository name from path, if the path doesn't match any
-// regular expressions in repoRegExps, nil string will be returned
-func parseRepository(path string) string {
-	for _, regExp := range repoRegExps {
-		subs := regExp.FindStringSubmatch(path)
-		// no match
-		if subs == nil {
-			continue
-		}
-
-		// match
-		// the subs should contain at least 2 matching texts, the first one matches
-		// the whole regular expression, and the second one matches the repository
-		// part
-		if len(subs) < 2 {
-			log.Warningf("unexpected length of sub matches: %d, should >= 2 ", len(subs))
-			continue
-		}
-		return subs[1]
-	}
-
-	return ""
-}

src/common/utils/registry/auth/path_test.go

@@ -1,43 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestParseRepository(t *testing.T) {
-	cases := []struct {
-		input  string
-		output string
-	}{
-		{"/v2", ""},
-		{"/v2/_catalog", ""},
-		{"/v2/library/tags/list", "library"},
-		{"/v2/tags/list", ""},
-		{"/v2/tags/list/tags/list", "tags/list"},
-		{"/v2/library/manifests/latest", "library"},
-		{"/v2/library/manifests/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", "library"},
-		{"/v2/library/blobs/sha256:eec76eedea59f7bf39a2713bfd995c82cfaa97724ee5b7f5aba253e07423d0ae", "library"},
-		{"/v2/library/blobs/uploads", "library"},
-		{"/v2/library/blobs/uploads/1234567890", "library"},
-	}
-
-	for _, c := range cases {
-		assert.Equal(t, c.output, parseRepository(c.input))
-	}
-}

src/common/utils/registry/auth/tokenauthorizer.go

@@ -1,354 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-	"net/url"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/docker/distribution/registry/auth/token"
-	"github.com/goharbor/harbor/src/common/http/modifier"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	token_util "github.com/goharbor/harbor/src/core/service/token"
-)
-
-const (
-	latency int = 10 // second, the network latency when token is received
-	scheme      = "bearer"
-)
-
-type tokenGenerator interface {
-	generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error)
-}
-
-// UserAgentModifier adds the "User-Agent" header to the request
-type UserAgentModifier struct {
-	UserAgent string
-}
-
-// Modify adds user-agent header to the request
-func (u *UserAgentModifier) Modify(req *http.Request) error {
-	req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.UserAgent)
-	return nil
-}
-
-// tokenAuthorizer implements registry.Modifier interface. It parses scopses
-// from the request, generates authentication token and modifies the requset
-// by adding the token
-type tokenAuthorizer struct {
-	registryURL  *url.URL // used to filter request
-	generator    tokenGenerator
-	client       *http.Client
-	cachedTokens map[string]*models.Token
-	sync.Mutex
-}
-
-// add token to the request
-func (t *tokenAuthorizer) Modify(req *http.Request) error {
-	// only handle requests sent to registry
-	goon, err := t.filterReq(req)
-	if err != nil {
-		return err
-	}
-
-	if !goon {
-		log.Debugf("the request %s is not sent to registry, skip", req.URL.String())
-		return nil
-	}
-
-	// parse scopes from request
-	scopes, err := parseScopes(req)
-	if err != nil {
-		return err
-	}
-
-	var token *models.Token
-	// try to get token from cache if the request is for empty scope(login)
-	// or single scope
-	if len(scopes) <= 1 {
-		key := ""
-		if len(scopes) == 1 {
-			key = scopeString(scopes[0])
-		}
-		token = t.getCachedToken(key)
-	}
-
-	// request a new token if the token is null
-	if token == nil {
-		token, err = t.generator.generate(scopes, t.registryURL.String())
-		if err != nil {
-			return err
-		}
-		// if the token is null(this happens if the registry needs no authentication), return
-		// directly. Or the token will be cached
-		if token == nil {
-			return nil
-		}
-		// only cache the token for empty scope(login) or single scope request
-		if len(scopes) <= 1 {
-			key := ""
-			if len(scopes) == 1 {
-				key = scopeString(scopes[0])
-			}
-			t.updateCachedToken(key, token)
-		}
-	}
-
-	tk := token.GetToken()
-	if len(tk) == 0 {
-		return errors.New("empty token content")
-	}
-
-	req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", tk))
-
-	return nil
-}
-
-func scopeString(scope *token.ResourceActions) string {
-	if scope == nil {
-		return ""
-	}
-	return fmt.Sprintf("%s:%s:%s", scope.Type, scope.Name, strings.Join(scope.Actions, ","))
-}
-
-// some requests are sent to backend storage, such as s3, this method filters
-// the requests only sent to registry
-func (t *tokenAuthorizer) filterReq(req *http.Request) (bool, error) {
-	// the registryURL is nil when the first request comes, init it with
-	// the scheme and host of the request which must be sent to the registry
-	if t.registryURL == nil {
-		u, err := url.Parse(buildPingURL(req.URL.Scheme + "://" + req.URL.Host))
-		if err != nil {
-			return false, err
-		}
-		t.registryURL = u
-	}
-
-	v2Index := strings.Index(req.URL.Path, "/v2/")
-	if v2Index == -1 {
-		return false, nil
-	}
-
-	if req.URL.Host != t.registryURL.Host || req.URL.Scheme != t.registryURL.Scheme ||
-		req.URL.Path[:v2Index+4] != t.registryURL.Path {
-		return false, nil
-	}
-
-	return true, nil
-}
-
-// parse scopes from the request according to its method, path and query string
-func parseScopes(req *http.Request) ([]*token.ResourceActions, error) {
-	scopes := []*token.ResourceActions{}
-
-	from := req.URL.Query().Get("from")
-	if len(from) != 0 {
-		scopes = append(scopes, &token.ResourceActions{
-			Type:    "repository",
-			Name:    from,
-			Actions: []string{"pull"},
-		})
-	}
-
-	var scope *token.ResourceActions
-	path := strings.TrimRight(req.URL.Path, "/")
-	repository := parseRepository(path)
-	if len(repository) > 0 {
-		// pull, push, delete blob/manifest
-		scope = &token.ResourceActions{
-			Type: "repository",
-			Name: repository,
-		}
-		switch req.Method {
-		case http.MethodGet, http.MethodHead:
-			scope.Actions = []string{"pull"}
-		case http.MethodPost, http.MethodPut, http.MethodPatch:
-			scope.Actions = []string{"pull", "push"}
-		case http.MethodDelete:
-			scope.Actions = []string{"*"}
-		default:
-			scope = nil
-			log.Warningf("unsupported method: %s", req.Method)
-		}
-	} else if catalog.MatchString(path) {
-		// catalog
-		scope = &token.ResourceActions{
-			Type:    "registry",
-			Name:    "catalog",
-			Actions: []string{"*"},
-		}
-	} else if base.MatchString(path) {
-		// base
-		scope = nil
-	} else {
-		// unknown
-		return scopes, fmt.Errorf("can not parse scope from the request: %s %s", req.Method, req.URL.Path)
-	}
-
-	if scope != nil {
-		scopes = append(scopes, scope)
-	}
-
-	strs := []string{}
-	for _, s := range scopes {
-		strs = append(strs, scopeString(s))
-	}
-	log.Debugf("scopes parsed from request: %s", strings.Join(strs, " "))
-
-	return scopes, nil
-}
-
-func (t *tokenAuthorizer) getCachedToken(scope string) *models.Token {
-	t.Lock()
-	defer t.Unlock()
-	token := t.cachedTokens[scope]
-	if token == nil {
-		return nil
-	}
-
-	issueAt, err := time.Parse(time.RFC3339, token.IssuedAt)
-	if err != nil {
-		log.Errorf("failed parse %s: %v", token.IssuedAt, err)
-		delete(t.cachedTokens, scope)
-		return nil
-	}
-
-	if issueAt.Add(time.Duration(token.ExpiresIn-latency) * time.Second).Before(time.Now().UTC()) {
-		delete(t.cachedTokens, scope)
-		return nil
-	}
-
-	log.Debugf("get token for scope %s from cache", scope)
-	return token
-}
-
-func (t *tokenAuthorizer) updateCachedToken(scope string, token *models.Token) {
-	t.Lock()
-	defer t.Unlock()
-	t.cachedTokens[scope] = token
-}
-
-// ping returns the realm, service and error
-func ping(client *http.Client, endpoint string) (string, string, error) {
-	resp, err := client.Get(endpoint)
-	if err != nil {
-		return "", "", err
-	}
-	defer resp.Body.Close()
-
-	challenges := ParseChallengeFromResponse(resp)
-	for _, challenge := range challenges {
-		if scheme == challenge.Scheme {
-			realm := challenge.Parameters["realm"]
-			service := challenge.Parameters["service"]
-			return realm, service, nil
-		}
-	}
-
-	log.Warningf("Schemas %v are unsupported", challenges)
-	return "", "", nil
-}
-
-// NewStandardTokenAuthorizer returns a standard token authorizer. The authorizer will request a token
-// from token server and add it to the origin request
-// If customizedTokenService is set, the token request will be sent to it instead of the server get from authorizer
-func NewStandardTokenAuthorizer(client *http.Client, credential Credential,
-	customizedTokenService ...string) modifier.Modifier {
-	generator := &standardTokenGenerator{
-		credential: credential,
-		client:     client,
-	}
-
-	// when the registry client is used inside Harbor, the token request
-	// can be posted to token service directly rather than going through nginx.
-	// If realm is set as the internal url of token service, this can resolve
-	// two problems:
-	// 1. performance issue
-	// 2. the realm field returned by registry is an IP which can not reachable
-	// inside Harbor
-	if len(customizedTokenService) > 0 && len(customizedTokenService[0]) > 0 {
-		generator.realm = customizedTokenService[0]
-	}
-
-	return &tokenAuthorizer{
-		cachedTokens: make(map[string]*models.Token),
-		generator:    generator,
-		client:       client,
-	}
-}
-
-// standardTokenGenerator implements interface tokenGenerator
-type standardTokenGenerator struct {
-	realm      string
-	service    string
-	credential Credential
-	client     *http.Client
-}
-
-// get token from token service
-func (s *standardTokenGenerator) generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error) {
-	// ping first if the realm or service is null
-	if len(s.realm) == 0 || len(s.service) == 0 {
-		realm, service, err := ping(s.client, endpoint)
-		if err != nil {
-			return nil, err
-		}
-		if len(realm) == 0 {
-			log.Warning("empty realm, skip")
-			return nil, nil
-		}
-		if len(s.realm) == 0 {
-			s.realm = realm
-		}
-		s.service = service
-	}
-
-	return getToken(s.client, s.credential, s.realm, s.service, scopes)
-}
-
-// NewRawTokenAuthorizer returns a token authorizer which calls method to create
-// token directly
-func NewRawTokenAuthorizer(username, service string) modifier.Modifier {
-	generator := &rawTokenGenerator{
-		service:  service,
-		username: username,
-	}
-
-	return &tokenAuthorizer{
-		cachedTokens: make(map[string]*models.Token),
-		generator:    generator,
-	}
-}
-
-// rawTokenGenerator implements interface tokenGenerator
-type rawTokenGenerator struct {
-	service  string
-	username string
-}
-
-// generate token directly
-func (r *rawTokenGenerator) generate(scopes []*token.ResourceActions, endpoint string) (*models.Token, error) {
-	return token_util.MakeToken(r.username, r.service, scopes)
-}
-
-func buildPingURL(endpoint string) string {
-	return fmt.Sprintf("%s/v2/", endpoint)
-}

src/common/utils/registry/auth/tokenauthorizer_test.go

@@ -1,222 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/docker/distribution/registry/auth/token"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/common/utils/test"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestFilterReq(t *testing.T) {
-	authorizer := tokenAuthorizer{}
-
-	// v2
-	req, err := http.NewRequest(http.MethodGet, "http://registry/v2/", nil)
-	require.Nil(t, err)
-	goon, err := authorizer.filterReq(req)
-	assert.Nil(t, err)
-	assert.True(t, goon)
-
-	// catalog
-	req, err = http.NewRequest(http.MethodGet, "http://registry/v2/_catalog?n=1000", nil)
-	require.Nil(t, err)
-	goon, err = authorizer.filterReq(req)
-	assert.Nil(t, err)
-	assert.True(t, goon)
-
-	// contains two v2 in path
-	req, err = http.NewRequest(http.MethodGet, "http://registry/v2/library/v2/tags/list", nil)
-	require.Nil(t, err)
-	goon, err = authorizer.filterReq(req)
-	assert.Nil(t, err)
-	assert.True(t, goon)
-
-	// different scheme
-	req, err = http.NewRequest(http.MethodGet, "https://registry/v2/library/golang/tags/list", nil)
-	require.Nil(t, err)
-	goon, err = authorizer.filterReq(req)
-	assert.Nil(t, err)
-	assert.False(t, goon)
-
-	// different host
-	req, err = http.NewRequest(http.MethodGet, "http://vmware.com/v2/library/golang/tags/list", nil)
-	require.Nil(t, err)
-	goon, err = authorizer.filterReq(req)
-	assert.Nil(t, err)
-	assert.False(t, goon)
-
-	// different path
-	req, err = http.NewRequest(http.MethodGet, "https://registry/s3/ssss", nil)
-	require.Nil(t, err)
-	goon, err = authorizer.filterReq(req)
-	assert.Nil(t, err)
-	assert.False(t, goon)
-}
-
-func TestParseScopes(t *testing.T) {
-	// contains from in query string
-	req, err := http.NewRequest(http.MethodGet, "http://registry/v2?from=library", nil)
-	require.Nil(t, err)
-	scopses, err := parseScopes(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(scopses))
-	assert.EqualValues(t, &token.ResourceActions{
-		Type: "repository",
-		Name: "library",
-		Actions: []string{
-			"pull"},
-	}, scopses[0])
-
-	// v2
-	req, err = http.NewRequest(http.MethodGet, "http://registry/v2", nil)
-	require.Nil(t, err)
-	scopses, err = parseScopes(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(scopses))
-
-	// catalog
-	req, err = http.NewRequest(http.MethodGet, "http://registry/v2/_catalog", nil)
-	require.Nil(t, err)
-	scopses, err = parseScopes(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(scopses))
-	assert.EqualValues(t, &token.ResourceActions{
-		Type: "registry",
-		Name: "catalog",
-		Actions: []string{
-			"*"},
-	}, scopses[0])
-
-	// manifest
-	req, err = http.NewRequest(http.MethodPut, "http://registry/v2/library/mysql/5.6/manifests/1", nil)
-	require.Nil(t, err)
-	scopses, err = parseScopes(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(scopses))
-	assert.EqualValues(t, &token.ResourceActions{
-		Type:    "repository",
-		Name:    "library/mysql/5.6",
-		Actions: []string{"pull", "push"},
-	}, scopses[0])
-
-	// invalid
-	req, err = http.NewRequest(http.MethodPut, "http://registry/other", nil)
-	require.Nil(t, err)
-	scopses, err = parseScopes(req)
-	assert.NotNil(t, err)
-}
-
-func TestGetAndUpdateCachedToken(t *testing.T) {
-	authorizer := &tokenAuthorizer{
-		cachedTokens: make(map[string]*models.Token),
-	}
-
-	// empty cache
-	token := authorizer.getCachedToken("")
-	assert.Nil(t, token)
-
-	// put a valid token into cache
-	token = &models.Token{
-		Token:     "token",
-		ExpiresIn: 60,
-		IssuedAt:  time.Now().Format(time.RFC3339),
-	}
-	authorizer.updateCachedToken("", token)
-	token2 := authorizer.getCachedToken("")
-	assert.EqualValues(t, token, token2)
-
-	// put a expired token into cache
-	token = &models.Token{
-		Token:     "token",
-		ExpiresIn: 60,
-		IssuedAt:  time.Now().Add(-time.Second * 120).Format("2006-01-02 15:04:05.999999999 -0700 MST"),
-	}
-	authorizer.updateCachedToken("", token)
-	token2 = authorizer.getCachedToken("")
-	assert.Nil(t, token2)
-}
-
-func TestModifyOfStandardTokenAuthorizer(t *testing.T) {
-	token := &models.Token{
-		Token:     "token",
-		ExpiresIn: 3600,
-		IssuedAt:  time.Now().String(),
-	}
-	data, err := json.Marshal(token)
-	require.Nil(t, err)
-
-	tokenHandler := test.Handler(&test.Response{
-		Body: data,
-	})
-
-	tokenServer := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "GET",
-			Pattern: "/service/token",
-			Handler: tokenHandler,
-		})
-	defer tokenServer.Close()
-
-	header := fmt.Sprintf("Bearer realm=\"%s/service/token\",service=\"registry\"",
-		tokenServer.URL)
-	pingHandler := test.Handler(&test.Response{
-		StatusCode: http.StatusUnauthorized,
-		Headers: map[string]string{
-			"WWW-Authenticate": header,
-		},
-	})
-	registryServer := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "GET",
-			Pattern: "/v2",
-			Handler: pingHandler,
-		})
-	defer registryServer.Close()
-
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/v2/", registryServer.URL), nil)
-	require.Nil(t, err)
-
-	authorizer := NewStandardTokenAuthorizer(http.DefaultClient, nil)
-
-	err = authorizer.Modify(req)
-	require.Nil(t, err)
-
-	tk := req.Header.Get("Authorization")
-	assert.Equal(t, strings.ToLower("Bearer "+token.Token), strings.ToLower(tk))
-}
-
-func TestUserAgentModifier(t *testing.T) {
-	agent := "harbor-registry-client"
-	modifier := &UserAgentModifier{
-		UserAgent: agent,
-	}
-	req, err := http.NewRequest(http.MethodGet, "http://registry/v2/", nil)
-	require.Nil(t, err)
-	modifier.Modify(req)
-	actual := req.Header.Get("User-Agent")
-	if actual != agent {
-		t.Errorf("expect request to have header User-Agent=%s, but got User-Agent=%s", agent, actual)
-	}
-}

src/common/utils/registry/auth/util.go

@@ -1,88 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-
-	"github.com/docker/distribution/registry/auth/token"
-	commonhttp "github.com/goharbor/harbor/src/common/http"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/common/utils/registry"
-)
-
-const (
-	service = "harbor-registry"
-)
-
-// GetToken requests a token against the endpoint using credential provided
-func GetToken(endpoint string, insecure bool, credential Credential,
-	scopes []*token.ResourceActions) (*models.Token, error) {
-	client := &http.Client{
-		Transport: registry.GetHTTPTransport(insecure),
-	}
-
-	return getToken(client, credential, endpoint, service, scopes)
-}
-
-func getToken(client *http.Client, credential Credential, realm, service string,
-	scopes []*token.ResourceActions) (*models.Token, error) {
-	u, err := url.Parse(realm)
-	if err != nil {
-		return nil, err
-	}
-	query := u.Query()
-	query.Add("service", service)
-	for _, scope := range scopes {
-		query.Add("scope", scopeString(scope))
-	}
-	u.RawQuery = query.Encode()
-
-	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	if credential != nil {
-		credential.Modify(req)
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, &commonhttp.Error{
-			Code:    resp.StatusCode,
-			Message: string(data),
-		}
-	}
-
-	token := &models.Token{}
-	if err = json.Unmarshal(data, token); err != nil {
-		return nil, err
-	}
-
-	return token, nil
-}

src/common/utils/registry/manifest_test.go

@@ -1,60 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
-	"testing"
-
-	"github.com/docker/distribution/manifest/schema2"
-)
-
-func TestUnMarshal(t *testing.T) {
-	b := []byte(`{  
-   "schemaVersion":2,
-   "mediaType":"application/vnd.docker.distribution.manifest.v2+json",
-   "config":{  
-      "mediaType":"application/vnd.docker.container.image.v1+json",
-      "size":1473,
-      "digest":"sha256:c54a2cc56cbb2f04003c1cd4507e118af7c0d340fe7e2720f70976c4b75237dc"
-   },
-   "layers":[  
-      {  
-         "mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip",
-         "size":974,
-         "digest":"sha256:c04b14da8d1441880ed3fe6106fb2cc6fa1c9661846ac0266b8a5ec8edf37b7c"
-      }
-   ]
-}`)
-
-	manifest, _, err := UnMarshal(schema2.MediaTypeManifest, b)
-	if err != nil {
-		t.Fatalf("failed to parse manifest: %v", err)
-	}
-
-	refs := manifest.References()
-	if len(refs) != 2 {
-		t.Fatalf("unexpected length of reference: %d != %d", len(refs), 2)
-	}
-
-	digest := "sha256:c54a2cc56cbb2f04003c1cd4507e118af7c0d340fe7e2720f70976c4b75237dc"
-	if refs[0].Digest.String() != digest {
-		t.Errorf("unexpected digest: %s != %s", refs[0].Digest.String(), digest)
-	}
-
-	digest = "sha256:c04b14da8d1441880ed3fe6106fb2cc6fa1c9661846ac0266b8a5ec8edf37b7c"
-	if refs[1].Digest.String() != digest {
-		t.Errorf("unexpected digest: %s != %s", refs[1].Digest.String(), digest)
-	}
-}

src/common/utils/registry/registry.go

@@ -1,185 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
-	"crypto/tls"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strings"
-
-	commonhttp "github.com/goharbor/harbor/src/common/http"
-	"github.com/goharbor/harbor/src/common/utils"
-)
-
-// Registry holds information of a registry entity
-type Registry struct {
-	Endpoint *url.URL
-	client   *http.Client
-}
-
-var defaultHTTPTransport, secureHTTPTransport, insecureHTTPTransport *http.Transport
-
-func init() {
-	defaultHTTPTransport = &http.Transport{}
-
-	secureHTTPTransport = &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: false,
-		},
-	}
-	insecureHTTPTransport = &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
-		},
-	}
-}
-
-// GetHTTPTransport returns HttpTransport based on insecure configuration
-func GetHTTPTransport(insecure ...bool) *http.Transport {
-	if len(insecure) == 0 {
-		return defaultHTTPTransport
-	}
-	if insecure[0] {
-		return insecureHTTPTransport
-	}
-	return secureHTTPTransport
-}
-
-// NewRegistry returns an instance of registry
-func NewRegistry(endpoint string, client *http.Client) (*Registry, error) {
-	u, err := utils.ParseEndpoint(endpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	registry := &Registry{
-		Endpoint: u,
-		client:   client,
-	}
-
-	return registry, nil
-}
-
-// Catalog ...
-func (r *Registry) Catalog() ([]string, error) {
-	repos := []string{}
-	aurl := r.Endpoint.String() + "/v2/_catalog?n=1000"
-
-	for len(aurl) > 0 {
-		req, err := http.NewRequest("GET", aurl, nil)
-		if err != nil {
-			return repos, err
-		}
-		resp, err := r.client.Do(req)
-		if err != nil {
-			return nil, parseError(err)
-		}
-
-		defer resp.Body.Close()
-		b, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return repos, err
-		}
-
-		if resp.StatusCode == http.StatusOK {
-			catalogResp := struct {
-				Repositories []string `json:"repositories"`
-			}{}
-
-			if err := json.Unmarshal(b, &catalogResp); err != nil {
-				return repos, err
-			}
-
-			repos = append(repos, catalogResp.Repositories...)
-			// Link: </v2/_catalog?last=library%2Fhello-world-25&n=100>; rel="next"
-			// Link: <http://domain.com/v2/_catalog?last=library%2Fhello-world-25&n=100>; rel="next"
-			link := resp.Header.Get("Link")
-			if strings.HasSuffix(link, `rel="next"`) && strings.Index(link, "<") >= 0 && strings.Index(link, ">") >= 0 {
-				aurl = link[strings.Index(link, "<")+1 : strings.Index(link, ">")]
-				if strings.Index(aurl, ":") < 0 {
-					aurl = r.Endpoint.String() + aurl
-				}
-			} else {
-				aurl = ""
-			}
-		} else {
-			return repos, &commonhttp.Error{
-				Code:    resp.StatusCode,
-				Message: string(b),
-			}
-		}
-	}
-	return repos, nil
-}
-
-// Ping checks by Head method
-func (r *Registry) Ping() error {
-	return r.ping(http.MethodHead)
-}
-
-// PingGet checks by Get method
-func (r *Registry) PingGet() error {
-	return r.ping(http.MethodGet)
-}
-
-func (r *Registry) ping(method string) error {
-	req, err := http.NewRequest(method, buildPingURL(r.Endpoint.String()), nil)
-	if err != nil {
-		return err
-	}
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		return parseError(err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusOK {
-		return nil
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	return &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-}
-
-// PingSimple checks whether the registry is available. It checks the connectivity and certificate (if TLS enabled)
-// only, regardless of credential.
-func (r *Registry) PingSimple() error {
-	err := r.Ping()
-	if err == nil {
-		return nil
-	}
-	httpErr, ok := err.(*commonhttp.Error)
-	if !ok {
-		return err
-	}
-	if httpErr.Code == http.StatusUnauthorized ||
-		httpErr.Code == http.StatusForbidden {
-		return nil
-	}
-	return httpErr
-}

src/common/utils/registry/registry_test.go

@@ -1,142 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/url"
-	"strconv"
-	"testing"
-
-	"github.com/goharbor/harbor/src/common/utils/test"
-)
-
-func TestPing(t *testing.T) {
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  http.MethodHead,
-			Pattern: "/v2/",
-			Handler: test.Handler(nil),
-		})
-	defer server.Close()
-
-	client, err := newRegistryClient(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for registry: %v", err)
-	}
-
-	if err = client.Ping(); err != nil {
-		t.Errorf("failed to ping registry: %v", err)
-	}
-}
-
-func TestCatalog(t *testing.T) {
-	repositories := make([]string, 0, 1001)
-	for i := 0; i < 1001; i++ {
-		repositories = append(repositories, strconv.Itoa(i))
-	}
-
-	handler := func(w http.ResponseWriter, r *http.Request) {
-		q := r.URL.Query()
-		last := q.Get("last")
-		n, err := strconv.Atoi(q.Get("n"))
-		if err != nil || n <= 0 {
-			n = 1000
-		}
-
-		length := len(repositories)
-
-		begin := length
-		if len(last) == 0 {
-			begin = 0
-		} else {
-			for i, repository := range repositories {
-				if repository == last {
-					begin = i + 1
-					break
-				}
-			}
-		}
-
-		end := begin + n
-		if end > length {
-			end = length
-		}
-
-		w.Header().Set(http.CanonicalHeaderKey("Content-Type"), "application/json")
-		if end < length {
-			u, err := url.Parse("/v2/_catalog")
-			if err != nil {
-				w.WriteHeader(http.StatusInternalServerError)
-				return
-			}
-			values := u.Query()
-			values.Add("last", repositories[end-1])
-			values.Add("n", strconv.Itoa(n))
-
-			u.RawQuery = values.Encode()
-
-			link := fmt.Sprintf("<%s>; rel=\"next\"", u.String())
-			w.Header().Set(http.CanonicalHeaderKey("link"), link)
-		}
-
-		repos := struct {
-			Repositories []string `json:"repositories"`
-		}{
-			Repositories: []string{},
-		}
-
-		if begin < length {
-			repos.Repositories = repositories[begin:end]
-		}
-
-		b, err := json.Marshal(repos)
-		if err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-
-		w.Write(b)
-
-	}
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "GET",
-			Pattern: "/v2/_catalog",
-			Handler: handler,
-		})
-	defer server.Close()
-
-	client, err := newRegistryClient(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for registry: %v", err)
-	}
-
-	repos, err := client.Catalog()
-	if err != nil {
-		t.Fatalf("failed to catalog repositories: %v", err)
-	}
-
-	if len(repos) != len(repositories) {
-		t.Errorf("unexpected length of repositories: %d != %d", len(repos), len(repositories))
-	}
-}
-
-func newRegistryClient(url string) (*Registry, error) {
-	return NewRegistry(url, &http.Client{})
-}

src/common/utils/registry/repository.go

@@ -1,535 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"sort"
-	"strconv"
-	"strings"
-
-	"github.com/docker/distribution/manifest/schema1"
-	"github.com/docker/distribution/manifest/schema2"
-	commonhttp "github.com/goharbor/harbor/src/common/http"
-	"github.com/goharbor/harbor/src/common/utils"
-)
-
-// Repository holds information of a repository entity
-type Repository struct {
-	Name     string
-	Endpoint *url.URL
-	client   *http.Client
-}
-
-// NewRepository returns an instance of Repository
-func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) {
-	name = strings.TrimSpace(name)
-
-	u, err := utils.ParseEndpoint(endpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	repository := &Repository{
-		Name:     name,
-		Endpoint: u,
-		client:   client,
-	}
-
-	return repository, nil
-}
-
-func parseError(err error) error {
-	if urlErr, ok := err.(*url.Error); ok {
-		if regErr, ok := urlErr.Err.(*commonhttp.Error); ok {
-			return regErr
-		}
-	}
-	return err
-}
-
-// ListTag ...
-func (r *Repository) ListTag() ([]string, error) {
-	tags := []string{}
-	aurl := buildTagListURL(r.Endpoint.String(), r.Name)
-
-	for len(aurl) > 0 {
-		req, err := http.NewRequest("GET", aurl, nil)
-		if err != nil {
-			return tags, err
-		}
-		resp, err := r.client.Do(req)
-		if err != nil {
-			return nil, parseError(err)
-		}
-
-		defer resp.Body.Close()
-		b, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return tags, err
-		}
-
-		if resp.StatusCode == http.StatusOK {
-			tagsResp := struct {
-				Tags []string `json:"tags"`
-			}{}
-
-			if err := json.Unmarshal(b, &tagsResp); err != nil {
-				return tags, err
-			}
-
-			tags = append(tags, tagsResp.Tags...)
-			// Link: </v2/library/hello-world/tags/list?last=......>; rel="next"
-			// Link: <http://domain.com/v2/library/hello-world/tags/list?last=......>; rel="next"
-			link := resp.Header.Get("Link")
-			if strings.HasSuffix(link, `rel="next"`) && strings.Index(link, "<") >= 0 && strings.Index(link, ">") >= 0 {
-				aurl = link[strings.Index(link, "<")+1 : strings.Index(link, ">")]
-				if strings.Index(aurl, ":") < 0 {
-					aurl = r.Endpoint.String() + aurl
-				}
-			} else {
-				aurl = ""
-			}
-		} else if resp.StatusCode == http.StatusNotFound {
-
-			// TODO remove the logic if the bug of registry is fixed
-			// It's a workaround for a bug of registry: when listing tags of
-			// a repository which is being pushed, a "NAME_UNKNOWN" error will
-			// been returned, while the catalog API can list this repository.
-			return tags, nil
-		} else {
-			return tags, &commonhttp.Error{
-				Code:    resp.StatusCode,
-				Message: string(b),
-			}
-		}
-	}
-	sort.Strings(tags)
-	return tags, nil
-}
-
-// ManifestExist ...
-func (r *Repository) ManifestExist(reference string) (digest string, exist bool, err error) {
-	req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
-	if err != nil {
-		return
-	}
-
-	req.Header.Add(http.CanonicalHeaderKey("Accept"), schema1.MediaTypeManifest)
-	req.Header.Add(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		err = parseError(err)
-		return
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusOK {
-		exist = true
-		digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
-		return
-	}
-
-	if resp.StatusCode == http.StatusNotFound {
-		return
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	err = &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-	return
-}
-
-// PullManifest ...
-func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) {
-	req, err := http.NewRequest("GET", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
-	if err != nil {
-		return
-	}
-
-	for _, mediaType := range acceptMediaTypes {
-		req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
-	}
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		err = parseError(err)
-		return
-	}
-
-	defer resp.Body.Close()
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	if resp.StatusCode == http.StatusOK {
-		digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
-		mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
-		payload = b
-		return
-	}
-
-	err = &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-
-	return
-}
-
-// PushManifest ...
-func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (digest string, err error) {
-	req, err := http.NewRequest("PUT", buildManifestURL(r.Endpoint.String(), r.Name, reference),
-		bytes.NewReader(payload))
-	if err != nil {
-		return
-	}
-	req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType)
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		err = parseError(err)
-		return
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
-		digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
-		return
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	err = &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-
-	return
-}
-
-// DeleteManifest ...
-func (r *Repository) DeleteManifest(digest string) error {
-	req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), r.Name, digest), nil)
-	if err != nil {
-		return err
-	}
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		return parseError(err)
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusAccepted {
-		return nil
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	return &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-}
-
-// MountBlob ...
-func (r *Repository) MountBlob(digest, from string) error {
-	req, err := http.NewRequest("POST", buildMountBlobURL(r.Endpoint.String(), r.Name, digest, from), nil)
-	req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode/100 != 2 {
-		defer resp.Body.Close()
-		b, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return err
-		}
-		return &commonhttp.Error{
-			Code:    resp.StatusCode,
-			Message: string(b),
-		}
-	}
-
-	return nil
-}
-
-// DeleteTag ...
-func (r *Repository) DeleteTag(tag string) error {
-	digest, exist, err := r.ManifestExist(tag)
-	if err != nil {
-		return err
-	}
-
-	if !exist {
-		return &commonhttp.Error{
-			Code: http.StatusNotFound,
-		}
-	}
-
-	return r.DeleteManifest(digest)
-}
-
-// BlobExist ...
-func (r *Repository) BlobExist(digest string) (bool, error) {
-	req, err := http.NewRequest("HEAD", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
-	if err != nil {
-		return false, err
-	}
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		return false, parseError(err)
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusOK {
-		return true, nil
-	}
-
-	if resp.StatusCode == http.StatusNotFound {
-		return false, nil
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return false, err
-	}
-
-	return false, &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-}
-
-// PullBlob : client must close data if it is not nil
-func (r *Repository) PullBlob(digest string) (size int64, data io.ReadCloser, err error) {
-	req, err := http.NewRequest("GET", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
-	if err != nil {
-		return
-	}
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		err = parseError(err)
-		return
-	}
-
-	if resp.StatusCode == http.StatusOK {
-		contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
-		size, err = strconv.ParseInt(contengLength, 10, 64)
-		if err != nil {
-			return
-		}
-		data = resp.Body
-		return
-	}
-	// can not close the connect if the status code is 200
-	defer resp.Body.Close()
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	err = &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-
-	return
-}
-
-func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID string, err error) {
-	req, err := http.NewRequest("POST", buildInitiateBlobUploadURL(r.Endpoint.String(), r.Name), nil)
-	req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		err = parseError(err)
-		return
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusAccepted {
-		location = resp.Header.Get(http.CanonicalHeaderKey("Location"))
-		uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID"))
-		return
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	err = &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-
-	return
-}
-
-func (r *Repository) monolithicBlobUpload(location, digest string, size int64, data io.Reader) error {
-	url, err := buildMonolithicBlobUploadURL(r.Endpoint.String(), location, digest)
-	if err != nil {
-		return err
-	}
-	req, err := http.NewRequest("PUT", url, data)
-	if err != nil {
-		return err
-	}
-	req.ContentLength = size
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		return parseError(err)
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusCreated {
-		return nil
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	return &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-}
-
-// PushBlob ...
-func (r *Repository) PushBlob(digest string, size int64, data io.Reader) error {
-	location, _, err := r.initiateBlobUpload(r.Name)
-	if err != nil {
-		return err
-	}
-	return r.monolithicBlobUpload(location, digest, size, data)
-}
-
-// DeleteBlob ...
-func (r *Repository) DeleteBlob(digest string) error {
-	req, err := http.NewRequest("DELETE", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
-	if err != nil {
-		return err
-	}
-
-	resp, err := r.client.Do(req)
-	if err != nil {
-		return parseError(err)
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusAccepted {
-		return nil
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	return &commonhttp.Error{
-		Code:    resp.StatusCode,
-		Message: string(b),
-	}
-}
-
-func buildPingURL(endpoint string) string {
-	return fmt.Sprintf("%s/v2/", endpoint)
-}
-
-func buildTagListURL(endpoint, repoName string) string {
-	return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName)
-}
-
-func buildManifestURL(endpoint, repoName, reference string) string {
-	return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference)
-}
-
-func buildBlobURL(endpoint, repoName, reference string) string {
-	return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repoName, reference)
-}
-
-func buildMountBlobURL(endpoint, repoName, digest, from string) string {
-	return fmt.Sprintf("%s/v2/%s/blobs/uploads/?mount=%s&from=%s", endpoint, repoName, digest, from)
-}
-
-func buildInitiateBlobUploadURL(endpoint, repoName string) string {
-	return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repoName)
-}
-
-func buildMonolithicBlobUploadURL(endpoint, location, digest string) (string, error) {
-	relative, err := isRelativeURL(location)
-	if err != nil {
-		return "", err
-	}
-	// when the registry enables "relativeurls", the location returned
-	// has no scheme and host part
-	if relative {
-		location = endpoint + location
-	}
-	query := ""
-	if strings.ContainsRune(location, '?') {
-		query = "&"
-	} else {
-		query = "?"
-	}
-	query += fmt.Sprintf("digest=%s", digest)
-	return fmt.Sprintf("%s%s", location, query), nil
-}
-
-func isRelativeURL(endpoint string) (bool, error) {
-	u, err := url.Parse(endpoint)
-	if err != nil {
-		return false, err
-	}
-	return !u.IsAbs(), nil
-}

src/common/utils/registry/repository_test.go

@@ -1,458 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/docker/distribution/manifest/schema2"
-	commonhttp "github.com/goharbor/harbor/src/common/http"
-	"github.com/goharbor/harbor/src/common/utils/test"
-)
-
-var (
-	repository = "library/hello-world"
-	tag        = "latest"
-
-	mediaType = schema2.MediaTypeManifest
-	manifest  = []byte("manifest")
-
-	blob = []byte("blob")
-
-	uuid = "0663ff44-63bb-11e6-8b77-86f30ca893d3"
-
-	digest = "sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b"
-)
-
-func TestBlobExist(t *testing.T) {
-	handler := func(w http.ResponseWriter, r *http.Request) {
-		path := r.URL.Path
-		dgt := path[strings.LastIndex(path, "/")+1:]
-		if dgt == digest {
-			w.Header().Add(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(blob)))
-			w.Header().Add(http.CanonicalHeaderKey("Docker-Content-Digest"), digest)
-			w.Header().Add(http.CanonicalHeaderKey("Content-Type"), "application/octet-stream")
-			return
-		}
-
-		w.WriteHeader(http.StatusNotFound)
-	}
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "HEAD",
-			Pattern: fmt.Sprintf("/v2/%s/blobs/", repository),
-			Handler: handler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		err = parseError(err)
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	exist, err := client.BlobExist(digest)
-	if err != nil {
-		t.Fatalf("failed to check the existence of blob: %v", err)
-	}
-
-	if !exist {
-		t.Errorf("blob should exist on registry, but it does not exist")
-	}
-
-	exist, err = client.BlobExist("invalid_digest")
-	if err != nil {
-		t.Fatalf("failed to check the existence of blob: %v", err)
-	}
-
-	if exist {
-		t.Errorf("blob should not exist on registry, but it exists")
-	}
-}
-
-func TestPullBlob(t *testing.T) {
-	handler := test.Handler(&test.Response{
-		Headers: map[string]string{
-			"Content-Length":        strconv.Itoa(len(blob)),
-			"Docker-Content-Digest": digest,
-			"Content-Type":          "application/octet-stream",
-		},
-		Body: blob,
-	})
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "GET",
-			Pattern: fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
-			Handler: handler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	size, reader, err := client.PullBlob(digest)
-	if err != nil {
-		t.Fatalf("failed to pull blob: %v", err)
-	}
-
-	if size != int64(len(blob)) {
-		t.Errorf("unexpected size of blob: %d != %d", size, len(blob))
-	}
-
-	b, err := ioutil.ReadAll(reader)
-	if err != nil {
-		t.Fatalf("failed to read from reader: %v", err)
-	}
-
-	if bytes.Compare(b, blob) != 0 {
-		t.Errorf("unexpected blob: %s != %s", string(b), string(blob))
-	}
-}
-
-func TestPushBlob(t *testing.T) {
-	location := ""
-	initUploadHandler := func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Add(http.CanonicalHeaderKey("Content-Length"), "0")
-		w.Header().Add(http.CanonicalHeaderKey("Location"), location)
-		w.Header().Add(http.CanonicalHeaderKey("Range"), "0-0")
-		w.Header().Add(http.CanonicalHeaderKey("Docker-Upload-UUID"), uuid)
-		w.WriteHeader(http.StatusAccepted)
-	}
-
-	monolithicUploadHandler := test.Handler(&test.Response{
-		StatusCode: http.StatusCreated,
-		Headers: map[string]string{
-			"Content-Length":        "0",
-			"Location":              fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
-			"Docker-Content-Digest": digest,
-		},
-	})
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "POST",
-			Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/", repository),
-			Handler: initUploadHandler,
-		},
-		&test.RequestHandlerMapping{
-			Method:  "PUT",
-			Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/%s", repository, uuid),
-			Handler: monolithicUploadHandler,
-		})
-	defer server.Close()
-	location = fmt.Sprintf("%s/v2/%s/blobs/uploads/%s", server.URL, repository, uuid)
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	if err = client.PushBlob(digest, int64(len(blob)), bytes.NewReader(blob)); err != nil {
-		t.Fatalf("failed to push blob: %v", err)
-	}
-}
-
-func TestDeleteBlob(t *testing.T) {
-	handler := test.Handler(&test.Response{
-		StatusCode: http.StatusAccepted,
-	})
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "DELETE",
-			Pattern: fmt.Sprintf("/v2/%s/blobs/%s", repository, digest),
-			Handler: handler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	if err = client.DeleteBlob(digest); err != nil {
-		t.Fatalf("failed to delete blob: %v", err)
-	}
-}
-
-func TestManifestExist(t *testing.T) {
-	handler := func(w http.ResponseWriter, r *http.Request) {
-		path := r.URL.Path
-		tg := path[strings.LastIndex(path, "/")+1:]
-		if tg == tag {
-			w.Header().Add(http.CanonicalHeaderKey("Docker-Content-Digest"), digest)
-			w.Header().Add(http.CanonicalHeaderKey("Content-Type"), mediaType)
-			return
-		}
-
-		w.WriteHeader(http.StatusNotFound)
-	}
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "HEAD",
-			Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
-			Handler: handler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	d, exist, err := client.ManifestExist(tag)
-	if err != nil {
-		t.Fatalf("failed to check the existence of manifest: %v", err)
-	}
-
-	if !exist || d != digest {
-		t.Errorf("manifest should exist on registry, but it does not exist")
-	}
-
-	_, exist, err = client.ManifestExist("invalid_tag")
-	if err != nil {
-		t.Fatalf("failed to check the existence of manifest: %v", err)
-	}
-
-	if exist {
-		t.Errorf("manifest should not exist on registry, but it exists")
-	}
-}
-
-func TestPullManifest(t *testing.T) {
-	handler := test.Handler(&test.Response{
-		Headers: map[string]string{
-			"Docker-Content-Digest": digest,
-			"Content-Type":          mediaType,
-		},
-		Body: manifest,
-	})
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "GET",
-			Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
-			Handler: handler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	d, md, payload, err := client.PullManifest(tag, []string{mediaType})
-	if err != nil {
-		t.Fatalf("failed to pull manifest: %v", err)
-	}
-
-	if d != digest {
-		t.Errorf("unexpected digest of manifest: %s != %s", d, digest)
-	}
-
-	if md != mediaType {
-		t.Errorf("unexpected media type of manifest: %s != %s", md, mediaType)
-	}
-
-	if bytes.Compare(payload, manifest) != 0 {
-		t.Errorf("unexpected manifest: %s != %s", string(payload), string(manifest))
-	}
-}
-
-func TestPushManifest(t *testing.T) {
-	handler := test.Handler(&test.Response{
-		StatusCode: http.StatusCreated,
-		Headers: map[string]string{
-			"Content-Length":        "0",
-			"Docker-Content-Digest": digest,
-			"Location":              "",
-		},
-	})
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "PUT",
-			Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, tag),
-			Handler: handler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	d, err := client.PushManifest(tag, mediaType, manifest)
-	if err != nil {
-		t.Fatalf("failed to pull manifest: %v", err)
-	}
-
-	if d != digest {
-		t.Errorf("unexpected digest of manifest: %s != %s", d, digest)
-	}
-}
-
-func TestDeleteTag(t *testing.T) {
-	manifestExistHandler := test.Handler(&test.Response{
-		Headers: map[string]string{
-			"Docker-Content-Digest": digest,
-			"Content-Type":          mediaType,
-		},
-	})
-
-	deleteManifestandler := test.Handler(&test.Response{
-		StatusCode: http.StatusAccepted,
-	})
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "HEAD",
-			Pattern: fmt.Sprintf("/v2/%s/manifests/", repository),
-			Handler: manifestExistHandler,
-		},
-		&test.RequestHandlerMapping{
-			Method:  "DELETE",
-			Pattern: fmt.Sprintf("/v2/%s/manifests/%s", repository, digest),
-			Handler: deleteManifestandler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	if err = client.DeleteTag(tag); err != nil {
-		t.Fatalf("failed to delete tag: %v", err)
-	}
-}
-
-func TestListTag(t *testing.T) {
-	handler := test.Handler(&test.Response{
-		Headers: map[string]string{
-			"Content-Type": "application/json",
-		},
-		Body: []byte(fmt.Sprintf("{\"name\": \"%s\",\"tags\": [\"%s\"]}", repository, tag)),
-	})
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "GET",
-			Pattern: fmt.Sprintf("/v2/%s/tags/list", repository),
-			Handler: handler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	tags, err := client.ListTag()
-	if err != nil {
-		t.Fatalf("failed to list tags: %v", err)
-	}
-
-	if len(tags) != 1 {
-		t.Fatalf("unexpected length of tags: %d != %d", len(tags), 1)
-	}
-
-	if tags[0] != tag {
-		t.Errorf("unexpected tag: %s != %s", tags[0], tag)
-	}
-}
-
-func TestParseError(t *testing.T) {
-	err := &url.Error{
-		Err: &commonhttp.Error{},
-	}
-	e := parseError(err)
-	if _, ok := e.(*commonhttp.Error); !ok {
-		t.Errorf("error type does not match registry error")
-	}
-}
-
-func newRepository(endpoint string) (*Repository, error) {
-	return NewRepository(repository, endpoint, &http.Client{})
-}
-
-func TestBuildMonolithicBlobUploadURL(t *testing.T) {
-	endpoint := "http://192.169.0.1"
-	digest := "sha256:ef15416724f6e2d5d5b422dc5105add931c1f2a45959cd4993e75e47957b3b55"
-
-	// absolute URL
-	location := "http://192.169.0.1/v2/library/golang/blobs/uploads/c9f84fd7-0198-43e3-80a7-dd13771cd7f0?_state=GabyCujPu0dpxiY8yYZTq"
-	expected := location + "&digest=" + digest
-	url, err := buildMonolithicBlobUploadURL(endpoint, location, digest)
-	require.Nil(t, err)
-	assert.Equal(t, expected, url)
-
-	// relative URL
-	location = "/v2/library/golang/blobs/uploads/c9f84fd7-0198-43e3-80a7-dd13771cd7f0?_state=GabyCujPu0dpxiY8yYZTq"
-	expected = endpoint + location + "&digest=" + digest
-	url, err = buildMonolithicBlobUploadURL(endpoint, location, digest)
-	require.Nil(t, err)
-	assert.Equal(t, expected, url)
-}
-
-func TestBuildMountBlobURL(t *testing.T) {
-	endpoint := "http://192.169.0.1"
-	repoName := "library/hello-world"
-	digest := "sha256:ef15416724f6e2d5d5b422dc5105add931c1f2a45959cd4993e75e47957b3b55"
-	from := "library/hi-world"
-	expected := fmt.Sprintf("%s/v2/%s/blobs/uploads/?mount=%s&from=%s", endpoint, repoName, digest, from)
-
-	actual := buildMountBlobURL(endpoint, repoName, digest, from)
-	assert.Equal(t, expected, actual)
-}
-
-func TestMountBlob(t *testing.T) {
-	mountHandler := func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusAccepted)
-	}
-
-	server := test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  "POST",
-			Pattern: fmt.Sprintf("/v2/%s/blobs/uploads/", repository),
-			Handler: mountHandler,
-		})
-	defer server.Close()
-
-	client, err := newRepository(server.URL)
-	if err != nil {
-		t.Fatalf("failed to create client for repository: %v", err)
-	}
-
-	if err = client.MountBlob(digest, "library/hi-world"); err != nil {
-		t.Fatalf("failed to mount blob: %v", err)
-	}
-}

src/core/api/admin_job.go

@@ -133,7 +133,7 @@ func (aj *AJAPI) list(name string) {
 
 // getSchedule gets admin job schedule ...
 func (aj *AJAPI) getSchedule(name string) {
-	adminJobSchedule := models.AdminJobSchedule{}
+	result := models.AdminJobRep{}
 
 	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
 		Name: name,
@@ -154,10 +154,11 @@ func (aj *AJAPI) getSchedule(name string) {
 			aj.SendInternalServerError(fmt.Errorf("failed to convert admin job response: %v", err))
 			return
 		}
-		adminJobSchedule.Schedule = adminJobRep.Schedule
+		result.Schedule = adminJobRep.Schedule
+		result.Parameters = adminJobRep.Parameters
 	}
 
-	aj.Data["json"] = adminJobSchedule
+	aj.Data["json"] = result
 	aj.ServeJSON()
 }
 
@@ -285,9 +286,10 @@ func (aj *AJAPI) submit(ajr *models.AdminJobReq) {
 	}
 
 	id, err := dao.AddAdminJob(&common_models.AdminJob{
-		Name: ajr.Name,
+		Name:       ajr.Name,
-		Kind: ajr.JobKind(),
+		Kind:       ajr.JobKind(),
-		Cron: ajr.CronString(),
+		Cron:       ajr.CronString(),
+		Parameters: ajr.ParamString(),
 	})
 	if err != nil {
 		aj.SendInternalServerError(err)
@@ -345,6 +347,7 @@ func convertToAdminJobRep(job *common_models.AdminJob) (models.AdminJobRep, erro
 		Name:         job.Name,
 		Kind:         job.Kind,
 		Status:       job.Status,
+		Parameters:   job.Parameters,
 		CreationTime: job.CreationTime,
 		UpdateTime:   job.UpdateTime,
 	}

src/core/api/chart_label_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/goharbor/harbor/src/chartserver"
 	"github.com/goharbor/harbor/src/common"
+	"github.com/goharbor/harbor/src/common/api"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/stretchr/testify/assert"
@@ -29,9 +30,9 @@ import (
 )
 
 var (
-	resourceLabelAPIPath                = "/api/chartrepo/library/charts/harbor/0.2.0/labels"
+	resourceLabelAPIPath                = fmt.Sprintf("/api/%s/chartrepo/library/charts/harbor/0.2.0/labels", api.APIVersion)
-	resourceLabelAPIPathWithFakeProject = "/api/chartrepo/not-exist/charts/harbor/0.2.0/labels"
+	resourceLabelAPIPathWithFakeProject = fmt.Sprintf("/api/%s/chartrepo/not-exist/charts/harbor/0.2.0/labels", api.APIVersion)
-	resourceLabelAPIPathWithFakeChart   = "/api/chartrepo/library/charts/not-exist/0.2.0/labels"
+	resourceLabelAPIPathWithFakeChart   = fmt.Sprintf("/api/%s/chartrepo/library/charts/not-exist/0.2.0/labels", api.APIVersion)
 	cProLibraryLabelID                  int64
 	mockChartServer                     *httptest.Server
 	oldChartController                  *chartserver.Controller

src/core/api/chart_repository.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/goharbor/harbor/src/chartserver"
 	"github.com/goharbor/harbor/src/common"
+	"github.com/goharbor/harbor/src/common/api"
 	"github.com/goharbor/harbor/src/common/rbac"
 	hlog "github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/config"
@@ -28,13 +29,9 @@ import (
 )
 
 const (
-	namespaceParam          = ":repo"
+	namespaceParam = ":repo"
-	nameParam               = ":name"
+	nameParam      = ":name"
-	filenameParam           = ":filename"
+	filenameParam  = ":filename"
-	defaultRepo             = "library"
-	rootUploadingEndpoint   = "/api/chartrepo/charts"
-	rootIndexEndpoint       = "/chartrepo/index.yaml"
-	chartRepoHealthEndpoint = "/api/chartrepo/health"
 
 	accessLevelPublic = iota
 	accessLevelRead
@@ -50,6 +47,13 @@ const (
 	chartPackageFileExtension = "tgz"
 )
 
+var (
+	defaultRepo             = "library"
+	rootUploadingEndpoint   = fmt.Sprintf("/api/%s/chartrepo/charts", api.APIVersion)
+	chartRepoHealthEndpoint = fmt.Sprintf("/api/%s/chartrepo/health", api.APIVersion)
+	rootIndexEndpoint       = "/chartrepo/index.yaml"
+)
+
 // chartController is a singleton instance
 var chartController *chartserver.Controller
 
@@ -108,7 +112,7 @@ func (cra *ChartRepositoryAPI) requireAccess(action rbac.Action, subresource ...
 	return cra.RequireProjectAccess(cra.namespace, action, subresource...)
 }
 
-// GetHealthStatus handles GET /api/chartrepo/health
+// GetHealthStatus handles GET /chartrepo/health
 func (cra *ChartRepositoryAPI) GetHealthStatus() {
 	// Check access
 	if !cra.SecurityCtx.IsAuthenticated() {

src/core/api/chart_repository_test.go

@@ -2,11 +2,13 @@ package api
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/goharbor/harbor/src/chartserver"
+	"github.com/goharbor/harbor/src/common/api"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/rbac"
 	"github.com/goharbor/harbor/src/core/promgr/metamgr"
@@ -18,7 +20,7 @@ var (
 )
 
 func TestIsMultipartFormData(t *testing.T) {
-	req, err := createRequest(http.MethodPost, "/api/chartrepo/charts")
+	req, err := createRequest(http.MethodPost, fmt.Sprintf("/api/%s/chartrepo/charts", api.APIVersion))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -56,7 +58,7 @@ func TestPrepareEnv(t *testing.T) {
 func TestGetHealthStatus(t *testing.T) {
 	status := make(map[string]interface{})
 	err := handleAndParse(&testingRequest{
-		url:        "/api/chartrepo/health",
+		url:        fmt.Sprintf("/api/%s/chartrepo/health", api.APIVersion),
 		method:     http.MethodGet,
 		credential: sysAdmin,
 	}, &status)
@@ -110,7 +112,7 @@ func TestDownloadChart(t *testing.T) {
 func TesListCharts(t *testing.T) {
 	charts := make([]*chartserver.ChartInfo, 0)
 	err := handleAndParse(&testingRequest{
-		url:        "/api/chartrepo/library/charts",
+		url:        fmt.Sprintf("/api/%s/chartrepo/library/charts", api.APIVersion),
 		method:     http.MethodGet,
 		credential: projAdmin,
 	}, &charts)
@@ -128,7 +130,7 @@ func TesListCharts(t *testing.T) {
 func TestListChartVersions(t *testing.T) {
 	chartVersions := make(chartserver.ChartVersions, 0)
 	err := handleAndParse(&testingRequest{
-		url:        "/api/chartrepo/library/charts/harbor",
+		url:        fmt.Sprintf("/api/%s/chartrepo/library/charts/harbor", api.APIVersion),
 		method:     http.MethodGet,
 		credential: projAdmin,
 	}, &chartVersions)
@@ -146,7 +148,7 @@ func TestListChartVersions(t *testing.T) {
 func TestGetChartVersion(t *testing.T) {
 	chartV := &chartserver.ChartVersionDetails{}
 	err := handleAndParse(&testingRequest{
-		url:        "/api/chartrepo/library/charts/harbor/0.2.0",
+		url:        fmt.Sprintf("/api/%s/chartrepo/library/charts/harbor/0.2.0", api.APIVersion),
 		method:     http.MethodGet,
 		credential: projAdmin,
 	}, chartV)
@@ -168,7 +170,7 @@ func TestGetChartVersion(t *testing.T) {
 func TestDeleteChartVersion(t *testing.T) {
 	runCodeCheckingCases(t, &codeCheckingCase{
 		request: &testingRequest{
-			url:        "/api/chartrepo/library/charts/harbor/0.2.1",
+			url:        fmt.Sprintf("/api/%s/chartrepo/library/charts/harbor/0.2.1", api.APIVersion),
 			method:     http.MethodDelete,
 			credential: projAdmin,
 		},
@@ -180,7 +182,7 @@ func TestDeleteChartVersion(t *testing.T) {
 func TestDeleteChart(t *testing.T) {
 	runCodeCheckingCases(t, &codeCheckingCase{
 		request: &testingRequest{
-			url:        "/api/chartrepo/library/charts/harbor",
+			url:        fmt.Sprintf("/api/%s/chartrepo/library/charts/harbor", api.APIVersion),
 			method:     http.MethodDelete,
 			credential: projAdmin,
 		},

src/core/api/harborapi_test.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/astaxie/beego"
 	"github.com/dghubble/sling"
+	"github.com/goharbor/harbor/src/common/api"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/job/test"
 	"github.com/goharbor/harbor/src/common/models"
@@ -113,21 +114,10 @@ func init() {
 	beego.Router("/api/projects/:id([0-9]+)/metadatas/", &MetadataAPI{}, "post:Post")
 	beego.Router("/api/projects/:id([0-9]+)/metadatas/:name", &MetadataAPI{}, "put:Put;delete:Delete")
 	beego.Router("/api/projects/:pid([0-9]+)/members/?:pmid([0-9]+)", &ProjectMemberAPI{})
-	beego.Router("/api/repositories", &RepositoryAPI{})
 	beego.Router("/api/statistics", &StatisticAPI{})
 	beego.Router("/api/users/?:id", &UserAPI{})
 	beego.Router("/api/usergroups/?:ugid([0-9]+)", &UserGroupAPI{})
 	beego.Router("/api/logs", &LogAPI{})
-	beego.Router("/api/repositories/*", &RepositoryAPI{}, "put:Put")
-	beego.Router("/api/repositories/*/labels", &RepositoryLabelAPI{}, "get:GetOfRepository;post:AddToRepository")
-	beego.Router("/api/repositories/*/labels/:id([0-9]+", &RepositoryLabelAPI{}, "delete:RemoveFromRepository")
-	beego.Router("/api/repositories/*/tags/:tag/labels", &RepositoryLabelAPI{}, "get:GetOfImage;post:AddToImage")
-	beego.Router("/api/repositories/*/tags/:tag/labels/:id([0-9]+", &RepositoryLabelAPI{}, "delete:RemoveFromImage")
-	beego.Router("/api/repositories/*/tags/:tag", &RepositoryAPI{}, "delete:Delete;get:GetTag")
-	beego.Router("/api/repositories/*/tags", &RepositoryAPI{}, "get:GetTags;post:Retag")
-	beego.Router("/api/repositories/*/tags/:tag/manifest", &RepositoryAPI{}, "get:GetManifests")
-	beego.Router("/api/repositories/*/signatures", &RepositoryAPI{}, "get:GetSignatures")
-	beego.Router("/api/repositories/top", &RepositoryAPI{}, "get:GetTopRepos")
 	beego.Router("/api/registries", &RegistryAPI{}, "get:List;post:Post")
 	beego.Router("/api/registries/ping", &RegistryAPI{}, "post:Ping")
 	beego.Router("/api/registries/:id([0-9]+)", &RegistryAPI{}, "get:Get;put:Put;delete:Delete")
@@ -183,15 +173,15 @@ func init() {
 	beego.Router("/api/projects/:pid([0-9]+)/immutabletagrules/:id([0-9]+)", &ImmutableTagRuleAPI{})
 	// Charts are controlled under projects
 	chartRepositoryAPIType := &ChartRepositoryAPI{}
-	beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")
-	beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts")
-	beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions")
-	beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "delete:DeleteChart")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "delete:DeleteChart")
-	beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion")
-	beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion")
-	beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
-	beego.Router("/api/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile")
-	beego.Router("/api/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
 
 	// Repository services
 	beego.Router("/chartrepo/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo")
@@ -199,8 +189,8 @@ func init() {
 	beego.Router("/chartrepo/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart")
 	// Labels for chart
 	chartLabelAPIType := &ChartLabelAPI{}
-	beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel")
-	beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel")
+	beego.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel")
 
 	quotaAPIType := &QuotaAPI{}
 	beego.Router("/api/quotas", quotaAPIType, "get:List")
@@ -226,11 +216,6 @@ func init() {
 	beego.Router("/api/repositories/*/tags/:tag/scan", scanAPI, "post:Scan;get:Report")
 	beego.Router("/api/repositories/*/tags/:tag/scan/:uuid/log", scanAPI, "get:Log")
 
-	// syncRegistry
-	if err := SyncRegistry(config.GlobalProjectMgr); err != nil {
-		log.Fatalf("failed to sync repositories from registry: %v", err)
-	}
-
 	if err := quota.Sync(config.GlobalProjectMgr, false); err != nil {
 		log.Fatalf("failed to sync quota from backend: %v", err)
 	}
@@ -357,48 +342,6 @@ func (a testapi) LogGet(user usrInfo) (int, []apilib.AccessLog, error) {
 	return code, successPayload, err
 }
 
-// // Delete a repository or a tag in a repository.
-// // Delete a repository or a tag in a repository.
-// // This endpoint let user delete repositories and tags with repo name and tag.\n
-// // @param repoName The name of repository which will be deleted.
-// // @param tag Tag of a repository.
-// // @return void
-// // func (a testapi) RepositoriesDelete(prjUsr UsrInfo, repoName string, tag string) (int, error) {
-// func (a testapi) RepositoriesDelete(prjUsr UsrInfo, repoName string, tag string) (int, error) {
-//	_sling := sling.New().Delete(a.basePath)
-
-//	// create path and map variables
-//	path := "/api/repositories"
-
-//	_sling = _sling.Path(path)
-
-//	type QueryParams struct {
-//		RepoName string `url:"repo_name,omitempty"`
-//		Tag      string `url:"tag,omitempty"`
-//	}
-
-//	_sling = _sling.QueryStruct(&QueryParams{RepoName: repoName, Tag: tag})
-//	// accept header
-//	accepts := []string{"application/json", "text/plain"}
-//	for key := range accepts {
-//		_sling = _sling.Set("Accept", accepts[key])
-//		break // only use the first Accept
-//	}
-
-//	req, err := _sling.Request()
-//	req.SetBasicAuth(prjUsr.Name, prjUsr.Passwd)
-//	// fmt.Printf("request %+v", req)
-
-//	client := &http.Client{}
-//	httpResponse, err := client.Do(req)
-//	defer httpResponse.Body.Close()
-
-//	if err != nil {
-//		// handle error
-//	}
-//	return httpResponse.StatusCode, err
-// }
-
 // Delete project by projectID
 func (a testapi) ProjectsDelete(prjUsr usrInfo, projectID string) (int, error) {
 	_sling := sling.New().Delete(a.basePath)
@@ -609,140 +552,6 @@ func (a testapi) PutProjectMember(authInfo usrInfo, projectID string, userID str
 	return httpStatusCode, err
 }
 
-// -------------------------Repositories Test---------------------------------------//
-// Return relevant repos of projectID
-func (a testapi) GetRepos(authInfo usrInfo, projectID, keyword string) (
-	int, interface{}, error) {
-	_sling := sling.New().Get(a.basePath)
-
-	path := "/api/repositories/"
-
-	_sling = _sling.Path(path)
-
-	type QueryParams struct {
-		ProjectID string `url:"project_id"`
-		Keyword   string `url:"q"`
-	}
-
-	_sling = _sling.QueryStruct(&QueryParams{
-		ProjectID: projectID,
-		Keyword:   keyword,
-	})
-	code, body, err := request(_sling, jsonAcceptHeader, authInfo)
-	if err != nil {
-		return 0, nil, err
-	}
-
-	if code == http.StatusOK {
-		repositories := []repoResp{}
-		if err = json.Unmarshal(body, &repositories); err != nil {
-			return 0, nil, err
-		}
-		return code, repositories, nil
-	}
-
-	return code, nil, nil
-}
-
-func (a testapi) GetTag(authInfo usrInfo, repository string, tag string) (int, *models.TagResp, error) {
-	_sling := sling.New().Get(a.basePath).Path(fmt.Sprintf("/api/repositories/%s/tags/%s", repository, tag))
-	code, data, err := request(_sling, jsonAcceptHeader, authInfo)
-	if err != nil {
-		return 0, nil, err
-	}
-
-	if code != http.StatusOK {
-		log.Printf("failed to get tag of %s:%s: %d %s \n", repository, tag, code, string(data))
-		return code, nil, nil
-	}
-
-	result := models.TagResp{}
-	if err := json.Unmarshal(data, &result); err != nil {
-		return 0, nil, err
-	}
-	return http.StatusOK, &result, nil
-}
-
-// Get tags of a relevant repository
-func (a testapi) GetReposTags(authInfo usrInfo, repoName string) (int, interface{}, error) {
-	_sling := sling.New().Get(a.basePath)
-
-	path := fmt.Sprintf("/api/repositories/%s/tags", repoName)
-
-	_sling = _sling.Path(path)
-
-	httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
-	if err != nil {
-		return 0, nil, err
-	}
-
-	if httpStatusCode != http.StatusOK {
-		return httpStatusCode, body, nil
-	}
-
-	result := []models.TagResp{}
-	if err := json.Unmarshal(body, &result); err != nil {
-		return 0, nil, err
-	}
-	return http.StatusOK, result, nil
-}
-
-// RetagImage retag image to another tag
-func (a testapi) RetagImage(authInfo usrInfo, repoName string, retag *apilib.Retag) (int, error) {
-	_sling := sling.New().Post(a.basePath)
-
-	path := fmt.Sprintf("/api/repositories/%s/tags", repoName)
-
-	_sling = _sling.Path(path)
-	_sling = _sling.BodyJSON(retag)
-
-	httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
-	return httpStatusCode, err
-}
-
-// Get manifests of a relevant repository
-func (a testapi) GetReposManifests(authInfo usrInfo, repoName string, tag string) (int, error) {
-	_sling := sling.New().Get(a.basePath)
-
-	path := fmt.Sprintf("/api/repositories/%s/tags/%s/manifest", repoName, tag)
-
-	_sling = _sling.Path(path)
-
-	httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
-	return httpStatusCode, err
-}
-
-// Get public repositories which are accessed most
-func (a testapi) GetReposTop(authInfo usrInfo, count string) (int, interface{}, error) {
-	_sling := sling.New().Get(a.basePath)
-
-	path := "/api/repositories/top"
-
-	_sling = _sling.Path(path)
-
-	type QueryParams struct {
-		Count string `url:"count"`
-	}
-
-	_sling = _sling.QueryStruct(&QueryParams{
-		Count: count,
-	})
-	code, body, err := request(_sling, jsonAcceptHeader, authInfo)
-	if err != nil {
-		return 0, nil, err
-	}
-
-	if code != http.StatusOK {
-		return code, body, err
-	}
-
-	result := []*repoResp{}
-	if err = json.Unmarshal(body, &result); err != nil {
-		return 0, nil, err
-	}
-	return http.StatusOK, result, nil
-}
-
 // --------------------Replication_Policy Test--------------------------------//
 
 // Create a new replication policy
@@ -836,54 +645,6 @@ func (a testapi) DeletePolicyByID(authInfo usrInfo, policyID string) (int, error
 	return httpStatusCode, err
 }
 
-// Return projects created by Harbor
-// func (a HarborApi) ProjectsGet (projectName string, isPublic int32) ([]Project, error) {
-//    }
-
-// Check if the project name user provided already exists.
-// func (a HarborApi) ProjectsHead (projectName string) (error) {
-// }
-
-// Get access logs accompany with a relevant project.
-// func (a HarborApi) ProjectsProjectIdLogsFilterPost (projectID int32, accessLog AccessLog) ([]AccessLog, error) {
-// }
-
-// Return a project's relevant role members.
-// func (a HarborApi) ProjectsProjectIdMembersGet (projectID int32) ([]Role, error) {
-// }
-
-// Add project role member accompany with relevant project and user.
-// func (a HarborApi) ProjectsProjectIdMembersPost (projectID int32, roles RoleParam) (error) {
-// }
-
-// Delete project role members accompany with relevant project and user.
-// func (a HarborApi) ProjectsProjectIdMembersUserIdDelete (projectID int32, userId int32) (error) {
-// }
-
-// Return role members accompany with relevant project and user.
-// func (a HarborApi) ProjectsProjectIdMembersUserIdGet (projectID int32, userId int32) ([]Role, error) {
-// }
-
-// Update project role members accompany with relevant project and user.
-// func (a HarborApi) ProjectsProjectIdMembersUserIdPut (projectID int32, userId int32, roles RoleParam) (error) {
-// }
-
-// Update properties for a selected project.
-// func (a HarborApi) ProjectsProjectIdPut (projectID int32, project Project) (error) {
-// }
-
-// Get repositories accompany with relevant project and repo name.
-// func (a HarborApi) RepositoriesGet (projectID int32, q string) ([]Repository, error) {
-// }
-
-// Get manifests of a relevant repository.
-// func (a HarborApi) RepositoriesManifestGet (repoName string, tag string) (error) {
-// }
-
-// Get tags of a relevant repository.
-// func (a HarborApi) RepositoriesTagsGet (repoName string) (error) {
-// }
-
 // Get registered users of Harbor.
 func (a testapi) UsersGet(userName string, authInfo usrInfo) (int, []apilib.User, error) {
 	_sling := sling.New().Get(a.basePath)

src/core/api/internal.go

@@ -47,15 +47,6 @@ func (ia *InternalAPI) Prepare() {
 	}
 }
 
-// SyncRegistry ...
-func (ia *InternalAPI) SyncRegistry() {
-	err := SyncRegistry(ia.ProjectMgr)
-	if err != nil {
-		ia.SendInternalServerError(err)
-		return
-	}
-}
-
 // RenameAdmin we don't provide flexibility in this API, as this is a workaround.
 func (ia *InternalAPI) RenameAdmin() {
 	if !dao.IsSuperUser(ia.SecurityCtx.GetUsername()) {

src/core/api/models/admin_job.go

@@ -73,6 +73,7 @@ type AdminJobRep struct {
 	ID           int64     `json:"id"`
 	Name         string    `json:"job_name"`
 	Kind         string    `json:"job_kind"`
+	Parameters   string    `json:"job_parameters"`
 	Status       string    `json:"job_status"`
 	UUID         string    `json:"-"`
 	Deleted      bool      `json:"deleted"`
@@ -151,6 +152,16 @@ func (ar *AdminJobReq) CronString() string {
 	return string(str)
 }
 
+// ParamString ...
+func (ar *AdminJobReq) ParamString() string {
+	str, err := json.Marshal(ar.Parameters)
+	if err != nil {
+		log.Debugf("failed to marshal json error, %v", err)
+		return ""
+	}
+	return string(str)
+}
+
 // ConvertSchedule converts different kinds of cron string into one standard for UI to show.
 // in the latest design, it uses {"type":"Daily","cron":"0 0 0 * * *"} as the cron item.
 // As for supporting migration from older version, it needs to convert {"parameter":{"daily_time":0},"type":"daily"}

src/core/api/models/admin_job_test.go

@@ -138,6 +138,19 @@ func TestCronString(t *testing.T) {
 	assert.True(t, strings.EqualFold(cronStr, "{\"type\":\"Daily\",\"Cron\":\"20 3 0 * * *\"}"))
 }
 
+func TestParamString(t *testing.T) {
+	adminJobPara := make(map[string]interface{})
+	adminJobPara["key1"] = "value1"
+	adminJobPara["key2"] = true
+	adminJobPara["key3"] = 88
+
+	adminjob := &AdminJobReq{
+		Parameters: adminJobPara,
+	}
+	paramStr := adminjob.ParamString()
+	assert.True(t, strings.EqualFold(paramStr, "{\"key1\":\"value1\",\"key2\":true,\"key3\":88}"))
+}
+
 func TestConvertSchedule(t *testing.T) {
 	schedule1 := "{\"type\":\"Daily\",\"cron\":\"20 3 0 * * *\"}"
 	converted1, err1 := ConvertSchedule(schedule1)

src/core/api/quota/registry/registry.go

@@ -15,23 +15,19 @@
 package registry
 
 import (
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/docker/distribution/manifest/schema1"
-	"github.com/docker/distribution/manifest/schema2"
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/models"
 	common_quota "github.com/goharbor/harbor/src/common/quota"
 	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/common/utils/registry"
 	"github.com/goharbor/harbor/src/core/api"
 	quota "github.com/goharbor/harbor/src/core/api/quota"
 	"github.com/goharbor/harbor/src/core/promgr"
-	coreutils "github.com/goharbor/harbor/src/core/utils"
+	"github.com/goharbor/harbor/src/pkg/registry"
 	"github.com/pkg/errors"
+	"strings"
+	"sync"
+	"time"
 )
 
 // Migrator ...
@@ -60,7 +56,7 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
 		err      error
 	)
 
-	reposInRegistry, err := api.Catalog()
+	reposInRegistry, err := registry.Cli.Catalog()
 	if err != nil {
 		return nil, err
 	}
@@ -392,11 +388,7 @@ func infoOfProject(project string, repoList []string) (quota.ProjectInfo, error)
 }
 
 func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
-	repoClient, err := coreutils.NewRepositoryClientForUI("harbor-core", repo)
+	tags, err := registry.Cli.ListTags(repo)
-	if err != nil {
-		return quota.RepoData{}, err
-	}
-	tags, err := repoClient.ListTag()
 	if err != nil {
 		return quota.RepoData{}, err
 	}
@@ -405,11 +397,7 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
 	var blobs []*models.Blob
 
 	for _, tag := range tags {
-		_, mediaType, payload, err := repoClient.PullManifest(tag, []string{
+		manifest, digest, err := registry.Cli.PullManifest(repo, tag)
-			schema1.MediaTypeManifest,
-			schema1.MediaTypeSignedManifest,
-			schema2.MediaTypeManifest,
-		})
 		if err != nil {
 			log.Error(err)
 			// To workaround issue: https://github.com/goharbor/harbor/issues/9299, just log the error and do not raise it.
@@ -417,28 +405,27 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
 			// User still can view there images with size 0 in harbor.
 			continue
 		}
-		manifest, desc, err := registry.UnMarshal(mediaType, payload)
+		mediaType, payload, err := manifest.Payload()
 		if err != nil {
-			log.Error(err)
 			return quota.RepoData{}, err
 		}
 		// self
 		afnb := &models.ArtifactAndBlob{
-			DigestAF:   desc.Digest.String(),
+			DigestAF:   digest,
-			DigestBlob: desc.Digest.String(),
+			DigestBlob: digest,
 		}
 		afnbs = append(afnbs, afnb)
 		// add manifest as a blob.
 		blob := &models.Blob{
-			Digest:       desc.Digest.String(),
+			Digest:       digest,
-			ContentType:  desc.MediaType,
+			ContentType:  mediaType,
-			Size:         desc.Size,
+			Size:         int64(len(payload)),
 			CreationTime: time.Now(),
 		}
 		blobs = append(blobs, blob)
 		for _, layer := range manifest.References() {
 			afnb := &models.ArtifactAndBlob{
-				DigestAF:   desc.Digest.String(),
+				DigestAF:   digest,
 				DigestBlob: layer.Digest.String(),
 			}
 			afnbs = append(afnbs, afnb)
@@ -454,7 +441,7 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
 			PID:          pid,
 			Repo:         repo,
 			Tag:          tag,
-			Digest:       desc.Digest.String(),
+			Digest:       digest,
 			Kind:         "Docker-Image",
 			CreationTime: time.Now(),
 		}

src/core/api/reg_gc.go

@@ -48,12 +48,18 @@ func (gc *GCAPI) Prepare() {
 //  "schedule": {
 //    "type": "Daily",
 //    "cron": "0 0 0 * * *"
+//  },
+//  "parameters": {
+//    "delete_untagged": true
 //  }
 //	}
 // create a manual trigger for GC
 // 	{
 //  "schedule": {
 //    "type": "Manual"
+//  },
+//  "parameters": {
+//    "delete_untagged": true
 //  }
 //	}
 func (gc *GCAPI) Post() {
@@ -64,9 +70,7 @@ func (gc *GCAPI) Post() {
 		return
 	}
 	ajr.Name = common_job.ImageGC
-	ajr.Parameters = map[string]interface{}{
+	ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
-		"redis_url_reg": os.Getenv("_REDIS_URL_REG"),
-	}
 	gc.submit(&ajr)
 	gc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10))
 }
@@ -77,6 +81,9 @@ func (gc *GCAPI) Post() {
 //  "schedule": {
 //    "type": "None",
 //    "cron": ""
+//  },
+//  "parameters": {
+//    "delete_untagged": true
 //  }
 //	}
 func (gc *GCAPI) Put() {
@@ -87,9 +94,7 @@ func (gc *GCAPI) Put() {
 		return
 	}
 	ajr.Name = common_job.ImageGC
-	ajr.Parameters = map[string]interface{}{
+	ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
-		"redis_url_reg": os.Getenv("_REDIS_URL_REG"),
-	}
 	gc.updateSchedule(ajr)
 }
 

src/core/api/reg_gc_test.go

@@ -7,10 +7,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-var adminJob001 apilib.AdminJobReq
-
 func TestGCPost(t *testing.T) {
 
+	adminJob001 := apilib.AdminJobReq{
+		Parameters: map[string]interface{}{"delete_untagged": false},
+	}
 	assert := assert.New(t)
 	apiTest := newHarborAPI()
 

src/core/api/repository_label.go

@@ -1,193 +0,0 @@
-// Copyright 2018 Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-
-	"github.com/goharbor/harbor/src/common"
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/common/rbac"
-	coreutils "github.com/goharbor/harbor/src/core/utils"
-)
-
-// RepositoryLabelAPI handles requests for adding/removing label to/from repositories and images
-type RepositoryLabelAPI struct {
-	LabelResourceAPI
-	repository *models.RepoRecord
-	tag        string
-	label      *models.Label
-}
-
-// Prepare ...
-func (r *RepositoryLabelAPI) Prepare() {
-	// Super
-	r.LabelResourceAPI.Prepare()
-
-	if !r.SecurityCtx.IsAuthenticated() {
-		r.SendUnAuthorizedError(errors.New("UnAuthorized"))
-		return
-	}
-
-	repository := r.GetString(":splat")
-	repo, err := dao.GetRepositoryByName(repository)
-	if err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to get repository %s: %v", repository, err))
-		return
-	}
-
-	if repo == nil {
-		r.SendNotFoundError(fmt.Errorf("repository %s not found", repository))
-		return
-	}
-	r.repository = repo
-
-	tag := r.GetString(":tag")
-	if len(tag) > 0 {
-		exist, err := imageExist(r.SecurityCtx.GetUsername(), repository, tag)
-		if err != nil {
-			r.SendInternalServerError(fmt.Errorf("failed to check the existence of image %s:%s: %v", repository, tag, err))
-			return
-		}
-		if !exist {
-			r.SendNotFoundError(fmt.Errorf("image %s:%s not found", repository, tag))
-			return
-		}
-		r.tag = tag
-	}
-
-	if r.Ctx.Request.Method == http.MethodDelete {
-		labelID, err := r.GetInt64FromPath(":id")
-		if err != nil {
-			r.SendInternalServerError(fmt.Errorf("failed to get ID parameter from path: %v", err))
-			return
-		}
-
-		label, ok := r.exists(labelID)
-		if !ok {
-			return
-		}
-
-		r.label = label
-	}
-}
-
-func (r *RepositoryLabelAPI) requireAccess(action rbac.Action, subresource ...rbac.Resource) bool {
-	if len(subresource) == 0 {
-		subresource = append(subresource, rbac.ResourceRepositoryLabel)
-	}
-
-	return r.RequireProjectAccess(r.repository.ProjectID, action, subresource...)
-}
-
-func (r *RepositoryLabelAPI) isValidLabelReq() bool {
-	p, err := r.ProjectMgr.Get(r.repository.ProjectID)
-	if err != nil {
-		r.SendInternalServerError(err)
-		return false
-	}
-
-	l := &models.Label{}
-	if err := r.DecodeJSONReq(l); err != nil {
-		r.SendBadRequestError(err)
-		return false
-	}
-
-	label, ok := r.validate(l.ID, p.ProjectID)
-	if !ok {
-		return false
-	}
-	r.label = label
-
-	return true
-}
-
-// GetOfImage returns labels of an image
-func (r *RepositoryLabelAPI) GetOfImage() {
-	if !r.requireAccess(rbac.ActionList, rbac.ResourceRepositoryTagLabel) {
-		return
-	}
-
-	r.getLabelsOfResource(common.ResourceTypeImage, fmt.Sprintf("%s:%s", r.repository.Name, r.tag))
-}
-
-// AddToImage adds the label to an image
-func (r *RepositoryLabelAPI) AddToImage() {
-	if !r.requireAccess(rbac.ActionCreate, rbac.ResourceRepositoryTagLabel) || !r.isValidLabelReq() {
-		return
-	}
-
-	rl := &models.ResourceLabel{
-		LabelID:      r.label.ID,
-		ResourceType: common.ResourceTypeImage,
-		ResourceName: fmt.Sprintf("%s:%s", r.repository.Name, r.tag),
-	}
-	r.markLabelToResource(rl)
-}
-
-// RemoveFromImage removes the label from an image
-func (r *RepositoryLabelAPI) RemoveFromImage() {
-	if !r.requireAccess(rbac.ActionDelete, rbac.ResourceRepositoryTagLabel) {
-		return
-	}
-
-	r.removeLabelFromResource(common.ResourceTypeImage,
-		fmt.Sprintf("%s:%s", r.repository.Name, r.tag), r.label.ID)
-}
-
-// GetOfRepository returns labels of a repository
-func (r *RepositoryLabelAPI) GetOfRepository() {
-	if !r.requireAccess(rbac.ActionList) {
-		return
-	}
-
-	r.getLabelsOfResource(common.ResourceTypeRepository, r.repository.RepositoryID)
-}
-
-// AddToRepository adds the label to a repository
-func (r *RepositoryLabelAPI) AddToRepository() {
-	if !r.requireAccess(rbac.ActionCreate) || !r.isValidLabelReq() {
-		return
-	}
-
-	rl := &models.ResourceLabel{
-		LabelID:      r.label.ID,
-		ResourceType: common.ResourceTypeRepository,
-		ResourceID:   r.repository.RepositoryID,
-	}
-	r.markLabelToResource(rl)
-}
-
-// RemoveFromRepository removes the label from a repository
-func (r *RepositoryLabelAPI) RemoveFromRepository() {
-	if !r.requireAccess(rbac.ActionDelete) {
-		return
-	}
-
-	r.removeLabelFromResource(common.ResourceTypeRepository, r.repository.RepositoryID, r.label.ID)
-}
-
-func imageExist(username, repository, tag string) (bool, error) {
-	client, err := coreutils.NewRepositoryClientForUI(username, repository)
-	if err != nil {
-		return false, err
-	}
-
-	_, exist, err := client.ManifestExist(tag)
-	return exist, err
-}

src/core/api/repository_label_test.go

@@ -1,255 +0,0 @@
-// Copyright 2018 Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
-	"fmt"
-	"net/http"
-	"testing"
-
-	"github.com/goharbor/harbor/src/common"
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-var (
-	resourceLabelAPIBasePath = "/api/repositories"
-	repo                     = "library/hello-world"
-	tag                      = "latest"
-	proLibraryLabelID        int64
-)
-
-func TestAddToImage(t *testing.T) {
-	sysLevelLabelID, err := dao.AddLabel(&models.Label{
-		Name:  "sys_level_label",
-		Level: common.LabelLevelSystem,
-	})
-	require.Nil(t, err)
-	defer dao.DeleteLabel(sysLevelLabelID)
-
-	proTestLabelID, err := dao.AddLabel(&models.Label{
-		Name:      "pro_test_label",
-		Level:     common.LabelLevelUser,
-		Scope:     common.LabelScopeProject,
-		ProjectID: 100,
-	})
-	require.Nil(t, err)
-	defer dao.DeleteLabel(proTestLabelID)
-
-	proLibraryLabelID, err = dao.AddLabel(&models.Label{
-		Name:      "pro_library_label",
-		Level:     common.LabelLevelUser,
-		Scope:     common.LabelScopeProject,
-		ProjectID: 1,
-	})
-	require.Nil(t, err)
-
-	cases := []*codeCheckingCase{
-		// 401
-		{
-			request: &testingRequest{
-				url: fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath,
-					repo, tag),
-				method: http.MethodPost,
-			},
-			code: http.StatusUnauthorized,
-		},
-		// 403
-		{
-			request: &testingRequest{
-				url: fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath,
-					repo, tag),
-				method:     http.MethodPost,
-				credential: projGuest,
-			},
-			code: http.StatusForbidden,
-		},
-		// 404 repo doesn't exist
-		{
-			request: &testingRequest{
-				url:        fmt.Sprintf("%s/library/non-exist-repo/tags/%s/labels", resourceLabelAPIBasePath, tag),
-				method:     http.MethodPost,
-				credential: projDeveloper,
-			},
-			code: http.StatusNotFound,
-		},
-		// 404 image doesn't exist
-		{
-			request: &testingRequest{
-				url:        fmt.Sprintf("%s/%s/tags/non-exist-tag/labels", resourceLabelAPIBasePath, repo),
-				method:     http.MethodPost,
-				credential: projDeveloper,
-			},
-			code: http.StatusNotFound,
-		},
-		// 400
-		{
-			request: &testingRequest{
-				url:        fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath, repo, tag),
-				method:     http.MethodPost,
-				credential: projDeveloper,
-			},
-			code: http.StatusBadRequest,
-		},
-		// 404 label doesn't exist
-		{
-			request: &testingRequest{
-				url: fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath,
-					repo, tag),
-				method:     http.MethodPost,
-				credential: projDeveloper,
-				bodyJSON: struct {
-					ID int64
-				}{
-					ID: 1000,
-				},
-			},
-			code: http.StatusNotFound,
-		},
-		// 400 system level label
-		{
-			request: &testingRequest{
-				url: fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath,
-					repo, tag),
-				method:     http.MethodPost,
-				credential: projDeveloper,
-				bodyJSON: struct {
-					ID int64
-				}{
-					ID: sysLevelLabelID,
-				},
-			},
-			code: http.StatusBadRequest,
-		},
-		// 400 try to add the label of project1 to the image under project2
-		{
-			request: &testingRequest{
-				url: fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath,
-					repo, tag),
-				method:     http.MethodPost,
-				credential: projDeveloper,
-				bodyJSON: struct {
-					ID int64
-				}{
-					ID: proTestLabelID,
-				},
-			},
-			code: http.StatusBadRequest,
-		},
-		// 200
-		{
-			request: &testingRequest{
-				url: fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath,
-					repo, tag),
-				method:     http.MethodPost,
-				credential: projDeveloper,
-				bodyJSON: struct {
-					ID int64
-				}{
-					ID: proLibraryLabelID,
-				},
-			},
-			code: http.StatusOK,
-		},
-	}
-	runCodeCheckingCases(t, cases...)
-}
-
-func TestGetOfImage(t *testing.T) {
-	labels := []*models.Label{}
-	err := handleAndParse(&testingRequest{
-		url:        fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath, repo, tag),
-		method:     http.MethodGet,
-		credential: projDeveloper,
-	}, &labels)
-	require.Nil(t, err)
-	require.Equal(t, 1, len(labels))
-	assert.Equal(t, proLibraryLabelID, labels[0].ID)
-}
-
-func TestRemoveFromImage(t *testing.T) {
-	runCodeCheckingCases(t, &codeCheckingCase{
-		request: &testingRequest{
-			url: fmt.Sprintf("%s/%s/tags/%s/labels/%d", resourceLabelAPIBasePath,
-				repo, tag, proLibraryLabelID),
-			method:     http.MethodDelete,
-			credential: projDeveloper,
-		},
-		code: http.StatusOK,
-	})
-
-	labels := []*models.Label{}
-	err := handleAndParse(&testingRequest{
-		url: fmt.Sprintf("%s/%s/tags/%s/labels", resourceLabelAPIBasePath,
-			repo, tag),
-		method:     http.MethodGet,
-		credential: projDeveloper,
-	}, &labels)
-	require.Nil(t, err)
-	require.Equal(t, 0, len(labels))
-}
-
-func TestAddToRepository(t *testing.T) {
-	runCodeCheckingCases(t, &codeCheckingCase{
-		request: &testingRequest{
-			url:    fmt.Sprintf("%s/%s/labels", resourceLabelAPIBasePath, repo),
-			method: http.MethodPost,
-			bodyJSON: struct {
-				ID int64
-			}{
-				ID: proLibraryLabelID,
-			},
-			credential: projDeveloper,
-		},
-		code: http.StatusOK,
-	})
-}
-
-func TestGetOfRepository(t *testing.T) {
-	labels := []*models.Label{}
-	err := handleAndParse(&testingRequest{
-		url:        fmt.Sprintf("%s/%s/labels", resourceLabelAPIBasePath, repo),
-		method:     http.MethodGet,
-		credential: projDeveloper,
-	}, &labels)
-	require.Nil(t, err)
-	require.Equal(t, 1, len(labels))
-	assert.Equal(t, proLibraryLabelID, labels[0].ID)
-}
-
-func TestRemoveFromRepository(t *testing.T) {
-	runCodeCheckingCases(t, &codeCheckingCase{
-		request: &testingRequest{
-			url: fmt.Sprintf("%s/%s/labels/%d", resourceLabelAPIBasePath,
-				repo, proLibraryLabelID),
-			method:     http.MethodDelete,
-			credential: projDeveloper,
-		},
-		code: http.StatusOK,
-	})
-
-	labels := []*models.Label{}
-	err := handleAndParse(&testingRequest{
-		url:        fmt.Sprintf("%s/%s/labels", resourceLabelAPIBasePath, repo),
-		method:     http.MethodGet,
-		credential: projDeveloper,
-	}, &labels)
-	require.Nil(t, err)
-	require.Equal(t, 0, len(labels))
-
-	dao.DeleteLabel(proLibraryLabelID)
-}

src/core/api/repository_test.go

@@ -1,452 +0,0 @@
-// Copyright 2018 Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package api
-
-import (
-	"fmt"
-	"net/http"
-	"testing"
-
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/testing/apitests/apilib"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestGetRepos(t *testing.T) {
-
-	assert := assert.New(t)
-	apiTest := newHarborAPI()
-	projectID := "1"
-	keyword := "library/hello-world"
-
-	fmt.Println("Testing Repos Get API")
-	// -------------------case 1 : response code = 200------------------------//
-	fmt.Println("case 1 : response code = 200")
-	code, repositories, err := apiTest.GetRepos(*admin, projectID, keyword)
-	if err != nil {
-		t.Errorf("failed to get repositories: %v", err)
-	} else {
-		assert.Equal(int(200), code, "response code should be 200")
-		if repos, ok := repositories.([]repoResp); ok {
-			require.Equal(t, int(1), len(repos), "the length of repositories should be 1")
-			assert.Equal(repos[0].Name, "library/hello-world", "unexpected repository name")
-		} else {
-			t.Error("unexpected response")
-		}
-	}
-
-	// -------------------case 2 : response code = 404------------------------//
-	fmt.Println("case 2 : response code = 404:project  not found")
-	projectID = "111"
-	httpStatusCode, _, err := apiTest.GetRepos(*admin, projectID, keyword)
-	if err != nil {
-		t.Error("Error whihle get repos by projectID", err.Error())
-		t.Log(err)
-	} else {
-		assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
-	}
-
-	// -------------------case 3 : response code = 400------------------------//
-	fmt.Println("case 3 : response code = 400,invalid project_id")
-	projectID = "ccc"
-	httpStatusCode, _, err = apiTest.GetRepos(*admin, projectID, keyword)
-	if err != nil {
-		t.Error("Error whihle get repos by projectID", err.Error())
-		t.Log(err)
-	} else {
-		assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
-	}
-
-	fmt.Printf("\n")
-}
-
-func TestGetReposTags(t *testing.T) {
-
-	assert := assert.New(t)
-	apiTest := newHarborAPI()
-
-	// -------------------case 1 : response code = 404------------------------//
-	fmt.Println("case 1 : response code = 404,repo not found")
-	repository := "errorRepos"
-	code, _, err := apiTest.GetReposTags(*admin, repository)
-	if err != nil {
-		t.Errorf("failed to get tags of repository %s: %v", repository, err)
-	} else {
-		assert.Equal(int(404), code, "httpStatusCode should be 404")
-	}
-	// -------------------case 2 : response code = 200------------------------//
-	fmt.Println("case 2 : response code = 200")
-	repository = "library/hello-world"
-	code, tags, err := apiTest.GetReposTags(*admin, repository)
-	if err != nil {
-		t.Errorf("failed to get tags of repository %s: %v", repository, err)
-	} else {
-		assert.Equal(int(200), code, "httpStatusCode should be 200")
-		if tg, ok := tags.([]models.TagResp); ok {
-			assert.Equal(1, len(tg), fmt.Sprintf("there should be only one tag, but now %v", tg))
-			assert.Equal(tg[0].Name, "latest", "the tag should be latest")
-		} else {
-			t.Error("unexpected response")
-		}
-	}
-
-	// -------------------case 3 : response code = 404------------------------//
-	fmt.Println("case 3 : response code = 404")
-	repository = "library/hello-world"
-	tag := "not_exist_tag"
-	code, result, err := apiTest.GetTag(*admin, repository, tag)
-	assert.Nil(err)
-	assert.Equal(http.StatusNotFound, code)
-
-	// -------------------case 4 : response code = 200------------------------//
-	fmt.Println("case 4 : response code = 200")
-	repository = "library/hello-world"
-	tag = "latest"
-	code, result, err = apiTest.GetTag(*admin, repository, tag)
-	assert.Nil(err)
-	assert.Equal(http.StatusOK, code)
-	assert.Equal(tag, result.Name)
-
-	fmt.Printf("\n")
-}
-
-func TestGetReposManifests(t *testing.T) {
-	var httpStatusCode int
-	var err error
-	var repoName string
-	var tag string
-
-	assert := assert.New(t)
-	apiTest := newHarborAPI()
-
-	fmt.Println("Testing ReposManifests Get API")
-	// -------------------case 1 : response code = 200------------------------//
-	fmt.Println("case 1 : response code = 200")
-	repoName = "library/hello-world"
-	tag = "latest"
-	httpStatusCode, err = apiTest.GetReposManifests(*admin, repoName, tag)
-	if err != nil {
-		t.Error("Error whihle get reposManifests by repoName and tag", err.Error())
-		t.Log(err)
-	} else {
-		assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
-	}
-	// -------------------case 2 : response code = 404------------------------//
-	fmt.Println("case 2 : response code = 404:tags error,manifest unknown")
-	tag = "l"
-	httpStatusCode, err = apiTest.GetReposManifests(*admin, repoName, tag)
-	if err != nil {
-		t.Error("Error whihle get reposManifests by repoName and tag", err.Error())
-		t.Log(err)
-	} else {
-		assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
-	}
-
-	// -------------------case 3 : response code = 404------------------------//
-	fmt.Println("case 3 : response code = 404,repo not found")
-	repoName = "111"
-	httpStatusCode, err = apiTest.GetReposManifests(*admin, repoName, tag)
-	if err != nil {
-		t.Error("Error whihle get reposManifests by repoName and tag", err.Error())
-		t.Log(err)
-	} else {
-		assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
-	}
-
-	fmt.Printf("\n")
-}
-
-func TestGetReposTop(t *testing.T) {
-
-	assert := assert.New(t)
-	apiTest := newHarborAPI()
-
-	fmt.Println("Testing ReposTop Get API")
-	// -------------------case 1 : response code = 400------------------------//
-	fmt.Println("case 1 : response code = 400,invalid count")
-	count := "cc"
-	code, _, err := apiTest.GetReposTop(*admin, count)
-	if err != nil {
-		t.Errorf("failed to get the most popular repositories: %v", err)
-	} else {
-		assert.Equal(int(400), code, "response code should be 400")
-	}
-
-	// -------------------case 2 : response code = 200------------------------//
-	fmt.Println("case 2 : response code = 200")
-	count = "1"
-	code, repos, err := apiTest.GetReposTop(*admin, count)
-	if err != nil {
-		t.Errorf("failed to get the most popular repositories: %v", err)
-	} else {
-		assert.Equal(int(200), code, "response code should be 200")
-		if r, ok := repos.([]*repoResp); ok {
-			assert.Equal(int(1), len(r), "the length should be 1")
-			assert.Equal(r[0].Name, "library/busybox", "the name of repository should be library/busybox")
-		} else {
-			t.Error("unexpected response")
-		}
-	}
-
-	fmt.Printf("\n")
-}
-
-func TestPopulateAuthor(t *testing.T) {
-	author := "author"
-	detail := &models.TagDetail{
-		Author: author,
-	}
-	populateAuthor(detail)
-	assert.Equal(t, author, detail.Author)
-
-	detail = &models.TagDetail{}
-	populateAuthor(detail)
-	assert.Equal(t, "", detail.Author)
-
-	maintainer := "maintainer"
-	detail = &models.TagDetail{
-		Config: &models.TagCfg{
-			Labels: map[string]string{
-				"Maintainer": maintainer,
-			},
-		},
-	}
-	populateAuthor(detail)
-	assert.Equal(t, maintainer, detail.Author)
-}
-
-func TestPutOfRepository(t *testing.T) {
-	base := "/api/repositories/"
-	desc := struct {
-		Description string `json:"description"`
-	}{
-		Description: "description_for_test",
-	}
-
-	cases := []*codeCheckingCase{
-		// 404
-		{
-			request: &testingRequest{
-				method:   http.MethodPut,
-				url:      base + "non_exist_repository",
-				bodyJSON: desc,
-			},
-			code: http.StatusNotFound,
-		},
-		// 401
-		{
-			request: &testingRequest{
-				method:   http.MethodPut,
-				url:      base + "library/hello-world",
-				bodyJSON: desc,
-			},
-			code: http.StatusUnauthorized,
-		},
-		// 403 non-member
-		{
-			request: &testingRequest{
-				method:     http.MethodPut,
-				url:        base + "library/hello-world",
-				bodyJSON:   desc,
-				credential: nonSysAdmin,
-			},
-			code: http.StatusForbidden,
-		},
-		// 403 project guest
-		{
-			request: &testingRequest{
-				method:     http.MethodPut,
-				url:        base + "library/hello-world",
-				bodyJSON:   desc,
-				credential: projGuest,
-			},
-			code: http.StatusForbidden,
-		},
-		// 200 project developer
-		{
-			request: &testingRequest{
-				method:     http.MethodPut,
-				url:        base + "library/hello-world",
-				bodyJSON:   desc,
-				credential: projDeveloper,
-			},
-			code: http.StatusOK,
-		},
-		// 200 project admin
-		{
-			request: &testingRequest{
-				method:     http.MethodPut,
-				url:        base + "library/hello-world",
-				bodyJSON:   desc,
-				credential: projAdmin,
-			},
-			code: http.StatusOK,
-		},
-		// 200 system admin
-		{
-			request: &testingRequest{
-				method:     http.MethodPut,
-				url:        base + "library/hello-world",
-				bodyJSON:   desc,
-				credential: sysAdmin,
-			},
-			code: http.StatusOK,
-		},
-	}
-	runCodeCheckingCases(t, cases...)
-
-	// verify that the description is changed
-	repositories := []*repoResp{}
-	err := handleAndParse(&testingRequest{
-		method: http.MethodGet,
-		url:    base,
-		queryStruct: struct {
-			ProjectID int64 `url:"project_id"`
-		}{
-			ProjectID: 1,
-		},
-	}, &repositories)
-	require.Nil(t, err)
-	var repository *repoResp
-	for _, repo := range repositories {
-		if repo.Name == "library/hello-world" {
-			repository = repo
-			break
-		}
-	}
-	require.NotNil(t, repository)
-	assert.Equal(t, desc.Description, repository.Description)
-}
-
-func TestRetag(t *testing.T) {
-	assert := assert.New(t)
-	apiTest := newHarborAPI()
-	repo := "library/hello-world"
-
-	fmt.Println("Testing Image Retag API")
-	// -------------------case 1 : response code = 200------------------------//
-	fmt.Println("case 1 : response code = 200")
-	retagReq := &apilib.Retag{
-		Tag:      "prd",
-		SrcImage: "library/hello-world:latest",
-		Override: true,
-	}
-	code, err := apiTest.RetagImage(*admin, repo, retagReq)
-	if err != nil {
-		t.Errorf("failed to retag: %v", err)
-	} else {
-		assert.Equal(int(200), code, "response code should be 200")
-	}
-
-	// -------------------case 2 : response code = 400------------------------//
-	fmt.Println("case 2 : response code = 400: invalid image value provided")
-	retagReq = &apilib.Retag{
-		Tag:      "prd",
-		SrcImage: "hello-world:latest",
-		Override: true,
-	}
-	httpStatusCode, err := apiTest.RetagImage(*admin, repo, retagReq)
-	if err != nil {
-		t.Errorf("failed to retag: %v", err)
-	} else {
-		assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
-	}
-
-	// -------------------case 3 : response code = 404------------------------//
-	fmt.Println("case 3 : response code = 404: source image not exist")
-	retagReq = &apilib.Retag{
-		Tag:      "prd",
-		SrcImage: "release/hello-world:notexist",
-		Override: true,
-	}
-	httpStatusCode, err = apiTest.RetagImage(*admin, repo, retagReq)
-	if err != nil {
-		t.Errorf("failed to retag: %v", err)
-	} else {
-		assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
-	}
-
-	// -------------------case 4 : response code = 404------------------------//
-	fmt.Println("case 4 : response code = 404: target project not exist")
-	retagReq = &apilib.Retag{
-		Tag:      "prd",
-		SrcImage: "library/hello-world:latest",
-		Override: true,
-	}
-	httpStatusCode, err = apiTest.RetagImage(*admin, "nonexist/hello-world", retagReq)
-	if err != nil {
-		t.Errorf("failed to retag: %v", err)
-	} else {
-		assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
-	}
-
-	// -------------------case 5 : response code = 401------------------------//
-	fmt.Println("case 5 : response code = 401, unathorized")
-	retagReq = &apilib.Retag{
-		Tag:      "prd",
-		SrcImage: "library/hello-world:latest",
-		Override: true,
-	}
-	httpStatusCode, err = apiTest.RetagImage(*unknownUsr, repo, retagReq)
-	if err != nil {
-		t.Errorf("failed to retag: %v", err)
-	} else {
-		assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 401")
-	}
-
-	// -------------------case 6 : response code = 409------------------------//
-	fmt.Println("case 6 : response code = 409, conflict")
-	retagReq = &apilib.Retag{
-		Tag:      "latest",
-		SrcImage: "library/hello-world:latest",
-		Override: false,
-	}
-	httpStatusCode, err = apiTest.RetagImage(*admin, repo, retagReq)
-	if err != nil {
-		t.Errorf("failed to retag: %v", err)
-	} else {
-		assert.Equal(int(409), httpStatusCode, "httpStatusCode should be 409")
-	}
-
-	// -------------------case 7 : response code = 400------------------------//
-	fmt.Println("case 7 : response code = 400")
-	retagReq = &apilib.Retag{
-		Tag:      ".0.1",
-		SrcImage: "library/hello-world:latest",
-		Override: true,
-	}
-	code, err = apiTest.RetagImage(*admin, repo, retagReq)
-	if err != nil {
-		t.Errorf("failed to retag: %v", err)
-	} else {
-		assert.Equal(int(400), code, "response code should be 400")
-	}
-
-	// -------------------case 8 : response code = 400------------------------//
-	fmt.Println("case 8 : response code = 400")
-	retagReq = &apilib.Retag{
-		Tag:      "v0.1",
-		SrcImage: "library/hello-world:latest",
-		Override: true,
-	}
-	code, err = apiTest.RetagImage(*admin, "library/Aaaa", retagReq)
-	if err != nil {
-		t.Errorf("failed to retag: %v", err)
-	} else {
-		assert.Equal(int(400), code, "response code should be 400")
-	}
-
-	fmt.Printf("\n")
-}

src/core/api/scan.go

@@ -15,6 +15,7 @@
 package api
 
 import (
+	"github.com/goharbor/harbor/src/pkg/registry"
 	"net/http"
 	"strconv"
 
@@ -22,7 +23,6 @@ import (
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/rbac"
 	"github.com/goharbor/harbor/src/common/utils"
-	coreutils "github.com/goharbor/harbor/src/core/utils"
 	"github.com/goharbor/harbor/src/jobservice/logger"
 	"github.com/goharbor/harbor/src/pkg/scan/errs"
 	"github.com/goharbor/harbor/src/pkg/scan/report"
@@ -192,20 +192,15 @@ func (sa *ScanAPI) Log() {
 // TODO: This can be removed if the registry access interface is ready.
 type digestGetter func(repo, tag string, username string) (string, error)
 
+// TODO this method should be reconsidered as the tags are stored in database
+// TODO rather than in registry
 func getDigest(repo, tag string, username string) (string, error) {
-	client, err := coreutils.NewRepositoryClientForUI(username, repo)
+	exist, digest, err := registry.Cli.ManifestExist(repo, tag)
 	if err != nil {
 		return "", err
 	}
-
+	if !exist {
-	digest, exists, err := client.ManifestExist(tag)
-	if err != nil {
-		return "", err
-	}
-
-	if !exists {
 		return "", errors.Errorf("tag %s does exist", tag)
 	}
-
 	return digest, nil
 }

src/core/api/search.go

@@ -23,7 +23,6 @@ import (
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/config"
-	coreutils "github.com/goharbor/harbor/src/core/utils"
 	"k8s.io/helm/cmd/helm/search"
 )
 
@@ -180,27 +179,10 @@ func filterRepositories(projects []*models.Project, keyword string) (
 		entry["project_public"] = project.IsPublic()
 		entry["pull_count"] = repository.PullCount
 
-		tags, err := getTags(repository.Name)
+		// TODO populate artifact count
-		if err != nil {
+		// entry["tags_count"] = len(tags)
-			return nil, err
-		}
-		entry["tags_count"] = len(tags)
 
 		result = append(result, entry)
 	}
 	return result, nil
 }
-
-func getTags(repository string) ([]string, error) {
-	client, err := coreutils.NewRepositoryClientForUI("harbor-core", repository)
-	if err != nil {
-		return nil, err
-	}
-
-	tags, err := client.ListTag()
-	if err != nil {
-		return nil, err
-	}
-
-	return tags, nil
-}

src/core/api/utils.go

@@ -1,280 +0,0 @@
-// Copyright 2018 Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
-	"fmt"
-	"net/http"
-	"sort"
-	"strings"
-
-	"github.com/goharbor/harbor/src/common/dao"
-	commonhttp "github.com/goharbor/harbor/src/common/http"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/common/utils"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/common/utils/registry"
-	"github.com/goharbor/harbor/src/common/utils/registry/auth"
-	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/promgr"
-	coreutils "github.com/goharbor/harbor/src/core/utils"
-)
-
-// SyncRegistry syncs the repositories of registry with database.
-func SyncRegistry(pm promgr.ProjectManager) error {
-
-	log.Infof("Start syncing repositories from registry to DB... ")
-
-	reposInRegistry, err := Catalog()
-	if err != nil {
-		log.Error(err)
-		return err
-	}
-
-	var repoRecordsInDB []*models.RepoRecord
-	repoRecordsInDB, err = dao.GetRepositories()
-	if err != nil {
-		log.Errorf("error occurred while getting all registories. %v", err)
-		return err
-	}
-
-	var reposInDB []string
-	for _, repoRecordInDB := range repoRecordsInDB {
-		reposInDB = append(reposInDB, repoRecordInDB.Name)
-	}
-
-	var reposToAdd []string
-	var reposToDel []string
-	reposToAdd, reposToDel, err = diffRepos(reposInRegistry, reposInDB, pm)
-	if err != nil {
-		return err
-	}
-
-	if len(reposToAdd) > 0 {
-		log.Infof("Start adding repositories into DB %v ... ", len(reposToAdd))
-		for _, repoToAdd := range reposToAdd {
-			project, _ := utils.ParseRepository(repoToAdd)
-			pullCount, err := dao.CountPull(repoToAdd)
-			if err != nil {
-				log.Errorf("Error happens when counting pull count from access log: %v", err)
-			}
-			pro, err := pm.Get(project)
-			if err != nil {
-				log.Errorf("failed to get project %s: %v", project, err)
-				continue
-			}
-			repoRecord := models.RepoRecord{
-				Name:      repoToAdd,
-				ProjectID: pro.ProjectID,
-				PullCount: pullCount,
-			}
-
-			if err := dao.AddRepository(repoRecord); err != nil {
-				log.Errorf("Error happens when adding the missing repository: %v", err)
-			} else {
-				log.Infof("Add repository: %s success.", repoToAdd)
-			}
-		}
-	}
-
-	if len(reposToDel) > 0 {
-		log.Debugf("Start deleting repositories from DB... ")
-		for _, repoToDel := range reposToDel {
-			if err := dao.DeleteRepository(repoToDel); err != nil {
-				log.Errorf("Error happens when deleting the repository: %v", err)
-			} else {
-				log.Debugf("Delete repository: %s success.", repoToDel)
-			}
-		}
-	}
-
-	log.Infof("Sync repositories from registry to DB is done.")
-	return nil
-}
-
-// Catalog ...
-func Catalog() ([]string, error) {
-	repositories := []string{}
-
-	rc, err := initRegistryClient()
-	if err != nil {
-		return repositories, err
-	}
-
-	repositories, err = rc.Catalog()
-	if err != nil {
-		return repositories, err
-	}
-
-	return repositories, nil
-}
-
-func diffRepos(reposInRegistry []string, reposInDB []string,
-	pm promgr.ProjectManager) ([]string, []string, error) {
-	var needsAdd []string
-	var needsDel []string
-
-	sort.Strings(reposInRegistry)
-	sort.Strings(reposInDB)
-
-	i, j := 0, 0
-	repoInR, repoInD := "", ""
-	for i < len(reposInRegistry) && j < len(reposInDB) {
-		repoInR = reposInRegistry[i]
-		repoInD = reposInDB[j]
-		d := strings.Compare(repoInR, repoInD)
-		if d < 0 {
-			i++
-			exist, err := projectExists(pm, repoInR)
-			if err != nil {
-				log.Errorf("failed to check the existence of project %s: %v", repoInR, err)
-				continue
-			}
-
-			if !exist {
-				continue
-			}
-
-			// TODO remove the workaround when the bug of registry is fixed
-			client, err := coreutils.NewRepositoryClientForUI("harbor-core", repoInR)
-			if err != nil {
-				return needsAdd, needsDel, err
-			}
-
-			exist, err = repositoryExist(repoInR, client)
-			if err != nil {
-				return needsAdd, needsDel, err
-			}
-
-			if !exist {
-				continue
-			}
-
-			needsAdd = append(needsAdd, repoInR)
-		} else if d > 0 {
-			needsDel = append(needsDel, repoInD)
-			j++
-		} else {
-			// TODO remove the workaround when the bug of registry is fixed
-			client, err := coreutils.NewRepositoryClientForUI("harbor-core", repoInR)
-			if err != nil {
-				return needsAdd, needsDel, err
-			}
-
-			exist, err := repositoryExist(repoInR, client)
-			if err != nil {
-				return needsAdd, needsDel, err
-			}
-
-			if !exist {
-				needsDel = append(needsDel, repoInD)
-			}
-
-			i++
-			j++
-		}
-	}
-
-	for i < len(reposInRegistry) {
-		repoInR = reposInRegistry[i]
-		i++
-		exist, err := projectExists(pm, repoInR)
-		if err != nil {
-			log.Errorf("failed to check whether project of %s exists: %v", repoInR, err)
-			continue
-		}
-
-		if !exist {
-			continue
-		}
-
-		client, err := coreutils.NewRepositoryClientForUI("harbor-core", repoInR)
-		if err != nil {
-			log.Errorf("failed to create repository client: %v", err)
-			continue
-		}
-
-		exist, err = repositoryExist(repoInR, client)
-		if err != nil {
-			log.Errorf("failed to check the existence of repository %s: %v", repoInR, err)
-			continue
-		}
-
-		if !exist {
-			continue
-		}
-
-		needsAdd = append(needsAdd, repoInR)
-	}
-
-	for j < len(reposInDB) {
-		needsDel = append(needsDel, reposInDB[j])
-		j++
-	}
-
-	return needsAdd, needsDel, nil
-}
-
-func projectExists(pm promgr.ProjectManager, repository string) (bool, error) {
-	project, _ := utils.ParseRepository(repository)
-	return pm.Exists(project)
-}
-
-func initRegistryClient() (r *registry.Registry, err error) {
-	endpoint, err := config.RegistryURL()
-	if err != nil {
-		return nil, err
-	}
-
-	addr := endpoint
-	if strings.Contains(endpoint, "://") {
-		addr = strings.Split(endpoint, "://")[1]
-	}
-
-	if err := utils.TestTCPConn(addr, 60, 2); err != nil {
-		return nil, err
-	}
-
-	authorizer := auth.DefaultBasicAuthorizer()
-	return registry.NewRegistry(endpoint, &http.Client{
-		Transport: registry.NewTransport(registry.GetHTTPTransport(), authorizer),
-	})
-}
-
-func buildReplicationURL() string {
-	url := config.InternalJobServiceURL()
-	return fmt.Sprintf("%s/api/jobs/replication", url)
-}
-
-func buildJobLogURL(jobID string, jobType string) string {
-	url := config.InternalJobServiceURL()
-	return fmt.Sprintf("%s/api/jobs/%s/%s/log", url, jobType, jobID)
-}
-
-func buildReplicationActionURL() string {
-	url := config.InternalJobServiceURL()
-	return fmt.Sprintf("%s/api/jobs/replication/actions", url)
-}
-
-func repositoryExist(name string, client *registry.Repository) (bool, error) {
-	tags, err := client.ListTag()
-	if err != nil {
-		if regErr, ok := err.(*commonhttp.Error); ok && regErr.Code == http.StatusNotFound {
-			return false, nil
-		}
-		return false, err
-	}
-	return len(tags) != 0, nil
-}

src/core/controllers/controllers_test.go

@@ -30,7 +30,6 @@ import (
 	utilstest "github.com/goharbor/harbor/src/common/utils/test"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/filter"
-	"github.com/goharbor/harbor/src/core/middlewares"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -101,8 +100,6 @@ func TestRedirectForOIDC(t *testing.T) {
 func TestAll(t *testing.T) {
 	config.InitWithSettings(utilstest.GetUnitTestConfig())
 	assert := assert.New(t)
-	err := middlewares.Init()
-	assert.Nil(err)
 
 	r, _ := http.NewRequest("POST", "/c/login", nil)
 	w := httptest.NewRecorder()

src/core/filter/security.go

@@ -23,6 +23,7 @@ import (
 	beegoctx "github.com/astaxie/beego/context"
 	"github.com/docker/distribution/reference"
 	"github.com/goharbor/harbor/src/common"
+	"github.com/goharbor/harbor/src/common/api"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/dao/group"
 	"github.com/goharbor/harbor/src/common/models"
@@ -209,7 +210,7 @@ func (oc *oidcCliReqCtxModifier) Modify(ctx *beegoctx.Context) bool {
 	path := ctx.Request.URL.Path
 	if path != "/service/token" &&
 		!strings.HasPrefix(path, "/chartrepo/") &&
-		!strings.HasPrefix(path, "/api/chartrepo/") {
+		!strings.HasPrefix(path, fmt.Sprintf("/api/%s/chartrepo/", api.APIVersion)) {
 		log.Debug("OIDC CLI modifier only handles request by docker CLI or helm CLI")
 		return false
 	}

src/core/main.go

@@ -229,26 +229,6 @@ func main() {
 
 	server.RegisterRoutes()
 
-	syncRegistry := os.Getenv("SYNC_REGISTRY")
-	sync, err := strconv.ParseBool(syncRegistry)
-	if err != nil {
-		log.Errorf("Failed to parse SYNC_REGISTRY: %v", err)
-		// if err set it default to false
-		sync = false
-	}
-	if sync {
-		if err := api.SyncRegistry(config.GlobalProjectMgr); err != nil {
-			log.Error(err)
-		}
-	} else {
-		log.Infof("Because SYNC_REGISTRY set false , no need to sync registry \n")
-	}
-
-	log.Info("Init proxy")
-	if err := middlewares.Init(); err != nil {
-		log.Fatalf("init proxy error, %v", err)
-	}
-
 	syncQuota := os.Getenv("SYNC_QUOTA")
 	doSyncQuota, err := strconv.ParseBool(syncQuota)
 	if err != nil {

src/core/middlewares/chain.go

@@ -19,15 +19,6 @@ import (
 
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/middlewares/chart"
-	"github.com/goharbor/harbor/src/core/middlewares/contenttrust"
-	"github.com/goharbor/harbor/src/core/middlewares/countquota"
-	"github.com/goharbor/harbor/src/core/middlewares/immutable"
-	"github.com/goharbor/harbor/src/core/middlewares/listrepo"
-	"github.com/goharbor/harbor/src/core/middlewares/readonly"
-	"github.com/goharbor/harbor/src/core/middlewares/regtoken"
-	"github.com/goharbor/harbor/src/core/middlewares/sizequota"
-	"github.com/goharbor/harbor/src/core/middlewares/url"
-	"github.com/goharbor/harbor/src/core/middlewares/vulnerable"
 	"github.com/justinas/alice"
 )
 
@@ -62,16 +53,7 @@ func (b *DefaultCreator) Create() *alice.Chain {
 
 func (b *DefaultCreator) geMiddleware(mName string) alice.Constructor {
 	middlewares := map[string]alice.Constructor{
-		CHART:        func(next http.Handler) http.Handler { return chart.New(next) },
+		CHART: func(next http.Handler) http.Handler { return chart.New(next) },
-		READONLY:     func(next http.Handler) http.Handler { return readonly.New(next) },
-		URL:          func(next http.Handler) http.Handler { return url.New(next) },
-		LISTREPO:     func(next http.Handler) http.Handler { return listrepo.New(next) },
-		CONTENTTRUST: func(next http.Handler) http.Handler { return contenttrust.New(next) },
-		VULNERABLE:   func(next http.Handler) http.Handler { return vulnerable.New(next) },
-		SIZEQUOTA:    func(next http.Handler) http.Handler { return sizequota.New(next) },
-		COUNTQUOTA:   func(next http.Handler) http.Handler { return countquota.New(next) },
-		IMMUTABLE:    func(next http.Handler) http.Handler { return immutable.New(next) },
-		REGTOKEN:     func(next http.Handler) http.Handler { return regtoken.New(next) },
 	}
 	return middlewares[mName]
 }

src/core/middlewares/chart/builder.go

@@ -20,6 +20,7 @@ import (
 	"regexp"
 	"strconv"
 
+	"github.com/goharbor/harbor/src/common/api"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
@@ -29,8 +30,10 @@ import (
 )
 
 var (
-	deleteChartVersionRe = regexp.MustCompile(`^/api/chartrepo/(?P<namespace>[^?#]+)/charts/(?P<name>[^?#]+)/(?P<version>[^?#]+)/?$`)
+	deleteChartVersionRePattern = fmt.Sprintf(`^/api/%s/chartrepo/(?P<namespace>[^?#]+)/charts/(?P<name>[^?#]+)/(?P<version>[^?#]+)/?$`, api.APIVersion)
-	createChartVersionRe = regexp.MustCompile(`^/api/chartrepo/(?P<namespace>[^?#]+)/charts/?$`)
+	deleteChartVersionRe        = regexp.MustCompile(deleteChartVersionRePattern)
+	createChartVersionRePattern = fmt.Sprintf(`^/api/%s/chartrepo/(?P<namespace>[^?#]+)/charts/?$`, api.APIVersion)
+	createChartVersionRe        = regexp.MustCompile(createChartVersionRePattern)
 )
 
 var (

src/core/middlewares/chart/handler_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/goharbor/harbor/src/chartserver"
+	"github.com/goharbor/harbor/src/common/api"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/core/middlewares/util"
 	"github.com/goharbor/harbor/src/pkg/types"
@@ -30,7 +31,7 @@ import (
 )
 
 func deleteChartVersion(projectName, chartName, version string) {
-	url := fmt.Sprintf("/api/chartrepo/%s/charts/%s/%s", projectName, chartName, version)
+	url := fmt.Sprintf("/api/%s/chartrepo/%s/charts/%s/%s", api.APIVersion, projectName, chartName, version)
 	req, _ := http.NewRequest(http.MethodDelete, url, nil)
 
 	next := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
@@ -43,7 +44,7 @@ func deleteChartVersion(projectName, chartName, version string) {
 }
 
 func uploadChartVersion(projectID int64, projectName, chartName, version string) {
-	url := fmt.Sprintf("/api/chartrepo/%s/charts/", projectName)
+	url := fmt.Sprintf("/api/%s/chartrepo/%s/charts/", api.APIVersion, projectName)
 	req, _ := http.NewRequest(http.MethodPost, url, nil)
 
 	info := &util.ChartVersionInfo{

src/core/middlewares/config.go

@@ -16,23 +16,8 @@ package middlewares
 
 // const variables
 const (
-	CHART        = "chart"
+	CHART = "chart"
-	READONLY     = "readonly"
-	URL          = "url"
-	LISTREPO     = "listrepo"
-	CONTENTTRUST = "contenttrust"
-	VULNERABLE   = "vulnerable"
-	SIZEQUOTA    = "sizequota"
-	COUNTQUOTA   = "countquota"
-	IMMUTABLE    = "immutable"
-	REGTOKEN     = "regtoken"
 )
 
 // ChartMiddlewares middlewares for chart server
 var ChartMiddlewares = []string{CHART}
-
-// Middlewares with sequential organization
-var Middlewares = []string{READONLY, URL, REGTOKEN, LISTREPO, CONTENTTRUST, VULNERABLE, SIZEQUOTA, IMMUTABLE, COUNTQUOTA}
-
-// MiddlewaresLocal ...
-var MiddlewaresLocal = []string{SIZEQUOTA, IMMUTABLE, COUNTQUOTA}

src/core/middlewares/contenttrust/handler.go

@@ -1,116 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package contenttrust
-
-import (
-	"github.com/goharbor/harbor/src/common/utils"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"github.com/goharbor/harbor/src/pkg/signature/notary"
-	"net/http"
-	"net/http/httptest"
-)
-
-// NotaryEndpoint ...
-var NotaryEndpoint = ""
-
-type contentTrustHandler struct {
-	next http.Handler
-}
-
-// New ...
-func New(next http.Handler) http.Handler {
-	return &contentTrustHandler{
-		next: next,
-	}
-}
-
-// ServeHTTP ...
-func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	doContentTrustCheck, image := validate(req)
-	if !doContentTrustCheck {
-		cth.next.ServeHTTP(rw, req)
-		return
-	}
-	rec := httptest.NewRecorder()
-	cth.next.ServeHTTP(rec, req)
-	if rec.Result().StatusCode == http.StatusOK {
-		match, err := matchNotaryDigest(image)
-		if err != nil {
-			http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError)
-			return
-		}
-		if !match {
-			log.Debugf("digest mismatch, failing the response.")
-			http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed)
-			return
-		}
-	}
-	util.CopyResp(rec, rw)
-}
-
-func validate(req *http.Request) (bool, util.ArtifactInfo) {
-	var img util.ArtifactInfo
-	imgRaw := req.Context().Value(util.ArtifactInfoCtxKey)
-	if imgRaw == nil || !config.WithNotary() {
-		return false, img
-	}
-	img, _ = req.Context().Value(util.ArtifactInfoCtxKey).(util.ArtifactInfo)
-	if img.Digest == "" {
-		return false, img
-	}
-	if scannerPull, ok := util.ScannerPullFromContext(req.Context()); ok && scannerPull {
-		return false, img
-	}
-	if !util.GetPolicyChecker().ContentTrustEnabled(img.ProjectName) {
-		return false, img
-	}
-	return true, img
-}
-
-func matchNotaryDigest(img util.ArtifactInfo) (bool, error) {
-	if NotaryEndpoint == "" {
-		NotaryEndpoint = config.InternalNotaryEndpoint()
-	}
-	targets, err := notary.GetInternalTargets(NotaryEndpoint, util.TokenUsername, img.Repository)
-	if err != nil {
-		return false, err
-	}
-	for _, t := range targets {
-		if utils.IsDigest(img.Reference) {
-			d, err := notary.DigestFromTarget(t)
-			if err != nil {
-				return false, err
-			}
-			if img.Digest == d {
-				return true, nil
-			}
-		} else {
-			if t.Tag == img.Reference {
-				log.Debugf("found reference: %s in notary, try to match digest.", img.Reference)
-				d, err := notary.DigestFromTarget(t)
-				if err != nil {
-					return false, err
-				}
-				if img.Digest == d {
-					return true, nil
-				}
-			}
-		}
-	}
-	log.Debugf("image: %#v, not found in notary", img)
-	return false, nil
-}

src/core/middlewares/contenttrust/handler_test.go

@@ -1,62 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package contenttrust
-
-import (
-	"net/http/httptest"
-	"os"
-	"testing"
-
-	"github.com/goharbor/harbor/src/common"
-	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	notarytest "github.com/goharbor/harbor/src/pkg/signature/notary/test"
-	"github.com/stretchr/testify/assert"
-)
-
-var endpoint = "10.117.4.142"
-var notaryServer *httptest.Server
-var token = ""
-
-func TestMain(m *testing.M) {
-	notaryServer = notarytest.NewNotaryServer(endpoint)
-	defer notaryServer.Close()
-	NotaryEndpoint = notaryServer.URL
-	var defaultConfig = map[string]interface{}{
-		common.ExtEndpoint:     "https://" + endpoint,
-		common.WithNotary:      true,
-		common.TokenExpiration: 30,
-	}
-	config.InitWithSettings(defaultConfig)
-	result := m.Run()
-	if result != 0 {
-		os.Exit(result)
-	}
-}
-
-func TestMatchNotaryDigest(t *testing.T) {
-	assert := assert.New(t)
-	// The data from common/utils/notary/helper_test.go
-	img1 := util.ArtifactInfo{Repository: "notary-demo/busybox", Reference: "1.0", ProjectName: "notary-demo", Digest: "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
-	img2 := util.ArtifactInfo{Repository: "notary-demo/busybox", Reference: "2.0", ProjectName: "notary-demo", Digest: "sha256:12345678"}
-
-	res1, err := matchNotaryDigest(img1)
-	assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)
-	assert.True(res1)
-
-	res2, err := matchNotaryDigest(img2)
-	assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2)
-	assert.False(res2)
-}

src/core/middlewares/countquota/builder.go

@@ -1,100 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package countquota
-
-import (
-	"fmt"
-	"net/http"
-	"strconv"
-
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor/quota"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-)
-
-var (
-	defaultBuilders = []interceptor.Builder{
-		&manifestDeletionBuilder{},
-		&manifestCreationBuilder{},
-	}
-)
-
-type manifestDeletionBuilder struct{}
-
-func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Interceptor, error) {
-	if match, _, _ := util.MatchDeleteManifest(req); !match {
-		return nil, nil
-	}
-
-	info, ok := util.ManifestInfoFromContext(req.Context())
-	if !ok {
-		var err error
-		info, err = util.ParseManifestInfoFromPath(req)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse manifest, error %v", err)
-		}
-
-		// Manifest info will be used by computeResourcesForDeleteManifest
-		*req = *(req.WithContext(util.NewManifestInfoContext(req.Context(), info)))
-	}
-
-	opts := []quota.Option{
-		quota.EnforceResources(config.QuotaPerProjectEnable()),
-		quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
-		quota.WithAction(quota.SubtractAction),
-		quota.StatusCode(http.StatusAccepted),
-		quota.MutexKeys(info.MutexKey("count")),
-		quota.OnResources(computeResourcesForManifestDeletion),
-		quota.OnFulfilled(func(http.ResponseWriter, *http.Request) error {
-			return dao.DeleteArtifactByDigest(info.ProjectID, info.Repository, info.Digest)
-		}),
-	}
-
-	return quota.New(opts...), nil
-}
-
-type manifestCreationBuilder struct{}
-
-func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Interceptor, error) {
-	if match, _, _ := util.MatchPushManifest(req); !match {
-		return nil, nil
-	}
-
-	info, ok := util.ManifestInfoFromContext(req.Context())
-	if !ok {
-		var err error
-		info, err = util.ParseManifestInfoFromReq(req)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse manifest, error %v", err)
-		}
-
-		// Manifest info will be used by computeResourcesForCreateManifest
-		*req = *(req.WithContext(util.NewManifestInfoContext(req.Context(), info)))
-	}
-
-	opts := []quota.Option{
-		quota.EnforceResources(config.QuotaPerProjectEnable()),
-		quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
-		quota.WithAction(quota.AddAction),
-		quota.StatusCode(http.StatusCreated),
-		quota.MutexKeys(info.MutexKey("count")),
-		quota.OnResources(computeResourcesForManifestCreation),
-		quota.OnFulfilled(afterManifestCreated),
-	}
-
-	return quota.New(opts...), nil
-}

src/core/middlewares/countquota/handler.go

@@ -1,89 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package countquota
-
-import (
-	"fmt"
-	"net/http"
-
-	"github.com/goharbor/harbor/src/common/quota"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-)
-
-type countQuotaHandler struct {
-	builders []interceptor.Builder
-	next     http.Handler
-}
-
-// New ...
-func New(next http.Handler, builders ...interceptor.Builder) http.Handler {
-	if len(builders) == 0 {
-		builders = defaultBuilders
-	}
-
-	return &countQuotaHandler{
-		builders: builders,
-		next:     next,
-	}
-}
-
-// ServeHTTP manifest ...
-func (h *countQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	interceptor, err := h.getInterceptor(req)
-	if err != nil {
-		log.Warningf("Error occurred when to handle request in count quota handler: %v", err)
-		http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in count quota handler: %v", err)),
-			http.StatusInternalServerError)
-		return
-	}
-
-	if interceptor == nil {
-		h.next.ServeHTTP(rw, req)
-		return
-	}
-
-	if err := interceptor.HandleRequest(req); err != nil {
-		log.Warningf("Error occurred when to handle request in count quota handler: %v", err)
-		if _, ok := err.(quota.Errors); ok {
-			util.FireQuotaEvent(req, 1, err.Error())
-			http.Error(rw, util.MarshalError("DENIED", fmt.Sprintf("Quota exceeded when processing the request of %v", err)), http.StatusForbidden)
-			return
-		}
-		http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in count quota handler: %v", err)),
-			http.StatusInternalServerError)
-		return
-	}
-
-	h.next.ServeHTTP(rw, req)
-
-	interceptor.HandleResponse(rw, req)
-}
-
-func (h *countQuotaHandler) getInterceptor(req *http.Request) (interceptor.Interceptor, error) {
-	for _, builder := range h.builders {
-		interceptor, err := builder.Build(req)
-		if err != nil {
-			return nil, err
-		}
-
-		if interceptor != nil {
-			return interceptor, nil
-		}
-	}
-
-	return nil, nil
-}

src/core/middlewares/countquota/handler_test.go

@@ -1,331 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package countquota
-
-import (
-	"fmt"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/docker/distribution"
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"github.com/goharbor/harbor/src/pkg/types"
-	"github.com/opencontainers/go-digest"
-	"github.com/stretchr/testify/suite"
-)
-
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
-func getProjectCountUsage(projectID int64) (int64, error) {
-	usage := models.QuotaUsage{Reference: "project", ReferenceID: fmt.Sprintf("%d", projectID)}
-	err := dao.GetOrmer().Read(&usage, "reference", "reference_id")
-	if err != nil {
-		return 0, err
-	}
-	used, err := types.NewResourceList(usage.Used)
-	if err != nil {
-		return 0, err
-	}
-
-	return used[types.ResourceCount], nil
-}
-
-func randomString(n int) string {
-	const letterBytes = "abcdefghijklmnopqrstuvwxyz"
-
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = letterBytes[rand.Intn(len(letterBytes))]
-	}
-
-	return string(b)
-}
-
-func doDeleteManifestRequest(projectID int64, projectName, name, dgt string, next ...http.HandlerFunc) int {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	url := fmt.Sprintf("/v2/%s/manifests/%s", repository, dgt)
-	req, _ := http.NewRequest("DELETE", url, nil)
-
-	ctx := util.NewManifestInfoContext(req.Context(), &util.ManifestInfo{
-		ProjectID:  projectID,
-		Repository: repository,
-		Digest:     dgt,
-	})
-
-	rr := httptest.NewRecorder()
-
-	var n http.HandlerFunc
-	if len(next) > 0 {
-		n = next[0]
-	} else {
-		n = func(w http.ResponseWriter, req *http.Request) {
-			w.WriteHeader(http.StatusAccepted)
-		}
-	}
-
-	h := New(http.HandlerFunc(n))
-	h.ServeHTTP(util.NewCustomResponseWriter(rr), req.WithContext(ctx))
-
-	return rr.Code
-}
-
-func doPutManifestRequest(projectID int64, projectName, name, tag, dgt string, withDupBlob bool, next ...http.HandlerFunc) int {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	url := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag)
-	req, _ := http.NewRequest("PUT", url, nil)
-
-	mfInfo := &util.ManifestInfo{
-		ProjectID:  projectID,
-		Repository: repository,
-		Tag:        tag,
-		Digest:     dgt,
-		References: []distribution.Descriptor{
-			{Digest: digest.FromString(randomString(15))},
-			{Digest: digest.FromString(randomString(15))},
-		},
-	}
-
-	ctx := util.NewManifestInfoContext(req.Context(), mfInfo)
-	if withDupBlob {
-		dupDigest := digest.FromString(randomString(15))
-		mfInfo.References = append(mfInfo.References, distribution.Descriptor{Digest: dupDigest})
-		mfInfo.References = append(mfInfo.References, distribution.Descriptor{Digest: dupDigest})
-	}
-
-	rr := httptest.NewRecorder()
-
-	var n http.HandlerFunc
-	if len(next) > 0 {
-		n = next[0]
-	} else {
-		n = func(w http.ResponseWriter, req *http.Request) {
-			w.WriteHeader(http.StatusCreated)
-		}
-	}
-
-	h := New(http.HandlerFunc(n))
-	h.ServeHTTP(util.NewCustomResponseWriter(rr), req.WithContext(ctx))
-
-	return rr.Code
-}
-
-type HandlerSuite struct {
-	suite.Suite
-}
-
-func (suite *HandlerSuite) addProject(projectName string) int64 {
-	projectID, err := dao.AddProject(models.Project{
-		Name:    projectName,
-		OwnerID: 1,
-	})
-
-	suite.Nil(err, fmt.Sprintf("Add project failed for %s", projectName))
-
-	return projectID
-}
-
-func (suite *HandlerSuite) checkCountUsage(expected, projectID int64) {
-	count, err := getProjectCountUsage(projectID)
-	suite.Nil(err, fmt.Sprintf("Failed to get count usage of project %d, error: %v", projectID, err))
-	suite.Equal(expected, count, "Failed to check count usage for project %d", projectID)
-}
-
-func (suite *HandlerSuite) TearDownTest() {
-	for _, table := range []string{
-		"artifact", "blob",
-		"artifact_blob", "project_blob",
-		"quota", "quota_usage",
-	} {
-		dao.ClearTable(table)
-	}
-}
-
-func (suite *HandlerSuite) TestPutManifestCreated() {
-	projectName := randomString(5)
-
-	projectID := suite.addProject(projectName)
-	defer func() {
-		dao.DeleteProject(projectID)
-	}()
-
-	dgt := digest.FromString(randomString(15)).String()
-	code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false)
-	suite.Equal(http.StatusCreated, code)
-	suite.checkCountUsage(1, projectID)
-
-	total, err := dao.GetTotalOfArtifacts(&models.ArtifactQuery{Digest: dgt})
-	suite.Nil(err)
-	suite.Equal(int64(1), total, "Artifact should be created")
-
-	// Push the photon:latest with photon:dev
-	code = doPutManifestRequest(projectID, projectName, "photon", "dev", dgt, false)
-	suite.Equal(http.StatusCreated, code)
-	suite.checkCountUsage(2, projectID)
-
-	total, err = dao.GetTotalOfArtifacts(&models.ArtifactQuery{Digest: dgt})
-	suite.Nil(err)
-	suite.Equal(int64(2), total, "Artifact should be created")
-
-	// Push the photon:latest with new image
-	newDgt := digest.FromString(randomString(15)).String()
-	code = doPutManifestRequest(projectID, projectName, "photon", "latest", newDgt, false)
-	suite.Equal(http.StatusCreated, code)
-	suite.checkCountUsage(2, projectID)
-
-	total, err = dao.GetTotalOfArtifacts(&models.ArtifactQuery{Digest: newDgt})
-	suite.Nil(err)
-	suite.Equal(int64(1), total, "Artifact should be updated")
-}
-
-func (suite *HandlerSuite) TestPutManifestCreatedDupBlobs() {
-	projectName := randomString(5)
-
-	projectID := suite.addProject(projectName)
-	defer func() {
-		dao.DeleteProject(projectID)
-	}()
-
-	dgt := digest.FromString(randomString(15)).String()
-	code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, true)
-	suite.Equal(http.StatusCreated, code)
-	suite.checkCountUsage(1, projectID)
-
-	var count int64
-	err := dao.GetOrmer().Raw("select count(*) from artifact_blob where digest_af = ?", dgt).QueryRow(&count)
-	suite.Nil(err)
-	// 4 = self + 3 distinct blobs
-	suite.Equal(int64(4), count)
-}
-
-func (suite *HandlerSuite) TestPutManifestFailed() {
-	projectName := randomString(5)
-
-	projectID := suite.addProject(projectName)
-	defer func() {
-		dao.DeleteProject(projectID)
-	}()
-
-	next := func(w http.ResponseWriter, req *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}
-
-	dgt := digest.FromString(randomString(15)).String()
-	code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false, next)
-	suite.Equal(http.StatusForbidden, code)
-	suite.checkCountUsage(0, projectID)
-
-	total, err := dao.GetTotalOfArtifacts(&models.ArtifactQuery{Digest: dgt})
-	suite.Nil(err)
-	suite.Equal(int64(0), total, "Artifact should not be created")
-}
-
-func (suite *HandlerSuite) TestDeleteManifestAccepted() {
-	projectName := randomString(5)
-
-	projectID := suite.addProject(projectName)
-	defer func() {
-		dao.DeleteProject(projectID)
-	}()
-
-	dgt := digest.FromString(randomString(15)).String()
-	code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false)
-	suite.Equal(http.StatusCreated, code)
-	suite.checkCountUsage(1, projectID)
-
-	code = doDeleteManifestRequest(projectID, projectName, "photon", dgt)
-	suite.Equal(http.StatusAccepted, code)
-	suite.checkCountUsage(0, projectID)
-}
-
-func (suite *HandlerSuite) TestDeleteManifestFailed() {
-	projectName := randomString(5)
-
-	projectID := suite.addProject(projectName)
-	defer func() {
-		dao.DeleteProject(projectID)
-	}()
-
-	dgt := digest.FromString(randomString(15)).String()
-	code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false)
-	suite.Equal(http.StatusCreated, code)
-	suite.checkCountUsage(1, projectID)
-
-	next := func(w http.ResponseWriter, req *http.Request) {
-		w.WriteHeader(http.StatusInternalServerError)
-	}
-
-	code = doDeleteManifestRequest(projectID, projectName, "photon", dgt, next)
-	suite.Equal(http.StatusInternalServerError, code)
-	suite.checkCountUsage(1, projectID)
-}
-
-func (suite *HandlerSuite) TestDeleteManifestInMultiProjects() {
-	projectName := randomString(5)
-
-	projectID := suite.addProject(projectName)
-	defer func() {
-		dao.DeleteProject(projectID)
-	}()
-
-	dgt := digest.FromString(randomString(15)).String()
-	code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false)
-	suite.Equal(http.StatusCreated, code)
-	suite.checkCountUsage(1, projectID)
-
-	{
-		projectName := randomString(5)
-
-		projectID := suite.addProject(projectName)
-		defer func() {
-			dao.DeleteProject(projectID)
-		}()
-
-		code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false)
-		suite.Equal(http.StatusCreated, code)
-		suite.checkCountUsage(1, projectID)
-
-		code = doDeleteManifestRequest(projectID, projectName, "photon", dgt)
-		suite.Equal(http.StatusAccepted, code)
-		suite.checkCountUsage(0, projectID)
-	}
-
-	code = doDeleteManifestRequest(projectID, projectName, "photon", dgt)
-	suite.Equal(http.StatusAccepted, code)
-	suite.checkCountUsage(0, projectID)
-}
-
-func TestMain(m *testing.M) {
-	config.Init()
-	dao.PrepareTestForPostgresSQL()
-
-	if result := m.Run(); result != 0 {
-		os.Exit(result)
-	}
-}
-
-func TestRunHandlerSuite(t *testing.T) {
-	suite.Run(t, new(HandlerSuite))
-}

src/core/middlewares/countquota/util.go

@@ -1,124 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package countquota
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-	"strings"
-
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/common/quota"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"github.com/goharbor/harbor/src/pkg/types"
-)
-
-// computeResourcesForManifestCreation returns count resource required for manifest
-// no count required if the tag of the repository exists in the project
-func computeResourcesForManifestCreation(req *http.Request) (types.ResourceList, error) {
-	info, ok := util.ManifestInfoFromContext(req.Context())
-	if !ok {
-		return nil, errors.New("manifest info missing")
-	}
-
-	// only count quota required when push new tag
-	if info.IsNewTag() {
-		return quota.ResourceList{quota.ResourceCount: 1}, nil
-	}
-
-	return nil, nil
-}
-
-// computeResourcesForManifestDeletion returns count resource will be released when manifest deleted
-// then result will be the sum of manifest count of the same repository in the project
-func computeResourcesForManifestDeletion(req *http.Request) (types.ResourceList, error) {
-	info, ok := util.ManifestInfoFromContext(req.Context())
-	if !ok {
-		return nil, errors.New("manifest info missing")
-	}
-
-	total, err := dao.GetTotalOfArtifacts(&models.ArtifactQuery{
-		PID:    info.ProjectID,
-		Repo:   info.Repository,
-		Digest: info.Digest,
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("error occurred when get artifacts %v ", err)
-	}
-
-	return types.ResourceList{types.ResourceCount: total}, nil
-}
-
-// afterManifestCreated the handler after manifest created success
-// it will create or update the artifact info in db, and then attach blobs to artifact
-func afterManifestCreated(w http.ResponseWriter, req *http.Request) error {
-	info, ok := util.ManifestInfoFromContext(req.Context())
-	if !ok {
-		return errors.New("manifest info missing")
-	}
-
-	artifact := info.Artifact()
-	if artifact.ID == 0 {
-		if _, err := dao.AddArtifact(artifact); err != nil {
-			return fmt.Errorf("error to add artifact, %v", err)
-		}
-	} else {
-		if err := dao.UpdateArtifact(artifact); err != nil {
-			return fmt.Errorf("error to update artifact, %v", err)
-		}
-	}
-
-	return attachBlobsToArtifact(info)
-}
-
-// attachBlobsToArtifact attach the blobs which from manifest to artifact
-func attachBlobsToArtifact(info *util.ManifestInfo) error {
-	temp := make(map[string]interface{})
-	artifactBlobs := []*models.ArtifactAndBlob{}
-
-	temp[info.Digest] = nil
-	// self
-	artifactBlobs = append(artifactBlobs, &models.ArtifactAndBlob{
-		DigestAF:   info.Digest,
-		DigestBlob: info.Digest,
-	})
-
-	// avoid the duplicate layers.
-	for _, reference := range info.References {
-		_, exist := temp[reference.Digest.String()]
-		if !exist {
-			temp[reference.Digest.String()] = nil
-			artifactBlobs = append(artifactBlobs, &models.ArtifactAndBlob{
-				DigestAF:   info.Digest,
-				DigestBlob: reference.Digest.String(),
-			})
-		}
-	}
-
-	if err := dao.AddArtifactNBlobs(artifactBlobs); err != nil {
-		if strings.Contains(err.Error(), dao.ErrDupRows.Error()) {
-			log.Warning("the artifact and blobs have already in the DB, it maybe an existing image with different tag")
-			return nil
-		}
-
-		return fmt.Errorf("error to add artifact and blobs in proxy response handler, %v", err)
-	}
-
-	return nil
-}

src/core/middlewares/immutable/builder.go

@@ -1,54 +0,0 @@
-package immutable
-
-import (
-	"fmt"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor/immutable"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"net/http"
-)
-
-var (
-	defaultBuilders = []interceptor.Builder{
-		&manifestDeletionBuilder{},
-		&manifestCreationBuilder{},
-	}
-)
-
-type manifestDeletionBuilder struct{}
-
-func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Interceptor, error) {
-	if match, _, _ := util.MatchDeleteManifest(req); !match {
-		return nil, nil
-	}
-
-	info, ok := util.ManifestInfoFromContext(req.Context())
-	if !ok {
-		var err error
-		info, err = util.ParseManifestInfoFromPath(req)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse manifest, error %v", err)
-		}
-	}
-
-	return immutable.NewDeleteMFInteceptor(info), nil
-}
-
-type manifestCreationBuilder struct{}
-
-func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Interceptor, error) {
-	if match, _, _ := util.MatchPushManifest(req); !match {
-		return nil, nil
-	}
-
-	info, ok := util.ManifestInfoFromContext(req.Context())
-	if !ok {
-		var err error
-		info, err = util.ParseManifestInfoFromReq(req)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse manifest, error %v", err)
-		}
-	}
-
-	return immutable.NewPushMFInteceptor(info), nil
-}

src/core/middlewares/immutable/handler.go

@@ -1,95 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package immutable
-
-import (
-	"errors"
-	"fmt"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
-	middlerware_err "github.com/goharbor/harbor/src/core/middlewares/util/error"
-	internal_errors "github.com/goharbor/harbor/src/internal/error"
-	"net/http"
-)
-
-type immutableHandler struct {
-	builders []interceptor.Builder
-	next     http.Handler
-}
-
-// New ...
-func New(next http.Handler, builders ...interceptor.Builder) http.Handler {
-	if len(builders) == 0 {
-		builders = defaultBuilders
-	}
-
-	return &immutableHandler{
-		builders: builders,
-		next:     next,
-	}
-}
-
-// ServeHTTP ...
-func (rh *immutableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-
-	interceptor, err := rh.getInterceptor(req)
-	if err != nil {
-		log.Warningf("Error occurred when to handle request in immutable handler: %v", err)
-		pkgE := internal_errors.New(fmt.Errorf("error occurred when to handle request in immutable handler: %v", err)).WithCode(internal_errors.GeneralCode)
-		msg := internal_errors.NewErrs(pkgE).Error()
-		http.Error(rw, msg, http.StatusInternalServerError)
-		return
-	}
-
-	if interceptor == nil {
-		rh.next.ServeHTTP(rw, req)
-		return
-	}
-
-	if err := interceptor.HandleRequest(req); err != nil {
-		log.Warningf("Error occurred when to handle request in immutable handler: %v", err)
-		var e *middlerware_err.ErrImmutable
-		if errors.As(err, &e) {
-			pkgE := internal_errors.New(e).WithCode(internal_errors.PreconditionCode)
-			msg := internal_errors.NewErrs(pkgE).Error()
-			http.Error(rw, msg, http.StatusPreconditionFailed)
-			return
-		}
-
-		pkgE := internal_errors.New(fmt.Errorf("error occurred when to handle request in immutable handler: %v", err)).WithCode(internal_errors.GeneralCode)
-		msg := internal_errors.NewErrs(pkgE).Error()
-		http.Error(rw, msg, http.StatusInternalServerError)
-		return
-	}
-
-	rh.next.ServeHTTP(rw, req)
-
-	interceptor.HandleResponse(rw, req)
-}
-
-func (rh *immutableHandler) getInterceptor(req *http.Request) (interceptor.Interceptor, error) {
-	for _, builder := range rh.builders {
-		interceptor, err := builder.Build(req)
-		if err != nil {
-			return nil, err
-		}
-
-		if interceptor != nil {
-			return interceptor, nil
-		}
-	}
-
-	return nil, nil
-}

src/core/middlewares/immutable/handler_test.go

@@ -1,151 +0,0 @@
-package immutable
-
-import (
-	"github.com/docker/distribution"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"github.com/opencontainers/go-digest"
-
-	"fmt"
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/pkg/immutabletag"
-	immu_model "github.com/goharbor/harbor/src/pkg/immutabletag/model"
-	"github.com/stretchr/testify/require"
-	"github.com/stretchr/testify/suite"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"testing"
-)
-
-type HandlerSuite struct {
-	suite.Suite
-}
-
-func doPutManifestRequest(projectID int64, projectName, name, tag, dgt string, next ...http.HandlerFunc) int {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	url := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag)
-	req, _ := http.NewRequest("PUT", url, nil)
-
-	mfInfo := &util.ManifestInfo{
-		ProjectID:  projectID,
-		Repository: repository,
-		Tag:        tag,
-		Digest:     dgt,
-		References: []distribution.Descriptor{
-			{Digest: digest.FromString(randomString(15))},
-			{Digest: digest.FromString(randomString(15))},
-		},
-	}
-	ctx := util.NewManifestInfoContext(req.Context(), mfInfo)
-	rr := httptest.NewRecorder()
-
-	var n http.HandlerFunc
-	if len(next) > 0 {
-		n = next[0]
-	} else {
-		n = func(w http.ResponseWriter, req *http.Request) {
-			w.WriteHeader(http.StatusCreated)
-		}
-	}
-
-	h := New(http.HandlerFunc(n))
-	h.ServeHTTP(util.NewCustomResponseWriter(rr), req.WithContext(ctx))
-
-	return rr.Code
-}
-
-func randomString(n int) string {
-	const letterBytes = "abcdefghijklmnopqrstuvwxyz"
-
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = letterBytes[rand.Intn(len(letterBytes))]
-	}
-
-	return string(b)
-}
-
-func (suite *HandlerSuite) addProject(projectName string) int64 {
-	projectID, err := dao.AddProject(models.Project{
-		Name:    projectName,
-		OwnerID: 1,
-	})
-	suite.Nil(err, fmt.Sprintf("Add project failed for %s", projectName))
-	return projectID
-}
-
-func (suite *HandlerSuite) addArt(pid int64, repo string, tag string) int64 {
-	afid, err := dao.AddArtifact(&models.Artifact{
-		PID:    pid,
-		Repo:   repo,
-		Tag:    tag,
-		Digest: digest.FromString(randomString(15)).String(),
-		Kind:   "Docker-Image",
-	})
-	suite.Nil(err, fmt.Sprintf("Add artifact failed for %s", repo))
-	return afid
-}
-
-func (suite *HandlerSuite) addImmutableRule(pid int64) int64 {
-	metadata := &immu_model.Metadata{
-		ProjectID: pid,
-		Priority:  1,
-		Action:    "immutable",
-		Template:  "immutable_template",
-		TagSelectors: []*immu_model.Selector{
-			{
-				Kind:       "doublestar",
-				Decoration: "matches",
-				Pattern:    "release-**",
-			},
-		},
-		ScopeSelectors: map[string][]*immu_model.Selector{
-			"repository": {
-				{
-					Kind:       "doublestar",
-					Decoration: "repoMatches",
-					Pattern:    "**",
-				},
-			},
-		},
-	}
-	id, err := immutabletag.ImmuCtr.CreateImmutableRule(metadata)
-	require.NoError(suite.T(), err, "nil error expected but got %s", err)
-	return id
-}
-
-func (suite *HandlerSuite) TestPutManifestCreated() {
-	projectName := randomString(5)
-
-	projectID := suite.addProject(projectName)
-	immuRuleID := suite.addImmutableRule(projectID)
-	afID := suite.addArt(projectID, projectName+"/photon", "release-1.10")
-	defer func() {
-		dao.DeleteProject(projectID)
-		dao.DeleteArtifact(afID)
-		immutabletag.ImmuCtr.DeleteImmutableRule(immuRuleID)
-	}()
-
-	dgt := digest.FromString(randomString(15)).String()
-	code1 := doPutManifestRequest(projectID, projectName, "photon", "release-1.10", dgt)
-	suite.Equal(http.StatusPreconditionFailed, code1)
-
-	code2 := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt)
-	suite.Equal(http.StatusCreated, code2)
-
-}
-
-func TestMain(m *testing.M) {
-	dao.PrepareTestForPostgresSQL()
-
-	if result := m.Run(); result != 0 {
-		os.Exit(result)
-	}
-}
-
-func TestRunHandlerSuite(t *testing.T) {
-	suite.Run(t, new(HandlerSuite))
-}

src/core/middlewares/inlet.go

@@ -1,58 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package middlewares
-
-import (
-	"errors"
-	"net/http"
-
-	"github.com/goharbor/harbor/src/common/security"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/registryproxy"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-)
-
-var head http.Handler
-var proxy http.Handler
-
-// Init initialize the Proxy instance and handler chain.
-func Init() error {
-	proxy = registryproxy.New()
-	if proxy == nil {
-		return errors.New("get nil when to create proxy")
-	}
-	return nil
-}
-
-// Handle handles the request.
-func Handle(rw http.ResponseWriter, req *http.Request) {
-	securityCtx, ok := security.FromContext(req.Context())
-	if !ok {
-		log.Errorf("failed to get security context in middlerware")
-		// error to get security context, use the default chain.
-		head = New(Middlewares).Create().Then(proxy)
-	} else {
-		// true: the request is from 127.0.0.1, only quota middlewares are applied to request
-		// false: the request is from outside, all of middlewares are applied to the request.
-		if securityCtx.IsSolutionUser() {
-			head = New(MiddlewaresLocal).Create().Then(proxy)
-		} else {
-			head = New(Middlewares).Create().Then(proxy)
-		}
-	}
-
-	customResW := util.NewCustomResponseWriter(rw)
-	head.ServeHTTP(customResW, req)
-}

src/core/middlewares/interceptor/immutable/deletemf.go

@@ -1,67 +0,0 @@
-package immutable
-
-import (
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/models"
-	common_util "github.com/goharbor/harbor/src/common/utils"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	middlerware_err "github.com/goharbor/harbor/src/core/middlewares/util/error"
-	"github.com/goharbor/harbor/src/pkg/art"
-	"github.com/goharbor/harbor/src/pkg/immutabletag/match/rule"
-	"net/http"
-)
-
-// NewDeleteMFInteceptor ....
-func NewDeleteMFInteceptor(mf *util.ManifestInfo) interceptor.Interceptor {
-	return &delmfInterceptor{
-		mf: mf,
-	}
-}
-
-type delmfInterceptor struct {
-	mf *util.ManifestInfo
-}
-
-// HandleRequest ...
-func (dmf *delmfInterceptor) HandleRequest(req *http.Request) (err error) {
-
-	artifactQuery := &models.ArtifactQuery{
-		Digest: dmf.mf.Digest,
-		Repo:   dmf.mf.Repository,
-		PID:    dmf.mf.ProjectID,
-	}
-	var afs []*models.Artifact
-	afs, err = dao.ListArtifacts(artifactQuery)
-	if err != nil {
-		log.Error(err)
-		return
-	}
-	if len(afs) == 0 {
-		return
-	}
-
-	for _, af := range afs {
-		_, repoName := common_util.ParseRepository(dmf.mf.Repository)
-		var matched bool
-		matched, err = rule.NewRuleMatcher().Match(dmf.mf.ProjectID, art.Candidate{
-			Repository:  repoName,
-			Tags:        []string{af.Tag},
-			NamespaceID: dmf.mf.ProjectID,
-		})
-		if err != nil {
-			log.Error(err)
-			return
-		}
-		if matched {
-			return middlerware_err.NewErrImmutable(repoName, af.Tag)
-		}
-	}
-
-	return
-}
-
-// HandleRequest ...
-func (dmf *delmfInterceptor) HandleResponse(w http.ResponseWriter, r *http.Request) {
-}

src/core/middlewares/interceptor/immutable/pushmf.go

@@ -1,65 +0,0 @@
-package immutable
-
-import (
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/models"
-	common_util "github.com/goharbor/harbor/src/common/utils"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	middlerware_err "github.com/goharbor/harbor/src/core/middlewares/util/error"
-	"github.com/goharbor/harbor/src/pkg/art"
-	"github.com/goharbor/harbor/src/pkg/immutabletag/match/rule"
-	"net/http"
-)
-
-// NewPushMFInteceptor ....
-func NewPushMFInteceptor(mf *util.ManifestInfo) interceptor.Interceptor {
-	return &pushmfInterceptor{
-		mf: mf,
-	}
-}
-
-type pushmfInterceptor struct {
-	mf *util.ManifestInfo
-}
-
-// HandleRequest ...
-func (pmf *pushmfInterceptor) HandleRequest(req *http.Request) (err error) {
-
-	_, repoName := common_util.ParseRepository(pmf.mf.Repository)
-	var matched bool
-	matched, err = rule.NewRuleMatcher().Match(pmf.mf.ProjectID, art.Candidate{
-		Repository:  repoName,
-		Tags:        []string{pmf.mf.Tag},
-		NamespaceID: pmf.mf.ProjectID,
-	})
-	if err != nil {
-		log.Error(err)
-		return
-	}
-	if !matched {
-		return
-	}
-
-	artifactQuery := &models.ArtifactQuery{
-		PID:  pmf.mf.ProjectID,
-		Repo: pmf.mf.Repository,
-		Tag:  pmf.mf.Tag,
-	}
-	var afs []*models.Artifact
-	afs, err = dao.ListArtifacts(artifactQuery)
-	if err != nil {
-		log.Error(err)
-		return
-	}
-	if len(afs) == 0 {
-		return
-	}
-
-	return middlerware_err.NewErrImmutable(repoName, pmf.mf.Tag)
-}
-
-// HandleRequest ...
-func (pmf *pushmfInterceptor) HandleResponse(w http.ResponseWriter, r *http.Request) {
-}

src/core/middlewares/listrepo/handler.go

@@ -1,104 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package listrepo
-
-import (
-	"encoding/json"
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"net/http"
-	"net/http/httptest"
-	"regexp"
-	"strconv"
-)
-
-const (
-	catalogURLPattern = `/v2/_catalog`
-)
-
-type listReposHandler struct {
-	next http.Handler
-}
-
-// New ...
-func New(next http.Handler) http.Handler {
-	return &listReposHandler{
-		next: next,
-	}
-}
-
-// ServeHTTP ...
-func (lrh listReposHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	var rec *httptest.ResponseRecorder
-	listReposFlag := matchListRepos(req)
-	if listReposFlag {
-		rec = httptest.NewRecorder()
-		lrh.next.ServeHTTP(rec, req)
-		if rec.Result().StatusCode != http.StatusOK {
-			util.CopyResp(rec, rw)
-			return
-		}
-		var ctlg struct {
-			Repositories []string `json:"repositories"`
-		}
-		decoder := json.NewDecoder(rec.Body)
-		if err := decoder.Decode(&ctlg); err != nil {
-			log.Errorf("Decode repositories error: %v", err)
-			util.CopyResp(rec, rw)
-			return
-		}
-		var entries []string
-		for repo := range ctlg.Repositories {
-			log.Debugf("the repo in the response %s", ctlg.Repositories[repo])
-			exist := dao.RepositoryExists(ctlg.Repositories[repo])
-			if exist {
-				entries = append(entries, ctlg.Repositories[repo])
-			}
-		}
-		type Repos struct {
-			Repositories []string `json:"repositories"`
-		}
-		resp := &Repos{Repositories: entries}
-		respJSON, err := json.Marshal(resp)
-		if err != nil {
-			log.Errorf("Encode repositories error: %v", err)
-			util.CopyResp(rec, rw)
-			return
-		}
-
-		for k, v := range rec.Header() {
-			rw.Header()[k] = v
-		}
-		clen := len(respJSON)
-		rw.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(clen))
-		rw.Write(respJSON)
-		return
-	}
-	lrh.next.ServeHTTP(rw, req)
-}
-
-// matchListRepos checks if the request looks like a request to list repositories.
-func matchListRepos(req *http.Request) bool {
-	if req.Method != http.MethodGet {
-		return false
-	}
-	re := regexp.MustCompile(catalogURLPattern)
-	s := re.FindStringSubmatch(req.URL.Path)
-	if len(s) == 1 {
-		return true
-	}
-	return false
-}

src/core/middlewares/listrepo/handler_test.go

@@ -1,37 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package listrepo
-
-import (
-	"github.com/stretchr/testify/assert"
-	"net/http"
-	"testing"
-)
-
-func TestMatchListRepos(t *testing.T) {
-	assert := assert.New(t)
-	req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil)
-	res1 := matchListRepos(req1)
-	assert.False(res1, "%s %v is not a request to list repos", req1.Method, req1.URL)
-
-	req2, _ := http.NewRequest("GET", "http://127.0.0.1:5000/v2/_catalog", nil)
-	res2 := matchListRepos(req2)
-	assert.True(res2, "%s %v is a request to list repos", req2.Method, req2.URL)
-
-	req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/_catalog", nil)
-	res3 := matchListRepos(req3)
-	assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL)
-
-}

src/core/middlewares/middlewares.go

@@ -15,6 +15,7 @@
 package middlewares
 
 import (
+	"github.com/goharbor/harbor/src/server/middleware/readonly"
 	"net/http"
 	"path"
 	"regexp"
@@ -29,11 +30,28 @@ import (
 )
 
 var (
-	blobURLRe = regexp.MustCompile("^/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String())
+	match         = regexp.MustCompile
+	numericRegexp = match(`[0-9]+`)
+
+	blobURLRe = match("^/v2/(" + reference.NameRegexp.String() + ")/blobs/" + reference.DigestRegexp.String())
 
 	// fetchBlobAPISkipper skip transaction middleware for fetch blob API
 	// because transaction use the ResponseBuffer for the response which will degrade the performance for fetch blob
 	fetchBlobAPISkipper = middleware.MethodAndPathSkipper(http.MethodGet, blobURLRe)
+
+	// readonlySkippers skip the post request when harbor sets to readonly.
+	readonlySkippers = []middleware.Skipper{
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/c/login")),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/c/userExists")),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/c/oidc/onboard")),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/service/notifications/jobs/adminjob/"+numericRegexp.String())),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/service/notifications/jobs/replication/"+numericRegexp.String())),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/service/notifications/jobs/replication/task/"+numericRegexp.String())),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/service/notifications/jobs/webhook/"+numericRegexp.String())),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/service/notifications/jobs/retention/task/"+numericRegexp.String())),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/service/notifications/jobs/schedules/"+numericRegexp.String())),
+		middleware.MethodAndPathSkipper(http.MethodPost, match("^/service/notifications/jobs/webhook/"+numericRegexp.String())),
+	}
 )
 
 // legacyAPISkipper skip middleware for legacy APIs
@@ -52,6 +70,7 @@ func legacyAPISkipper(r *http.Request) bool {
 func MiddleWares() []beego.MiddleWare {
 	return []beego.MiddleWare{
 		requestid.Middleware(),
+		readonly.Middleware(readonlySkippers...),
 		orm.Middleware(legacyAPISkipper),
 		transaction.Middleware(legacyAPISkipper, fetchBlobAPISkipper),
 	}

src/core/middlewares/middlewares_test.go

@@ -65,3 +65,34 @@ func Test_legacyAPISkipper(t *testing.T) {
 		})
 	}
 }
+
+func Test_readonlySkipper(t *testing.T) {
+	type args struct {
+		r *http.Request
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{"login", args{httptest.NewRequest(http.MethodPost, "/c/login", nil)}, true},
+		{"login get", args{httptest.NewRequest(http.MethodGet, "/c/login", nil)}, false},
+		{"onboard", args{httptest.NewRequest(http.MethodPost, "/c/oidc/onboard", nil)}, true},
+		{"user exist", args{httptest.NewRequest(http.MethodPost, "/c/userExists", nil)}, true},
+		{"user exist", args{httptest.NewRequest(http.MethodPost, "/service/notifications/jobs/adminjob/123456", nil)}, true},
+		{"user exist", args{httptest.NewRequest(http.MethodPost, "/service/notifications/jobs/adminjob/abcdefg", nil)}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var pass bool
+			for _, skipper := range readonlySkippers {
+				if got := skipper(tt.args.r); got == tt.want {
+					pass = true
+				}
+			}
+			if !pass {
+				t.Errorf("readonlySkippers() = %v, want %v", tt.args, tt.want)
+			}
+		})
+	}
+}

src/core/middlewares/readonly/hanlder.go

@@ -1,45 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package readonly
-
-import (
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"net/http"
-)
-
-type readonlyHandler struct {
-	next http.Handler
-}
-
-// New ...
-func New(next http.Handler) http.Handler {
-	return &readonlyHandler{
-		next: next,
-	}
-}
-
-// ServeHTTP ...
-func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	if config.ReadOnly() {
-		if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch || req.Method == http.MethodPut {
-			log.Warningf("The request is prohibited in readonly mode, url is: %s", req.URL.Path)
-			http.Error(rw, util.MarshalError("DENIED", "The system is in read only mode. Any modification is prohibited."), http.StatusForbidden)
-			return
-		}
-	}
-	rh.next.ServeHTTP(rw, req)
-}

src/core/middlewares/registryproxy/handler.go

@@ -1,61 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registryproxy
-
-import (
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/config"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-)
-
-type proxyHandler struct {
-	handler http.Handler
-}
-
-// New ...
-func New(urls ...string) http.Handler {
-	var registryURL string
-	var err error
-	if len(urls) > 1 {
-		log.Errorf("the parm, urls should have only 0 or 1 elements")
-		return nil
-	}
-	if len(urls) == 0 {
-		registryURL, err = config.RegistryURL()
-		if err != nil {
-			log.Error(err)
-			return nil
-		}
-	} else {
-		registryURL = urls[0]
-	}
-	targetURL, err := url.Parse(registryURL)
-	if err != nil {
-		log.Error(err)
-		return nil
-	}
-
-	return &proxyHandler{
-		handler: httputil.NewSingleHostReverseProxy(targetURL),
-	}
-
-}
-
-// ServeHTTP ...
-func (ph proxyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	ph.handler.ServeHTTP(rw, req)
-}

src/core/middlewares/regtoken/handler.go

@@ -1,72 +0,0 @@
-package regtoken
-
-import (
-	"github.com/docker/distribution/registry/auth"
-	"github.com/goharbor/harbor/src/common/rbac"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	pkg_token "github.com/goharbor/harbor/src/pkg/token"
-	"github.com/goharbor/harbor/src/pkg/token/claims/registry"
-	"net/http"
-	"strings"
-)
-
-// regTokenHandler is responsible for decoding the registry token in the docker pull request header,
-// as harbor adds customized claims action into registry auth token, the middlerware is for decode it and write it into
-// request context, then for other middlerwares in chain to use it to bypass request validation.
-type regTokenHandler struct {
-	next http.Handler
-}
-
-// New ...
-func New(next http.Handler) http.Handler {
-	return &regTokenHandler{
-		next: next,
-	}
-}
-
-// ServeHTTP ...
-func (r *regTokenHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	imgRaw := req.Context().Value(util.ArtifactInfoCtxKey)
-	if imgRaw == nil {
-		r.next.ServeHTTP(rw, req)
-		return
-	}
-	img, _ := req.Context().Value(util.ArtifactInfoCtxKey).(util.ArtifactInfo)
-	if img.Digest == "" {
-		r.next.ServeHTTP(rw, req)
-		return
-	}
-
-	parts := strings.Split(req.Header.Get("Authorization"), " ")
-	if len(parts) != 2 || strings.ToLower(parts[0]) != "bearer" {
-		r.next.ServeHTTP(rw, req)
-		return
-	}
-
-	rawToken := parts[1]
-	opt := pkg_token.DefaultTokenOptions()
-	regTK, err := pkg_token.Parse(opt, rawToken, &registry.Claim{})
-	if err != nil {
-		log.Errorf("failed to decode reg token: %v, the error is skipped and round the request to native registry.", err)
-		r.next.ServeHTTP(rw, req)
-		return
-	}
-
-	accessItems := []auth.Access{}
-	accessItems = append(accessItems, auth.Access{
-		Resource: auth.Resource{
-			Type: rbac.ResourceRepository.String(),
-			Name: img.Repository,
-		},
-		Action: rbac.ActionScannerPull.String(),
-	})
-
-	accessSet := regTK.Claims.(*registry.Claim).GetAccess()
-	for _, access := range accessItems {
-		if accessSet.Contains(access) {
-			*req = *(req.WithContext(util.NewScannerPullContext(req.Context(), true)))
-		}
-	}
-	r.next.ServeHTTP(rw, req)
-}

src/core/middlewares/regtoken/handler_test.go

@@ -1,55 +0,0 @@
-package regtoken
-
-import (
-	"fmt"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"github.com/stretchr/testify/suite"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"testing"
-)
-
-type HandlerSuite struct {
-	suite.Suite
-}
-
-func doPullManifestRequest(projectName, name, tag string, next ...http.HandlerFunc) int {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	url := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag)
-	req, _ := http.NewRequest("GET", url, nil)
-
-	token := "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6IkNWUTc6REM3NTpHVEROOkxTTUs6VUFJTjpIUUVWOlZVSDQ6Q0lRRDpRV01COlM0Qzc6U0c0STpGRUhYIn0.eyJpc3MiOiJoYXJib3ItdG9rZW4taXNzdWVyIiwic3ViIjoicm9ib3QkZGVtbzExIiwiYXVkIjoiaGFyYm9yLXJlZ2lzdHJ5IiwiZXhwIjoxNTcxNzYzOTI2LCJuYmYiOjE1NzE3NjM4NjYsImlhdCI6MTU3MTc2Mzg2NiwianRpIjoiTnRaZWx4Z01KTUU1MXlEMCIsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoibGlicmFyeS9oZWxsby13b3JsZCIsImFjdGlvbnMiOlsicHVzaCIsIioiLCJwdWxsIiwic2Nhbm5lcnB1bGwiXX1dfQ.GlWuvtoxmChnpvbWaG5901Z9-g63DrzyNUREWlDbR5gnNeuOKjLNyE4QpogAQKx2yYtcGxbqNL3VfJkExJ_gMS0Qw8e10utGOawwqD4oqf_J06eKq4HzpZJengZfcjMA4g2RoeOlqdVdwimB_PdX9vkBO1od0wX0Cc2v0p2w5TkibcThKRoeLeVs2oRewkKLuVHNSM8wwRIlAvpWJuNnvRCFlHRkLcZM_KpGXqT7H-PZETTisWCi1pMxeYEwIsDFLlTKdV8LaiDeDmH-RaLOsuyAySYEW9Ynk5K3P_dUl2c_SYQXloPyi0MvXxSn6EWE4eHF2oQDM_SvIzR9sOVB8TtjMjKKMQ4yr_mqgMcfEpnInJATExBR56wmxNdLESncHl8rUYCe2jCjQFuR9NGQA1tGdjI4NoBN-OVD0dBs9rm_mkb2tgD-3gEhyzAw6hg0uzDsF7bj5Aq8scoi42UurhX2bZM89s4-TWBp4DWuBG0HDiwpOiBvB3RMm6MpQxsqrl0hQm_WH18L6QCknAW2e3d_6DJWJ0eBzISrhDr7LkqJKl1J8pv4zqoh_EUVeLyzTmjEULm-VbnpVF4wW5yTLF3S6F7Ox4vwWtVfi1XQNVOcJDB3VPUsRgiTTuCW-ZGcBLw-OdIcwaJ3T_QZkEjUw1f6i1JcGa0Mpgl83aLiSdQ 0xc0003c77c0 map[alg:RS256 kid:CVQ7:DC75:GTDN:LSMK:UAIN:HQEV:VUH4:CIQD:QWMB:S4C7:SG4I:FEHX typ:JWT] 0xc000496000 GlWuvtoxmChnpvbWaG5901Z9-g63DrzyNUREWlDbR5gnNeuOKjLNyE4QpogAQKx2yYtcGxbqNL3VfJkExJ_gMS0Qw8e10utGOawwqD4oqf_J06eKq4HzpZJengZfcjMA4g2RoeOlqdVdwimB_PdX9vkBO1od0wX0Cc2v0p2w5TkibcThKRoeLeVs2oRewkKLuVHNSM8wwRIlAvpWJuNnvRCFlHRkLcZM_KpGXqT7H-PZETTisWCi1pMxeYEwIsDFLlTKdV8LaiDeDmH-RaLOsuyAySYEW9Ynk5K3P_dUl2c_SYQXloPyi0MvXxSn6EWE4eHF2oQDM_SvIzR9sOVB8TtjMjKKMQ4yr_mqgMcfEpnInJATExBR56wmxNdLESncHl8rUYCe2jCjQFuR9NGQA1tGdjI4NoBN-OVD0dBs9rm_mkb2tgD-3gEhyzAw6hg0uzDsF7bj5Aq8scoi42UurhX2bZM89s4-TWBp4DWuBG0HDiwpOiBvB3RMm6MpQxsqrl0hQm_WH18L6QCknAW2e3d_6DJWJ0eBzISrhDr7LkqJKl1J8pv4zqoh_EUVeLyzTmjEULm-VbnpVF4wW5yTLF3S6F7Ox4vwWtVfi1XQNVOcJDB3VPUsRgiTTuCW-ZGcBLw-OdIcwaJ3T_QZkEjUw1f6i1JcGa0Mpgl83aLiSdQ"
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
-	rr := httptest.NewRecorder()
-
-	var n http.HandlerFunc
-	if len(next) > 0 {
-		n = next[0]
-	} else {
-		n = func(w http.ResponseWriter, req *http.Request) {
-			w.WriteHeader(http.StatusNotFound)
-		}
-	}
-
-	h := New(http.HandlerFunc(n))
-	h.ServeHTTP(util.NewCustomResponseWriter(rr), req)
-
-	return rr.Code
-}
-
-func (suite *HandlerSuite) TestPullManifest() {
-	code1 := doPullManifestRequest("library", "photon", "release-1.10")
-	suite.Equal(http.StatusNotFound, code1)
-}
-
-func TestMain(m *testing.M) {
-	if result := m.Run(); result != 0 {
-		os.Exit(result)
-	}
-}
-
-func TestRunHandlerSuite(t *testing.T) {
-	suite.Run(t, new(HandlerSuite))
-}

src/core/middlewares/sizequota/builder.go

@@ -1,208 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sizequota
-
-import (
-	"fmt"
-	"net/http"
-	"strconv"
-
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor/quota"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-)
-
-var (
-	defaultBuilders = []interceptor.Builder{
-		&blobStreamUploadBuilder{},
-		&blobStorageQuotaBuilder{},
-		&manifestCreationBuilder{},
-		&manifestDeletionBuilder{},
-	}
-)
-
-// blobStreamUploadBuilder interceptor builder for PATCH /v2/<name>/blobs/uploads/<uuid>
-type blobStreamUploadBuilder struct{}
-
-func (*blobStreamUploadBuilder) Build(req *http.Request) (interceptor.Interceptor, error) {
-	if !match(req, http.MethodPatch, blobUploadURLRe) {
-		return nil, nil
-	}
-
-	s := blobUploadURLRe.FindStringSubmatch(req.URL.Path)
-	uuid := s[2]
-
-	onResponse := func(w http.ResponseWriter, req *http.Request) {
-		if !config.QuotaPerProjectEnable() {
-			return
-		}
-
-		size, err := parseUploadedBlobSize(w)
-		if err != nil {
-			log.Errorf("failed to parse uploaded blob size for upload %s, error: %v", uuid, err)
-			return
-		}
-
-		ok, err := setUploadedBlobSize(uuid, size)
-		if err != nil {
-			log.Errorf("failed to update blob update size for upload %s, error: %v", uuid, err)
-			return
-		}
-
-		if !ok {
-			// ToDo discuss what to do here.
-			log.Errorf("fail to set bunk: %s size: %d in redis, it causes unable to set correct quota for the artifact", uuid, size)
-		}
-	}
-
-	return interceptor.ResponseInterceptorFunc(onResponse), nil
-}
-
-// blobStorageQuotaBuilder interceptor builder for these requests
-// PUT  /v2/<name>/blobs/uploads/<uuid>?digest=<digest>
-// POST /v2/<name>/blobs/uploads/?mount=<digest>&from=<repository name>
-type blobStorageQuotaBuilder struct{}
-
-func (*blobStorageQuotaBuilder) Build(req *http.Request) (interceptor.Interceptor, error) {
-	parseBlobInfo := getBlobInfoParser(req)
-	if parseBlobInfo == nil {
-		return nil, nil
-	}
-
-	info, err := parseBlobInfo(req)
-	if err != nil {
-		return nil, err
-	}
-
-	// replace req with blob info context
-	*req = *(req.WithContext(util.NewBlobInfoContext(req.Context(), info)))
-
-	opts := []quota.Option{
-		quota.EnforceResources(config.QuotaPerProjectEnable()),
-		quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
-		quota.WithAction(quota.AddAction),
-		quota.StatusCode(http.StatusCreated), // NOTICE: mount blob and blob upload complete both return 201 when success
-		quota.OnResources(computeResourcesForBlob),
-		quota.MutexKeys(info.MutexKey()),
-		quota.OnFulfilled(func(http.ResponseWriter, *http.Request) error {
-			return syncBlobInfoToProject(info)
-		}),
-	}
-
-	return quota.New(opts...), nil
-}
-
-// manifestCreationBuilder interceptor builder for the request PUT /v2/<name>/manifests/<reference>
-type manifestCreationBuilder struct{}
-
-func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Interceptor, error) {
-	if match, _, _ := util.MatchPushManifest(req); !match {
-		return nil, nil
-	}
-
-	info, err := util.ParseManifestInfoFromReq(req)
-	if err != nil {
-		return nil, err
-	}
-
-	// Replace request with manifests info context
-	*req = *req.WithContext(util.NewManifestInfoContext(req.Context(), info))
-
-	// Sync manifest layers to blobs for foreign layers not pushed and they are not in blob table
-	if err := info.SyncBlobs(); err != nil {
-		log.Warningf("Failed to sync blobs, error: %v", err)
-	}
-
-	opts := []quota.Option{
-		quota.EnforceResources(config.QuotaPerProjectEnable()),
-		quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
-		quota.WithAction(quota.AddAction),
-		quota.StatusCode(http.StatusCreated),
-		quota.OnResources(computeResourcesForManifestCreation),
-		quota.MutexKeys(info.MutexKey("size")),
-		quota.OnFulfilled(func(http.ResponseWriter, *http.Request) error {
-			// manifest created, sync manifest itself as blob to blob and project_blob table
-			blobInfo, err := parseBlobInfoFromManifest(req)
-			if err != nil {
-				return err
-			}
-
-			if err := syncBlobInfoToProject(blobInfo); err != nil {
-				return err
-			}
-
-			// sync blobs from manifest which are not in project to project_blob table
-			blobs, err := info.GetBlobsNotInProject()
-			if err != nil {
-				return err
-			}
-
-			_, err = dao.AddBlobsToProject(info.ProjectID, blobs...)
-
-			return err
-		}),
-	}
-
-	return quota.New(opts...), nil
-}
-
-// deleteManifestBuilder interceptor builder for the request DELETE /v2/<name>/manifests/<reference>
-type manifestDeletionBuilder struct{}
-
-func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Interceptor, error) {
-	if match, _, _ := util.MatchDeleteManifest(req); !match {
-		return nil, nil
-	}
-
-	info, ok := util.ManifestInfoFromContext(req.Context())
-	if !ok {
-		var err error
-		info, err = util.ParseManifestInfoFromPath(req)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse manifest, error %v", err)
-		}
-
-		// Manifest info will be used by computeResourcesForDeleteManifest
-		*req = *(req.WithContext(util.NewManifestInfoContext(req.Context(), info)))
-	}
-
-	blobs, err := dao.GetBlobsByArtifact(info.Digest)
-	if err != nil {
-		return nil, fmt.Errorf("failed to query blobs of %s, error: %v", info.Digest, err)
-	}
-
-	mutexKeys := []string{info.MutexKey("size")}
-	for _, blob := range blobs {
-		mutexKeys = append(mutexKeys, info.BlobMutexKey(blob))
-	}
-
-	opts := []quota.Option{
-		quota.EnforceResources(config.QuotaPerProjectEnable()),
-		quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
-		quota.WithAction(quota.SubtractAction),
-		quota.StatusCode(http.StatusAccepted),
-		quota.OnResources(computeResourcesForManifestDeletion),
-		quota.MutexKeys(mutexKeys...),
-		quota.OnFulfilled(func(http.ResponseWriter, *http.Request) error {
-			blobs := info.ExclusiveBlobs
-			return dao.RemoveBlobsFromProject(info.ProjectID, blobs...)
-		}),
-	}
-
-	return quota.New(opts...), nil
-}

src/core/middlewares/sizequota/handler.go

@@ -1,89 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sizequota
-
-import (
-	"fmt"
-	"net/http"
-
-	"github.com/goharbor/harbor/src/common/quota"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/middlewares/interceptor"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-)
-
-type sizeQuotaHandler struct {
-	builders []interceptor.Builder
-	next     http.Handler
-}
-
-// New ...
-func New(next http.Handler, builders ...interceptor.Builder) http.Handler {
-	if len(builders) == 0 {
-		builders = defaultBuilders
-	}
-
-	return &sizeQuotaHandler{
-		builders: builders,
-		next:     next,
-	}
-}
-
-// ServeHTTP ...
-func (h *sizeQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	interceptor, err := h.getInterceptor(req)
-	if err != nil {
-		log.Warningf("Error occurred when to handle request in size quota handler: %v", err)
-		http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in size quota handler: %v", err)),
-			http.StatusInternalServerError)
-		return
-	}
-
-	if interceptor == nil {
-		h.next.ServeHTTP(rw, req)
-		return
-	}
-
-	if err := interceptor.HandleRequest(req); err != nil {
-		log.Warningf("Error occurred when to handle request in size quota handler: %v", err)
-		if _, ok := err.(quota.Errors); ok {
-			util.FireQuotaEvent(req, 1, err.Error())
-			http.Error(rw, util.MarshalError("DENIED", fmt.Sprintf("Quota exceeded when processing the request of %v", err)), http.StatusForbidden)
-			return
-		}
-		http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in size quota handler: %v", err)),
-			http.StatusInternalServerError)
-		return
-	}
-
-	h.next.ServeHTTP(rw, req)
-
-	interceptor.HandleResponse(rw, req)
-}
-
-func (h *sizeQuotaHandler) getInterceptor(req *http.Request) (interceptor.Interceptor, error) {
-	for _, builder := range h.builders {
-		interceptor, err := builder.Build(req)
-		if err != nil {
-			return nil, err
-		}
-
-		if interceptor != nil {
-			return interceptor, nil
-		}
-	}
-
-	return nil, nil
-}

src/core/middlewares/sizequota/handler_test.go

@@ -1,751 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sizequota
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strconv"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/docker/distribution"
-	"github.com/docker/distribution/manifest"
-	"github.com/docker/distribution/manifest/schema2"
-	"github.com/goharbor/harbor/src/common"
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/middlewares/countquota"
-	"github.com/goharbor/harbor/src/core/middlewares/util"
-	"github.com/goharbor/harbor/src/pkg/types"
-	"github.com/opencontainers/go-digest"
-	"github.com/stretchr/testify/suite"
-)
-
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
-func genUUID() string {
-	b := make([]byte, 16)
-
-	if _, err := rand.Read(b); err != nil {
-		return ""
-	}
-
-	return fmt.Sprintf("%X-%X-%X-%X-%X", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
-}
-
-func getProjectCountUsage(projectID int64) (int64, error) {
-	usage := models.QuotaUsage{Reference: "project", ReferenceID: fmt.Sprintf("%d", projectID)}
-	err := dao.GetOrmer().Read(&usage, "reference", "reference_id")
-	if err != nil {
-		return 0, err
-	}
-	used, err := types.NewResourceList(usage.Used)
-	if err != nil {
-		return 0, err
-	}
-
-	return used[types.ResourceCount], nil
-}
-
-func getProjectStorageUsage(projectID int64) (int64, error) {
-	usage := models.QuotaUsage{Reference: "project", ReferenceID: fmt.Sprintf("%d", projectID)}
-	err := dao.GetOrmer().Read(&usage, "reference", "reference_id")
-	if err != nil {
-		return 0, err
-	}
-	used, err := types.NewResourceList(usage.Used)
-	if err != nil {
-		return 0, err
-	}
-
-	return used[types.ResourceStorage], nil
-}
-
-func randomString(n int) string {
-	const letterBytes = "abcdefghijklmnopqrstuvwxyz"
-
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = letterBytes[rand.Intn(len(letterBytes))]
-	}
-
-	return string(b)
-}
-
-func makeManifest(configSize int64, layerSizes []int64) schema2.Manifest {
-	manifest := schema2.Manifest{
-		Versioned: manifest.Versioned{SchemaVersion: 2, MediaType: schema2.MediaTypeManifest},
-		Config: distribution.Descriptor{
-			MediaType: schema2.MediaTypeImageConfig,
-			Size:      configSize,
-			Digest:    digest.FromString(randomString(15)),
-		},
-	}
-
-	for _, size := range layerSizes {
-		manifest.Layers = append(manifest.Layers, distribution.Descriptor{
-			MediaType: schema2.MediaTypeLayer,
-			Size:      size,
-			Digest:    digest.FromString(randomString(15)),
-		})
-	}
-
-	return manifest
-}
-
-func manifestWithAdditionalLayers(raw schema2.Manifest, layerSizes []int64) schema2.Manifest {
-	var manifest schema2.Manifest
-
-	manifest.Versioned = raw.Versioned
-	manifest.Config = raw.Config
-	manifest.Layers = append(manifest.Layers, raw.Layers...)
-
-	for _, size := range layerSizes {
-		manifest.Layers = append(manifest.Layers, distribution.Descriptor{
-			MediaType: schema2.MediaTypeLayer,
-			Size:      size,
-			Digest:    digest.FromString(randomString(15)),
-		})
-	}
-
-	return manifest
-}
-
-func manifestWithAdditionalForeignLayers(raw schema2.Manifest, layerSizes []int64) schema2.Manifest {
-	var manifest schema2.Manifest
-
-	manifest.Versioned = raw.Versioned
-	manifest.Config = raw.Config
-	manifest.Layers = append(manifest.Layers, raw.Layers...)
-
-	for _, size := range layerSizes {
-		manifest.Layers = append(manifest.Layers, distribution.Descriptor{
-			MediaType: schema2.MediaTypeForeignLayer,
-			Size:      size,
-			Digest:    digest.FromString(randomString(15)),
-		})
-	}
-
-	return manifest
-}
-
-func digestOfManifest(manifest schema2.Manifest) string {
-	bytes, _ := json.Marshal(manifest)
-
-	return digest.FromBytes(bytes).String()
-}
-
-func sizeOfManifest(manifest schema2.Manifest) int64 {
-	bytes, _ := json.Marshal(manifest)
-
-	return int64(len(bytes))
-}
-
-func sizeOfImage(manifest schema2.Manifest) int64 {
-	totalSizeOfLayers := manifest.Config.Size
-
-	for _, layer := range manifest.Layers {
-		if layer.MediaType != schema2.MediaTypeForeignLayer {
-			totalSizeOfLayers += layer.Size
-		}
-	}
-
-	return sizeOfManifest(manifest) + totalSizeOfLayers
-}
-
-func doHandle(req *http.Request, next ...http.HandlerFunc) int {
-	rr := httptest.NewRecorder()
-
-	var n http.HandlerFunc
-	if len(next) > 0 {
-		n = next[0]
-	} else {
-		n = func(w http.ResponseWriter, req *http.Request) {
-			w.WriteHeader(http.StatusCreated)
-		}
-	}
-
-	h := New(http.HandlerFunc(n))
-	h.ServeHTTP(util.NewCustomResponseWriter(rr), req)
-
-	return rr.Code
-}
-
-func patchBlobUpload(projectName, name, uuid, blobDigest string, chunkSize int64) {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	url := fmt.Sprintf("/v2/%s/blobs/uploads/%s?digest=%s", repository, uuid, blobDigest)
-	req, _ := http.NewRequest(http.MethodPatch, url, nil)
-
-	doHandle(req, func(w http.ResponseWriter, req *http.Request) {
-		w.WriteHeader(http.StatusAccepted)
-		w.Header().Add("Range", fmt.Sprintf("0-%d", chunkSize-1))
-	})
-}
-
-func putBlobUpload(projectName, name, uuid, blobDigest string, blobSize ...int64) {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	url := fmt.Sprintf("/v2/%s/blobs/uploads/%s?digest=%s", repository, uuid, blobDigest)
-	req, _ := http.NewRequest(http.MethodPut, url, nil)
-	if len(blobSize) > 0 {
-		req.Header.Add("Content-Length", strconv.FormatInt(blobSize[0], 10))
-	}
-
-	doHandle(req, func(w http.ResponseWriter, req *http.Request) {
-		w.WriteHeader(http.StatusCreated)
-	})
-}
-
-func mountBlob(projectName, name, blobDigest, fromRepository string) {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	url := fmt.Sprintf("/v2/%s/blobs/uploads/?mount=%s&from=%s", repository, blobDigest, fromRepository)
-	req, _ := http.NewRequest(http.MethodPost, url, nil)
-
-	doHandle(req, func(w http.ResponseWriter, req *http.Request) {
-		w.WriteHeader(http.StatusCreated)
-	})
-}
-
-func deleteManifest(projectName, name, digest string, accepted ...func() bool) {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	url := fmt.Sprintf("/v2/%s/manifests/%s", repository, digest)
-	req, _ := http.NewRequest(http.MethodDelete, url, nil)
-
-	next := countquota.New(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		if len(accepted) > 0 {
-			if accepted[0]() {
-				w.WriteHeader(http.StatusAccepted)
-			} else {
-				w.WriteHeader(http.StatusNotFound)
-			}
-
-			return
-		}
-
-		w.WriteHeader(http.StatusAccepted)
-	}))
-
-	rr := httptest.NewRecorder()
-	h := New(next)
-	h.ServeHTTP(util.NewCustomResponseWriter(rr), req)
-}
-
-func putManifest(projectName, name, tag string, manifest schema2.Manifest) {
-	repository := fmt.Sprintf("%s/%s", projectName, name)
-
-	buf, _ := json.Marshal(manifest)
-
-	url := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag)
-	req, _ := http.NewRequest(http.MethodPut, url, bytes.NewReader(buf))
-	req.Header.Add("Content-Type", manifest.MediaType)
-
-	next := countquota.New(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		w.WriteHeader(http.StatusCreated)
-	}))
-
-	rr := httptest.NewRecorder()
-	h := New(next)
-	h.ServeHTTP(util.NewCustomResponseWriter(rr), req)
-}
-
-func pushImage(projectName, name, tag string, manifest schema2.Manifest) {
-	putBlobUpload(projectName, name, genUUID(), manifest.Config.Digest.String(), manifest.Config.Size)
-	for _, layer := range manifest.Layers {
-		if layer.MediaType != schema2.MediaTypeForeignLayer {
-			putBlobUpload(projectName, name, genUUID(), layer.Digest.String(), layer.Size)
-		}
-	}
-
-	putManifest(projectName, name, tag, manifest)
-}
-
-func withProject(f func(int64, string)) {
-	projectName := randomString(5)
-
-	projectID, err := dao.AddProject(models.Project{
-		Name:    projectName,
-		OwnerID: 1,
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	defer func() {
-		dao.DeleteProject(projectID)
-	}()
-
-	f(projectID, projectName)
-}
-
-type HandlerSuite struct {
-	suite.Suite
-}
-
-func (suite *HandlerSuite) checkCountUsage(expected, projectID int64) {
-	count, err := getProjectCountUsage(projectID)
-	suite.Nil(err, fmt.Sprintf("Failed to get count usage of project %d, error: %v", projectID, err))
-	suite.Equal(expected, count, "Failed to check count usage for project %d", projectID)
-}
-
-func (suite *HandlerSuite) checkStorageUsage(expected, projectID int64) {
-	value, err := getProjectStorageUsage(projectID)
-	suite.Nil(err, fmt.Sprintf("Failed to get storage usage of project %d, error: %v", projectID, err))
-	suite.Equal(expected, value, "Failed to check storage usage for project %d", projectID)
-}
-
-func (suite *HandlerSuite) TearDownTest() {
-	for _, table := range []string{
-		"artifact", "blob",
-		"artifact_blob", "project_blob",
-		"quota", "quota_usage",
-	} {
-		dao.ClearTable(table)
-	}
-}
-
-func (suite *HandlerSuite) TestPatchBlobUpload() {
-	withProject(func(projectID int64, projectName string) {
-		uuid := genUUID()
-		blobDigest := digest.FromString(randomString(15)).String()
-		patchBlobUpload(projectName, "photon", uuid, blobDigest, 1024)
-		size, err := getUploadedBlobSize(uuid)
-		suite.Nil(err)
-		suite.Equal(int64(1024), size)
-	})
-}
-
-func (suite *HandlerSuite) TestPutBlobUpload() {
-	withProject(func(projectID int64, projectName string) {
-		uuid := genUUID()
-		blobDigest := digest.FromString(randomString(15)).String()
-		putBlobUpload(projectName, "photon", uuid, blobDigest, 1024)
-		suite.checkStorageUsage(1024, projectID)
-
-		blob, err := dao.GetBlob(blobDigest)
-		suite.Nil(err)
-		suite.Equal(int64(1024), blob.Size)
-	})
-}
-
-func (suite *HandlerSuite) TestPutBlobUploadWithPatch() {
-	withProject(func(projectID int64, projectName string) {
-		uuid := genUUID()
-		blobDigest := digest.FromString(randomString(15)).String()
-		patchBlobUpload(projectName, "photon", uuid, blobDigest, 1024)
-
-		putBlobUpload(projectName, "photon", uuid, blobDigest)
-		suite.checkStorageUsage(1024, projectID)
-
-		blob, err := dao.GetBlob(blobDigest)
-		suite.Nil(err)
-		suite.Equal(int64(1024), blob.Size)
-	})
-}
-
-func (suite *HandlerSuite) TestMountBlob() {
-	withProject(func(projectID int64, projectName string) {
-		blobDigest := digest.FromString(randomString(15)).String()
-		putBlobUpload(projectName, "photon", genUUID(), blobDigest, 1024)
-		suite.checkStorageUsage(1024, projectID)
-
-		repository := fmt.Sprintf("%s/%s", projectName, "photon")
-
-		withProject(func(projectID int64, projectName string) {
-			mountBlob(projectName, "harbor", blobDigest, repository)
-			suite.checkStorageUsage(1024, projectID)
-		})
-	})
-}
-
-func (suite *HandlerSuite) TestPutManifestCreated() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(100, []int64{100, 100})
-
-		putBlobUpload(projectName, "photon", genUUID(), manifest.Config.Digest.String(), manifest.Config.Size)
-		for _, layer := range manifest.Layers {
-			putBlobUpload(projectName, "photon", genUUID(), layer.Digest.String(), layer.Size)
-		}
-
-		putManifest(projectName, "photon", "latest", manifest)
-
-		suite.checkStorageUsage(int64(300+sizeOfManifest(manifest)), projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestDeleteManifest() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-
-		deleteManifest(projectName, "photon", digestOfManifest(manifest))
-		suite.checkStorageUsage(0, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestImageWithForeignLayers() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := manifestWithAdditionalForeignLayers(makeManifest(1, []int64{2, 3, 4, 5}), []int64{6, 7})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-		suite.checkStorageUsage(sizeOfManifest(manifest)+1+2+3+4+5, projectID)
-
-		blobs, err := dao.GetBlobsByArtifact(digestOfManifest(manifest))
-		if suite.Nil(err) {
-			suite.Len(blobs, 8)
-		}
-
-		deleteManifest(projectName, "photon", digestOfManifest(manifest))
-		suite.checkStorageUsage(0, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestImageOverwrite() {
-	withProject(func(projectID int64, projectName string) {
-		manifest1 := makeManifest(1, []int64{2, 3, 4, 5})
-		size1 := sizeOfImage(manifest1)
-		pushImage(projectName, "photon", "latest", manifest1)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size1, projectID)
-
-		manifest2 := makeManifest(1, []int64{2, 3, 4, 5})
-		size2 := sizeOfImage(manifest2)
-		pushImage(projectName, "photon", "latest", manifest2)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size1+size2, projectID)
-
-		manifest3 := makeManifest(1, []int64{2, 3, 4, 5})
-		size3 := sizeOfImage(manifest2)
-		pushImage(projectName, "photon", "latest", manifest3)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size1+size2+size3, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestPushImageMultiTimes() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestPushImageToSameRepository() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "photon", "dev", manifest)
-		suite.checkCountUsage(2, projectID)
-		suite.checkStorageUsage(size, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestPushImageToDifferentRepositories() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "mysql", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "redis", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "postgres", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestPushImageToDifferentProjects() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "mysql", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-
-		withProject(func(id int64, name string) {
-			manifest := makeManifest(1, []int64{2, 3, 4, 5})
-			size := sizeOfImage(manifest)
-
-			pushImage(name, "mysql", "latest", manifest)
-			suite.checkStorageUsage(size, id)
-
-			suite.checkStorageUsage(size, projectID)
-		})
-	})
-}
-
-func (suite *HandlerSuite) TestDeleteManifestShareLayersInSameRepository() {
-	withProject(func(projectID int64, projectName string) {
-		manifest1 := makeManifest(1, []int64{2, 3, 4, 5})
-		size1 := sizeOfImage(manifest1)
-
-		pushImage(projectName, "mysql", "latest", manifest1)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size1, projectID)
-
-		manifest2 := manifestWithAdditionalLayers(manifest1, []int64{6, 7})
-		pushImage(projectName, "mysql", "dev", manifest2)
-		suite.checkCountUsage(2, projectID)
-
-		totalSize := size1 + sizeOfManifest(manifest2) + 6 + 7
-		suite.checkStorageUsage(totalSize, projectID)
-
-		deleteManifest(projectName, "mysql", digestOfManifest(manifest1))
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(totalSize-sizeOfManifest(manifest1), projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestDeleteManifestShareLayersInDifferentRepositories() {
-	withProject(func(projectID int64, projectName string) {
-		manifest1 := makeManifest(1, []int64{2, 3, 4, 5})
-		size1 := sizeOfImage(manifest1)
-
-		pushImage(projectName, "mysql", "latest", manifest1)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size1, projectID)
-
-		pushImage(projectName, "mysql", "dev", manifest1)
-		suite.checkCountUsage(2, projectID)
-		suite.checkStorageUsage(size1, projectID)
-
-		manifest2 := manifestWithAdditionalLayers(manifest1, []int64{6, 7})
-		pushImage(projectName, "mariadb", "latest", manifest2)
-		suite.checkCountUsage(3, projectID)
-
-		totalSize := size1 + sizeOfManifest(manifest2) + 6 + 7
-		suite.checkStorageUsage(totalSize, projectID)
-
-		deleteManifest(projectName, "mysql", digestOfManifest(manifest1))
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(totalSize-sizeOfManifest(manifest1), projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestDeleteManifestInSameRepository() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "photon", "dev", manifest)
-		suite.checkCountUsage(2, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		deleteManifest(projectName, "photon", digestOfManifest(manifest))
-		suite.checkCountUsage(0, projectID)
-		suite.checkStorageUsage(0, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestDeleteManifestInDifferentRepositories() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "mysql", "latest", manifest)
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "mysql", "5.6", manifest)
-		suite.checkCountUsage(2, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "redis", "latest", manifest)
-		suite.checkCountUsage(3, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		deleteManifest(projectName, "redis", digestOfManifest(manifest))
-		suite.checkCountUsage(2, projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		pushImage(projectName, "redis", "latest", manifest)
-		suite.checkCountUsage(3, projectID)
-		suite.checkStorageUsage(size, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestDeleteManifestInDifferentProjects() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "mysql", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-
-		withProject(func(id int64, name string) {
-			pushImage(name, "mysql", "latest", manifest)
-			suite.checkStorageUsage(size, id)
-
-			suite.checkStorageUsage(size, projectID)
-			deleteManifest(projectName, "mysql", digestOfManifest(manifest))
-			suite.checkCountUsage(0, projectID)
-			suite.checkStorageUsage(0, projectID)
-		})
-
-	})
-}
-
-func (suite *HandlerSuite) TestPushDeletePush() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-
-		deleteManifest(projectName, "photon", digestOfManifest(manifest))
-		suite.checkStorageUsage(0, projectID)
-
-		pushImage(projectName, "photon", "latest", manifest)
-		suite.checkStorageUsage(size, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestPushImageRace() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		size := sizeOfImage(manifest)
-
-		var wg sync.WaitGroup
-		for i := 0; i < 10; i++ {
-			wg.Add(1)
-			go func() {
-				defer wg.Done()
-				pushImage(projectName, "photon", "latest", manifest)
-			}()
-		}
-		wg.Wait()
-
-		suite.checkCountUsage(1, projectID)
-		suite.checkStorageUsage(size, projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestDeleteImageRace() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		pushImage(projectName, "photon", "latest", manifest)
-
-		count := 100
-		size := sizeOfImage(manifest)
-		for i := 0; i < count; i++ {
-			manifest := makeManifest(1, []int64{2, 3, 4, 5})
-			pushImage(projectName, "mysql", fmt.Sprintf("tag%d", i), manifest)
-			size += sizeOfImage(manifest)
-		}
-
-		suite.checkCountUsage(int64(count+1), projectID)
-		suite.checkStorageUsage(size, projectID)
-
-		var wg sync.WaitGroup
-		for i := 0; i < 10; i++ {
-			wg.Add(1)
-			go func(i int) {
-				defer wg.Done()
-				deleteManifest(projectName, "photon", digestOfManifest(manifest), func() bool {
-					return i == 0
-				})
-			}(i)
-		}
-		wg.Wait()
-
-		suite.checkCountUsage(int64(count), projectID)
-		suite.checkStorageUsage(size-sizeOfImage(manifest), projectID)
-	})
-}
-
-func (suite *HandlerSuite) TestDisableProjectQuota() {
-	withProject(func(projectID int64, projectName string) {
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		pushImage(projectName, "photon", "latest", manifest)
-
-		quotas, err := dao.ListQuotas(&models.QuotaQuery{
-			Reference:   "project",
-			ReferenceID: strconv.FormatInt(projectID, 10),
-		})
-
-		suite.Nil(err)
-		suite.Len(quotas, 1)
-	})
-
-	withProject(func(projectID int64, projectName string) {
-		cfg := config.GetCfgManager()
-		cfg.Set(common.QuotaPerProjectEnable, false)
-		defer cfg.Set(common.QuotaPerProjectEnable, true)
-
-		manifest := makeManifest(1, []int64{2, 3, 4, 5})
-		pushImage(projectName, "photon", "latest", manifest)
-
-		quotas, err := dao.ListQuotas(&models.QuotaQuery{
-			Reference:   "project",
-			ReferenceID: strconv.FormatInt(projectID, 10),
-		})
-
-		suite.Nil(err)
-		suite.Len(quotas, 0)
-	})
-}
-
-func TestMain(m *testing.M) {
-	config.Init()
-	dao.PrepareTestForPostgresSQL()
-
-	if result := m.Run(); result != 0 {
-		os.Exit(result)
-	}
-}
-
-func TestRunHandlerSuite(t *testing.T) {
-	suite.Run(t, new(HandlerSuite))
-}