Iterate all paginations when listing projects and repositories (#7660)

Iterate all paginations when listing projects and repositories

Signed-off-by: Wenkai Yin <yinw@vmware.com>

src/common/http/client.go

@@ -17,9 +17,13 @@ package http
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
+	"reflect"
+	"strings"
 
 	"github.com/goharbor/harbor/src/common/http/modifier"
 )
@@ -168,3 +172,63 @@ func (c *Client) do(req *http.Request) ([]byte, error) {
 
 	return data, nil
 }
+
+// GetAndIteratePagination iterates the pagination header and returns all resources
+// The parameter "v" must be a pointer to a slice
+func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
+	url, err := url.Parse(endpoint)
+	if err != nil {
+		return err
+	}
+
+	rv := reflect.ValueOf(v)
+	if rv.Kind() != reflect.Ptr {
+		return errors.New("v should be a pointer to a slice")
+	}
+	elemType := rv.Elem().Type()
+	if elemType.Kind() != reflect.Slice {
+		return errors.New("v should be a pointer to a slice")
+	}
+
+	resources := reflect.Indirect(reflect.New(elemType))
+	for len(endpoint) > 0 {
+		req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+		if err != nil {
+			return err
+		}
+		resp, err := c.Do(req)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode < 200 || resp.StatusCode > 299 {
+			return &Error{
+				Code:    resp.StatusCode,
+				Message: string(data),
+			}
+		}
+
+		res := reflect.New(elemType)
+		if err = json.Unmarshal(data, res.Interface()); err != nil {
+			return err
+		}
+		resources = reflect.AppendSlice(resources, reflect.Indirect(res))
+
+		endpoint = ""
+		link := resp.Header.Get("Link")
+		for _, str := range strings.Split(link, ",") {
+			if strings.HasSuffix(str, `rel="next"`) &&
+				strings.Index(str, "<") >= 0 &&
+				strings.Index(str, ">") >= 0 {
+				endpoint = url.Scheme + "://" + url.Host + str[strings.Index(str, "<")+1:strings.Index(str, ">")]
+				break
+			}
+		}
+	}
+	rv.Elem().Set(resources)
+	return nil
+}

src/replication/adapter/harbor/adapter.go

@@ -211,8 +211,8 @@ type project struct {
 
 func (a *adapter) getProjects(name string) ([]*project, error) {
 	projects := []*project{}
-	url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=1000", a.coreServiceURL, name)
+	url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=500", a.coreServiceURL, name)
-	if err := a.client.Get(url, &projects); err != nil {
+	if err := a.client.GetAndIteratePagination(url, &projects); err != nil {
 		return nil, err
 	}
 	return projects, nil
@@ -243,3 +243,12 @@ func (a *adapter) getProject(name string) (*project, error) {
 	}
 	return nil, nil
 }
+
+func (a *adapter) getRepositories(projectID int64) ([]*repository, error) {
+	repositories := []*repository{}
+	url := fmt.Sprintf("%s/api/repositories?project_id=%d&page=1&page_size=500", a.coreServiceURL, projectID)
+	if err := a.client.GetAndIteratePagination(url, &repositories); err != nil {
+		return nil, err
+	}
+	return repositories, nil
+}

src/replication/adapter/harbor/image_registry.go

@@ -64,9 +64,8 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
 	}
 	resources := []*model.Resource{}
 	for _, project := range projects {
-		repositories := []*repository{}
+		repositories, err := a.getRepositories(project.ID)
-		url := fmt.Sprintf("%s/api/repositories?project_id=%d", a.coreServiceURL, project.ID)
+		if err != nil {
-		if err = a.client.Get(url, &repositories); err != nil {
 			return nil, err
 		}
 		repositories, err = filterRepositories(repositories, filters)