diff --git a/src/common/http/client.go b/src/common/http/client.go index 03155fc9c..533212dc0 100644 --- a/src/common/http/client.go +++ b/src/common/http/client.go @@ -17,9 +17,13 @@ package http import ( "bytes" "encoding/json" + "errors" "io" "io/ioutil" "net/http" + "net/url" + "reflect" + "strings" "github.com/goharbor/harbor/src/common/http/modifier" ) @@ -168,3 +172,63 @@ func (c *Client) do(req *http.Request) ([]byte, error) { return data, nil } + +// GetAndIteratePagination iterates the pagination header and returns all resources +// The parameter "v" must be a pointer to a slice +func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error { + url, err := url.Parse(endpoint) + if err != nil { + return err + } + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return errors.New("v should be a pointer to a slice") + } + elemType := rv.Elem().Type() + if elemType.Kind() != reflect.Slice { + return errors.New("v should be a pointer to a slice") + } + + resources := reflect.Indirect(reflect.New(elemType)) + for len(endpoint) > 0 { + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return err + } + resp, err := c.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return &Error{ + Code: resp.StatusCode, + Message: string(data), + } + } + + res := reflect.New(elemType) + if err = json.Unmarshal(data, res.Interface()); err != nil { + return err + } + resources = reflect.AppendSlice(resources, reflect.Indirect(res)) + + endpoint = "" + link := resp.Header.Get("Link") + for _, str := range strings.Split(link, ",") { + if strings.HasSuffix(str, `rel="next"`) && + strings.Index(str, "<") >= 0 && + strings.Index(str, ">") >= 0 { + endpoint = url.Scheme + "://" + url.Host + str[strings.Index(str, "<")+1:strings.Index(str, ">")] + break + } + } + } + rv.Elem().Set(resources) + return nil +} diff --git a/src/replication/adapter/harbor/adapter.go b/src/replication/adapter/harbor/adapter.go index 74614ac6c..7cc83ef37 100644 --- a/src/replication/adapter/harbor/adapter.go +++ b/src/replication/adapter/harbor/adapter.go @@ -211,8 +211,8 @@ type project struct { func (a *adapter) getProjects(name string) ([]*project, error) { projects := []*project{} - url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=1000", a.coreServiceURL, name) - if err := a.client.Get(url, &projects); err != nil { + url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=500", a.coreServiceURL, name) + if err := a.client.GetAndIteratePagination(url, &projects); err != nil { return nil, err } return projects, nil @@ -243,3 +243,12 @@ func (a *adapter) getProject(name string) (*project, error) { } return nil, nil } + +func (a *adapter) getRepositories(projectID int64) ([]*repository, error) { + repositories := []*repository{} + url := fmt.Sprintf("%s/api/repositories?project_id=%d&page=1&page_size=500", a.coreServiceURL, projectID) + if err := a.client.GetAndIteratePagination(url, &repositories); err != nil { + return nil, err + } + return repositories, nil +} diff --git a/src/replication/adapter/harbor/image_registry.go b/src/replication/adapter/harbor/image_registry.go index 4d7ce68d5..87420c4d2 100644 --- a/src/replication/adapter/harbor/image_registry.go +++ b/src/replication/adapter/harbor/image_registry.go @@ -64,9 +64,8 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error } resources := []*model.Resource{} for _, project := range projects { - repositories := []*repository{} - url := fmt.Sprintf("%s/api/repositories?project_id=%d", a.coreServiceURL, project.ID) - if err = a.client.Get(url, &repositories); err != nil { + repositories, err := a.getRepositories(project.ID) + if err != nil { return nil, err } repositories, err = filterRepositories(repositories, filters)