From 60b32441eff3daceeae8ce822c40cbca16dab92a Mon Sep 17 00:00:00 2001
From: elee <eddy@gfxlabs.io>
Date: Thu, 3 Mar 2022 23:32:45 -0600
Subject: [PATCH] paging for gitlab

---
 .gitignore                                    |  5 ++-
 .../localhost/docker-compose.yml              | 32 +++++++++++++++
 internal/gitsources/gitlab/gitlab.go          | 41 +++++++++++++++----
 internal/gitsources/gitlab/gitlab_test.go     | 29 +++++++++++++
 4 files changed, 98 insertions(+), 9 deletions(-)
 create mode 100644 examples/docker-compose/localhost/docker-compose.yml
 create mode 100644 internal/gitsources/gitlab/gitlab_test.go

diff --git a/.gitignore b/.gitignore
index 139032e..b8ae620 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,7 @@
 vendor/
 bin/
 tools/
-webbundle/bindata.go
\ No newline at end of file
+webbundle/bindata.go
+
+configs/
+tmp/
diff --git a/examples/docker-compose/localhost/docker-compose.yml b/examples/docker-compose/localhost/docker-compose.yml
new file mode 100644
index 0000000..3966811
--- /dev/null
+++ b/examples/docker-compose/localhost/docker-compose.yml
@@ -0,0 +1,32 @@
+version: '3.8'
+services:
+    etcd:
+        image: quay.io/coreos/etcd
+        restart: unless-stopped
+        volumes:
+            - etcd-data:/etcd-data
+        environment:
+            ETCD_NAME: agola-etcd
+            ETCD_DATA_DIR: /etcd-data
+            ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
+            ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
+            ETCD_INITIAL_CLUSTER_TOKEN: agola-etcd-1
+    agola:
+        image: agola
+        restart: unless-stopped
+        volumes:
+            - /etc/timezone:/etc/timezone:ro
+            - /etc/localtime:/etc/localtime:ro
+            - /var/run/docker.sock:/var/run/docker.sock
+            - ./configs:/configs
+            - agola-data:/data/agola
+            - agola-tmp:/tmp/agola
+        ports:
+        - 8000:8000
+        command: ["serve", "--components", "all-base,executor", "--config", "/configs/config.yml"]
+        depends_on:
+            - etcd
+volumes:
+    agola-data:
+    agola-tmp:
+    etcd-data:
diff --git a/internal/gitsources/gitlab/gitlab.go b/internal/gitsources/gitlab/gitlab.go
index 682a962..cac08ff 100644
--- a/internal/gitsources/gitlab/gitlab.go
+++ b/internal/gitsources/gitlab/gitlab.go
@@ -34,6 +34,7 @@ import (
 )
 
 var (
+	PAGE_MAX           = 100
 	GitlabOauth2Scopes = []string{"api"}
 
 	branchRefPrefix     = "refs/heads/"
@@ -279,17 +280,41 @@ func (c *Client) CreateCommitStatus(repopath, commitSHA string, status gitsource
 }
 
 func (c *Client) ListUserRepos() ([]*gitsource.RepoInfo, error) {
-	// get only repos with permission greater or equal to maintainer
-	opts := &gitlab.ListProjectsOptions{MinAccessLevel: gitlab.AccessLevel(gitlab.MaintainerPermissions)}
-	remoteRepos, _, err := c.client.Projects.ListProjects(opts)
-	if err != nil {
-		return nil, errors.WithStack(err)
+	baseOpts := gitlab.ListProjectsOptions{
+		MinAccessLevel: gitlab.AccessLevel(gitlab.MaintainerPermissions),
+		ListOptions: gitlab.ListOptions{
+			Page:    1,
+			PerPage: 100,
+		},
 	}
 
-	repos := []*gitsource.RepoInfo{}
+	return c.listUserRepos(baseOpts)
+}
 
-	for _, rr := range remoteRepos {
-		repos = append(repos, fromGitlabRepo(rr))
+func (c *Client) listUserRepos(fromOptions gitlab.ListProjectsOptions) ([]*gitsource.RepoInfo, error) {
+	repos := []*gitsource.RepoInfo{}
+	// get only repos with permission greater or equal to maintainer
+	opts := &fromOptions
+	for i := 1; i <= PAGE_MAX; i++ {
+		remoteRepos, pageResp, err := c.client.Projects.ListProjects(opts)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		for _, rr := range remoteRepos {
+			repos = append(repos, fromGitlabRepo(rr))
+		}
+		opts.ListOptions.Page = pageResp.NextPage
+		if opts.ListOptions.Page == 0 {
+			break
+		}
+		if len(remoteRepos) == 0 {
+			break
+		}
+		if pageResp.TotalPages != 0 {
+			if pageResp.TotalPages >= pageResp.CurrentPage {
+			}
+		}
 	}
 
 	return repos, nil
diff --git a/internal/gitsources/gitlab/gitlab_test.go b/internal/gitsources/gitlab/gitlab_test.go
new file mode 100644
index 0000000..6df8134
--- /dev/null
+++ b/internal/gitsources/gitlab/gitlab_test.go
@@ -0,0 +1,29 @@
+package gitlab
+
+import (
+	"testing"
+
+	gitlab "github.com/xanzy/go-gitlab"
+)
+
+var c *Client
+
+func init() {
+	c, _ = New(Opts{SkipVerify: true, APIURL: "https://gitlab.com"})
+}
+
+func TestListUserReposLimit(t *testing.T) {
+	baseOpts := gitlab.ListProjectsOptions{
+		ListOptions: gitlab.ListOptions{
+			Page:    1,
+			PerPage: 1,
+		},
+	}
+	repos, err := c.listUserRepos(baseOpts)
+	if err != nil {
+		t.Errorf("err getting repos %v", err)
+	}
+	if len(repos) != PAGE_MAX {
+		t.Errorf("page fail expected %d to be %d", len(repos), PAGE_MAX)
+	}
+}