Merge "Deal with gitea pagination of repo lists"

playbooks/roles/gitea-git-repos/library/gitea_create_repos.py

@@ -99,8 +99,22 @@ class Gitea(object):
             self.log("Added gerrit to team:", org)
 
     def get_org_repo_list(self, org):
-        return [x['full_name'] for x in
-                self.get('/api/v1/orgs/{org}/repos'.format(org=org)).json()]
+        params = { 'limit': 50, 'page': 1 }
+        repos = []
+        gitea_data = self.get(
+                '/api/v1/orgs/{org}/repos'.format(org=org),
+                params=params
+            ).json()
+        while gitea_data:
+            repos.extend([x['full_name'] for x in gitea_data])
+            # Gitea paginates and returns an empty list at the end of the
+            # listing. 50 items is the max limit.
+            params['page'] += 1
+            gitea_data = self.get(
+                    '/api/v1/orgs/{org}/repos'.format(org=org),
+                    params=params
+                ).json()
+        return repos
 
     def get_csrf_token(self):
         resp = self.get('/')
@@ -182,9 +196,17 @@ class Gitea(object):
     def make_projects(self, projects, gitea_repos, csrf_token,
                       settings_thread_pool, branches_thread_pool, futures):
         for project in projects:
-            if project['project'] in gitea_repos:
             create = False
-            else:
+            if project['project'] not in gitea_repos:
+                try:
+                    self.get('/' + project['project'])
+                except requests.HTTPError:
+                    # If the project isn't in the listing we do an explicit
+                    # check for its existence. This is because gitea repo
+                    # listings require pagination and they don't use stable
+                    # sorting and that causes problems reliably producing a
+                    # complete repo list. If we cannot find the project
+                    # then create it.
                     create = True
             if create:
                 # TODO: use threadpool when we're running with

zuul.d/system-config-run.yaml

@@ -500,6 +500,9 @@
         - playbooks/service-gitea-lb.yaml
         - playbooks/service-gitea.yaml
         - playbooks/manage-projects.yaml
+        # Run twice to ensure that we noop properly when
+        # all projects are created in gitea.
+        - playbooks/manage-projects.yaml
       run_test_playbook: playbooks/test-gitea.yaml
     host-vars:
       gitea99.opendev.org: