From 16a4bdce028b4522c78352d8aabc10f8d67144ba Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 16 Mar 2021 08:22:42 -0700 Subject: [PATCH] Don't always update gitea project descriptions There is some correlation that running the manage-projects playbook gives our gitea fits. The bulk of the work done here is in trying to update the descriptions of all projects. There isn't a good way to see if the description is already set first so we just try and ignore errors. This creates potentially thousands of operations all at once and could be why things are sad. We move these operations under the always update flag which is not set on normal runs. If we really need to converge to a good updated state we can manually run the playbook/role with always update set. We also don't set a limit on the number of ThreadPoolExecutor workers which will default to 5 * NumProcs. Could be that tuning this down would make gitea happier. One other thought is that we may not be using request sessions properly for connection reuse. In particular requests notes that you need to set stream to False or read request content to return a connection back to the pool for reuse. We might look into this for further improvements. Change-Id: I6e6fb1eb08303e9da7e38cf493d1871364340000 --- playbooks/roles/gitea-git-repos/README.rst | 16 ++++++++++++++++ .../library/gitea_create_repos.py | 12 ++++++------ playbooks/test-manage-projects.yaml | 5 +++++ zuul.d/system-config-run.yaml | 5 +++-- 4 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 playbooks/test-manage-projects.yaml diff --git a/playbooks/roles/gitea-git-repos/README.rst b/playbooks/roles/gitea-git-repos/README.rst index 6770339df7..a94905cace 100644 --- a/playbooks/roles/gitea-git-repos/README.rst +++ b/playbooks/roles/gitea-git-repos/README.rst @@ -1 +1,17 @@ Create git repos on a gitea server + +**Role Variables** + +.. zuul:rolevar:: gitea_url + :default: https://localhost:3000 + + The gitea to talk to. This is relative to the remote ansible node which + means localhost:3000 is on the ansible inventory node. + +.. zuul:rolevar:: gitea_always_update + :default: false + + Whether or not all projects should be coerced to the configured and + desired state. This defaults to false because it can be expensive to run, + but if project attributes like issue trackers or descriptions update this + is used to make those changes. diff --git a/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py b/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py index d0f099e9c2..a87d2f05b8 100755 --- a/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py +++ b/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py @@ -243,12 +243,6 @@ class Gitea(object): # TODO: use threadpool when we're running with # https://github.com/go-gitea/gitea/pull/7493 self.make_gitea_project(project, csrf_token) - else: - # We don't need to create it but lets update descriptions - # since humans like that. - futures.append(settings_thread_pool.submit( - self.update_gitea_project_description, - project, csrf_token)) if create or self.always_update: futures.append(settings_thread_pool.submit( self.update_gitea_project_settings, @@ -256,6 +250,12 @@ class Gitea(object): futures.append(branches_thread_pool.submit( self.update_gitea_project_branches, project, csrf_token)) + if self.always_update: + # If we are not creating, but are trying to always update + # then we update the project description. + futures.append(settings_thread_pool.submit( + self.update_gitea_project_description, + project, csrf_token)) def run(self): futures = [] diff --git a/playbooks/test-manage-projects.yaml b/playbooks/test-manage-projects.yaml new file mode 100644 index 0000000000..2a08e8c3f7 --- /dev/null +++ b/playbooks/test-manage-projects.yaml @@ -0,0 +1,5 @@ +- hosts: "gitea:!disabled" + name: "Create repos on gitea servers" + roles: + - role: gitea-git-repos + gitea_always_update: true diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index e8fe02b77d..12375cf74c 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -539,8 +539,9 @@ # Run twice to ensure that we noop properly when # all projects are created in gitea. We also update # zuul's description to ensure that descriptions are - # updated - - playbooks/manage-projects.yaml + # updated. This uses a test specific playbook to set + # the always_update flag. + - playbooks/test-manage-projects.yaml run_test_playbook: playbooks/test-gitea.yaml host-vars: gitea99.opendev.org: