Translate gitea project creation to python

Sadly, as readable as the use of the uri module to do the interactions
with gitea is, more reent ansible changed how subprocesses are forked
and this makes iterating over all the projects in projects.yaml take
an incredibly long amount of time.

Instead of doing it in yaml, make a python module that takes the list
one time and does looping and requests calls. This should make it be
possible to run the actual gitea creation playbook in integration tests.

Change-Id: Ifff3291c1092e6df09ae339c9e7dddb5ee692685

playbooks/group_vars/gitea.yaml

@@ -1,3 +1,4 @@
+ansible_python_interpreter: python3
 gitea_root_email: infra-root@openstack.org
 gitea_gerrit_public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVuhTMAz1H2Jr9AC3py9A0vlNna6Sdt4yrvZOayxukPqQ7GPZd+Mo7MVyypxLD479N2mA09JAdsbq1eTiPP8ksEkB+dNxZzw8mY1653R/IXSW6J9xPcoDa88HF2s/xHN24IWzgiDjNNe79AQ+sKleByEQZ++xXny3MRpy258hKUvAtjjOLOnM1PBs8JNOzBL+UPgWRgSX6GG0qywJZqjD1Qx5kvH9RTRLi+tcMhEi4laN7BYvn4csY0sYzTzPG4ZTu3ootIJoRlQGtQ0LmoFO1vSwyEJUags6/ZZGjgy3jl3kwcU/b8ZnFlF4MDw1OB1QqMb4r6bMHbXNIupp4zJbz gerrit-replication-2014-04-25
 iptables_extra_public_tcp_ports:

playbooks/roles/gitea-git-repos/library/gitea_create_repos.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import time
+import requests
+import urllib.parse
+
+from ansible.module_utils.basic import AnsibleModule
+
+SB_REPO = 'https://storyboard.openstack.org/#!/project/{org}/{repo}'
+SB_FORMAT = 'https://storyboard.openstack.org/#!/story/{{index}}'
+LP_REPO = 'https://bugs.launchpad.net/{repo}'
+LP_FORMAT = 'https://bugs.launchpad.net/{repo}/+bug/{{index}}'
+
+
+
+class Gitea(object):
+
+    def __init__(self, url, password, always_update, projects):
+        self.url = url
+        self.password = password
+        self.always_update = always_update
+        self.projects = projects
+        self.orgs = { f['project'].split('/')[0] for f in  self.projects }
+
+    def request(self, method, endpoint, *args, **kwargs):
+        resp = requests.request(
+            method,
+            urllib.parse.urljoin(self.url, endpoint),
+            auth=('root', self.password),
+            verify=False,
+            *args, **kwargs)
+        resp.raise_for_status()
+        return resp
+
+    def get(self, endpoint, *args, **kwargs):
+        return self.request('GET', endpoint, *args, **kwargs)
+
+    def post(self, endpoint, *args, **kwargs):
+        return self.request('POST', endpoint, *args, **kwargs)
+
+    def put(self, endpoint, *args, **kwargs):
+        return self.request('PUT', endpoint, *args, **kwargs)
+
+    def get_gitea_orgs(self):
+        orgs = self.get("/api/v1/user/orgs").json()
+        return [f['username'] for f in orgs]
+
+    def make_gitea_org(self, org):
+        self.post(
+            '/api/v1/admin/users/root/orgs',
+            json=dict(username=org))
+
+    def ensure_gitea_teams(self, org):
+        team_list = self.get('/api/v1/orgs/{org}/teams'.format(org=org)).json()
+        owner_id = [f['id'] for f in team_list if f['name'] == 'Owners'][0]
+
+        org_owners = self.get(
+            '/api/v1/teams/{owner_id}/members'.format(owner_id=owner_id))
+        if 'gerrit' not in [f['username'] for f in org_owners.json()]:
+            self.put('/api/v1/teams/{owner_id}/members/gerrit'.format(
+                owner_id=owner_id))
+
+    def get_org_repo_list(self, org):
+        return self.get('/api/v1/orgs/{org}/repos'.format(org=org)).json()
+
+    def get_csrf_token(self):
+        resp = self.get('/')
+        return urllib.parse.unquote(resp.cookies.get('_csrf'))
+
+    def make_gitea_project(self, project, csrf_token):
+        org, repo = project['project'].split('/', 1)
+        resp = self.post(
+            '/api/v1/org/{org}/repos'.format(org=org),
+            json=dict(
+                auto_init=True,
+                description=project.get('description', '')[:255],
+                name=repo,
+                private=False,
+                readme='Default'))
+        if project.get('use-storyboard'):
+            external_tracker_url = SB_REPO.format(org=org, repo=repo)
+            tracker_url_format = SB_FORMAT
+        elif project.get('groups'):
+            external_tracker_url = LP_REPO.format(repo=project['groups'][0])
+            tracker_url_format = LP_FORMAT.format(repo=project['groups'][0])
+        else:
+            external_tracker_url = LP_REPO.format(repo=repo)
+            tracker_url_format = LP_FORMAT.format(repo=repo)
+
+        self.post(
+            '/{org}/{repo}/settings'.format(org=org, repo=repo),
+            data=dict(
+                _csrf=csrf_token,
+                action='advanced',
+                # enable_pulls is not provided, which disables it
+                # enable_wiki is not provided, which disables it
+                enable_external_wiki=False,
+                external_wiki_url='',
+                # enable_issues is on so that issue links work
+                enable_issues='on',
+                enable_external_tracker=True,
+                external_tracker_url=external_tracker_url,
+                tracker_url_format=tracker_url_format,
+                tracker_issue_style='numeric',
+            ))
+
+        for count in range(0, 5):
+            try:
+                return self.post(
+                    '/{org}/{repo}/settings/branches'.format(
+                        org=org, repo=repo),
+                    data=dict(
+                        _csrf=csrf_token,
+                        action='default_branch',
+                        branch='master',
+                    ))
+            except requests.exceptions.HTTPError as e:
+                time.sleep(3)
+        raise Exception("Could not update branch settings")
+
+    def run(self):
+        gitea_orgs = self.get_gitea_orgs()
+        gitea_repos = []
+        for org in self.orgs:
+            if org not in gitea_orgs:
+                self.make_gitea_org(org)
+            self.ensure_gitea_teams(org)
+            gitea_repos.extend(self.get_org_repo_list(org))
+        csrf_token = self.get_csrf_token()
+
+        for project in self.projects:
+            if project['project'] not in gitea_repos or self.always_update:
+                self.make_gitea_project(project, csrf_token)
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            url=dict(required=True),
+            password=dict(required=True),
+            projects=dict(required=True, type='list'),
+            always_update=dict(type='bool', default=True),
+        )
+    )
+
+    p = module.params
+
+    gitea = Gitea(
+        url=p.get('url'),
+        password=p.get('password'),
+        always_update=p.get('always_update'),
+        projects=p.get('projects'),
+    )
+    try:
+        gitea.run()
+    except Exception as e:
+        module.fail_json(msg=str(e), changed=True)
+
+    module.exit_json(changed=True)
+
+
+if __name__ == '__main__':
+    ansible_main()

playbooks/roles/gitea-git-repos/tasks/main.yaml

@@ -1,44 +1,8 @@
-- name: Get Gerrit project list
-  set_fact:
-    gerrit_projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}"
-- name: Parse Gerrit org list
-  set_fact:
-    gerrit_orgs: "{{ gerrit_projects | map(attribute='project') | map('regex_search', '^(.*?)/') | list | unique | select | map('regex_replace', '/', '') | list }}"
-- name: debug
-  debug:
-    msg: "{{ gerrit_orgs }}"
-- name: Get Gitea org list
-  # We assume that all the orgs we are interested in are owned by root
-  uri:
-    url: "{{ gitea_url }}/api/v1/user/orgs"
-    user: root
+- name: Create Gitea Repos and Org
+  gitea_create_repos:
+    url: "{{ gitea_url }}"
     password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    validate_certs: false
-    status_code: 200
-  register: gitea_org_list
-- name: Parse Gitea org list
-  set_fact:
-    gitea_orgs: "{{ gitea_org_list.json | map(attribute='username') | list }}"
-- name: Create orgs
-  loop: "{{ gerrit_orgs }}"
-  loop_control:
-    loop_var: org
-  include_tasks: 'setup-org.yaml'
-- name: Get a CSRF token
-  uri:
-    url: "{{ gitea_url }}/"
-    validate_certs: false
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-  register: gitea_token
-- name: Parse CSRF taken
-  set_fact:
-    gitea_token: "{{ gitea_token.cookies._csrf|regex_replace('%3D','=') }}"
-- name: Create repos
-  loop: "{{ gerrit_projects }}"
-  loop_control:
-    loop_var: project
-  include_tasks: 'setup-repo.yaml'
-  when: gitea_always_update or project.project not in gitea_repos
+    always_update: "{{ gitea_always_update }}"
+    # Lookup runs locally on the calling machine, so doesn't need
+    # /opt/project-config remotely
+    projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}"

playbooks/roles/gitea-git-repos/tasks/setup-org.yaml

@@ -1,56 +0,0 @@
-- name: Process org
-  debug:
-    msg: "Processing org {{ org }}"
-- name: Create org
-  when: org not in gitea_orgs
-  uri:
-    url: "{{ gitea_url }}/api/v1/admin/users/root/orgs"
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    validate_certs: false
-    status_code: 201
-    method: POST
-    body_format: json
-    body:
-      username: "{{ org }}"
-- name: Get org team list
-  uri:
-    url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/teams"
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    validate_certs: false
-    status_code: 200
-  register: gitea_org_team_list
-- name: Get org owners
-  uri:
-    url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members"
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    validate_certs: false
-    status_code: 200
-  register: gitea_org_members
-- name: Add Gerrit user to org
-  when: "'gerrit' not in gitea_org_members.json | map(attribute='username')"
-  uri:
-    url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members/gerrit"
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    validate_certs: false
-    status_code: 204
-    method: PUT
-- name: Get org repo list
-  uri:
-    url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/repos"
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    validate_certs: false
-    status_code: 200
-  register: gitea_org_repo_list
-- name: Parse org repo list
-  set_fact:
-    gitea_repos: "{{ gitea_org_repo_list.json | map(attribute='full_name') | list + gitea_repos | default([]) }}"

playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml

@@ -1,98 +0,0 @@
-- name: debug
-  debug:
-    msg: "{{ project }}"
-- name: Parse project name
-  set_fact:
-    org: "{{ project.project | regex_replace('^(.*)/(.*)$', '\\1') }}"
-    repo: "{{ project.project | regex_replace('^(.*)/(.*)$', '\\2') }}"
-- name: Create repo
-  when: project.project not in gitea_repos
-  uri:
-    url: "{{ gitea_url }}/api/v1/org/{{ org }}/repos"
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    validate_certs: false
-    status_code: 201
-    method: POST
-    body_format: json
-    body:
-      auto_init: true
-      description: "{{ (project.description | default(''))[:255] }}"
-      name: "{{ repo }}"
-      private: false
-      readme: Default
-  register: create_repo_result
-
-- name: Set storyboard tracker url
-  when: "'use-storyboard' in project and project['use-storyboard']"
-  set_fact:
-    external_tracker_url: "https://storyboard.openstack.org/#!/project/{{ org }}/{{ repo }}"
-- name: Set storyboard tracker url format
-  when: "'use-storyboard' in project and project['use-storyboard']"
-  set_fact:
-    tracker_url_format: "https://storyboard.openstack.org/#!/story/{index}"
-- name: Set launchpad tracker url
-  when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' not in project or not project['groups'])"
-  set_fact:
-    external_tracker_url: "https://bugs.launchpad.net/{{ repo }}"
-- name: Set launchpad tracker url format
-  when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' not in project or not project['groups'])"
-  set_fact:
-    tracker_url_format: "https://bugs.launchpad.net/{{ repo }}/+bug/{index}"
-- name: Set launchpad tracker url if group set
-  when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' in project and project['groups'])"
-  set_fact:
-    external_tracker_url: "https://bugs.launchpad.net/{{ project.groups[0] }}"
-- name: Set launchpad tracker url format if group set
-  when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' in project and project['groups'])"
-  set_fact:
-    tracker_url_format: "https://bugs.launchpad.net/{{ project.groups[0] }}/+bug/{index}"
-
-- name: Adjust repo settings
-  when: gitea_always_update or project.project not in gitea_repos
-  register: result
-  retries: 3
-  until: result is succeeded
-  delay: 5
-  uri:
-    url: "{{ gitea_url }}/{{ org }}/{{ repo }}/settings"
-    validate_certs: false
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    status_code: 302
-    method: POST
-    body_format: form-urlencoded
-    body:
-      _csrf: "{{ gitea_token }}"
-      action: advanced
-      # enable_pulls is not provided, which disables it
-      # enable_wiki is not provided, which disables it
-      enable_external_wiki: false
-      external_wiki_url:
-      # enable_issues is on so that issue links work
-      enable_issues: on
-      enable_external_tracker: true
-      external_tracker_url: "{{ external_tracker_url }}"
-      tracker_url_format: "{{ tracker_url_format }}"
-      tracker_issue_style: numeric
-- name: Set default branch
-  when: gitea_always_update or project.project not in gitea_repos
-  register: result
-  retries: 3
-  until: result is succeeded
-  delay: 5
-  uri:
-    url: "{{ gitea_url }}/{{ org }}/{{ repo }}/settings/branches"
-    validate_certs: false
-    user: root
-    password: "{{ gitea_root_password }}"
-    force_basic_auth: true
-    status_code: 302
-    method: POST
-    body_format: form-urlencoded
-    body:
-      _csrf: "{{ gitea_token }}"
-      action: default_branch
-      branch: master

playbooks/roles/gitea/tasks/main.yaml

@@ -32,6 +32,11 @@
   template:
     src: app.ini.j2
     dest: /var/gitea/conf/app.ini
+- name: Install requests
+  package:
+    name:
+      - python3-requests
+    state: present
 - name: Install docker-compose
   package:
     name:

playbooks/sync-gitea-projects.yaml

@@ -8,7 +8,6 @@
         repo: https://git.openstack.org/openstack-infra/project-config
         dest: /opt/project-config
         force: yes
-      register: gitinfo
 
 - hosts: "gitea:!disabled"
   name: "Create repos on gitea servers"
@@ -16,5 +15,4 @@
   max_fail_percentage: 1
   roles:
     - role: gitea-git-repos
-      project_config_ref: "{{ hostvars.localhost.gitinfo.after }}"
       gitea_always_update: true