diff --git a/playbooks/group_vars/gitea.yaml b/playbooks/group_vars/gitea.yaml index 1934a54338..19eedab34a 100644 --- a/playbooks/group_vars/gitea.yaml +++ b/playbooks/group_vars/gitea.yaml @@ -1,3 +1,4 @@ +ansible_python_interpreter: python3 gitea_root_email: infra-root@openstack.org gitea_gerrit_public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVuhTMAz1H2Jr9AC3py9A0vlNna6Sdt4yrvZOayxukPqQ7GPZd+Mo7MVyypxLD479N2mA09JAdsbq1eTiPP8ksEkB+dNxZzw8mY1653R/IXSW6J9xPcoDa88HF2s/xHN24IWzgiDjNNe79AQ+sKleByEQZ++xXny3MRpy258hKUvAtjjOLOnM1PBs8JNOzBL+UPgWRgSX6GG0qywJZqjD1Qx5kvH9RTRLi+tcMhEi4laN7BYvn4csY0sYzTzPG4ZTu3ootIJoRlQGtQ0LmoFO1vSwyEJUags6/ZZGjgy3jl3kwcU/b8ZnFlF4MDw1OB1QqMb4r6bMHbXNIupp4zJbz gerrit-replication-2014-04-25 iptables_extra_public_tcp_ports: diff --git a/playbooks/roles/gitea-git-repos/library/__init__.py b/playbooks/roles/gitea-git-repos/library/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py b/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py new file mode 100755 index 0000000000..c846ab9887 --- /dev/null +++ b/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time +import requests +import urllib.parse + +from ansible.module_utils.basic import AnsibleModule + +SB_REPO = 'https://storyboard.openstack.org/#!/project/{org}/{repo}' +SB_FORMAT = 'https://storyboard.openstack.org/#!/story/{{index}}' +LP_REPO = 'https://bugs.launchpad.net/{repo}' +LP_FORMAT = 'https://bugs.launchpad.net/{repo}/+bug/{{index}}' + + + +class Gitea(object): + + def __init__(self, url, password, always_update, projects): + self.url = url + self.password = password + self.always_update = always_update + self.projects = projects + self.orgs = { f['project'].split('/')[0] for f in self.projects } + + def request(self, method, endpoint, *args, **kwargs): + resp = requests.request( + method, + urllib.parse.urljoin(self.url, endpoint), + auth=('root', self.password), + verify=False, + *args, **kwargs) + resp.raise_for_status() + return resp + + def get(self, endpoint, *args, **kwargs): + return self.request('GET', endpoint, *args, **kwargs) + + def post(self, endpoint, *args, **kwargs): + return self.request('POST', endpoint, *args, **kwargs) + + def put(self, endpoint, *args, **kwargs): + return self.request('PUT', endpoint, *args, **kwargs) + + def get_gitea_orgs(self): + orgs = self.get("/api/v1/user/orgs").json() + return [f['username'] for f in orgs] + + def make_gitea_org(self, org): + self.post( + '/api/v1/admin/users/root/orgs', + json=dict(username=org)) + + def ensure_gitea_teams(self, org): + team_list = self.get('/api/v1/orgs/{org}/teams'.format(org=org)).json() + owner_id = [f['id'] for f in team_list if f['name'] == 'Owners'][0] + + org_owners = self.get( + '/api/v1/teams/{owner_id}/members'.format(owner_id=owner_id)) + if 'gerrit' not in [f['username'] for f in org_owners.json()]: + self.put('/api/v1/teams/{owner_id}/members/gerrit'.format( + owner_id=owner_id)) + + def get_org_repo_list(self, org): + return self.get('/api/v1/orgs/{org}/repos'.format(org=org)).json() + + def get_csrf_token(self): + resp = self.get('/') + return urllib.parse.unquote(resp.cookies.get('_csrf')) + + def make_gitea_project(self, project, csrf_token): + org, repo = project['project'].split('/', 1) + resp = self.post( + '/api/v1/org/{org}/repos'.format(org=org), + json=dict( + auto_init=True, + description=project.get('description', '')[:255], + name=repo, + private=False, + readme='Default')) + if project.get('use-storyboard'): + external_tracker_url = SB_REPO.format(org=org, repo=repo) + tracker_url_format = SB_FORMAT + elif project.get('groups'): + external_tracker_url = LP_REPO.format(repo=project['groups'][0]) + tracker_url_format = LP_FORMAT.format(repo=project['groups'][0]) + else: + external_tracker_url = LP_REPO.format(repo=repo) + tracker_url_format = LP_FORMAT.format(repo=repo) + + self.post( + '/{org}/{repo}/settings'.format(org=org, repo=repo), + data=dict( + _csrf=csrf_token, + action='advanced', + # enable_pulls is not provided, which disables it + # enable_wiki is not provided, which disables it + enable_external_wiki=False, + external_wiki_url='', + # enable_issues is on so that issue links work + enable_issues='on', + enable_external_tracker=True, + external_tracker_url=external_tracker_url, + tracker_url_format=tracker_url_format, + tracker_issue_style='numeric', + )) + + for count in range(0, 5): + try: + return self.post( + '/{org}/{repo}/settings/branches'.format( + org=org, repo=repo), + data=dict( + _csrf=csrf_token, + action='default_branch', + branch='master', + )) + except requests.exceptions.HTTPError as e: + time.sleep(3) + raise Exception("Could not update branch settings") + + def run(self): + gitea_orgs = self.get_gitea_orgs() + gitea_repos = [] + for org in self.orgs: + if org not in gitea_orgs: + self.make_gitea_org(org) + self.ensure_gitea_teams(org) + gitea_repos.extend(self.get_org_repo_list(org)) + csrf_token = self.get_csrf_token() + + for project in self.projects: + if project['project'] not in gitea_repos or self.always_update: + self.make_gitea_project(project, csrf_token) + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict( + url=dict(required=True), + password=dict(required=True), + projects=dict(required=True, type='list'), + always_update=dict(type='bool', default=True), + ) + ) + + p = module.params + + gitea = Gitea( + url=p.get('url'), + password=p.get('password'), + always_update=p.get('always_update'), + projects=p.get('projects'), + ) + try: + gitea.run() + except Exception as e: + module.fail_json(msg=str(e), changed=True) + + module.exit_json(changed=True) + + +if __name__ == '__main__': + ansible_main() diff --git a/playbooks/roles/gitea-git-repos/tasks/main.yaml b/playbooks/roles/gitea-git-repos/tasks/main.yaml index d356e9c8a8..e4f3f5c5d9 100644 --- a/playbooks/roles/gitea-git-repos/tasks/main.yaml +++ b/playbooks/roles/gitea-git-repos/tasks/main.yaml @@ -1,44 +1,8 @@ -- name: Get Gerrit project list - set_fact: - gerrit_projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}" -- name: Parse Gerrit org list - set_fact: - gerrit_orgs: "{{ gerrit_projects | map(attribute='project') | map('regex_search', '^(.*?)/') | list | unique | select | map('regex_replace', '/', '') | list }}" -- name: debug - debug: - msg: "{{ gerrit_orgs }}" -- name: Get Gitea org list - # We assume that all the orgs we are interested in are owned by root - uri: - url: "{{ gitea_url }}/api/v1/user/orgs" - user: root +- name: Create Gitea Repos and Org + gitea_create_repos: + url: "{{ gitea_url }}" password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 200 - register: gitea_org_list -- name: Parse Gitea org list - set_fact: - gitea_orgs: "{{ gitea_org_list.json | map(attribute='username') | list }}" -- name: Create orgs - loop: "{{ gerrit_orgs }}" - loop_control: - loop_var: org - include_tasks: 'setup-org.yaml' -- name: Get a CSRF token - uri: - url: "{{ gitea_url }}/" - validate_certs: false - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - register: gitea_token -- name: Parse CSRF taken - set_fact: - gitea_token: "{{ gitea_token.cookies._csrf|regex_replace('%3D','=') }}" -- name: Create repos - loop: "{{ gerrit_projects }}" - loop_control: - loop_var: project - include_tasks: 'setup-repo.yaml' - when: gitea_always_update or project.project not in gitea_repos + always_update: "{{ gitea_always_update }}" + # Lookup runs locally on the calling machine, so doesn't need + # /opt/project-config remotely + projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}" diff --git a/playbooks/roles/gitea-git-repos/tasks/setup-org.yaml b/playbooks/roles/gitea-git-repos/tasks/setup-org.yaml deleted file mode 100644 index a4938842e1..0000000000 --- a/playbooks/roles/gitea-git-repos/tasks/setup-org.yaml +++ /dev/null @@ -1,56 +0,0 @@ -- name: Process org - debug: - msg: "Processing org {{ org }}" -- name: Create org - when: org not in gitea_orgs - uri: - url: "{{ gitea_url }}/api/v1/admin/users/root/orgs" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 201 - method: POST - body_format: json - body: - username: "{{ org }}" -- name: Get org team list - uri: - url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/teams" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 200 - register: gitea_org_team_list -- name: Get org owners - uri: - url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 200 - register: gitea_org_members -- name: Add Gerrit user to org - when: "'gerrit' not in gitea_org_members.json | map(attribute='username')" - uri: - url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members/gerrit" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 204 - method: PUT -- name: Get org repo list - uri: - url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/repos" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 200 - register: gitea_org_repo_list -- name: Parse org repo list - set_fact: - gitea_repos: "{{ gitea_org_repo_list.json | map(attribute='full_name') | list + gitea_repos | default([]) }}" diff --git a/playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml b/playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml deleted file mode 100644 index b13ca74e80..0000000000 --- a/playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml +++ /dev/null @@ -1,98 +0,0 @@ -- name: debug - debug: - msg: "{{ project }}" -- name: Parse project name - set_fact: - org: "{{ project.project | regex_replace('^(.*)/(.*)$', '\\1') }}" - repo: "{{ project.project | regex_replace('^(.*)/(.*)$', '\\2') }}" -- name: Create repo - when: project.project not in gitea_repos - uri: - url: "{{ gitea_url }}/api/v1/org/{{ org }}/repos" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 201 - method: POST - body_format: json - body: - auto_init: true - description: "{{ (project.description | default(''))[:255] }}" - name: "{{ repo }}" - private: false - readme: Default - register: create_repo_result - -- name: Set storyboard tracker url - when: "'use-storyboard' in project and project['use-storyboard']" - set_fact: - external_tracker_url: "https://storyboard.openstack.org/#!/project/{{ org }}/{{ repo }}" -- name: Set storyboard tracker url format - when: "'use-storyboard' in project and project['use-storyboard']" - set_fact: - tracker_url_format: "https://storyboard.openstack.org/#!/story/{index}" -- name: Set launchpad tracker url - when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' not in project or not project['groups'])" - set_fact: - external_tracker_url: "https://bugs.launchpad.net/{{ repo }}" -- name: Set launchpad tracker url format - when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' not in project or not project['groups'])" - set_fact: - tracker_url_format: "https://bugs.launchpad.net/{{ repo }}/+bug/{index}" -- name: Set launchpad tracker url if group set - when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' in project and project['groups'])" - set_fact: - external_tracker_url: "https://bugs.launchpad.net/{{ project.groups[0] }}" -- name: Set launchpad tracker url format if group set - when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' in project and project['groups'])" - set_fact: - tracker_url_format: "https://bugs.launchpad.net/{{ project.groups[0] }}/+bug/{index}" - -- name: Adjust repo settings - when: gitea_always_update or project.project not in gitea_repos - register: result - retries: 3 - until: result is succeeded - delay: 5 - uri: - url: "{{ gitea_url }}/{{ org }}/{{ repo }}/settings" - validate_certs: false - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - status_code: 302 - method: POST - body_format: form-urlencoded - body: - _csrf: "{{ gitea_token }}" - action: advanced - # enable_pulls is not provided, which disables it - # enable_wiki is not provided, which disables it - enable_external_wiki: false - external_wiki_url: - # enable_issues is on so that issue links work - enable_issues: on - enable_external_tracker: true - external_tracker_url: "{{ external_tracker_url }}" - tracker_url_format: "{{ tracker_url_format }}" - tracker_issue_style: numeric -- name: Set default branch - when: gitea_always_update or project.project not in gitea_repos - register: result - retries: 3 - until: result is succeeded - delay: 5 - uri: - url: "{{ gitea_url }}/{{ org }}/{{ repo }}/settings/branches" - validate_certs: false - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - status_code: 302 - method: POST - body_format: form-urlencoded - body: - _csrf: "{{ gitea_token }}" - action: default_branch - branch: master diff --git a/playbooks/roles/gitea/tasks/main.yaml b/playbooks/roles/gitea/tasks/main.yaml index 8a1aba61b6..7e51a49a9a 100644 --- a/playbooks/roles/gitea/tasks/main.yaml +++ b/playbooks/roles/gitea/tasks/main.yaml @@ -32,6 +32,11 @@ template: src: app.ini.j2 dest: /var/gitea/conf/app.ini +- name: Install requests + package: + name: + - python3-requests + state: present - name: Install docker-compose package: name: diff --git a/playbooks/sync-gitea-projects.yaml b/playbooks/sync-gitea-projects.yaml index bd25a86fb0..b12af45b08 100644 --- a/playbooks/sync-gitea-projects.yaml +++ b/playbooks/sync-gitea-projects.yaml @@ -8,7 +8,6 @@ repo: https://git.openstack.org/openstack-infra/project-config dest: /opt/project-config force: yes - register: gitinfo - hosts: "gitea:!disabled" name: "Create repos on gitea servers" @@ -16,5 +15,4 @@ max_fail_percentage: 1 roles: - role: gitea-git-repos - project_config_ref: "{{ hostvars.localhost.gitinfo.after }}" gitea_always_update: true