Add gitea k8s resource definitions and playbook

This adds k8s resource definitions for running gitea, and an ansible
playbook to create them.

It also includes ansible playbooks to create orgs and repos.

Change-Id: Ib64597512c6a85d7e1495d18ae42b242f9af5a67

kubernetes/gitea/README

@@ -0,0 +1,2 @@
+This directory contains Ansible playbooks and Kubernetes resource
+definitions for running gitea.

kubernetes/gitea/app.ini.j2

@@ -0,0 +1,74 @@
+[server]
+APP_DATA_PATH    = /data/gitea
+SSH_DOMAIN       = localhost
+HTTP_PORT        = 3000
+ROOT_URL         = http://38.108.68.96/
+DISABLE_SSH      = false
+SSH_PORT         = 22
+LFS_CONTENT_PATH = /data/git/lfs
+DOMAIN           = localhost
+LFS_START_SERVER = true
+LFS_JWT_SECRET   = {{ lfs_jwt_secret }}
+OFFLINE_MODE     = false
+
+[database]
+DB_TYPE  = mysql
+HOST     = gitea-pxc.gitea-db.svc.cluster.local:3306
+NAME     = gitea
+USER     = {{ db_username }}
+PASSWD   = {{ db_password }}
+SSL_MODE = disable
+LOG_SQL  = false
+
+[indexer]
+ISSUE_INDEXER_PATH = /data/gitea/indexers/issues.bleve
+REPO_INDEXER_ENABLED = true
+
+[session]
+PROVIDER_CONFIG = /data/gitea/sessions
+PROVIDER        = file
+
+[picture]
+AVATAR_UPLOAD_PATH      = /data/gitea/avatars
+DISABLE_GRAVATAR        = false
+ENABLE_FEDERATED_AVATAR = true
+
+[attachment]
+PATH = /data/gitea/attachments
+
+[log]
+ROOT_PATH = /logs
+LEVEL     = Info
+
+[security]
+INSTALL_LOCK   = true
+SECRET_KEY     = {{ secret_key }}
+INTERNAL_TOKEN = {{ internal_token }}
+
+[service]
+DISABLE_REGISTRATION              = false
+REQUIRE_SIGNIN_VIEW               = false
+REGISTER_EMAIL_CONFIRM            = false
+ENABLE_NOTIFY_MAIL                = false
+ALLOW_ONLY_EXTERNAL_REGISTRATION  = false
+ENABLE_CAPTCHA                    = false
+DEFAULT_KEEP_EMAIL_PRIVATE        = false
+DEFAULT_ALLOW_CREATE_ORGANIZATION = true
+DEFAULT_ENABLE_TIMETRACKING       = true
+NO_REPLY_ADDRESS                  = noreply.example.org
+
+[mailer]
+ENABLED = false
+
+[openid]
+ENABLE_OPENID_SIGNIN = true
+ENABLE_OPENID_SIGNUP = true
+
+[markup.pandoc]
+ENABLED = true
+; List of file extensions that should be rendered by an external command
+FILE_EXTENSIONS = .rst
+; External command to render all matching extensions
+RENDER_COMMAND = "/usr/bin/pandoc"
+; Input is not a standard input but a file
+IS_INPUT_FILE = false

kubernetes/gitea/gitea-playbook.yaml

@@ -0,0 +1,120 @@
+- hosts: localhost
+  tasks:
+    # Deploy the service
+    - name: Set up gitea namespace
+      k8s:
+        state: present
+        definition: "{{ lookup('template', 'k8s/namespace.yaml') | from_yaml }}"
+    - name: Set up gitea secrets
+      k8s:
+        state: present
+        definition: "{{ lookup('template', 'k8s/secret.yaml') | from_yaml }}"
+    - name: Set up gitea configmap
+      k8s:
+        state: present
+        definition:
+          apiVersion: v1
+          kind: ConfigMap
+          metadata:
+            name: gitea-conf
+            namespace: gitea
+          data:
+            # Note: we are not asking ansible to template this, it
+            # will be run by jinja-init
+            app.ini.j2: "{{ lookup('file', 'app.ini.j2') }}"
+    - name: Set up gitea deployment
+      k8s:
+        state: present
+        definition: "{{ lookup('template', 'k8s/deployment.yaml') | from_yaml }}"
+    - name: Set up gitea service
+      k8s:
+        state: present
+        definition: "{{ lookup('template', 'k8s/service.yaml') | from_yaml }}"
+
+    # Bootstrap
+    # TODO: wait until service is up
+    - name: Get service IP
+      k8s:
+        namespace: gitea
+        kind: Service
+        name: gitea-service
+      register: gitea_service
+    - name: Set service url fact
+      set_fact:
+        gitea_url: "http://{{ gitea_service.result.status.loadBalancer.ingress[0].ip }}"
+    - name: Check if root user exists
+      uri:
+        url: "{{ gitea_url }}/api/v1/users/root"
+        status_code: 200, 404
+      register: root_user_check
+    - name: Create root user
+      when: root_user_check.status==404
+      block:
+        - name: Find gitea pods
+          k8s_facts:
+            namespace: gitea
+            kind: Pod
+            label_selectors:
+              - "app = gitea"
+          register: gitea_pods
+        - name: Create root user
+          command: "kubectl exec {{ gitea_pods.resources[0].metadata.name }} -n gitea -c gitea -- gitea admin create-user --name root --password {{ gitea_root_password }} --email {{ gitea_root_email }} --admin"
+          no_log: true
+    - name: Check if gerrit user exists
+      uri:
+        url: "{{ gitea_url }}/api/v1/users/gerrit"
+        status_code: 200, 404
+      register: gerrit_user_check
+    - name: Create gerrit user
+      when: gerrit_user_check.status==404
+      no_log: true
+      uri:
+        url: "{{ gitea_url }}/api/v1/admin/users"
+        method: POST
+        user: root
+        password: "{{ gitea_root_password }}"
+        force_basic_auth: true
+        status_code: 201
+        body_format: json
+        body:
+          email: "gerrit@review.opendev.org"
+          full_name: Gerrit
+          login_name: gerrit
+          password: "{{ gitea_gerrit_password }}"
+          send_notify: false
+          source_id: 0
+          username: gerrit
+    - name: Check if gerrit ssh key exists
+      uri:
+        user: root
+        password: "{{ gitea_root_password }}"
+        force_basic_auth: true
+        url: "{{ gitea_url }}/api/v1/users/gerrit/keys"
+        status_code: 200
+      register: gerrit_key_check
+      no_log: true
+    - name: Delete old gerrit ssh key
+      when: gerrit_key_check.json | length > 0 and gerrit_key_check.json[0].key != gitea_gerrit_public_key
+      no_log: true
+      uri:
+        user: root
+        password: "{{ gitea_root_password }}"
+        force_basic_auth: true
+        url: "{{ gitea_url }}/api/v1/user/keys/{{ gerrit_key_check.json[0].id }}"
+        method: DELETE
+        status_code: 204
+    - name: Add gerrit ssh key
+      when: gerrit_key_check.json | length == 0
+      no_log: true
+      uri:
+        user: root
+        password: "{{ gitea_root_password }}"
+        force_basic_auth: true
+        url: "{{ gitea_url }}/api/v1/admin/users/gerrit/keys"
+        method: POST
+        status_code: 201
+        body_format: json
+        body:
+          key: "{{ gitea_gerrit_public_key }}"
+          read_only: false
+          title: "Gerrit replication key"

kubernetes/gitea/k8s/deployment.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitea
+  namespace: gitea
+  labels:
+    app: gitea
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: gitea
+  template:
+    metadata:
+      labels:
+        app: gitea
+    spec:
+      initContainers:
+        - name: gitea-config
+          image: opendevorg/gitea-init
+          env:
+            - {name: VERBOSE, value: '1'}
+          volumeMounts:
+            - {name: config-template, mountPath: /config_src}
+            - {name: gitea-conf, mountPath: /conf}
+            - {name: gitea-data, mountPath: /data}
+            - {name: secrets, mountPath: /secrets}
+      containers:
+        - name: gitea
+          image: opendevorg/gitea
+          ports:
+            - containerPort: 3000
+          volumeMounts:
+            - name: gitea-data
+              mountPath: /data
+            - name: gitea-conf
+              mountPath: /custom/conf
+            - name: logs
+              mountPath: /logs
+        - name: openssh
+          image: opendevorg/gitea-openssh
+          ports:
+            - containerPort: 22
+          volumeMounts:
+            - name: gitea-data
+              mountPath: /data
+            - name: gitea-conf
+              mountPath: /custom/conf
+            - name: logs
+              mountPath: /logs
+      volumes:
+        - name: gitea-data
+          flexVolume:
+            driver: ceph.rook.io/rook
+            fsType: ceph
+            options:
+              fsName: rookfs
+              clusterNamespace: rook-ceph
+              clusterName: rook-ceph
+        - name: config-template
+          configMap:
+            name: gitea-conf
+        - name: gitea-conf
+          emptyDir:
+        - name: logs
+          emptyDir:
+        - name: secrets
+          secret:
+            secretName: gitea-app

kubernetes/gitea/k8s/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gitea

kubernetes/gitea/k8s/secret.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gitea-app
+  namespace: gitea
+type: Opaque
+stringData:
+  secret_key: {{ gitea_secret_key }}
+  internal_token: {{ gitea_internal_token }}
+  lfs_jwt_secret: {{ gitea_lfs_jwt_secret }}
+  db_username: {{ gitea_db_username }}
+  db_password: {{ gitea_db_password }}

kubernetes/gitea/k8s/service.yaml

@@ -0,0 +1,18 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: gitea-service
+  namespace: gitea
+spec:
+  selector:
+    app: gitea
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 3000
+      name: http
+    - protocol: TCP
+      port: 22
+      targetPort: 22
+      name: ssh
+  type: LoadBalancer

kubernetes/gitea/setup-org.yaml

@@ -0,0 +1,56 @@
+- name: Process org
+  debug:
+    msg: "Processing org {{ org }}"
+- name: Create org
+  when: org not in gitea_orgs
+  uri:
+    url: "{{ gitea_url }}/api/v1/admin/users/root/orgs"
+    user: root
+    password: "{{ gitea_root_password }}"
+    force_basic_auth: true
+    status_code: 201
+    method: POST
+    body_format: json
+    body:
+      username: "{{ org }}"
+- name: Get org team list
+  uri:
+    url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/teams"
+    user: root
+    password: "{{ gitea_root_password }}"
+    force_basic_auth: true
+    status_code: 200
+  register: gitea_org_team_list
+- name: Get org owners
+  uri:
+    url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members"
+    user: root
+    password: "{{ gitea_root_password }}"
+    force_basic_auth: true
+    status_code: 200
+  register: gitea_org_members
+- name: Add Gerrit user to org
+  when: "'gerrit' not in gitea_org_members.json | map(attribute='username')"
+  uri:
+    url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members/gerrit"
+    user: root
+    password: "{{ gitea_root_password }}"
+    force_basic_auth: true
+    status_code: 204
+    method: PUT
+- name: Get org repo list
+  uri:
+    url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/repos"
+    user: root
+    password: "{{ gitea_root_password }}"
+    force_basic_auth: true
+    status_code: 200
+  register: gitea_org_repo_list
+- name: Parse org repo list
+  set_fact:
+    gitea_org_repos: "{{ gitea_org_repo_list.json | map(attribute='name') | list }}"
+- name: Create repos in org
+  loop: "{{ (gerrit_projects.keys() | map('regex_search', '^' + org + '/.*') | select | map('regex_replace', '^.*/', '') | list) }}"
+  loop_control:
+    loop_var: repo
+  include_tasks: 'setup-repo.yaml'

kubernetes/gitea/setup-repo.yaml

@@ -0,0 +1,35 @@
+- name: debug
+  debug:
+    msg: "{{ gerrit_projects[org+'/'+repo] }}"
+- name: Create repo
+  when: repo not in gitea_org_repos
+  uri:
+    url: "{{ gitea_url }}/api/v1/org/{{ org }}/repos"
+    user: root
+    password: "{{ gitea_root_password }}"
+    force_basic_auth: true
+    status_code: 201
+    method: POST
+    body_format: json
+    body:
+      auto_init: false
+      description: "{{ gerrit_projects[org+'/'+repo]['description'] | default('') }}"
+      name: "{{ repo }}"
+      private: false
+  register: create_repo_result
+- name: Get created repo id
+  when: "create_repo_result.json is defined"
+  set_fact:
+    repo_id: "{{ create_repo_result.json['id'] }}"
+- name: Prepare sql query
+  when: "repo_id is defined"
+  set_fact:
+    sql_statement: |
+      start transaction;
+      delete from repo_unit where repo_id = {{ repo_id }} and `type` in (2, 3, 5, 7);
+      insert into repo_unit (repo_id, `type`, config, created_unix) values ({{ repo_id }}, 7, "{""ExternalTrackerURL"":""https://storyboard.openstack.org/#!/project/{{ org }}/{{ repo }}"",""ExternalTrackerFormat"":""https://storyboard.openstack.org/#!/story/{index}"",""ExternalTrackerStyle"":""""}", unix_timestamp());
+      commit;
+- name: Adjust repo settings
+  when: "sql_statement is defined"
+  command: |
+    /home/corvus/opendev/kubectl exec gitea-pxc-0 -c database -n gitea-db -- mysql gitea -e '{{ sql_statement }}'

kubernetes/gitea/sync-repos.yaml

@@ -0,0 +1,36 @@
+- hosts: localhost
+  vars:
+    gitea_url: http://38.108.68.66
+  tasks:
+    - name: Get Gerrit project list
+      uri:
+        url: "https://review.openstack.org/projects/"
+        status_code: 200
+        return_content: true
+      register: gerrit_project_list
+    - name: Parse Gerrit project list
+      set_fact:
+        gerrit_projects: "{{ gerrit_project_list.content[4:] | from_json }}"
+    - name: Parse Gerrit org list
+      set_fact:
+        gerrit_orgs: "{{ gerrit_projects.keys() | map('regex_search', '^(.*?)/') | list | unique | select | map('regex_replace', '/', '') | list }}"
+    - name: debug
+      debug:
+        msg: "{{ gerrit_orgs }}"
+    - name: Get Gitea org list
+      # We assume that all the orgs we are interested in are owned by root
+      uri:
+        url: "{{ gitea_url }}/api/v1/user/orgs"
+        user: root
+        password: "{{ gitea_root_password }}"
+        force_basic_auth: true
+        status_code: 200
+      register: gitea_org_list
+    - name: Parse Gitea org list
+      set_fact:
+        gitea_orgs: "{{ gitea_org_list.json | map(attribute='username') | list }}"
+    - name: Create orgs
+      loop: "{{ gerrit_orgs }}"
+      loop_control:
+        loop_var: org
+      include_tasks: 'setup-org.yaml'