Manage opendev.org cert with LE

This is the first step in managing the opendev.org cert with LE. We
modify gitea01.opendev.org only to request the cert so that if this
breaks the other 7 giteas can continue to serve opendev.org. When we are
happy with the results we can merge the followup change to update the
other 7 giteas.

Depends-On: https://review.opendev.org/694182
Change-Id: I9587b8c2896975aa0148cc3d9b37f325a0be8970

.zuul.yaml

@@ -958,6 +958,7 @@
           label: ubuntu-bionic
     vars:
       run_playbooks:
+        - playbooks/service-letsencrypt.yaml
         - playbooks/service-gitea-lb.yaml
         - playbooks/remote_puppet_git.yaml
       run_test_playbook: playbooks/test-gitea.yaml
@@ -979,6 +980,7 @@
       - playbooks/roles/gitea/
       - playbooks/roles/gitea-git-repos/
       - playbooks/roles/haproxy/
+      - playbooks/roles/letsencrypt-create-certs/handlers/restart_gitea.yaml
       - testinfra/test_gitea.py
       - testinfra/test_gitea_lb.py
       # From gitea_files -- If we rebuild the image, we want to run

inventory/groups.yaml

@@ -68,6 +68,8 @@ groups:
     - mirror[0-9]*.opendev.org
     - files[0-9]*.open*.org
     - static.openstack.org
+    - gitea01.opendev.org
+    - gitea99.opendev.org
   logstash:
     - logstash[0-9]*.open*.org
   logstash-worker:

playbooks/host_vars/gitea01.opendev.org.yaml

@@ -0,0 +1,4 @@
+letsencrypt_certs:
+  gitea01-main:
+    - gitea01.opendev.org
+    - opendev.org

playbooks/roles/gitea/tasks/main.yaml

@@ -20,14 +20,6 @@
     - logs
     - certs
     - db
-- name: Write TLS private key
-  copy:
-    content: "{{ gitea_tls_key }}"
-    dest: /var/gitea/certs/key.pem
-- name: Write TLS certificate
-  copy:
-    content: "{{ gitea_tls_cert }}"
-    dest: /var/gitea/certs/cert.pem
 - name: Write app.ini
   template:
     src: app.ini.j2

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -58,3 +58,11 @@
 
 - name: letsencrypt updated mirror01-openafs-provider-opendev-org-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
+- name: letsencrypt updated gitea99-main
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_gitea.yaml
+
+# We split out handlers for each gitea host as handlers should be run in order
+# This allows us to do a rolling restart of the gitea backends.
+- name: letsencrypt updated gitea01-main
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_gitea.yaml

playbooks/roles/letsencrypt-create-certs/handlers/restart_gitea.yaml

@@ -0,0 +1,49 @@
+- name: Ensure gitea cert directy exists
+  file:
+    state: directory
+    path: "/var/gitea/certs"
+    owner: 1000
+    group: 1000
+
+- name: Put key in place
+  copy:
+    remote_src: yes
+    src: /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
+    dest: /var/gitea/certs/key.pem
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Put cert in place
+  copy:
+    remote_src: yes
+    # Gitea doesn't seem to accept separate ca chain and cert files.
+    # I believe it wants a single combined file as per fullchain.cer.
+    src: /etc/letsencrypt-certs/{{ inventory_hostname }}/fullchain.cer
+    dest: /var/gitea/certs/cert.pem
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Check for running gitea
+  command: pgrep -f gitea
+  ignore_errors: yes
+  register: gitea_pids
+
+- name: Restart gitea if running
+  when: gitea_pids.rc == 0
+  block:
+    - name: Restart gitea web
+      shell:
+        cmd: docker-compose restart gitea-web
+        chdir: /etc/gitea-docker/
+
+    - name: Wait for service to start and have valid users
+      uri:
+        url: "https://localhost:3000/api/v1/users/root"
+        validate_certs: false
+        status_code: 200, 404
+      register: root_user_check
+      delay: 1
+      retries: 300
+      until: root_user_check and root_user_check.status in (200, 404)

playbooks/zuul/run-base.yaml

@@ -85,6 +85,7 @@
         - host_vars/bridge.openstack.org.yaml
         - host_vars/letsencrypt01.opendev.org.yaml
         - host_vars/letsencrypt02.opendev.org.yaml
+        - host_vars/gitea99.opendev.org.yaml
         - host_vars/mirror01.openafs.provider.opendev.org.yaml
         - host_vars/mirror-update01.opendev.org.yaml
         - host_vars/backup-test01.opendev.org.yaml

playbooks/zuul/templates/group_vars/gitea.yaml.j2

@@ -7,54 +7,3 @@ gitea_db_password: 5bfuOBKtltff0XZX
 gitea_root_password: BUbBcpToMwR05ZCB
 gitea_no_log: false
 gitea_gerrit_password: yVpMWIUIvT7f6NwA
-gitea_tls_cert: |
-  -----BEGIN CERTIFICATE-----
-  MIIDXTCCAkWgAwIBAgIJANOV6XqCusL0MA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
-  BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
-  aWRnaXRzIFB0eSBMdGQwHhcNMTkwMjE1MjIwNjI0WhcNMTkwMzE3MjIwNjI0WjBF
-  MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
-  ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
-  CgKCAQEAvXpjO7ViMSG5IuSi7Y76wGUML2WpVyjGKeJur2BQkQQwy+5daUAwM0sr
-  sSa31IDya9hlDetQpLFE1QPFrwkNe2MT9+V/vIJJDoRbt2Tgrzj1ZL/DSws1FikF
-  L7vI8Je0Hb4Ylhd66xeuoz3jQW6ky9huJi8ZEkc4DNa1ehkyZd2nUXsu5DizQEU6
-  b+I5LneikWPrMSNOMSw3BrC9P6j9X8/j2Txpmkww3sC+TegsQKQSNTBvz8HUM6m6
-  OlT/yezjkNCDd/HHR49veMiOgvwJK6ZVGXl7Pg/tb+piXlI4lrXD0tjzEY+4jPJW
-  6m55r3l+yFvVoomStAjc7mDDnYul+wIDAQABo1AwTjAdBgNVHQ4EFgQUbVQz03pc
-  RO167fYlsXNtSFPP7oYwHwYDVR0jBBgwFoAUbVQz03pcRO167fYlsXNtSFPP7oYw
-  DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAQV+91n6+Wb4kj3byEJL9
-  X75geYQ7oz2HgWyJ8EB/cfxhZDxe4AqaTOnTsz2hf+QLh46wnc1Kkwn6REtq2izn
-  uLRYQJ1RklhGFMNEanweMwwVOcqsclFzX/u5dDl6jGaVaz2G/chvhPScmqoZGc9u
-  4K0DE5kQTHwYwyBSuOmZ0K+zlEzTaXt5Uadc8OpQ8Axx8sR9yhb5mDq2To6jBjU3
-  aT8Nwcpc2QchAA/dlJFfqm9YHCjcqtPdBuNrsRHP3FABr8OlmNTx3hm6ox7Zhijx
-  ROGRUmwjV78T87Z1gF5cpBEUj5BgiyMyoaK5HjWg9HJfPolul20PN88o+n17hkK8
-  lw==
-  -----END CERTIFICATE-----
-gitea_tls_key: |
-  -----BEGIN PRIVATE KEY-----
-  MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC9emM7tWIxIbki
-  5KLtjvrAZQwvZalXKMYp4m6vYFCRBDDL7l1pQDAzSyuxJrfUgPJr2GUN61CksUTV
-  A8WvCQ17YxP35X+8gkkOhFu3ZOCvOPVkv8NLCzUWKQUvu8jwl7QdvhiWF3rrF66j
-  PeNBbqTL2G4mLxkSRzgM1rV6GTJl3adRey7kOLNARTpv4jkud6KRY+sxI04xLDcG
-  sL0/qP1fz+PZPGmaTDDewL5N6CxApBI1MG/PwdQzqbo6VP/J7OOQ0IN38cdHj294
-  yI6C/AkrplUZeXs+D+1v6mJeUjiWtcPS2PMRj7iM8lbqbnmveX7IW9WiiZK0CNzu
-  YMOdi6X7AgMBAAECggEBAJcQLnF6KTD2q/3vvx4a8jvV1CMtsBb3QRY/mvNjnJgh
-  eS39eqfhLwyWD92K+uEHdT8aJWc1hvPnCPOzsDXex8rpsQ/g/zgxv0E9sUnDuYa5
-  qJuMb40zAD4Msj/ePVPj/wv/dOalDbDFDszDGJ4gMm76vMbgoeJ6uWsy+zi/QfkH
-  oI36pUnk165oGQtLVljKhclVpFcdno+E1LhrGpTgkHHNgx7P3J0mpsmjhIuQy9qk
-  Ugp9sPdvevgiduLW3qAWurn0lbQ1xcXt+BrGsqEU9m5wY6r4RLdqvHqfwgRCNOAC
-  blfXLacvh48Hpic4/LzXZmif83F6ntK4gierOp7aq+ECgYEA7ieBsvG5Dz2IasSu
-  n/1cNv7OtGn0cRuaW4zChraR4gKwOt93TL8jB0vjFr2Dp7SQsLdKfzuWMgnuI5wG
-  GZzx6nKM9hboCnh8p7jTF08HdAcXp5Bmfq8e9TUz2OUuU88PPcZDFwBL7Pk/lSn+
-  L7U3zLnjzqkbcqiyH4khWef22JkCgYEAy60g/Nnc+AWFhJToKEbd1JRwDDyYa6Ub
-  7zmcR0C1e3sUXfZf67qBEeXVNPV7mOwQ94ff/A7InzckAIAeWPT95idZ2MTdC097
-  NWC81IAvJODK/Y69AuPcyz69QYnRLKUfPwE4iTl77iev8tIwXDbkdhiW6dq4O93z
-  843PGEnkq7MCgYEAkr9XRSt7q+9votKlA8K70st6FWOAkz2+BJGcwCO5irm7W9ud
-  CHZyoClbugR3Hpy915Zp2jKeXyENU3XtsFSsIJoLUAxXWTRbI4JY2HEDF7TTF5Z8
-  Aa3o9pGc7BZ0UIIzUw5bAs5U+qWvTzu7/Cu/QXB99jbvydw3PgVivqKX0WkCgYAn
-  jZSFZe2igLgAGkbHY5O6r6Tey3myFdtJ5r8xmyBjPXCkGq9gANUF28M+yJlbBiT5
-  XPqjYV+Wg8fLDRZXoiQYaPXqwbhHdQTxRbsF7Wq6V6kz+l88S3HaSnHIY3IqoFpk
-  CuGmzHIDutNRbX4Uulg9kuLjwSTcA2tXledsyRTOPwKBgHHRUWkzf2GHHOcla9td
-  TEUmEM3gpXQmtjec976VjSnD7N8aitTfknLUtyq7f3VfPA/Oj/eug2lNX9+cCKMG
-  0nN3kZLVaUvxJ5YPiaQ9EzGqRoDOMto+CRksuSnBUGDcvpBX6Z+09qZgzqP3En1K
-  eZ6Mi1Y0bWwKXCyd8tbbqi9p
-  -----END PRIVATE KEY-----

playbooks/zuul/templates/host_vars/gitea99.opendev.org.yaml.j2

@@ -0,0 +1,4 @@
+letsencrypt_certs:
+  gitea99-main:
+    - gitea99.opendev.org
+    - opendev.org