Add support for LetsEncrypt-managed certs

Add support for automatic provisioning and renewal of HTTPS
certificates via LetsEncrypt.

Spec is available at:
https://etherpad.opendev.org/p/kolla-ansible-letsencrypt-https

Depends-On: https://review.opendev.org/c/openstack/kolla/+/887347
Co-Authored-By: Michal Arbet <michal.arbet@ultimum.io>
Implements: blueprint letsencrypt-https
Change-Id: I35317ea0343f0db74ddc0e587862e95408e9e106

ansible/group_vars/all.yml

@@ -406,6 +406,7 @@ grafana_server_listen_port: "{{ grafana_server_port }}"
 
 haproxy_stats_port: "1984"
 haproxy_monitor_port: "61313"
+haproxy_ssh_port: "2985"
 
 heat_internal_fqdn: "{{ kolla_internal_fqdn }}"
 heat_external_fqdn: "{{ kolla_external_fqdn }}"
@@ -450,6 +451,8 @@ keystone_ssh_port: "8023"
 
 kuryr_port: "23750"
 
+letsencrypt_webserver_port: "8081"
+
 magnum_internal_fqdn: "{{ kolla_internal_fqdn }}"
 magnum_external_fqdn: "{{ kolla_external_fqdn }}"
 magnum_api_port: "9511"
@@ -834,6 +837,7 @@ enable_ironic_pxe_uefi: "no"
 enable_ironic_prometheus_exporter: "{{ enable_ironic | bool and enable_prometheus | bool }}"
 enable_iscsid: "{{ enable_cinder | bool and enable_cinder_backend_iscsi | bool }}"
 enable_kuryr: "no"
+enable_letsencrypt: "no"
 enable_magnum: "no"
 enable_manila: "no"
 enable_manila_backend_generic: "no"
@@ -993,7 +997,8 @@ kolla_tls_backend_key: "{{ kolla_certificates_dir }}/backend-key.pem"
 #####################
 # ACME client options
 #####################
-acme_client_servers: []
+acme_client_lego: "server lego {{ api_interface_address }}:{{ letsencrypt_webserver_port }}"
+acme_client_servers: "{% set arr = [] %}{% if enable_letsencrypt | bool %}{{ arr.append(acme_client_lego) }}{% endif %}{{ arr }}"
 
 ####################
 # Keystone options

ansible/inventory/all-in-one

@@ -191,6 +191,9 @@ control
 [venus:children]
 monitoring
 
+[letsencrypt:children]
+loadbalancer
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -687,3 +690,9 @@ venus
 
 [venus-manager:children]
 venus
+
+[letsencrypt-webserver:children]
+letsencrypt
+
+[letsencrypt-lego:children]
+letsencrypt

ansible/inventory/multinode

@@ -209,6 +209,9 @@ control
 [venus:children]
 monitoring
 
+[letsencrypt:children]
+loadbalancer
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -706,3 +709,9 @@ venus
 
 [venus-manager:children]
 venus
+
+[letsencrypt-webserver:children]
+letsencrypt
+
+[letsencrypt-lego:children]
+letsencrypt

ansible/roles/certificates/tasks/generate.yml

@@ -67,6 +67,7 @@
         dest: "{{ kolla_external_fqdn_cert }}"
         mode: "0660"
   when:
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_external | bool
 
 - block:
@@ -77,6 +78,7 @@
         remote_src: yes
         mode: "0660"
   when:
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_external | bool
     - kolla_enable_tls_internal | bool
     - kolla_same_external_internal_vip | bool
@@ -137,5 +139,6 @@
         dest: "{{ kolla_internal_fqdn_cert }}"
         mode: "0660"
   when:
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_internal | bool
     - not kolla_same_external_internal_vip | bool

ansible/roles/common/templates/cron-logrotate-letsencrypt.conf.j2

@@ -0,0 +1,3 @@
+"/var/log/kolla/letsencrypt/*.log"
+{
+}

ansible/roles/haproxy-config/defaults/main.yml

@@ -5,6 +5,7 @@ haproxy_service_template: "haproxy_single_service_split.cfg.j2"
 haproxy_frontend_http_extra:
   - "option httplog"
   - "option forwardfor"
+haproxy_frontend_redirect_extra: []
 haproxy_frontend_tcp_extra:
   - "option tcplog"
 haproxy_backend_http_extra: []

ansible/roles/haproxy-config/templates/haproxy_single_service_split.cfg.j2

@@ -1,7 +1,7 @@
 #jinja2: lstrip_blocks: True
-{%- set external_tls_bind_info = 'ssl crt /etc/haproxy/haproxy.pem' if kolla_enable_tls_external|bool else '' %}
+{%- set external_tls_bind_info = 'ssl crt /etc/haproxy/certificates/haproxy.pem' if kolla_enable_tls_external|bool else '' %}
 {%- set external_tls_bind_info = "%s %s" % (external_tls_bind_info, haproxy_http2_protocol) if kolla_enable_tls_external|bool and haproxy_enable_http2|bool else external_tls_bind_info %}
-{%- set internal_tls_bind_info = 'ssl crt /etc/haproxy/haproxy-internal.pem' if kolla_enable_tls_internal|bool else '' %}
+{%- set internal_tls_bind_info = 'ssl crt /etc/haproxy/certificates/haproxy-internal.pem' if kolla_enable_tls_internal|bool else '' %}
 {%- set internal_tls_bind_info = "%s %s" % (internal_tls_bind_info, haproxy_http2_protocol) if kolla_enable_tls_internal|bool and haproxy_enable_http2|bool else internal_tls_bind_info %}
 
 {%- macro userlist_macro(service_name, auth_user, auth_pass) %}
@@ -10,7 +10,7 @@ userlist {{ service_name }}-user
 {% endmacro %}
 
 {%- macro frontend_macro(service_name, service_port, service_mode, external,
-                         frontend_http_extra, frontend_tcp_extra) %}
+                         frontend_http_extra, frontend_redirect_extra, frontend_tcp_extra) %}
 frontend {{ service_name }}_front
     {% if service_mode == 'redirect' %}
     mode http
@@ -50,7 +50,10 @@ frontend {{ service_name }}_front
     {{ "bind %s:%s %s"|e|format(vip_address, service_port, tls_option)|trim() }}
     {# Redirect mode sets a redirect scheme instead of a backend #}
     {% if service_mode == 'redirect' %}
-    redirect scheme https code 301 if !{ ssl_fc }
+    redirect scheme https code 301 if !{ ssl_fc } !{ path_reg ^/.well-known/acme-challenge/.+ }
+        {% for redirect_option in frontend_redirect_extra %}
+    {{ redirect_option }}
+        {% endfor %}
     {% else %}
     default_backend {{ service_name }}_back
     {% endif %}
@@ -133,6 +136,7 @@ backend {{ service_name }}_back
         {% set frontend_tcp_extra = haproxy_service.frontend_tcp_extra|default([]) + haproxy_frontend_tcp_extra %}
         {% set backend_tcp_extra = haproxy_service.backend_tcp_extra|default([]) %}
         {% set frontend_http_extra = haproxy_service.frontend_http_extra|default([]) + haproxy_frontend_http_extra %}
+        {% set frontend_redirect_extra = haproxy_service.frontend_redirect_extra|default([]) + haproxy_frontend_redirect_extra %}
         {% set backend_http_extra = haproxy_service.backend_http_extra|default([]) %}
         {% set tls_backend = haproxy_service.tls_backend|default(false) %}
         {# Allow for basic auth #}
@@ -144,7 +148,7 @@ backend {{ service_name }}_back
         {% if with_frontend %}
             {% if not (external|bool and haproxy_single_external_frontend|bool and mode == 'http') %}
 {{ frontend_macro(haproxy_name, haproxy_service.port, mode, external,
-                  frontend_http_extra, frontend_tcp_extra) }}
+                  frontend_http_extra, frontend_redirect_extra, frontend_tcp_extra) }}
             {% endif %}
         {% endif %}
         {# Redirect (to https) is a special case, as it does not include a backend #}

ansible/roles/horizon/defaults/main.yml

@@ -49,6 +49,8 @@ horizon_services:
         external: false
         port: "{{ horizon_port }}"
         listen_port: "{{ horizon_listen_port }}"
+        frontend_redirect_extra:
+          - "use_backend acme_client_back if { path_reg ^/.well-known/acme-challenge/.+ }"
       horizon_external:
         enabled: "{{ enable_horizon }}"
         mode: "http"
@@ -68,6 +70,8 @@ horizon_services:
         external_fqdn: "{{ horizon_external_fqdn }}"
         port: "{{ horizon_port }}"
         listen_port: "{{ horizon_listen_port }}"
+        frontend_redirect_extra:
+          - "use_backend acme_client_back if { path_reg ^/.well-known/acme-challenge/.+ }"
       acme_client:
         enabled: "{{ enable_horizon }}"
         with_frontend: false

ansible/roles/letsencrypt/defaults/main.yml

@@ -0,0 +1,60 @@
+---
+letsencrypt_services:
+  letsencrypt-lego:
+    container_name: letsencrypt_lego
+    group: letsencrypt-lego
+    enabled: true
+    image: "{{ letsencrypt_lego_image_full }}"
+    volumes: "{{ letsencrypt_lego_default_volumes + letsencrypt_lego_extra_volumes }}"
+    dimensions: "{{ letsencrypt_lego_dimensions }}"
+  letsencrypt-webserver:
+    container_name: letsencrypt_webserver
+    group: letsencrypt-webserver
+    enabled: true
+    image: "{{ letsencrypt_webserver_image_full }}"
+    volumes: "{{ letsencrypt_webserver_default_volumes + letsencrypt_webserver_extra_volumes }}"
+    dimensions: "{{ letsencrypt_webserver_dimensions }}"
+
+
+##############
+# LetsEncrypt
+##############
+letsencrypt_tag: "{{ openstack_tag }}"
+letsencrypt_logging_debug: "{{ openstack_logging_debug }}"
+
+letsencrypt_lego_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/letsencrypt-lego"
+letsencrypt_lego_tag: "{{ letsencrypt_tag }}"
+letsencrypt_lego_image_full: "{{ letsencrypt_lego_image }}:{{ letsencrypt_lego_tag }}"
+
+letsencrypt_webserver_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/letsencrypt-webserver"
+letsencrypt_webserver_tag: "{{ letsencrypt_tag }}"
+letsencrypt_webserver_image_full: "{{ letsencrypt_webserver_image }}:{{ letsencrypt_webserver_tag }}"
+
+letsencrypt_lego_dimensions: "{{ default_container_dimensions }}"
+letsencrypt_webserver_dimensions: "{{ default_container_dimensions }}"
+
+letsencrypt_lego_default_volumes:
+  - "{{ node_config_directory }}/letsencrypt-lego/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "letsencrypt:/etc/letsencrypt"
+  - "kolla_logs:/var/log/kolla/"
+letsencrypt_lego_extra_volumes: "{{ default_extra_volumes }}"
+
+letsencrypt_webserver_default_volumes:
+  - "{{ node_config_directory }}/letsencrypt-webserver/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "letsencrypt:/etc/letsencrypt"
+  - "kolla_logs:/var/log/kolla/"
+letsencrypt_webserver_extra_volumes: "{{ default_extra_volumes }}"
+
+letsencrypt_cert_server: "https://acme-v02.api.letsencrypt.org/directory"
+# attempt to renew Let's Encrypt certificate every 4 hours
+letsencrypt_cron_renew_schedule: "0 */4 * * *"
+# The email used for certificate registration and recovery contact. Required.
+letsencrypt_email: ""
+letsencrypt_cert_valid_days: "30"
+
+letsencrypt_external_fqdns:
+  - "{{ kolla_external_fqdn }}"
+letsencrypt_internal_fqdns:
+  - "{{ kolla_internal_fqdn }}"

ansible/roles/letsencrypt/handlers/main.yml

@@ -0,0 +1,34 @@
+---
+- name: Restart letsencrypt-webserver container
+  vars:
+    service_name: "letsencrypt-webserver"
+    service: "{{ letsencrypt_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+    healthcheck: "{{ service.healthcheck | default(omit) }}"
+    environment: "{{ service.environment | default(omit) }}"
+  when:
+    - kolla_action != "config"
+
+- name: Restart letsencrypt-lego container
+  vars:
+    service_name: "letsencrypt-lego"
+    service: "{{ letsencrypt_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+    healthcheck: "{{ service.healthcheck | default(omit) }}"
+    environment: "{{ service.environment | default(omit) }}"
+  when:
+    - kolla_action != "config"

ansible/roles/letsencrypt/tasks/check-containers.yml

@@ -0,0 +1,18 @@
+---
+- name: Check LetsEncrypt containers
+  become: true
+  kolla_docker:
+    action: "compare_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes }}"
+    dimensions: "{{ item.value.dimensions }}"
+    healthcheck: "{{ item.value.healthcheck | default(omit) }}"
+    environment: "{{ item.value.environment | default(omit) }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ letsencrypt_services }}"
+  notify:
+    - "Restart {{ item.key }} container"

ansible/roles/letsencrypt/tasks/config.yml

@@ -0,0 +1,66 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item.key }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ letsencrypt_services }}"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item.key }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ letsencrypt_services }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: Copying over letsencrypt-webserver.conf
+  vars:
+    service: "{{ letsencrypt_services['letsencrypt-webserver'] }}"
+  become: true
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/letsencrypt-webserver/letsencrypt-webserver.conf"
+    mode: "0660"
+  with_first_found:
+    - "{{ node_custom_config }}/letsencrypt/{{ inventory_hostname }}/letsencrypt-webserver.conf"
+    - "{{ node_custom_config }}/letsencrypt/letsencrypt-webserver.conf"
+    - "letsencrypt-webserver.conf.j2"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  notify:
+    - Restart letsencrypt-webserver container
+
+- name: Copying files for letsencrypt-lego
+  vars:
+    service: "{{ letsencrypt_services['letsencrypt-lego'] }}"
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ node_config_directory }}/letsencrypt-lego/{{ item.dest }}"
+    mode: "0660"
+  become: true
+  with_items:
+    - { src: "crontab.j2", dest: "crontab" }
+    - { src: "id_rsa.j2", dest: "id_rsa" }
+    - { src: "letsencrypt-lego-run.sh.j2", dest: "letsencrypt-lego-run.sh" }
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  notify:
+    - Restart letsencrypt-lego container
+
+- include_tasks: copy-certs.yml
+  when:
+    - kolla_copy_ca_into_containers | bool

ansible/roles/letsencrypt/tasks/config_validate.yml

@@ -0,0 +1 @@
+---

ansible/roles/letsencrypt/tasks/copy-certs.yml

@@ -0,0 +1,6 @@
+---
+- name: "Copy certificates and keys for {{ project_name }}"
+  import_role:
+    role: service-cert-copy
+  vars:
+    project_services: "{{ letsencrypt_services }}"

ansible/roles/letsencrypt/tasks/deploy-containers.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: check-containers.yml

ansible/roles/letsencrypt/tasks/deploy.yml

@@ -0,0 +1,7 @@
+---
+- import_tasks: config.yml
+
+- import_tasks: check-containers.yml
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/letsencrypt/tasks/loadbalancer.yml

@@ -0,0 +1,7 @@
+---
+- name: "Configure loadbalancer for {{ project_name }}"
+  import_role:
+    name: loadbalancer-config
+  vars:
+    project_services: "{{ letsencrypt_services }}"
+  tags: always

ansible/roles/letsencrypt/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include_tasks: "{{ kolla_action }}.yml"

ansible/roles/letsencrypt/tasks/precheck.yml

@@ -0,0 +1,33 @@
+---
+- name: Get container facts
+  become: true
+  kolla_container_facts:
+    container_engine: "{{ kolla_container_engine }}"
+    name:
+      - letsencrypt_webserver
+  register: container_facts
+
+- name: Checking free port for LetsEncrypt server
+  vars:
+    service: "{{ letsencrypt_services['letsencrypt-webserver'] }}"
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ letsencrypt_webserver_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - container_facts['letsencrypt_webserver'] is not defined
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+
+- name: Validating letsencrypt email variable
+  run_once: true
+  vars:
+    replace: "valid"
+  assert:
+    that: letsencrypt_email | regex_replace('.*@.*$', replace) == "valid"
+    fail_msg: "Letsencrypt contact email value didn't pass validation."
+  when:
+    - enable_letsencrypt | bool
+    - kolla_enable_tls_external | bool

ansible/roles/letsencrypt/tasks/pull.yml

@@ -0,0 +1,11 @@
+---
+- name: Pulling LetsEncrypt images
+  become: true
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ item.value.image }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ letsencrypt_services }}"

ansible/roles/letsencrypt/tasks/reconfigure.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: deploy.yml

ansible/roles/letsencrypt/tasks/stop.yml

@@ -0,0 +1,6 @@
+---
+- import_role:
+    role: service-stop
+  vars:
+    project_services: "{{ letsencrypt_services }}"
+    service_name: "{{ project_name }}"

ansible/roles/letsencrypt/tasks/upgrade.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: deploy.yml

ansible/roles/letsencrypt/templates/crontab.j2

@@ -0,0 +1,8 @@
+PATH=/usr/local/bin:/usr/bin:/bin
+
+{% if kolla_external_vip_address != kolla_internal_vip_address and kolla_external_fqdn != kolla_external_vip_address %}
+{{ letsencrypt_cron_renew_schedule }} /usr/bin/letsencrypt-certificates --external --fqdns {% for fqdn in letsencrypt_external_fqdns %}{{ fqdn }}{% if not loop.last %},{% endif %}{% endfor %} --days {{ letsencrypt_cert_valid_days }} --port {{ letsencrypt_webserver_port }} --mail {{ letsencrypt_email }} --acme {{ letsencrypt_cert_server }} --vips {% if not kolla_same_external_internal_vip %}{{ kolla_external_vip_address }},{% endif %}{{ kolla_internal_vip_address }} --haproxies-ssh {% for host in groups['loadbalancer'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ haproxy_ssh_port }}{% if not loop.last %},{% endif %}{% endfor %} 2>&1 | tee -a /var/log/kolla/letsencrypt/letsencrypt-lego.log
+{% endif %}
+{% if kolla_external_vip_address == kolla_internal_vip_address and kolla_internal_fqdn != kolla_internal_vip_address %}
+{{ letsencrypt_cron_renew_schedule }} /usr/bin/letsencrypt-certificates --internal --fqdns {% for fqdn in letsencrypt_internal_fqdns %}{{ fqdn }}{% if not loop.last %},{% endif %}{% endfor %} --days {{ letsencrypt_cert_valid_days }} --port {{ letsencrypt_webserver_port }} --mail {{ letsencrypt_email }} --acme {{ letsencrypt_cert_server }} --vips {% if not kolla_same_external_internal_vip %}{{ kolla_external_vip_address }},{% endif %}{{ kolla_internal_vip_address }} --haproxies-ssh {% for host in groups['loadbalancer'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ haproxy_ssh_port }}{% if not loop.last %},{% endif %}{% endfor %} 2>&1 | tee -a /var/log/kolla/letsencrypt/letsencrypt-lego.log
+{% endif %}

ansible/roles/letsencrypt/templates/id_rsa.j2

@@ -0,0 +1 @@
+{{ haproxy_ssh_key.private_key }}

ansible/roles/letsencrypt/templates/letsencrypt-lego-run.sh.j2

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+{% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %}
+
+{% if kolla_external_vip_address != kolla_internal_vip_address and kolla_external_fqdn != kolla_external_vip_address %}
+/usr/bin/letsencrypt-certificates --external --fqdns {% for fqdn in letsencrypt_external_fqdns %}{{ fqdn }}{% if not loop.last %},{% endif %}{% endfor %} --days {{ letsencrypt_cert_valid_days }} --port {{ letsencrypt_webserver_port }} --mail {{ letsencrypt_email }} --acme {{ letsencrypt_cert_server }} --vips {% if not kolla_same_external_internal_vip %}{{ kolla_external_vip_address }},{% endif %}{{ kolla_internal_vip_address }} --haproxies-ssh {% for host in groups['loadbalancer'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ haproxy_ssh_port }}{% if not loop.last %},{% endif %}{% endfor %} 2>&1 | tee -a /var/log/kolla/letsencrypt/letsencrypt-lego.log
+{% endif %}
+{% if kolla_external_vip_address == kolla_internal_vip_address and kolla_internal_fqdn != kolla_internal_vip_address %}
+/usr/bin/letsencrypt-certificates --internal --fqdns {% for fqdn in letsencrypt_internal_fqdns %}{{ fqdn }}{% if not loop.last %},{% endif %}{% endfor %} --days {{ letsencrypt_cert_valid_days }} --port {{ letsencrypt_webserver_port }} --mail {{ letsencrypt_email }} --acme {{ letsencrypt_cert_server }} --vips {% if not kolla_same_external_internal_vip %}{{ kolla_external_vip_address }},{% endif %}{{ kolla_internal_vip_address }} --haproxies-ssh {% for host in groups['loadbalancer'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ haproxy_ssh_port }}{% if not loop.last %},{% endif %}{% endfor %} 2>&1 | tee -a /var/log/kolla/letsencrypt/letsencrypt-lego.log
+{% endif %}
+
+{{ cron_cmd }}

ansible/roles/letsencrypt/templates/letsencrypt-lego.json.j2

@@ -0,0 +1,26 @@
+{% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %}
+{% set cron_path = '/var/spool/cron/crontabs/root' if kolla_base_distro in ['ubuntu', 'debian'] else '/var/spool/cron/root' %}
+{
+    "command": "/usr/local/bin/letsencrypt-lego-run.sh",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/letsencrypt-lego-run.sh",
+            "dest": "/usr/local/bin/letsencrypt-lego-run.sh",
+            "owner": "root",
+            "perm": "0700"
+        },
+        {
+            "source": "{{ container_config_directory }}/crontab",
+            "dest": "{{ cron_path }}",
+            "owner": "root",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/id_rsa",
+            "dest": "/var/lib/letsencrypt/.ssh/id_rsa",
+            "owner": "letsencrypt",
+            "perm": "0600"
+        }
+    ]
+}
+

ansible/roles/letsencrypt/templates/letsencrypt-webserver.conf.j2

@@ -0,0 +1,19 @@
+Listen {{ api_interface_address }}:8081
+
+ServerSignature Off
+ServerTokens Prod
+TraceEnable off
+KeepAliveTimeout 60
+
+<VirtualHost {{ api_interface_address }}:8081>
+    DocumentRoot /etc/letsencrypt/http-01
+    ErrorLog "/var/log/kolla/letsencrypt/letsencrypt-webserver-error.log"
+    CustomLog "/var/log/kolla/letsencrypt/letsencrypt-webserver-access.log" common
+
+    <Directory "/etc/letsencrypt/http-01/">
+        Options None
+        AllowOverride None
+        Require all granted
+    </Directory>
+
+</VirtualHost>

ansible/roles/letsencrypt/templates/letsencrypt-webserver.json.j2

@@ -0,0 +1,14 @@
+{% set letsencrypt_apache_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %}
+{% set apache_binary = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %}
+
+{
+    "command": "/usr/sbin/{{ apache_binary }} -DFOREGROUND",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/letsencrypt-webserver.conf",
+            "dest": "/etc/{{ letsencrypt_apache_dir }}/letsencrypt-webserver.conf",
+            "owner": "letsencrypt",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/letsencrypt/vars/main.yml

@@ -0,0 +1,2 @@
+---
+project_name: "letsencrypt"

ansible/roles/loadbalancer/defaults/main.yml

@@ -26,6 +26,14 @@ loadbalancer_services:
     privileged: True
     volumes: "{{ keepalived_default_volumes + keepalived_extra_volumes }}"
     dimensions: "{{ keepalived_dimensions }}"
+  haproxy-ssh:
+    container_name: "haproxy_ssh"
+    group: loadbalancer
+    enabled: "{{ enable_letsencrypt | bool }}"
+    image: "{{ haproxy_ssh_image_full }}"
+    volumes: "{{ haproxy_ssh_default_volumes }}"
+    dimensions: "{{ haproxy_ssh_dimensions }}"
+    healthcheck: "{{ haproxy_ssh_healthcheck }}"
 
 
 ####################
@@ -43,6 +51,10 @@ proxysql_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker
 proxysql_tag: "{{ openstack_tag }}"
 proxysql_image_full: "{{ proxysql_image }}:{{ proxysql_tag }}"
 
+haproxy_ssh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/haproxy-ssh"
+haproxy_ssh_tag: "{{ haproxy_tag }}"
+haproxy_ssh_image_full: "{{ haproxy_ssh_image }}:{{ haproxy_ssh_tag }}"
+
 syslog_server: "{{ api_interface_address }}"
 syslog_haproxy_facility: "local1"
 
@@ -59,6 +71,7 @@ haproxy_defaults_max_connections: 10000
 haproxy_dimensions: "{{ default_container_dimensions }}"
 proxysql_dimensions: "{{ default_container_dimensions }}"
 keepalived_dimensions: "{{ default_container_dimensions }}"
+haproxy_ssh_dimensions: "{{ default_container_dimensions }}"
 
 haproxy_enable_healthchecks: "{{ enable_container_healthchecks }}"
 haproxy_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
@@ -86,11 +99,27 @@ proxysql_healthcheck:
   test: "{% if proxysql_enable_healthchecks | bool %}{{ proxysql_healthcheck_test }}{% else %}NONE{% endif %}"
   timeout: "{{ proxysql_healthcheck_timeout }}"
 
+haproxy_ssh_enable_healthchecks: "{{ enable_container_healthchecks }}"
+haproxy_ssh_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
+haproxy_ssh_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
+haproxy_ssh_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
+haproxy_ssh_healthcheck_test: ["CMD-SHELL", "healthcheck_listen sshd {{ haproxy_ssh_port }}"]
+haproxy_ssh_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
+haproxy_ssh_healthcheck:
+  interval: "{{ haproxy_ssh_healthcheck_interval }}"
+  retries: "{{ haproxy_ssh_healthcheck_retries }}"
+  start_period: "{{ haproxy_ssh_healthcheck_start_period }}"
+  test: "{% if haproxy_ssh_enable_healthchecks | bool %}{{ haproxy_ssh_healthcheck_test }}{% else %}NONE{% endif %}"
+  timeout: "{{ haproxy_ssh_healthcheck_timeout }}"
+
+
 haproxy_default_volumes:
   - "{{ node_config_directory }}/haproxy/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "haproxy_socket:/var/lib/kolla/haproxy/"
+  - "letsencrypt_certificates:/etc/haproxy/certificates"
+
 proxysql_default_volumes:
   - "{{ node_config_directory }}/proxysql/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
@@ -105,6 +134,13 @@ keepalived_default_volumes:
   - "/lib/modules:/lib/modules:ro"
   - "{{ 'haproxy_socket:/var/lib/kolla/haproxy/' if enable_haproxy | bool else '' }}"
   - "{{ 'proxysql_socket:/var/lib/kolla/proxysql/' if enable_proxysql | bool else '' }}"
+haproxy_ssh_default_volumes:
+  - "{{ node_config_directory }}/haproxy-ssh/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
+  - "haproxy_socket:/var/lib/kolla/haproxy/"
+  - "{{ 'letsencrypt:/etc/letsencrypt' if enable_letsencrypt | bool else omit }}"
+  - "{{ 'letsencrypt_certificates:/etc/haproxy/certificates' if enable_letsencrypt | bool else omit }}"
 
 haproxy_extra_volumes: "{{ default_extra_volumes }}"
 proxysql_extra_volumes: "{{ default_extra_volumes }}"
@@ -143,8 +179,7 @@ haproxy_defaults_balance: "roundrobin"
 haproxy_host_ipv4_tcp_retries2: "KOLLA_UNSET"
 
 # HAProxy socket admin permissions enable
-haproxy_socket_level_admin: "no"
+haproxy_socket_level_admin: "{{ enable_letsencrypt | bool }}"
-
 kolla_externally_managed_cert: False
 
 # Allow to disable keepalived tracking script (e.g. for single node environments

ansible/roles/loadbalancer/handlers/main.yml

@@ -333,3 +333,19 @@
     - service.enabled | bool
   listen:
     - Wait for virtual IP to appear
+
+- name: Restart haproxy-ssh container
+  vars:
+    service_name: "haproxy-ssh"
+    service: "{{ loadbalancer_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes | reject('equalto', '') | list }}"
+    dimensions: "{{ service.dimensions }}"
+    healthcheck: "{{ service.healthcheck | default(omit) }}"
+  when:
+    - kolla_action != "config"

ansible/roles/loadbalancer/tasks/config.yml

@@ -80,8 +80,10 @@
   become: true
   with_dict: "{{ loadbalancer_services }}"
   when:
+    - keepalived_track_script_enabled | bool
     - inventory_hostname in groups[service.group]
     - item.key != 'keepalived'
+    - item.key != 'haproxy-ssh'
     - not item.value.enabled | bool
       or not inventory_hostname in groups[item.value.group]
     - service.enabled | bool
@@ -102,6 +104,7 @@
     - inventory_hostname in groups[service.group]
     - inventory_hostname in groups[item.value.group]
     - item.key != 'keepalived'
+    - item.key != 'haproxy-ssh'
     - item.value.enabled | bool
     - service.enabled | bool
   notify:
@@ -214,6 +217,7 @@
     mode: "0660"
   become: true
   when:
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_external | bool
     - not kolla_externally_managed_cert | bool
     - inventory_hostname in groups[service.group]
@@ -232,6 +236,7 @@
     mode: "0660"
   become: true
   when:
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_internal | bool
     - not kolla_externally_managed_cert | bool
     - inventory_hostname in groups[service.group]
@@ -280,3 +285,20 @@
     - "proxysql/proxysql_run.sh.j2"
   notify:
     - Restart proxysql container
+
+- name: Copying files for haproxy-ssh
+  vars:
+    haproxy_ssh: "{{ loadbalancer_services['haproxy-ssh'] }}"
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ node_config_directory }}/haproxy-ssh/{{ item.dest }}"
+    mode: "0600"
+  become: true
+  with_items:
+    - { src: "haproxy-ssh/sshd_config.j2", dest: "sshd_config" }
+    - { src: "haproxy-ssh/id_rsa.pub", dest: "id_rsa.pub" }
+  when:
+    - inventory_hostname in groups[haproxy_ssh.group]
+    - haproxy_ssh.enabled | bool
+  notify:
+    - Restart haproxy-ssh container

ansible/roles/loadbalancer/tasks/precheck.yml

@@ -63,6 +63,7 @@
   changed_when: false
   when:
     - not kolla_externally_managed_cert | bool
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_external | bool
 
 - name: Assert that external haproxy certificate exists
@@ -72,6 +73,7 @@
     fail_msg: "External haproxy certificate file is not found. It is configured via 'kolla_external_fqdn_cert'"
   when:
     - not kolla_externally_managed_cert | bool
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_external | bool
 
 - name: Checking if internal haproxy certificate exists
@@ -83,6 +85,7 @@
   changed_when: false
   when:
     - not kolla_externally_managed_cert | bool
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_internal | bool
 
 - name: Assert that internal haproxy certificate exists
@@ -92,6 +95,7 @@
     fail_msg: "Internal haproxy certificate file is not found. It is configured via 'kolla_internal_fqdn_cert'"
   when:
     - not kolla_externally_managed_cert | bool
+    - not enable_letsencrypt | bool
     - kolla_enable_tls_internal | bool
 
 - name: Checking the kolla_external_vip_interface is present

ansible/roles/loadbalancer/templates/haproxy-ssh/haproxy-ssh.json.j2

@@ -0,0 +1,17 @@
+{
+    "command": "/usr/sbin/sshd -D",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/sshd_config",
+            "dest": "/etc/ssh/sshd_config",
+            "owner": "root",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/id_rsa.pub",
+            "dest": "/var/lib/haproxy/.ssh/authorized_keys",
+            "owner": "haproxy",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/loadbalancer/templates/haproxy-ssh/id_rsa.pub

@@ -0,0 +1 @@
+{{ haproxy_ssh_key.public_key }}

ansible/roles/loadbalancer/templates/haproxy-ssh/sshd_config.j2

@@ -0,0 +1,5 @@
+Port {{ haproxy_ssh_port }}
+ListenAddress {{ api_interface_address }}
+
+SyslogFacility AUTHPRIV
+UsePAM yes

ansible/roles/loadbalancer/templates/haproxy/haproxy.json.j2

@@ -18,7 +18,7 @@
             "dest": "/etc/haproxy/services.d",
             "owner": "root",
             "perm": "0700"
-        },
+        }{% if kolla_enable_tls_external | bool and not enable_letsencrypt | bool %},
         {
             "source": "{{ container_config_directory }}/external-frontend-map",
             "dest": "/etc/haproxy/external-frontend-map",
@@ -28,17 +28,19 @@
         },
         {
             "source": "{{ container_config_directory }}/haproxy.pem",
-            "dest": "/etc/haproxy/haproxy.pem",
+            "dest": "/etc/haproxy/certificates/haproxy.pem",
-            "owner": "root",
+            "owner": "haproxy",
             "perm": "0600",
             "optional": {{ (not kolla_enable_tls_external | bool) | string | lower }}
-        },
+        }{% endif %}
+        {% if kolla_enable_tls_internal | bool and not enable_letsencrypt | bool %},
         {
             "source": "{{ container_config_directory }}/haproxy-internal.pem",
-            "dest": "/etc/haproxy/haproxy-internal.pem",
+            "dest": "/etc/haproxy/certificates/haproxy-internal.pem",
-            "owner": "root",
+            "owner": "haproxy",
             "perm": "0600",
             "optional": {{ (not kolla_enable_tls_internal | bool) | string | lower }}
         }
+        {% endif %}
     ]
 }

ansible/roles/loadbalancer/templates/haproxy/haproxy_run.sh.j2

@@ -1,9 +1,40 @@
 #!/bin/bash -x
 
-# We need to run haproxy with one `-f` for each service, because including an
+{% if kolla_enable_tls_internal | bool or kolla_enable_tls_external | bool %}
-# entire config directory was not a feature until version 1.7 of HAProxy.
+{% if kolla_enable_tls_external | bool %}
-# So, append "-f $cfg" to the haproxy command for each service file.
+if [ ! -e "/etc/haproxy/certificates/haproxy.pem" ]; then
-# This will run haproxy_cmd *exactly once*.
+    # Generate temporary self-signed cert
+    # This means external tls is enabled but the certificate was not copied
+    # to the container - so letsencrypt is enabled
+    #
+    # Let's generate certificate to make haproxy happy, lego will
+    # replace it in a while
+    ssl_tmp_dir=$(mktemp -d)
+    openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes -keyout ${ssl_tmp_dir}/haproxy$$.key -out ${ssl_tmp_dir}/haproxy$$.crt -subj "/CN={{ kolla_external_fqdn }}"
+    cat ${ssl_tmp_dir}/haproxy$$.crt ${ssl_tmp_dir}/haproxy$$.key> /etc/haproxy/certificates/haproxy.pem
+    rm -rf ${ssl_tmp_dir}
+    chown haproxy:haproxy /etc/haproxy/certificates/haproxy.pem
+    chmod 0660 /etc/haproxy/certificates/haproxy.pem
+fi
+{% endif %}
+{% if kolla_enable_tls_internal | bool %}
+if [ ! -e "/etc/haproxy/certificates/haproxy-internal.pem" ]; then
+    # Generate temporary self-signed cert
+    # This means external tls is enabled but the certificate was not copied
+    # to the container - so letsencrypt is enabled
+    #
+    # Let's generate certificate to make haproxy happy, lego will
+    # replace it in a while
+    ssl_tmp_dir=$(mktemp -d)
+    openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes -keyout ${ssl_tmp_dir}/haproxy-internal$$.key -out ${ssl_tmp_dir}/haproxy-internal$$.crt -subj "/CN={{ kolla_internal_fqdn }}"
+    cat ${ssl_tmp_dir}/haproxy-internal$$.crt ${ssl_tmp_dir}/haproxy-internal$$.key> /etc/haproxy/certificates/haproxy-internal.pem
+    rm -rf ${ssl_tmp_dir}
+    chown haproxy:haproxy /etc/haproxy/certificates/haproxy-internal.pem
+    chmod 0660 /etc/haproxy/certificates/haproxy-internal.pem
+fi
+{% endif %}
+{% endif %}
+
 find /etc/haproxy/services.d/ -mindepth 1 -print0 | \
     xargs -0 -Icfg echo -f cfg | \
     xargs /usr/sbin/haproxy -W -db -p /run/haproxy.pid -f /etc/haproxy/haproxy.cfg

ansible/site.yml

@@ -41,6 +41,7 @@
         - enable_iscsid_{{ enable_iscsid | bool }}
         - enable_keystone_{{ enable_keystone | bool }}
         - enable_kuryr_{{ enable_kuryr | bool }}
+        - enable_letsencrypt_{{ enable_letsencrypt | bool }}
         - enable_loadbalancer_{{ enable_loadbalancer | bool }}
         - enable_magnum_{{ enable_magnum | bool }}
         - enable_manila_{{ enable_manila | bool }}
@@ -200,6 +201,11 @@
             tasks_from: loadbalancer
           tags: keystone
           when: enable_keystone | bool
+        - include_role:
+            name: letsencrypt
+            tasks_from: loadbalancer
+          tags: letsencrypt
+          when: enable_letsencrypt | bool
         - include_role:
             name: magnum
             tasks_from: loadbalancer
@@ -340,6 +346,16 @@
         - enable_haproxy | bool
         - kolla_action in ['deploy', 'reconfigure', 'upgrade', 'config']
 
+- name: Apply role letsencrypt
+  gather_facts: false
+  hosts:
+    - letsencrypt
+    - '&enable_letsencrypt_True'
+  serial: '{{ kolla_serial|default("0") }}'
+  roles:
+    - { role: letsencrypt,
+        tags: letsencrypt }
+
 - name: Apply role collectd
   gather_facts: false
   hosts:

doc/source/admin/tls.rst

@@ -288,6 +288,35 @@ disable verification of the backend certificate:
 
 .. _admin-tls-generating-a-private-ca:
 
+Generating TLS certificates with Let's Encrypt
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Let's Encrypt is a free, automated, and open certificate authority.
+
+To enable OpenStack to deploy the Let's Encrypt container to fetch
+certificates from the Let's Encrypt certificate authority, the following
+must be configured in ``globals.yml``:
+
+.. code-block:: yaml
+
+  enable_letsencrypt: "yes"
+  letsencrypt_email: "<The email used for registration and recovery contact>"
+
+The Let's Encrypt container will attempt to renew your certificates every 12
+hours. If the certificates are renewed, they will automatically be deployed
+to the HAProxy containers using SSH.
+
+.. note::
+
+  If ``letsencrypt_email`` is not valid email, letsencrypt role will
+  not work correctly.
+
+.. note::
+
+  If ``enable_letsencrypt`` is set to true, haproxy's socket will run with
+  admin access level. This is needed so Let's Encrypt can interact
+  with HAProxy.
+
 Generating a Private Certificate Authority
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

etc/kolla/globals.yml

@@ -270,6 +270,19 @@ workaround_ansible_issue_8743: yes
 # Please read the docs for more details.
 #acme_client_servers: []
 
+####################
+# LetsEncrypt options
+####################
+# This option is required for letsencrypt role to work properly.
+#letsencrypt_email: ""
+
+####################
+# LetsEncrypt certificate server options
+####################
+#letsencrypt_cert_server: "https://acme-v02.api.letsencrypt.org/directory"
+# attempt to renew Let's Encrypt certificate every 12 hours
+#letsencrypt_cron_renew_schedule:  "0   */12   *   *   *"
+
 ################
 # Region options
 ################

etc/kolla/passwords.yml

@@ -205,6 +205,10 @@ neutron_ssh_key:
   private_key:
   public_key:
 
+haproxy_ssh_key:
+  private_key:
+  public_key:
+
 ####################
 # Gnocchi options
 ####################

kolla_ansible/cmd/genpwd.py

@@ -137,8 +137,9 @@ def main():
 
     # SSH key pair
     ssh_keys = ['kolla_ssh_key', 'nova_ssh_key',
-                'keystone_ssh_key', 'bifrost_ssh_key', 'octavia_amp_ssh_key',
+                'keystone_ssh_key', 'bifrost_ssh_key',
-                'neutron_ssh_key']
+                'octavia_amp_ssh_key', 'neutron_ssh_key',
+                'haproxy_ssh_key']
 
     # If these keys are None, leave them as None
     blank_keys = ['docker_registry_password']

releasenotes/notes/add-lets-encrypt-intergration-9e5f9846536379af.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - Add Lets Encrypt TLS certificate service integration into Openstack
+    deployment. Enables trusted TLS certificate generation option for
+    secure communcation with OpenStack HAProxy instances using
+    ``letsencrypt_email``, ``kolla_internal_fqdn`` and/or
+    ``kolla_external_fqdn`` is required. One container runs an Apache
+    ACME client webserver and one runs Lego for certificate retrieval
+    and renewal. The Lego container starts a cron job which attempts
+    to renew certificates every 12 hours.

tests/deploy.sh

@@ -6,18 +6,63 @@ set -o errexit
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
+function init_pebble {
+
+    sudo echo "[i] Pulling letsencrypt/pebble" > /tmp/logs/ansible/certificates
+    sudo docker pull letsencrypt/pebble &>> /tmp/logs/ansible/certificates
+
+    sudo echo "[i] Force removing old pebble container" &>> /tmp/logs/ansible/certificates
+    sudo docker rm -f pebble &>> /tmp/logs/ansible/certificates
+
+    sudo echo "[i] Run new pebble container" &>> /tmp/logs/ansible/certificates
+    sudo docker run --name pebble --rm -d -e "PEBBLE_VA_NOSLEEP=1" -e "PEBBLE_VA_ALWAYS_VALID=1" --net=host letsencrypt/pebble &>> /tmp/logs/ansible/certificates
+
+    sudo echo "[i] Wait for pebble container be up" &>> /tmp/logs/ansible/certificates
+    # wait until pebble starts
+    while ! sudo docker logs pebble | grep -q "Listening on"; do
+        sleep 1
+    done
+    sudo echo "[i] Wait for pebble container done" &>> /tmp/logs/ansible/certificates
+
+    sudo echo "[i] Pebble container logs" &>> /tmp/logs/ansible/certificates
+    sudo docker logs pebble &>> /tmp/logs/ansible/certificates
+}
+
+function pebble_cacert {
+
+    sudo docker cp pebble:/test/certs/pebble.minica.pem /etc/kolla/certificates/ca/pebble-root.crt
+    sudo curl -k -s -o /etc/kolla/certificates/ca/pebble.crt -v https://127.0.0.1:15000/roots/0
+}
+
+function certificates {
 
-function deploy {
     RAW_INVENTORY=/etc/kolla/inventory
-
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
-    #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
-    sudo chmod -R 777 /etc/kolla
     # generate self-signed certificates for the optional internal TLS tests
     if [[ "$TLS_ENABLED" = "True" ]]; then
         kolla-ansible -i ${RAW_INVENTORY} -vvv certificates > /tmp/logs/ansible/certificates
     fi
+    if [[ "$LE_ENABLED" = "True" ]]; then
+        init_pebble
+        pebble_cacert
+    fi
+
+    #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
+    sudo chmod -R 777 /etc/kolla
+}
+
+
+function deploy {
+
+    RAW_INVENTORY=/etc/kolla/inventory
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
+
+    #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
+    sudo chmod -R 777 /etc/kolla
+
+    certificates
+
     # Actually do the deployment
     kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/deploy-prechecks
     kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull

tests/run.yml

@@ -21,11 +21,12 @@
         need_build_image: "{{ kolla_build_images | default(false) }}"
         build_image_tag: "change_{{ zuul.change | default('none') }}"
         openstack_core_enabled: "{{ openstack_core_enabled }}"
-        openstack_core_tested: "{{ scenario in ['core', 'cephadm', 'zun', 'cells', 'swift', 'ovn'] }}"
+        openstack_core_tested: "{{ scenario in ['core', 'cephadm', 'zun', 'cells', 'swift', 'ovn', 'lets-encrypt'] }}"
         dashboard_enabled: "{{ openstack_core_enabled }}"
         upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt"
         docker_image_tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}"
         kolla_ansible_venv_path: "{{ ansible_env.HOME }}/kolla-ansible-venv"
+        kolla_internal_fqdn: "kolla.example.com"
 
     - name: Install dig for Designate testing
       become: true
@@ -46,6 +47,18 @@
       vars:
         disk_type: "{{ 'ceph-lvm' if scenario in ['cephadm'] else scenario }}"
 
+    - name: Update /etc/hosts with internal API FQDN
+      blockinfile:
+        dest: /etc/hosts
+        marker: "# {mark} ANSIBLE GENERATED INTERNAL API FQDN"
+        block: |
+          {{ kolla_internal_vip_address }} {{ kolla_internal_fqdn }}
+          192.0.2.1 pebble
+      become: True
+      when:
+        - scenario == "lets-encrypt"
+
+
 - hosts: primary
   any_errors_fatal: true
   vars:
@@ -397,6 +410,7 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             TLS_ENABLED: "{{ tls_enabled }}"
+            LE_ENABLED: "{{ le_enabled }}"
             KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
             HAS_UPGRADE: "{{ is_upgrade | bool | ternary('yes', 'no') }}"
 
@@ -410,6 +424,7 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             TLS_ENABLED: "{{ tls_enabled }}"
+            LE_ENABLED: "{{ le_enabled }}"
           when: dashboard_enabled
 
         - name: Run init-core-openstack.sh script

tests/setup_gate.sh

@@ -94,6 +94,10 @@ function prepare_images {
         GATE_IMAGES="^cron,^fluentd,^haproxy,^keepalived,^kolla-toolbox,^mariadb"
     fi
 
+    if [[ $SCENARIO == "lets-encrypt" ]]; then
+        GATE_IMAGES+=",^letsencrypt,^haproxy"
+    fi
+
     if [[ $SCENARIO == "prometheus-opensearch" ]]; then
         GATE_IMAGES="^cron,^fluentd,^grafana,^haproxy,^keepalived,^kolla-toolbox,^mariadb,^memcached,^opensearch,^prometheus,^rabbitmq"
     fi

tests/templates/globals-default.j2

@@ -206,3 +206,13 @@ keepalived_track_script_enabled: "no"
 neutron_modules_extra:
   - name: 'nf_conntrack_tftp'
   - name: 'nf_nat_tftp'
+
+{% if scenario == "lets-encrypt" %}
+enable_letsencrypt: "yes"
+rabbitmq_enable_tls: "yes"
+letsencrypt_email: "usero@openstack.test"
+letsencrypt_cert_server: "https://pebble:14000/dir"
+kolla_internal_fqdn: "{{ kolla_internal_fqdn }}"
+kolla_enable_tls_backend: "no"
+kolla_admin_openrc_cacert: "{% raw %}{{ kolla_certificates_dir }}{% endraw %}/ca/pebble.crt"
+{% endif %}

tests/templates/inventory.j2

@@ -258,6 +258,9 @@ control
 [venus:children]
 monitoring
 
+[letsencrypt:children]
+loadbalancer
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -749,3 +752,9 @@ venus
 
 [venus-manager:children]
 venus
+
+[letsencrypt-webserver:children]
+letsencrypt
+
+[letsencrypt-lego:children]
+letsencrypt

zuul.d/base.yaml

@@ -47,6 +47,7 @@
       neutron_tenant_network_prefix_length: "24"
       neutron_tenant_network_dns_server: "8.8.8.8"
       tls_enabled: false
+      le_enabled: false
       configure_swap_size: 0
     roles:
       - zuul: zuul/zuul-jobs
@@ -245,3 +246,18 @@
       - ^kolla_ansible/
       - ^tests/run-hashi-vault.yml
       - ^tests/test-hashicorp-vault-passwords.sh
+
+- job:
+    name: kolla-ansible-lets-encrypt-base
+    parent: kolla-ansible-base
+    voting: false
+    files:
+      - ^ansible/roles/letsencrypt/
+      - ^ansible/roles/loadbalancer/
+      - ^tests/test-core-openstack.sh
+      - ^tests/test-dashboard.sh
+      - ^tests/deploy.sh
+    vars:
+      scenario: lets-encrypt
+      tls_enabled: true
+      le_enabled: true

zuul.d/jobs.yaml

@@ -405,6 +405,22 @@
     vars:
       base_distro: ubuntu
 
+- job:
+    name: kolla-ansible-ubuntu-lets-encrypt
+    parent: kolla-ansible-lets-encrypt-base
+    nodeset: kolla-ansible-jammy-multi
+    vars:
+      base_distro: ubuntu
+      install_type: source
+
+- job:
+    name: kolla-ansible-rocky9-lets-encrypt
+    parent: kolla-ansible-lets-encrypt-base
+    nodeset: kolla-ansible-rocky9-multi
+    vars:
+      base_distro: rocky
+      install_type: source
+
 - job:
     name: kolla-ansible-rocky9-prometheus-opensearch
     parent: kolla-ansible-prometheus-opensearch-base

zuul.d/project.yaml

@@ -62,6 +62,8 @@
         - kolla-ansible-rocky9-upgrade-cephadm
         - kolla-ansible-ubuntu-upgrade-cephadm
         - kolla-ansible-rocky9-hashi-vault
+        - kolla-ansible-ubuntu-lets-encrypt
+        - kolla-ansible-rocky9-lets-encrypt
     check-arm64:
       jobs:
         - kolla-ansible-debian-aarch64