Add support for encrypting backend Neutron API Server

Add TLS support for backend Neutron API Server communication using
HAProxy to perform TLS termination. When used in conjunction with
enabling TLS for service API endpoints, network communication will be
encrypted end to end, from client through HAProxy to the Neutron
service.

Change-Id: Ib333a1f1bd12491df72a9e52d961161210e2d330
Partially-Implements: blueprint add-ssl-internal-network

ansible/group_vars/all.yml

@@ -380,6 +380,7 @@ neutron_internal_fqdn: "{{ kolla_internal_fqdn }}"
 neutron_external_fqdn: "{{ kolla_external_fqdn }}"
 neutron_server_port: "9696"
 neutron_server_listen_port: "{{ neutron_server_port }}"
+neutron_tls_proxy_stats_port: "9697"
 
 nova_internal_fqdn: "{{ kolla_internal_fqdn }}"
 nova_external_fqdn: "{{ kolla_external_fqdn }}"
@@ -979,6 +980,7 @@ neutron_public_endpoint: "{{ public_protocol }}://{{ neutron_external_fqdn | put
 # "sriovnet1" and tunnel_interface used here as placeholders
 neutron_sriov_physnet_mappings:
   sriovnet1: "{{ tunnel_interface }}"
+neutron_enable_tls_backend: "{{ kolla_enable_tls_backend }}"
 
 #######################
 # Nova options

ansible/roles/common/defaults/main.yml

@@ -70,6 +70,7 @@ fluentd_image_full: "{{ fluentd_image }}:{{ fluentd_tag }}"
 syslog_swift_facility: "local0"
 syslog_haproxy_facility: "local1"
 syslog_glance_tls_proxy_facility: "local2"
+syslog_neutron_tls_proxy_facility: "local4"
 
 kolla_toolbox_default_volumes:
   - "{{ node_config_directory }}/kolla-toolbox/:{{ container_config_directory }}/:ro"
@@ -153,6 +154,8 @@ fluentd_input_openstack_services:
     enabled: "{{ enable_murano | bool }}"
   - name: neutron
     enabled: "{{ enable_neutron | bool }}"
+  - name: neutron-tls-proxy
+    enabled: "{{ neutron_enable_tls_backend | bool }}"
   - name: nova
     enabled: "{{ enable_nova | bool }}"
   - name: octavia

ansible/roles/common/tasks/config.yml

@@ -203,6 +203,8 @@
       - { name: "monasca", enabled: "{{ enable_monasca | bool }}" }
       - { name: "murano", enabled: "{{ enable_murano | bool }}" }
       - { name: "neutron", enabled: "{{ enable_neutron | bool }}" }
+      - { name: "neutron-tls-proxy", enabled: "{{ neutron_enable_tls_backend |
+      bool }}" }
       - { name: "nova", enabled: "{{ enable_nova | bool }}" }
       - { name: "octavia", enabled: "{{ enable_octavia | bool }}" }
       - { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq | bool }}" }

ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2

@@ -43,6 +43,13 @@
     </record>
 </filter>
 
+<filter syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
+    @type record_transformer
+    <record>
+        programname neutron-tls-proxy
+    </record>
+</filter>
+
 # Rename internal Fluent message field to match other logs. This removes
 # all other fields by default, including the original message field. This is
 # intented to avoid duplication of the log message and to prevent passing

ansible/roles/common/templates/conf/output/00-local.conf.j2

@@ -180,3 +180,64 @@
 {% endif %}
 </match>
 {% endif %}
+
+{% if neutron_enable_tls_backend | bool %}
+<match syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
+  @type copy
+  <store>
+    @type file
+    path /var/log/kolla/neutron-tls-proxy/neutron-tls-proxy.*.log
+    output_tag false
+    output_time false
+    append true
+    compress gzip
+    <buffer>
+      timekey_use_utc
+    </buffer>
+  </store>
+{% if log_direct_to_elasticsearch %}
+  <store>
+       type elasticsearch
+       host {{ elasticsearch_address }}
+       port {{ elasticsearch_port }}
+       scheme {{ fluentd_elasticsearch_scheme }}
+{% if fluentd_elasticsearch_path != '' %}
+       path {{ fluentd_elasticsearch_path }}
+{% endif %}
+{% if fluentd_elasticsearch_scheme == 'https' %}
+       ssl_version {{ fluentd_elasticsearch_ssl_version }}
+       ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
+{% endif %}
+{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
+       user {{ fluentd_elasticsearch_user }}
+       password {{ fluentd_elasticsearch_password }}
+{% endif %}
+       logstash_format true
+       logstash_prefix {{ kibana_log_prefix }}
+       flush_interval 15s
+       reconnect_on_error true
+       buffer_type file
+       buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
+  </store>
+{% elif enable_monasca | bool %}
+  <store>
+     @type monasca
+     keystone_url {{ keystone_internal_url }}
+     monasca_api {{ monasca_log_api_internal_endpoint }}
+     monasca_api_version v2.0
+     username {{ monasca_agent_user }}
+     password {{ monasca_agent_password }}
+     domain_id default
+     project_name {{ monasca_control_plane_project }}
+     message_field_name Payload
+     buffer_type file
+     buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
+     max_retry_wait 1800s
+     disable_retry_limit true
+     <buffer>
+       chunk_limit_size 8m
+     </buffer>
+  </store>
+{% endif %}
+</match>
+{% endif %}

ansible/roles/common/templates/cron-logrotate-neutron-tls-proxy.conf.j2

@@ -0,0 +1,3 @@
+"/var/log/kolla/neutron-tls-proxy/*.log"
+{
+}

ansible/roles/common/templates/fluentd.json.j2

@@ -40,6 +40,13 @@
             "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
             "recurse": true
         },
+{% endif %}
+{% if neutron_enable_tls_backend | bool %}
+        {
+            "path": "/var/log/kolla/neutron-tls-proxy",
+            "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
+            "recurse": true
+        },
 {% endif %}
         {
             "path": "/var/log/kolla/swift",

ansible/roles/neutron/defaults/main.yml

@@ -13,13 +13,13 @@ neutron_services:
     healthcheck: "{{ neutron_server_healthcheck }}"
     haproxy:
       neutron_server:
-        enabled: "{{ enable_neutron }}"
+        enabled: "{{ enable_neutron | bool and not neutron_enable_tls_backend | bool}}"
         mode: "http"
         external: false
         port: "{{ neutron_server_port }}"
         listen_port: "{{ neutron_server_listen_port }}"
       neutron_server_external:
-        enabled: "{{ enable_neutron }}"
+        enabled: "{{ enable_neutron | bool and not neutron_enable_tls_backend | bool}}"
         mode: "http"
         external: true
         port: "{{ neutron_server_port }}"
@@ -174,6 +174,30 @@ neutron_services:
     host_in_groups: "{{ inventory_hostname in groups['ironic-neutron-agent'] }}"
     volumes: "{{ ironic_neutron_agent_default_volumes + ironic_neutron_agent_extra_volumes }}"
     dimensions: "{{ ironic_neutron_agent_dimensions }}"
+  neutron-tls-proxy:
+    container_name: neutron_tls_proxy
+    group: neutron-server
+    host_in_groups: "{{ inventory_hostname in groups['neutron-server'] }}"
+    enabled: "{{ neutron_enable_tls_backend }}"
+    image: "{{ neutron_tls_proxy_image_full }}"
+    volumes: "{{ neutron_tls_proxy_default_volumes + neutron_tls_proxy_extra_volumes }}"
+    dimensions: "{{ neutron_tls_proxy_dimensions }}"
+    healthcheck: "{{ neutron_tls_proxy_healthcheck }}"
+    haproxy:
+      neutron_tls_proxy:
+        enabled: "{{ enable_neutron | bool and neutron_enable_tls_backend | bool }}"
+        mode: "http"
+        external: false
+        port: "{{ neutron_server_port }}"
+        listen_port: "{{ neutron_server_listen_port }}"
+        tls_backend: "yes"
+      neutron_tls_proxy_external:
+        enabled: "{{ enable_neutron | bool and neutron_enable_tls_backend | bool }}"
+        mode: "http"
+        external: true
+        port: "{{ neutron_server_port }}"
+        listen_port: "{{ neutron_server_listen_port }}"
+        tls_backend: "yes"
 
 ####################
 # Database
@@ -245,6 +269,10 @@ ironic_neutron_agent_image: "{{ docker_registry ~ '/' if docker_registry else ''
 ironic_neutron_agent_tag: "{{ neutron_tag }}"
 ironic_neutron_agent_image_full: "{{ ironic_neutron_agent_image }}:{{ ironic_neutron_agent_tag }}"
 
+neutron_tls_proxy_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ neutron_install_type }}-haproxy"
+neutron_tls_proxy_tag: "{{ neutron_tag }}"
+neutron_tls_proxy_image_full: "{{ neutron_tls_proxy_image }}:{{ neutron_tls_proxy_tag }}"
+
 neutron_agent_dimensions: "{{ default_container_dimensions }}"
 
 neutron_dhcp_agent_dimensions: "{{ neutron_agent_dimensions }}"
@@ -261,6 +289,7 @@ neutron_bgp_dragent_dimensions: "{{ default_container_dimensions }}"
 neutron_infoblox_ipam_agent_dimensions: "{{ default_container_dimensions }}"
 neutron_metering_agent_dimensions: "{{ neutron_agent_dimensions }}"
 ironic_neutron_agent_dimensions: "{{ default_container_dimensions }}"
+neutron_tls_proxy_dimensions: "{{ default_container_dimensions }}"
 
 neutron_dhcp_agent_enable_healthchecks: "{{ enable_container_healthchecks }}"
 neutron_dhcp_agent_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
@@ -344,7 +373,7 @@ neutron_server_enable_healthchecks: "{{ enable_container_healthchecks }}"
 neutron_server_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
 neutron_server_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
 neutron_server_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
-neutron_server_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{{ api_interface_address |  put_address_in_context('url') }}:{{ neutron_server_listen_port }}"]
+neutron_server_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{% if neutron_enable_tls_backend | bool %}localhost{% else %}{{ api_interface_address |  put_address_in_context('url') }}{% endif %}:{{ neutron_server_listen_port }}"]
 neutron_server_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
 neutron_server_healthcheck:
   interval: "{{ neutron_server_healthcheck_interval }}"
@@ -353,6 +382,19 @@ neutron_server_healthcheck:
   test: "{% if neutron_server_enable_healthchecks | bool %}{{ neutron_server_healthcheck_test }}{% else %}NONE{% endif %}"
   timeout: "{{ neutron_server_healthcheck_timeout }}"
 
+neutron_tls_proxy_enable_healthchecks: "{{ enable_container_healthchecks }}"
+neutron_tls_proxy_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
+neutron_tls_proxy_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
+neutron_tls_proxy_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
+neutron_tls_proxy_healthcheck_test: ["CMD-SHELL", "healthcheck_curl -u {{ haproxy_user }}:{{ haproxy_password }} {{ api_interface_address |  put_address_in_context('url') }}:{{ neutron_tls_proxy_stats_port }}"]
+neutron_tls_proxy_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
+neutron_tls_proxy_healthcheck:
+  interval: "{{ neutron_tls_proxy_healthcheck_interval }}"
+  retries: "{{ neutron_tls_proxy_healthcheck_retries }}"
+  start_period: "{{ neutron_tls_proxy_healthcheck_start_period }}"
+  test: "{% if neutron_tls_proxy_enable_healthchecks | bool %}{{ neutron_tls_proxy_healthcheck_test }}{% else %}NONE{% endif %}"
+  timeout: "{{ neutron_tls_proxy_healthcheck_timeout }}"
+
 neutron_sriov_agent_enable_healthchecks: "{{ enable_container_healthchecks }}"
 neutron_sriov_agent_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
 neutron_sriov_agent_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
@@ -459,6 +501,11 @@ ironic_neutron_agent_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
   - "kolla_logs:/var/log/kolla/"
+neutron_tls_proxy_default_volumes:
+  - "{{ node_config_directory }}/neutron-tls-proxy/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
+  - "kolla_logs:/var/log/kolla/"
 
 neutron_extra_volumes: "{{ default_extra_volumes }}"
 neutron_dhcp_agent_extra_volumes: "{{ neutron_extra_volumes }}"
@@ -475,6 +522,7 @@ neutron_bgp_dragent_extra_volumes: "{{ neutron_extra_volumes }}"
 neutron_infoblox_ipam_agent_extra_volumes: "{{ neutron_extra_volumes }}"
 neutron_metering_agent_extra_volumes: "{{ neutron_extra_volumes }}"
 ironic_neutron_agent_extra_volumes: "{{ neutron_extra_volumes }}"
+neutron_tls_proxy_extra_volumes: "{{ neutron_extra_volumes }}"
 
 ####################
 # OpenStack
@@ -712,3 +760,23 @@ neutron_ks_users:
 # SRIOV
 ####################
 neutron_sriov_physnets: "{{ neutron_sriov_physnet_mappings.items() | map('join', ':') | join(',') }}"
+
+####################
+# Backend TLS proxy
+####################
+syslog_server: "{{ api_interface_address }}"
+syslog_neutron_tls_proxy_facility: "local4"
+
+neutron_tls_proxy_max_connections: 40000
+neutron_tls_proxy_processes: 1
+neutron_tls_proxy_process_cpu_map: "no"
+neutron_tls_proxy_defaults_max_connections: 10000
+neutron_tls_proxy_http_request_timeout: "10s"
+neutron_tls_proxy_http_keep_alive_timeout: "10s"
+neutron_tls_proxy_queue_timeout: "1m"
+neutron_tls_proxy_connect_timeout: "10s"
+neutron_tls_proxy_client_timeout: "1m"
+neutron_tls_proxy_server_timeout: "1m"
+neutron_tls_proxy_check_timeout: "10s"
+# Check http://www.haproxy.org/download/1.5/doc/configuration.txt for available options
+neutron_tls_proxy_defaults_balance: "roundrobin"

ansible/roles/neutron/handlers/main.yml

@@ -154,6 +154,22 @@
   when:
     - kolla_action != "config"
 
+- name: Restart neutron-tls-proxy container
+  vars:
+    service_name: "neutron-tls-proxy"
+    service: "{{ neutron_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+    healthcheck: "{{ service.healthcheck | default(omit) }}"
+  when:
+    - kolla_action != "config"
+
 - name: Restart neutron-metadata-agent container
   vars:
     service_name: "neutron-metadata-agent"

ansible/roles/neutron/tasks/config.yml

@@ -22,7 +22,21 @@
 
 - include_tasks: copy-certs.yml
   when:
-    - kolla_copy_ca_into_containers | bool
+    - kolla_copy_ca_into_containers | bool or neutron_enable_tls_backend | bool
+
+- name: Creating TLS backend PEM File
+  vars:
+    neutron_tls_proxy: "{{ neutron_services['neutron-tls-proxy'] }}"
+  assemble:
+    src: "{{ node_config_directory }}/neutron-tls-proxy/"
+    dest: "{{ node_config_directory }}/neutron-tls-proxy/neutron-cert-and-key.pem"
+    mode: "0660"
+    regexp: "^neutron-(cert|key)\\.pem$"
+    remote_src: true
+  become: true
+  when:
+    - neutron_tls_proxy.enabled | bool
+    - neutron_tls_proxy.host_in_groups | bool
 
 - name: Check if policies shall be overwritten
   stat:
@@ -438,5 +452,24 @@
   notify:
     - "Restart {{ item.0 }} container"
 
+- name: Copying over neutron-tls-proxy.cfg
+  vars:
+    neutron_tls_proxy: "{{ neutron_services['neutron-tls-proxy'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/neutron-tls-proxy/neutron-tls-proxy.cfg"
+    mode: "0660"
+  become: true
+  with_first_found:
+    - "{{ node_custom_config }}/neutron/{{ inventory_hostname }}/neutron-tls-proxy.cfg"
+    - "{{ node_custom_config }}/neutron/neutron-tls-proxy.cfg"
+    - "neutron-tls-proxy.cfg.j2"
+  when:
+    - neutron_tls_proxy.enabled | bool
+    - neutron_tls_proxy.host_in_groups | bool
+  notify:
+    - Restart neutron-tls-proxy container
+
+
 - import_tasks: check-containers.yml
   when: kolla_action != "config"

ansible/roles/neutron/templates/neutron-tls-proxy.cfg.j2

@@ -0,0 +1,47 @@
+#jinja2: lstrip_blocks: True
+global
+    chroot /var/lib/haproxy
+    user neutron
+    group neutron
+    daemon
+    log {{ syslog_server }}:{{ syslog_udp_port }} {{ syslog_neutron_tls_proxy_facility }}
+    maxconn {{ neutron_tls_proxy_max_connections }}
+    nbproc {{ neutron_tls_proxy_processes }}
+    {% if (neutron_tls_proxy_processes | int > 1) and (neutron_tls_proxy_process_cpu_map | bool) %}
+        {% for cpu_idx in range(0, neutron_tls_proxy_processes) %}
+    cpu-map {{ cpu_idx + 1 }} {{ cpu_idx }}
+        {% endfor %}
+    {% endif %}
+    ssl-default-bind-ciphers DEFAULT:!MEDIUM:!3DES
+    ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11
+    tune.ssl.default-dh-param 4096
+
+defaults
+    log global
+    option redispatch
+    retries 3
+    timeout http-request {{ neutron_tls_proxy_http_request_timeout }}
+    timeout http-keep-alive {{ neutron_tls_proxy_http_keep_alive_timeout }}
+    timeout queue {{ neutron_tls_proxy_queue_timeout }}
+    timeout connect {{ neutron_tls_proxy_connect_timeout }}
+    timeout client {{ neutron_tls_proxy_client_timeout }}
+    timeout server {{ neutron_tls_proxy_server_timeout }}
+    timeout check {{ neutron_tls_proxy_check_timeout }}
+    balance {{ neutron_tls_proxy_defaults_balance }}
+    maxconn {{ neutron_tls_proxy_defaults_max_connections }}
+
+listen stats
+   bind {{ api_interface_address }}:{{ neutron_tls_proxy_stats_port }}
+   mode http
+   stats enable
+   stats uri /
+   stats refresh 15s
+   stats realm Haproxy\ Stats
+   stats auth {{ haproxy_user }}:{{ haproxy_password }}
+
+frontend neutron_backend_tls
+    bind {{ api_interface_address }}:{{ neutron_server_listen_port }} ssl crt /etc/neutron/certs/neutron-cert-and-key.pem
+    default_backend neutron_server
+
+backend neutron_server
+    server neutron-server 127.0.0.1:{{ neutron_server_listen_port }} check

ansible/roles/neutron/templates/neutron-tls-proxy.json.j2

@@ -0,0 +1,17 @@
+{
+    "command": "/usr/sbin/haproxy -W -db -p /run/haproxy.pid -f /etc/neutron/neutron-tls-proxy.cfg",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/neutron-tls-proxy.cfg",
+            "dest": "/etc/neutron/neutron-tls-proxy.cfg",
+            "owner": "neutron",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/neutron-cert-and-key.pem",
+            "dest": "/etc/neutron/certs/neutron-cert-and-key.pem",
+            "owner": "neutron",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/neutron/templates/neutron.conf.j2

@@ -7,8 +7,12 @@ log_dir = /var/log/kolla/neutron
 # NOTE(elemoine): set use_stderr to False or the logs will also be sent to
 # stderr and collected by Docker
 use_stderr = False
-
+{% if neutron_enable_tls_backend | bool %}
+bind_host = 127.0.0.1
+{% else %}
 bind_host = {{ api_interface_address }}
+{% endif %}
+
 bind_port = {{ neutron_server_listen_port }}
 
 api_paste_config = /usr/share/neutron/api-paste.ini

doc/source/admin/advanced-configuration.rst

@@ -264,6 +264,10 @@ If Glance TLS backend is enabled (``glance_enable_tls_backend``), the syslog
 facility for the ``glance_tls_proxy`` service uses ``local2`` by default. This
 can be set via ``syslog_glance_tls_proxy_facility``.
 
+If Neutron TLS backend is enabled (``neutron_enable_tls_backend``), the syslog
+facility for the ``neutron_tls_proxy`` service uses ``local4`` by default.
+This can be set via ``syslog_neutron_tls_proxy_facility``.
+
 Mount additional Docker volumes in containers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/admin/tls.rst

@@ -218,8 +218,8 @@ Enabling TLS on the backend services secures communication between the
 HAProxy listing on the internal/external VIP and the OpenStack
 services. It also enables secure end-to-end communication between OpenStack
 services that support TLS termination. The OpenStack services that support
-backend TLS termination in Ussuri are: Keystone, Glance, Heat, Placement,
+backend TLS termination in Victoria are: Nova, Ironic, Neutron, Keystone,
-Horizon, Barbican, and Cinder.
+Glance, Heat, Placement, Horizon, Barbican, and Cinder.
 
 The configuration variables that control back-end TLS for service endpoints
 are:

releasenotes/notes/encrypt-additional-services-backend-haproxy-29467a9771e99917.yaml

@@ -2,6 +2,6 @@
 features:
   - |
     Adds configuration options to enable backend TLS encryption from HAProxy
-    to the Nova and Ironic services. When used in conjunction with enabling TLS
+    to the Nova, Ironic, and Neutron services. When used in conjunction with
-    for service API endpoints, network communcation will be encrypted end to
+    enabling TLS for service API endpoints, network communcation will be
-    end, from client through HAProxy to the backend service.
+    encrypted end to end, from client through HAProxy to the backend service.