From 2d920745a45eec166e2c381aa7b15b1168b03887 Mon Sep 17 00:00:00 2001
From: "Michal Jastrzebski (inc0)" <michal.jastrzebski@intel.com>
Date: Fri, 17 Jul 2015 12:20:31 +0200
Subject: [PATCH] Ansible install of HAProxy

This commit consists HAProxy ansible bits including config generation,
container deployment and hot reloads.

Closes-Bug: #1477915
Co-Authored-By: Sam Yaple <sam@yaple.net>
Change-Id: Ie93fa68fdb6b2885889c992ff1267d38b68e0cbc
Partially-implements: blueprint ansible-service
---
 ansible/inventory/all-in-one                  |  3 +
 ansible/inventory/multinode                   |  3 +
 ansible/roles/haproxy/defaults/main.yml       | 16 ++++
 ansible/roles/haproxy/tasks/config.yml        | 14 ++++
 ansible/roles/haproxy/tasks/main.yml          |  4 +
 ansible/roles/haproxy/tasks/start.yml         | 14 ++++
 .../roles/haproxy/templates/haproxy.cfg.j2    | 73 +++++++++++++++++++
 ansible/site.yml                              |  8 +-
 docker/centos/binary/haproxy/Dockerfile       |  2 +-
 .../binary/haproxy/ensure_latest_config.sh    |  1 +
 docker/common/haproxy/config-external.sh      |  3 +-
 docker/common/haproxy/ensure_latest_config.sh | 12 +++
 docker/common/haproxy/start.sh                | 15 ++--
 13 files changed, 158 insertions(+), 10 deletions(-)
 create mode 100755 ansible/roles/haproxy/defaults/main.yml
 create mode 100755 ansible/roles/haproxy/tasks/config.yml
 create mode 100755 ansible/roles/haproxy/tasks/main.yml
 create mode 100755 ansible/roles/haproxy/tasks/start.yml
 create mode 100755 ansible/roles/haproxy/templates/haproxy.cfg.j2
 mode change 100644 => 100755 ansible/site.yml
 create mode 120000 docker/centos/binary/haproxy/ensure_latest_config.sh
 create mode 100755 docker/common/haproxy/ensure_latest_config.sh

diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 7fd25f9d66..c79d87eb97 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -12,6 +12,9 @@ localhost       ansible_connection=local
 
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
+[haproxy:children]
+support
+
 [database:children]
 support
 
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 5df5a783fe..116f813ff2 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -15,6 +15,9 @@ compute01
 
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
+[haproxy:children]
+support
+
 [database:children]
 support
 
diff --git a/ansible/roles/haproxy/defaults/main.yml b/ansible/roles/haproxy/defaults/main.yml
new file mode 100755
index 0000000000..1d8b444a62
--- /dev/null
+++ b/ansible/roles/haproxy/defaults/main.yml
@@ -0,0 +1,16 @@
+---
+project_name: "haproxy"
+
+
+####################
+# Docker
+####################
+docker_haproxy_registry: "{{ docker_registry }}"
+docker_haproxy_namespace: "{{ docker_namespace }}"
+kolla_haproxy_base_distro: "{{ kolla_base_distro }}"
+kolla_haproxy_install_type: "{{ kolla_install_type }}"
+kolla_haproxy_container_name: "haproxy"
+
+docker_haproxy_image: "{{ docker_haproxy_registry }}{{ docker_haproxy_namespace }}/{{ kolla_haproxy_base_distro }}-{{ kolla_haproxy_install_type }}-{{ kolla_haproxy_container_name }}"
+docker_haproxy_tag: "{{ openstack_release }}"
+docker_haproxy_image_full: "{{ docker_haproxy_image }}:{{ docker_haproxy_tag }}"
diff --git a/ansible/roles/haproxy/tasks/config.yml b/ansible/roles/haproxy/tasks/config.yml
new file mode 100755
index 0000000000..6aa4c0bba8
--- /dev/null
+++ b/ansible/roles/haproxy/tasks/config.yml
@@ -0,0 +1,14 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/haproxy/"
+    state: "directory"
+    recurse: "yes"
+
+- name: Copying over config(s)
+  template:
+    src: "haproxy.cfg.j2"
+    dest: "{{ node_config_directory }}/haproxy/haproxy.cfg"
+
+- name: Allowing non-local IP binding
+  sysctl: name="net.ipv4.ip_nonlocal_bind" value=1 sysctl_set=yes
diff --git a/ansible/roles/haproxy/tasks/main.yml b/ansible/roles/haproxy/tasks/main.yml
new file mode 100755
index 0000000000..1f16915ad9
--- /dev/null
+++ b/ansible/roles/haproxy/tasks/main.yml
@@ -0,0 +1,4 @@
+---
+- include: config.yml
+
+- include: start.yml
diff --git a/ansible/roles/haproxy/tasks/start.yml b/ansible/roles/haproxy/tasks/start.yml
new file mode 100755
index 0000000000..cbb577b0f2
--- /dev/null
+++ b/ansible/roles/haproxy/tasks/start.yml
@@ -0,0 +1,14 @@
+---
+- include: ../../start.yml
+  vars:
+    container_image: "{{ docker_haproxy_image_full }}"
+    container_name: "haproxy"
+    container_volumes:
+      - "{{ node_config_directory }}/haproxy/:/opt/kolla/haproxy/:ro"
+    container_environment:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+
+- name: Ensuring latest haproxy config is used
+  command: docker exec haproxy /opt/kolla/ensure_latest_config.sh
+  register: status
+  changed_when: status.stdout.find('changed') != -1
diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2
new file mode 100755
index 0000000000..07fe7aacf4
--- /dev/null
+++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2
@@ -0,0 +1,73 @@
+global
+  daemon
+  maxconn 4000
+
+defaults
+  mode http
+  maxconn 4000
+  option redispatch
+  retries 3
+  timeout http-request 10s
+  timeout queue 1m
+  timeout connect 10s
+  timeout client 1m
+  timeout server 1m
+  timeout check 10s
+
+listen mariadb
+  mode tcp
+  option mysql-check user haproxy
+  option tcpka
+
+  # TODO(SamYaple): Make DB port configurable throughout Ansible
+  bind {{ kolla_internal_address }}:3306
+{% for host in groups['database'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:3306 check inter 2000 rise 2 fall 5 {% if not loop.first %}backup{% endif %}
+
+{% endfor %}
+
+listen rabbitmq
+  mode tcp
+  option tcpka
+
+  # TODO(SamYaple): Make rabbitmq port configurable throughout Ansible
+  bind {{ kolla_internal_address }}:5672
+{% for host in groups['message-broker'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:5672 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen keystone_public
+  bind {{ kolla_internal_address }}:{{ keystone_public_port }}
+{% for host in groups['keystone'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:{{ keystone_public_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen keystone_admin
+  bind {{ kolla_internal_address }}:{{ keystone_admin_port }}
+{% for host in groups['keystone'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:{{ keystone_admin_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen glance_registry
+  bind {{ kolla_internal_address }}:{{ glance_registry_port }}
+{% for host in groups['glance'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:{{ glance_registry_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen glance_api
+  bind {{ kolla_internal_address }}:{{ glance_api_port }}
+{% for host in groups['glance'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:{{ glance_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen nova_api
+  bind {{ kolla_internal_address }}:{{ nova_api_port }}
+{% for host in groups['nova-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:{{ nova_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen neutron_server
+  bind {{ kolla_internal_address }}:{{ neutron_server_port }}
+{% for host in groups['neutron-server'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:{{ neutron_server_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
diff --git a/ansible/site.yml b/ansible/site.yml
old mode 100644
new mode 100755
index 5abb1d07de..1b708080be
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -1,8 +1,14 @@
 ---
+- hosts: haproxy
+  roles:
+    - haproxy
+  tags: haproxy
+
 - hosts: database
   roles:
     - database
-  tags: database
+  tags:
+    - database
 
 - hosts: message-broker
   roles:
diff --git a/docker/centos/binary/haproxy/Dockerfile b/docker/centos/binary/haproxy/Dockerfile
index 40f2af6105..8e1a05f18a 100755
--- a/docker/centos/binary/haproxy/Dockerfile
+++ b/docker/centos/binary/haproxy/Dockerfile
@@ -6,6 +6,6 @@ RUN yum -y install \
     && yum clean all
 
 COPY start.sh /
-COPY config-external.sh /opt/kolla/
+COPY config-external.sh ensure_latest_config.sh /opt/kolla/
 
 CMD ["/start.sh"]
diff --git a/docker/centos/binary/haproxy/ensure_latest_config.sh b/docker/centos/binary/haproxy/ensure_latest_config.sh
new file mode 120000
index 0000000000..80c50f32a0
--- /dev/null
+++ b/docker/centos/binary/haproxy/ensure_latest_config.sh
@@ -0,0 +1 @@
+../../../common/haproxy/ensure_latest_config.sh
\ No newline at end of file
diff --git a/docker/common/haproxy/config-external.sh b/docker/common/haproxy/config-external.sh
index d267eb195f..0c41a5bdbb 100755
--- a/docker/common/haproxy/config-external.sh
+++ b/docker/common/haproxy/config-external.sh
@@ -4,8 +4,7 @@ TARGET="/etc/haproxy/haproxy.cfg"
 OWNER="root"
 
 if [[ -f "$SOURCE" ]]; then
-    rm $TARGET
-    cp $SOURCE $TARGET
+    cp -f $SOURCE $TARGET
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
diff --git a/docker/common/haproxy/ensure_latest_config.sh b/docker/common/haproxy/ensure_latest_config.sh
new file mode 100755
index 0000000000..b91189ca57
--- /dev/null
+++ b/docker/common/haproxy/ensure_latest_config.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+CURRENT_CONFIG_HASH=$(sha1sum /etc/haproxy/haproxy.cfg | cut -f1 -d' ')
+NEW_CONFIG_HASH=$(sha1sum /opt/kolla/haproxy/haproxy.cfg | cut -f1 -d' ')
+
+if [[ $CURRENT_CONFIG_HASH != $NEW_CONFIG_HASH ]]; then
+    changed=changed
+    source /opt/kolla/config-external.sh
+    /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -sf $(cat /run/haproxy.pid)
+fi
+
+echo $changed
diff --git a/docker/common/haproxy/start.sh b/docker/common/haproxy/start.sh
index 68600f44ee..c391847677 100755
--- a/docker/common/haproxy/start.sh
+++ b/docker/common/haproxy/start.sh
@@ -2,11 +2,7 @@
 set -o errexit
 
 CMD='/usr/sbin/haproxy'
-# Parameters:
-# -db for non-daemon execution and logging to stdout
-# -p pidfile to specify pidfile and allow hot reconfiguration
-# loop which generates -f file.conf for each file in /etc/haproxy and /etc/haproxy/conf.d
-ARGS="-db -f /etc/haproxy/haproxy.cfg"
+ARGS="-f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid"
 
 # Loading common functions.
 source /opt/kolla/kolla-common.sh
@@ -14,4 +10,11 @@ source /opt/kolla/kolla-common.sh
 # Config-internal script exec out of this function, it does not return here.
 set_configs
 
-exec $CMD $ARGS
+# We are intentionally not using exec so we can reload the haproxy config later
+$CMD $ARGS
+
+# TODO(SamYaple): This has the potential for a race condition triggered by a
+#                 config reload that could cause the container to exit
+while [[ -e "/proc/$(cat /run/haproxy.pid)" ]]; do
+    sleep 5
+done