Add Zuul load balancer

This adds a load balancer for zuul-web and fingergw.

Change-Id: Id5aa01151f64f3c85e1532ad66999ef9471c5896

inventory/service/group_vars/zuul-lb.yaml

@@ -0,0 +1,23 @@
+zuul_lb_listeners:
+  - name: balance_zuul_http
+    bind:
+      - ':::80'
+    servers:
+      - name: 'zuul02.opendev.org'
+        address: '104.130.246.31:80'
+  - name: balance_zuul_https
+    bind:
+      - ':::443'
+    servers:
+      - name: 'zuul02.opendev.org'
+        address: '104.130.246.31:443'
+  - name: balance_zuul_finger
+    bind:
+      - ':::79'
+    servers:
+      - name: 'zuul02.opendev.org'
+        address: '104.130.246.31:79'
+iptables_extra_public_tcp_ports:
+  - 443
+  - 80
+  - 79

inventory/service/groups.yaml

@@ -211,6 +211,8 @@ groups:
     - wiki-dev[0-9]*.openstack.org
   zookeeper:
     - zk[0-9]*.open*.org
+  zuul-lb:
+    - zuul-lb[0-9]*.opendev.org
   zuul:
     - ze[0-9]*.opendev.org
     - zm[0-9]*.opendev.org

playbooks/roles/zuul-lb/README.rst

@@ -0,0 +1,9 @@
+Install the zuul-lb services
+
+This configures haproxy
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_lb_listeners
+
+   The backends to configure

playbooks/roles/zuul-lb/tasks/main.yaml

@@ -0,0 +1,5 @@
+- name: Install haproxy with zuul config
+  include_role:
+    name: haproxy
+  vars:
+    haproxy_config_template: zuul-haproxy.cfg.j2

playbooks/roles/zuul-lb/templates/zuul-haproxy.cfg.j2

@@ -0,0 +1,36 @@
+global
+  uid  1000
+  gid  1000
+  log  /dev/log local0
+  maxconn  4000
+  pidfile  /var/haproxy/run/haproxy.pid
+  stats  socket /var/haproxy/run/stats uid 1000 gid 1000 mode 0600 level admin
+
+defaults
+  log-format  "%ci:%cp [%t] %ft [%bi]:%bp %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
+  log  global
+  maxconn  8000
+  option  redispatch
+  retries  3
+  stats  enable
+  timeout  http-request 10s
+  timeout  queue 1m
+  timeout  connect 10s
+  timeout  client 2m
+  timeout  server 2m
+  timeout  check 10s
+
+{% for listener in zuul_lb_listeners %}
+listen {{ listener.name }}
+  {% for bind in listener.bind %}
+  bind {{ bind }}
+  {% endfor %}
+  mode tcp
+  balance source
+  option tcp-check
+
+  {% for server in listener.servers %}
+  server {{ server.name }} {{ server.address }} check
+  {% endfor %}
+
+{% endfor %}

playbooks/service-zuul-lb.yaml

@@ -0,0 +1,6 @@
+- hosts: "zuul-lb:!disabled"
+  name: "Base: configure zuul load balancer"
+  roles:
+    - iptables
+    - install-docker
+    - zuul-lb

playbooks/zuul/run-base.yaml

@@ -68,6 +68,7 @@
         - group_vars/registry.yaml
         - group_vars/control-plane-clouds.yaml
         - group_vars/afs-client.yaml
+        - group_vars/zuul-lb.yaml
         - group_vars/zuul.yaml
         - group_vars/zuul-executor.yaml
         - group_vars/zuul-merger.yaml

playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2

@@ -0,0 +1,19 @@
+zuul_lb_listeners:
+  - name: balance_zuul_http
+    bind:
+      - ":::80"
+    servers:
+      - name: "zuul02.opendev.org"
+        address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:80"
+  - name: balance_zuul_https
+    bind:
+      - ":::443"
+    servers:
+      - name: "zuul02.opendev.org"
+        address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:443"
+  - name: balance_zuul_finger
+    bind:
+      - ":::79"
+    servers:
+      - name: "zuul02.opendev.org"
+        address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:79"

testinfra/test_zuul_lb.py

@@ -0,0 +1,34 @@
+# Copyright 2018 Red Hat, Inc.
+# Copyright 2022 Acme Gating, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+
+testinfra_hosts = ['zuul-lb01.opendev.org']
+
+
+def test_zuul_listening(host):
+    zuul_https = host.socket("tcp://0.0.0.0:443")
+    assert zuul_https.is_listening
+    zuul_http = host.socket("tcp://0.0.0.0:80")
+    assert zuul_http.is_listening
+    zuul_finger = host.socket("tcp://0.0.0.0:79")
+    assert zuul_finger.is_listening
+
+def test_haproxy_statsd_running(host):
+    cmd = host.run("docker inspect haproxy-docker_haproxy-statsd_1")
+    out = json.loads(cmd.stdout)
+    assert out[0]["State"]["Status"] == "running"
+    assert out[0]["RestartCount"] == 0

zuul.d/infra-prod.yaml

@@ -386,6 +386,21 @@
       - roles/kerberos-client/
       - roles/openafs-client/
 
+- job:
+    name: infra-prod-service-zuul-lb
+    parent: infra-prod-service-base
+    description: Run service-zuul-lb.yaml playbook.
+    vars:
+      playbook_name: service-zuul-lb.yaml
+    files:
+      - inventory/base
+      - playbooks/service-zuul-lb.yaml
+      - inventory/service/group_vars/zuul-lb.yaml
+      - playbooks/roles/pip3/
+      - playbooks/roles/iptables/
+      - playbooks/roles/install-docker/
+      - playbooks/roles/haproxy/
+
 - job:
     name: infra-prod-service-review
     parent: infra-prod-service-base

zuul.d/project.yaml

@@ -542,6 +542,10 @@
               # should reconfigure after any project updates
               - name: infra-prod-manage-projects
                 soft: true
+        - infra-prod-service-zuul-lb: &infra-prod-service-zuul-lb
+            dependencies:
+              - name: system-config-promote-image-haproxy-statsd
+                soft: true
         - infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview
             dependencies:
               - name: infra-prod-letsencrypt
@@ -632,6 +636,7 @@
         - infra-prod-service-review: *infra-prod-service-review
         - infra-prod-service-zookeeper: *infra-prod-service-zookeeper
         - infra-prod-service-zuul: *infra-prod-service-zuul
+        - infra-prod-service-zuul-lb: *infra-prod-service-zuul-lb
         - infra-prod-service-zuul-preview: *infra-prod-service-zuul-preview
         - infra-prod-run-accessbot: *infra-prod-run-accessbot
         - infra-prod-manage-projects: *infra-prod-manage-projects

zuul.d/system-config-run.yaml

@@ -858,6 +858,8 @@
           label: ubuntu-focal
         - name: zuul02.opendev.org
           label: ubuntu-focal
+        - name: zuul-lb01.opendev.org
+          label: ubuntu-focal
     required-projects:
       - openstack/project-config
       - opendev/system-config
@@ -866,6 +868,7 @@
         - playbooks/letsencrypt.yaml
         - playbooks/service-zookeeper.yaml
         - playbooks/service-zuul.yaml
+        - playbooks/service-zuul-lb.yaml
         # Test our ad hoc restart playbook works
         - playbooks/zuul_restart.yaml
     host-vars:
@@ -887,19 +890,25 @@
       bridge.openstack.org:
         host_copy_output:
           '/etc/hosts': logs
+      zuul-lb01.opendev.org:
+        host_copy_output:
+          '/var/haproxy/etc': logs
     files:
       - playbooks/bootstrap-bridge.yaml
       - playbooks/service-zookeeper.yaml
       - playbooks/service-zuul.yaml
+      - playbooks/service-zuul-lb.yaml
       - inventory/service/group_vars/zuul
+      - inventory/service/group_vars/zuul-lb.yaml
       - inventory/service/group_vars/zookeeper.yaml
       - inventory/service/host_vars/zk\d+
       - inventory/service/host_vars/zuul02.opendev.org
       - playbooks/roles/zookeeper/
-      - playbooks/roles/install-apt-repo
+      - playbooks/roles/install-apt-repo/
-      - playbooks/roles/zuul
+      - playbooks/roles/zuul.*
-      - playbooks/zuul/templates/group_vars/zuul
+      - playbooks/zuul/templates/group_vars/zuul.*
       - playbooks/zuul/templates/group_vars/zookeeper.yaml
+      - playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2
       - playbooks/zuul/templates/host_vars/zk\d+
       - playbooks/zuul/templates/host_vars/zuul02.opendev.org
       - playbooks/zuul_restart.yaml