opendev/system-config
Code Issues Proposed changes

Add Zuul load balancer

Browse Source 
This adds a load balancer for zuul-web and fingergw.

Change-Id: Id5aa01151f64f3c85e1532ad66999ef9471c5896
This commit is contained in:
James E. Blair 2022-02-10 09:28:46 -08:00
parent 2c5bc279d6
commit 2a9553ef25
12 changed files with 167 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
inventory/service/group_vars/zuul-lb.yaml Normal file
View File

@ -0,0 +1,23 @@
zuul_lb_listeners:
- name: balance_zuul_http
bind:
- ':::80'
servers:
- name: 'zuul02.opendev.org'
address: '104.130.246.31:80'
- name: balance_zuul_https
bind:
- ':::443'
servers:
- name: 'zuul02.opendev.org'
address: '104.130.246.31:443'
- name: balance_zuul_finger
bind:
- ':::79'
servers:
- name: 'zuul02.opendev.org'
address: '104.130.246.31:79'
iptables_extra_public_tcp_ports:
- 443
- 80
- 79

2
inventory/service/groups.yaml
View File

@ -211,6 +211,8 @@ groups:
- wiki-dev[0-9]*.openstack.org
zookeeper:
- zk[0-9]*.open*.org
zuul-lb:
- zuul-lb[0-9]*.opendev.org
zuul:
- ze[0-9]*.opendev.org
- zm[0-9]*.opendev.org

9
playbooks/roles/zuul-lb/README.rst Normal file
View File

@ -0,0 +1,9 @@
Install the zuul-lb services
This configures haproxy
**Role Variables**
.. zuul:rolevar:: zuul_lb_listeners
The backends to configure

5
playbooks/roles/zuul-lb/tasks/main.yaml Normal file
View File

@ -0,0 +1,5 @@
- name: Install haproxy with zuul config
include_role:
name: haproxy
vars:
haproxy_config_template: zuul-haproxy.cfg.j2

36
playbooks/roles/zuul-lb/templates/zuul-haproxy.cfg.j2 Normal file
View File

@ -0,0 +1,36 @@
global
uid 1000
gid 1000
log /dev/log local0
maxconn 4000
pidfile /var/haproxy/run/haproxy.pid
stats socket /var/haproxy/run/stats uid 1000 gid 1000 mode 0600 level admin
defaults
log-format "%ci:%cp [%t] %ft [%bi]:%bp %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
log global
maxconn 8000
option redispatch
retries 3
stats enable
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 2m
timeout server 2m
timeout check 10s
{% for listener in zuul_lb_listeners %}
listen {{ listener.name }}
{% for bind in listener.bind %}
bind {{ bind }}
{% endfor %}
mode tcp
balance source
option tcp-check
{% for server in listener.servers %}
server {{ server.name }} {{ server.address }} check
{% endfor %}
{% endfor %}

6
playbooks/service-zuul-lb.yaml Normal file
View File

@ -0,0 +1,6 @@
- hosts: "zuul-lb:!disabled"
name: "Base: configure zuul load balancer"
roles:
- iptables
- install-docker
- zuul-lb

1
playbooks/zuul/run-base.yaml
View File

@ -68,6 +68,7 @@
- group_vars/registry.yaml
- group_vars/control-plane-clouds.yaml
- group_vars/afs-client.yaml
- group_vars/zuul-lb.yaml
- group_vars/zuul.yaml
- group_vars/zuul-executor.yaml
- group_vars/zuul-merger.yaml

19
playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2 Normal file
View File

@ -0,0 +1,19 @@
zuul_lb_listeners:
- name: balance_zuul_http
bind:
- ":::80"
servers:
- name: "zuul02.opendev.org"
address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:80"
- name: balance_zuul_https
bind:
- ":::443"
servers:
- name: "zuul02.opendev.org"
address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:443"
- name: balance_zuul_finger
bind:
- ":::79"
servers:
- name: "zuul02.opendev.org"
address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:79"

34
testinfra/test_zuul_lb.py Normal file
View File

@ -0,0 +1,34 @@
# Copyright 2018 Red Hat, Inc.
# Copyright 2022 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
testinfra_hosts = ['zuul-lb01.opendev.org']
def test_zuul_listening(host):
zuul_https = host.socket("tcp://0.0.0.0:443")
assert zuul_https.is_listening
zuul_http = host.socket("tcp://0.0.0.0:80")
assert zuul_http.is_listening
zuul_finger = host.socket("tcp://0.0.0.0:79")
assert zuul_finger.is_listening
def test_haproxy_statsd_running(host):
cmd = host.run("docker inspect haproxy-docker_haproxy-statsd_1")
out = json.loads(cmd.stdout)
assert out[0]["State"]["Status"] == "running"
assert out[0]["RestartCount"] == 0

15
zuul.d/infra-prod.yaml
View File

@ -386,6 +386,21 @@
- roles/kerberos-client/
- roles/openafs-client/
- job:
name: infra-prod-service-zuul-lb
parent: infra-prod-service-base
description: Run service-zuul-lb.yaml playbook.
vars:
playbook_name: service-zuul-lb.yaml
files:
- inventory/base
- playbooks/service-zuul-lb.yaml
- inventory/service/group_vars/zuul-lb.yaml
- playbooks/roles/pip3/
- playbooks/roles/iptables/
- playbooks/roles/install-docker/
- playbooks/roles/haproxy/
- job:
name: infra-prod-service-review
parent: infra-prod-service-base

5
zuul.d/project.yaml
View File

@ -542,6 +542,10 @@
# should reconfigure after any project updates
- name: infra-prod-manage-projects
soft: true
- infra-prod-service-zuul-lb: &infra-prod-service-zuul-lb
dependencies:
- name: system-config-promote-image-haproxy-statsd
soft: true
- infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview
dependencies:
- name: infra-prod-letsencrypt
@ -632,6 +636,7 @@
- infra-prod-service-review: *infra-prod-service-review
- infra-prod-service-zookeeper: *infra-prod-service-zookeeper
- infra-prod-service-zuul: *infra-prod-service-zuul
- infra-prod-service-zuul-lb: *infra-prod-service-zuul-lb
- infra-prod-service-zuul-preview: *infra-prod-service-zuul-preview
- infra-prod-run-accessbot: *infra-prod-run-accessbot
- infra-prod-manage-projects: *infra-prod-manage-projects

15
zuul.d/system-config-run.yaml
View File

@ -858,6 +858,8 @@
label: ubuntu-focal
- name: zuul02.opendev.org
label: ubuntu-focal
- name: zuul-lb01.opendev.org
label: ubuntu-focal
required-projects:
- openstack/project-config
- opendev/system-config
@ -866,6 +868,7 @@
- playbooks/letsencrypt.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-zuul.yaml
- playbooks/service-zuul-lb.yaml
# Test our ad hoc restart playbook works
- playbooks/zuul_restart.yaml
host-vars:
@ -887,19 +890,25 @@
bridge.openstack.org:
host_copy_output:
'/etc/hosts': logs
zuul-lb01.opendev.org:
host_copy_output:
'/var/haproxy/etc': logs
files:
- playbooks/bootstrap-bridge.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-zuul.yaml
- playbooks/service-zuul-lb.yaml
- inventory/service/group_vars/zuul
- inventory/service/group_vars/zuul-lb.yaml
- inventory/service/group_vars/zookeeper.yaml
- inventory/service/host_vars/zk\d+
- inventory/service/host_vars/zuul02.opendev.org
- playbooks/roles/zookeeper/
- playbooks/roles/install-apt-repo
- playbooks/roles/zuul
- playbooks/zuul/templates/group_vars/zuul
- playbooks/roles/install-apt-repo/
- playbooks/roles/zuul.*
- playbooks/zuul/templates/group_vars/zuul.*
- playbooks/zuul/templates/group_vars/zookeeper.yaml
- playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2
- playbooks/zuul/templates/host_vars/zk\d+
- playbooks/zuul/templates/host_vars/zuul02.opendev.org
- playbooks/zuul_restart.yaml