diff --git a/dev/functions b/dev/functions index 65723c89d..90bfe3c94 100644 --- a/dev/functions +++ b/dev/functions @@ -40,6 +40,15 @@ function config_defaults { # Whether to deploy seed services. export KAYOBE_SEED_SERVICE_DEPLOY=${KAYOBE_SEED_SERVICE_DEPLOY:-1} + # Whether to provision a VM for the infra VM host. + export KAYOBE_INFRA_VM_PROVISION=${KAYOBE_INFRA_VM_PROVISION:-1} + + # Whether to configure the infra VM host. + export KAYOBE_INFRA_VM_HOST_CONFIGURE=${KAYOBE_INFRA_VM_HOST_CONFIGURE:-1} + + # Whether to deploy infra VM services. + export KAYOBE_INFRA_VM_SERVICE_DEPLOY=${KAYOBE_INFRA_VM_SERVICE_DEPLOY:-1} + # Whether to use the 'kolla-ansible certificates' command to generate X.509 # certificates. export KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES=${KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES:-0} @@ -340,6 +349,28 @@ function seed_upgrade { run_kayobe seed service upgrade } +function infra_vm_deploy { + # Deploy a kayobe infra VM. + environment_setup + + control_host_bootstrap + + if [[ ${KAYOBE_INFRA_VM_PROVISION} = 1 ]]; then + echo "Provisioning the infra VM" + run_kayobe infra vm provision + fi + + if [[ ${KAYOBE_INFRA_VM_HOST_CONFIGURE} = 1 ]]; then + echo "Configuring the infra VM host" + run_kayobe infra vm host configure + fi + + if [[ ${KAYOBE_INFRA_VM_SERVICE_DEPLOY} = 1 ]]; then + echo "Deploying containerised infra VM services" + run_kayobe infra vm service deploy + fi +} + function overcloud_deploy { # Deploy a kayobe control plane. echo "Deploying a kayobe development environment. This consists of a " diff --git a/dev/infra-vm-deploy.sh b/dev/infra-vm-deploy.sh new file mode 100755 index 000000000..b1b219b7b --- /dev/null +++ b/dev/infra-vm-deploy.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# Simple script to stand up a development environment for infra VMs using +# kayobe. This should be executed from the hypervisor. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${PARENT}/functions" + + +function main { + config_init + infra_vm_deploy +} + +main diff --git a/playbooks/kayobe-infra-vm-base/infra-vms-group-vars.j2 b/playbooks/kayobe-infra-vm-base/infra-vms-group-vars.j2 new file mode 100644 index 000000000..109707e11 --- /dev/null +++ b/playbooks/kayobe-infra-vm-base/infra-vms-group-vars.j2 @@ -0,0 +1,5 @@ +--- +aio_interface: eth0 + +# Route via the seed-hypervisor to the outside world. +aio_gateway: 192.168.33.4 diff --git a/playbooks/kayobe-infra-vm-base/overrides.yml.j2 b/playbooks/kayobe-infra-vm-base/overrides.yml.j2 new file mode 100644 index 000000000..63c751dcc --- /dev/null +++ b/playbooks/kayobe-infra-vm-base/overrides.yml.j2 @@ -0,0 +1,55 @@ +--- +# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as +# Ansible is run directly on the controller. +disable_selinux_do_reboot: false + +# Use the OpenStack infra's Dockerhub mirror. +docker_registry_mirrors: + - "http://{{ zuul_site_mirror_fqdn }}:8082/" + +kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}" +kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}" +kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}" +kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}" +kolla_openstack_logging_debug: True +pip_upper_constraints_file: "/tmp/upper-constraints.txt" + +# The hosts used by Zuul may or may not have Virtualization Technology (VT) +# enabled. Don't fail if it's disabled. +libvirt_host_require_vt: false + +# Nested virtualisation is not working well in CI currently. Force the use of +# QEMU. +libvirt_vm_engine: "qemu" + +# Use the CI infra's PyPI mirror. +pip_local_mirror: true +pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" +pip_trusted_hosts: + - "{{ zuul_site_mirror_fqdn }}" + +# Try with only a single VCPU, word on the street is that QEMU doesn't play +# nicely with more than one. +infra_vm_vcpus: 1 + +# Reduce the memory footprint of the infra VM. +infra_vm_memory_mb: "{{ 1 * 1024 }}" + +# Use cirros rather than CentOS for the VM. +infra_vm_bootstrap_user: cirros +infra_vm_root_image: /opt/cache/files/cirros-0.5.2-x86_64-disk.img + +# Cirros doesn't load cdom drivers by default. +vm_configdrive_device: disk + +# Cirros is Debian family, but doesn't support path globs in +# /etc/network/interfaces. +configdrive_os_family: Debian +configdrive_debian_network_interfaces_supports_glob: false + +# NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during ifdown, +# and removes the bridge if there are no interfaces left. When Kayobe bounces +# veth links plugged into the bridge, it causes the bridge which has the IP we +# are using for SSH to be removed. Use a dummy interface. +aio_bridge_ports: + - dummy1 diff --git a/playbooks/kayobe-infra-vm-base/pre.yml b/playbooks/kayobe-infra-vm-base/pre.yml new file mode 100644 index 000000000..9a520db8b --- /dev/null +++ b/playbooks/kayobe-infra-vm-base/pre.yml @@ -0,0 +1,57 @@ +--- +- hosts: primary + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" + tasks: + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - + # braio - to exist with an IP address of 192.168.33.4. + - import_role: + name: kayobe-network-bootstrap + vars: + bridge_interface: braio + bridge_ip: 192.168.33.4 + bridge_prefix: 24 + bridge_port_interface: dummy1 + + # NOTE(mgoddard): Configure IP forwarding and NAT to allow communication + # from the infra VM to the outside world. + + # FIXME(mgoddard): use a libvirt network? + - name: Ensure NAT is configured + iptables: + chain: POSTROUTING + table: nat + out_interface: "{{ ansible_default_ipv4.interface }}" + jump: MASQUERADE + become: true + + # FIXME(mgoddard): use a libvirt network? + - name: Ensure IP forwarding is enabled + sysctl: + name: net.ipv4.conf.all.forwarding + value: 1 + become: true + + - name: Ensure SELinux is disabled + selinux: + state: disabled + become: True + when: ansible_os_family == 'RedHat' + + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + + - name: Ensure infra-vms group variables exist + template: + src: infra-vms-group-vars.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/inventory/group_vars/infra-vms/network-interfaces" + + - name: Ensure kayobe is installed + shell: + cmd: dev/install.sh &> {{ logs_dir }}/ansible/install + chdir: "{{ kayobe_src_dir }}" + executable: /bin/bash diff --git a/playbooks/kayobe-infra-vm-base/run.yml b/playbooks/kayobe-infra-vm-base/run.yml new file mode 100644 index 000000000..bc0106793 --- /dev/null +++ b/playbooks/kayobe-infra-vm-base/run.yml @@ -0,0 +1,17 @@ +--- +- hosts: primary + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" + # The CirrOS image does not support much beyond logging in. + KAYOBE_INFRA_VM_HOST_CONFIGURE: 0 + KAYOBE_INFRA_VM_SERVICE_DEPLOY: 0 + tasks: + - name: Ensure seed hypervisor is deployed + shell: + cmd: "{{ kayobe_src_dir }}/dev/seed-hypervisor-deploy.sh &> {{ logs_dir }}/ansible/seed-hypervisor-deploy" + executable: /bin/bash + + - name: Ensure infra VM is deployed + shell: + cmd: "{{ kayobe_src_dir }}/dev/infra-vm-deploy.sh &> {{ logs_dir }}/ansible/infra-vm-deploy" + executable: /bin/bash diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 9eaea0305..f0e6bb2b3 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -229,3 +229,25 @@ name: kayobe-seed-vm-ubuntu-focal parent: kayobe-seed-vm-base nodeset: kayobe-ubuntu-focal + +- job: + name: kayobe-infra-vm-base + parent: kayobe-base + description: | + Base job for testing infra VM provisioning. + + Configures the primary VM as a libvirt hypervisor, and provisions an + infra VM. + pre-run: playbooks/kayobe-infra-vm-base/pre.yml + run: playbooks/kayobe-infra-vm-base/run.yml + timeout: 5400 + +- job: + name: kayobe-infra-vm-centos8s + parent: kayobe-infra-vm-base + nodeset: kayobe-centos8s + +- job: + name: kayobe-infra-vm-ubuntu-focal + parent: kayobe-infra-vm-base + nodeset: kayobe-ubuntu-focal diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 0952a8a50..01b6d4aad 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -23,6 +23,8 @@ - kayobe-seed-upgrade-ubuntu-focal - kayobe-seed-vm-centos8s - kayobe-seed-vm-ubuntu-focal + - kayobe-infra-vm-centos8s + - kayobe-infra-vm-ubuntu-focal gate: queue: kayobe @@ -43,3 +45,5 @@ - kayobe-seed-upgrade-ubuntu-focal - kayobe-seed-vm-centos8s - kayobe-seed-vm-ubuntu-focal + - kayobe-infra-vm-centos8s + - kayobe-infra-vm-ubuntu-focal