From 4ccbb37673944d654df9c5b1130d510d6385ee8e Mon Sep 17 00:00:00 2001 From: "Meadows, Alan (am240k)" Date: Tue, 9 Feb 2021 10:57:35 -0800 Subject: [PATCH] initial vino-builder commit Change-Id: Ie660a061f8912a9e15b4e716f4c08365b809be03 --- vino-builder/Dockerfile | 45 ++++ vino-builder/assets/entrypoint.sh | 29 +++ .../playbooks/roles/libvirt/defaults/main.yml | 1 + .../roles/libvirt/library/core_allocation.py | 155 ++++++++++++++ .../roles/libvirt/tasks/create-domain.yaml | 52 +++++ .../roles/libvirt/tasks/create-network.yaml | 52 +++++ .../roles/libvirt/tasks/create-storage.yaml | 51 +++++ .../playbooks/roles/libvirt/tasks/main.yml | 38 ++++ .../playbooks/sample-vino-ansible-input.yaml | 198 ++++++++++++++++++ .../assets/playbooks/vino-builder.yaml | 46 ++++ 10 files changed, 667 insertions(+) create mode 100644 vino-builder/Dockerfile create mode 100644 vino-builder/assets/entrypoint.sh create mode 100644 vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml create mode 100644 vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py create mode 100644 vino-builder/assets/playbooks/roles/libvirt/tasks/create-domain.yaml create mode 100644 vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml create mode 100644 vino-builder/assets/playbooks/roles/libvirt/tasks/create-storage.yaml create mode 100644 vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml create mode 100644 vino-builder/assets/playbooks/sample-vino-ansible-input.yaml create mode 100644 vino-builder/assets/playbooks/vino-builder.yaml diff --git a/vino-builder/Dockerfile b/vino-builder/Dockerfile new file mode 100644 index 0000000..f3b9d72 --- /dev/null +++ b/vino-builder/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:18.04 + +SHELL ["bash", "-exc"] +ENV DEBIAN_FRONTEND noninteractive + +ARG k8s_version=v1.18.3 +ARG kubectl_url=https://storage.googleapis.com/kubernetes-release/release/"${k8s_version}"/bin/linux/amd64/kubectl + + +# Update distro and install common reqs +RUN apt-get update ;\ + apt-get dist-upgrade -y ;\ + apt-get install -y \ + python3-minimal \ + python3-pip \ + python3-setuptools \ + python3-libvirt \ + libvirt-clients \ + python3-netaddr \ + python3-lxml \ + curl \ + make \ + sudo \ + iproute2 \ + bridge-utils \ + iputils-ping \ + net-tools \ + less \ + jq \ + vim \ + openssh-client ;\ + curl -sSLo /usr/local/bin/kubectl "${kubectl_url}" ;\ + chmod +x /usr/local/bin/kubectl ;\ + pip3 install --upgrade pip ;\ + pip3 install --upgrade wheel ;\ + pip3 install --upgrade ansible ;\ + rm -rf /var/lib/apt/lists/* + +COPY assets /opt/assets/ +RUN cp -ravf /opt/assets/* / ;\ + rm -rf /opt/assets + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT /entrypoint.sh diff --git a/vino-builder/assets/entrypoint.sh b/vino-builder/assets/entrypoint.sh new file mode 100644 index 0000000..f9f69af --- /dev/null +++ b/vino-builder/assets/entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +TIMEOUT=300 +while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + echo "Waiting for libvirt socket at /var/run/libvirt/libvirt-sock" + sleep 1 + else + echo "ERROR: libvirt did not start in time (socket missing) /var/run/libvirt/libvirt-sock" + exit 1 + fi +done + +ansible-playbook -v -e @/var/lib/vino-builder/vino-builder-config.yaml /playbooks/vino-builder.yaml \ No newline at end of file diff --git a/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml b/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml new file mode 100644 index 0000000..d2a4f49 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml @@ -0,0 +1 @@ +libvirt_uri: qemu:///system \ No newline at end of file diff --git a/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py b/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py new file mode 100644 index 0000000..6f6566b --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py @@ -0,0 +1,155 @@ +#!/usr/bin/python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# generate_baremetal_macs method ripped from +# openstack/tripleo-incubator/scripts/configure-vm + +import math +import random +import sys +import fnmatch +import os +from itertools import chain +import json + +DOCUMENTATION = ''' +--- +module: core_allocation +version_added: "1.0" +short_description: Allocate numa aligned cores for libvirt domains and track allocations +description: + - Generate numa aligned cores for libvirt domains and track allocations +''' + +PATH_SYS_DEVICES_NODE = "/sys/devices/system/node" + +def _parse_range(rng): + parts = rng.split('-') + if 1 > len(parts) > 2: + raise ValueError("Bad range: '%s'" % (rng,)) + parts = [int(i) for i in parts] + start = parts[0] + end = start if len(parts) == 1 else parts[1] + if start > end: + end, start = start, end + return range(start, end + 1) + +def _parse_range_list(rngs): + return sorted(set(chain(*[_parse_range(rng) for rng in rngs.split(',')]))) + +def get_numa_cores(): + """Return cores as a dict of numas each with their expanded core lists""" + numa_core_dict = {} + for root, dir, files in os.walk(PATH_SYS_DEVICES_NODE): + for numa in fnmatch.filter(dir, "node*"): + numa_path = os.path.join(PATH_SYS_DEVICES_NODE, numa) + cpulist = os.path.join(numa_path, "cpulist") + with open(cpulist, 'r') as f: + parsed_range_list = _parse_range_list(f.read()) + numa_core_dict[numa] = parsed_range_list + return numa_core_dict + +def allocate_cores(nodes, exclude_cpu): + """Return""" + + core_state = {} + + try: + f = open('/etc/libvirt/vino-cores.json', 'r') + core_state = json.loads(f.read()) + except: + pass + + # instantiate initial inventory - we don't support the inventory + # changing (e.g. adding cores) + if 'inventory' not in core_state: + core_state['inventory'] = get_numa_cores() + + # explode exclude cpu list - we don't support adjusting this after-the-fact + # right now + if 'exclude' not in core_state: + exclude_core_list = _parse_range_list(exclude_cpu) + core_state['exclude'] = exclude_core_list + + # reduce inventory by exclude + if 'available' not in core_state: + core_state['available'] = {} + for numa in core_state['inventory'].keys(): + numa_available = [x for x in core_state['inventory'][numa] if x not in core_state['exclude']] + core_state['available'][numa] = numa_available + + if 'assignments' not in core_state: + core_state['assignments'] = {} + + # walk the nodes, consuming inventory or discovering previous allocations + # address the case where previous != desired - delete previous, re-run + for node in nodes: + + for num_node in range(0, node['count']): + + # generate a unique name such as master-0, master-1 + node_name = node['name'] + '-' + str(num_node) + + # extract the core count + core_count = int(node['instance']['vcpu']) + + # discover any previous allocation + if 'assignments' in core_state: + if node_name in core_state['assignments']: + if len(core_state['assignments'][node_name]) == core_count: + continue + else: + # TODO: support releasing the cores and adding them back + # to available + raise Exception("Existing assignment exists for node %s but does not match current core count needed" % node_name) + + # allocate the cores + allocated=False + for numa in core_state['available']: + if core_count <= len(core_state['available'][numa]): + allocated=True + cores_to_use = core_state['available'][numa][:core_count] + core_state['assignments'][node_name] = cores_to_use + core_state['available'][numa] = core_state['available'][numa][core_count:] + break + else: + continue + if not allocated: + raise Exception("Unable to find sufficient cores (%s) for node %s (available was %r)" % (core_count, node_name, core_state['available'])) + + # return a dict of nodes: cores + # or error if insufficient + with open('/etc/libvirt/vino-cores.json', 'w') as f: + f.write(json.dumps(core_state)) + + return core_state['assignments'] + + +def main(): + module = AnsibleModule( + argument_spec=dict( + nodes=dict(required=True, type='list'), + exclude_cpu=dict(required=True, type='str') + ) + ) + result = allocate_cores(module.params["nodes"], + module.params["exclude_cpu"]) + module.exit_json(**result) + +# see http://docs.ansible.com/developing_modules.html#common-module-boilerplate +from ansible.module_utils.basic import AnsibleModule # noqa + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/create-domain.yaml b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-domain.yaml new file mode 100644 index 0000000..9df7192 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-domain.yaml @@ -0,0 +1,52 @@ +- name: debug print loop + debug: + msg: "outer item={{ node }} inner item={{item}}" + loop: "{{ range(0,node.count)|list }}" + +- name: debug print virsh xml domain + debug: + msg: "{{ libvirtDomains[node.name]['domainTemplate'] }}" + loop: "{{ range(0,node.count)|list }}" + +- name: get state of existing volumes + shell: | + virsh vol-list vino-default + register: vol_list + +- name: write out domain volume request xml + copy: content="{{libvirtDomains[node.name]['volumeTemplate']}}" dest=/tmp/vol-{{item}}.xml + loop: "{{ range(0,node.count)|list }}" + +- name: create domain volume if it doesn't exist + shell: | + virsh vol-create vino-default /tmp/vol-{{item}}.xml + loop: "{{ range(0,node.count)|list }}" + when: "node.name + '-' + item|string not in vol_list.stdout" + +- name: ensure vino instance state directory exists + file: + path: /var/lib/libvirt/vino-instances + state: directory + recurse: yes + owner: root + group: root + +# the virt community plugin does not handle pushing out updates +# to domains, so we must shell out here instead + +- name: write out domain volume request xml + copy: content="{{libvirtDomains[node.name]['domainTemplate']}}" dest=/tmp/domain-{{item}}.xml + loop: "{{ range(0,node.count)|list }}" + +- name: virsh define domain + shell: | + virsh define /tmp/domain-{{item}}.xml + loop: "{{ range(0,node.count)|list }}" + +# - name: set vm to running +# virt: +# name: "{{ node.name + '-' + item|string}}" +# state: running +# # autostart: yes +# loop: "{{ range(0,node.count)|list }}" +# ignore_errors: true diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml new file mode 100644 index 0000000..8614592 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml @@ -0,0 +1,52 @@ +# Facts will be available as 'ansible_libvirt_networks' +- name: initially gather facts on existing virsh networks + virt_net: + command: facts + name: management # this attribute is not needed but required + uri: "{{ libvirt_uri }}" + ignore_errors: true + +- name: Print value of ansible networks + debug: + msg: "Value of ansible_libvirt_networks is {{ ansible_libvirt_networks }}" + +# TODO(alanmeadows): deal with updates as once its defined we will +# never re-define it +- name: add networks defined if they do not already exist + virt_net: + command: define + # looks like setting name here is a redundant, the name is anyways taken from the template xml file, but should set it to make virt_pool module happy. + name: "{{ item.name }}" + xml: "{{ item.libvirtTemplate }}" + uri: "{{ libvirt_uri }}" + when: 'item.name not in ansible_libvirt_networks' + vars: + nodebridgegw: ipam.bridge_ip + +# Re-gather Facts will be available as 'ansible_libvirt_networks' +- name: re-gather facts on existing virsh networks + virt_net: + command: facts + name: management + uri: "{{ libvirt_uri }}" + ignore_errors: true + +- name: start the network + virt_net: + command: create + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" + when: item.name in ansible_libvirt_networks and ansible_libvirt_networks[item.name].state != "active" + +# these are idempotent so require no conditional checks +- name: autostart the network + virt_net: + autostart: yes + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" + +- name: activate the network + virt_net: + state: active + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" \ No newline at end of file diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/create-storage.yaml b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-storage.yaml new file mode 100644 index 0000000..a2581cb --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-storage.yaml @@ -0,0 +1,51 @@ +# Facts will be available as 'ansible_libvirt_pools' +- name: initially gather facts on existing virsh pool + virt_pool: + command: facts + uri: "{{ libvirt_uri }}" + +- name: Print value of ansible storage pools + debug: + msg: "Value of ansible_libvirt_pools is {{ ansible_libvirt_pools }}" + +- name: write out storage xml template + copy: content="{{item.libvirtTemplate}}" dest="/tmp/storage-{{item.name}}.xml" + +- name: define the storage pool + shell: "virsh pool-define /tmp/storage-{{item.name}}.xml" + +# Re-gather facts after definining additional pools available as 'ansible_libvirt_pools' +- name: re-gather facts on existing virsh pools after defining missing pools + virt_pool: + command: facts + uri: "{{ libvirt_uri }}" + +- name: build the storage pool + virt_pool: + command: build + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" + when: item.name in ansible_libvirt_pools and ansible_libvirt_pools[item.name].state != "active" + +- name: start the storage pool + virt_pool: + command: create + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" + when: item.name in ansible_libvirt_pools and ansible_libvirt_pools[item.name].state != "active" + +# these are idempotent so require no conditional checks + +# TODO: we are not actually defining configs on the host +# for some reason we cannot do this +- name: autostart the storage pool + virt_pool: + autostart: yes + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" + +- name: activate the storage pool + virt_pool: + state: active + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" \ No newline at end of file diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml b/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml new file mode 100644 index 0000000..1f47451 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml @@ -0,0 +1,38 @@ +########################################## +# configure storage # +########################################## + +- name: create storage + include_tasks: create-storage.yaml + loop: "{{ libvirtStorage }}" + +########################################## +# configure networks # +########################################## + +- name: create network + include_tasks: create-network.yaml + loop: "{{ libvirtNetworks }}" + +########################################## +# configure domains # +########################################## + +- name: allocate domain cores + core_allocation: + nodes: "{{ nodes }}" + exclude_cpu: "{{ configuration.cpuExclude }}" + register: node_core_map + when: nodes + +- name: debug print node_core_map + debug: + msg: "node_core_map = {{ node_core_map }}" + +- name: define domain outer loop + include_tasks: create-domain.yaml + loop: "{{ nodes }}" + loop_control: + loop_var: node + + diff --git a/vino-builder/assets/playbooks/sample-vino-ansible-input.yaml b/vino-builder/assets/playbooks/sample-vino-ansible-input.yaml new file mode 100644 index 0000000..a73bc71 --- /dev/null +++ b/vino-builder/assets/playbooks/sample-vino-ansible-input.yaml @@ -0,0 +1,198 @@ +configuration: + cpuExclude: 0-1,54-60 + redfishCredentialSecret: + name: redfishSecret + namespace: airship-system +networks: + - name: management + subnet: 192.168.2.0/20 + allocationStart: 192.168.2.10 + allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) + routes: + - to: 10.0.0.0/24 + via: "{{ ipam.bridge_ip | default(omit) }}" # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw` + dns_servers: ["135.188.34.124"] + - name: mobility-gn + subnet: 169.0.0.0/24 + routes: + - to: 0.0.0.0/0 + via: 169.0.0.1 + allocationStart: 169.0.0.10 + allocationStop: 169.0.0.254 +libvirtNetworks: + - name: management + libvirtTemplate: | + + management + + + + + + + + + + +# - name: mobility-gn +# libvirtTemplate: +libvirtStorage: + - name: vino-default + libvirtTemplate: | + + vino-default + + /var/lib/libvirt/vino + + 0711 + 0 + 0 + + + +libvirtDomains: + master: + volumeTemplate: | + {% set nodename = node.name + '-' + item|string %} + + {{ nodename }} + 0 + {{ node.instance.rootSize }} + + domainTemplate: | + {% set nodename = node.name + '-' + item|string %} + + {{ nodename }} + {{ nodename | hash('md5') }} + + {% for flavor in node.labels %} + {% for key in flavor.keys() %} + {% if key == 'vm-flavor' %} + {{ flavor[key] }} + {% endif %} + {% endfor %} + {% endfor %} + {{ ansible_date_time.date }} + + {{ node.instance.memory }} + {% if node.instance.hugepages %} + + + + + {% endif %} + {{ node.instance.vcpu }} + # function to produce list of cpus, in same numa (controled by bool), state will need to be tracked via file on hypervisor host. gotpl psudo: + + 8192 + {% for core in node_core_map[nodename] %} + + {% endfor %} + + + + /machine + + + hvm + + + + + + + + + + + + + destroy + restart + destroy + + /usr/bin/qemu-system-x86_64 + + # for each disk requested + + + + + + + + +
+ + + + + + +
+ + + # for each interface defined in vino, e.g. + + + + +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + +42424:+104 + + + worker-standard: + libvirtTemplate: ... +nodes: + - name: master + labels: + - vm-flavor: master + instance: + memory: 8 + vcpu: 2 + hugepages: true + rootSize: 30 + count: 2 + BMHNetworkTemplate: + name: configMapFooThatsGoTplForNetwork + namespace: foo + field: bmhnetwork + - name: worker-standard + labels: + - vm-flavor: worker-standard + instance: + memory: 8 + vcpu: 2 + hugepages: true + rootSize: 30 + count: 0 + libvirtTemplate: | + foobar + BMHNetworkTemplate: + name: configMapFooThatsGoTplForNetwork + namespace: foo + field: bmhnetwork \ No newline at end of file diff --git a/vino-builder/assets/playbooks/vino-builder.yaml b/vino-builder/assets/playbooks/vino-builder.yaml new file mode 100644 index 0000000..6ba068c --- /dev/null +++ b/vino-builder/assets/playbooks/vino-builder.yaml @@ -0,0 +1,46 @@ + +# Licensed under the Apache License, Version 2.0 (the "License"); + +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + # - host-annotator that populates the k8s node object with approprite annotations + # - report back information such as: + # - vminfra-bridge ip address as label to k8s node + # - sushy-tools ip endpoint for BMC control + # - vino-builder (ansible) that that consumes the `ConfigMap` that contains everything necessary for libvirt to define the virtual machines and networks on the host and does both green-field generation of VM resources and understands if the `ConfigMap` changed and will handle those lifecycle updates. There is no need to stage or coordinate changes to these `ConfigMap` resources as they will result in a no-op `virsh update` which only take effect with a VM stop/start. + # - do the following (assumption is all of this is idempotent for day 2): + # - interogate host + # - prevalidate (is kvm loaded, etc) + # - define host facts (eg cpu list, vf list, etc) + # - interogate existing vms or state recording somewhere + # - collect resources in use + # - what cores are in use + # - what vfs are in use + # - memory in use + # - define libvirt networks + # - define libvirt storage pools + # - ensure appropriate qcows exist + # - define libvirt domains + # - ensure mem/cpu aligned in one numa + # - new domain validation (only on new domains): + # - do a simple domain start/destroy test via redfish. + # - wait for dhcp req on admin interface? + +--- +- hosts: localhost + + tasks: + + # generate libvirt definitions for storage, networks, and domains + - name: process libvirt definitions + include_role: + name: libvirt \ No newline at end of file