Merge pull request #2 from stackhpc/vbmc

Add Virtual BMC configuration

ansible/action_plugins/tenks_schedule.py

@@ -49,7 +49,7 @@ class ActionModule(ActionBase):
         vms = []
         idx = 0
         for typ, cnt in six.iteritems(task_vars['specs']):
-            for _ in six.range(cnt):
+            for _ in six.moves.range(cnt):
                 vm = deepcopy(task_vars['vm_types'][typ])
                 # Sequentially number the VM and volume names.
                 vm['name'] = "%s%d" % (task_vars['vm_name_prefix'], idx)

ansible/deploy.yml

@@ -5,7 +5,8 @@
 
 - hosts: libvirt
   tasks:
-    - include_role:
+    - name: Configure host for Libvirt
+      include_role:
         name: stackhpc.libvirt-host
       vars:
         libvirt_host_pools:
@@ -18,6 +19,13 @@
             group: "{{ libvirt_pool_group }}"
         libvirt_host_require_vt: "{{ libvirt_require_vt }}"
 
+    - name: Set up Virtual BMC daemon
+      include_role:
+        name: virtualbmc-daemon
+      vars:
+        vbmcd_virtualenv_path: "{{ virtualenv_path }}"
+        vbmcd_python_upper_contraints_url: "{{ python_upper_constraints_url }}"
+
 # Ensure we have facts about all hypervisors before scheduling begins.
 - hosts: hypervisors
   gather_facts: true
@@ -44,10 +52,35 @@
              | default([]) | subelements('physical_networks') }}
 
 - hosts: libvirt
-  tasks:
-    - include_tasks: libvirt_create_vms.yml
   vars:
     # Allocations are stored in the localhost's vars.
-        vms: >-
+    nodes: >-
       {{ hostvars['localhost'].allocations.result[inventory_hostname]
            | default([]) }}
+  tasks:
+    - name: Check that enough ports are available for Virtual BMC
+      fail:
+        msg: >
+          {{ nodes | count }} nodes were specified to be added to Virtual BMC,
+          but only {{ ipmi_port_range_end - ipmi_port_range_start }} ports are
+          available for use by Virtual BMC.
+      when: >-
+          (nodes | count) > (ipmi_port_range_end - ipmi_port_range_start)
+
+    - name: Create Libvirt VMs
+      include_tasks: libvirt_create_vms.yml
+      vars:
+        vms: "{{ nodes }}"
+
+    - name: Register domains with Virtual BMC
+      include_role:
+        name: virtualbmc-domain
+      vars:
+        vbmc_domain: "{{ domain }}"
+        vbmc_ipmi_port: "{{ ipmi_port_range_start + port_offset }}"
+        vbmc_virtualenv_path: "{{ virtualenv_path }}"
+        vbmc_log_directory: "{{ log_directory }}"
+      loop: "{{ nodes | map(attribute='name') | list }}"
+      loop_control:
+        loop_var: domain
+        index_var: port_offset

ansible/group_vars/hypervisors

@@ -10,6 +10,11 @@ system_requirements:
 # not exist at this location, one will be created.
 virtualenv_path: "{{ '/'.join([ansible_env['HOME'], 'tenks-venv']) }}"
 
+# The URL of the upper constraints file to pass to pip when installing Python
+# packages.
+python_upper_constraints_url: >-
+  https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt
+
 # Naming scheme for bridges created by tenks for physical networks is
 # {{ bridge_prefix + i }}, where `i` is the index of the physical network in
 # physnet_mappings (sorted alphabetically by key).
@@ -29,9 +34,5 @@ veth_vm_ovs_suffix: '-ovs'
 # seem right.
 veth_vm_source_suffix: '-tap'
 
-console_log_directory: /var/log/tenks/console_logs/
-
-# The URL of the upper constraints file to pass to pip when installing Python
-# packages.
-python_upper_contraints_url: >-
-  https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt
+# Directory in which to store Tenks logs.
+log_directory: /var/log/tenks/

ansible/group_vars/libvirt

@@ -11,3 +11,7 @@ libvirt_pool_group: "{{ ansible_user_id }}"
 # By default, allow QEMU without hardware virtualisation since this is a
 # development tool.
 libvirt_require_vt: false
+
+# The range of ports available for use for node IPMI traffic.
+ipmi_port_range_start: 6230
+ipmi_port_range_end: 6240

ansible/host_setup.yml

@@ -4,9 +4,9 @@
     name: "{{ system_requirements }}"
   become: true
 
-- name: Ensure console log directory exists
+- name: Ensure log directory exists
   file:
-    path: "{{ console_log_directory }}"
+    path: "{{ log_directory }}"
     state: directory
   become: true
 
@@ -43,11 +43,3 @@
   loop: "{{ physnet_mappings | dictsort }}"
   loop_control:
     index_var: idx
-
-- name: Ensure Python requirements are installed
-  pip:
-    requirements: >-
-      {{ '/'.join([(playbook_dir | dirname), 'venv-requirements.txt']) }}
-    extra_args: >-
-      -c {{ python_upper_contraints_url }}
-    virtualenv: "{{ virtualenv_path }}"

ansible/libvirt_create_vms.yml

@@ -3,7 +3,7 @@
   include_role:
     name: stackhpc.libvirt-vm
   vars:
-    libvirt_vm_default_console_log_dir: "{{ console_log_directory }}"
+    libvirt_vm_default_console_log_dir: "{{ log_directory }}"
     # Configure VM definitions for the Libvirt provider.
     libvirt_vms: >-
       {{ vms | map('set_libvirt_interfaces')

ansible/roles/virtualbmc-daemon/README.md

@@ -0,0 +1,17 @@
+Virtual BMC Daemon
+==================
+
+This role sets up the Virtual BMC daemon in systemd.
+
+Requirements
+------------
+
+- systemd
+
+Role Variables
+--------------
+
+- `vbmcd_virtualenv_path`: The path to the virtualenv in which to install
+  Virtual BMC.
+- `vbmcd_python_upper_constraints_url`: The URL of the upper constraints file
+  to pass to pip when installing Python packages.

ansible/roles/virtualbmc-daemon/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# The path to the virtualenv in which to install Virtual BMC.
+vbmcd_virtualenv_path:
+# The URL of the upper constraints file to pass to pip when installing Python
+# packages.
+vbmcd_python_upper_constraints_url:

ansible/roles/virtualbmc-daemon/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Ensure Python requirements are installed
+  pip:
+    requirements: "{{ '/'.join([role_path, 'files', 'requirements.txt']) }}"
+    extra_args: >-
+      -c {{ vbmcd_python_upper_contraints_url }}
+    virtualenv: "{{ vbmcd_virtualenv_path }}"
+
+- name: Ensure Virtual BMC systemd service is configured
+  template:
+    src: templates/{{ item }}.j2
+    dest: /etc/systemd/system/{{ item }}
+    owner: root
+    group: root
+    mode: 0644
+  become: true
+  register: service_file
+  loop:
+    - "{{ service }}.service"
+
+- name: Ensure Virtual BMC systemd service is started and enabled
+  systemd:
+    name: "{{ service }}"
+    enabled: yes
+    state: started
+    daemon_reload: "{{ service_file.changed }}"
+  become: true

ansible/roles/virtualbmc-daemon/templates/vbmcd.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=Virtual BMC daemon
+
+[Service]
+Type=simple
+Restart=on-failure
+ExecStart="{{ vbmcd_virtualenv_path }}/bin/vbmcd" --foreground

ansible/roles/virtualbmc-daemon/vars/main.yml

@@ -0,0 +1,3 @@
+---
+# The name of the Virtual BMC daemon systemd service.
+service: vbmcd

ansible/roles/virtualbmc-domain/README.md

@@ -0,0 +1,27 @@
+Virtual BMC Domain
+==================
+
+This role ensures a Libvirt domain is added to and started in Virtual BMC.
+
+Requirements
+------------
+
+- Virtual BMC installed in a virtualenv
+- Virtual BMC daemon running
+
+Role Variables
+--------------
+
+- `vbmc_domain`: The name of the Libvirt domain to be added to Virtual BMC.
+- `vbmc_virtualenv_path`: The path to the virtualenv in which Virtual BMC is
+  installed.
+- `vbmc_ipmi_listen_address`: The address on which Virtual BMC will listen for
+  IPMI traffic. Default is 0.0.0.0.
+- `vbmc_ipmi_port`: The port on which Virtual BMC will listen for IPMI traffic.
+  Default is 6230.
+- `vbmc_ipmi_username`: The IPMI username that Virtual BMC will use. Default is
+  'username'.
+- `vbmc_ipmi_password`: The IPMI password that Virtual BMC will use. Default is
+  'password'.
+- `vbmc_log_directory`: The directory in which to store Virtual BMC logs. If
+   `None`, output will not be logged to a file. Default is `None`.

ansible/roles/virtualbmc-domain/defaults/main.yml

@@ -0,0 +1,16 @@
+---
+# The address on which VBMC will listen for IPMI traffic for this domain.
+vbmc_ipmi_listen_address: 0.0.0.0
+# The port on which VBMC will listen for IPMI traffic for this domain.
+vbmc_ipmi_port: 6230
+# The IPMI username that VBMC will use.
+vbmc_ipmi_username: username
+# The IPMI password that VBMC will use.
+vbmc_ipmi_password: password
+
+# The name of the Libvirt domain to be added to Virtual BMC.
+vbmc_domain:
+# The directory in which to store VBMC logs.
+vbmc_log_directory:
+# The path to the virtualenv in which Virtual BMC is installed.
+vbmc_virtualenv_path:

ansible/roles/virtualbmc-domain/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+- name: Set VBMC command string
+  set_fact:
+    # vbmcd should already be running, so --no-daemon stops vbmc from spawning
+    # another instance of the daemon.
+    cmd: >-
+      '{{ vbmc_virtualenv_path }}/bin/vbmc'
+        --no-daemon
+        {% if vbmc_log_directory is not none %}
+        --log-file '{{ vbmc_log_directory }}/vbmc-{{ domain }}.log'
+        {% endif %}
+
+# Even if the VM is present in VBMC, we can't guarantee that it's configured
+# correctly. It's easiest to delete and re-add it; this should involve minimal
+# downtime.
+- name: Ensure VM is stopped and deleted in VBMC
+  command: >-
+    {{ cmd }} {{ item }} '{{ domain }}'
+  loop:
+    - stop
+    - delete
+  register: res
+  changed_when: res.rc == 0
+  failed_when:
+    - res.rc != 0
+    - "'No domain with matching name' not in res.stderr"
+  become: true
+
+# The commands above tend to return before the daemon has completed the action.
+# Check here to be safe.
+- name: Wait to ensure socket is closed
+  wait_for:
+    host: "{{ vbmc_ipmi_listen_address }}"
+    port: "{{ vbmc_ipmi_port }}"
+    state: stopped
+    timeout: 15
+
+- name: Ensure domain is added to VBMC
+  command: >-
+    {{ cmd }} add '{{ domain }}'
+      --port {{ vbmc_ipmi_port }}
+      --username '{{ vbmc_ipmi_username }}'
+      --password '{{ vbmc_ipmi_password }}'
+      --address {{ vbmc_ipmi_listen_address }}
+  become: true
+
+- name: Ensure VM is started in VBMC
+  command: >
+    {{ cmd }} start '{{ domain }}'
+  register: res
+  # Retry a few times in case the VBMC daemon has been slow to process the last
+  # few commands.
+  until: res is succeeded
+  become: true