Use virtualbmc for virtual BM nodes

This patch adds installation of virtualbmc and plugging VMs into it as
part of 'bifrost-create-vm-nodes' role.

The baremetal data file is now populated with nodes using
"agent_ipmitool" driver.

All things defaulting to some *_ssh drivers are replaced with
corresponding *_ipmitool drivers.

Most libvirt and VM related tasks (including some SSH key management)
and vars are moved from 'bifrost-ironic-install' role to
'bifrost-create-vm-nodes' role, as ironic istelf now should barely care
about libvirt as no *_ssh drivers are used.

Change-Id: If42082e5b1fe8b83a364e27efc549bb74a19ab2f
Closes-Bug: #1659876

bifrost/inventory.py

@@ -241,7 +241,7 @@ def _process_baremetal_csv(data_source, groups, hostvars):
                 host['provisioning_ipv4_address'] = host['ipv4_address']
 
             # Default Driver unless otherwise defined or determined.
-            host['driver'] = "agent_ssh"
+            host['driver'] = "agent_ipmitool"
 
             if len(row) > 15:
                 driver = _val_or_none(row, 16)

bifrost/tests/functional/test_inventory_functional.py

@@ -49,12 +49,13 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
  "x86_64", "disk_size": "1024", "cpus": "2"}}, "hostname0":
- {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ssh",
+ {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ipmitool",
  "name": "hostname0", "ipv4_address": "192.168.1.2",
  "provisioning_ipv4_address": "192.168.1.2", "ansible_ssh_host":
- "192.168.1.2", "driver_info": {"power": {"ssh_virt_type": "virsh",
+ "192.168.1.2", "driver_info": {"power": {"ipmi_address": "192.0.2.2",
- "ssh_key_filename": "/home/ironic/.ssh/id_rsa", "ssh_username":
+ "ipmi_password": "undefined", "ipmi_username": "root",
- "ironic", "ssh_port": 22, "ssh_address": "192.0.2.2"}}, "nics":
+ "ipmi_target_address": null, "ipmi_target_channel": null,
+ "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:05"}], "properties": {"ram": "8192",
  "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"}}}""".replace('\n', '')
         expected_groups = """{"baremetal": {"hosts": ["hostname0",
@@ -170,12 +171,13 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
  "x86_64", "disk_size": "1024", "cpus": "2"}}, "hostname0":
- {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ssh",
+ {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ipmitool",
  "name": "hostname0", "ipv4_address": "192.168.1.2", "ansible_ssh_host":
  "192.168.1.2", "provisioning_ipv4_address": "192.168.1.2",
- "driver_info": {"power": {"ssh_virt_type": "virsh",
+ "driver_info": {"power": {"ipmi_address": "192.0.2.2",
- "ssh_key_filename": "/home/ironic/.ssh/id_rsa", "ssh_username":
+ "ipmi_password": "undefined", "ipmi_username": "root",
- "ironic", "ssh_port": 22, "ssh_address": "192.0.2.2"}}, "nics":
+ "ipmi_target_address": null, "ipmi_target_channel": null,
+ "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:05"}], "properties": {"ram": "8192",
  "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"}}}""".replace('\n', '')
 

playbooks/roles/bifrost-create-vm-nodes/defaults/required_defaults_CentOS.yml

@@ -1,4 +1,5 @@
 ---
+sgabios_dir: /usr/share/sgabios/
 libvirt_service_name: libvirtd
 required_packages:
  - qemu-img
@@ -21,5 +22,7 @@ required_packages:
  - libvirt-daemon-kvm
  - libvirt-python
  - python-lxml
+ - pkgconfig
+ - libvirt-devel
 test_vm_machine: "pc"
 test_vm_emulator: "/usr/libexec/qemu-kvm"

playbooks/roles/bifrost-create-vm-nodes/defaults/required_defaults_Debian.yml

@@ -1,4 +1,5 @@
 ---
+sgabios_dir: /usr/share/qemu/
 libvirt_service_name: libvirt-bin
 required_packages:
  - libvirt-bin
@@ -8,3 +9,5 @@ required_packages:
  - sgabios
  - python-libvirt
  - python-lxml
+ - pkg-config
+ - libvirt-dev

playbooks/roles/bifrost-create-vm-nodes/defaults/required_defaults_RedHat.yml

@@ -1,4 +1,5 @@
 ---
+sgabios_dir: /usr/share/sgabios/
 libvirt_service_name: libvirtd
 required_packages:
  - qemu-img
@@ -6,10 +7,11 @@ required_packages:
  - qemu-kvm
  - qemu-kvm-common
  - qemu-system-x86
- - sgabios
  - sgabios-bin
  - libvirt
  - libvirt-client
  - libvirt-daemon
- - python-libvirt
+ - libvirt-python
  - python-lxml
+ - pkgconfig
+ - libvirt-devel

playbooks/roles/bifrost-create-vm-nodes/defaults/required_defaults_Suse.yml

@@ -1,4 +1,5 @@
 ---
+sgabios_dir: /usr/share/sgabios/
 libvirt_service_name: libvirtd
 required_packages:
  - qemu-tools
@@ -10,3 +11,5 @@ required_packages:
  - libvirt-daemon
  - libvirt-python
  - python-lxml
+ - pkg-config
+ - libvirt-devel

playbooks/roles/bifrost-create-vm-nodes/tasks/create_vm.yml

@@ -88,29 +88,50 @@
         output for additional details"
   when: "{{ vm_name not in existing_vms.list_vms }}"
 
+# TODO(pas-ha) replace 'command: vbmc ...' tasks
+# with a custom Ansible module using vbmc Python API
+- name: get list of nodes from virtualbmc
+  command: vbmc list
+  register: vbmc_list
+
+- name: delete vm from virtualbmc if it is there
+  command: vbmc delete {{ vm_name }}
+  when: "{{ vm_name in vbmc_list.stdout }}"
+
+- set_fact:
+    virtual_ipmi_port: "{{ (test_vm_ipmi_port_start|default(623) | int ) + (testvm_json_data | length) }}"
+
+- name: plug vm into vbmc
+  command: vbmc add {{ vm_name }} --libvirt-uri {{ test_vm_libvirt_uri }} --port {{ virtual_ipmi_port }}
+
+- name: start virtualbmc
+  command: vbmc start {{ vm_name }}
+
 - name: get XML of the vm
   virt:
     name: "{{ vm_name }}"
     command: get_xml
   register: testvm_xml
 
+# NOTE(pas-ha) relies on our XML template for VM that defines a single NIC
 - name: get MAC from vm XML
   set_fact:
     vm_mac: "{{ (testvm_xml.get_xml | regex_findall(\"<mac address='.*'/>\") | first).split('=') | last | regex_replace(\"['/>]\", '') }}"
 
+# NOTE(pas-ha) using default username and password set by virtualbmc - "admin" and "password" respectively
+# see vbmc add --help
 - name: set the json entry for vm
   set_fact:
     testvm_data:
       name: "{{ vm_name }}"
       uuid: "{{ vm_name | to_uuid }}"
-      driver: "agent_ssh"
+      driver: "{{ test_vm_node_driver|default('agent_ipmitool') }}"
       driver_info:
         power:
-          ssh_address: "192.168.122.1"
+          ipmi_address: "192.168.122.1"
-          ssh_port: "22"
+          ipmi_port: "{{ virtual_ipmi_port }}"
-          ssh_username: "ironic"
+          ipmi_username: "admin"
-          ssh_key_filename: "/home/ironic/.ssh/id_rsa"
+          ipmi_password: "password"
-          ssh_virt_type: "virsh"
       nics:
         - mac: "{{ vm_mac }}"
       ansible_ssh_host: "192.168.122.{{ testvm_json_data | length + 2 }}"

playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml

@@ -93,7 +93,7 @@
     test_vm_node_names: "{{ generated_test_vm_node_names }}"
   when: "{{ test_vm_node_names | length == 0 }}"
 
-- name: create placeholder var for vm entries in CSV format
+- name: create placeholder var for vm entries in JSON format
   set_fact:
     testvm_json_data: {}
 

playbooks/roles/bifrost-create-vm-nodes/tasks/prepare_libvirt.yml

@@ -18,6 +18,32 @@
 - name: "Restart libvirt service"
   service: name="{{libvirt_service_name}}" state=restarted
 
+# NOTE(Shrews) We need to enable ip forwarding for the libvirt bridge to
+# operate properly with dnsmasq. This should be done before starting dnsmasq.
+- name: "Enable IP forwarding in sysctl"
+  sysctl:
+    name: "net.ipv4.ip_forward"
+    value: 1
+    sysctl_set: yes
+    state: present
+    reload: yes
+
+# NOTE(Shrews) Ubuntu packaging+apparmor issue prevents libvirt from loading
+# the ROM from /usr/share/misc.
+- name: "Look for sgabios in {{ sgabios_dir }}"
+  stat: path={{ sgabios_dir }}/sgabios.bin
+  register: test_sgabios_qemu
+
+- name: "Look for sgabios in /usr/share/misc"
+  stat: path=/usr/share/misc/sgabios.bin
+  register: test_sgabios_misc
+
+- name: "Place sgabios.bin"
+  command: cp /usr/share/misc/sgabios.bin /usr/share/qemu/sgabios.bin
+  when: >
+    test_sgabios_qemu == false and
+    test_sgabios_misc == true
+
 - name: ensure libvirt network is present
   virt_net:
     name: "{{ test_vm_network }}"
@@ -80,3 +106,7 @@
     state: directory
     path: "{{ test_vm_logdir }}"
     mode: "o+rx"
+
+- name: install virtualbmc
+  pip:
+    name: virtualbmc

playbooks/roles/bifrost-ironic-install/defaults/required_defaults_Debian_family.yml

@@ -5,10 +5,8 @@ init_ext: .conf
 syslinux_tftp_dir: /usr/lib/syslinux/
 ipxe_dir: /usr/lib/ipxe/
 ipxe_full_binary: ipxe.pxe
-sgabios_dir: /usr/share/qemu/
 ironic_rootwrap_dir: /usr/local/bin/
 nginx_user: www-data
-virt_group: libvirtd
 mysql_service_name: mysql
 tftp_service_name: tftpd-hpa
 required_packages:
@@ -36,8 +34,6 @@ required_packages:
   - kpartx
   - qemu-utils
   - python-openwsman
-  - sgabios
-  - libvirt-bin
   - debootstrap
   - uuid-runtime
   - curl

playbooks/roles/bifrost-ironic-install/defaults/required_defaults_Debian_jessie.yml

@@ -5,9 +5,7 @@ init_ext: .service
 syslinux_tftp_dir: /usr/lib/PXELINUX/
 ipxe_dir: /usr/lib/ipxe/
 ipxe_full_binary: ipxe.pxe
-sgabios_dir: /usr/share/qemu/
 nginx_user: www-data
-virt_group: libvirt
 mysql_service_name: mysql
 required_packages:
   - mysql-server
@@ -34,8 +32,6 @@ required_packages:
   - genisoimage
   - kpartx
   - qemu-utils
-  - sgabios
-  - libvirt-bin
   - debootstrap
   - uuid-runtime
   - dnsmasq

playbooks/roles/bifrost-ironic-install/defaults/required_defaults_RedHat_family.yml

@@ -5,10 +5,8 @@ init_ext: .service
 syslinux_tftp_dir: /var/lib/tftpboot
 ipxe_dir: /usr/share/ipxe/
 ipxe_full_binary: ipxe.lkrn
-sgabios_dir: /usr/share/sgabios/
 ironic_rootwrap_dir: /usr/bin/
 nginx_user: nginx
-virt_group: libvirt
 mysql_service_name: mariadb
 tftp_service_name: tftp
 required_packages:
@@ -37,10 +35,8 @@ required_packages:
   - qemu-kvm-tools
   - qemu-img
   - openwsman-python
-  - sgabios-bin
   - libselinux-python
   - policycoreutils-python
-  - libvirt-daemon
   - debootstrap
   - gcc
   - python2-pip

playbooks/roles/bifrost-ironic-install/defaults/required_defaults_Suse_family.yml

@@ -10,10 +10,8 @@ syslinux_tftp_dir: /usr/share/syslinux
 download_ipxe: true
 ipxe_dir: /usr/local/share/ipxe
 ipxe_full_binary: ipxe.lkrn
-sgabios_dir: /usr/share/sgabios/
 ironic_rootwrap_dir: /usr/bin/
 nginx_user: nginx
-virt_group: libvirt
 mysql_service_name: mysql
 tftp_service_name: tftp
 required_packages:
@@ -41,9 +39,7 @@ required_packages:
   - kpartx
   - qemu-tools
   - openwsman-python
-  - qemu-sgabios
   - policycoreutils-python
-  - libvirt-daemon
   - debootstrap
   - iptables
   - tar

playbooks/roles/bifrost-ironic-install/defaults/required_defaults_Ubuntu_16.04.yml

@@ -27,8 +27,6 @@ required_packages:
   - kpartx
   - qemu-utils
   - python-openwsman
-  - sgabios
-  - libvirt-bin
   - debootstrap
   - uuid-runtime
   - curl

playbooks/roles/bifrost-ironic-install/tasks/bootstrap.yml

@@ -155,69 +155,6 @@
   command: ironic-dbsync --config-file /etc/ironic/ironic.conf upgrade
   environment: "{{ bifrost_venv_env if enable_venv else '{}' }}"
   when: test_created_db.changed | bool == false
-- name: "Do RedHat-specific changes for libvirt"
-  include: redhat_libvirt_changes.yml
-  when: ansible_os_family == 'RedHat'
-- name: "Add ironic user to virtualization group"
-  user: name=ironic group="{{ virt_group }}" append=yes
-  when: testing | bool == true
-- name: "Identify ssh_private_key from ssh_private_key_path"
-  include: set_ssh_private_key.yml
-  when: >
-    testing | bool == false and
-    ssh_private_key is undefined and
-    ssh_private_key_path is defined
-- name: "Create SSH directory for ironic user"
-  file:
-    path: "/home/ironic/.ssh"
-    owner: "ironic"
-    group: "ironic"
-    mode: 0700
-    state: directory
-  when: >
-    testing | bool == true or
-    ssh_private_key is defined
-- name: "Set private key file"
-  copy:
-    content: "{{ ssh_private_key }}"
-    dest: /home/ironic/.ssh/id_rsa
-    owner: ironic
-    group: ironic
-    mode: 0600
-  no_log: true
-  when: >
-    testing | bool == false and
-    ssh_private_key is defined
-- name: "Check for ironic user SSH key"
-  stat: path=/home/ironic/.ssh/id_rsa
-  register: test_ironic_pvt_key
-- name: "Generate SSH key for ironic user"
-  command: ssh-keygen -f /home/ironic/.ssh/id_rsa -N ""
-  when: >
-    testing | bool == true and
-    test_ironic_pvt_key.stat.exists | bool == false
-- name: "Set ownership on ironic SSH private key"
-  file:
-    name: /home/ironic/.ssh/id_rsa
-    owner: ironic
-    group: ironic
-    mode: 0600
-    state: file
-  when: >
-    testing | bool == true and
-    test_ironic_pvt_key.stat.exists | bool == false
-- name: "Set ownership on ironic SSH public key"
-  file:
-    name: /home/ironic/.ssh/id_rsa.pub
-    owner: ironic
-    group: ironic
-    mode: 0644
-    state: file
-  when: testing | bool == true and test_ironic_pvt_key.stat.exists | bool == false
-- name: "Create authorized_keys file for ironic user"
-  command: >
-    cp -p /home/ironic/.ssh/id_rsa.pub /home/ironic/.ssh/authorized_keys
-  when: testing | bool == true
 - name: "Create service folder if systemd template is defined"
   file:
     path: "{{ init_dest_dir }}"
@@ -283,30 +220,6 @@
      include_dhcp_server | bool == true and
      test_libvirt_dnsmasq.stat.exists | bool == true and
      testing | bool == true
-# NOTE(Shrews) We need to enable ip forwarding for the libvirt bridge to
-# operate properly with dnsmasq. This should be done before starting dnsmasq.
-- name: "Enable IP forwarding in sysctl"
-  sysctl:
-    name: "net.ipv4.ip_forward"
-    value: 1
-    sysctl_set: yes
-    state: present
-    reload: yes
-  when: testing | bool == true
-# NOTE(Shrews) Ubuntu packaging+apparmor issue prevents libvirt from loading
-# the ROM from /usr/share/misc.
-- name: "Look for sgabios in {{ sgabios_dir }}"
-  stat: path={{ sgabios_dir }}/sgabios.bin
-  register: test_sgabios_qemu
-- name: "Look for sgabios in /usr/share/misc"
-  stat: path=/usr/share/misc/sgabios.bin
-  register: test_sgabios_misc
-- name: "Place sgabios.bin"
-  command: cp /usr/share/misc/sgabios.bin /usr/share/qemu/sgabios.bin
-  when: >
-    test_sgabios_qemu == false and
-    test_sgabios_misc == true and
-    testing | bool == true
 - name: "Deploy nginx configuration file for serving HTTP requests"
   template: src=nginx.conf.j2 dest=/etc/nginx/nginx.conf
 - name: "Download Ironic Python Agent kernel & image"

playbooks/roles/bifrost-ironic-install/tasks/redhat_libvirt_changes.yml

@@ -1,25 +0,0 @@
-# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
----
-- name: "Create libvirt group on RedHat/Centos"
-  group: name=libvirt
-# NOTE(TheJulia): Modify the supplied libvirtd config as by default,
-# access to libvirt is restricted to the root group via polkit.
-- name: "Update libvirt configuration file on RedHat/CentOS so the libvirt group has access"
-  lineinfile: dest=/etc/libvirt/libvirtd.conf regexp=^unix_sock_group line='unix_sock_group = "libvirt"'
-- name: "Change libvirt authentication to none as RedHat/CentOS use polkit by default"
-  lineinfile: dest=/etc/libvirt/libvirtd.conf regexp=^auth_unix_rw line='auth_unix_rw = "none"'
-- name: "Change libvirt socket permissions to be restricted on RedHat/CentOS"
-  lineinfile: dest=/etc/libvirt/libvirtd.conf regexp=^unix_sock_rw_perms line='unix_sock_rw_perms = "0770"'

playbooks/roles/bifrost-ironic-install/templates/ironic.conf.j2

@@ -10,7 +10,7 @@
 # control instead of ironic, noop is the only available network driver.
 enabled_network_interfaces = noop
 {% if testing | bool == true %}
-enabled_drivers = agent_ssh,pxe_ssh
+enabled_drivers = agent_ipmitool,pxe_ipmitool
 debug = true
 {% else %}
 enabled_drivers = {{ enabled_drivers }}

releasenotes/notes/nossh-16577ec5c3b82594.yaml

@@ -0,0 +1,33 @@
+---
+features:
+  - |
+    Bifrost starts to phase out support of SSH power and management drivers
+    in ironic as these drivers are unsupported and targeted for removal
+    from ironic in Pike release.
+
+    Bifrost testing is moved to ``ipmitools``-based ironic drivers and
+    virtual hardware exposed via 'virtualbmc' utility.
+
+    Default ironic drivers set up by bifrost are changed to
+    ``pxe_ipmitool`` and ``agent_ipmitool``.
+
+    Default driver for CSV baremetal data files in bifrost's dynamic inventory
+    is changed to ``agent_ipmitool``.
+
+    ``bifrost-ironic-install`` role is decoupled from libvirt, and
+    all preparation for testing with 'virtual' hardware are now handled in
+    ``bifrost-create-vm-nodes`` role.
+upgrade:
+  - |
+    Default ironic drivers set up by bifrost are changed to
+    ``pxe_ipmitool`` and ``agent_ipmitool``.
+
+  - |
+    Default driver for CSV baremetal data files in bifrost's dynamic inventory
+    is changed to ``agent_ipmitool``.
+    Those bifrost users relying on such behavior must explicitly set the
+    driver in CSV baremetal data file.
+deprecations:
+  - |
+    Handling of ``*_ssh`` drivers for CSV baremetal data format is
+    deprecated and will be removed in Queens release.