Improve bare metal compute node management

Adds these new commands:

kayobe baremetal compute inspect
kayobe baremetal compute manage
kayobe baremetal compute provide

These can be used to set the provision state of ironic nodes in the
baremetal-compute group.

.gitignore

@@ -59,6 +59,7 @@ ansible/roles/stackhpc.drac/
 ansible/roles/stackhpc.drac-facts/
 ansible/roles/stackhpc.os-flavors/
 ansible/roles/stackhpc.os-images/
+ansible/roles/stackhpc.os-ironic-state/
 ansible/roles/stackhpc.os-openstackclient/
 ansible/roles/stackhpc.os-networks/
 ansible/roles/stackhpc.os-projects/

ansible/baremetal-compute-inspect.yml

@@ -0,0 +1,49 @@
+---
+# This playbook will ensure that all baremetal compute nodes in the
+# baremetal-compute ansible group are inspected. The nodes should be in the
+# 'manageable' state.
+
+# We install shade in a virtualenv on one of the controllers, and delegate to
+# it when executing the stackhpc.os-ironic-state role.
+
+- name: Ensure dependencies are installed and the virtual environment is activated
+  hosts: controllers[0]
+  gather_facts: False
+  vars:
+    venv: "{{ virtualenv_path }}/shade"
+  roles:
+    - role: stackhpc.os-shade
+      os_shade_venv: "{{ venv }}"
+
+    - role: activate-virtualenv
+      activate_virtualenv_path: "{{ venv }}"
+
+- name: Ensure baremetal compute nodes are inspected in ironic
+  hosts: baremetal-compute
+  gather_facts: False
+  vars:
+    controller_host: "{{ groups['controllers'][0] }}"
+    # Whether to wait for the state transition to complete.
+    baremetal_compute_wait: True
+    # Time to wait for state transition to complete, if baremetal_compute_wait
+    # is True.
+    baremetal_compute_timeout: 1200
+  tasks:
+    - name: Ensure baremetal compute nodes are inspected in ironic
+      os_ironic_inspect:
+        auth_type: "{{ openstack_auth_type }}"
+        auth: "{{ openstack_auth }}"
+        name: "{{ inventory_hostname }}"
+        timeout: "{{ baremetal_compute_timeout }}"
+        wait: "{{ baremetal_compute_wait }}"
+      delegate_to: "{{ controller_host }}"
+      vars:
+        # NOTE: Without this, the controller's ansible_host variable will not
+        # be respected when using delegate_to.
+        ansible_host: "{{ hostvars[controller_host].ansible_host | default(controller_host) }}"
+
+- name: Deactivate the virtual environment on the controller
+  hosts: controllers[0]
+  gather_facts: False
+  roles:
+    - role: deactivate-virtualenv

ansible/baremetal-compute-manage.yml

@@ -0,0 +1,44 @@
+---
+# This playbook will ensure that all baremetal compute nodes in the overcloud
+# ironic inventory are manageable. Supported initial states include 'enroll',
+# 'manageable', and 'available'.
+
+# We install shade in a virtualenv on one of the controllers, and delegate to
+# it when executing the stackhpc.os-ironic-state role.
+
+- name: Ensure baremetal compute nodes are available in ironic
+  hosts: controllers[0]
+  gather_facts: False
+  vars:
+    venv: "{{ virtualenv_path }}/shade"
+  roles:
+    - role: stackhpc.os-shade
+      os_shade_venv: "{{ venv }}"
+
+    - role: activate-virtualenv
+      activate_virtualenv_path: "{{ venv }}"
+
+- name: Ensure baremetal compute nodes are manageable in ironic
+  hosts: baremetal-compute
+  gather_facts: False
+  vars:
+    # Whether to wait for the state transition to complete.
+    baremetal_compute_wait: True
+    # Time to wait for state transition to complete, if baremetal_compute_wait
+    # is True.
+    baremetal_compute_timeout: 1200
+  roles:
+    - role: stackhpc.os-ironic-state
+      os_ironic_state_auth_type: "{{ openstack_auth_type }}"
+      os_ironic_state_auth: "{{ openstack_auth }}"
+      os_ironic_state_name: "{{ inventory_hostname }}"
+      os_ironic_state_provision_state: "manage"
+      os_ironic_state_wait: "{{ baremetal_compute_wait }}"
+      os_ironic_state_timeout: "{{ baremetal_compute_timeout }}"
+      os_ironic_state_delegate_to: "{{ groups['controllers'][0] }}"
+
+- name: Ensure baremetal compute nodes are available in ironic
+  hosts: controllers[0]
+  gather_facts: False
+  roles:
+    - role: deactivate-virtualenv

ansible/baremetal-compute-provide.yml

@@ -0,0 +1,44 @@
+---
+# This playbook will ensure that all baremetal compute nodes in the overcloud
+# ironic inventory are available. Supported initial states include 'enroll' and
+# 'manageable'.
+
+# We install shade in a virtualenv on one of the controllers, and delegate to
+# it when executing the stackhpc.os-ironic-state role.
+
+- name: Ensure baremetal compute nodes are available in ironic
+  hosts: controllers[0]
+  gather_facts: False
+  vars:
+    venv: "{{ virtualenv_path }}/shade"
+  roles:
+    - role: stackhpc.os-shade
+      os_shade_venv: "{{ venv }}"
+
+    - role: activate-virtualenv
+      activate_virtualenv_path: "{{ venv }}"
+
+- name: Ensure baremetal compute nodes are available in ironic
+  hosts: baremetal-compute
+  gather_facts: False
+  vars:
+    # Whether to wait for the state transition to complete.
+    baremetal_compute_wait: True
+    # Time to wait for state transition to complete, if baremetal_compute_wait
+    # is True.
+    baremetal_compute_timeout: 1200
+  roles:
+    - role: stackhpc.os-ironic-state
+      os_ironic_state_auth_type: "{{ openstack_auth_type }}"
+      os_ironic_state_auth: "{{ openstack_auth }}"
+      os_ironic_state_name: "{{ inventory_hostname }}"
+      os_ironic_state_provision_state: "provide"
+      os_ironic_state_wait: "{{ baremetal_compute_wait }}"
+      os_ironic_state_timeout: "{{ baremetal_compute_timeout }}"
+      os_ironic_state_delegate_to: "{{ groups['controllers'][0] }}"
+
+- name: Ensure baremetal compute nodes are available in ironic
+  hosts: controllers[0]
+  gather_facts: False
+  roles:
+    - role: deactivate-virtualenv

ansible/compute-node-provide.yml

@@ -1,88 +0,0 @@
----
-# This playbook will ensure that all baremetal compute nodes in the overcloud
-# ironic inventory are available. Supported initial states include 'enroll' and
-# 'manageable'.
-
-- name: Ensure baremetal compute nodes are available in ironic
-  hosts: controllers[0]
-  vars:
-    venv: "{{ virtualenv_path }}/shade"
-    # Set this to a colon-separated list of baremetal compute node hostnames to
-    # provide.  If unset, all baremetal compute nodes will be provided.
-    compute_node_limit: ""
-    compute_node_limit_list: "{{ compute_node_limit.split(':') }}"
-  roles:
-    - role: stackhpc.os-openstackclient
-      os_openstackclient_venv: "{{ venv }}"
-  tasks:
-    - name: Ensure required Python packages are installed
-      pip:
-        name: "{{ item.name }}"
-        state: present
-        virtualenv: "{{ venv }}"
-      with_items:
-        - name: python-ironicclient
-
-    - name: Get a list of ironic nodes
-      shell: >
-        source {{ venv }}/bin/activate &&
-        openstack baremetal node list --fields name provision_state -f json --no-maintenance
-      register: ironic_node_list
-      changed_when: False
-      environment: "{{ openstack_auth_env }}"
-
-    - name: Initialise a fact containing the ironic nodes
-      set_fact:
-        ironic_nodes: []
-
-    - name: Update a fact containing the ironic nodes
-      set_fact:
-        ironic_nodes: "{{ ironic_nodes + [item] }}"
-      with_items: "{{ ironic_node_list.stdout | from_json }}"
-      when: >
-        {{ not compute_node_limit or
-           item['Name'] in compute_node_limit_list }}
-
-    - name: Ensure ironic nodes are managed
-      shell: >
-        source {{ venv }}/bin/activate &&
-        openstack baremetal node manage {{ item['Name'] }}
-      with_items: "{{ ironic_nodes }}"
-      when: item['Provisioning State'] == 'enroll'
-      environment: "{{ openstack_auth_env }}"
-
-    - name: Ensure ironic nodes are available
-      shell: >
-        source {{ venv }}/bin/activate &&
-        openstack baremetal node provide {{ item['Name'] }}
-      with_items: "{{ ironic_nodes }}"
-      when: item['Provisioning State'] in ['enroll', 'manageable']
-      environment: "{{ openstack_auth_env }}"
-
-    - name: Get a list of ironic nodes
-      shell: >
-        source {{ venv }}/bin/activate &&
-        openstack baremetal node list -f json -c Name -c 'Provisioning State' --no-maintenance
-      register: ironic_node_list
-      changed_when: False
-      environment: "{{ openstack_auth_env }}"
-
-    - name: Initialise a fact containing the ironic nodes
-      set_fact:
-        ironic_nodes: []
-
-    - name: Limit ironic nodes to the specified list
-      set_fact:
-        ironic_nodes: "{{ ironic_nodes + [item] }}"
-      with_items: "{{ ironic_node_list.stdout | from_json }}"
-      when: >
-        {{ not compute_node_limit or
-           item['Name'] in compute_node_limit_list }}
-
-    - name: Fail if any ironic nodes are not available
-      fail:
-        msg: >
-          Failed to make baremetal compute node {{ item['Name'] }} available in ironic.
-          Provisioning state is {{ item['Provisioning State'] }}.
-      with_items: "{{ ironic_nodes }}"
-      when: item['Provisioning State'] != 'available'

ansible/compute-node-provide.yml

@@ -0,0 +1 @@
+baremetal-compute-provide.yml

ansible/roles/activate-virtualenv/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# Path to a virtualenv to activate.
+activate_virtualenv_path:

ansible/roles/activate-virtualenv/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Set a fact containing the current python interpreter
+  set_fact:
+    activate_virtualenv_current_python_interpreter: "{{ ansible_python_interpreter | default('/usr/bin/python') }}"
+
+# Note that setting this via a play or task variable seems to not
+# evaluate the Jinja variable reference, so we use set_fact.
+- name: Update the Ansible python interpreter fact to point to the virtualenv
+  set_fact:
+    ansible_python_interpreter: "{{ activate_virtualenv_path }}/bin/python"

ansible/roles/deactivate-virtualenv/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+# Path to a python interpreter to set as the ansible_python_interpreter
+# variable. The default uses a variable set by the activate-virtualenv role
+# containing the original python interpreter before entering the virtualenv.
+deactivate_virtualenv_python_interpreter: "{{ activate_virtualenv_current_python_interpreter | default('/usr/bin/python') }}"

ansible/roles/deactivate-virtualenv/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+# This variable is unset before we set it, and it does not appear to be
+# possible to unset a variable in Ansible.
+- name: Set a fact to reset the Ansible python interpreter
+  set_fact:
+    ansible_python_interpreter: "{{ deactivate_virtualenv_python_interpreter }}"

doc/source/administration.rst

@@ -125,6 +125,32 @@ any of these hosts are not expected to be active (e.g. prior to overcloud
 deployment), the set of target hosts may be limited using the ``--limit``
 argument.
 
+Baremetal Compute Node Management
+=================================
+
+When enrolling new hardware or performing maintenance, it can be useful to be
+able to manage many bare metal compute nodes simulteneously.
+
+In all cases, commands are delegated to one of the controller hosts, and
+executed concurrently. Note that ansible's ``forks`` configuration option,
+which defaults to 5, may limit the number of nodes configured concurrently.
+
+To move the baremetal compute nodes to the ``manageable`` provision state::
+
+    (kayobe) $ kayobe baremetal compute manage
+
+To move the baremetal compute nodes to the ``available`` provision state::
+
+    (kayobe) $ kayobe baremetal compute provide
+
+To trigger hardware inspection on the baremetal compute nodes::
+
+    (kayobe) $ kayobe baremetal compute inspect
+
+By default these commands wait for the state transition to complete for each
+node. This behavior can be changed by overriding the variable
+``baremetal_compute_wait`` via ``-e baremetal_compute_wait=False``
+
 Running Kayobe Playbooks on Demand
 ==================================
 

doc/source/release-notes.rst

@@ -28,6 +28,9 @@ Features
   be added to the ``[compute]`` group.
 * Adds support for multiple external networks.  ``external_net_names`` should
   be a list of names of networks.
+* Adds commands for management of baremetal compute nodes - ``kayobe baremetal
+  compute inspect``, ``kayobe baremetal compute manage``, and ``kayobe
+  baremetal compute provide``.
 
 Upgrade Notes
 -------------

kayobe/cli/commands.py

@@ -997,3 +997,31 @@ class NetworkConnectivityCheck(KayobeAnsibleMixin, VaultMixin, Command):
         self.app.LOG.debug("Performing network connectivity check")
         playbooks = _build_playbook_list("network-connectivity")
         self.run_kayobe_playbooks(parsed_args, playbooks)
+
+
+class BaremetalComputeInspect(KayobeAnsibleMixin, VaultMixin, Command):
+    """Perform hardware inspection on baremetal compute nodes."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Performing hardware inspection on baremetal "
+                           "compute nodes")
+        playbooks = _build_playbook_list("baremetal-compute-inspect")
+        self.run_kayobe_playbooks(parsed_args, playbooks)
+
+
+class BaremetalComputeManage(KayobeAnsibleMixin, VaultMixin, Command):
+    """Put baremetal compute nodes into the manageable provision state."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Making baremetal compute nodes manageable")
+        playbooks = _build_playbook_list("baremetal-compute-manage")
+        self.run_kayobe_playbooks(parsed_args, playbooks)
+
+
+class BaremetalComputeProvide(KayobeAnsibleMixin, VaultMixin, Command):
+    """Put baremetal compute nodes into the available provision state."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Making baremetal compute nodes available")
+        playbooks = _build_playbook_list("baremetal-compute-provide")
+        self.run_kayobe_playbooks(parsed_args, playbooks)

kayobe/tests/unit/cli/test_commands.py

@@ -179,3 +179,57 @@ class TestCase(unittest.TestCase):
             ),
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    def test_baremetal_compute_inspect(self, mock_run):
+        command = commands.BaremetalComputeInspect(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [
+                    "ansible/baremetal-compute-inspect.yml",
+                ],
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    def test_baremetal_compute_manage(self, mock_run):
+        command = commands.BaremetalComputeManage(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [
+                    "ansible/baremetal-compute-manage.yml",
+                ],
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    def test_baremetal_compute_provide(self, mock_run):
+        command = commands.BaremetalComputeProvide(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [
+                    "ansible/baremetal-compute-provide.yml",
+                ],
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)

requirements.yml

@@ -12,6 +12,7 @@
 - src: stackhpc.drac-facts
 - src: stackhpc.os-flavors
 - src: stackhpc.os-images
+- src: stackhpc.os-ironic-state
 - src: stackhpc.os-networks
 - src: stackhpc.os-openstackclient
 - src: stackhpc.os-projects

setup.py

@@ -51,6 +51,9 @@ setup(
             'kayobe-vault-password-helper = kayobe.cmd.kayobe_vault_password_helper:main',
         ],
         'kayobe.cli': [
+            'baremetal_compute_inspect = kayobe.cli.commands:BaremetalComputeInspect',
+            'baremetal_compute_manage = kayobe.cli.commands:BaremetalComputeManage',
+            'baremetal_compute_provide = kayobe.cli.commands:BaremetalComputeProvide',
             'control_host_bootstrap = kayobe.cli.commands:ControlHostBootstrap',
             'control_host_upgrade = kayobe.cli.commands:ControlHostUpgrade',
             'configuration_dump = kayobe.cli.commands:ConfigurationDump',