Allow building configdrive from JSON in the API

Extend the API with the ability to build config drives from meta_data,
network_data and user_data, where meta_data and network_data are JSON
objects, and user_data is either a JSON object, a JSON array or
raw contents as a string.

This change uses openstacksdk (which is already an indirect dependency)
for building config drives.

Change-Id: Ie1f399a4cb6d4fe5afec79341d3bccc0f81204b2
Story: #2005083
Task: #29663

api-ref/source/baremetal-api-v1-node-management.inc

@@ -352,6 +352,10 @@ detailed documentation of the Ironic State Machine is available
   A node can be rescued or unrescued by setting the node's provision target state to
   ``rescue`` or ``unrescue`` respectively.
 
+.. versionadded:: 1.56
+   A ``configdrive`` can be a JSON object with ``meta_data``, ``network_data``
+   and ``user_data``.
+
 Normal response code: 202
 
 Error codes:

api-ref/source/parameters.yaml

@@ -545,12 +545,20 @@ conductor_group:
   type: string
 configdrive:
   description: |
-    A gzip'ed and base-64 encoded config drive, to be written to a partition
+    A config drive to be written to a partition on the Node's boot disk. Can be
-    on the Node's boot disk. This parameter is only accepted when setting the
+    a full gzip'ed and base-64 encoded image or a JSON object with the keys:
-    state to "active" or "rebuild".
+
+    * ``meta_data`` (optional) - JSON object with the standard meta data.
+      Ironic will provide the defaults for the ``uuid`` and ``name`` fields.
+    * ``network_data`` (optional) - JSON object with networking configuration.
+    * ``user_data`` (optional) - user data. May be a string (which will be
+      UTF-8 encoded); a JSON object, or a JSON array.
+
+    This parameter is only accepted when setting the state to "active" or
+    "rebuild".
   in: body
   required: false
-  type: string or gzip+b64 blob
+  type: string or object
 console_enabled:
   description: |
     Indicates whether console access is enabled or disabled on this node.

ironic/api/controllers/v1/node.py

@@ -600,7 +600,7 @@ class NodeStatesController(rest.RestController):
 
     @METRICS.timer('NodeStatesController.provision')
     @expose.expose(None, types.uuid_or_name, wtypes.text,
-                   wtypes.text, types.jsontype, wtypes.text,
+                   types.jsontype, types.jsontype, wtypes.text,
                    status_code=http_client.ACCEPTED)
     def provision(self, node_ident, target, configdrive=None,
                   clean_steps=None, rescue_password=None):
@@ -616,8 +616,8 @@ class NodeStatesController(rest.RestController):
         :param node_ident: UUID or logical name of a node.
         :param target: The desired provision state of the node or verb.
         :param configdrive: Optional. A gzipped and base64 encoded
-            configdrive. Only valid when setting provision state
+            configdrive or a dict to build a configdrive from. Only valid when
-            to "active" or "rebuild".
+            setting provision state to "active" or "rebuild".
         :param clean_steps: An ordered list of cleaning steps that will be
             performed on the node. A cleaning step is a dictionary with
             required keys 'interface' and 'step', and optional key 'args'. If
@@ -681,8 +681,7 @@ class NodeStatesController(rest.RestController):
                 action=target, node=rpc_node.uuid,
                 state=rpc_node.provision_state)
 
-        if configdrive:
+        api_utils.check_allow_configdrive(target, configdrive)
-            api_utils.check_allow_configdrive(target)
 
         if clean_steps and target != ir_states.VERBS['clean']:
             msg = (_('"clean_steps" is only valid when setting target '

ironic/api/controllers/v1/utils.py

@@ -17,6 +17,8 @@ import inspect
 import re
 
 import jsonpatch
+import jsonschema
+from jsonschema import exceptions as json_schema_exc
 import os_traits
 from oslo_config import cfg
 from oslo_utils import uuidutils
@@ -586,7 +588,30 @@ def check_allow_driver_detail(detail):
              'opr': versions.MINOR_30_DYNAMIC_DRIVERS})
 
 
-def check_allow_configdrive(target):
+_CONFIG_DRIVE_SCHEMA = {
+    'anyOf': [
+        {
+            'type': 'object',
+            'properties': {
+                'meta_data': {'type': 'object'},
+                'network_data': {'type': 'object'},
+                'user_data': {
+                    'type': ['object', 'array', 'string', 'null']
+                }
+            },
+            'additionalProperties': False
+        },
+        {
+            'type': ['string', 'null']
+        }
+    ]
+}
+
+
+def check_allow_configdrive(target, configdrive=None):
+    if not configdrive:
+        return
+
     allowed_targets = [states.ACTIVE]
     if allow_node_rebuild_with_configdrive():
         allowed_targets.append(states.REBUILD)
@@ -597,6 +622,21 @@ def check_allow_configdrive(target):
         raise wsme.exc.ClientSideError(
             msg, status_code=http_client.BAD_REQUEST)
 
+    try:
+        jsonschema.validate(configdrive, _CONFIG_DRIVE_SCHEMA)
+    except json_schema_exc.ValidationError as e:
+        msg = _('Invalid configdrive format: %s') % e
+        raise wsme.exc.ClientSideError(
+            msg, status_code=http_client.BAD_REQUEST)
+
+    if isinstance(configdrive, dict) and not allow_build_configdrive():
+        msg = _('Providing a JSON object for configdrive is only supported'
+                ' starting with API version %(base)s.%(opr)s') % {
+                    'base': versions.BASE_VERSION,
+                    'opr': versions.MINOR_56_BUILD_CONFIGDRIVE}
+        raise wsme.exc.ClientSideError(
+            msg, status_code=http_client.BAD_REQUEST)
+
 
 def check_allow_filter_by_fault(fault):
     """Check if filtering nodes by fault is allowed.
@@ -1094,3 +1134,11 @@ def check_policy(policy_name):
     """
     cdict = pecan.request.context.to_policy_values()
     policy.authorize(policy_name, cdict, cdict)
+
+
+def allow_build_configdrive():
+    """Check if building configdrive is allowed.
+
+    Version 1.56 of the API added support for building configdrive.
+    """
+    return pecan.request.version.minor >= versions.MINOR_56_BUILD_CONFIGDRIVE

ironic/api/controllers/v1/versions.py

@@ -92,6 +92,8 @@ BASE_VERSION = 1
 # v1.52: Add allocation API.
 # v1.53: Add support for Smart NIC port
 # v1.54: Add events support.
+# v1.55: Add deploy templates API.
+# v1.56: Add support for building configdrives.
 
 MINOR_0_JUNO = 0
 MINOR_1_INITIAL_VERSION = 1
@@ -149,6 +151,7 @@ MINOR_52_ALLOCATION = 52
 MINOR_53_PORT_SMARTNIC = 53
 MINOR_54_EVENTS = 54
 MINOR_55_DEPLOY_TEMPLATES = 55
+MINOR_56_BUILD_CONFIGDRIVE = 56
 
 # When adding another version, update:
 # - MINOR_MAX_VERSION
@@ -156,7 +159,7 @@ MINOR_55_DEPLOY_TEMPLATES = 55
 #   explanation of what changed in the new version
 # - common/release_mappings.py, RELEASE_MAPPING['master']['api']
 
-MINOR_MAX_VERSION = MINOR_55_DEPLOY_TEMPLATES
+MINOR_MAX_VERSION = MINOR_56_BUILD_CONFIGDRIVE
 
 # String representations of the minor and maximum versions
 _MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_1_INITIAL_VERSION)

ironic/common/release_mappings.py

@@ -131,7 +131,7 @@ RELEASE_MAPPING = {
         }
     },
     'master': {
-        'api': '1.55',
+        'api': '1.56',
         'rpc': '1.48',
         'objects': {
             'Allocation': ['1.0'],

ironic/conductor/manager.py

@@ -3623,6 +3623,8 @@ def do_node_deploy(task, conductor_id=None, configdrive=None):
 
     try:
         if configdrive:
+            if isinstance(configdrive, dict):
+                configdrive = utils.build_configdrive(node, configdrive)
             _store_configdrive(node, configdrive)
     except (exception.SwiftOperationError, exception.ConfigInvalid) as e:
         with excutils.save_and_reraise_exception():

ironic/conductor/utils.py

@@ -15,8 +15,10 @@
 import collections
 import time
 
+from openstack.baremetal import configdrive as os_configdrive
 from oslo_config import cfg
 from oslo_log import log
+from oslo_serialization import jsonutils
 from oslo_service import loopingcall
 from oslo_utils import excutils
 import six
@@ -1316,3 +1318,30 @@ def validate_deploy_templates(task):
     # Gather deploy steps from matching deploy templates, validate them.
     user_steps = _get_steps_from_deployment_templates(task)
     _validate_user_deploy_steps(task, user_steps)
+
+
+def build_configdrive(node, configdrive):
+    """Build a configdrive from provided meta_data, network_data and user_data.
+
+    If uuid or name are not provided in the meta_data, they're defauled to the
+    node's uuid and name accordingly.
+
+    :param node: an Ironic node object.
+    :param configdrive: A configdrive as a dict with keys ``meta_data``,
+        ``network_data`` and ``user_data`` (all optional).
+    :returns: A gzipped and base64 encoded configdrive as a string.
+    """
+    meta_data = configdrive.setdefault('meta_data', {})
+    meta_data.setdefault('uuid', node.uuid)
+    if node.name:
+        meta_data.setdefault('name', node.name)
+
+    user_data = configdrive.get('user_data')
+    if isinstance(user_data, (dict, list)):
+        user_data = jsonutils.dump_as_bytes(user_data)
+    elif user_data:
+        user_data = user_data.encode('utf-8')
+
+    LOG.debug('Building a configdrive for node %s', node.uuid)
+    return os_configdrive.build(meta_data, user_data=user_data,
+                                network_data=configdrive.get('network_data'))

ironic/tests/unit/api/controllers/v1/test_node.py

@@ -4121,6 +4121,42 @@ class TestPut(test_api_base.BaseApiTest):
         self.assertEqual(urlparse.urlparse(ret.location).path,
                          expected_location)
 
+    def test_provision_with_deploy_configdrive_as_dict(self):
+        ret = self.put_json('/nodes/%s/states/provision' % self.node.uuid,
+                            {'target': states.ACTIVE,
+                             'configdrive': {'user_data': 'foo'}},
+                            headers={api_base.Version.string: '1.56'})
+        self.assertEqual(http_client.ACCEPTED, ret.status_code)
+        self.assertEqual(b'', ret.body)
+        self.mock_dnd.assert_called_once_with(context=mock.ANY,
+                                              node_id=self.node.uuid,
+                                              rebuild=False,
+                                              configdrive={'user_data': 'foo'},
+                                              topic='test-topic')
+
+    def test_provision_with_deploy_configdrive_as_dict_all_fields(self):
+        fake_cd = {'user_data': {'serialize': 'me'},
+                   'meta_data': {'hostname': 'example.com'},
+                   'network_data': {'links': []}}
+        ret = self.put_json('/nodes/%s/states/provision' % self.node.uuid,
+                            {'target': states.ACTIVE,
+                             'configdrive': fake_cd},
+                            headers={api_base.Version.string: '1.56'})
+        self.assertEqual(http_client.ACCEPTED, ret.status_code)
+        self.assertEqual(b'', ret.body)
+        self.mock_dnd.assert_called_once_with(context=mock.ANY,
+                                              node_id=self.node.uuid,
+                                              rebuild=False,
+                                              configdrive=fake_cd,
+                                              topic='test-topic')
+
+    def test_provision_with_deploy_configdrive_as_dict_unsupported(self):
+        ret = self.put_json('/nodes/%s/states/provision' % self.node.uuid,
+                            {'target': states.ACTIVE,
+                             'configdrive': {'user_data': 'foo'}},
+                            expect_errors=True)
+        self.assertEqual(http_client.BAD_REQUEST, ret.status_code)
+
     def test_provision_with_rebuild(self):
         node = self.node
         node.provision_state = states.ACTIVE

ironic/tests/unit/api/controllers/v1/test_utils.py

@@ -501,18 +501,48 @@ class TestApiUtils(base.TestCase):
     def test_check_allow_configdrive_fails(self, mock_request):
         mock_request.version.minor = 35
         self.assertRaises(wsme.exc.ClientSideError,
-                          utils.check_allow_configdrive, states.DELETED)
+                          utils.check_allow_configdrive, states.DELETED,
+                          "abcd")
+        self.assertRaises(wsme.exc.ClientSideError,
+                          utils.check_allow_configdrive, states.ACTIVE,
+                          {'meta_data': {}})
         mock_request.version.minor = 34
         self.assertRaises(wsme.exc.ClientSideError,
-                          utils.check_allow_configdrive, states.REBUILD)
+                          utils.check_allow_configdrive, states.REBUILD,
+                          "abcd")
 
     @mock.patch.object(pecan, 'request', spec_set=['version'])
     def test_check_allow_configdrive(self, mock_request):
         mock_request.version.minor = 35
-        utils.check_allow_configdrive(states.ACTIVE)
+        utils.check_allow_configdrive(states.ACTIVE, "abcd")
-        utils.check_allow_configdrive(states.REBUILD)
+        utils.check_allow_configdrive(states.REBUILD, "abcd")
         mock_request.version.minor = 34
-        utils.check_allow_configdrive(states.ACTIVE)
+        utils.check_allow_configdrive(states.ACTIVE, "abcd")
+
+    @mock.patch.object(pecan, 'request', spec_set=['version'])
+    def test_check_allow_configdrive_as_dict(self, mock_request):
+        mock_request.version.minor = 56
+        utils.check_allow_configdrive(states.ACTIVE, {'meta_data': {}})
+        utils.check_allow_configdrive(states.ACTIVE, {'meta_data': {},
+                                                      'network_data': {},
+                                                      'user_data': {}})
+        utils.check_allow_configdrive(states.ACTIVE, {'user_data': 'foo'})
+        utils.check_allow_configdrive(states.ACTIVE, {'user_data': ['foo']})
+
+    @mock.patch.object(pecan, 'request', spec_set=['version'])
+    def test_check_allow_configdrive_as_dict_invalid(self, mock_request):
+        mock_request.version.minor = 56
+        self.assertRaises(wsme.exc.ClientSideError,
+                          utils.check_allow_configdrive, states.REBUILD,
+                          {'foo': 'bar'})
+        for key in ['meta_data', 'network_data']:
+            self.assertRaises(wsme.exc.ClientSideError,
+                              utils.check_allow_configdrive, states.REBUILD,
+                              {key: 'a string'})
+        for key in ['meta_data', 'network_data', 'user_data']:
+            self.assertRaises(wsme.exc.ClientSideError,
+                              utils.check_allow_configdrive, states.REBUILD,
+                              {key: 42})
 
     @mock.patch.object(pecan, 'request', spec_set=['version'])
     def test_allow_rescue_interface(self, mock_request):

ironic/tests/unit/conductor/test_manager.py

@@ -2035,26 +2035,29 @@ class DoNodeDeployTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
         mock_store.assert_called_once_with(task.node, configdrive)
 
     @mock.patch.object(manager, '_store_configdrive')
-    def _test__do_node_deploy_ok(self, mock_store, configdrive=None):
+    def _test__do_node_deploy_ok(self, mock_store, configdrive=None,
+                                 expected_configdrive=None):
+        expected_configdrive = expected_configdrive or configdrive
         self._start_service()
         with mock.patch.object(fake.FakeDeploy,
                                'deploy', autospec=True) as mock_deploy:
             mock_deploy.return_value = None
-            node = obj_utils.create_test_node(
+            self.node = obj_utils.create_test_node(
-                self.context, driver='fake-hardware',
+                self.context, driver='fake-hardware', name=None,
                 provision_state=states.DEPLOYING,
                 target_provision_state=states.ACTIVE)
-            task = task_manager.TaskManager(self.context, node.uuid)
+            task = task_manager.TaskManager(self.context, self.node.uuid)
 
             manager.do_node_deploy(task, self.service.conductor.id,
                                    configdrive=configdrive)
-            node.refresh()
+            self.node.refresh()
-            self.assertEqual(states.ACTIVE, node.provision_state)
+            self.assertEqual(states.ACTIVE, self.node.provision_state)
-            self.assertEqual(states.NOSTATE, node.target_provision_state)
+            self.assertEqual(states.NOSTATE, self.node.target_provision_state)
-            self.assertIsNone(node.last_error)
+            self.assertIsNone(self.node.last_error)
             mock_deploy.assert_called_once_with(mock.ANY, mock.ANY)
             if configdrive:
-                mock_store.assert_called_once_with(task.node, configdrive)
+                mock_store.assert_called_once_with(task.node,
+                                                   expected_configdrive)
             else:
                 self.assertFalse(mock_store.called)
 
@@ -2065,6 +2068,48 @@ class DoNodeDeployTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
         configdrive = 'foo'
         self._test__do_node_deploy_ok(configdrive=configdrive)
 
+    @mock.patch('openstack.baremetal.configdrive.build', autospec=True)
+    def test__do_node_deploy_configdrive_as_dict(self, mock_cd):
+        mock_cd.return_value = 'foo'
+        configdrive = {'user_data': 'abcd'}
+        self._test__do_node_deploy_ok(configdrive=configdrive,
+                                      expected_configdrive='foo')
+        mock_cd.assert_called_once_with({'uuid': self.node.uuid},
+                                        network_data=None,
+                                        user_data=b'abcd')
+
+    @mock.patch('openstack.baremetal.configdrive.build', autospec=True)
+    def test__do_node_deploy_configdrive_as_dict_with_meta_data(self, mock_cd):
+        mock_cd.return_value = 'foo'
+        configdrive = {'meta_data': {'uuid': uuidutils.generate_uuid(),
+                                     'name': 'new-name',
+                                     'hostname': 'example.com'}}
+        self._test__do_node_deploy_ok(configdrive=configdrive,
+                                      expected_configdrive='foo')
+        mock_cd.assert_called_once_with(configdrive['meta_data'],
+                                        network_data=None,
+                                        user_data=None)
+
+    @mock.patch('openstack.baremetal.configdrive.build', autospec=True)
+    def test__do_node_deploy_configdrive_with_network_data(self, mock_cd):
+        mock_cd.return_value = 'foo'
+        configdrive = {'network_data': {'links': []}}
+        self._test__do_node_deploy_ok(configdrive=configdrive,
+                                      expected_configdrive='foo')
+        mock_cd.assert_called_once_with({'uuid': self.node.uuid},
+                                        network_data={'links': []},
+                                        user_data=None)
+
+    @mock.patch('openstack.baremetal.configdrive.build', autospec=True)
+    def test__do_node_deploy_configdrive_and_user_data_as_dict(self, mock_cd):
+        mock_cd.return_value = 'foo'
+        configdrive = {'user_data': {'user': 'data'}}
+        self._test__do_node_deploy_ok(configdrive=configdrive,
+                                      expected_configdrive='foo')
+        mock_cd.assert_called_once_with({'uuid': self.node.uuid},
+                                        network_data=None,
+                                        user_data=b'{"user": "data"}')
+
     @mock.patch.object(swift, 'SwiftAPI')
     @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare')
     def test__do_node_deploy_configdrive_swift_error(self, mock_prepare,

lower-constraints.txt

@@ -61,7 +61,7 @@ munch==2.2.0
 netaddr==0.7.19
 netifaces==0.10.6
 openstackdocstheme==1.18.1
-openstacksdk==0.12.0
+openstacksdk==0.25.0
 os-api-ref==1.4.0
 os-client-config==1.29.0
 os-service-types==1.2.0

releasenotes/notes/build-configdrive-5b3b9095824faf4e.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Adds support for building config drives. Starting with API version 1.56,
+    the ``configdrive`` parameter of ``/v1/nodes/<node>/states/provision`` can
+    be a JSON object with optional keys ``meta_data`` (JSON object),
+    ``network_data`` (JSON object) and ``user_data`` (JSON object, array or
+    string). See `story 2005083
+    <https://storyboard.openstack.org/#!/story/2005083>`_ for more details.

requirements.txt

@@ -47,3 +47,4 @@ jsonschema<3.0.0,>=2.6.0 # MIT
 psutil>=3.2.2 # BSD
 futurist>=1.2.0 # Apache-2.0
 tooz>=1.58.0 # Apache-2.0
+openstacksdk>=0.25.0 # Apache-2.0