Add ./bifrost-cli deploy and refactor bifrost-deploy-nodes-dynamic

A new simplified command is added for deploying nodes, optionally
specifying an image.

The underlying role is updated to allow specifying a full image URL,
a configdrive URL or contents and a full checksum.

Change-Id: I6c99b01dc827c0bd2ef98eff73de4dfbac433fe1

bifrost/cli.py

@@ -191,14 +191,20 @@ def cmd_install(args):
         "See documentation for next steps")
 
 
-def cmd_enroll(args):
+def configure_inventory(args):
     inventory = os.path.join(PLAYBOOKS, 'inventory', 'bifrost_inventory.py')
-    if os.path.exists(args.inventory):
+    if not args.inventory:
+        os.environ['BIFROST_INVENTORY_SOURCE'] = 'ironic'
+    elif os.path.exists(args.inventory):
         nodes_inventory = os.path.abspath(args.inventory)
         os.environ['BIFROST_INVENTORY_SOURCE'] = nodes_inventory
     else:
         sys.exit('Inventory file %s cannot be found' % args.inventory)
+    return inventory
 
+
+def cmd_enroll(args):
+    inventory = configure_inventory(args)
     ansible('enroll-dynamic.yaml',
             inventory=inventory,
             verbose=args.debug,
@@ -206,6 +212,33 @@ def cmd_enroll(args):
             extra_vars=args.extra_vars)
 
 
+def cmd_deploy(args):
+    inventory = configure_inventory(args)
+    try:
+        configdrive = json.loads(args.configdrive)
+    except (ValueError, TypeError):
+        configdrive = args.configdrive
+
+    extra_vars = args.extra_vars or []
+    if configdrive:
+        # Need to preserve JSON
+        extra_vars.append(json.dumps({'deploy_config_drive': configdrive}))
+
+    if (args.image and not args.image.startswith('file://') and not
+            args.image_checksum):
+        raise TypeError('An --image-checksum is required with --image '
+                        'when the image is not a local file')
+
+    ansible('deploy-dynamic.yaml',
+            inventory=inventory,
+            verbose=args.debug,
+            deploy_image_source=args.image,
+            deploy_image_type=args.image_type,
+            deploy_image_checksum=args.image_checksum,
+            wait_for_node_deploy=args.wait,
+            extra_vars=extra_vars)
+
+
 def parse_args():
     parser = argparse.ArgumentParser("Bifrost CLI")
     parser.add_argument('--debug', action='store_true',
@@ -303,6 +336,23 @@ def parse_args():
     enroll.add_argument('-e', '--extra-vars', action='append',
                         help='additional vars to pass to ansible')
 
+    deploy = subparsers.add_parser(
+        'deploy', help='Deploy bare metal nodes')
+    deploy.set_defaults(func=cmd_deploy)
+    deploy.add_argument('inventory', nargs='?',
+                        help='file with the inventory, skip to use Ironic')
+    deploy.add_argument('--image', help='image URL to deploy')
+    deploy.add_argument('--image-checksum',
+                        help='checksum of the image to deploy')
+    deploy.add_argument('--partition', action='store_const',
+                        const='partition', dest='image_type',
+                        help='the image is a partition image')
+    deploy.add_argument('--configdrive', help='URL or JSON with a configdrive')
+    deploy.add_argument('--wait', action='store_true',
+                        help='wait for deployment to be finished')
+    deploy.add_argument('-e', '--extra-vars', action='append',
+                        help='additional vars to pass to ansible')
+
     args = parser.parse_args()
     if getattr(args, 'func', None) is None:
         parser.print_usage(file=sys.stderr)

doc/source/install/index.rst

@@ -300,8 +300,8 @@ See the built-in documentation for more details:
     ./bifrost-cli install --help
 
 The Ansible variables generated for installation are stored in a JSON file
-(``bifrost-install-env.json`` by default) that should be passed via the ``-e``
+(``baremetal-install-env.json`` by default) that should be passed via the
-flag to subsequent playbook or command invokations.
+``-e`` flag to subsequent playbook or command invokations.
 
 .. _custom-ipa-images:
 

doc/source/user/howto.rst

@@ -162,7 +162,13 @@ in an ``instance_info`` variable, for example:
         "name": "testvm1",
         "instance_info": {
           "image_source": "http://image.server/image.qcow2",
-          "image_checksum": "<md5 checksum>"
+          "image_checksum": "<md5 checksum>",
+          "configdrive": {
+            "meta_data": {
+              "public_keys": {"0": "ssh-rsa ..."},
+              "hostname": "vm1.example.com"
+            }
+          }
         }
       }
   }
@@ -181,16 +187,6 @@ Starting with the Wallaby cycle, you can use ``bifrost-cli`` for enrolling:
 
     ./bifrost-cli enroll /tmp/baremetal.json
 
-Utilizing the dynamic inventory module, enrollment is as simple as setting
-the ``BIFROST_INVENTORY_SOURCE`` environment variable to your inventory data
-source, and then executing the enrollment playbook:
-
-.. code-block:: bash
-
-  export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.json
-  cd playbooks
-  ansible-playbook -vvvv -i inventory/bifrost_inventory.py enroll-dynamic.yaml
-
 Note that enrollment is a one-time operation. The Ansible module *does not*
 synchronize data for existing nodes.  You should use the ironic CLI to do this
 manually at the moment.
@@ -208,35 +204,72 @@ utilize configuration drives to convey basic configuration information to the
 each host. This configuration information includes an SSH key to allow a user
 to login to the system.
 
-To utilize the dynamic inventory based deployment:
+Starting with the Yoga cycle, you can use ``bifrost-cli`` for deploying. If
+you used ``bifrost-cli`` for installation, you should pass its environment
+variables, as well as the inventory file (see `JSON file format`_):
 
 .. code-block:: bash
 
-  export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.json
+    ./bifrost-cli deploy /tmp/baremetal.json \
-  cd playbooks
+        -e @baremetal-install-env.json
-  ansible-playbook -vvvv -i inventory/bifrost_inventory.py deploy-dynamic.yaml
-
-If you used ``bifrost-cli`` for installation, you should pass its environment
-variables::
-
-  export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.json
-  cd playbooks
-  ansible-playbook -vvvv \
-    -i inventory/bifrost_inventory.py \
-    -e @../bifrost-install-env.json \
-    deploy-dynamic.yaml
 
 .. note::
+   By default, the playbook will return once the deploy has started. Pass
+   the ``--wait`` flag to wait for completion.
 
-  Before running the above command, ensure that the value for
+The inventory file may override some deploy settings, such as images or even
-  `ssh_public_key_path` in ``./playbooks/inventory/group_vars/baremetal``
+the complete ``instance_info``, per node.  If you omit it, all nodes from
-  refers to a valid public key file, or set the ssh_public_key_path option
+Ironic will be deployed using the Bifrost defaults:
-  on the ansible-playbook command line by setting the variable.
-  Example: "-e ssh_public_key_path=~/.ssh/id_rsa.pub"
 
-The image, downloaded or generated during installation, is used by default.
+.. code-block:: bash
-Please see `JSON file format`_ for information on how to override the image per
+
-node.
+    ./bifrost-cli deploy -e @baremetal-install-env.json
+
+Command line parameters
+-----------------------
+
+By default the playbooks use the image, downloaded or built during
+installation. You can also use a custom image:
+
+.. code-block:: bash
+
+    ./bifrost-cli deploy -e @baremetal-install-env.json \
+        --image http://example.com/images/my-image.qcow2 \
+        --image-checksum 91ebfb80743bb98c59f787c9dc1f3cef \
+
+You can also provide a custom configdrive URL (or its content) instead of
+the one Bifrost builds for you:
+
+.. code-block:: bash
+
+    ./bifrost-cli deploy -e @baremetal-install-env.json \
+        --config-drive '{"meta_data": {"public_keys": {"0": "'"$(cat ~/.ssh/id_rsa.pub)"'"}}}' \
+
+File images do not require a checksum:
+
+.. code-block:: bash
+
+    ./bifrost-cli deploy -e @baremetal-install-env.json \
+        --image file:///var/lib/ironic/custom-image.qcow2
+
+.. note:: Files must be readable by Ironic. Your home directory is often not.
+
+Partition images can de deployed by specifying an image type:
+
+.. code-block:: bash
+
+    ./bifrost-cli deploy -e @baremetal-install-env.json \
+        --image http://example.com/images/my-image.qcow2 \
+        --image-checksum 91ebfb80743bb98c59f787c9dc1f3cef \
+        --partition
+
+.. note::
+   The default root partition size is 10 GiB. Set the ``deploy_root_gb``
+   parameter to override or use a first-boot service such as cloud-init to
+   grow the root partition automatically.
+
+Redeploy Hardware
+=================
 
 If the hosts need to be re-deployed, the dynamic redeploy playbook may be used:
 
@@ -249,6 +282,42 @@ If the hosts need to be re-deployed, the dynamic redeploy playbook may be used:
 This playbook will undeploy the hosts, followed by a deployment, allowing
 a configurable timeout for the hosts to transition in each step.
 
+Use playbooks instead of bifrost-cli
+====================================
+
+Using playbooks directly allows you full control over what is executed by
+Bifrost, with what variables and using what inventory.
+
+Utilizing the dynamic inventory module, enrollment is as simple as setting
+the ``BIFROST_INVENTORY_SOURCE`` environment variable to your inventory data
+source, and then executing the enrollment playbook:
+
+.. code-block:: bash
+
+  export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.json
+  cd playbooks
+  ansible-playbook -vvvv -i inventory/bifrost_inventory.py enroll-dynamic.yaml
+
+To utilize the dynamic inventory based deployment:
+
+.. code-block:: bash
+
+  export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.json
+  cd playbooks
+  ansible-playbook -vvvv -i inventory/bifrost_inventory.py deploy-dynamic.yaml
+
+If you used ``bifrost-cli`` for installation, you should pass its environment
+variables:
+
+.. code-block:: bash
+
+  export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.json
+  cd playbooks
+  ansible-playbook -vvvv \
+    -i inventory/bifrost_inventory.py \
+    -e @../baremetal-install-env.json \
+    deploy-dynamic.yaml
+
 Deployment and configuration of operating systems
 =================================================
 
@@ -273,6 +342,11 @@ Due to the nature of the design, it would be relatively easy for a user to
 import automatic growth or reconfiguration steps either in the image to be
 deployed, or in post-deployment steps via custom Ansible playbooks.
 
+To be able to access nodes via SSH, ensure that the value for
+`ssh_public_key_path` in ``./playbooks/inventory/group_vars/baremetal``
+refers to a valid public key file, or set the ``ssh_public_key_path`` variable
+on the command line, e.g. ``-e ssh_public_key_path=~/.ssh/id_rsa.pub``.
+
 Advanced topics
 ===============
 

playbooks/deploy-dynamic.yaml

@@ -11,5 +11,8 @@
   roles:
     - role: bifrost-configdrives-dynamic
       delegate_to: "{{ groups['target'][0] if groups['target'] is defined else 'localhost' }}"
+      when:
+        - deploy_config_drive is undefined
+        - instance_info is undefined or instance_info.configdrive is undefined
     - role: bifrost-deploy-nodes-dynamic
       delegate_to: "{{ groups['target'][0] if groups['target'] is defined else 'localhost' }}"

playbooks/roles/bifrost-configdrives-dynamic/defaults/main.yml

@@ -7,6 +7,13 @@ write_interfaces_file: false
 http_boot_folder: /var/lib/ironic/httpboot
 # Default location to the ssh public key for the user operating Bifrost.
 #ssh_public_key_path: "/path/to/id_rsa.pub"
+deploy_url_protocol: "http"
+
+file_url_port: "8080"
+network_interface: "virbr0"
+ans_network_interface: "{{ network_interface | replace('-', '_') }}"
+ans_hostname: "{{ groups['target'][0] if groups['target'] is defined else 'localhost' }}"
+internal_ip: "{{ hostvars[ans_hostname]['ansible_' + ans_network_interface]['ipv4']['address'] }}"
 
 # Default interface name
 # TODO(TheJulia): Remove this default.

playbooks/roles/bifrost-configdrives-dynamic/tasks/main.yml

@@ -130,3 +130,7 @@
     state: absent
     force: yes
     name: "{{ variable_configdrive_location.path }}"
+
+- name: "Set the configdrive URL"
+  set_fact:
+    deploy_config_drive: "{{ deploy_url_protocol }}://{{ internal_ip }}:{{ file_url_port }}/configdrive-{{ uuid }}.iso.gz"

playbooks/roles/bifrost-deploy-nodes-dynamic/README.md

@@ -15,6 +15,14 @@ bifrost-configdrives-dynamic, however that is unnecessary IF the host has a
 dictionary named instance_info defined as that will be used as overriding
 values.
 
+There are two ways to specify the image information:
+- Set `instance_info` in the Ironic format (all `deploy_image_` properties
+  are ignored in this case).
+- Set `deploy_image_source` to a URL and `deploy_image_checksum` to a checksum
+  value or a URL with checksums (optional for `file://` images).
+- Set `deploy_image_filename` to a file name in the HTTP directory (or rely
+  on the Bifrost defaults).
+
 Role Variables
 --------------
 
@@ -27,12 +35,25 @@ network_interface: This is the network interface that the nodes receive
                    variable does not have a default in this role and expects to
                    receive this information from the calling playbook.
 
+deploy_image_source: The URL of the image to deploy. The default is derived
+                     from `deploy_image_filename` and the address of the HTTP
+                     server.
+
+deploy_image_checksum: The checksum (or its URL) of the image to deploy.
+                       By default a checksum of `deploy_image_path` is used.
+
 deploy_image_filename: This is the filename of the image to deploy, which is
                        combined with the network_interface variable to generate
-                       a URL used to set the ironic instance image_source. This
+                       a URL used to set the ironic instance image_source.
-                       variable does not have a default in this role and
+                       The default is `deployment_image.qcow2`.
-                       expects to receive this information from the calling
+
-                       playbook.
+deploy_image_path: This is the full path to the image to be deployed.
+                   This is as ironic requires the MD5 hash of the file to be
+                   deployed for validation during the deployment process. As a
+                   result of this requirement, the hash is automatically
+                   collected and submitted to ironic with the node deployment
+                   request. The default is deploy_image_filename in the HTTP
+                   server path.
 
 deploy_url_protocol: The protocol to utilize to access config_drive and
                      image_source files. The default is to utilize HTTP in
@@ -40,25 +61,24 @@ deploy_url_protocol: The protocol to utilize to access config_drive and
                      allows a user to change that default if they have
                      a modified local webserver configuration.
 
-deploy_image: This is the full path to the image to be deployed to the system.
-              This is as ironic requires the MD5 hash of the file to be
-              deployed for validation during the deployment process.  As a
-              result of this requirement, the hash is automatically collected
-              and submitted to ironic with the node deployment request. This
-              variable does not have a default in this role and expects to
-              receive this information from the calling playbook.
-
 deploy_image_rootfs: This is the UUID of the root filesystem contained in the
                      deployment image. It is usually not required to specify
                      this unless software RAID based deployment is performed.
                      See https://docs.openstack.org/ironic/latest/admin/raid.html#image-requirements
                      for more information.
 
+deploy_image_type: The type of the image: "whole-disk" or "partition".
+                   Will not be passed by default.
+
+deploy_configdrive: The URL or the contents of the configdrive to use.
+                    By default the URL generated by the
+                    `bifrost-configdrives-dynamic` role is used.
+
 instance_info: A dictionary containing the information to define an instance.
                By default, this is NOT expected to be defined, however if
                defined it is passed in whole to the deployment step.  This
-               value will override deploy_image_filename, deploy_image,
+               value will override deploy_image_filename, deploy_image_path,
-               deploy_image_rootfs and network_interface variables. Key-value
+               deploy_image_rootfs and deploy_image_type variables. Key-value
                pairs that are generally expected are image_source,
                image_checksum, root_gb, however, any supported key/value can be
                submitted to the API.
@@ -119,7 +139,7 @@ NOTE: The example below assumes bifrost's default and that an instance_info
         connection: local
         become: no
         roles:
-          - role: bifrost-configdrives
+          - role: bifrost-configdrives-dynamic
           - role: bifrost-deploy-nodes-dynamic
 
 License

playbooks/roles/bifrost-deploy-nodes-dynamic/defaults/main.yml

@@ -3,10 +3,15 @@
 file_url_port: "8080"
 network_interface: "virbr0"
 ans_network_interface: "{{ network_interface | replace('-', '_') }}"
-internal_ip: "{{ hostvars[inventory_hostname]['ansible_' + ans_network_interface]['ipv4']['address'] }}"
+ans_hostname: "{{ groups['target'][0] if groups['target'] is defined else 'localhost' }}"
+internal_ip: "{{ hostvars[ans_hostname]['ansible_' + ans_network_interface]['ipv4']['address'] }}"
+
 http_boot_folder: "/var/lib/ironic/httpboot"
 deploy_image_filename: "deployment_image.qcow2"
-deploy_image: "{{http_boot_folder}}/{{deploy_image_filename}}"
+# Backward compatibility: used to be called deploy_image
+deploy_image_path: "{{ deploy_image | default(http_boot_folder + '/' + deploy_image_filename) }}"
+deploy_image_source: "{{ deploy_url_protocol }}://{{ internal_ip }}:{{ file_url_port }}/{{ deploy_image_filename }}"
+deploy_root_gb: 10
 inventory_dhcp: false
 inventory_dhcp_static_ip: true
 inventory_dns: false

playbooks/roles/bifrost-deploy-nodes-dynamic/tasks/main.yml

@@ -17,9 +17,6 @@
 # the pass-through could mean that the user could deploy
 # things that are not directly accessible or reasonable
 # to be inspected.
-- name: "Obtain setup facts"
-  setup:
-    gather_timeout: "{{ fact_gather_timeout }}"
 
 - import_role:
     name: bifrost-cloud-config
@@ -73,7 +70,45 @@
   become: yes
   when: inventory_dhcp | bool or inventory_dns | bool
 
-- name: "Deploy to hardware - Using custom instance_info."
+- name: "Create instance info"
+  block:
+
+    - name: "Figure out image checksum"
+      block:
+
+        - name: "Collect the checksum of the deployment image."
+          stat:
+            path: "{{ deploy_image_path }}"
+            get_checksum: yes
+            checksum_algorithm: md5
+          register: test_deploy_image
+          become: yes
+
+        - name: "Error if deploy_image_path is not present, and instance_info is not defined"
+          fail:
+            msg: "The user-defined deploy_image_path, which is the image to be written to the remote node(s) upon deployment, was not found. Cannot proceed."
+          when: not test_deploy_image.stat.exists
+
+        - name: "Set the calculated checksum"
+          set_fact:
+            deploy_image_checksum: "{{ test_deploy_image.stat.checksum }}"
+
+      when:
+        - deploy_image_checksum is not defined
+        - not deploy_image_source.startswith('file://')
+
+    - name: "Set generated instance_info"
+      set_fact:
+        instance_info:
+          image_source: "{{ deploy_image_source }}"
+          image_checksum: "{{ deploy_image_checksum | default(omit) }}"
+          image_rootfs_uuid: "{{ deploy_image_rootfs | default(omit) }}"
+          image_type: "{{ deploy_image_type | default(omit) }}"
+          root_gb: "{{ deploy_root_gb if deploy_image_type | default('') == 'partition' else omit }}"
+
+  when: instance_info is not defined or instance_info == {}
+
+- name: "Deploy to hardware"
   openstack.cloud.baremetal_node_action:
     cloud: "{{ cloud_name | default(omit) }}"
     auth_type: "{{ auth_type | default(omit) }}"
@@ -82,42 +117,8 @@
     ironic_url: "{{ ironic_url | default(omit) }}"
     uuid: "{{ uuid }}"
     state: present
-    config_drive: "{{ deploy_url_protocol }}://{{ internal_ip }}:{{ file_url_port }}/configdrive-{{ uuid }}.iso.gz"
+    # Allow instance_info in the inventory to override configdrive
+    config_drive: "{{ instance_info.configdrive | default(deploy_config_drive) | default(omit) }}"
     instance_info: "{{ instance_info }}"
     wait: "{{ wait_for_node_deploy }}"
     timeout: " {{ wait_timeout | default(1800) }}"
-  when: instance_info is defined and instance_info | to_json != '{}'
-
-- name: "Collect the checksum of the deployment image."
-  stat:
-    path: "{{ deploy_image }}"
-    get_checksum: yes
-    checksum_algorithm: md5
-  register: test_deploy_image
-  become: yes
-  when: instance_info is not defined or ( instance_info is defined and instance_info | to_json == '{}' )
-
-- name: "Error if deploy_image is not present, and instance_info is not defined"
-  fail: msg="The user-defined deploy_image, which is the image to be written to the remote node(s) upon deployment, was not found. Cannot proceed."
-  when:
-    - instance_info is not defined
-    - not test_deploy_image.stat.exists
-
-- name: "Deploy to hardware - bifrost default"
-  openstack.cloud.baremetal_node_action:
-    cloud: "{{ cloud_name | default(omit) }}"
-    auth_type: "{{ auth_type | default(omit) }}"
-    auth: "{{ auth | default(omit) }}"
-    ca_cert: "{{ tls_certificate_path | default(omit) }}"
-    ironic_url: "{{ ironic_url | default(omit) }}"
-    uuid: "{{ uuid }}"
-    state: present
-    config_drive: "{{ deploy_url_protocol }}://{{ internal_ip }}:{{ file_url_port }}/configdrive-{{ uuid }}.iso.gz"
-    instance_info:
-      image_source: "{{ deploy_url_protocol }}://{{ internal_ip }}:{{ file_url_port }}/{{ deploy_image_filename }}"
-      image_checksum: "{{ test_deploy_image.stat.checksum }}"
-      image_disk_format: "qcow2"
-      image_rootfs_uuid: "{{ deploy_image_rootfs | default(omit) }}"
-    wait: "{{ wait_for_node_deploy }}"
-    timeout: " {{ wait_timeout | default(1800) }}"
-  when: instance_info is not defined or ( instance_info is defined and instance_info | to_json == '{}' )

releasenotes/notes/cli-deploy-6202c0801b7b2079.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Adds a new CLI command ``./bifrost-cli deploy`` that runs the deploy
+    playbook, optionally specifying a custom image.
+  - |
+    Adds a new way to specify a custom image for the
+    ``bifrost-deploy-nodes-dynamic`` role by setting the new parameters
+    ``deploy_image_source`` and ``deploy_image_checksum``.
+  - |
+    Allows customizing the configdrive URL or JSON for the
+    ``bifrost-deploy-nodes-dynamic`` role by setting the new parameter
+    ``deploy_config_drive``.
+deprecations:
+  - |
+    The ``deploy_image`` parameter of the ``bifrost-deploy-nodes-dynamic`` role
+    is deprecated in favour of ``deploy_image_path``.