bootstrap-host: Prepare disk for machinectl storage

Currently we prepare an available secondary disk, but do not
use it. The preparation taking place now was useful when the
backing file system for LXC was 'dir', but it is now machinectl
which uses a btrfs partition instead.

This patch implements the preparation of a btrfs sparse file
if there is no secondary disk, or the preparation of a secondary
btrfs disk if there is one.

The user_variables override for lxc_host_machine_volume_size
is removed as the default value in the role is perfectly adequate
for AIO deployment needs.

We also add some diagnostic information to help troubleshoot
issues if they arise.

Change-Id: Icadbcfd420bbe7c14a04606790cab09a1dce03ce

scripts/scripts-library.sh

@@ -318,6 +318,17 @@ function get_instance_info {
             "/openstack/log/instance-info/host_packages_info_${TS}.log" || true
           ;;
   esac
+
+  # Storage reports
+  btrfs filesystem usage /var/lib/machines > \
+    "/openstack/log/instance-info/machines_usage_${TS}.log" || true
+  btrfs filesystem show /var/lib/machines >> \
+    "/openstack/log/instance-info/machines_show_${TS}.log" || true
+  btrfs filesystem df /var/lib/machines >> \
+    "/openstack/log/instance-info/machines_df_${TS}.log" || true
+  btrfs qgroup show --human-readable -pcre --iec /var/lib/machines >> \
+    "/openstack/log/instance-info/machines_quotas_${TS}.log" || true
+  df -h > "/openstack/log/instance-info/report_fs_df_${TS}.log" || true
 }
 
 function get_pip {

tests/roles/bootstrap-host/defaults/main.yml

@@ -63,9 +63,14 @@ bootstrap_user_variables_extra_templates:
 bootstrap_host_swap_size: "{% if ansible_memory_mb['real']['total'] < 8*1024 %}4{% else %}8{% endif %}"
 
 ## Loopback volumes
-# Sparse loopback disks are used for Cinder, Swift and Nova (instance storage).
+# Sparse loopback disks are used for the containers if there is no secondary
+# disk available to partition for btrfs. They are also used for Ceph, Cinder,
+# Swift and Nova (instance storage).
 # The size of the loopback volumes can be customized here (in gigabytes).
 #
+# Size of the machines loopback disk in gigabytes (GB).
+bootstrap_host_loopback_machines_size: 128
+#
 # Boolean option to deploy the loopback disk for Cinder
 bootstrap_host_loopback_cinder: yes
 # Size of the Cinder loopback disk in gigabytes (GB).

tests/roles/bootstrap-host/tasks/main.yml

@@ -50,6 +50,15 @@
   tags:
     - prepare-data-disk
 
+# Prepare the Machines storage loopback disk
+# This is only necessary when there is no secondary disk
+# available to partition for btrfs
+- include: prepare_loopback_machines.yml
+  when:
+    - bootstrap_host_data_disk_device == None
+  tags:
+    - prepare-loopback-machines
+
 # Prepare the swap space loopback disk
 # This is only necessary if there isn't swap already
 - include: prepare_loopback_swap.yml

tests/roles/bootstrap-host/tasks/prepare_data_disk.yml

@@ -45,18 +45,20 @@
   with_items:
     - "parted --script /dev/{{ bootstrap_host_data_disk_device | regex_replace('!','/') }} mklabel gpt"
     - "parted --align optimal --script /dev/{{ bootstrap_host_data_disk_device | regex_replace('!','/') }} mkpart openstack-data1 ext4 0% 40%"
-    - "parted --align optimal --script /dev/{{ bootstrap_host_data_disk_device | regex_replace('!','/') }} mkpart openstack-data2 ext4 40% 100%"
+    - "parted --align optimal --script /dev/{{ bootstrap_host_data_disk_device | regex_replace('!','/') }} mkpart openstack-data2 btrfs 40% 100%"
   tags:
     - create-data-disk-partitions
 
 - name: Format the partitions
   filesystem:
-    fstype: ext4
-    dev: "{{ item }}"
+    fstype: "{{ item.fstype }}"
+    dev: "{{ item.dev }}"
   when: data_disk_partitions.rc == 1 or bootstrap_host_data_disk_device_force | bool
   with_items:
-    - "/dev/{{ bootstrap_host_data_disk_device | regex_replace('!(.*)$','/\\1p') }}1"
-    - "/dev/{{ bootstrap_host_data_disk_device | regex_replace('!(.*)$','/\\1p') }}2"
+    - dev: "/dev/{{ bootstrap_host_data_disk_device | regex_replace('!(.*)$','/\\1p') }}1"
+      fstype: "ext4"
+    - dev: "/dev/{{ bootstrap_host_data_disk_device | regex_replace('!(.*)$','/\\1p') }}2"
+      fstype: "btrfs"
   tags:
     - format-data-partitions
 
@@ -64,10 +66,14 @@
   mount:
     name: "{{ item.mount_point }}"
     src: "{{ item.device }}"
-    fstype: ext4
+    fstype: "{{ item.fstype }}"
     state: mounted
   with_items:
-    - { mount_point: /openstack, device: "/dev/{{ bootstrap_host_data_disk_device | regex_replace('!(.*)$','/\\1p') }}1"}
-    - { mount_point: /var/lib/lxc, device: "/dev/{{ bootstrap_host_data_disk_device | regex_replace('!(.*)$','/\\1p') }}2"}
+    - mount_point: /openstack
+      device: "/dev/{{ bootstrap_host_data_disk_device | regex_replace('!(.*)$','/\\1p') }}1"
+      fstype: ext4
+    - mount_point: /var/lib/machines
+      device: "/dev/{{ bootstrap_host_data_disk_device | regex_replace('!(.*)$','/\\1p') }}2"
+      fstype: btrfs
   tags:
     - mount-data-partitions

tests/roles/bootstrap-host/tasks/prepare_loopback_machines.yml

@@ -0,0 +1,37 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create sparse machines file
+  command: "truncate -s {{ bootstrap_host_loopback_machines_size }}G /openstack/machines.img"
+  args:
+    creates: /openstack/machines.img
+  tags:
+    - machines-file-create
+
+- name: Format the machines file
+  filesystem:
+    fstype: btrfs
+    dev: /openstack/machines.img
+  tags:
+    - machines-format-file
+
+- name: Create the mount points, fstab entries and mount the file systems
+  mount:
+    name: /var/lib/machines
+    src: /openstack/machines.img
+    fstype: btrfs
+    state: mounted
+  tags:
+    - machines-file-mount

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -165,10 +165,6 @@ cache_timeout: {{ cache_timeout }}
 # AIO build time. Options are: [machinectl, overlayfs, btrfs, zfs, dir, lvm]
 lxc_container_backing_store: "machinectl"
 
-# Allow the container store sufficient space to build everything we need.
-#  Size is in GB:
-lxc_host_machine_volume_size: 1024
-
 ## Enable LBaaSv2 in the AIO
 neutron_plugin_base:
   - router

tests/roles/bootstrap-host/vars/redhat.yml

@@ -17,6 +17,7 @@ rdo_package: "https://rdoproject.org/repos/openstack-pike/rdo-release-pike.rpm"
 
 packages_install:
   - bridge-utils
+  - btrfs-progs
   - curl
   - dbus
   - ethtool

tests/roles/bootstrap-host/vars/suse.yml

@@ -20,6 +20,7 @@ opensuse_openstack_repos:
 
 packages_install:
   - bridge-utils
+  - btrfsprogs
   - curl
   - dbus-1
   - ethtool

tests/roles/bootstrap-host/vars/ubuntu.yml

@@ -16,6 +16,7 @@
 packages_install:
   - apt-transport-https
   - bridge-utils
+  - btrfs-tools
   - build-essential
   - curl
   - dbus