MNAIO: Add ability to deploy Ceph

This commit enables overriding the default storage options of
cinder/swift to a Ceph backed storage for the multinode all in one
deployments.

We also correct the README and build.sh script to show/use the
current defaults correctly for VM_DISK_SIZE and INFRA_VM_SERVER_RAM.

Change-Id: I9e1f1b09d1bcf224f4afa765c585baf28e6cafa8

multi-node-aio/README.rst

@@ -107,12 +107,11 @@ to change this password please edit the pre-seed files.
 Set an external inventory used for the MNAIO:
   ``MNAIO_INVENTORY=${MNAIO_INVENTORY:-playbooks/inventory}``
 
-
 Set to instruct the preseed what the default network is expected to be:
   ``DEFAULT_NETWORK="${DEFAULT_NETWORK:-eth0}"``
 
 Set the VM disk size in gigabytes:
-  ``VM_DISK_SIZE="${VM_DISK_SIZE:-252}"``
+  ``VM_DISK_SIZE="${VM_DISK_SIZE:-92160}"``
 
 Instruct the system do all of the required host setup:
   ``SETUP_HOST=${SETUP_HOST:-true}``
@@ -181,7 +180,7 @@ Instruct the system to use a set amount of ram for compute VM type:
   ``COMPUTE_VM_SERVER_RAM=${COMPUTE_VM_SERVER_RAM:-8196}``
 
 Instruct the system to use a set amount of ram for infra VM type:
-  ``INFRA_VM_SERVER_RAM=${INFRA_VM_SERVER_RAM:-8196}``
+  ``INFRA_VM_SERVER_RAM=${INFRA_VM_SERVER_RAM:-16384}``
 
 Instruct the system to use a set amount of ram for load balancer VM type:
   ``LOADBALANCER_VM_SERVER_RAM=${LOADBALANCER_VM_SERVER_RAM:-1024}``
@@ -198,6 +197,8 @@ Instruct the system where to obtain iPXE kernels (looks for ipxe.lkrn, ipxe.efi,
 Instruct the system to use a customized iPXE script during boot of VMs:
   ``IPXE_PATH_URL=${IPXE_PATH_URL:-''}``
 
+Instruct the system to use CEPH block & object storage instead of the default LVM/swift:
+  ``ENABLE_CEPH_STORAGE=${ENABLE_CEPH_STORAGE:-false}``
 
 Re-kicking VM(s)
 ----------------
@@ -296,3 +297,11 @@ available via a URL, they can be downloaded and used on a fresh host as follows.
     # Then kick off the VM's from those images
     ansible-playbook ${ANSIBLE_PARAMETERS} playbooks/deploy-vms.yml
 
+Using Ceph-backed Block and Object Storage
+------------------------------------------
+
+To make use of Ceph in the environment, set ``ENABLE_CEPH_STORAGE`` to
+``true``. This will disable the use of Swift as the Object Storage back-end
+and disable the use of LVM as the Block Storage back-end, replacing both of
+these with Ceph services.
+

multi-node-aio/build.sh

@@ -26,6 +26,9 @@ ansible vm_hosts \
 
 export MNAIO_ANSIBLE_PARAMETERS=${MNAIO_ANSIBLE_PARAMETERS:-""}
 
+# TODO(odyssey4me):
+# Replace this build override mechanism to just respect the
+# group_vars instead of duplicating defaults here.
 ansible-playbook -vv \
                  -i ${MNAIO_INVENTORY:-"playbooks/inventory"} \
                  -e setup_host=${SETUP_HOST:-"true"} \
@@ -52,10 +55,11 @@ ansible-playbook -vv \
                  -e default_ubuntu_mirror_directory=${DEFAULT_MIRROR_DIR:-"/ubuntu"} \
                  -e cinder_vm_server_ram=${CINDER_VM_SERVER_RAM:-"2048"} \
                  -e compute_vm_server_ram=${COMPUTE_VM_SERVER_RAM:-"8196"} \
-                 -e infra_vm_server_ram=${INFRA_VM_SERVER_RAM:-"8196"} \
+                 -e infra_vm_server_ram=${INFRA_VM_SERVER_RAM:-"16384"} \
                  -e loadbalancer_vm_server_ram=${LOADBALANCER_VM_SERVER_RAM:-"2048"} \
                  -e logging_vm_server_ram=${LOGGING_VM_SERVER_RAM:-"2048"} \
                  -e swift_vm_server_ram=${SWIFT_VM_SERVER_RAM:-"2048"} \
+                 -e enable_ceph_storage=${ENABLE_CEPH_STORAGE=-"false"} \
                  -e container_tech=${CONTAINER_TECH:-"lxc"} \
                  -e ipxe_kernel_base_url=${IPXE_KERNEL_BASE_URL:-"http://boot.ipxe.org"} \
                  -e ipxe_path_url=${IPXE_PATH_URL:-""} ${MNAIO_ANSIBLE_PARAMETERS} \

multi-node-aio/playbooks/deploy-vms.yml

@@ -177,6 +177,8 @@
       copy:
         content: "{{ item.1.get_xml }}"
         dest: "/etc/libvirt/qemu/{{ item.1.item }}.xml"
+      when:
+        - server_vm | default(false) | bool
       delegate_to: "{{ item.0 }}"
       with_nested:
         - "{{ groups['vm_hosts'] }}"
@@ -309,8 +311,8 @@
         state: restarted
 
 
-- name: Make space for swift and cinder volumes
+- name: Make space for swift/cinder/ceph volumes
-  hosts: cinder_hosts:swift_hosts
+  hosts: cinder_hosts:swift_hosts:ceph_hosts:&vm_servers
   gather_facts: "{{ gather_facts | default(true) }}"
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
@@ -348,27 +350,27 @@
         _force_format_disks: "{{ (_remove_mounts is changed) or (_remove_lvs is changed) }}"
 
 - name: Setup cinder host volume
-  hosts: cinder_hosts
+  hosts: cinder_hosts:&vm_servers
   gather_facts: "{{ gather_facts | default(true) }}"
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
     - deploy-vms
   tasks:
-    - name: Create cinder-volumes lv
+    - name: Create cinder-volumes LV
       lvol:
         vg: vmvg00
         lv: cinder-volumes00
         size: "100%FREE"
         shrink: false
 
-    - name: Create data cinder-volumes group
+    - name: Create data cinder-volumes VG
       lvg:
         vg: cinder-volumes
         pvs: "/dev/vmvg00/cinder-volumes00"
 
 
 - name: Setup swift host volume
-  hosts: swift_hosts
+  hosts: swift_hosts:&vm_servers
   gather_facts: "{{ gather_facts | default(true) }}"
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
@@ -405,3 +407,30 @@
         - disk1
         - disk2
         - disk3
+
+- name: Setup ceph OSD volumes
+  hosts: ceph_hosts:&vm_servers
+  gather_facts: "{{ gather_facts | default(true) }}"
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  tags:
+    - deploy-vms
+  tasks:
+    - name: Create ceph OSD journal LV's
+      lvol:
+        vg: vmvg00
+        lv: "{{ item }}"
+        size: "{{ ceph_journal_size }}"
+      with_items:
+        - journal1
+        - journal2
+        - journal3
+
+    - name: Create ceph OSD disk LV's
+      lvol:
+        vg: vmvg00
+        lv: "{{ item }}"
+        size: "{{ ceph_osds_size }}"
+      with_items:
+        - data1
+        - data2
+        - data3

multi-node-aio/playbooks/group_vars/all.yml

@@ -147,7 +147,7 @@ mnaio_host_networks:
 
 osa_enable_infra: true
 osa_enable_identity: true
-osa_enable_block_storage: true
+osa_enable_block_storage: "{{ not (enable_ceph_storage | bool) }}"
 osa_enable_image: true
 osa_enable_lbaas: false
 osa_enable_compute: true
@@ -155,9 +155,14 @@ osa_enable_orchestration: true
 osa_enable_dashboard: true
 osa_enable_network: true
 osa_enable_meter: false
-osa_enable_object_storage: true
+osa_enable_object_storage: "{{ not (enable_ceph_storage | bool) }}"
 osa_enable_legacy_os_infra: "{{ (osa_enable_image | bool) and (osa_enable_orchestration | bool) and (osa_enable_dashboard | bool) and (osa_enable_compute | bool) }}"
 osa_disable_serial: false
 osa_enable_elk_metrics: false
 osa_enable_os_profiler: false
 osa_enable_uwsgi_stats: false
+
+# Ceph
+enable_ceph_storage: false
+ceph_journal_size: 5120
+ceph_osds_filesystem_type: 'xfs'

multi-node-aio/playbooks/group_vars/ceph_hosts.yml

@@ -0,0 +1,30 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in witing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_os_family: "{{ images[default_vm_image]['image_type'] }}"
+
+server_domain_name: 'openstack.local'
+server_vm: "{{ enable_ceph_storage | bool }}"
+server_vm_ram: '{{ ceph_vm_server_ram | default(1024) }}'
+server_vm_vcpus: '{{ ceph_vm_server_vcpus | default(2) }}'
+server_vm_primary_network: 'dhcp'
+server_image: "{{ default_vm_image }}"
+server_default_interface: 'eth0'
+server_preseed_ks: 'vm'
+server_extra_options: ''
+
+ceph_osds_size: 20480
+
+default_vm_storage: "{{ ceph_vm_disk_size | default(122880) }}"

multi-node-aio/playbooks/host_vars/ceph1.yml

@@ -0,0 +1,50 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in witing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_os_family: "{{ images[default_vm_image]['image_type'] }}"
+
+server_hostname: 'ceph1'
+server_vm_fixed_addr: '10.0.2.145'
+server_mac_address: '52:54:00:bd:80:13'
+
+server_networks:
+  dhcp:
+    iface: 'eth0'
+    inet_type: 'dhcp'
+    vm_int_iface: vm-br-dhcp
+  mgmt:
+    iface: 'eth1'
+    inet_type: 'static'
+    address: '10.0.236.145/22'
+    vm_int_iface: vm-br-eth1
+  flat:
+    iface: 'eth2'
+    inet_type: 'static'
+    address: '10.0.248.145/22'
+    vm_int_iface: vm-br-eth2
+  vlan:
+    iface: 'eth3'
+    inet_type: 'manual'
+    vm_int_iface: vm-br-eth3
+  vxlan:
+    iface: 'eth4'
+    inet_type: 'static'
+    address: '10.0.240.145/22'
+    vm_int_iface: vm-br-eth4
+  storage:
+    iface: 'eth5'
+    inet_type: 'static'
+    address: '10.0.244.145/22'
+    vm_int_iface: vm-br-eth5

multi-node-aio/playbooks/host_vars/ceph2.yml

@@ -0,0 +1,50 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in witing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_os_family: "{{ images[default_vm_image]['image_type'] }}"
+
+server_hostname: 'ceph2'
+server_vm_fixed_addr: '10.0.2.161'
+server_mac_address: '52:54:00:bd:80:14'
+
+server_networks:
+  dhcp:
+    iface: 'eth0'
+    inet_type: 'dhcp'
+    vm_int_iface: vm-br-dhcp
+  mgmt:
+    iface: 'eth1'
+    inet_type: 'static'
+    address: '10.0.236.146/22'
+    vm_int_iface: vm-br-eth1
+  flat:
+    iface: 'eth2'
+    inet_type: 'static'
+    address: '10.0.248.146/22'
+    vm_int_iface: vm-br-eth2
+  vlan:
+    iface: 'eth3'
+    inet_type: 'manual'
+    vm_int_iface: vm-br-eth3
+  vxlan:
+    iface: 'eth4'
+    inet_type: 'static'
+    address: '10.0.240.146/22'
+    vm_int_iface: vm-br-eth4
+  storage:
+    iface: 'eth5'
+    inet_type: 'static'
+    address: '10.0.244.146/22'
+    vm_int_iface: vm-br-eth5

multi-node-aio/playbooks/host_vars/ceph3.yml

@@ -0,0 +1,50 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in witing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_os_family: "{{ images[default_vm_image]['image_type'] }}"
+
+server_hostname: 'ceph3'
+server_vm_fixed_addr: '10.0.2.162'
+server_mac_address: '52:54:00:bd:80:15'
+
+server_networks:
+  dhcp:
+    iface: 'eth0'
+    inet_type: 'dhcp'
+    vm_int_iface: vm-br-dhcp
+  mgmt:
+    iface: 'eth1'
+    inet_type: 'static'
+    address: '10.0.236.147/22'
+    vm_int_iface: vm-br-eth1
+  flat:
+    iface: 'eth2'
+    inet_type: 'static'
+    address: '10.0.248.147/22'
+    vm_int_iface: vm-br-eth2
+  vlan:
+    iface: 'eth3'
+    inet_type: 'manual'
+    vm_int_iface: vm-br-eth3
+  vxlan:
+    iface: 'eth4'
+    inet_type: 'static'
+    address: '10.0.240.147/22'
+    vm_int_iface: vm-br-eth4
+  storage:
+    iface: 'eth5'
+    inet_type: 'static'
+    address: '10.0.244.147/22'
+    vm_int_iface: vm-br-eth5

multi-node-aio/playbooks/inventory/hosts

@@ -10,6 +10,9 @@ mnaio1         ansible_host=10.0.2.1 ansible_connection=local
 infra1         ansible_host=10.0.236.100
 infra2         ansible_host=10.0.236.101
 infra3         ansible_host=10.0.236.102
+ceph1          ansible_host=10.0.236.145
+ceph2          ansible_host=10.0.236.146
+ceph3          ansible_host=10.0.236.147
 cinder1        ansible_host=10.0.236.130
 cinder2        ansible_host=10.0.236.131
 compute1       ansible_host=10.0.236.120
@@ -41,6 +44,9 @@ mnaio1
 # The group "pxe_servers" is used to create PXE specific configurations for
 #  each host machines.
 [pxe_servers]
+ceph1
+ceph2
+ceph3
 cinder1
 cinder2
 compute1
@@ -94,3 +100,33 @@ cinder2
 [compute_hosts]
 compute1
 compute2
+
+# The group "ceph_hosts" is used in the MNAIO setup to create the basic drive
+# setup for a ceph deployment
+[ceph_hosts]
+ceph1
+ceph2
+ceph3
+
+######################## CEPH-ANSIBLE RELATED GROUPINGS ########################
+
+# Used by ceph-ansible to determine which hosts to deploy to
+[ceph]
+ceph1 monitor_address=10.0.244.145
+ceph2 monitor_address=10.0.244.146
+ceph3 monitor_address=10.0.244.147
+
+# Where to install the ceph monitoring service for ceph-ansible
+[mons:children]
+infra_hosts
+
+# Where to install the osd services for ceph-ansible
+[osds:children]
+ceph
+
+# Where to install the radosgw services for ceph-ansible
+[rgws:children]
+ceph
+
+# Where to install the ceph metadata services if needed for ceph-ansible
+[mdss]

multi-node-aio/playbooks/osa/openstack_user_config.yml

@@ -70,7 +70,12 @@ global_overrides:
           - cinder_api
           - cinder_volume
           - nova_compute
+{% if (osa_enable_object_storage | bool) and not (enable_ceph_storage | bool) %}
           - swift_proxy
+{% endif %}
+{% if enable_ceph_storage | bool %}
+          - ceph-osd
+{% endif %}
     - network:
         container_bridge: "br-lbaas"
         container_type: "veth"
@@ -83,7 +88,7 @@ global_overrides:
           - "octavia-worker"
           - "octavia-housekeeping"
           - "octavia-health-monitor"
-{% if osa_enable_object_storage | bool %}
+{% if (osa_enable_object_storage | bool) and not (enable_ceph_storage | bool) %}
   swift:
     part_power: 8
     storage_network: 'br-storage'
@@ -103,7 +108,7 @@ global_overrides:
 ###
 ### Anchors
 ###
-{% if osa_enable_block_storage | bool %}
+{% if (osa_enable_block_storage | bool) and not (enable_ceph_storage | bool) %}
 cinder_block: &cinder_block
 {%   for host in groups['cinder_hosts'] %}
   {{ hostvars[host]['server_hostname'] }}:
@@ -159,7 +164,7 @@ log_block: &log_block
 {% endfor %}
 
 
-{% if osa_enable_object_storage | bool %}
+{% if (osa_enable_object_storage | bool) and not (enable_ceph_storage | bool) %}
 swift_block: &swift_block
 {%   for host in groups['swift_hosts'] %}
   {{ hostvars[host]['server_hostname'] }}:
@@ -170,6 +175,30 @@ swift_block: &swift_block
 {% endif %}
 
 
+{% if enable_ceph_storage | bool %}
+ceph_osds_block: &ceph_osds_block
+{% for host in groups['ceph_hosts'] %}
+  {{ hostvars[host]['server_hostname'] }}:
+    ip: {{ hostvars[host]['server_networks']['mgmt']['address'].split('/')[0] }}
+    container_vars:
+      container_tech: "{{ default_container_tech }}"
+      lvm_volumes:
+        - data: data1
+          data_vg: vmvg00
+          journal: journal1
+          journal_vg: vmvg00
+        - data: data2
+          data_vg: vmvg00
+          journal: journal2
+          journal_vg: vmvg00
+        - data: data3
+          data_vg: vmvg00
+          journal: journal3
+          journal_vg: vmvg00
+{% endfor %}
+{% endif %}
+
+
 ###
 ### Infrastructure
 ###
@@ -188,6 +217,17 @@ log_hosts: *log_block
 haproxy_hosts: *loadbalancer_block
 {% endif %}
 
+{% if enable_ceph_storage | bool %}
+# Ceph Mon Hosts
+ceph-mon_hosts: *infra_block
+
+# Ceph RGW Hosts
+ceph-rgw_hosts: *infra_block
+
+# Ceph OSDs Hosts
+ceph-osd_hosts: *ceph_osds_block
+{% endif %}
+
 ###
 ### OpenStack
 ###
@@ -203,13 +243,15 @@ os-infra_hosts: *infra_block
 identity_hosts: *infra_block
 {% endif %}
 
-{% if osa_enable_block_storage | bool %}
+{% if (osa_enable_block_storage | bool) or (enable_ceph_storage | bool)  %}
 # cinder api services
 storage-infra_hosts: *infra_block
 
+{% if not (enable_ceph_storage | bool) %}
 # cinder storage host (LVM-backed)
 storage_hosts: *cinder_block
 {% endif %}
+{% endif %}
 
 {% if osa_enable_image | bool %}
 # glance
@@ -258,7 +300,7 @@ metrics_hosts: *infra_block
 metering-compute_hosts: *compute_block
 {% endif %}
 
-{% if osa_enable_object_storage | bool %}
+{% if (osa_enable_object_storage | bool) and not (enable_ceph_storage | bool) %}
 # swift storage hosts
 swift_hosts: *swift_block
 

multi-node-aio/playbooks/osa/user_mnaio_variables.yml

@@ -1,5 +1,4 @@
 ---
-
 # Tempest is being configured to use a known network
 tempest_public_subnet_cidr: 10.0.248.0/26
 
@@ -40,6 +39,40 @@ glance_glance_api_conf_overrides:
 galera_innodb_buffer_pool_size: 1024M
 galera_gcache_size: 128M
 
+{% if enable_ceph_storage | bool %}
+### Ceph Config Overrides
+## Ceph cluster fsid (must be generated before first run)
+generate_fsid: true
+
+## ceph-ansible settings
+## See https://github.com/ceph/ceph-ansible/tree/master/group_vars for
+## additional configuration options availble.
+monitor_address_block: 10.0.236.0/22
+public_network: 10.0.236.0/22
+cluster_network: 10.0.244.0/22
+osd_scenario: lvm
+journal_size: {{ ceph_journal_size }}
+# ceph-ansible automatically creates pools & keys for OpenStack services
+openstack_config: true
+cinder_ceph_client: cinder
+glance_ceph_client: glance
+glance_default_store: rbd
+glance_rbd_store_pool: images
+nova_libvirt_images_rbd_pool: vms
+{% raw %}
+cinder_backends:
+  ceph:
+    volume_driver: cinder.volume.drivers.rbd.RBDDriver
+    rbd_pool: volumes
+    rbd_ceph_conf: /etc/ceph/ceph.conf
+    rbd_store_chunk_size: 8
+    volume_backend_name: rbddriver
+    rbd_user: "{{ cinder_ceph_client }}"
+    rbd_secret_uuid: "{{ cinder_ceph_client_uuid }}"
+    report_discard_supported: true
+{% endraw %}
+{% endif %}
+
 {% if osa_enable_elk_metrics | bool %}
 # NOTE(d34dh0r53): The disk detection to determine the logstash_queue_type
 # can fail spectacularly on an MNAIO so I'm hard setting it here to bypass