From c86e66c9dc3595199b4b88a526e2c204c12364c8 Mon Sep 17 00:00:00 2001
From: Sam Yaple <sam@yaple.net>
Date: Sat, 29 Aug 2015 11:49:57 +0000
Subject: [PATCH] Ceph Ansible support

Add the initial playbooks for making ceph and ansible play nice
together.

This does not include all of the openstack changes to make things like
nova, glance, and cinder work. This will simply build the ceph cluster
and thats it. The next patchset will do the OpenStack integration.

DocImpact
Change-Id: Ie1697dde5f92e833652933a80f0004f31b641330
Partially-Implements: blueprint ceph-container
---
 ansible/group_vars/all.yml                    |   1 +
 ansible/inventory/all-in-one                  |   6 +
 ansible/inventory/multinode                   |   6 +
 ansible/library/bslurp.py                     | 189 ++++++++++++++++++
 ansible/library/find_disks.py                 |  86 ++++++++
 ansible/roles/ceph/defaults/main.yml          |  24 +++
 ansible/roles/ceph/tasks/bootstrap_mons.yml   |  57 ++++++
 ansible/roles/ceph/tasks/bootstrap_osds.yml   |  48 +++++
 ansible/roles/ceph/tasks/config.yml           |  22 ++
 .../roles/ceph/tasks/distribute_keyrings.yml  |  34 ++++
 ansible/roles/ceph/tasks/generate_cluster.yml |  30 +++
 ansible/roles/ceph/tasks/main.yml             |  16 ++
 ansible/roles/ceph/tasks/start_mons.yml       |  22 ++
 ansible/roles/ceph/tasks/start_osds.yml       |  50 +++++
 ansible/roles/ceph/templates/ceph.conf.j2     |   9 +
 ansible/site.yml                              |   4 +
 docker/ceph/ceph-base/Dockerfile.j2           |   4 +
 docker/ceph/ceph-mon/Dockerfile.j2            |   1 +
 docker/ceph/ceph-mon/config-external.sh       |  30 +++
 docker/ceph/ceph-mon/fetch_ceph_keys.py       |  67 +++++++
 docker/ceph/ceph-mon/start.sh                 |  12 +-
 docker/ceph/ceph-osd/config-external.sh       |  10 +
 docker/ceph/ceph-osd/start.sh                 |  42 +++-
 docker/openstack-base/Dockerfile.j2           |   3 +-
 etc/kolla/config/ceph.conf                    |   0
 etc/kolla/passwords.yml                       |   7 +
 26 files changed, 761 insertions(+), 19 deletions(-)
 create mode 100644 ansible/library/bslurp.py
 create mode 100644 ansible/library/find_disks.py
 create mode 100644 ansible/roles/ceph/defaults/main.yml
 create mode 100644 ansible/roles/ceph/tasks/bootstrap_mons.yml
 create mode 100644 ansible/roles/ceph/tasks/bootstrap_osds.yml
 create mode 100644 ansible/roles/ceph/tasks/config.yml
 create mode 100644 ansible/roles/ceph/tasks/distribute_keyrings.yml
 create mode 100644 ansible/roles/ceph/tasks/generate_cluster.yml
 create mode 100644 ansible/roles/ceph/tasks/main.yml
 create mode 100644 ansible/roles/ceph/tasks/start_mons.yml
 create mode 100644 ansible/roles/ceph/tasks/start_osds.yml
 create mode 100644 ansible/roles/ceph/templates/ceph.conf.j2
 create mode 100755 docker/ceph/ceph-mon/fetch_ceph_keys.py
 mode change 100644 => 100755 docker/ceph/ceph-mon/start.sh
 mode change 100644 => 100755 docker/ceph/ceph-osd/start.sh
 create mode 100644 etc/kolla/config/ceph.conf

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 3a4419723b..e4e903f9f1 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -140,6 +140,7 @@ enable_nova: "yes"
 enable_rabbitmq: "yes"
 
 # Additional optional OpenStack services are specified here
+enable_ceph: "no"
 enable_cinder: "no"
 enable_heat: "yes"
 enable_horizon: "yes"
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 8b782f72ac..d820c3e606 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -54,6 +54,12 @@ control
 [murano:children]
 control
 
+[ceph-mon:children]
+control
+
+[ceph-osd:children]
+storage
+
 
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 924c4ac844..51f9d055ca 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -62,6 +62,12 @@ control
 [murano:children]
 control
 
+[ceph-mon:children]
+control
+
+[ceph-osd:children]
+storage
+
 
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
diff --git a/ansible/library/bslurp.py b/ansible/library/bslurp.py
new file mode 100644
index 0000000000..485b4c3b46
--- /dev/null
+++ b/ansible/library/bslurp.py
@@ -0,0 +1,189 @@
+#!/usr/bin/python
+
+# Copyright 2015 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This module has been relicensed from the source below:
+# https://github.com/SamYaple/yaodu/blob/master/ansible/library/bslurp
+
+DOCUMENTATION = '''
+---
+module: bslurp
+short_description: Slurps a file from a remote node
+description:
+     - Used for fetching a binary blob containing the file, then push that file
+       to other hosts.
+options:
+  src:
+    description:
+      - File to fetch. When dest is used, src is expected to be a str with data
+    required: True
+    type: str
+  compress:
+    description:
+      - Compress file with zlib
+    default: True
+    type: bool
+  dest:
+    description:
+      - Where to write out binary blob
+    required: False
+    type: str
+  mode:
+    description:
+      - Destination file permissions
+    default: '0644'
+    type: str
+  sha1:
+    description:
+      - sha1 hash of the underlying data
+    default: None
+    type: bool
+author: Sam Yaple
+'''
+
+EXAMPLES = '''
+Distribute a file from single to many host:
+
+- hosts: web_servers
+  tasks:
+    - name: Pull in web config
+      bslurp: src="/path/to/file"
+      register: file_data
+      run_once: True
+    - name: Push if changed
+      bslurp:
+        src: "{{ file_data.content }}"
+        dest: "{{ file_data.source }}"
+        mode: "{{ file_data.mode }}"
+        sha1: "{{ file_data.sha1 }}"
+
+Distribute multiple files from single to many host:
+
+- hosts: web_servers
+  tasks:
+    - name: Pull in web config
+      bslurp: src="{{ item }}"
+      with_items:
+        - "/path/to/file1"
+        - "/path/to/file2"
+        - "/path/to/file3"
+      register: file_data
+      run_once: True
+    - name: Push if changed
+      bslurp:
+        src: "{{ item.content }}"
+        dest: "{{ item.source }}"
+        mode: "{{ item.mode }}"
+        sha1: "{{ item.sha1 }}"
+      with_items: file_data.results
+
+Distribute a to file many host without compression; Change permissions on dest:
+
+- hosts: web_servers
+  tasks:
+    - name: Pull in web config
+      bslurp: src="/path/to/file"
+      register: file_data
+      run_once: True
+    - name: Push if changed
+      bslurp:
+        src: "{{ file_data.content }}"
+        dest: "/new/path/to/file"
+        mode: "0777"
+        compress: False
+        sha1: "{{ file_data.sha1 }}"
+'''
+
+import base64
+import hashlib
+import os
+import sys
+import zlib
+
+def copy_from_host(module):
+    compress = module.params.get('compress')
+    src = module.params.get('src')
+
+    if not os.path.exists(src):
+        module.fail_json(msg="file not found: {}".format(src))
+    if not os.access(src, os.R_OK):
+        module.fail_json(msg="file is not readable: {}".format(src))
+
+    mode = oct(os.stat(src).st_mode & 0777)
+
+    with open(src, 'rb') as f:
+        raw_data = f.read()
+
+    sha1 = hashlib.sha1(raw_data).hexdigest()
+    data = zlib.compress(raw_data) if compress else raw_data
+
+    module.exit_json(content=base64.b64encode(data), sha1=sha1, mode=mode,
+                     source=src)
+
+
+def copy_to_host(module):
+    compress = module.params.get('compress')
+    dest = module.params.get('dest')
+    mode = int(module.params.get('mode'), 0)
+    sha1 = module.params.get('sha1')
+    src = module.params.get('src')
+
+    data = base64.b64decode(src)
+    raw_data = zlib.decompress(data) if compress else data
+
+    if sha1:
+        if os.path.exists(dest):
+            if os.access(dest, os.R_OK):
+                with open(dest, 'rb') as f:
+                    if hashlib.sha1(f.read()).hexdigest() == sha1:
+                        module.exit_json(changed=False)
+            else:
+                module.exit_json(failed=True, changed=False,
+                                 msg='file is not accessible: {}'.format(dest))
+
+        if sha1 != hashlib.sha1(raw_data).hexdigest():
+            module.exit_json(failed=True, changed=False,
+                             msg='sha1 sum does not match data')
+
+    with os.fdopen(os.open(dest, os.O_WRONLY | os.O_CREAT, mode), 'wb') as f:
+        f.write(raw_data)
+
+    module.exit_json(changed=True)
+
+def main():
+    module = AnsibleModule(
+        argument_spec = dict(
+            compress = dict(default=True, type='bool'),
+            dest = dict(type='str'),
+            mode = dict(default='0644', type='str'),
+            sha1 = dict(default=None, type='str'),
+            src = dict(required=True, type='str')
+        )
+    )
+
+    dest = module.params.get('dest')
+
+    try:
+        if dest:
+            copy_to_host(module)
+        else:
+            copy_from_host(module)
+    except Exception as e:
+        module.exit_json(failed=True, changed=True, msg=repr(e))
+
+# import module snippets
+from ansible.module_utils.basic import *
+if __name__ == '__main__':
+    main()
diff --git a/ansible/library/find_disks.py b/ansible/library/find_disks.py
new file mode 100644
index 0000000000..fe2a638b25
--- /dev/null
+++ b/ansible/library/find_disks.py
@@ -0,0 +1,86 @@
+#!/usr/bin/python
+
+# Copyright 2015 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This module has been relicensed from the source below:
+# https://github.com/SamYaple/yaodu/blob/master/ansible/library/ceph_osd_list
+
+DOCUMENTATION = '''
+---
+module: find_disks
+short_description: Return list of devices containing a specfied label
+description:
+     - This will return a list of all devices with a GPT partition label with
+       the name specified.
+options:
+  partition_name:
+    description:
+      - Partition name
+    required: True
+    type: bool
+author: Sam Yaple
+'''
+
+EXAMPLES = '''
+- hosts: ceph-osd
+  tasks:
+    - name: Return all valid formated devices with the name KOLLA_CEPH_OSD
+      ceph_osd_list:
+          partition_name: 'KOLLA_CEPH_OSD'
+      register: osds
+'''
+
+import sys
+import subprocess
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec = dict(
+            partition_name = dict(required=True, type='str')
+        )
+    )
+
+    partition_name = module.params.get('partition_name')
+
+    try:
+        # This should all really be done differently. Unfortunately there is no
+        # decent python library for dealing with disks like we need to here.
+        disks = subprocess.check_output("parted -l", shell=True).split('\n')
+        ret = list()
+
+        for line in disks:
+            d = line.split(' ')
+            if d[0] == 'Disk':
+                dev = d[1][:-1]
+
+            if line.find(partition_name) != -1:
+                # This process returns an error code when no results return
+                # We can ignore that, it is safe
+                p = subprocess.Popen("blkid " + dev + "*", shell=True, stdout=subprocess.PIPE)
+                fs_uuid = p.communicate()[0]
+                # The dev doesn't need to have a uuid, will be '' otherwise
+                if fs_uuid:
+                    fs_uuid = fs_uuid.split('"')[1]
+                ret.append({'device': dev, 'fs_uuid': fs_uuid})
+
+        module.exit_json(disks=ret)
+    except Exception as e:
+        module.exit_json(failed=True, msg=repr(e))
+
+# import module snippets
+from ansible.module_utils.basic import *
+if __name__ == '__main__':
+    main()
diff --git a/ansible/roles/ceph/defaults/main.yml b/ansible/roles/ceph/defaults/main.yml
new file mode 100644
index 0000000000..08fffde130
--- /dev/null
+++ b/ansible/roles/ceph/defaults/main.yml
@@ -0,0 +1,24 @@
+---
+project_name: "ceph"
+
+
+####################
+# Docker
+####################
+ceph_mon_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceph-mon"
+ceph_mon_tag: "{{ openstack_release }}"
+ceph_mon_image_full: "{{ ceph_mon_image }}:{{ ceph_mon_tag }}"
+
+ceph_osd_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceph-osd"
+ceph_osd_tag: "{{ openstack_release }}"
+ceph_osd_image_full: "{{ ceph_osd_image }}:{{ ceph_osd_tag }}"
+
+ceph_data_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-data"
+ceph_data_tag: "{{ openstack_release }}"
+ceph_data_image_full: "{{ ceph_data_image }}:{{ ceph_data_tag }}"
+
+
+####################
+# Ceph
+####################
+osd_initial_weight: "1"
diff --git a/ansible/roles/ceph/tasks/bootstrap_mons.yml b/ansible/roles/ceph/tasks/bootstrap_mons.yml
new file mode 100644
index 0000000000..853056b885
--- /dev/null
+++ b/ansible/roles/ceph/tasks/bootstrap_mons.yml
@@ -0,0 +1,57 @@
+---
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_ceph_cluster state=absent
+  changed_when: False
+  always_run: True
+  run_once: True
+
+- name: Creating temp file on localhost
+  local_action: copy content=None dest=/tmp/kolla_ceph_cluster mode=0600
+  changed_when: False
+  always_run: True
+  run_once: True
+
+# TODO(SamYaple): Improve failed_when check
+- name: Checking if a previous cluster exists
+  command: docker exec ceph_mon_data stat /etc/ceph/ceph.monmap
+  register: exists
+  changed_when: False
+  failed_when: False
+  always_run: True
+
+- name: Writing hostname of host with existing cluster files to temp file
+  local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_ceph_cluster mode=0600
+  changed_when: False
+  always_run: True
+  when: exists.rc == 0
+
+- name: Registering host from temp file
+  set_fact:
+    delegate_host: "{{ lookup('file', '/tmp/kolla_ceph_cluster') }}"
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_ceph_cluster state=absent
+  changed_when: False
+  always_run: True
+  run_once: True
+
+- name: Starting Ceph Monitor data container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: ceph_mon_data
+    image: "{{ ceph_data_image_full }}"
+    volumes:
+      - "/etc/ceph/"
+      - "/var/lib/ceph/"
+
+- include: generate_cluster.yml
+  when: delegate_host == 'None' and inventory_hostname == groups['ceph-mon'][0]
diff --git a/ansible/roles/ceph/tasks/bootstrap_osds.yml b/ansible/roles/ceph/tasks/bootstrap_osds.yml
new file mode 100644
index 0000000000..59163f8d39
--- /dev/null
+++ b/ansible/roles/ceph/tasks/bootstrap_osds.yml
@@ -0,0 +1,48 @@
+---
+- name: Looking up disks to bootstrap for Ceph
+  find_disks:
+      partition_name: 'KOLLA_CEPH_OSD_BOOTSTRAP'
+  register: osds_bootstrap
+  when: inventory_hostname in groups['ceph-osd']
+
+- name: Bootstrapping Ceph OSDs
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "no"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    privileged: True
+    name: "bootstrap_osd_{{ item.0 }}"
+    image: "{{ ceph_osd_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/ceph-osd/:/opt/kolla/ceph-osd/:ro"
+      - "/dev/:/dev/"
+    env:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      OSD_DEV: "{{ item.1.device }}"
+      OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
+  with_indexed_items: osds_bootstrap['disks']|default([])
+  when: inventory_hostname in groups['ceph-osd']
+
+# https://github.com/ansible/ansible-modules-core/pull/1031
+- name: Waiting for bootstrap containers to exit
+  command: docker wait "bootstrap_osd_{{ item.0 }}"
+  register: bootstrap_result
+  run_once: True
+  failed_when: bootstrap_result.stdout != "0"
+  with_indexed_items: osds_bootstrap['disks']|default([])
+  when: inventory_hostname in groups['ceph-osd']
+
+- name: Cleaning up bootstrap containers
+  docker:
+    name: "bootstrap_osd_{{ item.0 }}"
+    image: "{{ ceph_osd_image_full }}"
+    state: absent
+  with_indexed_items: osds_bootstrap['disks']|default([])
+  when: inventory_hostname in groups['ceph-osd']
diff --git a/ansible/roles/ceph/tasks/config.yml b/ansible/roles/ceph/tasks/config.yml
new file mode 100644
index 0000000000..d242ea80c8
--- /dev/null
+++ b/ansible/roles/ceph/tasks/config.yml
@@ -0,0 +1,22 @@
+---
+- include: ../../config.yml
+  vars:
+    service_name: "ceph-mon"
+    config_source:
+      - "roles/ceph/templates/ceph.conf.j2"
+      - "/etc/kolla/config/ceph.conf"
+    config_template_dest:
+      - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_minimal"
+      - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_augment"
+    config_dest: "{{ node_config_directory }}/{{ service_name }}/ceph.conf"
+
+- include: ../../config.yml
+  vars:
+    service_name: "ceph-osd"
+    config_source:
+      - "roles/ceph/templates/ceph.conf.j2"
+      - "/etc/kolla/config/ceph.conf"
+    config_template_dest:
+      - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_minimal"
+      - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_augment"
+    config_dest: "{{ node_config_directory }}/{{ service_name }}/ceph.conf"
diff --git a/ansible/roles/ceph/tasks/distribute_keyrings.yml b/ansible/roles/ceph/tasks/distribute_keyrings.yml
new file mode 100644
index 0000000000..10ec0a8ddf
--- /dev/null
+++ b/ansible/roles/ceph/tasks/distribute_keyrings.yml
@@ -0,0 +1,34 @@
+---
+- name: Fetching Ceph keyrings
+  command: docker exec ceph_mon fetch_ceph_keys.py
+  register: ceph_files_json
+  changed_when: "{{ (ceph_files_json.stdout | from_json).changed }}"
+  failed_when: "{{ (ceph_files_json.stdout | from_json).failed }}"
+  delegate_to: "{{ delegate_host }}"
+  run_once: True
+
+- name: Reading json from variable
+  set_fact:
+    ceph_files: "{{ (ceph_files_json.stdout | from_json) }}"
+
+- name: Pushing Ceph keyring for OSDs
+  bslurp:
+    src: "{{ item.content }}"
+    dest: "{{ node_config_directory }}/ceph-osd/{{ item.filename }}"
+    mode: 0600
+    sha1: "{{ item.sha1 }}"
+  with_items:
+    - "{{ ceph_files['ceph.client.admin.keyring'] }}"
+  when: inventory_hostname in groups['ceph-osd']
+
+- name: Pushing Ceph keyrings for Mons
+  bslurp:
+    src: "{{ item.content }}"
+    dest: "{{ node_config_directory }}/ceph-mon/{{ item.filename }}"
+    mode: 0600
+    sha1: "{{ item.sha1 }}"
+  with_items:
+    - "{{ ceph_files['ceph.client.admin.keyring'] }}"
+    - "{{ ceph_files['ceph.client.mon.keyring'] }}"
+    - "{{ ceph_files['ceph.monmap'] }}"
+  when: inventory_hostname in groups['ceph-mon']
diff --git a/ansible/roles/ceph/tasks/generate_cluster.yml b/ansible/roles/ceph/tasks/generate_cluster.yml
new file mode 100644
index 0000000000..1a0f16b1b8
--- /dev/null
+++ b/ansible/roles/ceph/tasks/generate_cluster.yml
@@ -0,0 +1,30 @@
+---
+- name: Generating Initial Ceph keyrings and monmap
+  docker:
+    detach: False
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "no"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: ceph_mon
+    image: "{{ ceph_mon_image_full }}"
+    volumes: "{{ node_config_directory }}/ceph-mon/:/opt/kolla/ceph-mon/:ro"
+    volumes_from:
+      - "ceph_mon_data"
+    env:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+
+- name: Waiting for a few settings for cluster to generate keys
+  command: sleep 3
+  changed_when: False
+
+- name: Setting host for cluster files
+  set_fact:
+    delegate_host: "{{ ansible_hostname }}"
diff --git a/ansible/roles/ceph/tasks/main.yml b/ansible/roles/ceph/tasks/main.yml
new file mode 100644
index 0000000000..63dca3460f
--- /dev/null
+++ b/ansible/roles/ceph/tasks/main.yml
@@ -0,0 +1,16 @@
+---
+- include: config.yml
+
+- include: bootstrap_mons.yml
+  when: inventory_hostname in groups['ceph-mon']
+
+- include: distribute_keyrings.yml
+
+- include: start_mons.yml
+  when: inventory_hostname in groups['ceph-mon']
+
+- include: bootstrap_osds.yml
+  when: inventory_hostname in groups['ceph-osd']
+
+- include: start_osds.yml
+  when: inventory_hostname in groups['ceph-osd']
diff --git a/ansible/roles/ceph/tasks/start_mons.yml b/ansible/roles/ceph/tasks/start_mons.yml
new file mode 100644
index 0000000000..ab50d73554
--- /dev/null
+++ b/ansible/roles/ceph/tasks/start_mons.yml
@@ -0,0 +1,22 @@
+---
+- name: Starting ceph-mon container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: ceph_mon
+    image: "{{ ceph_mon_image_full }}"
+    volumes: "{{ node_config_directory }}/ceph-mon/:/opt/kolla/ceph-mon/:ro"
+    volumes_from:
+      - "ceph_mon_data"
+    env:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+  when: inventory_hostname in groups['ceph-mon']
diff --git a/ansible/roles/ceph/tasks/start_osds.yml b/ansible/roles/ceph/tasks/start_osds.yml
new file mode 100644
index 0000000000..593ec7b15b
--- /dev/null
+++ b/ansible/roles/ceph/tasks/start_osds.yml
@@ -0,0 +1,50 @@
+---
+- name: Looking up OSDs for Ceph
+  find_disks:
+      partition_name: 'KOLLA_CEPH_DATA'
+  register: osds
+
+- name: Mounting Ceph OSD volumes
+  mount:
+    src: "UUID={{ item.fs_uuid }}"
+    fstype: xfs
+    state: mounted
+    name: "/var/lib/ceph/osd/{{ item.fs_uuid }}"
+  with_items: osds.disks
+
+- name: Gathering OSD IDs
+  command: 'cat /var/lib/ceph/osd/{{ item.fs_uuid }}/whoami'
+  with_items: osds.disks
+  register: id
+  changed_when: False
+  failed_when: id.rc != 0
+
+- name: Starting ceph-osds container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    pid: host
+    privileged: True
+    name: "ceph_osd_{{ item.0.stdout }}"
+    image: "{{ ceph_osd_image_full }}"
+    volumes:
+      - "/var/lib/ceph/osd/{{ item.1.fs_uuid }}:/var/lib/ceph/osd/ceph-{{ item.0.stdout }}"
+      - "{{ node_config_directory }}/ceph-osd/:/opt/kolla/ceph-osd/:ro"
+      - "/dev/:/dev/"
+    env:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      OSD_ID: "{{ item.0.stdout }}"
+      OSD_DEV: "{{ item.1.device }}"
+  with_together:
+    - id.results
+    - osds.disks
+  when: inventory_hostname in groups['ceph-osd']
+        and osds.disks
diff --git a/ansible/roles/ceph/templates/ceph.conf.j2 b/ansible/roles/ceph/templates/ceph.conf.j2
new file mode 100644
index 0000000000..9518742d73
--- /dev/null
+++ b/ansible/roles/ceph/templates/ceph.conf.j2
@@ -0,0 +1,9 @@
+[global]
+fsid = {{ ceph_cluster_fsid }}
+mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_hostname'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+
+mon host = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+
+auth cluster required = cephx
+auth service required = cephx
+auth client required = cephx
diff --git a/ansible/site.yml b/ansible/site.yml
index 5052000b67..3ba043b8ef 100755
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -1,4 +1,8 @@
 ---
+- hosts: [ceph-mon, ceph-osd]
+  roles:
+    - { role: ceph, tags: ceph, when: enable_ceph | bool }
+
 - hosts: [haproxy, mariadb, rabbitmq, cinder-api, glance-api, keystone, nova-api, neutron-server, swift-proxy-server]
   roles:
     - { role: haproxy, tags: haproxy, when: enable_haproxy | bool }
diff --git a/docker/ceph/ceph-base/Dockerfile.j2 b/docker/ceph/ceph-base/Dockerfile.j2
index b18890b5d3..363346bce7 100644
--- a/docker/ceph/ceph-base/Dockerfile.j2
+++ b/docker/ceph/ceph-base/Dockerfile.j2
@@ -15,3 +15,7 @@ RUN apt-get install -y --no-install-recommends \
     && apt-get clean
 
 {% endif %}
+
+RUN useradd --user-group ceph \
+    && mkdir -p /home/ceph \
+    && chown -R ceph: /home/ceph
diff --git a/docker/ceph/ceph-mon/Dockerfile.j2 b/docker/ceph/ceph-mon/Dockerfile.j2
index e9d47fc708..e8a1337ac9 100644
--- a/docker/ceph/ceph-mon/Dockerfile.j2
+++ b/docker/ceph/ceph-mon/Dockerfile.j2
@@ -3,5 +3,6 @@ MAINTAINER Kolla Project (https://launchpad.net/kolla)
 
 COPY start.sh /
 COPY config-external.sh /opt/kolla/
+COPY fetch_ceph_keys.py /usr/bin/
 
 CMD ["/start.sh"]
diff --git a/docker/ceph/ceph-mon/config-external.sh b/docker/ceph/ceph-mon/config-external.sh
index 75488239c7..289cffc86b 100644
--- a/docker/ceph/ceph-mon/config-external.sh
+++ b/docker/ceph/ceph-mon/config-external.sh
@@ -8,3 +8,33 @@ if [[ -f "$SOURCE" ]]; then
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.client.admin.keyring"
+TARGET="/etc/ceph/ceph.client.admin.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.client.mon.keyring"
+TARGET="/etc/ceph/ceph.client.mon.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
+
+SOURCE="/opt/kolla/ceph-mon/ceph.monmap"
+TARGET="/etc/ceph/ceph.monmap"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
diff --git a/docker/ceph/ceph-mon/fetch_ceph_keys.py b/docker/ceph/ceph-mon/fetch_ceph_keys.py
new file mode 100755
index 0000000000..ab2be9c16f
--- /dev/null
+++ b/docker/ceph/ceph-mon/fetch_ceph_keys.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+# Copyright 2015 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a stripped down version of an ansible module I wrote in Yaodu to
+# achieve the same goals we have for Kolla. I have relicensed it for Kolla
+# https://github.com/SamYaple/yaodu/blob/master/ansible/library/bslurp
+
+# Basically this module will fetch the admin and mon keyrings as well as the
+# monmap file. It then hashes the content, compresses them, and finally it
+# converts them to base64 to be safely transported around with ansible
+
+import base64
+import hashlib
+import json
+import os
+import sys
+import zlib
+
+
+def json_exit(msg=None, failed=False, changed=False):
+    if type(msg) is not dict:
+        msg = {'msg': str(msg)}
+    msg.update({'failed': failed, 'changed': changed})
+    print(json.dumps(msg))
+    sys.exit()
+
+
+def read_file(filename):
+    filename_path = os.path.join('/etc/ceph', filename)
+
+    if not os.path.exists(filename_path):
+        json_exit("file not found: {}".format(filename_path), failed=True)
+    if not os.access(filename_path, os.R_OK):
+        json_exit("file not readable: {}".format(filename_path), failed=True)
+
+    with open(filename_path, 'rb') as f:
+        raw_data = f.read()
+
+    return {'content': base64.b64encode(zlib.compress(raw_data)),
+            'sha1': hashlib.sha1(raw_data).hexdigest(),
+            'filename': filename}
+
+
+def main():
+    admin_keyring = 'ceph.client.admin.keyring'
+    mon_keyring = 'ceph.client.mon.keyring'
+    monmap = 'ceph.monmap'
+
+    files = [admin_keyring, mon_keyring, monmap]
+    json_exit({filename: read_file(filename) for filename in files})
+
+
+if __name__ == '__main__':
+    main()
diff --git a/docker/ceph/ceph-mon/start.sh b/docker/ceph/ceph-mon/start.sh
old mode 100644
new mode 100755
index eb5d7f5afd..3afaa3ad40
--- a/docker/ceph/ceph-mon/start.sh
+++ b/docker/ceph/ceph-mon/start.sh
@@ -3,11 +3,11 @@
 set -o errexit
 
 CMD="/usr/bin/ceph-mon"
-ARGS="-d -i ${MON_NAME} --public-addr ${MON_IP}:6789"
+ARGS="-d -i $(hostname) --public-addr ${MON_IP}:6789"
 
 # Setup common paths
-KEYRING_ADMIN="/etc/ceph/ceph.admin.keyring"
-KEYRING_MON="/etc/ceph/ceph.mon.keyring"
+KEYRING_ADMIN="/etc/ceph/ceph.client.admin.keyring"
+KEYRING_MON="/etc/ceph/ceph.client.mon.keyring"
 MONMAP="/etc/ceph/ceph.monmap"
 MON_DIR="/var/lib/ceph/mon/ceph-$(hostname)"
 
@@ -21,7 +21,7 @@ set_configs
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     # Lookup our fsid from the ceph.conf
-    FSID="$(awk '/^fsid/ {print $3; exit}' ${ceph_conf})"
+    FSID="$(awk '/^fsid/ {print $3; exit}' /etc/ceph/ceph.conf)"
 
     # Generating initial keyrings and monmap
     ceph-authtool --create-keyring "${KEYRING_MON}" --gen-key -n mon. --cap mon 'allow *'
@@ -29,8 +29,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_ADMIN}"
     monmaptool --create --add "$(hostname)" "${MON_IP}" --fsid "${FSID}" "${MONMAP}"
 
-    # TODO(SamYaple): Return json parsible output to ansible
-    exit 0
+    echo "Sleeping until keys are fetched"
+    /bin/sleep infinity
 fi
 
 # This section runs on every mon that does not have a keyring already.
diff --git a/docker/ceph/ceph-osd/config-external.sh b/docker/ceph/ceph-osd/config-external.sh
index 8e17487edb..8928f91cbd 100644
--- a/docker/ceph/ceph-osd/config-external.sh
+++ b/docker/ceph/ceph-osd/config-external.sh
@@ -8,3 +8,13 @@ if [[ -f "$SOURCE" ]]; then
     chown ${OWNER}: $TARGET
     chmod 0644 $TARGET
 fi
+
+SOURCE="/opt/kolla/ceph-osd/ceph.client.admin.keyring"
+TARGET="/etc/ceph/ceph.client.admin.keyring"
+OWNER="ceph"
+
+if [[ -f "$SOURCE" ]]; then
+    cp $SOURCE $TARGET
+    chown ${OWNER}: $TARGET
+    chmod 0600 $TARGET
+fi
diff --git a/docker/ceph/ceph-osd/start.sh b/docker/ceph/ceph-osd/start.sh
old mode 100644
new mode 100755
index cf9114a62e..84549ae8c3
--- a/docker/ceph/ceph-osd/start.sh
+++ b/docker/ceph/ceph-osd/start.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
+set -o xtrace
 set -o errexit
 
-CMD="/usr/bin/ceph-osd"
-ARGS="-f -d -i ${OSD_ID} --osd-journal ${OSD_DIR}/journal -k ${OSD_DIR}/keyring"
-
 # Loading common functions.
 source /opt/kolla/kolla-common.sh
 
@@ -13,23 +11,45 @@ set_configs
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
-    # Creating a new label for the disk
-    parted "${OSD_DEV}" -s -- mklabel gpt
+    JOURNAL_PARTUUID=$(uuidgen)
+
+    # Formating disk for ceph
+    sgdisk --zap-all -- "${OSD_DEV}"
+    sgdisk --new=2:1M:5G --change-name=2:KOLLA_CEPH_JOURNAL --typecode=2:45B0969E-9B03-4F30-B4C6-B4B80CEFF106 --partition-guid=2:${JOURNAL_PARTUUID} --mbrtogpt -- "${OSD_DEV}"
+    sgdisk --largest-new=1 --change-name=1:KOLLA_CEPH_DATA --typecode=1:4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D -- "${OSD_DEV}"
+    # This command may throw errors that we can safely ignore
+    partprobe || true
+
+    # We look up the appropriate device path with partition.
+    OSD_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?1)"
+    JOURNAL_PARTITION="${OSD_PARTITION%?}2"
 
-    # Preparing the OSD for use with Ceph
-    ceph-disk prepare "${OSD_DEV}"
     OSD_ID="$(ceph osd create)"
     OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
     mkdir -p "${OSD_DIR}"
-    mount "${OSD_DEV}1" "${OSD_DIR}"
-    ceph-osd -i "${OSD_ID}" --mkfs --mkkey
-    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow proflie osd' -i "${OSD_DIR}/keyring"
+    mkfs.xfs -f "${OSD_PARTITION}"
+    mount "${OSD_PARTITION}" "${OSD_DIR}"
 
-    # Adding osd to crush map
+    # This will through an error about no key existing. That is normal. It then
+    # creates the key in the next step.
+    ceph-osd -i "${OSD_ID}" --mkfs --osd-journal="${JOURNAL_PARTITION}" --mkkey
+    ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring"
+    umount "${OSD_PARTITION}"
+
+    # These commands only need to be run once per host but are safe to run
+    # repeatedly. This can be improved later or if any problems arise.
     ceph osd crush add-bucket "$(hostname)" host
     ceph osd crush move "$(hostname)" root=default
+
+    # Adding osd to crush map
     ceph osd crush add "${OSD_ID}" "${OSD_INITIAL_WEIGHT}" host="$(hostname)"
     exit 0
 fi
 
+# We look up the appropriate journal since we cannot rely on symlinks
+JOURNAL_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?2)"
+OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
+CMD="/usr/bin/ceph-osd"
+ARGS="-f -d -i ${OSD_ID} --osd-journal ${JOURNAL_PARTITION} -k ${OSD_DIR}/keyring"
+
 exec $CMD $ARGS
diff --git a/docker/openstack-base/Dockerfile.j2 b/docker/openstack-base/Dockerfile.j2
index 1880512afc..f7dec9e219 100644
--- a/docker/openstack-base/Dockerfile.j2
+++ b/docker/openstack-base/Dockerfile.j2
@@ -40,7 +40,6 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
         python-tuskarclient \
         python-zaqarclient \
         python-openstackclient \
-        MySQL-python \
-        numpy
+        MySQL-python
 
 {% endif %}
diff --git a/etc/kolla/config/ceph.conf b/etc/kolla/config/ceph.conf
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml
index 3d7ec924fa..c2c077b1c5 100644
--- a/etc/kolla/passwords.yml
+++ b/etc/kolla/passwords.yml
@@ -3,6 +3,12 @@
 # Ansible vault for locking down the secrets properly.
 
 
+###################
+# Ceph options
+####################
+ceph_cluster_fsid: "5fba2fbc-551d-11e5-a8ce-01ef4c5cf93c"
+
+
 ###################
 # Database options
 ####################
@@ -46,6 +52,7 @@ heat_domain_admin_password: "password"
 murano_database_password: "password"
 murano_keystone_password: "password"
 
+
 ####################
 # RabbitMQ options
 ####################