Integrate gnocchi with ceph

Gnocchi previously lacked high availability. We consider a lack of HA
in our a vast majority of operator oriented services to be a defective
design choice. this change integrates gnocchi with ceph to resolve the
the lack of HA.

Closes-Bug: #1626623
Change-Id: I71c5137842cb48bc4af0e50a2952df5631d0d6df

ansible/group_vars/all.yml

@@ -348,6 +348,13 @@ glance_backend_ceph: "{{ enable_ceph }}"
 ceilometer_database_type: "mongodb"
 
 
+#################
+# Gnocchi options
+#################
+# Vaid options are [file, ceph]
+gnocchi_backend_storage: "{{ 'ceph' if enable_ceph|bool else 'file' }}"
+
+
 #################################
 # Cinder options
 #################################
@@ -391,6 +398,7 @@ ceph_pool_type: "replicated"
 ceph_cinder_pool_name: "volumes"
 ceph_cinder_backup_pool_name: "backups"
 ceph_glance_pool_name: "images"
+ceph_gnocchi_pool_name: "gnocchi"
 ceph_nova_pool_name: "vms"
 
 ceph_erasure_profile: "k=4 m=2 ruleset-failure-domain=host"

ansible/roles/gnocchi/defaults/main.yml

@@ -2,6 +2,19 @@
 project_name: "gnocchi"
 
 
+####################
+# Ceph
+####################
+ceph_gnocchi_pool_type: "{{ ceph_pool_type }}"
+ceph_gnocchi_cache_mode: "{{ ceph_cache_mode }}"
+
+# Due to Ansible issues on include, you cannot override these variables. Please
+# override the variables they reference instead.
+gnocchi_pool_name: "{{ ceph_gnocchi_pool_name }}"
+gnocchi_pool_type: "{{ ceph_gnocchi_pool_type }}"
+gnocchi_cache_mode: "{{ ceph_gnocchi_cache_mode }}"
+
+
 ####################
 # Database
 ####################

ansible/roles/gnocchi/tasks/ceph.yml

@@ -0,0 +1,45 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+  when: inventory_hostname in groups[item]
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-metricd"
+    - "gnocchi-statsd"
+
+- name: Copying over config(s)
+  template:
+    src: roles/ceph/templates/ceph.conf.j2
+    dest: "{{ node_config_directory }}/{{ item }}/ceph.conf"
+  when: inventory_hostname in groups[item]
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-metricd"
+    - "gnocchi-statsd"
+
+- include: ../../ceph_pools.yml
+  vars:
+    pool_name: "{{ gnocchi_pool_name }}"
+    pool_type: "{{ gnocchi_pool_type }}"
+    cache_mode: "{{ gnocchi_cache_mode }}"
+
+# TODO(SamYaple): Improve changed_when tests
+- name: Pulling cephx keyring
+  command: docker exec ceph_mon ceph auth get-or-create client.gnocchi mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ ceph_gnocchi_pool_name }}, allow rwx pool={{ ceph_gnocchi_pool_name }}-cache'
+  register: cephx_key
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  run_once: True
+
+- name: Pushing cephx keyring
+  copy:
+    content: "{{ cephx_key.stdout }}\n\r"
+    dest: "{{ node_config_directory }}/{{ item }}/ceph.client.gnocchi.keyring"
+    mode: "0600"
+  when: inventory_hostname in groups[item]
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-metricd"
+    - "gnocchi-statsd"

ansible/roles/gnocchi/tasks/deploy.yml

@@ -1,4 +1,14 @@
 ---
+- include: ceph.yml
+  when:
+    - enable_ceph|bool
+    - gnocchi_backend_storage == 'ceph'
+
+- include: external_ceph.yml
+  when:
+    - enable_ceph|bool == False
+    - gnocchi_backend_storage == 'ceph'
+
 - include: register.yml
   when: inventory_hostname in groups['gnocchi-api']
 

ansible/roles/gnocchi/tasks/external_ceph.yml

@@ -0,0 +1,30 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+  when: inventory_hostname in groups[item]
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-metricd"
+    - "gnocchi-statsd"
+
+- name: Copy over ceph.conf file
+  copy:
+    src: "{{ node_custom_config }}/{{ item }}/ceph.conf"
+    dest: "{{ node_config_directory }}/{{ item }}/ceph.conf"
+  when: inventory_hostname in groups[item]
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-metricd"
+    - "gnocchi-statsd"
+
+- name: Copy over ceph gnocchi keyring
+  copy:
+    src: "{{ node_custom_config }}/{{ item }}/ceph.client.gnocchi.keyring"
+    dest: "{{ node_config_directory }}/{{ item }}/ceph.client.gnocchi.keryring"
+  when: inventory_hostname in groups[item]
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-metricd"
+    - "gnocchi-statsd"

ansible/roles/gnocchi/templates/gnocchi-api.json.j2

@@ -20,7 +20,19 @@
             "dest": "/etc/{{ gnocchi_dir }}/wsgi-gnocchi.conf",
             "owner": "gnocchi",
             "perm": "0644"
-        }
+        }{% if gnocchi_backend_storage == 'ceph' %},
+        {
+            "source": "{{ container_config_directory }}/ceph.conf",
+            "dest": "/etc/ceph/ceph.conf",
+            "owner": "gnocchi",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/ceph.client.gnocchi.keyring",
+            "dest": "/etc/ceph/ceph.client.gnocchi.keyring",
+            "owner": "gnocchi",
+            "perm": "0600"
+        }{% endif %}
 
     ]
 }

ansible/roles/gnocchi/templates/gnocchi-metricd.json.j2

@@ -6,6 +6,18 @@
             "dest": "/etc/gnocchi/gnocchi.conf",
             "owner": "gnocchi",
             "perm": "0600"
-        }
+        }{% if gnocchi_backend_storage == 'ceph' %},
+        {
+            "source": "{{ container_config_directory }}/ceph.conf",
+            "dest": "/etc/ceph/ceph.conf",
+            "owner": "gnocchi",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/ceph.client.gnocchi.keyring",
+            "dest": "/etc/ceph/ceph.client.gnocchi.keyring",
+            "owner": "gnocchi",
+            "perm": "0600"
+        }{% endif %}
     ]
 }

ansible/roles/gnocchi/templates/gnocchi-statsd.json.j2

@@ -6,6 +6,18 @@
             "dest": "/etc/gnocchi/gnocchi.conf",
             "owner": "gnocchi",
             "perm": "0600"
-        }
+        }{% if gnocchi_backend_storage == 'ceph' %},
+        {
+            "source": "{{ container_config_directory }}/ceph.conf",
+            "dest": "/etc/ceph/ceph.conf",
+            "owner": "gnocchi",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/ceph.client.gnocchi.keyring",
+            "dest": "/etc/ceph/ceph.client.gnocchi.keyring",
+            "owner": "gnocchi",
+            "perm": "0600"
+        }{% endif %}
     ]
 }

ansible/roles/gnocchi/templates/gnocchi.conf.j2

@@ -44,5 +44,13 @@ memcache_secret_key = {{ memcache_secret_key }}
 memcached_servers = {% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}
 
 [storage]
+{% if gnocchi_backend_storage == 'file' %}
 driver = file
 file_basepath = /var/lib/gnocchi
+{% elif gnocchi_backend_storage == 'ceph' %}
+driver = ceph
+ceph_pool = {{ gnocchi_pool_name }}
+ceph_username = gnocchi
+ceph_keyring = /etc/ceph/ceph.client.gnocchi.keyring
+ceph_conffile = /etc/ceph/ceph.conf
+{% endif %}

docker/gnocchi/gnocchi-base/Dockerfile.j2

@@ -13,7 +13,8 @@ MAINTAINER {{ maintainer }}
             'python-gnocchiclient',
             'httpd',
             'mod_wsgi',
-            'python-ldappool'
+            'python-ldappool',
+            'python-rados'
         ] %}
 
 {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }}
@@ -24,7 +25,8 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf
             'gnocchi-common',
             'apache2',
             'libapache2-mod-wsgi',
-            'python-ldappool'
+            'python-ldappool',
+            'python-rados'
         ] %}
 
 {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }}
@@ -37,7 +39,8 @@ RUN truncate -s 0 /etc/apache2/ports.conf
        {% set gnocchi_base_packages = [
             'httpd',
             'mod_wsgi',
-            'python-ldappool'
+            'python-ldappool',
+            'python-rados'
         ] %}
 {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }}
 RUN mkdir -p /var/www/cgi-bin/gnocchi \
@@ -47,7 +50,8 @@ RUN mkdir -p /var/www/cgi-bin/gnocchi \
        {% set gnocchi_base_packages = [
             'apache2',
             'libapache2-mod-wsgi',
-            'python-ldappool'
+            'python-ldappool',
+            'python-rados'
         ] %}
 
 {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }}
@@ -58,7 +62,7 @@ ADD gnocchi-base-archive /gnocchi-base-source
 
 RUN ln -s gnocchi-base-source/* gnocchi \
     && useradd --user-group gnocchi \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt gnocchiclient /gnocchi[keystone,mysql,file] \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt gnocchiclient /gnocchi[keystone,mysql,file,ceph] \
     && mkdir -p /etc/gnocchi /var/log/kolla/gnocchi /home/gnocchi \
     && cp -r /gnocchi/etc/gnocchi/* /etc/gnocchi/ \
     && chown -R gnocchi: /etc/gnocchi /var/log/kolla/gnocchi

etc/kolla/globals.yml

@@ -192,6 +192,14 @@ kolla_internal_vip_address: "10.10.10.254"
 # Valid options are [ mongodb, mysql ]
 #ceilometer_database_type: "mongodb"
 
+
+#######################
+# Gnocchi options
+#######################
+# Valid options are [ file, ceph ]
+#gnocchi_backend_storage: "{{ 'ceph' if enable_ceph|bool else 'file' }}"
+
+
 #######################
 # Cinder options
 #######################

releasenotes/notes/integrate-gnocchi-with-ceph-a6d5f81f4d8b0391.yaml

@@ -0,0 +1,3 @@
+---
+fixes:
+  - Integrates gnocchi with ceph to resolve the the lack of HA.