Support mon and osd to be named with hostname

In the current deployment of ceph, the node name of osd and the name
of mon are both IP, and other daemons use hostname.

This commit adds support for naming mon and osd nodes using hostname,
and does not change the default ip-named way.

Change-Id: I22bef72dcd8fc8bcd391ae30e4643520250fd556

ansible/group_vars/all.yml

@@ -68,6 +68,9 @@ container_proxy:
 # to the api_interface.  Allow the bind address to be an override.
 api_interface_address: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
 
+# This is used to get the ip corresponding to the storage_interface.
+storage_interface_address: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+
 ################
 # Chrony options
 ################
@@ -968,6 +971,16 @@ ceph_pool_pgp_num: 8
 # Valid options are [ filestore, bluestore]
 ceph_osd_store_type: "bluestore"
 
+# Set the host type for ceph daemons
+# Valid options are [ IP, HOSTNAME, FQDN, INVENTORY ]
+# Note: For existing clusters, please don't modify this parameter. Otherwise,
+# the existing mon will be invalidated, and the existing osd crush map will
+# be changed.
+ceph_mon_host_type: "IP"
+ceph_mgr_host_type: "INVENTORY"
+ceph_osd_host_type: "IP"
+ceph_mds_host_type: "INVENTORY"
+
 #####################
 # VMware support
 ######################

ansible/roles/ceph/defaults/main.yml

@@ -77,6 +77,27 @@ partition_name_osd_bootstrap: "{{ 'KOLLA_CEPH_OSD_BOOTSTRAP_BS' if ceph_osd_stor
 partition_name_cache_bootstrap: "{{ 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' }}"
 partition_name_osd_data: "{{ 'KOLLA_CEPH_DATA_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_DATA' }}"
 
+ceph_mon_hostname: "{%- if ceph_mon_host_type == 'HOSTNAME' -%}{{ ansible_hostname }}
+                    {%- elif ceph_mon_host_type == 'FQDN' -%}{{ ansible_fqdn }}
+                    {%- elif ceph_mon_host_type == 'INVENTORY' -%}{{ inventory_hostname }}
+                    {%- else -%}{{ storage_interface_address }}
+                    {%- endif %}"
+ceph_mgr_hostname: "{%- if ceph_mgr_host_type == 'HOSTNAME' -%}{{ ansible_hostname }}
+                    {%- elif ceph_mgr_host_type == 'FQDN' -%}{{ ansible_fqdn }}
+                    {%- elif ceph_mgr_host_type == 'INVENTORY' -%}{{ inventory_hostname }}
+                    {%- else -%}{{ storage_interface_address }}
+                    {%- endif %}"
+ceph_osd_hostname: "{%- if ceph_osd_host_type == 'HOSTNAME' -%}{{ ansible_hostname }}
+                    {%- elif ceph_osd_host_type == 'FQDN' -%}{{ ansible_fqdn }}
+                    {%- elif ceph_osd_host_type == 'INVENTORY' -%}{{ inventory_hostname }}
+                    {%- else -%}{{ storage_interface_address }}
+                    {%- endif %}"
+ceph_mds_hostname: "{%- if ceph_mds_host_type == 'HOSTNAME' -%}{{ ansible_hostname }}
+                    {%- elif ceph_mds_host_type == 'FQDN' -%}{{ ansible_fqdn }}
+                    {%- elif ceph_mds_host_type == 'INVENTORY' -%}{{ inventory_hostname }}
+                    {%- else -%}{{ storage_interface_address }}
+                    {%- endif %}"
+
 ####################
 ## Ceph_rgw_keystone
 ####################

ansible/roles/ceph/tasks/bootstrap_osds.yml

@@ -59,7 +59,7 @@
       USE_EXTERNAL_JOURNAL: "{{ item.1.external_journal | bool }}"
       OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}"
       OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
-      HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      HOSTNAME: "{{ ceph_osd_hostname }}"
       OSD_STORETYPE: "{{ ceph_osd_store_type }}"
       OSD_BS_DEV: "{{ item.1.device | default('') }}"
       OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}"
@@ -120,7 +120,7 @@
       USE_EXTERNAL_JOURNAL: "{{ item.1.external_journal | bool }}"
       OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}"
       OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
-      HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      HOSTNAME: "{{ ceph_osd_hostname }}"
       OSD_STORETYPE: "{{ ceph_osd_store_type }}"
       OSD_BS_DEV: "{{ item.1.device | default('') }}"
       OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}"

ansible/roles/ceph/tasks/generate_cluster.yml

@@ -8,8 +8,8 @@
     environment:
       KOLLA_BOOTSTRAP:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      MON_IP: "{{ storage_interface_address }}"
-      HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      HOSTNAME: "{{ ceph_mon_hostname }}"
     image: "{{ ceph_mon_image_full }}"
     labels:
       BOOTSTRAP:

ansible/roles/ceph/tasks/start_mdss.yml

@@ -22,24 +22,19 @@
 - name: Geting ceph mds keyring
   become: true
   kolla_ceph_keyring:
-    name: "mds.{{ hostvars[item]['inventory_hostname'] }}"
+    name: "mds.{{ ceph_mds_hostname }}"
     caps: "{{ ceph_client_mds_keyring_caps }}"
   register: ceph_mds_auth
-  run_once: true
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  with_items: "{{ groups['ceph-mds'] }}"
 
 - name: Pushing ceph mds keyring to ceph-mds
   become: true
   copy:
     content: |
-      [mds.{{ item.item }}]
+      [mds.{{ ceph_mds_hostname }}]
-          key = {{ item.keyring.key }}
+          key = {{ ceph_mds_auth.keyring.key }}
-    dest: "{{ node_config_directory }}/ceph-mds/ceph.mds.{{ inventory_hostname }}.keyring"
+    dest: "{{ node_config_directory }}/ceph-mds/ceph.mds.{{ ceph_mds_hostname }}.keyring"
     mode: "0600"
-  when:
-    - inventory_hostname == item.item
-  with_items: "{{ ceph_mds_auth.results }}"
 
 - name: Starting ceph-mds container
   become: true

ansible/roles/ceph/tasks/start_mgrs.yml

@@ -2,24 +2,19 @@
 - name: Getting ceph mgr keyring
   become: true
   kolla_ceph_keyring:
-    name: "mgr.{{ item }}"
+    name: "mgr.{{ ceph_mgr_hostname }}"
     caps: "{{ ceph_client_mgr_keyring_caps }}"
   register: ceph_mgr_keyring
-  run_once: true
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  with_items: "{{ groups['ceph-mgr'] }}"
 
 - name: Pushing ceph mgr keyring to ceph-mgr
   become: true
   copy:
     content: |
-      [mgr.{{ item.item }}]
+      [mgr.{{ ceph_mgr_hostname }}]
-          key = {{ item.keyring.key }}
+          key = {{ ceph_mgr_keyring.keyring.key }}
-    dest: "{{ node_config_directory }}/ceph-mgr/ceph.mgr.{{ inventory_hostname }}.keyring"
+    dest: "{{ node_config_directory }}/ceph-mgr/ceph.mgr.{{ ceph_mgr_hostname }}.keyring"
     mode: "0600"
-  when:
-    - inventory_hostname == item.item
-  with_items: "{{ ceph_mgr_keyring.results }}"
 
 - name: Starting ceph-mgr container
   become: true

ansible/roles/ceph/tasks/start_mons.yml

@@ -6,7 +6,7 @@
     common_options: "{{ docker_common_options }}"
     environment:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      HOSTNAME: "{{ ceph_mon_hostname }}"
     image: "{{ ceph_mon_image_full }}"
     name: "ceph_mon"
     volumes:

ansible/roles/ceph/templates/ceph-mds.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/usr/bin/ceph-mds -f -c /etc/ceph/ceph.conf -i {{ inventory_hostname }}",
+    "command": "/usr/bin/ceph-mds -f -c /etc/ceph/ceph.conf -i {{ ceph_mds_hostname }}",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/ceph.conf",
@@ -8,8 +8,8 @@
             "perm": "0600"
         },
         {
-            "source": "{{ container_config_directory }}/ceph.mds.{{ inventory_hostname }}.keyring",
+            "source": "{{ container_config_directory }}/ceph.mds.{{ ceph_mds_hostname }}.keyring",
-            "dest": "/var/lib/ceph/mds/ceph-{{ inventory_hostname }}/keyring",
+            "dest": "/var/lib/ceph/mds/ceph-{{ ceph_mds_hostname }}/keyring",
             "owner": "root",
             "perm": "0644"
         }

ansible/roles/ceph/templates/ceph-mgr.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/usr/bin/ceph-mgr -f -i {{ inventory_hostname }}",
+    "command": "/usr/bin/ceph-mgr -f -i {{ ceph_mgr_hostname }}",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/ceph.conf",
@@ -8,8 +8,8 @@
             "perm": "0600"
         },
         {
-            "source": "{{ container_config_directory }}/ceph.mgr.{{ inventory_hostname }}.keyring",
+            "source": "{{ container_config_directory }}/ceph.mgr.{{ ceph_mgr_hostname }}.keyring",
-            "dest": "/var/lib/ceph/mgr/ceph-{{ inventory_hostname }}/keyring",
+            "dest": "/var/lib/ceph/mgr/ceph-{{ ceph_mgr_hostname }}/keyring",
             "owner": "ceph",
             "perm": "0600"
         },

ansible/roles/ceph/templates/ceph-mon.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/usr/bin/ceph-mon -f {% if ceph_debug %}-d{% endif %} -i {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }} --public-addr {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}:6789",
+    "command": "/usr/bin/ceph-mon -f {% if ceph_debug %}-d{% endif %} -i {{ ceph_mon_hostname }} --public-addr {{ storage_interface_address }}:6789",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/ceph.conf",

ansible/roles/ceph/templates/ceph-osd.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/usr/bin/ceph-osd -f {% if ceph_debug %}-d{% endif %} --public-addr {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }} --cluster-addr {{ hostvars[inventory_hostname]['ansible_' + cluster_interface]['ipv4']['address'] }}",
+    "command": "/usr/bin/ceph-osd -f {% if ceph_debug %}-d{% endif %} --public-addr {{ storage_interface_address }} --cluster-addr {{ hostvars[inventory_hostname]['ansible_' + cluster_interface]['ipv4']['address'] }}",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/ceph.conf",

ansible/roles/ceph/templates/ceph.conf.j2

@@ -6,7 +6,15 @@ log to stderr = false
 err to stderr = false
 
 fsid = {{ ceph_cluster_fsid }}
+{% if ceph_mon_host_type == 'HOSTNAME' %}
+mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_hostname'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+{% elif ceph_mon_host_type == 'FQDN' %}
+mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_fqdn'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+{% elif ceph_mon_host_type == 'INVENTORY' %}
+mon initial members = {% for host in groups['ceph-mon'] %}{{ host }}{% if not loop.last %}, {% endif %}{% endfor %}
+{%- else %}
 mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + hostvars[host]['storage_interface']]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+{% endif %}
 
 mon host = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + hostvars[host]['storage_interface']]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %}