From d8c15ad4e8a2235396a0f40d2fe53112d23495c5 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 14 Aug 2019 09:13:29 +0000 Subject: [PATCH] CI: Add Ceph-Ansible jobs * Adding zuul centos-source/ubuntu-source ceph-ansible jobs * Jobs will deploy all Ceph integrated OpenStack components, i.e. cinder, glance, nova * Will utilize core openstack testing script Depends-On: https://review.opendev.org/685032 Depends-On: https://review.opendev.org/698301 Implements: blueprint ceph-ansible Change-Id: I233082b46785f74014177f579aeac887a25b2ae2 --- tests/check-config.sh | 2 ++ tests/deploy-ceph-ansible.sh | 30 ++++++++++++++++ tests/get_logs.sh | 15 +++++--- tests/run.yml | 55 ++++++++++++++++++++++++++++-- tests/setup_disks.sh | 7 ++++ tests/templates/ceph-ansible.j2 | 33 ++++++++++++++++++ tests/templates/ceph-inventory.j2 | 14 ++++++++ tests/templates/globals-default.j2 | 9 +++++ tests/test-core-openstack.sh | 4 +-- tools/setup_gate.sh | 4 +++ zuul.d/base.yaml | 11 ++++++ zuul.d/jobs.yaml | 18 ++++++++++ zuul.d/project.yaml | 8 +++++ 13 files changed, 201 insertions(+), 9 deletions(-) create mode 100755 tests/deploy-ceph-ansible.sh create mode 100644 tests/templates/ceph-ansible.j2 create mode 100644 tests/templates/ceph-inventory.j2 diff --git a/tests/check-config.sh b/tests/check-config.sh index c8c172c7cb..8337460400 100755 --- a/tests/check-config.sh +++ b/tests/check-config.sh @@ -18,8 +18,10 @@ function check_config { -not -path /etc/kolla \ -not -name admin-openrc.sh \ -not -name globals.yml \ + -not -name ceph-ansible.yml \ -not -name header \ -not -name inventory \ + -not -name ceph-inventory \ -not -name kolla-build.conf \ -not -name passwords.yml \ -not -name passwords.yml.old \ diff --git a/tests/deploy-ceph-ansible.sh b/tests/deploy-ceph-ansible.sh new file mode 100755 index 0000000000..9c4ef72975 --- /dev/null +++ b/tests/deploy-ceph-ansible.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + +function setup_ceph_ansible { + # FIXME(mnasiadka): Use python3 when we move to CentOS 8 + # (there are no python3 selinux bindings for 3 on C7) + # see https://bugs.centos.org/view.php?id=16389 + + # Prepare virtualenv for ceph-ansible deployment + virtualenv --system-site-packages ~/ceph-venv + ~/ceph-venv/bin/pip install -Ir requirements.txt + ~/ceph-venv/bin/pip install -IU selinux +} + +function deploy_ceph_ansible { + RAW_INVENTORY=/etc/kolla/ceph-inventory + + . ~/ceph-venv/bin/activate + + cp site-container.yml.sample site-container.yml + ansible-playbook -i ${RAW_INVENTORY} -e @/etc/kolla/ceph-ansible.yml -vvv site-container.yml --skip-tags=with_pkg &> /tmp/logs/ansible/deploy-ceph +} + +setup_ceph_ansible +deploy_ceph_ansible diff --git a/tests/get_logs.sh b/tests/get_logs.sh index b6e450db4c..58a5e8b397 100644 --- a/tests/get_logs.sh +++ b/tests/get_logs.sh @@ -78,13 +78,20 @@ copy_logs() { (docker info && docker images && docker ps -a && docker network ls && docker inspect $(docker ps -aq)) > ${LOG_DIR}/system_logs/docker-info.txt # ceph related logs - if [[ $(docker ps --filter name=ceph_mon --format "{{.Names}}") ]]; then - docker exec ceph_mon ceph --connect-timeout 5 -s > ${LOG_DIR}/kolla/ceph/ceph_s.txt + # NOTE(mnasiadka): regex to match both ceph_mon and ceph-mon-$hostname + for container in $(docker ps --filter name=ceph.\?mon --format "{{.Names}}"); do + if [ $container == "ceph_mon" ]; then + CEPH_LOG_DIR="${LOG_DIR}/kolla/ceph" + else + CEPH_LOG_DIR="${LOG_DIR}/ceph" + mkdir -p ${CEPH_LOG_DIR} + fi + docker exec ${container} ceph --connect-timeout 5 -s > ${CEPH_LOG_DIR}/ceph_s.txt # NOTE(yoctozepto): osd df removed on purpose to avoid CI POST_FAILURE due to a possible hang: # as of ceph mimic it hangs when MON is operational but MGR not # its usefulness is mediocre and having POST_FAILUREs is bad - docker exec ceph_mon ceph --connect-timeout 5 osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt - fi + docker exec ${container} ceph --connect-timeout 5 osd tree > ${CEPH_LOG_DIR}/ceph_osd_tree.txt + done # bifrost related logs if [[ $(docker ps --filter name=bifrost_deploy --format "{{.Names}}") ]]; then diff --git a/tests/run.yml b/tests/run.yml index b77eeb5205..ceff836b97 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -17,10 +17,11 @@ kolla_ansible_src_dir: "{{ ansible_env.PWD }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/" + ceph_ansible_src_dir: "{{ ansible_env.PWD }}/src/github.com/ceph/ceph-ansible" need_build_image: false build_image_tag: "change_{{ zuul.change | default('none') }}" openstack_core_enabled: "{{ openstack_core_enabled }}" - openstack_core_tested: "{{ scenario in ['core', 'ceph', 'cinder-lvm', 'cells', 'swift'] }}" + openstack_core_tested: "{{ scenario in ['core', 'ceph', 'ceph-ansible', 'cinder-lvm', 'cells', 'swift'] }}" dashboard_enabled: "{{ openstack_core_enabled }}" # TODO(mgoddard): Remove when previous_release is ussuri. playbook_python_version: "{{ '2' if is_upgrade and previous_release == 'train' else '3' }}" @@ -33,10 +34,10 @@ - name: Prepare disks for a storage service script: "setup_disks.sh {{ disk_type }}" - when: scenario in ['cinder-lvm', 'ceph', 'zun', 'swift'] + when: scenario in ['cinder-lvm', 'ceph', 'ceph-ansible', 'zun', 'swift'] become: true vars: - disk_type: "{{ ceph_storetype if scenario == 'ceph' else scenario }}" + disk_type: "{{ ceph_storetype if scenario in ['ceph', 'ceph-ansible'] else scenario }}" ceph_storetype: "{{ hostvars[inventory_hostname].get('ceph_osd_storetype') }}" - hosts: primary @@ -132,6 +133,14 @@ - src: "tests/templates/ironic-overrides.j2" dest: /etc/kolla/config/ironic.conf when: "{{ scenario == 'ironic' }}" + # Ceph-Ansible inventory + - src: "tests/templates/ceph-inventory.j2" + dest: /etc/kolla/ceph-inventory + when: "{{ scenario == 'ceph-ansible' }}" + # ceph-ansible.yml + - src: "tests/templates/ceph-ansible.j2" + dest: /etc/kolla/ceph-ansible.yml + when: "{{ scenario == 'ceph-ansible' }}" when: item.when | default(true) - block: @@ -257,6 +266,46 @@ # ready to deploy the control plane services. Control flow now depends on # the scenario being exercised. + # Deploy ceph-ansible on ceph-ansible scenarios + - block: + - name: Run deploy-ceph-ansible.sh script + script: + cmd: deploy-ceph-ansible.sh + executable: /bin/bash + chdir: "{{ ceph_ansible_src_dir }}" + + - name: Ensure required kolla config directories exist + file: + state: directory + name: "/etc/kolla/config/{{ item.name }}" + mode: 0777 + with_items: "{{ ceph_ansible_services }}" + + - name: copy ceph.conf to enabled services + copy: + src: "/etc/ceph/ceph.conf" + dest: "/etc/kolla/config/{{ item.name }}/ceph.conf" + remote_src: True + with_items: "{{ ceph_ansible_services }}" + + - name: copy keyrings to enabled services + copy: + remote_src: True + src: "/etc/ceph/{{ item.keyring }}" + dest: "/etc/kolla/config/{{ item.name }}/{{ item.keyring }}" + with_items: "{{ ceph_ansible_services }}" + become: True + + vars: + ceph_ansible_services: + - { name: 'cinder/cinder-volume', keyring: "ceph.client.cinder.keyring" } + - { name: 'cinder/cinder-backup', keyring: "ceph.client.cinder.keyring" } + - { name: 'cinder/cinder-backup', keyring: "ceph.client.cinder-backup.keyring" } + - { name: 'glance', keyring: "ceph.client.glance.keyring" } + - { name: 'nova', keyring: "ceph.client.nova.keyring" } + - { name: 'nova', keyring: "ceph.client.cinder.keyring" } + when: scenario == "ceph-ansible" + # Deploy control plane. For upgrade jobs this is the previous release. - block: - name: Run deploy.sh script diff --git a/tests/setup_disks.sh b/tests/setup_disks.sh index 4f3fd9168d..bf0ab419be 100644 --- a/tests/setup_disks.sh +++ b/tests/setup_disks.sh @@ -54,6 +54,13 @@ elif [ $1 = 'bluestore' ]; then LOOP=$(losetup -f) losetup $LOOP /opt/data/kolla/ceph-osd0-d.img parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_OSD0_D 1 -1 +elif [ $1 = 'ceph-lvm' ]; then + free_device=$(losetup -f) + fallocate -l 10G /var/lib/ceph-osd1.img + losetup $free_device /var/lib/ceph-osd1.img + pvcreate $free_device + vgcreate cephvg $free_device + lvcreate -l 100%FREE -n cephlv cephvg else echo "Unknown type" >&2 exit 1 diff --git a/tests/templates/ceph-ansible.j2 b/tests/templates/ceph-ansible.j2 new file mode 100644 index 0000000000..17aaf7f797 --- /dev/null +++ b/tests/templates/ceph-ansible.j2 @@ -0,0 +1,33 @@ +# ceph-ansible group vars +ceph_stable_release: "nautilus" +monitor_interface: "{{ api_interface_name }}" +radosgw_interface: "{{ api_interface_name }}" + +public_network: "{{ api_network_prefix }}0/{{ api_network_prefix_length }}" + +configure_firewall: false + +docker: true +containerized_deployment: true + +dashboard_enabled: false + +openstack_config: true +{% raw %} +openstack_pools: + - "{{ openstack_glance_pool }}" + - "{{ openstack_cinder_pool }}" + - "{{ openstack_cinder_backup_pool }}" + - "{{ openstack_nova_pool }}" + +openstack_keys: + - { name: client.glance, caps: { mon: "profile rbd", osd: "profile rbd pool=volumes, profile rbd pool={{ openstack_glance_pool.name }}"}, mode: "0600" } + - { name: client.cinder, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_cinder_pool.name }}, profile rbd pool={{ openstack_nova_pool.name }}, profile rbd pool={{ openstack_glance_pool.name }}"}, mode: "0600" } + - { name: client.cinder-backup, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_cinder_backup_pool.name }}"}, mode: "0600" } + - { name: client.nova, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_nova_pool.name }}, profile rbd pool={{ openstack_cinder_pool.name }}, profile rbd pool={{ openstack_glance_pool.name }}"}, mode: "0600" } +{% endraw %} + +# osds +lvm_volumes: + - data: cephlv + data_vg: cephvg diff --git a/tests/templates/ceph-inventory.j2 b/tests/templates/ceph-inventory.j2 new file mode 100644 index 0000000000..17edcdbdca --- /dev/null +++ b/tests/templates/ceph-inventory.j2 @@ -0,0 +1,14 @@ +[storage] +{% for host in hostvars %} +{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }} ceph_osd_store_type={{ 'filestore' if host == 'primary' else 'bluestore' }} +{% endfor %} + +# Ceph-Ansible hosts +[mons:children] +storage + +[mgrs:children] +storage + +[osds:children] +storage diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 2966bf4e52..3db840af9f 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -127,3 +127,12 @@ enable_mariadb: "yes" enable_memcached: "no" enable_rabbitmq: "no" {% endif %} + +{% if scenario == "ceph-ansible" %} +# kolla-ansible vars +enable_cinder: "yes" +# External Ceph +glance_backend_ceph: "yes" +cinder_backend_ceph: "yes" +nova_backend_ceph: "yes" +{% endif %} diff --git a/tests/test-core-openstack.sh b/tests/test-core-openstack.sh index 351c9048ac..267b45f25a 100755 --- a/tests/test-core-openstack.sh +++ b/tests/test-core-openstack.sh @@ -11,7 +11,7 @@ function test_smoke { openstack --debug compute service list openstack --debug network agent list openstack --debug orchestration service list - if [[ $SCENARIO == "ceph" ]] || [[ $SCENARIO == "cinder-lvm" ]]; then + if [[ $SCENARIO == "ceph" ]] || [[ $SCENARIO == "ceph-ansible" ]] | [[ $SCENARIO == "cinder-lvm" ]]; then openstack --debug volume service list fi } @@ -28,7 +28,7 @@ function test_instance_boot { fi echo "SUCCESS: Server creation" - if [[ $SCENARIO == "ceph" ]] || [[ $SCENARIO == "cinder-lvm" ]]; then + if [[ $SCENARIO == "ceph" ]] || [ $SCENARIO == "ceph-ansible" ] || [[ $SCENARIO == "cinder-lvm" ]]; then echo "TESTING: Cinder volume attachment" openstack volume create --size 2 test_volume attempt=1 diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index 46b41cda6d..813836c770 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -39,6 +39,10 @@ function setup_config { GATE_IMAGES+=",ceph,cinder" fi + if [[ $SCENARIO == "ceph-ansible" ]]; then + GATE_IMAGES+=",cinder" + fi + if [[ $SCENARIO == "cinder-lvm" ]]; then GATE_IMAGES+=",cinder,iscsid,tgtd" fi diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index d1f338cb09..f053bc7d58 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -93,3 +93,14 @@ - ^tests/test-swift.sh vars: scenario: swift + +- job: + name: kolla-ansible-ceph-ansible-base + parent: kolla-ansible-base + voting: false + vars: + scenario: ceph-ansible + ceph_osd_storetype: ceph-lvm + required-projects: + - name: github.com/ceph/ceph-ansible + override-checkout: v4.0.7 diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index d5505d6e30..d1f26e2a54 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -95,6 +95,24 @@ secondary2: ceph_osd_storetype: bluestore +- job: + name: kolla-ansible-centos-source-ceph-ansible + parent: kolla-ansible-ceph-ansible-base + nodeset: kolla-ansible-centos-multi + timeout: 9000 + vars: + base_distro: centos + install_type: source + +- job: + name: kolla-ansible-ubuntu-source-ceph-ansible + parent: kolla-ansible-ceph-ansible-base + nodeset: kolla-ansible-bionic-multi + timeout: 9000 + vars: + base_distro: ubuntu + install_type: source + - job: name: kolla-ansible-ubuntu-source-cinder-lvm parent: kolla-ansible-base diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 35090c5113..3c7f166368 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -70,6 +70,14 @@ files: - ^ansible/roles/mariadb/ - ^tests/test-mariadb.sh + - kolla-ansible-centos-source-ceph-ansible: + files: + - ^ansible/roles/(cinder|glance|gnocchi|nova-cell)/ + - ^tests/deploy-ceph-ansible.sh + - kolla-ansible-ubuntu-source-ceph-ansible: + files: + - ^ansible/roles/(cinder|glance|gnocchi|nova-cell)/ + - ^tests/deploy-ceph-ansible.sh check-arm64: jobs: - kolla-ansible-debian-source-aarch64