Merge "Eliminate installing pip on host/containers"

2018-08-10 20:41:36 +00:00 · 2018-08-10 20:41:36 +00:00 · cac78c2a65
commit cac78c2a65
parent 610d1f945d 191e0b8660
6 changed files with 138 additions and 134 deletions
--- a/inventory/group_vars/utility_all.yml
+++ b/inventory/group_vars/utility_all.yml
@ -23,9 +23,11 @@ galera_client_drop_config_file: true
 utility_package_state: "{{ package_state }}"
 utility_pip_package_state: "latest"

+# Path to the utility host openstack client venv binaries
+utility_venv_bin: "/openstack/venvs/utility-{{ openstack_release }}/bin"
+
 # Distribution packages to be installed into the utility container
 utility_distro_packages:
-  - curl
  - git

 utility_distro_openstack_clients_packages:
--- a/playbooks/ceph-rgw-keystone-setup.yml
+++ b/playbooks/ceph-rgw-keystone-setup.yml
@ -14,120 +14,95 @@
 # limitations under the License.

 - name: Configure keystone for radosgw
-  hosts: utility_all
+  hosts: "{{ openstack_service_setup_host | default('localhost') }}"
  user: root
  vars_files:
    - "defaults/{{ install_method }}_install.yml"
+  vars:
+    ansible_python_interpreter: >-
+      {{ ((openstack_service_setup_host | default('localhost')) == 'localhost') | ternary(ansible_playbook_python, ansible_python['executable']) }}
+  tags:
+    - ceph-rgw
+    - ceph-rgw-setup
+    - rgw-service-add
  tasks:
-    - name: Ensure RGW service
-      keystone:
-        command: "ensure_service"
-        endpoint: "{{ keystone_service_adminurl }}"
-        login_user: "{{ keystone_admin_user_name }}"
-        login_password: "{{ keystone_auth_admin_password }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
-        service_name: "{{ radosgw_service_name }}"
+    - name: Add service to the keystone service catalog
+      os_keystone_service:
+        cloud: default
+        state: present
+        name: "{{ radosgw_service_name }}"
        service_type: "{{ radosgw_service_type }}"
        description: "{{ radosgw_service_description }}"
-        insecure: "{{ keystone_service_adminuri_insecure }}"
-      run_once: true
+        interface: admin
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
      register: add_service
      until: add_service is success
      retries: 5
-      delay: 2
-      no_log: True
+      delay: 10
      tags:
        - ceph-rgw-setup
        - rgw-service-add

-    - name: Ensure RGW user
-      keystone:
-        command: "ensure_user"
-        endpoint: "{{ keystone_service_adminurl }}"
-        login_user: "{{ keystone_admin_user_name }}"
-        login_password: "{{ keystone_auth_admin_password }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
-        user_name: "{{ radosgw_admin_user }}"
-        tenant_name: "{{ radosgw_admin_tenant }}"
-        role_name: "{{ radosgw_role_name | default('service') }}"
+    - name: Add service user
+      os_user:
+        cloud: default
+        state: present
+        name: "{{ radosgw_admin_user }}"
        password: "{{ radosgw_admin_password }}"
-        insecure: "{{ keystone_service_adminuri_insecure }}"
-      run_once: true
+        domain: default
+        default_project: "{{ radosgw_admin_tenant }}"
+        interface: admin
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
      register: add_user
      until: add_user is success
      retries: 5
      delay: 10
      no_log: True
-      tags:
-        - ceph-rgw-setup
-        - rgw-service-add

-    - name: Ensure RGW user to admin role
-      keystone:
-        command: "ensure_user_role"
-        endpoint: "{{ keystone_service_adminurl }}"
-        login_user: "{{ keystone_admin_user_name }}"
-        login_password: "{{ keystone_auth_admin_password }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
-        user_name: "{{ radosgw_admin_user }}"
-        tenant_name: "{{ radosgw_admin_tenant }}"
-        role_name: "{{ radosgw_role_name | default('admin') }}"
-        insecure: "{{ keystone_service_adminuri_insecure }}"
-      run_once: true
-      register: add_admin_role
-      until: add_admin_role is success
+    - name: Add service user to roles
+      os_user_role:
+        cloud: default
+        state: present
+        user: "{{ radosgw_admin_user }}"
+        role: "{{ radosgw_role_name | default('admin') }}"
+        project: "{{ radosgw_admin_tenant }}"
+        interface: admin
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
+      register: add_user_role
+      until: add_user_role is success
      retries: 5
      delay: 10
-      no_log: True
-      tags:
-        - ceph-rgw-setup
-        - rgw-service-add

-    - name: Ensure swiftoperator role
-      keystone:
-        command: "ensure_role"
-        endpoint: "{{ keystone_service_adminurl }}"
-        login_user: "{{ keystone_admin_user_name }}"
-        login_password: "{{ keystone_auth_admin_password }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
-        role_name: "swiftoperator"
-        insecure: "{{ keystone_service_adminuri_insecure }}"
-      run_once: true
-      register: add_swiftoperator_role
-      until: add_swiftoperator_role is success
+    - name: Add service role
+      os_keystone_role:
+        cloud: default
+        state: present
+        name: "swiftoperator"
+        interface: admin
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
+      register: add_role
+      until: add_role is success
      retries: 5
      delay: 10
-      no_log: True
-      tags:
-        - ceph-rgw-setup
-        - rgw-service-add

-    - name: Ensure RGW endpoint
-      keystone:
-        command: "ensure_endpoint"
-        endpoint: "{{ keystone_service_adminurl }}"
-        login_user: "{{ keystone_admin_user_name }}"
-        login_password: "{{ keystone_auth_admin_password }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
-        region_name: "{{ radosgw_service_region }}"
-        service_name: "{{ radosgw_service_name }}"
-        service_type: "{{ radosgw_service_type }}"
-        insecure: "{{ keystone_service_adminuri_insecure }}"
-        endpoint_list:
-          - url: "{{ radosgw_service_publicurl }}"
-            interface: "public"
-          - url: "{{ radosgw_service_adminurl }}"
-            interface: "admin"
-          - url: "{{ radosgw_service_internalurl }}"
-            interface: "internal"
-      run_once: true
-      register: add_endpoint
-      until: add_endpoint is success
+    - name: Add endpoints to keystone endpoint catalog
+      os_keystone_endpoint:
+        cloud: default
+        state: present
+        service: "{{ radosgw_service_name }}"
+        endpoint_interface: "{{ item.interface }}"
+        url: "{{ item.url }}"
+        region: "{{ radosgw_service_region }}"
+        interface: admin
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
+      register: add_service
+      until: add_service is success
      retries: 5
      delay: 10
-      no_log: True
-      tags:
-        - ceph-rgw-setup
-        - rgw-service-add
-  tags:
-    - ceph-rgw
+      with_items:
+        - interface: "public"
+          url: "{{ radosgw_service_publicurl }}"
+        - interface: "internal"
+          url: "{{ radosgw_service_internalurl }}"
+        - interface: "admin"
+          url: "{{ radosgw_service_adminurl }}"
--- a/playbooks/repo-use.yml
+++ b/playbooks/repo-use.yml
@ -13,21 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# TODO(evrardjp): Remove this playbook when the repo_build process
-# is done upfront. This would mean that the openstack_hosts role
-# can run once and for all directly after the
-# lxc-container-create playbook.
 - name: Configure all nodes to use the repo container for python/apt packages
  hosts: "{{ openstack_host_group | default('hosts') }}:all_containers"
+  vars:
+    pip_install: no
+    pip_configure: yes
  vars_files:
-    - defaults/repo_packages/openstack_services.yml
+    - "defaults/repo_packages/openstack_services.yml"
    - "defaults/{{ install_method }}_install.yml"
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  tags:
+    - always
+    - repo-config
  pre_tasks:
    - include: common-tasks/package-cache-proxy.yml
    - include: common-tasks/set-pip-vars.yml
  roles:
    - pip_install
-  environment: "{{ deployment_environment_variables | default({}) }}"
-  tags:
-    - always
-    - repo-config
--- a/playbooks/utility-install.yml
+++ b/playbooks/utility-install.yml
@ -16,6 +16,11 @@
 - name: Setup the utility location(s)
  hosts: utility_all
  user: root
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  vars_files:
+    - "defaults/{{ install_method }}_install.yml"
+  tags:
+    - utility
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
      static: no
@ -58,31 +63,6 @@
        update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
        cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(cache_timeout, omit) }}"

-    - name: Get list of python clients
-      shell: "curl -s {{ repo_release_path }}/requirements_absolute_requirements.txt  | grep client | cut -d'=' -f1"
-      args:
-        warn: no
-      register: client_list
-      run_once: true
-      changed_when: false
-      when: install_method == "source"
-      tags:
-        - always
-        - skip_ansible_lint
-
-    - name: Install pip packages
-      pip:
-        name: "{{ client_list.stdout_lines | union(utility_pip_packages) }}"
-        state: "{{ utility_pip_package_state }}"
-        extra_args: >-
-          {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
-          {{ pip_install_options | default('') }}
-      register: install_packages
-      until: install_packages is success
-      retries: 5
-      delay: 2
-      when: install_method == "source"
-
    - name: Distribute private ssh key
      copy:
        content: "{{ utility_ssh_private_key }}"
@ -91,8 +71,60 @@
        owner: root
        group: root
      when: utility_ssh_private_key is defined
-  environment: "{{ deployment_environment_variables | default({}) }}"
-  vars_files:
-    - "defaults/{{ install_method }}_install.yml"
-  tags:
-    - utility
+
+    - name: Install openstack clients (source-based install)
+      when:
+        - install_method == "source"
+      block:
+        - name: Get list of repo packages
+          uri:
+            url: "{{ repo_release_path }}/requirements_absolute_requirements.txt"
+            return_content: yes
+          register: _abs_reqs
+          run_once: true
+          tags:
+            - always
+
+        - name: Derive the list of openstack clients
+          set_fact:
+            _openstack_client_list: >-
+              {%- set package_list = [] %}
+              {%- for l in _abs_reqs.content.split('\n') %}
+              {%-   if (l is match('^python_.*client==.*$')) or (l is match('^(aodh|gnocchi)client==.*$')) %}
+              {%-     set _ = package_list.append(l | regex_replace('==.*$', '')) %}
+              {%-   endif %}
+              {%- endfor %}
+              {{- package_list }}
+          run_once: true
+          tags:
+            - always
+
+        - name: Create the virtualenv (if it does not exist)
+          command: "virtualenv --never-download --no-site-packages {{ utility_venv_bin | dirname }}"
+          args:
+            creates: "{{ utility_venv_bin }}/activate"
+
+        - name: Install pip packages
+          pip:
+            name: "{{ _openstack_client_list | union(utility_pip_packages) }}"
+            state: "{{ utility_pip_package_state }}"
+            virtualenv: "{{ utility_venv_bin | dirname }}"
+            virtualenv_site_packages: "no"
+            extra_args: >-
+              {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
+              {{ pip_install_options | default('') }}
+          register: install_packages
+          until: install_packages is success
+          retries: 5
+          delay: 2
+
+        - name: Create symlinks for openstack clients
+          shell: |
+            {% set _bin_name = item | regex_replace('^(?:python_)?(\w*)(?:client)$', '\\1') %}
+            if [[ -e "{{ utility_venv_bin }}/{{ _bin_name }}" ]]; then
+              ln -sfn {{ utility_venv_bin }}/{{ _bin_name }} /usr/local/bin/{{ _bin_name }}
+            fi
+          args:
+            executable: /bin/bash
+          with_items: "{{ _openstack_client_list }}"
+
--- a/scripts/bootstrap-ansible.sh
+++ b/scripts/bootstrap-ansible.sh
@ -83,19 +83,16 @@ case ${DISTRO_ID} in
          python3 python3-dev \
          libssl-dev libffi-dev \
          python-apt python3-apt \
-          python-pip \
          python-virtualenv
        ;;
    opensuse)
        zypper -n install -l git-core curl autoconf gcc gcc-c++ \
            netcat-openbsd python python-xml python-devel gcc \
-            libffi-devel libopenssl-devel python-pip \
-            python-virtualenv
+            libffi-devel libopenssl-devel python-virtualenv
        # Leap ships with python3.4 which is not supported by ansible and as
        # such we are using python2
        # See https://github.com/ansible/ansible/issues/24180
        PYTHON_EXEC_PATH="/usr/bin/python2"
-        alternatives --set pip /usr/bin/pip2.7 || true
        ;;
 esac

--- a/tests/bootstrap-aio.yml
+++ b/tests/bootstrap-aio.yml
@ -19,7 +19,6 @@
  user: root
  roles:
    - role: "sshd"
-    - role: "pip_install"
    - role: "bootstrap-host"
  vars:
    ansible_python_interpreter: "/usr/bin/python"