Use a common python build/install role

In order to radically simplify how we prepare the service
venvs, we use a common role to do the wheel builds and the
venv preparation. This makes the process far simpler to
understand, because the role does its own building and
installing. It also reduces the code maintenance burden,
because instead of duplicating the build processes in the
repo_build role and the service role - we only have it all
done in a single place.

Given that the role now handles everything, and has sensible
defaults, we can remove the *_venv_tag and *_venv_download_url
in group_vars.

This is by no means the final stop in the simplification
process, but it is a step forward. The will be work to follow
which:

1. Changes how we define the versions of each service we wish
   to install. Currently this requires the use of the py_pkgs
   plugin, but we'd like to eliminate that to simplify the
   mechanism to something more intuitive.
2. Changes how we set the pip install arguments for the venv
   build. Right now the repo_build consolidates all constraints
   into a single set. That code is complex, difficult to
   understand and prohibits multi-series deployments.
3. Changes how we configure pip. Given that we no longer do
   pip installs onto the host, and only use venvs, we do not
   need to place pip.conf in a global location. We can explore
   other options, including perhaps not placing pip.conf at all.
4. Removes the repo_build process entirely. Once the roles are
   doing everything that's required, the repo_build process will
   be redundant and can be removed.

Depends-On: https://review.openstack.org/557039
Depends-On: https://review.openstack.org/557041
Depends-On: https://review.openstack.org/557042
Depends-On: https://review.openstack.org/557043
Depends-On: https://review.openstack.org/557047
Depends-On: https://review.openstack.org/557049
Depends-On: https://review.openstack.org/557050
Depends-On: https://review.openstack.org/557052
Depends-On: https://review.openstack.org/557053
Depends-On: https://review.openstack.org/557055
Depends-On: https://review.openstack.org/557059
Depends-On: https://review.openstack.org/557061
Depends-On: https://review.openstack.org/567692
Depends-On: https://review.openstack.org/599238
Depends-On: https://review.openstack.org/599240
Depends-On: https://review.openstack.org/599244
Depends-On: https://review.openstack.org/599247
Depends-On: https://review.openstack.org/599256
Depends-On: https://review.openstack.org/599434
Depends-On: https://review.openstack.org/599437
Change-Id: Ic31bd1c9f1c3eea61af50210d93aa96f9c797d92

inventory/group_vars/aodh_all.yml

@@ -22,7 +22,3 @@ aodh_service_adminuri: "{{ openstack_service_publicuri_proto|default(aodh_servic
 
 # Ensure that the package state matches the global setting
 aodh_package_state: "{{ package_state }}"
-
-# venv fetch configuration
-aodh_venv_tag: "{{ venv_tag }}"
-aodh_venv_download_url: "{{ venv_base_download_url }}/aodh-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/barbican_all.yml

@@ -16,7 +16,3 @@
 barbican_service_region: "{{ service_region }}"
 barbican_service_in_ldap: "{{ service_ldap_backend_enabled }}"
 barbican_keystone_auth: yes
-
-# venv fetch configuration
-barbican_venv_tag: "{{ venv_tag }}"
-barbican_venv_download_url: "{{ venv_base_download_url }}/barbican-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/ceilometer_all.yml

@@ -28,10 +28,6 @@ ceilometer_gnocchi_enabled: "{{ groups['gnocchi_all'] is defined and groups['gno
 # Ensure that the package state matches the global setting
 ceilometer_package_state: "{{ package_state }}"
 
-# venv fetch configuration
-ceilometer_venv_tag: "{{ venv_tag }}"
-ceilometer_venv_download_url: "{{ venv_base_download_url }}/ceilometer-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
-
 # Swift vars used when swift is enabled
 swift_system_user_name: "{{ hostvars['localhost']['swift_system_user_name'] }}"
 swift_system_shell: "{{ hostvars['localhost']['swift_system_shell'] }}"

inventory/group_vars/cinder_all.yml

@@ -44,9 +44,5 @@ cinder_package_state: "{{ package_state }}"
 # The system user for all cinder services
 cinder_system_user_name: cinder
 
-# venv fetch configuration
-cinder_venv_tag: "{{ venv_tag }}"
-cinder_venv_download_url: "{{ venv_base_download_url }}/cinder-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
-
 # If there are any Barbican hosts in the environment, then enable its usage
 cinder_barbican_enabled: "{{ (groups['barbican_all'] is defined) and (groups['barbican_all'] | length > 0) }}"

inventory/group_vars/congress_all.yml

@@ -18,7 +18,3 @@ congress_service_in_ldap: "{{ service_ldap_backend_enabled }}"
 
 # Ensure that the package state matches the global setting
 congress_package_state: "{{ package_state }}"
-
-# venv fetch configuration
-congress_venv_tag: "{{ venv_tag }}"
-congress_venv_download_url: "{{ venv_base_download_url }}/congress-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/designate_all.yml

@@ -18,7 +18,3 @@ designate_ceilometer_enabled: "{{ (groups['designate_all'] is defined) and (grou
 
 # Ensure that the package state matches the global setting
 designate_package_state: "{{ package_state }}"
-
-# venv fetch configuration
-designate_venv_tag: "{{ venv_tag }}"
-designate_venv_download_url: "{{ venv_base_download_url }}/designate-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/glance_all.yml

@@ -25,10 +25,6 @@ glance_default_store: "{{ ((groups['swift_all'] is defined) and (groups['swift_a
 # Ensure that the package state matches the global setting
 glance_package_state: "{{ package_state }}"
 
-# venv fetch configuration
-glance_venv_tag: "{{ venv_tag }}"
-glance_venv_download_url: "{{ venv_base_download_url }}/glance-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
-
 # glance default list of bind mounts
 glance_container_bind_mounts:
   - bind_dir_path: "/var/lib/glance/images"

inventory/group_vars/gnocchi_all.yml

@@ -27,10 +27,6 @@ gnocchi_package_state: "{{ package_state }}"
 # Ensure that keystone authentication is enabled for gnocchi
 gnocchi_keystone_auth: "{{ (groups['keystone_all'] is defined) and (groups['keystone_all'] | length > 0) }}"
 
-# venv fetch configuration
-gnocchi_venv_tag: "{{ venv_tag }}"
-gnocchi_venv_download_url: "{{ venv_base_download_url }}/gnocchi-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
-
 # Gnocchi default list of bind mounts
 gnocchi_container_bind_mounts:
   - bind_dir_path: "/var/lib/gnocchi"

inventory/group_vars/heat_all.yml

@@ -25,7 +25,3 @@ heat_package_state: "{{ package_state }}"
 # Only enable the heat cinder backups functionality if the cinder backup
 # service is enabled.
 heat_cinder_backups_enabled: "{{ hostvars['localhost']['cinder_service_backup_program_enabled'] }}"
-
-# venv fetch configuration
-heat_venv_tag: "{{ venv_tag }}"
-heat_venv_download_url: "{{ venv_base_download_url }}/heat-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/horizon_all.yml

@@ -35,7 +35,3 @@ horizon_package_state: "{{ package_state }}"
 horizon_enable_sahara_ui: "{{ (groups['sahara_all'] is defined) and (groups['sahara_all'] | length > 0) }}"
 # If there are any Trove hosts in the environment, then enable trove-dashboard
 horizon_enable_trove_ui: "{{ (groups['trove_all'] is defined) and (groups['trove_all'] | length > 0) }}"
-
-# venv fetch configuration
-horizon_venv_tag: "{{ venv_tag }}"
-horizon_venv_download_url: "{{ venv_base_download_url }}/horizon-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/ironic_all.yml

@@ -17,7 +17,3 @@ ironic_service_name: ironic
 
 # Ensure that the package state matches the global setting
 ironic_package_state: "{{ package_state }}"
-
-# venv fetch configuration
-ironic_venv_tag: "{{ venv_tag }}"
-ironic_venv_download_url: "{{ venv_base_download_url }}/ironic-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/keystone_all.yml

@@ -33,7 +33,3 @@ keystone_set_real_ip_from: "{{ groups['haproxy'] | map('extract', hostvars, 'con
 
 # Ensure that the package state matches the global setting
 keystone_package_state: "{{ package_state }}"
-
-# venv fetch configuration
-keystone_venv_tag: "{{ venv_tag }}"
-keystone_venv_download_url: "{{ venv_base_download_url }}/keystone-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/magnum_all.yml

@@ -25,7 +25,3 @@ magnum_service_region: "{{ service_region }}"
 
 # Ensure that the package state matches the global setting
 magnum_package_state: "{{ package_state }}"
-
-# venv fetch configuration
-magnum_venv_tag: "{{ venv_tag }}"
-magnum_venv_download_url: "{{ venv_base_download_url }}/magnum-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/neutron_all.yml

@@ -24,8 +24,4 @@ neutron_service_in_ldap: "{{ service_ldap_backend_enabled }}"
 # Ensure that the package state matches the global setting
 neutron_package_state: "{{ package_state }}"
 
-# venv fetch configuration
-neutron_venv_tag: "{{ venv_tag }}"
-neutron_venv_download_url: "{{ venv_base_download_url }}/neutron-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
-
 neutron_lbaas_octavia: "{{ (groups['octavia_all'] is defined) and (groups['octavia_all'] | length > 0) }}"

inventory/group_vars/nova_all.yml

@@ -59,10 +59,6 @@ nova_package_state: "{{ package_state }}"
 # The system user for all nova services
 nova_system_user_name: nova
 
-# venv fetch configuration
-nova_venv_tag: "{{ venv_tag }}"
-nova_venv_download_url: "{{ venv_base_download_url }}/nova-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
-
 # TODO: (andymccr) remove this once https://review.openstack.org/#/c/428120/ merges
 nova_reserved_host_disk_mb: 0
 

inventory/group_vars/octavia_all.yml

@@ -16,7 +16,3 @@
 octavia_service_internalurl: "{{ octavia_service_internaluri }}/v1/%(project_id)s"
 octavia_service_user_name: octavia
 octavia_service_region: "{{ service_region }}"
-
-# venv fetch configuration
-octavia_venv_tag: "{{ venv_tag }}"
-octavia_venv_download_url: "{{ venv_base_download_url }}/octavia-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/sahara_all.yml

@@ -16,7 +16,3 @@ sahara_ceilometer_enabled: "{{ (groups['ceilometer_all'] is defined) and (groups
 
 sahara_service_region: "{{ service_region }}"
 sahara_service_in_ldap: "{{ service_ldap_backend_enabled }}"
-
-# venv fetch configuration
-sahara_venv_tag: "{{ venv_tag }}"
-sahara_venv_download_url: "{{ venv_base_download_url }}/sahara-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/swift_all.yml

@@ -31,10 +31,6 @@ swift_package_state: "{{ package_state }}"
 # Used to optionally filter Gnocchi-originated traffic in Ceilometermiddleware
 swift_gnocchi_enabled: "{{ (groups['gnocchi_all'] is defined and groups['gnocchi_all'] | length > 0) }}"
 
-# venv fetch configuration
-swift_venv_tag: "{{ venv_tag }}"
-swift_venv_download_url: "{{ venv_base_download_url }}/swift-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
-
 # Ceilometer vars used when ceilometer is enabled
 ceilometer_service_user_name: "{{ hostvars['localhost']['ceilometer_service_user_name'] }}"
 ceilometer_service_tenant_name: "{{ hostvars['localhost']['ceilometer_service_tenant_name'] }}"

inventory/group_vars/tacker_all.yml

@@ -26,7 +26,3 @@ tacker_gnocchi_enabled: "{{ groups['gnocchi_all'] is defined and groups['gnocchi
 
 # Ensure that the package state matches the global setting
 tacker_package_state: "{{ package_state }}"
-
-# venv fetch configuration
-tacker_venv_tag: "{{ venv_tag }}"
-tacker_venv_download_url: "{{ venv_base_download_url }}/tacker-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/trove_all.yml

@@ -22,7 +22,3 @@ trove_ceilometer_enabled: "{{ (groups['trove_all'] is defined) and (groups['trov
 
 # Ensure that the package state matches the global setting
 trove_package_state: "{{ package_state }}"
-
-# venv fetch configuration
-trove_venv_tag: "{{ venv_tag }}"
-trove_venv_download_url: "{{ venv_base_download_url }}/trove-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

inventory/group_vars/utility_all.yml

@@ -65,14 +65,6 @@ tempest_service_available_swift: "{{ (groups['swift_all'] is defined and groups[
 tempest_service_available_congress: "{{ groups['congress_all'] is defined and groups['congress_all'] | length > 0 }}"
 
 tempest_log_dir: /var/log/utility
-tempest_venv_tag: "{{ openstack_release }}"
 
 # This sets the tempest group to the utility group
 tempest_main_group: utility_all
-
-#
-# Rally settings
-#
-
-rally_venv_tag: "{{ venv_tag }}"
-rally_venv_download_url: "{{ venv_base_download_url }}/rally-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"

playbooks/defaults/source_install.yml

@@ -13,12 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-cinder_bin: "/openstack/venvs/cinder-{{ cinder_venv_tag }}/bin"
-glance_bin: "/openstack/venvs/glance-{{ glance_venv_tag }}/bin"
-keystone_bin: "/openstack/venvs/keystone-{{ keystone_venv_tag }}/bin"
-nova_bin: "/openstack/venvs/nova-{{ nova_venv_tag }}/bin"
-octavia_bin: "/openstack/venvs/octavia-{{ octavia_venv_tag }}/bin"
-tempest_venv_bin: "/openstack/venvs/tempest-{{ tempest_venv_tag }}/bin"
+cinder_bin: "/openstack/venvs/cinder-{{ venv_tag }}/bin"
+glance_bin: "/openstack/venvs/glance-{{ venv_tag }}/bin"
+keystone_bin: "/openstack/venvs/keystone-{{ venv_tag }}/bin"
+nova_bin: "/openstack/venvs/nova-{{ venv_tag }}/bin"
+octavia_bin: "/openstack/venvs/octavia-{{ venv_tag }}/bin"
+tempest_venv_bin: "/openstack/venvs/tempest-{{ venv_tag }}/bin"
 
 # URL for the frozen internal openstack repo.
 repo_release_path: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/{{ os_distro_version }}"

playbooks/utility-install.yml

@@ -100,24 +100,16 @@
           tags:
             - always
 
-        - name: Create the virtualenv (if it does not exist)
-          command: "virtualenv --never-download --no-site-packages {{ utility_venv_bin | dirname }}"
-          args:
-            creates: "{{ utility_venv_bin }}/activate"
-
-        - name: Install pip packages
-          pip:
-            name: "{{ _openstack_client_list | union(utility_pip_packages) }}"
-            state: "{{ utility_pip_package_state }}"
-            virtualenv: "{{ utility_venv_bin | dirname }}"
-            virtualenv_site_packages: "no"
-            extra_args: >-
+        - name: Install the python venv
+          include_role:
+            name: "python_venv_build"
+            private: yes
+          vars:
+            venv_install_destination_path: "{{ utility_venv_bin | dirname }}"
+            venv_pip_install_args: >-
               {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
               {{ pip_install_options | default('') }}
-          register: install_packages
-          until: install_packages is success
-          retries: 5
-          delay: 2
+            venv_pip_packages: "{{ _openstack_client_list | union(utility_pip_packages) }}"
 
         - name: Create symlinks for openstack clients
           shell: |