Fix deployment when improper version of python-openssl is installed

Without this fix, the bifrost deployment will lead to several
issues due to the incompatibility of system python-openssl and
pip version of pyOpenSSL

Change-Id: I101c2918ed81809536cf912c4aaa54a2f2052a25

lower-constraints.txt

@@ -49,6 +49,7 @@ pyflakes==0.8.1
 Pygments==2.2.0
 pyinotify==0.9.6
 PyNaCl==1.2.1
+pyOpenSSL==18.0.0
 pyparsing==2.2.0
 python-dateutil==2.7.0
 python-mimeparse==1.6.0

playbooks/roles/bifrost-create-dib-image/tasks/main.yml

@@ -13,10 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
+- include: roles/common/venv_python_path.yml
 - name: Ensure required packages are installed
   package:
     name: "{{ dib_host_required_packages }}"
     state: present
+  environment: "{{ venv }}"
 # If attempting to utilize a base Ubuntu image, diskimage-builder
 # is the recommended, and default path.
 - name: "Test if image is present"

playbooks/roles/bifrost-ironic-install/tasks/download_ipa_image.yml

@@ -15,6 +15,8 @@
 ---
 # This is overly complex, however get_url will always re-retrieve the file
 # if it already exists, and this is to prevent that behavior.
+- include: roles/common/venv_python_path.yml
+
 - name: "Test if IPA kernel is present"
   stat: path={{ ipa_kernel }}
   register: test_ipa_kernel_present
@@ -24,6 +26,7 @@
     get_url: url="{{ ipa_kernel_upstream_checksum_url }}" dest="{{ ipa_kernel }}.{{ ipa_kernel_upstream_checksum_algo }}" timeout=300
     register: ipa_kernel_checksum_result
     ignore_errors: yes
+    environment: "{{ venv }}"
   - debug: msg="WARNING!!! {{ ipa_kernel_upstream_checksum_algo }} file not found at {{ ipa_kernel_upstream_checksum_url }}"
     when: ipa_kernel_checksum_result is defined and ipa_kernel_checksum_result.status_code is defined and ipa_kernel_checksum_result.status_code == 404
   - fail: msg="FATAL {{ ipa_kernel_upstream_checksum_algo }} file not found at {{ ipa_kernel_upstream_checksum_url }} GOT {{ ipa_kernel_checksum_result }}"
@@ -54,6 +57,7 @@
          (ipa_kernel_download_done|failed and ipa_kernel_download_done.status_code == 404)
   retries: 5
   delay: 10
+  environment: "{{ venv }}"
   when: test_ipa_kernel_present.stat.exists == false
 
 - name: "Test if IPA image is present"
@@ -65,6 +69,7 @@
     get_url: url="{{ ipa_ramdisk_upstream_checksum_url }}" dest="{{ ipa_ramdisk }}.{{ ipa_ramdisk_upstream_checksum_algo }}" timeout=300
     register: ipa_ramdisk_checksum_result
     ignore_errors: yes
+    environment: "{{ venv }}"
   - debug: msg="WARNING!!! {{ ipa_ramdisk_upstream_checksum_algo }} file not found at {{ ipa_ramdisk_upstream_checksum_url }}"
     when: ipa_ramdisk_checksum_result is defined and ipa_ramdisk_checksum_result.status_code is defined and ipa_ramdisk_checksum_result.status_code == 404
   - fail: msg="FATAL {{ ipa_ramdisk_upstream_checksum_algo }} file not found at {{ ipa_ramdisk_upstream_checksum_url }}"
@@ -95,4 +100,5 @@
          (ipa_ramdisk_download_done|failed and ipa_ramdisk_download_done.status_code == 404)
   retries: 5
   delay: 10
+  environment: "{{ venv }}"
   when: test_ipa_image_present.stat.exists == false

playbooks/roles/bifrost-ironic-install/tasks/get_ipxe.yml

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
+- include: roles/common/venv_python_path.yml
+
 - name: Create {{ ipxe_dir }}
   file:
     name={{ ipxe_dir }}
@@ -33,3 +35,4 @@
   with_items:
     - undionly.kpxe
     - ipxe.pxe
+  environment: "{{ venv }}"

playbooks/roles/bifrost-ironic-install/tasks/install.yml

@@ -13,12 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
+- include: roles/common/venv_python_path.yml
+
 - name: "Update Package Cache"
   apt: update_cache=yes
+  environment: "{{ venv }}"
   when: ansible_os_family == 'Debian'
 
 - name: "Install packages"
   action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+  environment: "{{ venv }}"
   with_items: "{{ required_packages }}"
 
 - name: "If running in CI, set source install facts just to be sure"
@@ -27,11 +31,6 @@
     ironicclient_source_install: true
   when: ci_testing | bool == true
 
-- name: "If VENV is set in the environment, enable installation into venv"
-  set_fact:
-    enable_venv: true
-  when: lookup('env', 'VENV') | length > 0
-
 # NOTE(TheJulia) While we don't necessarilly require /opt/stack any longer
 # and it should already be created by the Ansible setup, we will leave this
 # here for the time being.
@@ -49,6 +48,7 @@
      package=proliantutils
      state=present
      extra_args="-c {{ upper_constraints_file }}"
+  environment: "{{ venv }}"
   when: skip_install is not defined
 
 - name: "UcsSdk - Install from pip"
@@ -56,11 +56,13 @@
     package=UcsSdk
     version=0.8.1.9
     extra_args="-c {{ upper_constraints_file }}"
+  environment: "{{ venv }}"
   when: skip_install is not defined
 
 # TODO(dtantsur): only do this is the iscsi deploy interface is enabled
 - name: "Install iSCSI client if PXE driver support is enabled"
   action: "{{ ansible_pkg_mgr }} name={{ iscsi_client_package }} state=present"
+  environment: "{{ venv }}"
   when: skip_install is not defined
 
 - name: "Diskimage-builder - Install"
@@ -135,6 +137,8 @@
 # NOTE(pas-ha) even when install into virtualenv is requested,
 # we need to install shade into system for enroll-dynamic to succeed
 - block:
+  - name: install pyOpenSSL>18.0.0 from PyPI
+    command: "{{ hostvars[inventory_hostname].ansible_python.executable }} -m pip install pyOpenSSL>=18.0.0"
   - name: install shade from PyPI
     command: "{{ hostvars[inventory_hostname].ansible_python.executable }} -m pip install shade"
     when: not (shade_source_install | default(false) | bool)

playbooks/roles/bifrost-ironic-install/tasks/keystone_setup.yml

@@ -16,10 +16,10 @@
 # We should likely address that at some point, however I think a user
 # should be the driver of that work.
 
-- name: "If VENV is set in the environment, enable installation into venv"
+- name: "Initialize default venv"
   set_fact:
-    enable_venv: true
-  when: lookup('env', 'VENV') | length > 0
+    venv:
+      OS_IDENTITY_API_VERSION: "3"
 
 - name: "Error if credentials are undefined."
   fail:
@@ -39,6 +39,8 @@
       ironic.keystone.default_username is undefined or
       ironic.keystone.default_password is undefined
 
+- include: roles/common/venv_python_path.yml
+
 - name: "Ensure service project is present"
   os_project:
     name: "{{ ironic.service_catalog.project_name }}"
@@ -53,8 +55,7 @@
       project_name: "{{ keystone.bootstrap.project_name | default('admin') }}"
       project_domain_id: "default"
       user_domain_id: "default"
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Create service user for ironic"
@@ -72,8 +73,7 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Associate ironic user to admin role"
@@ -89,8 +89,7 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Create keystone service record for ironic"
@@ -107,8 +106,7 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Check ironic admin endpoint exists"
@@ -214,8 +212,7 @@
       project_name: "{{ keystone.bootstrap.project_name | default('admin') }}"
       project_domain_id: "default"
       user_domain_id: "default"
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Create baremetal_observer role"
@@ -229,8 +226,7 @@
       project_name: "{{ keystone.bootstrap.project_name | default('admin') }}"
       project_domain_id: "default"
       user_domain_id: "default"
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Create baremetal project"
@@ -247,8 +243,7 @@
       project_name: "{{ keystone.bootstrap.project_name | default('admin') }}"
       project_domain_id: "default"
       user_domain_id: "default"
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Create bifrost user"
@@ -265,8 +260,7 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Associate bifrost user with baremetal_admin"
@@ -282,6 +276,5 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true

playbooks/roles/bifrost-ironic-install/tasks/keystone_setup_inspector.yml

@@ -16,10 +16,10 @@
 # We should likely address that at some point, however I think a user
 # should be the driver of that work.
 
-- name: "If VENV is set in the environment, enable installation into venv"
+- name: "Initialize default venv"
   set_fact:
-    enable_venv: true
-  when: lookup('env', 'VENV') | length > 0
+    venv:
+      OS_IDENTITY_API_VERSION: "3"
 
 - name: "Error if credentials are undefined."
   fail:
@@ -38,6 +38,8 @@
       ironic_inspector.keystone.default_username is undefined or
       ironic_inspector.keystone.default_password is undefined
 
+- include: roles/common/venv_python_path.yml
+
 - name: "Create service user for ironic-inspector"
   os_user:
     name: "{{ ironic_inspector.service_catalog.username }}"
@@ -53,8 +55,7 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Associate ironic_inspector user to admin role"
@@ -70,8 +71,7 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Create keystone service record for ironic-inspector"
@@ -88,8 +88,7 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Check ironic-inspector admin endpoint exists"
@@ -198,8 +197,7 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true
 
 - name: "Associate inspector_user with baremetal_admin"
@@ -215,6 +213,5 @@
       project_domain_id: "default"
       user_domain_id: "default"
     wait: yes
-  environment:
-    OS_IDENTITY_API_VERSION: "3"
+  environment: "{{ venv }}"
   no_log: true

playbooks/roles/bifrost-keystone-install/tasks/install.yml

@@ -11,14 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
+- include: roles/common/venv_python_path.yml
+
 - name: "Install packages"
   action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
   with_items: "{{ required_packages }}"
-
-- name: "If VENV is set in the environment, enable installation into venv"
-  set_fact:
-    enable_venv: true
-  when: lookup('env', 'VENV') | length > 0
+  environment: "{{ venv }}"
 
 # NOTE(TheJulia) While we don't necessarilly require /opt/stack any longer
 # and it should already be created by the Ansible setup, we will leave this

playbooks/roles/common/venv_python_path.yml

@@ -0,0 +1,33 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: "If VENV is set in the environment, enable installation into venv"
+  set_fact:
+    enable_venv: true
+  when: lookup('env', 'VENV') | length > 0
+
+- name: "Retrieve venv python path"
+  shell: "echo \"import sys\\nprint(':'.join(sys.path))\" | {{ ansible_python.get('executable', '/usr/bin/python').split('/')[-1] }}"
+  environment: "{{ bifrost_venv_env | default({}) }}"
+  register: venv_pythonpath_result
+  when: enable_venv
+
+- name: "Compute venv python path"
+  set_fact:
+    venv_pythonpath:
+      PYTHONPATH: "{{ venv_pythonpath_result.get('stdout', '') }}"
+  when: enable_venv
+
+- name: "Compute proper complete venv including proper Python path"
+  set_fact:
+    venv: "{{ venv | default({}) | combine(bifrost_venv_env | default({})) | combine(venv_pythonpath | default({})) }}"

requirements.txt

@@ -5,4 +5,5 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0
 oslo.config>=5.2.0 # Apache-2.0
 oslo.log>=3.36.0 # Apache-2.0
 paramiko>=2.0.0 # LGPLv2.1+
+pyOpenSSL>=18.0.0 # Apache-2.0
 setuptools!=24.0.0,!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2,!=36.2.0,>=21.0.0 # PSF/ZPL