Support installing PyPI packages via a mirror

Add the ability to specify a list of users for whom a PyPI mirror should
be configured for commands such as `pip`.  This is accomplished by
installing a couple of configuration files in each user's home directory.

Change-Id: I21304b32c0c686c8dde2e3e1c0d2e2cd07af1eef
Story: 2003315

ansible/pip.yml

@@ -0,0 +1,8 @@
+---
+- name: Configure local PyPi mirror
+  hosts: seed-hypervisor:seed:overcloud
+  tags:
+    - pip
+  roles:
+    - role: pip
+  gather_facts: false

ansible/roles/pip/defaults/main.yml

@@ -0,0 +1,18 @@
+---
+pip_local_mirror: false
+
+# Users for which the necessary configuration will be put in place in order to
+# install PyPI packages from a mirror
+# NB: The Kolla user will be automatically added to this list if the above is
+# set to true
+pip_applicable_users:
+  - "{{ kayobe_ansible_user }}"
+  - root
+
+# PyPI local package mirror URL
+pip_index_url: ""
+
+# Optional: a list of 'trusted' hosts for which SSL verification will be
+# disabled
+pip_trusted_hosts: []
+

ansible/roles/pip/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- include_tasks: pip_local_mirror.yml
+  loop: "{{ pip_applicable_users }}"
+  loop_control:
+    loop_var: user
+  when: pip_local_mirror | bool

ansible/roles/pip/tasks/pip_local_mirror.yml

@@ -0,0 +1,51 @@
+---
+- name: Set a fact about the virtualenv
+  set_fact:
+    virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}"
+  when:
+    - ansible_python_interpreter is defined
+    - not ansible_python_interpreter.startswith('/bin')
+    - not ansible_python_interpreter.startswith('/usr/bin')
+
+- name: Deactivate the virtualenv
+  include_role:
+    name: deactivate-virtualenv
+  when: virtualenv is defined
+
+- name: Create local .pip directory for {{ user }}
+  file:
+    path: "~{{ user }}/.pip"
+    state: directory
+  become: True
+  become_user: "{{ user }}"
+
+- name: Create pip.conf for {{ user }}
+  copy:
+    content: |
+      [global]
+      index-url = {{ pip_index_url }}
+      {% if pip_trusted_hosts | length > 0 -%}
+      trusted-host =
+        {% for host in pip_trusted_hosts | unique -%}
+        {{ host }}
+        {% endfor -%}
+      {% endif -%}
+    dest: "~{{ user}}/.pip/pip.conf"
+  become: True
+  become_user: "{{ user }}"
+
+- name: Create .pydistutils.cfg for {{ user }}
+  copy:
+    content: |
+      [easy_install]
+      index-url = {{ pip_index_url }}
+    dest: "~{{ user}}/.pydistutils.cfg"
+  become: True
+  become_user: "{{ user }}"
+
+- name: Activate the virtualenv
+  include_role:
+    name: activate-virtualenv
+  vars:
+    activate_virtualenv_path: "{{ virtualenv }}"
+  when: virtualenv is defined

etc/kayobe/pip.yml

@@ -0,0 +1,23 @@
+---
+
+# Use a local PyPi mirror for installing Pip packages
+#pip_local_mirror: false
+
+# Users for which the necessary configuration will be put in place in order to
+# install PyPI packages from a mirror
+# NB: The Kolla user will be automatically added to this list if the above is
+# set to true
+#pip_applicable_users:
+#  - "{{ kayobe_ansible_user }}"
+#  - root
+
+# PyPI local package mirror URL
+#pip_index_url: ""
+
+# Optional: a list of 'trusted' hosts for which SSL verification will be
+# disabled
+#pip_trusted_hosts: []
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes

kayobe/cli/commands.py

@@ -264,6 +264,7 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
     * Configure a user account for use by kayobe for SSH access.
     * Optionally, create a virtualenv for remote target hosts.
     * Configure user accounts, group associations, and authorised SSH keys.
+    * Configure a PyPI mirror.
     * Configure Yum repos.
     * Configure the host's network interfaces.
     * Set sysctl parameters.
@@ -284,8 +285,8 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
             sys.exit(1)
         playbooks = _build_playbook_list(
             "ip-allocation", "ssh-known-host", "kayobe-ansible-user",
-            "kayobe-target-venv", "users", "yum", "dev-tools", "network",
+            "pip", "kayobe-target-venv", "users", "yum", "dev-tools",
-            "sysctl", "ntp", "seed-hypervisor-libvirt-host")
+            "network", "sysctl", "ntp", "seed-hypervisor-libvirt-host")
         self.run_kayobe_playbooks(parsed_args, playbooks,
                                   limit="seed-hypervisor")
 
@@ -347,6 +348,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Optionally, create a virtualenv for remote target hosts.
     * Optionally, wipe unmounted disk partitions (--wipe-disks).
     * Configure user accounts, group associations, and authorised SSH keys.
+    * Configure a PyPI mirror.
     * Configure Yum repos.
     * Disable SELinux.
     * Configure the host's network interfaces.
@@ -392,7 +394,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         # Run kayobe playbooks.
         playbooks = _build_playbook_list(
             "ip-allocation", "ssh-known-host", "kayobe-ansible-user",
-            "kayobe-target-venv")
+            "pip", "kayobe-target-venv")
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
         playbooks += _build_playbook_list(
@@ -419,10 +421,16 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kolla_ansible_seed(parsed_args, "bootstrap-servers",
                                     extra_vars=extra_vars)
 
+        # Re-run the Pip role after we've bootstrapped the Kolla user
+        extra_vars = {}
+        kolla_ansible_user = hostvars.get("kolla_ansible_user")
+        extra_vars["pip_applicable_users"] = [kolla_ansible_user]
+
         # Run final kayobe playbooks.
         playbooks = _build_playbook_list(
-            "kolla-target-venv", "kolla-host", "docker")
+            "pip", "kolla-target-venv", "kolla-host", "docker")
-        self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed")
+        self.run_kayobe_playbooks(parsed_args, playbooks,
+                                  extra_vars=extra_vars, limit="seed")
 
 
 class SeedHostPackageUpdate(KayobeAnsibleMixin, VaultMixin, Command):
@@ -679,6 +687,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Optionally, create a virtualenv for remote target hosts.
     * Optionally, wipe unmounted disk partitions (--wipe-disks).
     * Configure user accounts, group associations, and authorised SSH keys.
+    * Configure a PyPI mirror.
     * Configure Yum repos.
     * Disable SELinux.
     * Configure the host's network interfaces.
@@ -723,7 +732,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         # Kayobe playbooks.
         playbooks = _build_playbook_list(
             "ip-allocation", "ssh-known-host", "kayobe-ansible-user",
-            "kayobe-target-venv")
+            "pip", "kayobe-target-venv")
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
         playbooks += _build_playbook_list(
@@ -751,10 +760,17 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers",
                                          extra_vars=extra_vars)
 
+        # Re-run the Pip role after we've bootstrapped the Kolla user
+        extra_vars = {}
+        kolla_ansible_user = hostvars.get("kolla_ansible_user")
+        extra_vars["pip_applicable_users"] = [kolla_ansible_user]
+
         # Further kayobe playbooks.
         playbooks = _build_playbook_list(
-            "kolla-target-venv", "kolla-host", "docker", "ceph-block-devices")
+            "pip", "kolla-target-venv", "kolla-host",
-        self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
+            "docker", "ceph-block-devices")
+        self.run_kayobe_playbooks(parsed_args, playbooks,
+                                  extra_vars=extra_vars, limit="overcloud")
 
 
 class OvercloudHostPackageUpdate(KayobeAnsibleMixin, VaultMixin, Command):

kayobe/tests/unit/cli/test_commands.py

@@ -106,6 +106,7 @@ class TestCase(unittest.TestCase):
                     "ansible/ip-allocation.yml",
                     "ansible/ssh-known-host.yml",
                     "ansible/kayobe-ansible-user.yml",
+                    "ansible/pip.yml",
                     "ansible/kayobe-target-venv.yml",
                     "ansible/users.yml",
                     "ansible/yum.yml",
@@ -171,6 +172,7 @@ class TestCase(unittest.TestCase):
                     "ansible/ip-allocation.yml",
                     "ansible/ssh-known-host.yml",
                     "ansible/kayobe-ansible-user.yml",
+                    "ansible/pip.yml",
                     "ansible/kayobe-target-venv.yml",
                     "ansible/users.yml",
                     "ansible/yum.yml",
@@ -194,11 +196,13 @@ class TestCase(unittest.TestCase):
             mock.call(
                 mock.ANY,
                 [
+                    "ansible/pip.yml",
                     "ansible/kolla-target-venv.yml",
                     "ansible/kolla-host.yml",
                     "ansible/docker.yml",
                 ],
                 limit="seed",
+                extra_vars={'pip_applicable_users': [None]},
             ),
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
@@ -574,6 +578,7 @@ class TestCase(unittest.TestCase):
                     "ansible/ip-allocation.yml",
                     "ansible/ssh-known-host.yml",
                     "ansible/kayobe-ansible-user.yml",
+                    "ansible/pip.yml",
                     "ansible/kayobe-target-venv.yml",
                     "ansible/users.yml",
                     "ansible/yum.yml",
@@ -596,12 +601,14 @@ class TestCase(unittest.TestCase):
             mock.call(
                 mock.ANY,
                 [
+                    "ansible/pip.yml",
                     "ansible/kolla-target-venv.yml",
                     "ansible/kolla-host.yml",
                     "ansible/docker.yml",
                     "ansible/ceph-block-devices.yml",
                 ],
                 limit="overcloud",
+                extra_vars={"pip_applicable_users": [None]},
             ),
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)

releasenotes/notes/pip-local-mirror-d9a80257c8441970.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Introduces a new option - ``pip_local_mirror`` - to configure Pip package installation via a user-defined (often local) PyPi mirror.  This is set on a per-user basis, and by default this is for the Kayobe Ansible user, the Kolla Ansible user, and root.
+    See `Story 2003315 <https://storyboard.openstack.org/#!/story/2003315>`_