From 1ee93450ca35e99cbb3f3f229531c00cac3caa3d Mon Sep 17 00:00:00 2001 From: Nick Jones Date: Mon, 6 Aug 2018 17:26:27 +0100 Subject: [PATCH] Support installing PyPI packages via a mirror Add the ability to specify a list of users for whom a PyPI mirror should be configured for commands such as `pip`. This is accomplished by installing a couple of configuration files in each user's home directory. Change-Id: I21304b32c0c686c8dde2e3e1c0d2e2cd07af1eef Story: 2003315 --- ansible/pip.yml | 8 +++ ansible/roles/pip/defaults/main.yml | 18 +++++++ ansible/roles/pip/tasks/main.yml | 6 +++ ansible/roles/pip/tasks/pip_local_mirror.yml | 51 +++++++++++++++++++ etc/kayobe/pip.yml | 23 +++++++++ kayobe/cli/commands.py | 32 +++++++++--- kayobe/tests/unit/cli/test_commands.py | 7 +++ .../pip-local-mirror-d9a80257c8441970.yaml | 5 ++ 8 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 ansible/pip.yml create mode 100644 ansible/roles/pip/defaults/main.yml create mode 100644 ansible/roles/pip/tasks/main.yml create mode 100644 ansible/roles/pip/tasks/pip_local_mirror.yml create mode 100644 etc/kayobe/pip.yml create mode 100644 releasenotes/notes/pip-local-mirror-d9a80257c8441970.yaml diff --git a/ansible/pip.yml b/ansible/pip.yml new file mode 100644 index 000000000..24aea98da --- /dev/null +++ b/ansible/pip.yml @@ -0,0 +1,8 @@ +--- +- name: Configure local PyPi mirror + hosts: seed-hypervisor:seed:overcloud + tags: + - pip + roles: + - role: pip + gather_facts: false diff --git a/ansible/roles/pip/defaults/main.yml b/ansible/roles/pip/defaults/main.yml new file mode 100644 index 000000000..4377c819e --- /dev/null +++ b/ansible/roles/pip/defaults/main.yml @@ -0,0 +1,18 @@ +--- +pip_local_mirror: false + +# Users for which the necessary configuration will be put in place in order to +# install PyPI packages from a mirror +# NB: The Kolla user will be automatically added to this list if the above is +# set to true +pip_applicable_users: + - "{{ kayobe_ansible_user }}" + - root + +# PyPI local package mirror URL +pip_index_url: "" + +# Optional: a list of 'trusted' hosts for which SSL verification will be +# disabled +pip_trusted_hosts: [] + diff --git a/ansible/roles/pip/tasks/main.yml b/ansible/roles/pip/tasks/main.yml new file mode 100644 index 000000000..e85eb4b84 --- /dev/null +++ b/ansible/roles/pip/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- include_tasks: pip_local_mirror.yml + loop: "{{ pip_applicable_users }}" + loop_control: + loop_var: user + when: pip_local_mirror | bool diff --git a/ansible/roles/pip/tasks/pip_local_mirror.yml b/ansible/roles/pip/tasks/pip_local_mirror.yml new file mode 100644 index 000000000..df6889426 --- /dev/null +++ b/ansible/roles/pip/tasks/pip_local_mirror.yml @@ -0,0 +1,51 @@ +--- +- name: Set a fact about the virtualenv + set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin') + - not ansible_python_interpreter.startswith('/usr/bin') + +- name: Deactivate the virtualenv + include_role: + name: deactivate-virtualenv + when: virtualenv is defined + +- name: Create local .pip directory for {{ user }} + file: + path: "~{{ user }}/.pip" + state: directory + become: True + become_user: "{{ user }}" + +- name: Create pip.conf for {{ user }} + copy: + content: | + [global] + index-url = {{ pip_index_url }} + {% if pip_trusted_hosts | length > 0 -%} + trusted-host = + {% for host in pip_trusted_hosts | unique -%} + {{ host }} + {% endfor -%} + {% endif -%} + dest: "~{{ user}}/.pip/pip.conf" + become: True + become_user: "{{ user }}" + +- name: Create .pydistutils.cfg for {{ user }} + copy: + content: | + [easy_install] + index-url = {{ pip_index_url }} + dest: "~{{ user}}/.pydistutils.cfg" + become: True + become_user: "{{ user }}" + +- name: Activate the virtualenv + include_role: + name: activate-virtualenv + vars: + activate_virtualenv_path: "{{ virtualenv }}" + when: virtualenv is defined diff --git a/etc/kayobe/pip.yml b/etc/kayobe/pip.yml new file mode 100644 index 000000000..29a84f099 --- /dev/null +++ b/etc/kayobe/pip.yml @@ -0,0 +1,23 @@ +--- + +# Use a local PyPi mirror for installing Pip packages +#pip_local_mirror: false + +# Users for which the necessary configuration will be put in place in order to +# install PyPI packages from a mirror +# NB: The Kolla user will be automatically added to this list if the above is +# set to true +#pip_applicable_users: +# - "{{ kayobe_ansible_user }}" +# - root + +# PyPI local package mirror URL +#pip_index_url: "" + +# Optional: a list of 'trusted' hosts for which SSL verification will be +# disabled +#pip_trusted_hosts: [] + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 1c177c032..afc1245da 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -264,6 +264,7 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, * Configure a user account for use by kayobe for SSH access. * Optionally, create a virtualenv for remote target hosts. * Configure user accounts, group associations, and authorised SSH keys. + * Configure a PyPI mirror. * Configure Yum repos. * Configure the host's network interfaces. * Set sysctl parameters. @@ -284,8 +285,8 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, sys.exit(1) playbooks = _build_playbook_list( "ip-allocation", "ssh-known-host", "kayobe-ansible-user", - "kayobe-target-venv", "users", "yum", "dev-tools", "network", - "sysctl", "ntp", "seed-hypervisor-libvirt-host") + "pip", "kayobe-target-venv", "users", "yum", "dev-tools", + "network", "sysctl", "ntp", "seed-hypervisor-libvirt-host") self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed-hypervisor") @@ -347,6 +348,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Optionally, create a virtualenv for remote target hosts. * Optionally, wipe unmounted disk partitions (--wipe-disks). * Configure user accounts, group associations, and authorised SSH keys. + * Configure a PyPI mirror. * Configure Yum repos. * Disable SELinux. * Configure the host's network interfaces. @@ -392,7 +394,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, # Run kayobe playbooks. playbooks = _build_playbook_list( "ip-allocation", "ssh-known-host", "kayobe-ansible-user", - "kayobe-target-venv") + "pip", "kayobe-target-venv") if parsed_args.wipe_disks: playbooks += _build_playbook_list("wipe-disks") playbooks += _build_playbook_list( @@ -419,10 +421,16 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kolla_ansible_seed(parsed_args, "bootstrap-servers", extra_vars=extra_vars) + # Re-run the Pip role after we've bootstrapped the Kolla user + extra_vars = {} + kolla_ansible_user = hostvars.get("kolla_ansible_user") + extra_vars["pip_applicable_users"] = [kolla_ansible_user] + # Run final kayobe playbooks. playbooks = _build_playbook_list( - "kolla-target-venv", "kolla-host", "docker") - self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed") + "pip", "kolla-target-venv", "kolla-host", "docker") + self.run_kayobe_playbooks(parsed_args, playbooks, + extra_vars=extra_vars, limit="seed") class SeedHostPackageUpdate(KayobeAnsibleMixin, VaultMixin, Command): @@ -679,6 +687,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Optionally, create a virtualenv for remote target hosts. * Optionally, wipe unmounted disk partitions (--wipe-disks). * Configure user accounts, group associations, and authorised SSH keys. + * Configure a PyPI mirror. * Configure Yum repos. * Disable SELinux. * Configure the host's network interfaces. @@ -723,7 +732,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, # Kayobe playbooks. playbooks = _build_playbook_list( "ip-allocation", "ssh-known-host", "kayobe-ansible-user", - "kayobe-target-venv") + "pip", "kayobe-target-venv") if parsed_args.wipe_disks: playbooks += _build_playbook_list("wipe-disks") playbooks += _build_playbook_list( @@ -751,10 +760,17 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers", extra_vars=extra_vars) + # Re-run the Pip role after we've bootstrapped the Kolla user + extra_vars = {} + kolla_ansible_user = hostvars.get("kolla_ansible_user") + extra_vars["pip_applicable_users"] = [kolla_ansible_user] + # Further kayobe playbooks. playbooks = _build_playbook_list( - "kolla-target-venv", "kolla-host", "docker", "ceph-block-devices") - self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") + "pip", "kolla-target-venv", "kolla-host", + "docker", "ceph-block-devices") + self.run_kayobe_playbooks(parsed_args, playbooks, + extra_vars=extra_vars, limit="overcloud") class OvercloudHostPackageUpdate(KayobeAnsibleMixin, VaultMixin, Command): diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 37595d4a0..f63f0937b 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -106,6 +106,7 @@ class TestCase(unittest.TestCase): "ansible/ip-allocation.yml", "ansible/ssh-known-host.yml", "ansible/kayobe-ansible-user.yml", + "ansible/pip.yml", "ansible/kayobe-target-venv.yml", "ansible/users.yml", "ansible/yum.yml", @@ -171,6 +172,7 @@ class TestCase(unittest.TestCase): "ansible/ip-allocation.yml", "ansible/ssh-known-host.yml", "ansible/kayobe-ansible-user.yml", + "ansible/pip.yml", "ansible/kayobe-target-venv.yml", "ansible/users.yml", "ansible/yum.yml", @@ -194,11 +196,13 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ + "ansible/pip.yml", "ansible/kolla-target-venv.yml", "ansible/kolla-host.yml", "ansible/docker.yml", ], limit="seed", + extra_vars={'pip_applicable_users': [None]}, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -574,6 +578,7 @@ class TestCase(unittest.TestCase): "ansible/ip-allocation.yml", "ansible/ssh-known-host.yml", "ansible/kayobe-ansible-user.yml", + "ansible/pip.yml", "ansible/kayobe-target-venv.yml", "ansible/users.yml", "ansible/yum.yml", @@ -596,12 +601,14 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ + "ansible/pip.yml", "ansible/kolla-target-venv.yml", "ansible/kolla-host.yml", "ansible/docker.yml", "ansible/ceph-block-devices.yml", ], limit="overcloud", + extra_vars={"pip_applicable_users": [None]}, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) diff --git a/releasenotes/notes/pip-local-mirror-d9a80257c8441970.yaml b/releasenotes/notes/pip-local-mirror-d9a80257c8441970.yaml new file mode 100644 index 000000000..52a0ed83a --- /dev/null +++ b/releasenotes/notes/pip-local-mirror-d9a80257c8441970.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Introduces a new option - ``pip_local_mirror`` - to configure Pip package installation via a user-defined (often local) PyPi mirror. This is set on a per-user basis, and by default this is for the Kayobe Ansible user, the Kolla Ansible user, and root. + See `Story 2003315 `_