openstack/kayobe
Code Issues Proposed changes

Support installing PyPI packages via a mirror

Browse Source 
Add the ability to specify a list of users for whom a PyPI mirror should
be configured for commands such as `pip`.  This is accomplished by
installing a couple of configuration files in each user's home directory.

Change-Id: I21304b32c0c686c8dde2e3e1c0d2e2cd07af1eef
Story: 2003315
This commit is contained in:
Nick Jones 2018-08-06 17:26:27 +01:00 committed by Mark Goddard
parent 1ee04846f6
commit 1ee93450ca
8 changed files with 142 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ansible/pip.yml Normal file
View File

@ -0,0 +1,8 @@
---
- name: Configure local PyPi mirror
hosts: seed-hypervisor:seed:overcloud
tags:
- pip
roles:
- role: pip
gather_facts: false

18
ansible/roles/pip/defaults/main.yml Normal file
View File

@ -0,0 +1,18 @@
---
pip_local_mirror: false
# Users for which the necessary configuration will be put in place in order to
# install PyPI packages from a mirror
# NB: The Kolla user will be automatically added to this list if the above is
# set to true
pip_applicable_users:
- "{{ kayobe_ansible_user }}"
- root
# PyPI local package mirror URL
pip_index_url: ""
# Optional: a list of 'trusted' hosts for which SSL verification will be
# disabled
pip_trusted_hosts: []

6
ansible/roles/pip/tasks/main.yml Normal file
View File

@ -0,0 +1,6 @@
---
- include_tasks: pip_local_mirror.yml
loop: "{{ pip_applicable_users }}"
loop_control:
loop_var: user
when: pip_local_mirror | bool

51
ansible/roles/pip/tasks/pip_local_mirror.yml Normal file
View File

@ -0,0 +1,51 @@
---
- name: Set a fact about the virtualenv
set_fact:
virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}"
when:
- ansible_python_interpreter is defined
- not ansible_python_interpreter.startswith('/bin')
- not ansible_python_interpreter.startswith('/usr/bin')
- name: Deactivate the virtualenv
include_role:
name: deactivate-virtualenv
when: virtualenv is defined
- name: Create local .pip directory for {{ user }}
file:
path: "~{{ user }}/.pip"
state: directory
become: True
become_user: "{{ user }}"
- name: Create pip.conf for {{ user }}
copy:
content: |
[global]
index-url = {{ pip_index_url }}
{% if pip_trusted_hosts | length > 0 -%}
trusted-host =
{% for host in pip_trusted_hosts | unique -%}
{{ host }}
{% endfor -%}
{% endif -%}
dest: "~{{ user}}/.pip/pip.conf"
become: True
become_user: "{{ user }}"
- name: Create .pydistutils.cfg for {{ user }}
copy:
content: |
[easy_install]
index-url = {{ pip_index_url }}
dest: "~{{ user}}/.pydistutils.cfg"
become: True
become_user: "{{ user }}"
- name: Activate the virtualenv
include_role:
name: activate-virtualenv
vars:
activate_virtualenv_path: "{{ virtualenv }}"
when: virtualenv is defined

23
etc/kayobe/pip.yml Normal file
View File

@ -0,0 +1,23 @@
---
# Use a local PyPi mirror for installing Pip packages
#pip_local_mirror: false
# Users for which the necessary configuration will be put in place in order to
# install PyPI packages from a mirror
# NB: The Kolla user will be automatically added to this list if the above is
# set to true
#pip_applicable_users:
# - "{{ kayobe_ansible_user }}"
# - root
# PyPI local package mirror URL
#pip_index_url: ""
# Optional: a list of 'trusted' hosts for which SSL verification will be
# disabled
#pip_trusted_hosts: []
###############################################################################
# Dummy variable to allow Ansible to accept this file.
workaround_ansible_issue_8743: yes

32
kayobe/cli/commands.py
View File

@ -264,6 +264,7 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
* Configure a user account for use by kayobe for SSH access.
* Optionally, create a virtualenv for remote target hosts.
* Configure user accounts, group associations, and authorised SSH keys.
* Configure a PyPI mirror.
* Configure Yum repos.
* Configure the host's network interfaces.
* Set sysctl parameters.
@ -284,8 +285,8 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
sys.exit(1)
playbooks = _build_playbook_list(
"ip-allocation", "ssh-known-host", "kayobe-ansible-user",
"kayobe-target-venv", "users", "yum", "dev-tools", "network",
"sysctl", "ntp", "seed-hypervisor-libvirt-host")
"pip", "kayobe-target-venv", "users", "yum", "dev-tools",
"network", "sysctl", "ntp", "seed-hypervisor-libvirt-host")
self.run_kayobe_playbooks(parsed_args, playbooks,
limit="seed-hypervisor")
@ -347,6 +348,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
* Optionally, create a virtualenv for remote target hosts.
* Optionally, wipe unmounted disk partitions (--wipe-disks).
* Configure user accounts, group associations, and authorised SSH keys.
* Configure a PyPI mirror.
* Configure Yum repos.
* Disable SELinux.
* Configure the host's network interfaces.
@ -392,7 +394,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
# Run kayobe playbooks.
playbooks = _build_playbook_list(
"ip-allocation", "ssh-known-host", "kayobe-ansible-user",
"kayobe-target-venv")
"pip", "kayobe-target-venv")
if parsed_args.wipe_disks:
playbooks += _build_playbook_list("wipe-disks")
playbooks += _build_playbook_list(
@ -419,10 +421,16 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
self.run_kolla_ansible_seed(parsed_args, "bootstrap-servers",
extra_vars=extra_vars)
# Re-run the Pip role after we've bootstrapped the Kolla user
extra_vars = {}
kolla_ansible_user = hostvars.get("kolla_ansible_user")
extra_vars["pip_applicable_users"] = [kolla_ansible_user]
# Run final kayobe playbooks.
playbooks = _build_playbook_list(
"kolla-target-venv", "kolla-host", "docker")
self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed")
"pip", "kolla-target-venv", "kolla-host", "docker")
self.run_kayobe_playbooks(parsed_args, playbooks,
extra_vars=extra_vars, limit="seed")
class SeedHostPackageUpdate(KayobeAnsibleMixin, VaultMixin, Command):
@ -679,6 +687,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
* Optionally, create a virtualenv for remote target hosts.
* Optionally, wipe unmounted disk partitions (--wipe-disks).
* Configure user accounts, group associations, and authorised SSH keys.
* Configure a PyPI mirror.
* Configure Yum repos.
* Disable SELinux.
* Configure the host's network interfaces.
@ -723,7 +732,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
# Kayobe playbooks.
playbooks = _build_playbook_list(
"ip-allocation", "ssh-known-host", "kayobe-ansible-user",
"kayobe-target-venv")
"pip", "kayobe-target-venv")
if parsed_args.wipe_disks:
playbooks += _build_playbook_list("wipe-disks")
playbooks += _build_playbook_list(
@ -751,10 +760,17 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers",
extra_vars=extra_vars)
# Re-run the Pip role after we've bootstrapped the Kolla user
extra_vars = {}
kolla_ansible_user = hostvars.get("kolla_ansible_user")
extra_vars["pip_applicable_users"] = [kolla_ansible_user]
# Further kayobe playbooks.
playbooks = _build_playbook_list(
"kolla-target-venv", "kolla-host", "docker", "ceph-block-devices")
self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
"pip", "kolla-target-venv", "kolla-host",
"docker", "ceph-block-devices")
self.run_kayobe_playbooks(parsed_args, playbooks,
extra_vars=extra_vars, limit="overcloud")
class OvercloudHostPackageUpdate(KayobeAnsibleMixin, VaultMixin, Command):

7
kayobe/tests/unit/cli/test_commands.py
View File

@ -106,6 +106,7 @@ class TestCase(unittest.TestCase):
"ansible/ip-allocation.yml",
"ansible/ssh-known-host.yml",
"ansible/kayobe-ansible-user.yml",
"ansible/pip.yml",
"ansible/kayobe-target-venv.yml",
"ansible/users.yml",
"ansible/yum.yml",
@ -171,6 +172,7 @@ class TestCase(unittest.TestCase):
"ansible/ip-allocation.yml",
"ansible/ssh-known-host.yml",
"ansible/kayobe-ansible-user.yml",
"ansible/pip.yml",
"ansible/kayobe-target-venv.yml",
"ansible/users.yml",
"ansible/yum.yml",
@ -194,11 +196,13 @@ class TestCase(unittest.TestCase):
mock.call(
mock.ANY,
[
"ansible/pip.yml",
"ansible/kolla-target-venv.yml",
"ansible/kolla-host.yml",
"ansible/docker.yml",
],
limit="seed",
extra_vars={'pip_applicable_users': [None]},
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@ -574,6 +578,7 @@ class TestCase(unittest.TestCase):
"ansible/ip-allocation.yml",
"ansible/ssh-known-host.yml",
"ansible/kayobe-ansible-user.yml",
"ansible/pip.yml",
"ansible/kayobe-target-venv.yml",
"ansible/users.yml",
"ansible/yum.yml",
@ -596,12 +601,14 @@ class TestCase(unittest.TestCase):
mock.call(
mock.ANY,
[
"ansible/pip.yml",
"ansible/kolla-target-venv.yml",
"ansible/kolla-host.yml",
"ansible/docker.yml",
"ansible/ceph-block-devices.yml",
],
limit="overcloud",
extra_vars={"pip_applicable_users": [None]},
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)

5
releasenotes/notes/pip-local-mirror-d9a80257c8441970.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
Introduces a new option - ``pip_local_mirror`` - to configure Pip package installation via a user-defined (often local) PyPi mirror. This is set on a per-user basis, and by default this is for the Kayobe Ansible user, the Kolla Ansible user, and root.
See `Story 2003315 <https://storyboard.openstack.org/#!/story/2003315>`_