From c6e5c9a74e915915f1aaf1948c46a3f8afe9b823 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 15 Sep 2014 11:38:32 -0500 Subject: [PATCH] Adjusted pip install config and repos This PR ensures that the environment is always pointing at a frozen repository of PIP packages which consists of prebuilt wheels. A script was created to allow for packages and apps to be easily updated. Resolves: * https://github.com/rcbops/ansible-lxc-rpc/issues/113 * https://github.com/rcbops/ansible-lxc-rpc/issues/120 Begins resolving: * https://github.com/rcbops/ansible-lxc-rpc/issues/31 * https://github.com/rcbops/ansible-lxc-rpc/issues/53 set all project group_vars to a given sha added retry added tempest --- requirements.txt | 4 + rpc_deployment/inventory/group_vars/all.yml | 7 +- .../inventory/group_vars/cinder_all.yml | 4 +- .../inventory/group_vars/glance_all.yml | 4 +- .../inventory/group_vars/heat_all.yml | 4 +- .../inventory/group_vars/keystone_all.yml | 2 +- .../inventory/group_vars/kibana.yml | 2 +- .../inventory/group_vars/neutron_all.yml | 4 +- .../inventory/group_vars/nova_all.yml | 4 +- .../infrastructure/galera-post-config.yml | 1 + .../playbooks/infrastructure/repo-install.yml | 0 ...{host-common.yml => containers-common.yml} | 0 rpc_deployment/playbooks/setup/host-setup.yml | 3 +- rpc_deployment/roles/common/tasks/get_pip.yml | 50 +- .../common/templates/pip/base/global.conf.j2 | 4 + .../templates/pip/link/rpc_release.link | 1 + .../common/templates/pip/pip-link-build.py | 177 +++++ .../roles/common/templates/rpc-release | 2 +- .../openstack_common/tasks/pip_install.yml | 3 +- .../vars/config_vars/container_config.yml | 29 + .../vars/repo_packages/all_common.yml | 78 +++ .../vars/repo_packages/ansible_rpc_lxc.yml | 22 + rpc_deployment/vars/repo_packages/cinder.yml | 45 ++ .../vars/repo_packages/elasticsearch.yml | 29 + rpc_deployment/vars/repo_packages/galera.yml | 49 ++ rpc_deployment/vars/repo_packages/glance.yml | 40 ++ rpc_deployment/vars/repo_packages/haproxy.yml | 21 + rpc_deployment/vars/repo_packages/heat.yml | 40 ++ rpc_deployment/vars/repo_packages/holland.yml | 36 + rpc_deployment/vars/repo_packages/hosts.yml | 29 + .../vars/repo_packages/keystone.yml | 50 ++ .../vars/repo_packages/keystonemiddleware.yml | 24 + rpc_deployment/vars/repo_packages/kibana.yml | 24 + .../vars/repo_packages/logstash.yml | 28 + .../vars/repo_packages/memcached.yml | 19 + rpc_deployment/vars/repo_packages/neutron.yml | 43 ++ rpc_deployment/vars/repo_packages/nova.yml | 41 ++ .../vars/repo_packages/nova_libvirt.yml | 22 + .../vars/repo_packages/nova_spice_console.yml | 17 + .../openstack_global_requirements.yml | 27 + .../repo_packages/python_barbicanclient.yml | 24 + .../repo_packages/python_ceilometerclient.yml | 24 + .../repo_packages/python_cinderclient.yml | 24 + .../repo_packages/python_designateclient.yml | 24 + .../repo_packages/python_glanceclient.yml | 24 + .../vars/repo_packages/python_heatclient.yml | 24 + .../repo_packages/python_ironicclient.yml | 24 + .../repo_packages/python_keystoneclient.yml | 24 + .../repo_packages/python_neutronclient.yml | 24 + .../vars/repo_packages/python_novaclient.yml | 24 + .../repo_packages/python_openstackclient.yml | 24 + .../repo_packages/python_saharaclient.yml | 24 + .../vars/repo_packages/python_swiftclient.yml | 24 + .../vars/repo_packages/python_troveclient.yml | 24 + .../repo_packages/python_tuskarclient.yml | 24 + .../vars/repo_packages/python_zaqarclient.yml | 24 + .../vars/repo_packages/rabbitmq.yml | 26 + rpc_deployment/vars/repo_packages/raxmon.yml | 34 + rpc_deployment/vars/repo_packages/readme.rst | 39 ++ .../vars/repo_packages/rpc_support.yml | 23 + rpc_deployment/vars/repo_packages/rsyslog.yml | 19 + rpc_deployment/vars/repo_packages/swift.yml | 26 + rpc_deployment/vars/repo_packages/tempest.yml | 26 + .../vars/repo_packages/turbolift.yml | 23 + rpc_deployment/vars/repo_packages/utility.yml | 30 + scripts/rpc-wheel-builder.py | 632 ++++++++++++++++++ 66 files changed, 2230 insertions(+), 21 deletions(-) create mode 100644 rpc_deployment/playbooks/infrastructure/repo-install.yml rename rpc_deployment/playbooks/setup/{host-common.yml => containers-common.yml} (100%) create mode 100644 rpc_deployment/roles/common/templates/pip/base/global.conf.j2 create mode 100644 rpc_deployment/roles/common/templates/pip/link/rpc_release.link create mode 100644 rpc_deployment/roles/common/templates/pip/pip-link-build.py create mode 100644 rpc_deployment/vars/config_vars/container_config.yml create mode 100644 rpc_deployment/vars/repo_packages/all_common.yml create mode 100644 rpc_deployment/vars/repo_packages/ansible_rpc_lxc.yml create mode 100644 rpc_deployment/vars/repo_packages/cinder.yml create mode 100644 rpc_deployment/vars/repo_packages/elasticsearch.yml create mode 100644 rpc_deployment/vars/repo_packages/galera.yml create mode 100644 rpc_deployment/vars/repo_packages/glance.yml create mode 100644 rpc_deployment/vars/repo_packages/haproxy.yml create mode 100644 rpc_deployment/vars/repo_packages/heat.yml create mode 100644 rpc_deployment/vars/repo_packages/holland.yml create mode 100644 rpc_deployment/vars/repo_packages/hosts.yml create mode 100644 rpc_deployment/vars/repo_packages/keystone.yml create mode 100644 rpc_deployment/vars/repo_packages/keystonemiddleware.yml create mode 100644 rpc_deployment/vars/repo_packages/kibana.yml create mode 100644 rpc_deployment/vars/repo_packages/logstash.yml create mode 100644 rpc_deployment/vars/repo_packages/memcached.yml create mode 100644 rpc_deployment/vars/repo_packages/neutron.yml create mode 100644 rpc_deployment/vars/repo_packages/nova.yml create mode 100644 rpc_deployment/vars/repo_packages/nova_libvirt.yml create mode 100644 rpc_deployment/vars/repo_packages/nova_spice_console.yml create mode 100644 rpc_deployment/vars/repo_packages/openstack_global_requirements.yml create mode 100644 rpc_deployment/vars/repo_packages/python_barbicanclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_ceilometerclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_cinderclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_designateclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_glanceclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_heatclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_ironicclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_keystoneclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_neutronclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_novaclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_openstackclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_saharaclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_swiftclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_troveclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_tuskarclient.yml create mode 100644 rpc_deployment/vars/repo_packages/python_zaqarclient.yml create mode 100644 rpc_deployment/vars/repo_packages/rabbitmq.yml create mode 100644 rpc_deployment/vars/repo_packages/raxmon.yml create mode 100644 rpc_deployment/vars/repo_packages/readme.rst create mode 100644 rpc_deployment/vars/repo_packages/rpc_support.yml create mode 100644 rpc_deployment/vars/repo_packages/rsyslog.yml create mode 100644 rpc_deployment/vars/repo_packages/swift.yml create mode 100644 rpc_deployment/vars/repo_packages/tempest.yml create mode 100644 rpc_deployment/vars/repo_packages/turbolift.yml create mode 100644 rpc_deployment/vars/repo_packages/utility.yml create mode 100755 scripts/rpc-wheel-builder.py diff --git a/requirements.txt b/requirements.txt index c26242813e..8651e03970 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,7 @@ prettytable==0.7.2 pycrypto==2.6.1 wsgiref==0.1.2 pexpect==3.3 +requests==2.4.1 +cloudlib==0.0.8 +pip==1.5.6 +wheel==0.24.0 diff --git a/rpc_deployment/inventory/group_vars/all.yml b/rpc_deployment/inventory/group_vars/all.yml index de292e2ca2..20c73fac99 100644 --- a/rpc_deployment/inventory/group_vars/all.yml +++ b/rpc_deployment/inventory/group_vars/all.yml @@ -40,8 +40,8 @@ external_vip_address: "{{ external_lb_vip_address }}" ## URL for the frozen rpc repo -rpc_repo_url: "http://dc0e2a2ef0676c3453b1-31bb9324d3aeab0d08fa434012c1e64d.r5.cf1.rackcdn.com" - +rpc_repo_url: "http://rpc-slushee.rackspace.com" +rpc_release: "9.0.0" ## GPG Keys gpg_keys: @@ -55,7 +55,7 @@ apt_common_repos: apt_lxc_common_repos: - { repo: "ppa:ubuntu-lxc/stable", state: "present" } -get_pip_url: "https://bootstrap.pypa.io/get-pip.py" +get_pip_url: "http://rpc-slushee.rackspace.com/downloads/get-pip.py" ## Users that will not be created via container_common @@ -92,6 +92,7 @@ apt_common_packages: - libffi-dev - libxml2-dev - libxslt1-dev + - libsqlite3-dev - mariadb-client - libmariadbclient-dev diff --git a/rpc_deployment/inventory/group_vars/cinder_all.yml b/rpc_deployment/inventory/group_vars/cinder_all.yml index 85b5bacc44..40a2c3ea0b 100644 --- a/rpc_deployment/inventory/group_vars/cinder_all.yml +++ b/rpc_deployment/inventory/group_vars/cinder_all.yml @@ -59,10 +59,12 @@ service_names: - cinder-volume ## Git Source +## ALL of this has been relocated to vars/repo_packages +## TODO(someone) this should be removed once the repo bits are all figured out. git_repo: https://git.openstack.org/openstack/cinder git_fallback_repo: https://github.com/openstack/cinder git_etc_example: etc/cinder/ -git_install_branch: stable/icehouse +git_install_branch: 36406c9bb0bac99b5daf1b8194df73685cbe49ea service_pip_dependencies: - pywbem diff --git a/rpc_deployment/inventory/group_vars/glance_all.yml b/rpc_deployment/inventory/group_vars/glance_all.yml index 5a5c048163..38f1e1a783 100644 --- a/rpc_deployment/inventory/group_vars/glance_all.yml +++ b/rpc_deployment/inventory/group_vars/glance_all.yml @@ -71,10 +71,12 @@ service_names: flavor: "keystone+cachemanagement" ## Git Source +## ALL of this has been relocated to vars/repo_packages +## TODO(someone) this should be removed once the repo bits are all figured out. git_repo: https://git.openstack.org/openstack/glance git_fallback_repo: https://github.com/openstack/glance git_etc_example: etc/ -git_install_branch: stable/icehouse +git_install_branch: d0453ae453adeacf19d121f08f8f9691f11f435a service_pip_dependencies: - warlock diff --git a/rpc_deployment/inventory/group_vars/heat_all.yml b/rpc_deployment/inventory/group_vars/heat_all.yml index 9c489e2407..111f35ebd9 100644 --- a/rpc_deployment/inventory/group_vars/heat_all.yml +++ b/rpc_deployment/inventory/group_vars/heat_all.yml @@ -64,10 +64,12 @@ heat_waitcondition_server_url: "http://{{ internal_vip_address }}:8000/v1/waitco heat_metadata_server_url: "http://{{ internal_vip_address }}:8000" ## Git Source +## ALL of this has been relocated to vars/repo_packages +## TODO(someone) this should be removed once the repo bits are all figured out. git_repo: https://git.openstack.org/openstack/heat git_fallback_repo: https://github.com/openstack/heat git_etc_example: etc/heat -git_install_branch: stable/icehouse +git_install_branch: 93f3c78a8bd9ecb79fa3d719468b371e67519969 service_pip_dependencies: - MySQL-python diff --git a/rpc_deployment/inventory/group_vars/keystone_all.yml b/rpc_deployment/inventory/group_vars/keystone_all.yml index 1f1f80d38b..f9ef13dca5 100644 --- a/rpc_deployment/inventory/group_vars/keystone_all.yml +++ b/rpc_deployment/inventory/group_vars/keystone_all.yml @@ -57,7 +57,7 @@ system_group: keystone git_repo: https://git.openstack.org/openstack/keystone git_fallback_repo: https://github.com/openstack/keystone git_etc_example: etc/ -git_install_branch: stable/icehouse +git_install_branch: 7c2bee76936461043671adc6d100f20775c4212f # Common PIP Packages service_pip_dependencies: diff --git a/rpc_deployment/inventory/group_vars/kibana.yml b/rpc_deployment/inventory/group_vars/kibana.yml index c33f6ab7c3..9228742ef5 100644 --- a/rpc_deployment/inventory/group_vars/kibana.yml +++ b/rpc_deployment/inventory/group_vars/kibana.yml @@ -22,7 +22,7 @@ verbose: True container_lvm_fstype: ext4 container_lvm_fssize: 5GB -kibana_url: https://download.elasticsearch.org/kibana/kibana/kibana-3.1.0.tar.gz +kibana_url: http://rpc-slushee.rackspace.com/downloads/kibana-3.1.0.tar.gz kibana_root: /opt/kibana diff --git a/rpc_deployment/inventory/group_vars/neutron_all.yml b/rpc_deployment/inventory/group_vars/neutron_all.yml index 420deb29de..392cfa2fe7 100644 --- a/rpc_deployment/inventory/group_vars/neutron_all.yml +++ b/rpc_deployment/inventory/group_vars/neutron_all.yml @@ -77,10 +77,12 @@ service_names: - neutron-server ## Git Source +## ALL of this has been relocated to vars/repo_packages +## TODO(someone) this should be removed once the repo bits are all figured out. git_repo: https://git.openstack.org/openstack/neutron git_fallback_repo: https://github.com/openstack/neutron git_etc_example: etc/ -git_install_branch: stable/icehouse +git_install_branch: fd1221124b8f53a61731e51b5f923af44ec5598f service_pip_dependencies: - MySQL-python diff --git a/rpc_deployment/inventory/group_vars/nova_all.yml b/rpc_deployment/inventory/group_vars/nova_all.yml index 4c54cc5bdc..a65fa90200 100644 --- a/rpc_deployment/inventory/group_vars/nova_all.yml +++ b/rpc_deployment/inventory/group_vars/nova_all.yml @@ -68,10 +68,12 @@ nova_compute_driver: libvirt.LibvirtDriver nova_max_age: 0 ## Git Source +## ALL of this has been relocated to vars/repo_packages +## TODO(someone) this should be removed once the repo bits are all figured out. git_repo: https://git.openstack.org/openstack/nova git_fallback_repo: https://github.com/openstack/nova git_etc_example: etc/nova/ -git_install_branch: stable/icehouse +git_install_branch: 9ada982ccdb974d43c1c94e59eb4b84e35a7997d service_pip_dependencies: - MySQL-python diff --git a/rpc_deployment/playbooks/infrastructure/galera-post-config.yml b/rpc_deployment/playbooks/infrastructure/galera-post-config.yml index 3b59e08839..dc73beffbe 100644 --- a/rpc_deployment/playbooks/infrastructure/galera-post-config.yml +++ b/rpc_deployment/playbooks/infrastructure/galera-post-config.yml @@ -22,3 +22,4 @@ user: root roles: - galera_post_config + diff --git a/rpc_deployment/playbooks/infrastructure/repo-install.yml b/rpc_deployment/playbooks/infrastructure/repo-install.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rpc_deployment/playbooks/setup/host-common.yml b/rpc_deployment/playbooks/setup/containers-common.yml similarity index 100% rename from rpc_deployment/playbooks/setup/host-common.yml rename to rpc_deployment/playbooks/setup/containers-common.yml diff --git a/rpc_deployment/playbooks/setup/host-setup.yml b/rpc_deployment/playbooks/setup/host-setup.yml index 054e3bfb25..f3fbe4540f 100644 --- a/rpc_deployment/playbooks/setup/host-setup.yml +++ b/rpc_deployment/playbooks/setup/host-setup.yml @@ -16,5 +16,4 @@ - include: setup-common.yml - include: build-containers.yml - include: restart-containers.yml -- include: host-common.yml - +- include: containers-common.yml diff --git a/rpc_deployment/roles/common/tasks/get_pip.yml b/rpc_deployment/roles/common/tasks/get_pip.yml index 057dbd493d..f819519974 100644 --- a/rpc_deployment/roles/common/tasks/get_pip.yml +++ b/rpc_deployment/roles/common/tasks/get_pip.yml @@ -14,21 +14,57 @@ # limitations under the License. - name: Create /opt - file: > - state=directory - path=/opt + file: + state: "directory" + path: "/opt" - name: Get Modern PIP - get_url: > - url="{{ get_pip_url }}" - dest="/opt/get-pip.py" - validate_certs=no + get_url: + url: "{{ get_pip_url }}" + dest: "/opt/get-pip.py" + validate_certs: "no" register: get_pip until: get_pip|success retries: 3 +- name: Create pip config directory + file: + path: "{{ item }}" + state: "directory" + group: "{{ ansible_user_id }}" + owner: "{{ ansible_user_id }}" + with_items: + - "{{ ansible_env.HOME }}/.pip" + - "{{ ansible_env.HOME }}/.pip/base" + - "{{ ansible_env.HOME }}/.pip/links.d" + +- name: Drop pip config file(s) + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ ansible_user_id }}" + group: "{{ ansible_user_id }}" + mode: "{{ item.mode }}" + with_items: + - { src: "pip/pip-link-build.py", dest: "{{ ansible_env.HOME }}/.pip/pip-link-build.py", mode: "0755" } + - { src: "pip/base/global.conf.j2", dest: "{{ ansible_env.HOME }}/.pip/base/global.conf", mode: "0644" } + +- name: Drop release pip link + template: + src: "pip/link/rpc_release.link" + dest: "{{ ansible_env.HOME }}/.pip/links.d/rpc_release.link" + owner: "{{ ansible_user_id }}" + group: "{{ ansible_user_id }}" + mode: "0644" + register: pip_link + +- name: Execute pip config builder + command: "{{ ansible_env.HOME }}/.pip/pip-link-build.py" + changed_when: false + - name: Install PIP shell: "python /opt/get-pip.py" + changed_when: false register: pip_install until: pip_install|success retries: 3 diff --git a/rpc_deployment/roles/common/templates/pip/base/global.conf.j2 b/rpc_deployment/roles/common/templates/pip/base/global.conf.j2 new file mode 100644 index 0000000000..ad5ca97ad8 --- /dev/null +++ b/rpc_deployment/roles/common/templates/pip/base/global.conf.j2 @@ -0,0 +1,4 @@ +[global] +no-index = true +pre = true +timeout = 120 diff --git a/rpc_deployment/roles/common/templates/pip/link/rpc_release.link b/rpc_deployment/roles/common/templates/pip/link/rpc_release.link new file mode 100644 index 0000000000..38211488e3 --- /dev/null +++ b/rpc_deployment/roles/common/templates/pip/link/rpc_release.link @@ -0,0 +1 @@ +{{ rpc_repo_url }}/python_packages/{{ rpc_release }}/ diff --git a/rpc_deployment/roles/common/templates/pip/pip-link-build.py b/rpc_deployment/roles/common/templates/pip/pip-link-build.py new file mode 100644 index 0000000000..a892ad4d3d --- /dev/null +++ b/rpc_deployment/roles/common/templates/pip/pip-link-build.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2014, Kevin Carter + +""" +This script will build a pip.conf file dynamically based on a simple +configuration layout. The purpose of this script is to allow automation to +deploy parts of the main `pip.conf` file incrementally creating links and +sections as needed. + +Structure: + $HOME/.pip/ + $HOME/.pip/base + $HOME/.pip/links.d + +creates: + $HOME/.pip/pip.conf + + +* The script reads all configuration files from the base directory and then + applies the sections to the main config file at "$HOME/.pip/pip.conf" +* Within the [install] section will be generated with the value `find-links` + built from the link files found in "$HOME/.pip/links.d". +""" + +import ConfigParser +import os + + +def config_files(config_dir_path, extension='.link'): + """Discover all link files. + + :param config_dir_path: ``str`` Path to link directory + :param extension: ``str`` Extension for files + :return: ``list`` + """ + link_files = [] + for root_path, _, pip_files in os.walk(config_dir_path): + for f in pip_files: + if f.endswith(extension): + link_files.append(os.path.join(root_path, f)) + else: + return link_files + + +def pip_links(links_files): + """Read all link files. + + :param links_files: ``list`` List of files to read containing links + :return: `list`` + """ + links = [] + for link in links_files: + with open(link, 'rb') as f: + links.extend(f.readlines()) + else: + return links + + +def load_config(config_file): + """Load config from a file. + + :param config_file: ``str`` path to config file + :return: ``object`` + """ + config = ConfigParser.ConfigParser() + if config_file is None: + return config + + try: + with open(config_file) as f: + config.readfp(f) + except IOError: + return config + else: + return config + + +def set_links(links): + """Set all links and ensure there are no blank lines. + + :param links: ``list`` List of all raw links + :return: ``str`` + """ + pip_find_links = [] + for link in links: + if link != '\n' or not link: + pip_find_links.append(link.rstrip('\n')) + + links = [i for i in list(set(pip_find_links))] + return '\n%s' % '\n'.join(links) + + +def build_main_config(add_conf, main_config): + """Build configuration from all found conf files. + + :param add_conf: ``object`` ConfigParser object + :param main_config: ``object`` ConfigParser object + """ + for section in add_conf.sections(): + try: + main_config.add_section(section) + except ConfigParser.DuplicateSectionError: + pass + + for k, v in add_conf.items(section): + main_config.set(section, k, v) + + +def build_install_section(main_dir_path, main_config): + """Build the install section with links. + + :param main_dir_path: ``str`` Directory path + :param main_config: ``object`` ConfigParser object + """ + links = [] + links_dir = os.path.join(main_dir_path, 'links.d') + if os.path.isdir(links_dir): + _link = config_files(config_dir_path=links_dir, extension='.link') + links.extend(pip_links(_link)) + + # Add install section if not already found + try: + main_config.add_section('install') + except ConfigParser.DuplicateSectionError: + pass + + # Get all items from the install section + try: + install_items = main_config.items('install') + except ConfigParser.NoSectionError: + install_items = None + + link_strings = set_links(links) + if install_items: + for item in install_items: + if item[0] != 'find-links': + main_config.set('install', *item) + + main_config.set('install', 'find-links', link_strings) + + +def main(): + """Run the main application.""" + main_file_path = os.path.expanduser('~/.pip/pip.conf') + main_config = load_config(config_file=None) + + main_dir_path = os.path.dirname(main_file_path) + base_dir_path = os.path.join(main_dir_path, 'base') + if os.path.isdir(base_dir_path): + _confs = config_files(base_dir_path, extension='.conf') + for _conf in _confs: + _config = load_config(config_file=_conf) + build_main_config(_config, main_config) + + build_install_section(main_dir_path, main_config) + + # Write out the config file + with open(main_file_path, 'wb') as f: + main_config.write(f) + + +if __name__ == '__main__': + main() diff --git a/rpc_deployment/roles/common/templates/rpc-release b/rpc_deployment/roles/common/templates/rpc-release index 1966acf2da..00558229f7 100644 --- a/rpc_deployment/roles/common/templates/rpc-release +++ b/rpc_deployment/roles/common/templates/rpc-release @@ -1,4 +1,4 @@ DISTRIB_ID="RPC" -DISTRIB_RELEASE="9.0.0" +DISTRIB_RELEASE="{{ rpc_release }}" DISTRIB_CODENAME="wee-5ive" DISTRIB_DESCRIPTION="Rackspace Private Cloud" diff --git a/rpc_deployment/roles/openstack_common/tasks/pip_install.yml b/rpc_deployment/roles/openstack_common/tasks/pip_install.yml index 648e14b877..65c955f200 100644 --- a/rpc_deployment/roles/openstack_common/tasks/pip_install.yml +++ b/rpc_deployment/roles/openstack_common/tasks/pip_install.yml @@ -19,6 +19,7 @@ extra_args="{{ pip_install_options|default('') }} --allow-all-external" register: pip_install until: pip_install|success - retries: 3 + retries: 5 async: 1800 + delay: 5 poll: 5 diff --git a/rpc_deployment/vars/config_vars/container_config.yml b/rpc_deployment/vars/config_vars/container_config.yml new file mode 100644 index 0000000000..a8e07db0be --- /dev/null +++ b/rpc_deployment/vars/config_vars/container_config.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Container Template Config +container_template: rpc +container_release: trusty + +# only used when the lxc vg is present on the target +container_lvm_fstype: ext4 +container_lvm_fssize: 5GB + +# Parameters on what the conatiner will be built with +container_config: /etc/lxc/lxc-rpc.conf + +# Baseline container configuration options. +container_config_options: + - "lxc.aa_profile=lxc-openstack" diff --git a/rpc_deployment/vars/repo_packages/all_common.yml b/rpc_deployment/vars/repo_packages/all_common.yml new file mode 100644 index 0000000000..712b19e26a --- /dev/null +++ b/rpc_deployment/vars/repo_packages/all_common.yml @@ -0,0 +1,78 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## URL for the frozen rpc repo +rpc_repo_url: "http://rpc-slushee.rackspace.com" + +## GPG Keys +gpg_keys: + - { key_name: 'mariadb', keyserver: 'hkp://keyserver.ubuntu.com:80', hash_id: '0xcbcb082a1bb943db' } + +apt_common_repos: + - { repo: "deb http://mirror.jmu.edu/pub/mariadb/repo/5.5/ubuntu {{ ansible_distribution_release }} main", state: "present" } + +# apt_container_keys: +# - { url: "{{ rpc_repo_url }}/repo.gpg", state: "present" } + +## Repositories. the name: key value is only used in the ``containers-setup.yml`` play +# Set the name to whatever the domain name is of the ``rpc_repo_url`` value with any +# Hyphens and periods replaced with underlines. This will ensure that when ansible +# runs into the ``apt_common_repos`` option is set correctly. +# apt_common_repos: +# - { repo: "deb [arch=amd64] {{ rpc_repo_url }} rpc-LA main", state: "present", name: "rpc_slushee_rackspace_com" } +# - { repo: "deb [arch=amd64] {{ rpc_repo_url }} LA main", state: "present", name: "rpc_slushee_rackspace_com" } +# - { repo: "deb [arch=amd64] {{ rpc_repo_url }} universe-LA universe", state: "present", name: "rpc_slushee_rackspace_com" } + +## Python pip +get_pip_url: "{{ rpc_repo_url }}/downloads/get-pip.py" +get_pip_sha256sum: "d43dc33a5670d69dd14a9be1f2b2fa27ebf124ec1b212a47425331040f742a9b" + +pip_common_packages: + - wheel + - setuptools + - pip + + +## Base Packages +apt_common_packages: + - aptitude + - vlan + - python-software-properties + - python-dev + - build-essential + - git-core + - rsyslog + - lvm2 + - dmeventd + - libkmod-dev + - libkmod2 + - libssl-dev + - bridge-utils + - cgroup-lite + - sqlite3 + - iptables + - sshpass + - libffi-dev + - libxml2-dev + - libxslt1-dev + - mariadb-client + - libmariadbclient-dev + +# Util packages that are installed when repos are put in place +common_util_packages: + - curl + - wget + - time + - rsync diff --git a/rpc_deployment/vars/repo_packages/ansible_rpc_lxc.yml b/rpc_deployment/vars/repo_packages/ansible_rpc_lxc.yml new file mode 100644 index 0000000000..1ef184793e --- /dev/null +++ b/rpc_deployment/vars/repo_packages/ansible_rpc_lxc.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://github.com/rcbops/ansible-lxc-rpc" +git_install_branch: 22dd0baba7b1e0f496794ee59c968738df1a257d + +pip_wheel_name: ansible-lxc-rpc diff --git a/rpc_deployment/vars/repo_packages/cinder.yml b/rpc_deployment/vars/repo_packages/cinder.yml new file mode 100644 index 0000000000..5db523bac5 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/cinder.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: cinder + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/cinder +git_fallback_repo: https://github.com/openstack/cinder +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 36406c9bb0bac99b5daf1b8194df73685cbe49ea + +pip_wheel_name: cinder + +service_pip_dependencies: + - pywbem + - ecdsa + - MySQL-python + - python-memcached + - pycrypto + - python-cinderclient + - python-keystoneclient + - keystonemiddleware + +container_packages: + - libpq-dev + - libkmod-dev + - libkmod2 + - dmeventd + - tgt + - parted + - qemu-utils diff --git a/rpc_deployment/vars/repo_packages/elasticsearch.yml b/rpc_deployment/vars/repo_packages/elasticsearch.yml new file mode 100644 index 0000000000..15f1f03ab7 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/elasticsearch.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: elasticsearch + +apt_container_keys: + - { url: "http://packages.elasticsearch.org/GPG-KEY-elasticsearch", state: "present" } + +apt_container_repos: + - { repo: "deb http://packages.elasticsearch.org/elasticsearch/1.2/debian stable main", state: "present"} + +container_packages: + - elasticsearch + - openjdk-7-jre + +service_pip_dependencies: + - requests diff --git a/rpc_deployment/vars/repo_packages/galera.yml b/rpc_deployment/vars/repo_packages/galera.yml new file mode 100644 index 0000000000..19e4956890 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/galera.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: galera + +# The package name for mariaDB is set as a variable +# so that it can be used in debconf later in the +# "galera_common" role. +mariadb_server_package: "mariadb-galera-server-5.5" + +container_packages: + - mariadb-client + - "{{ mariadb_server_package }}" + - galera + - python-software-properties + - software-properties-common + - debconf-utils + - rsync + - xtrabackup + +service_pip_dependencies: + - MySQL-python + +# When installing container packages this bol will drop a +# run level policy to ensure that no services are started +# upon installation. +run_policy_deny: true + +debconf_items: + - question: "mysql-server/root_password" + name: "{{ mariadb_server_package }}" + value: "{{ mysql_password }}" + vtype: "string" + - question: "mysql-server/root_password_again" + name: "{{ mariadb_server_package }}" + value: "{{ mysql_password }}" + vtype: "string" diff --git a/rpc_deployment/vars/repo_packages/glance.yml b/rpc_deployment/vars/repo_packages/glance.yml new file mode 100644 index 0000000000..d769120e80 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/glance.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: glance + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/glance +git_fallback_repo: https://github.com/openstack/glance +git_dest: "/opt/{{ repo_path }}" +git_install_branch: d0453ae453adeacf19d121f08f8f9691f11f435a + +pip_wheel_name: glance + +container_packages: + - rsync + +service_pip_dependencies: + - warlock + - MySQL-python + - python-memcached + - pycrypto + - python-glanceclient + - python-swiftclient + - python-cinderclient + - python-keystoneclient + - keystonemiddleware diff --git a/rpc_deployment/vars/repo_packages/haproxy.yml b/rpc_deployment/vars/repo_packages/haproxy.yml new file mode 100644 index 0000000000..dfd69e6f79 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/haproxy.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: haproxy + +container_packages: + - haproxy + - hatop + - vim-haproxy diff --git a/rpc_deployment/vars/repo_packages/heat.yml b/rpc_deployment/vars/repo_packages/heat.yml new file mode 100644 index 0000000000..1e1c98fe37 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/heat.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: heat + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/heat +git_fallback_repo: https://github.com/openstack/heat +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 93f3c78a8bd9ecb79fa3d719468b371e67519969 + +pip_wheel_name: heat + +container_packages: + - rsync + - libxslt1.1 + +service_pip_dependencies: + - MySQL-python + - python-memcached + - pycrypto + - python-heatclient + - python-keystoneclient + - python-troveclient + - python-ceilometerclient + - keystonemiddleware diff --git a/rpc_deployment/vars/repo_packages/holland.yml b/rpc_deployment/vars/repo_packages/holland.yml new file mode 100644 index 0000000000..a8d7498dac --- /dev/null +++ b/rpc_deployment/vars/repo_packages/holland.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: holland + +repo_path: "holland_{{ git_install_branch | replace('/', '_') }}" + +git_repo: https://github.com/holland-backup/holland +git_install_branch: "v1.0.10" +git_dest: "/opt/{{ repo_path }}" +# git_repo_plugins are other installable packages contained within the same git repo +git_repo_plugins: + - { path: "plugins", package: "holland.lib.common" } + - { path: "plugins", package: "holland.lib.mysql" } + - { path: "plugins", package: "holland.backup.xtrabackup" } + +pip_wheel_name: holland + +container_packages: + - mariadb-client + - xtrabackup + +service_pip_dependencies: + - MySQL-python diff --git a/rpc_deployment/vars/repo_packages/hosts.yml b/rpc_deployment/vars/repo_packages/hosts.yml new file mode 100644 index 0000000000..a2aaf6ec29 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/hosts.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apt_host_repos: + - { repo: "ppa:ubuntu-lxc/stable", state: "present" } + +host_packages: + - sysstat + - vlan + - python3-lxc + - lxc-templates + - liblxc1 + - libssl-dev + - libffi-dev + +container_cache_tarball: "{{ rpc_repo_url }}/downloads/rpc-trusty-container.tgz" +container_tarball_sha256sum: 12dd6de0d17948cfe0a38e945fb083a58a12e4c395cd4391ed2dcf75085c1303 diff --git a/rpc_deployment/vars/repo_packages/keystone.yml b/rpc_deployment/vars/repo_packages/keystone.yml new file mode 100644 index 0000000000..e407bf2a82 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/keystone.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: keystone + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/keystone +git_fallback_repo: https://github.com/openstack/keystone +git_dest: "/opt/{{ repo_path }}" +git_etc_example: etc/ +git_install_branch: 7c2bee76936461043671adc6d100f20775c4212f + +pip_wheel_name: keystone + +container_packages: + - libsasl2-dev + - debhelper + - dh-apparmor + - docutils-common + - libjs-sphinxdoc + - libjs-underscore + - libxslt1.1 + - libldap2-dev + - apache2 + - apache2-utils + - libapache2-mod-wsgi + +# Common PIP Packages +service_pip_dependencies: + - repoze.lru + - pbr + - MySQL-python + - pycrypto + - python-memcached + - python-keystoneclient + - keystonemiddleware diff --git a/rpc_deployment/vars/repo_packages/keystonemiddleware.yml b/rpc_deployment/vars/repo_packages/keystonemiddleware.yml new file mode 100644 index 0000000000..523143ddb4 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/keystonemiddleware.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/keystonemiddleware +git_fallback_repo: https://github.com/openstack/keystonemiddleware +git_dest: "/opt/{{ repo_path }}" +git_install_branch: f61fe33f558ba3f02fc2400507d5f71de21e6d12 + +pip_wheel_name: keystonemiddleware diff --git a/rpc_deployment/vars/repo_packages/kibana.yml b/rpc_deployment/vars/repo_packages/kibana.yml new file mode 100644 index 0000000000..c9973382db --- /dev/null +++ b/rpc_deployment/vars/repo_packages/kibana.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: kibana + +kibana_url: "{{ rpc_repo_url }}/downloads/kibana-3.1.0.tar.gz" +kibana_sha256sum: "059a4b6b507b9ff771901d12035e499b0e8d1cae7d9e5284633e19da6c294e07" + +container_packages: + - apache2 + - python-passlib + - openjdk-7-jre diff --git a/rpc_deployment/vars/repo_packages/logstash.yml b/rpc_deployment/vars/repo_packages/logstash.yml new file mode 100644 index 0000000000..35ff3159d0 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/logstash.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: logstash + +# Apt repos for ELK +apt_container_keys: + - { url: "http://packages.elasticsearch.org/GPG-KEY-elasticsearch", state: "present" } + +apt_container_repos: + - { repo: "deb http://packages.elasticsearch.org/logstash/1.4/debian stable main", state: "present"} + +container_packages: + - logstash + - logstash-contrib + - openjdk-7-jre diff --git a/rpc_deployment/vars/repo_packages/memcached.yml b/rpc_deployment/vars/repo_packages/memcached.yml new file mode 100644 index 0000000000..1a7fa58b07 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/memcached.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: memcached + +container_packages: + - memcached diff --git a/rpc_deployment/vars/repo_packages/neutron.yml b/rpc_deployment/vars/repo_packages/neutron.yml new file mode 100644 index 0000000000..7de5b4cc15 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/neutron.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: neutron + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/neutron +git_fallback_repo: https://github.com/openstack/neutron +git_dest: "/opt/{{ repo_path }}" +git_etc_example: etc/ +git_install_branch: fd1221124b8f53a61731e51b5f923af44ec5598f +pip_wheel_name: neutron + +container_packages: + - libpq-dev + - dnsmasq-base + - dnsmasq-utils + +service_pip_dependencies: + - MySQL-python + - python-memcached + - pycrypto + - repoze.lru + - configobj + - cliff + - python-novaclient + - python-neutronclient + - python-keystoneclient + - keystonemiddleware diff --git a/rpc_deployment/vars/repo_packages/nova.yml b/rpc_deployment/vars/repo_packages/nova.yml new file mode 100644 index 0000000000..1868b6b9fb --- /dev/null +++ b/rpc_deployment/vars/repo_packages/nova.yml @@ -0,0 +1,41 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: nova + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/nova +git_fallback_repo: https://github.com/openstack/nova +git_dest: "/opt/{{ repo_path }}" +git_etc_example: etc/nova/ +git_install_branch: 9ada982ccdb974d43c1c94e59eb4b84e35a7997d + +pip_wheel_name: nova + +container_packages: + - libpq-dev + - open-iscsi + - vlan + - kpartx + +service_pip_dependencies: + - MySQL-python + - python-memcached + - pycrypto + - python-keystoneclient + - python-novaclient + - keystonemiddleware diff --git a/rpc_deployment/vars/repo_packages/nova_libvirt.yml b/rpc_deployment/vars/repo_packages/nova_libvirt.yml new file mode 100644 index 0000000000..0f87fed2d7 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/nova_libvirt.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +container_packages:: + - libvirt-bin + - python-libvirt + - qemu + - qemu-utils + - genisoimage + - sysfsutils diff --git a/rpc_deployment/vars/repo_packages/nova_spice_console.yml b/rpc_deployment/vars/repo_packages/nova_spice_console.yml new file mode 100644 index 0000000000..8e4540d7b1 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/nova_spice_console.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +container_packages: + - spice-html5 diff --git a/rpc_deployment/vars/repo_packages/openstack_global_requirements.yml b/rpc_deployment/vars/repo_packages/openstack_global_requirements.yml new file mode 100644 index 0000000000..3e336ffc21 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/openstack_global_requirements.yml @@ -0,0 +1,27 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/requirements" +git_fallback_repo: "https://github.com/openstack/requirements" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: c99cbdc7d3a1e5f5b29b9f1507fad2e62e09485c +requirements_file: global-requirements.txt + +pip_wheel_name: requirements + +pip_requirements_file: "{{ git_dest }}/global-requirements.txt" diff --git a/rpc_deployment/vars/repo_packages/python_barbicanclient.yml b/rpc_deployment/vars/repo_packages/python_barbicanclient.yml new file mode 100644 index 0000000000..07e7490c57 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_barbicanclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-barbicanclient" +git_fallback_repo: "https://github.com/openstack/python-barbicanclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 2.2.1 + +pip_wheel_name: python-barbicanclient diff --git a/rpc_deployment/vars/repo_packages/python_ceilometerclient.yml b/rpc_deployment/vars/repo_packages/python_ceilometerclient.yml new file mode 100644 index 0000000000..c28413a275 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_ceilometerclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-ceilometerclient" +git_fallback_repo: "https://github.com/openstack/python-ceilometerclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 1.0.9 + +pip_wheel_name: python-ceilometerclient diff --git a/rpc_deployment/vars/repo_packages/python_cinderclient.yml b/rpc_deployment/vars/repo_packages/python_cinderclient.yml new file mode 100644 index 0000000000..8b8e8c8c38 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_cinderclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-cinderclient" +git_fallback_repo: "https://github.com/openstack/python-cinderclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 1.0.9 + +pip_wheel_name: python-cinderclient diff --git a/rpc_deployment/vars/repo_packages/python_designateclient.yml b/rpc_deployment/vars/repo_packages/python_designateclient.yml new file mode 100644 index 0000000000..4a3cc41443 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_designateclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-designateclient" +git_fallback_repo: "https://github.com/openstack/python-designateclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 1.0.3 + +pip_wheel_name: python-designateclient diff --git a/rpc_deployment/vars/repo_packages/python_glanceclient.yml b/rpc_deployment/vars/repo_packages/python_glanceclient.yml new file mode 100644 index 0000000000..70c09360ab --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_glanceclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-glanceclient" +git_fallback_repo: "https://github.com/openstack/python-glanceclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 0.13.1 + +pip_wheel_name: python-glanceclient diff --git a/rpc_deployment/vars/repo_packages/python_heatclient.yml b/rpc_deployment/vars/repo_packages/python_heatclient.yml new file mode 100644 index 0000000000..98229dd6b0 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_heatclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-heatclient" +git_fallback_repo: "https://github.com/openstack/python-heatclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 0.2.10 + +pip_wheel_name: python-heatclient diff --git a/rpc_deployment/vars/repo_packages/python_ironicclient.yml b/rpc_deployment/vars/repo_packages/python_ironicclient.yml new file mode 100644 index 0000000000..3b72133767 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_ironicclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-ironicclient" +git_fallback_repo: "https://github.com/openstack/python-ironicclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 0.2.1 + +pip_wheel_name: python-ironicclient diff --git a/rpc_deployment/vars/repo_packages/python_keystoneclient.yml b/rpc_deployment/vars/repo_packages/python_keystoneclient.yml new file mode 100644 index 0000000000..2f50af3392 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_keystoneclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-keystoneclient" +git_fallback_repo: "https://github.com/openstack/python-keystoneclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 0.10.1 + +pip_wheel_name: python-keystoneclient diff --git a/rpc_deployment/vars/repo_packages/python_neutronclient.yml b/rpc_deployment/vars/repo_packages/python_neutronclient.yml new file mode 100644 index 0000000000..1de31b9006 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_neutronclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-neutronclient" +git_fallback_repo: "https://github.com/openstack/python-neutronclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 2.3.6 + +pip_wheel_name: python-neutronclient diff --git a/rpc_deployment/vars/repo_packages/python_novaclient.yml b/rpc_deployment/vars/repo_packages/python_novaclient.yml new file mode 100644 index 0000000000..d034d590c8 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_novaclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-novaclient" +git_fallback_repo: "https://github.com/openstack/python-novaclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 2.18.1 + +pip_wheel_name: python-novaclient diff --git a/rpc_deployment/vars/repo_packages/python_openstackclient.yml b/rpc_deployment/vars/repo_packages/python_openstackclient.yml new file mode 100644 index 0000000000..7fd30896ed --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_openstackclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-openstackclient" +git_fallback_repo: "https://github.com/openstack/python-openstackclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 0.4.0 + +pip_wheel_name: python-openstackclient diff --git a/rpc_deployment/vars/repo_packages/python_saharaclient.yml b/rpc_deployment/vars/repo_packages/python_saharaclient.yml new file mode 100644 index 0000000000..1937a87576 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_saharaclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-saharaclient" +git_fallback_repo: "https://github.com/openstack/python-saharaclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 0.7.1 + +pip_wheel_name: python-saharaclient diff --git a/rpc_deployment/vars/repo_packages/python_swiftclient.yml b/rpc_deployment/vars/repo_packages/python_swiftclient.yml new file mode 100644 index 0000000000..abd18f1a51 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_swiftclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-swiftclient" +git_fallback_repo: "https://github.com/openstack/python-swiftclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 2.2.0 + +pip_wheel_name: python-swiftclient diff --git a/rpc_deployment/vars/repo_packages/python_troveclient.yml b/rpc_deployment/vars/repo_packages/python_troveclient.yml new file mode 100644 index 0000000000..168f988f18 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_troveclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-troveclient" +git_fallback_repo: "https://github.com/openstack/python-troveclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 1.0.5 + +pip_wheel_name: python-troveclient diff --git a/rpc_deployment/vars/repo_packages/python_tuskarclient.yml b/rpc_deployment/vars/repo_packages/python_tuskarclient.yml new file mode 100644 index 0000000000..1fe456ea1e --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_tuskarclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-tuskarclient" +git_fallback_repo: "https://github.com/openstack/python-tuskarclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 0.1.8 + +pip_wheel_name: python-tuskarclient diff --git a/rpc_deployment/vars/repo_packages/python_zaqarclient.yml b/rpc_deployment/vars/repo_packages/python_zaqarclient.yml new file mode 100644 index 0000000000..50738715de --- /dev/null +++ b/rpc_deployment/vars/repo_packages/python_zaqarclient.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://git.openstack.org/openstack/python-zaqarclient" +git_fallback_repo: "https://github.com/openstack/python-zaqarclient" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 0.0.2 + +pip_wheel_name: python-zaqarclient diff --git a/rpc_deployment/vars/repo_packages/rabbitmq.yml b/rpc_deployment/vars/repo_packages/rabbitmq.yml new file mode 100644 index 0000000000..60e27f4577 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/rabbitmq.yml @@ -0,0 +1,26 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: rabbitmq + +apt_container_keys: + - { url: "http://www.rabbitmq.com/rabbitmq-signing-key-public.asc", state: "present" } + +apt_container_repos: + - { repo: "deb http://www.rabbitmq.com/debian/ testing main", state: "present" } + +container_packages: + - rabbitmq-server + - erlang diff --git a/rpc_deployment/vars/repo_packages/raxmon.yml b/rpc_deployment/vars/repo_packages/raxmon.yml new file mode 100644 index 0000000000..7c18e8bfe9 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/raxmon.yml @@ -0,0 +1,34 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: raxmon + +git_repo: https://github.com/rcbops/rpc-maas +git_install_branch: "v9.0.0" +git_dest: /usr/lib/rackspace-monitoring-agent/plugins/ + +apt_container_keys: + - { url: "https://monitoring.api.rackspacecloud.com/pki/agent/linux.asc", state: "present" } + +apt_container_repos: + - { repo: "deb http://stable.packages.cloudmonitoring.rackspace.com/ubuntu-14.04-x86_64 cloudmonitoring main", state: "present" } + +container_packages: + - rackspace-monitoring-agent + +service_pip_dependencies: + - rackspace-monitoring-cli + +pip_requirements_file: "/usr/lib/rackspace-monitoring-agent/plugins/requirements.txt" diff --git a/rpc_deployment/vars/repo_packages/readme.rst b/rpc_deployment/vars/repo_packages/readme.rst new file mode 100644 index 0000000000..69a3c7ffd0 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/readme.rst @@ -0,0 +1,39 @@ +Ansible Openstack LXC Packages +############################## +:date: 2014-09-01 09:57 +:tags: rackspace, lxc, openstack, cloud, ansible +:category: \*nix + +Packages Downloads and Installables +=================================== + +Any and all packages that need to be installed for this repository to work +should be specified here in the, ``repo_packages`` directory. The files in +this directory are given to plays as additional options. The options have +several default actions which are all processed by the following roles: +**package_source_archive**, **package_source_install**, +**package_system_install**. Inside these files all download-able objects +such as tar-balls and random files should also be specified. While the packaging +roles may not be used to process these links the stated purpose of this +directory is to have anything that is "installable" in a single location with +the goal to allow for easily manipulation of requirements as they change. + +Defaults processed by the **package_source_archive**, +**package_source_install**, **package_system_install** roles: + * gpg_keys: ``list`` of ``hashes`` with keys: key_name, keyserver, hash_id. + * apt_container_keys: ``list`` of ``hashes`` with keys: url, state + * apt_container_repos: ``list`` of ``hashes`` with keys: repo, state + * debconf_items: ``list`` of ``hashes`` with keys: question, name, value, vtype + * run_policy_deny: ``boolean`` true or false: When installing container packages this ``boolean`` will drop a run level policy to ensure that no services are started upon installation. + * repo_path: ``string`` used to set the "relative path" to an online repository without the domain name. also used as the target directory when downloading a given git repository. + * git_repo: ``string`` URI to the git repo to clone from. + * git_fallback_repo: ``string`` URI to an alternative git repo to clone from when **git_repo** fails. + * git_dest: ``string`` full path to place a cloned git repository. This will normally incorporate the **repo_path** variable for consistency purposes. + * git_install_branch: ``string`` branch, tag or SHA of a git repo to clone into. + * git_repo_plugins: ``list`` of ``hashes`` with keys: path, package | This is used to install additional packages which may be installable from the same base repo. + * pip_wheel_name: ``string`` pip package name to FIRST attempt installation of. + * service_pip_dependencies: ``list`` of ``strings``. + * container_packages: ``list`` of ``strings``. + * pip_common_packages: ``list`` of ``strings``. + * apt_common_packages: ``list`` of ``strings``. + * common_util_packages: ``list`` of ``strings``. diff --git a/rpc_deployment/vars/repo_packages/rpc_support.yml b/rpc_deployment/vars/repo_packages/rpc_support.yml new file mode 100644 index 0000000000..9d5b2b966d --- /dev/null +++ b/rpc_deployment/vars/repo_packages/rpc_support.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +common_util_packages: + - vim + - linux-crashdump + - iotop + - dstat + - tcpdump + - lsof + - htop diff --git a/rpc_deployment/vars/repo_packages/rsyslog.yml b/rpc_deployment/vars/repo_packages/rsyslog.yml new file mode 100644 index 0000000000..460f8b4483 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/rsyslog.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: rsyslog + +apt_container_repos: + - { repo: "ppa:adiscon/v8-stable", state: "present" } diff --git a/rpc_deployment/vars/repo_packages/swift.yml b/rpc_deployment/vars/repo_packages/swift.yml new file mode 100644 index 0000000000..c5bccbc67f --- /dev/null +++ b/rpc_deployment/vars/repo_packages/swift.yml @@ -0,0 +1,26 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: swift + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/swift +git_fallback_repo: https://github.com/openstack/swift +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 8d02147d04a41477383de8e13bea6ac3fd2cade0 + +pip_wheel_name: swift diff --git a/rpc_deployment/vars/repo_packages/tempest.yml b/rpc_deployment/vars/repo_packages/tempest.yml new file mode 100644 index 0000000000..1783fe31cb --- /dev/null +++ b/rpc_deployment/vars/repo_packages/tempest.yml @@ -0,0 +1,26 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: tempest + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/tempest +git_fallback_repo: https://github.com/openstack/tempest +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 9d7d28b66f9aa3fe1220f418cc2d13dd19984429 + +pip_wheel_name: tempest diff --git a/rpc_deployment/vars/repo_packages/turbolift.yml b/rpc_deployment/vars/repo_packages/turbolift.yml new file mode 100644 index 0000000000..30866b52c3 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/turbolift.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_path: "{{ pip_wheel_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: "https://github.com/cloudnull/turbolift" +git_dest: "/opt/{{ repo_path }}" +git_install_branch: 5e4fa93532ac4bf17d1424a2bf58d364d202c406 + +pip_wheel_name: turbolift diff --git a/rpc_deployment/vars/repo_packages/utility.yml b/rpc_deployment/vars/repo_packages/utility.yml new file mode 100644 index 0000000000..5dba223f0c --- /dev/null +++ b/rpc_deployment/vars/repo_packages/utility.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: utility + +container_packages: + - ruby1.9.1 + - openjdk-7-jre + +service_pip_dependencies: + - python-openstackclient + - python-cinderclient + - python-glanceclient + - python-heatclient + - python-keystoneclient + - python-neutronclient + - python-novaclient + - python-swiftclient diff --git a/scripts/rpc-wheel-builder.py b/scripts/rpc-wheel-builder.py new file mode 100755 index 0000000000..cc60668861 --- /dev/null +++ b/scripts/rpc-wheel-builder.py @@ -0,0 +1,632 @@ +#!/usr/bin/env python +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2014, Kevin Carter + +import argparse +import datetime +import json +import multiprocessing +import os +import subprocess +import sys +import tempfile +import time +import urlparse + +from distutils import version + +import requests +import yaml + +from cloudlib import logger + + +PYTHON_PACKAGES = { + 'base_release': {}, + 'known_release': {}, + 'from_git': {} +} + +GIT_REQUIREMENTS_MAP = { + 'github.com': 'https://raw.githubusercontent.com/%(path)s/%(branch)s' + '/%(file)s', + 'openstack.org': 'https://git.openstack.org/cgit/%(path)s/plain' + '/%(file)s?id=%(branch)s' +} + +VERSION_DESCRIPTORS = [ + '>=', '<=', '==', '!=', '<', '>' +] + + +class IndicatorThread(object): + """Creates a visual indicator while normally performing actions.""" + + def __init__(self, work_q=None, system=True, debug=False): + """System Operations Available on Load. + + :param work_q: + :param system: + """ + + self.debug = debug + self.work_q = work_q + self.system = system + self.job = None + + def __enter__(self): + if self.debug is False: + self.indicator_thread() + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.debug is False: + print('Done.') + self.job.terminate() + + def indicator(self): + """Produce the spinner.""" + + while self.system: + busy_chars = ['|', '/', '-', '\\'] + for bc in busy_chars: + note = 'Please Wait... ' + sys.stdout.write('\rProcessing - [ %s ] - %s' % (bc, note)) + sys.stdout.flush() + time.sleep(.1) + self.system = self.system + + def indicator_thread(self): + """indicate that we are performing work in a thread.""" + + self.job = multiprocessing.Process(target=self.indicator) + self.job.start() + return self.job + + +class LoggerWriter(object): + @property + def fileno(self): + return LOG.handlers[0].stream.fileno + + +def get_file_names(path, ext=None): + """Return a list of all files in the vars/repo_packages directory. + + :param path: ``str`` $PATH to search for files + :param ext: ``str`` extension filter for specific files. + """ + + paths = os.walk(os.path.abspath(path)) + files = [] + for fpath, _, afiles in paths: + for afile in afiles: + if ext is not None: + if afile.endswith(ext): + files.append(os.path.join(fpath, afile)) + else: + files.append(os.path.join(fpath, afile)) + else: + return files + + +def requirements_parse(pkgs): + """Parse all requirements. + + :param pkgs: ``list`` list of all requirements to parse. + """ + for pkg in pkgs: + LOG.debug('Parsing python dependencies: %s', pkg) + split_pkg = pkg.split(',') + for version_descriptor in VERSION_DESCRIPTORS: + if version_descriptor in split_pkg[0]: + name, ver = split_pkg[0].split(version_descriptor) + ver = '%s%s' % (version_descriptor, ver) + if len(split_pkg) > 1: + versions = split_pkg[1:] + versions.insert(0, ver) + else: + versions = [ver] + + break + else: + name = split_pkg[0] + versions = None + + base_release = PYTHON_PACKAGES['base_release'] + if name in base_release: + saved_versions = base_release[name] + if versions is not None: + if '==' in versions: + _lv = version.LooseVersion + if _lv(versions) < _lv(saved_versions): + versions = saved_versions + LOG.debug( + 'New version found for replacement: [ %s ]', + versions + ) + + if isinstance(versions, list): + base_release[name.lower()] = '%s%s' % (name, ','.join(versions)) + elif versions is not None: + base_release[name.lower()] = '%s%s' % (name, versions) + else: + base_release[name.lower()] = name + + +def package_dict(var_file): + """Process variable file for Python requirements. + + :param var_file: ``str`` path to yaml file. + """ + LOG.debug('Opening [ %s ]', var_file) + with open(var_file, 'rb') as f: + package_vars = yaml.safe_load(f.read()) + + pip_pkgs = package_vars.get('service_pip_dependencies') + if pip_pkgs: + requirements_parse(pkgs=pip_pkgs) + + git_repo = package_vars.get('git_repo') + if git_repo: + LOG.debug('Building git type package [ %s ]', git_repo) + git_url = urlparse.urlsplit(git_repo) + repo_name = os.path.basename(git_url.path) + repo = PYTHON_PACKAGES['from_git'][repo_name] = {} + repo['branch'] = package_vars.get('git_install_branch', 'master') + repo['full_url'] = git_repo + repo['project'] = repo_name + + setup_file = None + for k, v in GIT_REQUIREMENTS_MAP.iteritems(): + if k in git_repo: + requirements_request = v % { + 'path': git_url.path.lstrip('/'), + 'file': package_vars.get( + 'requirements_file', 'requirements.txt' + ), + 'branch': repo['branch'] + } + req = requests.get(requirements_request) + if req.status_code == 200: + requirements = [ + i.split()[0] for i in req.text.splitlines() + if i + if not i.startswith('#') + ] + repo['requirements'] = requirements + requirements_parse(pkgs=requirements) + + setup_request = v % { + 'path': git_url.path.lstrip('/'), + 'file': 'setup.py', + 'branch': repo['branch'] + } + setup = requests.head(setup_request) + if setup.status_code == 200: + setup_file = True + break + + git_req = 'git+%s@%s' + known_release = PYTHON_PACKAGES['known_release'] + if setup_file is True: + known_release[repo_name] = git_req % ( + repo['full_url'], repo['branch'] + ) + + git_repo_plugins = package_vars.get('git_repo_plugins') + if git_repo_plugins: + for grp in git_repo_plugins: + LOG.debug( + 'Building git type package with plugins [ %s ]', + git_repo_plugins + ) + plugin = '%s/%s' % ( + grp['path'].strip('/'), + grp['package'].lstrip('/') + ) + known_release[grp['package']] = git_req % ( + git_url.geturl(), + '%s#egg=%s&subdirectory=%s' % ( + repo['branch'], + grp['package'].strip('/'), + plugin + ) + ) + + +def retryloop(attempts, timeout=None, delay=None, backoff=1, obj=None): + """Enter the amount of retries you want to perform. + + The timeout allows the application to quit on "X". + delay allows the loop to wait on fail. Useful for making REST calls. + + Example: + Function for retring an action. + for retry in retryloop(attempts=10, timeout=30, delay=1, backoff=1): + something + if somecondition: + retry() + + :param attempts: + :param timeout: + :param delay: + :param backoff: + """ + + starttime = time.time() + success = set() + for _ in range(attempts): + success.add(True) + yield success.clear + if success: + return + duration = time.time() - starttime + if timeout is not None and duration > timeout: + break + if delay: + time.sleep(delay) + delay *= backoff + + error = ( + 'RetryError: FAILED TO PROCESS [ %s ] after [ %s ] Attempts' % ( + obj, + attempts + ) + ) + _error_handler(msg=error) + + +def build_wheel(wheel_dir, build_dir, dist, quiet=False, make_opts=None): + """Execute python wheel build command. + + :param wheel_dir: ``str`` $PATH to local save directory + :param build_dir: ``str`` $PATH to temp build directory + :param dist: ``str`` $PATH to requirements file + """ + command = [ + 'pip', + 'wheel', + '--timeout', + '120', + '--wheel-dir', + wheel_dir, + '--allow-all-external', + '--build', + build_dir + ] + + if make_opts is not None: + for make_opt in make_opts: + command.append(make_opt) + + command.extend(['--requirement', dist]) + build_command = ' '.join(command) + LOG.info('Command: %s' % build_command) + for retry in retryloop(3, obj=build_command, delay=2, backoff=1): + try: + with IndicatorThread(debug=quiet): + ret_data = subprocess.check_call( + command, + stdout=LoggerWriter(), + stderr=LoggerWriter() + ) + + LOG.info('Command return code: [ %s ]', ret_data) + if ret_data: + raise subprocess.CalledProcessError(ret_data, build_command) + except subprocess.CalledProcessError as exp: + LOG.warn( + 'Process failure. Error: [ %s ]. Removing build directory' + ' for retry. Check log for more detauls.', str(exp) + ) + retry() + + +def remove_dirs(directory): + """Delete a directory recursively. + + :param directory: ``str`` $PATH to directory. + """ + LOG.info('Removing directory [ %s ]', directory) + for file_name in get_file_names(path=directory): + LOG.debug('Removing file [ %s ]', file_name) + os.remove(file_name) + + dir_names = [] + for dir_name, _, _ in os.walk(directory): + dir_names.append(dir_name) + + dir_names = sorted(dir_names, reverse=True) + for dir_name in dir_names: + try: + LOG.debug('Removing subdirectory [ %s ]', dir_name) + os.removedirs(dir_name) + except OSError: + pass + + +def _requirements_maker(name, wheel_dir, release, build_dir, quiet, make_opts): + requirements_file_lines = [] + for value in sorted(release.values()): + requirements_file_lines.append('%s\n' % value) + + requirements_file = os.path.join(wheel_dir, name) + with open(requirements_file, 'wb') as f: + f.writelines(requirements_file_lines) + + build_wheel( + wheel_dir=wheel_dir, + build_dir=build_dir, + dist=requirements_file, + quiet=quiet, + make_opts=make_opts + ) + + +def make_wheels(wheel_dir, build_dir, quiet): + """Build wheels of all installed packages that don't already have one. + + :param wheel_dir: ``str`` $PATH to local save directory + :param build_dir: ``str`` $PATH to temp build directory + """ + + _requirements_maker( + name='rpc_base_requirements.txt', + wheel_dir=wheel_dir, + release=PYTHON_PACKAGES['base_release'], + build_dir=build_dir, + quiet=quiet, + make_opts=None + ) + + _requirements_maker( + name='rpc_known_requirements.txt', + wheel_dir=wheel_dir, + release=PYTHON_PACKAGES['known_release'], + build_dir=build_dir, + quiet=quiet, + make_opts=['--no-deps'] + ) + + remove_dirs(directory=build_dir) + remove_dirs( + directory=os.path.join( + tempfile.gettempdir(), + 'pip_build_root' + ) + ) + + +def ensure_consistency(): + """Iterate through the known data set and remove duplicates.""" + + LOG.info('Ensuring the package list is consistent') + for key in PYTHON_PACKAGES['known_release'].keys(): + PYTHON_PACKAGES['base_release'].pop(key, None) + + +def new_setup(user_args, input_path, output_path, quiet): + """Discover all yaml files in the input directory.""" + + LOG.info('Discovering input file(s)') + var_files = None + if os.path.isdir(user_args['input']): + var_files = get_file_names(path=input_path, ext='.yml') + else: + if not input_path.endswith(('.yml', '.yaml')): + error = ( + 'The file you specified, [ %s ] does not have a valid yaml' + ' extension. Please check your file and try again.' + % input_path + ) + _error_handler(msg=error) + else: + var_files = [input_path] + + # Populate the package dict + LOG.info('Building the package list') + with IndicatorThread(debug=quiet): + for var_file in var_files: + package_dict(var_file=var_file) + + # Ensure no general packages take precedence over the explicit ones + ensure_consistency() + + # Get a timestamp and create a report file + utctime = datetime.datetime.utcnow() + utctime = utctime.strftime("%Y%m%d_%H%M%S") + backup_name = 'python-build-report-%s.json' % utctime + output_report_file = os.path.join( + output_path, + 'json-reports', + backup_name + ) + _mkdirs(os.path.dirname(output_report_file)) + + # Generate a timestamped report file + LOG.info('Generating packaging report [ %s ]', output_report_file) + with open(output_report_file, 'wb') as f: + f.write( + json.dumps( + PYTHON_PACKAGES, + indent=2, + sort_keys=True + ) + ) + + +def _error_handler(msg, system_exit=True): + """Handle and error logging and exit the application if needed. + + :param msg: ``str`` message to log + :param system_exit: ``bol`` if true the system will exit with an error. + """ + LOG.error(msg) + if system_exit is True: + raise SystemExit(msg) + + +def _user_args(): + """Setup argument Parsing.""" + + parser = argparse.ArgumentParser( + usage='%(prog)s', + description='Rackspace Openstack, Python wheel builder', + epilog='Python package builder Licensed "Apache 2.0"' + ) + file_input = parser.add_mutually_exclusive_group(required=True) + file_input.add_argument( + '-i', + '--input', + help='Path to the directory where the repo_packages/ file or filess' + ' are. This can be set to a directory or a file. If the path is' + ' a directory all .yml files will be scanned for python packages' + ' and git repositories.', + default=None + ) + file_input.add_argument( + '--pre-input', + help='Path to a already built json file which contains the python' + ' packages and git repositories required.', + default=None + ) + parser.add_argument( + '-o', + '--output', + help='Path to the location where the built Python package files will' + ' be stored.', + required=True, + default=None + ) + parser.add_argument( + '--build-dir', + help='Path to temporary build directory. If unset a auto generated' + ' temporary directory will be used.', + required=False, + default=None + ) + opts = parser.add_mutually_exclusive_group() + opts.add_argument( + '--debug', + help='Enable debug mode', + action='store_true', + default=False + ) + opts.add_argument( + '--quiet', + help='Enables quiet mode, this disables all stdout', + action='store_true', + default=False + ) + + return vars(parser.parse_args()) + + +def _get_abs_path(path): + """Return the absolute path for a given path. + + :param path: ``str`` $PATH to be created + :returns: ``str`` + """ + return os.path.abspath( + os.path.expanduser( + path + ) + ) + + +def _mkdirs(path): + """Create a directory. + + :param path: ``str`` $PATH to be created + """ + if not os.path.exists(path): + LOG.info('Creating directory [ %s ]' % path) + os.makedirs(path) + else: + if not os.path.isdir(path): + error = ( + 'Path [ %s ] can not be created, it exists and is not already' + ' a directory.' % path + ) + _error_handler(msg=error) + + +def main(): + """Run the main app. + + This application will create all Python wheel files from within an + environment. The purpose is to create pre-compiled python wheels from + the RPC playbooks. + """ + + # Parse input arguments + user_args = _user_args() + + # Load the logging + _logging = logger.LogSetup(debug_logging=user_args['debug']) + if user_args['quiet'] is True: + stream = False + else: + stream = True + + _logging.default_logger( + name='rpc_wheel_builder', + enable_stream=stream + ) + + global LOG + LOG = logger.getLogger(name='rpc_wheel_builder') + + # Create the output path + output_path = _get_abs_path(path=user_args['output']) + LOG.info('Getting output path') + _mkdirs(path=output_path) + + # Create the build path + LOG.info('Getting build path') + if user_args['build_dir'] is not None: + build_path = _get_abs_path(path=user_args['build_dir']) + _mkdirs(path=build_path) + else: + build_path = tempfile.mkdtemp(prefix='rpc_wheels_build_') + pre_input = user_args['pre_input'] + if pre_input: + pre_input_path = _get_abs_path(path=user_args['pre_input']) + with open(pre_input_path, 'rb') as f: + global PYTHON_PACKAGES + PYTHON_PACKAGES = json.loads(f.read()) + else: + # Get the input path + LOG.info('Getting input path') + input_path = _get_abs_path(path=user_args['input']) + new_setup( + user_args=user_args, + input_path=input_path, + output_path=output_path, + quiet=user_args['quiet'] + ) + + # Create all of the python package wheels + make_wheels( + wheel_dir=output_path, + build_dir=build_path, + quiet=user_args['quiet'] + ) + + +if __name__ == "__main__": + main()