From 1bb5121e0bc98a3c184eb8f264741c6176604c6f Mon Sep 17 00:00:00 2001 From: Dmitriy Rabotyagov Date: Thu, 13 Jan 2022 16:19:49 +0200 Subject: [PATCH] Move git_requirements plugin to collection Depends-On: https://review.opendev.org/c/openstack/openstack-ansible-plugins/+/824563 Change-Id: Iabb0386f0d020fbc95ea23701ef2feda01b999f2 --- playbooks/library/git_requirements.py | 335 ---------------------- scripts/get-ansible-role-requirements.yml | 2 +- scripts/openstack-ansible.rc | 2 +- 3 files changed, 2 insertions(+), 337 deletions(-) delete mode 100644 playbooks/library/git_requirements.py diff --git a/playbooks/library/git_requirements.py b/playbooks/library/git_requirements.py deleted file mode 100644 index 3ea9e0652a..0000000000 --- a/playbooks/library/git_requirements.py +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -from ansible.module_utils.basic import AnsibleModule -import git -import itertools -import multiprocessing -import os -import signal -import time - -DOCUMENTATION = """ ---- -module: git_requirements -short_description: Module to run a multithreaded git clone - -options: - repo_info: - description: - - List of repo information dictionaries containing at - a minimum a key entry "src" with the source git URL - to clone for each repo. In these dictionaries, one - can further specify: - "path" - destination clone location - "version" - git version to checkout - "refspec" - git refspec to checkout - "depth" - clone depth level - "shallow_since" - get repo history starting from that date - "force" - require git clone uses "--force" - default_path: - description: - Default git clone path (str) in case not - specified on an individual repo basis in - repo_info. Defaults to "master". Not - required. - default_version: - description: - Default git version (str) in case not - specified on an individual repo basis in - repo_info. Defaults to "master". Not - required. - default_refspec: - description: - Default git repo refspec (str) in case not - specified on an individual repo basis in - repo_info. Defaults to "". Not required. - default_depth: - description: - Default clone depth (int) in case not specified - on an individual repo basis. Defaults to None. - Not required. - default_shallow_since: - description: - Default shallow date (str) strating from which - git history will be fetched. Defaults to None. - Has prescedence over depth. - Not required. - retries: - description: - Integer number of retries allowed in case of git - clone failure. Defaults to 1. Not required. - delay: - description: - Integer time delay (seconds) between git clone - retries in case of failure. Defaults to 0. Not - required. - force: - description: - Boolean. Apply --force flags to git clones wherever - possible. Defaults to False. Not required. - core_multiplier: - description: - Integer multiplier on the number of cores - present on the machine to use for - multithreading. For example, on a 2 core - machine, a multiplier of 4 would use 8 - threads. Defaults to 4. Not required. -""" - -EXAMPLES = r""" - -- name: Clone repos - git_requirements: - repo_info: "[{'src':'https://github.com/ansible/', - 'name': 'ansible' - 'dest': '/etc/opt/ansible'}]" -""" - - -def init_signal(): - signal.signal(signal.SIGINT, signal.SIG_IGN) - - -def check_out_version(repo, version, pull=False, fetch=True, force=False, - refspec=None, tag=False, depth=None, - shallow_since=None): - if fetch: - try: - if shallow_since: - depth = None - repo.git.fetch(tags=tag, force=force, refspec=refspec, - depth=depth, shallow_since=shallow_since) - except Exception as e: - return ["Failed to fetch %s\n%s" % (repo.working_dir, str(e))] - - try: - repo.git.checkout(version, force=force) - except Exception as e: - return [ - "Failed to check out version %s for %s\n%s" % - (version, repo.working_dir, str(e))] - - if repo.is_dirty(untracked_files=True) and force: - try: - repo.git.clean(force=force) - except Exception as e: - return [ - "Failed to clean up repository% s\n%s" % - (repo.working_dir, str(e))] - - if pull: - try: - repo.git.pull(force=force, refspec=refspec, depth=depth) - except Exception as e: - return ["Failed to pull repo %s\n%s" % (repo.working_dir, str(e))] - return [] - - -def pull_wrapper(info): - role_info = info - retries = info[1]["retries"] - delay = info[1]["delay"] - for i in range(retries): - success = pull_role(role_info) - if success: - return True - else: - time.sleep(delay) - info[2].append(["Role {0} failed after {1} retries\n".format(role_info[0], - retries)]) - return False - - -def pull_role(info): - role, config, failures = info - - required_version = role["version"] - version_hash = False - if 'version' in role: - # If the version is the length of a hash then treat is as one - if len(required_version) == 40: - version_hash = True - - def get_repo(dest): - try: - return git.Repo(dest) - except Exception: - failtxt = "Role in {0} is broken/not a git repo.".format( - role["dest"]) - failtxt += "Please delete or fix it manually" - failures.append(failtxt) - return False - - # if repo exists - if os.path.exists(role["dest"]): - repo = get_repo(role["dest"]) - if not repo: - return False # go to next role - repo_url = list(repo.remote().urls)[0] - if repo_url != role["src"]: - repo.remote().set_url(role["src"]) - - # if they want master then fetch, checkout and pull to stay at latest - # master - if required_version == "master": - fail = check_out_version(repo, required_version, pull=True, - force=config["force"], - refspec=role["refspec"], - depth=role["depth"], - shallow_since=role["shallow_since"]) - - # If we have a hash then reset it to - elif version_hash: - fail = check_out_version(repo, required_version, - force=config["force"], - refspec=role["refspec"], - depth=role["depth"], - shallow_since=role["shallow_since"]) - else: - # describe can fail in some cases so be careful: - try: - current_version = repo.git.describe(tags=True) - except Exception: - current_version = "" - if current_version == required_version and not config["force"]: - fail = [] - pass - else: - fail = check_out_version(repo, required_version, - force=config["force"], - refspec=role["refspec"], - depth=role["depth"], - shallow_since=role["shallow_since"], - tag=True) - - else: - try: - # If we have a hash id then treat this a little differently - shallow_since = role.get('shallow_since') - if shallow_since: - depth = None - else: - depth = role.get('depth') - - if version_hash: - repo = git.Repo.clone_from(role["src"], role["dest"], - branch='master', - no_single_branch=True, - depth=depth, - shallow_since=shallow_since,) - if not repo: - return False # go to next role - fail = check_out_version(repo, required_version, - force=config["force"], - refspec=role["refspec"], - depth=depth, - fetch=False, - shallow_since=shallow_since,) - else: - git.Repo.clone_from(role["src"], role["dest"], - branch=required_version, - depth=depth, - shallow_since=shallow_since, - no_single_branch=True) - fail = [] - - except Exception as e: - fail = ('Failed cloning repo %s\n%s' % (role["dest"], str(e))) - - if fail == []: - return True - else: - failures.append(fail) - return False - - -def set_default(dictionary, key, defaults): - if key not in dictionary.keys(): - dictionary[key] = defaults[key] - - -def main(): - # Define variables - failures = multiprocessing.Manager().list() - - # Data we can pass in to the module - fields = { - "repo_info": {"required": True, "type": "list"}, - "default_path": {"required": True, - "type": "str"}, - "default_version": {"required": False, - "type": "str", - "default": "master"}, - "default_refspec": {"required": False, - "type": "str", - "default": None}, - "default_depth": {"required": False, - "type": "int", - "default": None}, - "default_shallow_since": {"required": False, - "type": str, - "default": None}, - "retries": {"required": False, - "type": "int", - "default": 1}, - "delay": {"required": False, - "type": "int", - "default": 0}, - "force": {"required": False, - "type": "bool", - "default": False}, - "core_multiplier": {"required": False, - "type": "int", - "default": 4}, - - } - - # Pull in module fields and pass into variables - module = AnsibleModule(argument_spec=fields) - - git_repos = module.params['repo_info'] - defaults = { - "path": module.params["default_path"], - "depth": module.params["default_depth"], - "shallow_since": module.params["default_shallow_since"], - "version": module.params["default_version"], - "refspec": module.params["default_refspec"] - } - config = { - "retries": module.params["retries"], - "delay": module.params["delay"], - "force": module.params["force"], - "core_multiplier": module.params["core_multiplier"] - } - - # Set up defaults - for repo in git_repos: - for key in ["path", "refspec", "version", "depth", "shallow_since"]: - set_default(repo, key, defaults) - if "name" not in repo.keys(): - repo["name"] = os.path.basename(repo["src"]) - repo["dest"] = os.path.join(repo["path"], repo["name"]) - - # Define varibles - failures = multiprocessing.Manager().list() - core_count = multiprocessing.cpu_count() * config["core_multiplier"] - - # Load up process and pass in interrupt and core process count - p = multiprocessing.Pool(core_count, init_signal) - - clone_success = p.map(pull_wrapper, zip(git_repos, - itertools.repeat(config), - itertools.repeat(failures)), - chunksize=1) - p.close() - - success = all(i for i in clone_success) - if success: - module.exit_json(msg=str(git_repos), changed=True) - else: - module.fail_json(msg=("Module failed"), meta=failures) - - -if __name__ == '__main__': - main() diff --git a/scripts/get-ansible-role-requirements.yml b/scripts/get-ansible-role-requirements.yml index 963b03154e..84d56e2abc 100644 --- a/scripts/get-ansible-role-requirements.yml +++ b/scripts/get-ansible-role-requirements.yml @@ -137,7 +137,7 @@ - name: Clone git repos block: - name: Clone git repos (parallel) - git_requirements: + openstack.osa.git_requirements: default_path: "{{ role_path_default }}" default_depth: "{{ role_clone_default_depth }}" default_version: "master" diff --git a/scripts/openstack-ansible.rc b/scripts/openstack-ansible.rc index d9aa49f5f4..59773e4fa8 100644 --- a/scripts/openstack-ansible.rc +++ b/scripts/openstack-ansible.rc @@ -26,7 +26,7 @@ export ANSIBLE_ROLES_PATH="${ANSIBLE_ROLES_PATH:-/etc/ansible/roles:OSA_PLAYBOOK export ANSIBLE_COLLECTIONS_PATHS="${ANSIBLE_COLLECTIONS_PATHS:-/etc/ansible}" export ANSIBLE_COLLECTIONS_PATH="${ANSIBLE_COLLECTIONS_PATH:-/etc/ansible}" -export ANSIBLE_LIBRARY="${ANSIBLE_LIBRARY:-OSA_PLAYBOOK_PATH/library:/etc/ansible/roles/ceph-ansible/library}" +export ANSIBLE_LIBRARY="${ANSIBLE_LIBRARY:-/etc/ansible/roles/ceph-ansible/library}" export ANSIBLE_FILTER_PLUGINS="${ANSIBLE_FILTER_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/filter}" export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_ACTION_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/actions}" export ANSIBLE_CALLBACK_PLUGINS="${ANSIBLE_CALLBACK_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/callback}"