From 98d008c6affcc5e6eeb8ed146cd74cad77c144cb Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 10 Dec 2014 08:19:28 -0800 Subject: [PATCH] Rework package-installs to collapse on build host Instead of doing the work in the image of parsing through the element's package-install declarations, we can squash it on hostside, where we have both YAML and JSON available to us, and then emit a single pre-processed file into the target to be used later. Change-Id: I3f182aa3aae0a79b2f3ea4e66c1878ad12878b0a --- .../bin/package-installs-squash | 82 ++++++++++ .../package-installs/bin/package-installs-v2 | 144 ++++++------------ .../11-create-package-installs-dir | 14 -- .../extra-data.d/99-squash-package-install | 5 + .../install.d/00-package-installs | 2 +- .../install.d/99-package-uninstalls | 2 +- .../post-install.d/00-package-installs | 2 +- .../post-install.d/99-package-uninstalls | 2 +- .../pre-install.d/02-package-installs | 2 +- .../pre-install.d/99-package-uninstalls | 2 +- 10 files changed, 136 insertions(+), 121 deletions(-) create mode 100755 elements/package-installs/bin/package-installs-squash delete mode 100755 elements/package-installs/extra-data.d/11-create-package-installs-dir create mode 100755 elements/package-installs/extra-data.d/99-squash-package-install diff --git a/elements/package-installs/bin/package-installs-squash b/elements/package-installs/bin/package-installs-squash new file mode 100755 index 000000000..b79e6417c --- /dev/null +++ b/elements/package-installs/bin/package-installs-squash @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import collections +import functools +import json +import os + +import yaml + + +def collect_data(data, filename, element_name): + try: + objs = json.load(open(filename)) + except ValueError: + objs = yaml.load(open(filename)) + for pkg_name, params in objs.items(): + if not params: + params = {} + phase = params.get('phase', 'install.d') + install = "install" + if 'uninstall' in params: + install = "uninstall" + + data[phase][install].append((pkg_name, element_name)) + return data + + +def main(): + parser = argparse.ArgumentParser( + description="Produce a single packages-installs file from all of" + " the available package-installs files") + parser.add_argument('--elements', required=True, + help="Which elements to squash") + parser.add_argument('--path', required=True, + help="Elements path to search for elements") + parser.add_argument('outfile', help="Location of the output file") + args = parser.parse_args() + + # Replicate the logic of finding the first element, because we can't + # operate on the post-copied hooks dir, since we lose element context + element_dirs = list() + for element_name in args.elements.split(): + for elements_dir in args.path.split(':'): + potential_path = os.path.join(elements_dir, element_name) + if os.path.exists(potential_path): + element_dirs.append((elements_dir, element_name)) + + # Collect the merge of all of the existing install files in the elements + # that are the first on the ELEMENT_PATH + final_dict = collections.defaultdict( + functools.partial(collections.defaultdict, list)) + for (elements_dir, element_name) in element_dirs: + for file_type in ('json', 'yaml'): + target_file = os.path.join( + elements_dir, element_name, "package-installs.%s" % file_type) + if not os.path.exists(target_file): + continue + final_dict = collect_data(final_dict, target_file, element_name) + + # Write the resulting file + with open(args.outfile, 'w') as outfile: + json.dump( + final_dict, outfile, + indent=True, separators=(',', ': '), sort_keys=False) + +if __name__ == '__main__': + main() diff --git a/elements/package-installs/bin/package-installs-v2 b/elements/package-installs/bin/package-installs-v2 index 86eca98aa..7b04fb17e 100755 --- a/elements/package-installs/bin/package-installs-v2 +++ b/elements/package-installs/bin/package-installs-v2 @@ -15,131 +15,73 @@ # under the License. import argparse -from json import load as json_load -import os +import json import subprocess import sys -from yaml import load as yaml_load - - -class PackageInstalls(object): - - @classmethod - def phase_to_attr(cls, phase): - return phase.replace('.', '_').replace('-', '_') - - @classmethod - def from_yaml_path(cls, path): - with open(path) as fp: - return PackageInstalls.from_native_objs(yaml_load(fp)) - - @classmethod - def from_json_path(cls, path): - with open(path) as fp: - return PackageInstalls.from_native_objs(json_load(fp)) - - @classmethod - def from_native_objs(cls, objs): - init_args = {} - for pkg_name, params in objs.items(): - uninstall = False - phase = "install.d" - - try: - phase = params["phase"] - except (KeyError, TypeError): - pass - - try: - uninstall = bool(params["uninstall"]) - except (KeyError, TypeError): - pass - - init_arg = PackageInstalls.phase_to_attr(phase) - if uninstall: - init_arg = init_arg + '_uninst' - - init_args[init_arg] = init_args.get(init_arg, []) + [pkg_name] - return PackageInstalls(**init_args) - - def __init__(self, **phase_installs): - for phase, pkgs in phase_installs.items(): - setattr(self, phase, pkgs) - - -class PackageInstallsController(object): - - def __init__(self, path='/usr/share/package-installs'): - self.path = path - - def package_installs(self): - for yaml_path in os.listdir(self.path): - full_path = os.path.join(self.path, yaml_path) - if full_path.endswith('.yaml'): - pi = PackageInstalls.from_yaml_path(full_path) - elif full_path.endswith('.json'): - pi = PackageInstalls.from_json_path(full_path) - else: - print("No decoder known for %s, skipping" % full_path) - continue - yield (yaml_path[:-5], pi) - def main(): parser = argparse.ArgumentParser( description="Install or uninstall packages for a specific phase based" " on package-installs files.") - parser.add_argument('--phase', + parser.add_argument('--phase', required=True, help="Install phase to filter on. Valid options are" " 'install.d' or pre-install.d") parser.add_argument('--uninstall', action="store_true", help="Only show packages to uninstall. By default only" " packages to install are shown") - args, extra = parser.parse_known_args() + parser.add_argument('-n', '--noop', action="store_true", + help="Don't actually install, just print the command") + parser.add_argument('infile', help="File to process") + args = parser.parse_args() - if not args.phase: - print("Please specify an install phase.") - sys.exit(1) + packages = json.load(open(args.infile)) + if args.uninstall: + install = "uninstall" + else: + install = "install" - pi_c = PackageInstallsController() - pkgs = [] - for element, pi in pi_c.package_installs(): - installs_attr = PackageInstalls.phase_to_attr(args.phase) - if args.uninstall: - installs_attr += '_uninst' + pkgs = list() + if args.phase in packages and install in packages[args.phase]: + install_packages = packages[args.phase][install] + else: + print("Nothing to %s" % install) + sys.exit(0) + + for (pkg, element) in install_packages: + print("%sing %s from %s" % (install, pkg, element)) + pkg_map_args = ["pkg-map", "--missing-ok", "--element", element] + pkg_map_args.append(pkg) try: - phase_installs = getattr(pi, installs_attr) - except AttributeError: - continue - - for pkg in phase_installs: - print("Installing %s from %s" % (phase_installs, element)) - pkg_map_args = ["pkg-map", "--missing-ok", "--element", element] - pkg_map_args += phase_installs - - try: - map_output = subprocess.check_output(pkg_map_args) - except subprocess.CalledProcessError as e: - if e.returncode == 1: + map_output = subprocess.check_output( + pkg_map_args, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + if e.returncode == 1: + if args.noop: + pkgs.append(pkg) + continue + else: print("pkg-map failed with error %s" % e.output) sys.exit(1) - elif e.returncode == 2: - pkgs += phase_installs - continue - pkgs += map_output.strip().split('\n') + elif e.returncode == 2: + pkgs.append(pkg) + continue + pkgs.extend(map_output.strip().split('\n')) install_args = ["install-packages"] if args.uninstall: install_args.append("-e") - install_args.extend(pkgs) + install_args.extend(list(set(pkgs))) - try: - subprocess.check_output(install_args) - except subprocess.CalledProcessError as e: - print("install failed with error %s" % e.output) - sys.exit(1) + if args.noop: + print(" ".join(install_args)) + else: + try: + subprocess.check_output(install_args) + except subprocess.CalledProcessError as e: + print("install failed with error %s" % e.output) + sys.exit(1) if __name__ == '__main__': diff --git a/elements/package-installs/extra-data.d/11-create-package-installs-dir b/elements/package-installs/extra-data.d/11-create-package-installs-dir deleted file mode 100755 index 0aaa50c21..000000000 --- a/elements/package-installs/extra-data.d/11-create-package-installs-dir +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -eux -set -o pipefail - -dest_dir="$TMP_MOUNT_PATH/usr/share/package-installs/" -sudo mkdir -p $dest_dir - -for ELEMENT in $IMAGE_ELEMENT ; do - for DIR in ${ELEMENTS_PATH//:/ }; do - if [ -f "$DIR/$ELEMENT/package-installs.yaml" ]; then - sudo cp $DIR/$ELEMENT/package-installs.yaml $dest_dir/$ELEMENT.yaml - fi - done -done diff --git a/elements/package-installs/extra-data.d/99-squash-package-install b/elements/package-installs/extra-data.d/99-squash-package-install new file mode 100755 index 000000000..74a31075c --- /dev/null +++ b/elements/package-installs/extra-data.d/99-squash-package-install @@ -0,0 +1,5 @@ +#!/bin/bash +set -eux +set -o pipefail + +sudo $(dirname $0)/../bin/package-installs-squash --elements="$IMAGE_ELEMENT" --path=$ELEMENTS_PATH $TMP_MOUNT_PATH/tmp/package-installs.json diff --git a/elements/package-installs/install.d/00-package-installs b/elements/package-installs/install.d/00-package-installs index 540c8c02c..9b8cc696a 100755 --- a/elements/package-installs/install.d/00-package-installs +++ b/elements/package-installs/install.d/00-package-installs @@ -4,4 +4,4 @@ set -eux set -o pipefail package-installs -d $(dirname $0) -package-installs-v2 --phase install.d +package-installs-v2 --phase install.d /tmp/package-installs.json diff --git a/elements/package-installs/install.d/99-package-uninstalls b/elements/package-installs/install.d/99-package-uninstalls index b2c91679a..afedd382d 100755 --- a/elements/package-installs/install.d/99-package-uninstalls +++ b/elements/package-installs/install.d/99-package-uninstalls @@ -4,4 +4,4 @@ set -eux set -o pipefail package-uninstalls -d $(dirname $0) -package-installs-v2 --phase install.d --uninstall +package-installs-v2 --phase install.d --uninstall /tmp/package-installs.json diff --git a/elements/package-installs/post-install.d/00-package-installs b/elements/package-installs/post-install.d/00-package-installs index 0ed5cdedb..3251f8e1f 100755 --- a/elements/package-installs/post-install.d/00-package-installs +++ b/elements/package-installs/post-install.d/00-package-installs @@ -4,4 +4,4 @@ set -eux set -o pipefail package-installs -d $(dirname $0) -package-installs-v2 --phase post-install.d +package-installs-v2 --phase post-install.d /tmp/package-installs.json diff --git a/elements/package-installs/post-install.d/99-package-uninstalls b/elements/package-installs/post-install.d/99-package-uninstalls index f7fb6e02a..dec560eec 100755 --- a/elements/package-installs/post-install.d/99-package-uninstalls +++ b/elements/package-installs/post-install.d/99-package-uninstalls @@ -4,4 +4,4 @@ set -eux set -o pipefail package-uninstalls -d $(dirname $0) -package-installs-v2 --phase post-install.d --uninstall +package-installs-v2 --phase post-install.d --uninstall /tmp/package-installs.json diff --git a/elements/package-installs/pre-install.d/02-package-installs b/elements/package-installs/pre-install.d/02-package-installs index dbf76c739..36d5ba224 100755 --- a/elements/package-installs/pre-install.d/02-package-installs +++ b/elements/package-installs/pre-install.d/02-package-installs @@ -4,4 +4,4 @@ set -eux set -o pipefail package-installs -d $(dirname $0) -package-installs-v2 --phase pre-install.d +package-installs-v2 --phase pre-install.d /tmp/package-installs.json diff --git a/elements/package-installs/pre-install.d/99-package-uninstalls b/elements/package-installs/pre-install.d/99-package-uninstalls index ebccedc01..c96a9314c 100755 --- a/elements/package-installs/pre-install.d/99-package-uninstalls +++ b/elements/package-installs/pre-install.d/99-package-uninstalls @@ -4,4 +4,4 @@ set -eux set -o pipefail package-uninstalls -d $(dirname $0) -package-installs-v2 --phase pre-install.d --uninstall +package-installs-v2 --phase pre-install.d --uninstall /tmp/package-installs.json