Rework package-installs to collapse on build host
Instead of doing the work in the image of parsing through the element's package-install declarations, we can squash it on hostside, where we have both YAML and JSON available to us, and then emit a single pre-processed file into the target to be used later. Change-Id: I3f182aa3aae0a79b2f3ea4e66c1878ad12878b0a
This commit is contained in:
Monty Taylor
Gregory Haynes
parent
1273cd5532
commit
98d008c6af
82
elements/package-installs/bin/package-installs-squash
Executable file
82
elements/package-installs/bin/package-installs-squash
Executable file
@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def collect_data(data, filename, element_name):
|
||||
try:
|
||||
objs = json.load(open(filename))
|
||||
except ValueError:
|
||||
objs = yaml.load(open(filename))
|
||||
for pkg_name, params in objs.items():
|
||||
if not params:
|
||||
params = {}
|
||||
phase = params.get('phase', 'install.d')
|
||||
install = "install"
|
||||
if 'uninstall' in params:
|
||||
install = "uninstall"
|
||||
|
||||
data[phase][install].append((pkg_name, element_name))
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Produce a single packages-installs file from all of"
|
||||
" the available package-installs files")
|
||||
parser.add_argument('--elements', required=True,
|
||||
help="Which elements to squash")
|
||||
parser.add_argument('--path', required=True,
|
||||
help="Elements path to search for elements")
|
||||
parser.add_argument('outfile', help="Location of the output file")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Replicate the logic of finding the first element, because we can't
|
||||
# operate on the post-copied hooks dir, since we lose element context
|
||||
element_dirs = list()
|
||||
for element_name in args.elements.split():
|
||||
for elements_dir in args.path.split(':'):
|
||||
potential_path = os.path.join(elements_dir, element_name)
|
||||
if os.path.exists(potential_path):
|
||||
element_dirs.append((elements_dir, element_name))
|
||||
|
||||
# Collect the merge of all of the existing install files in the elements
|
||||
# that are the first on the ELEMENT_PATH
|
||||
final_dict = collections.defaultdict(
|
||||
functools.partial(collections.defaultdict, list))
|
||||
for (elements_dir, element_name) in element_dirs:
|
||||
for file_type in ('json', 'yaml'):
|
||||
target_file = os.path.join(
|
||||
elements_dir, element_name, "package-installs.%s" % file_type)
|
||||
if not os.path.exists(target_file):
|
||||
continue
|
||||
final_dict = collect_data(final_dict, target_file, element_name)
|
||||
|
||||
# Write the resulting file
|
||||
with open(args.outfile, 'w') as outfile:
|
||||
json.dump(
|
||||
final_dict, outfile,
|
||||
indent=True, separators=(',', ': '), sort_keys=False)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -15,131 +15,73 @@
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
from json import load as json_load
|
||||
import os
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from yaml import load as yaml_load
|
||||
|
||||
|
||||
class PackageInstalls(object):
|
||||
|
||||
@classmethod
|
||||
def phase_to_attr(cls, phase):
|
||||
return phase.replace('.', '_').replace('-', '_')
|
||||
|
||||
@classmethod
|
||||
def from_yaml_path(cls, path):
|
||||
with open(path) as fp:
|
||||
return PackageInstalls.from_native_objs(yaml_load(fp))
|
||||
|
||||
@classmethod
|
||||
def from_json_path(cls, path):
|
||||
with open(path) as fp:
|
||||
return PackageInstalls.from_native_objs(json_load(fp))
|
||||
|
||||
@classmethod
|
||||
def from_native_objs(cls, objs):
|
||||
init_args = {}
|
||||
for pkg_name, params in objs.items():
|
||||
uninstall = False
|
||||
phase = "install.d"
|
||||
|
||||
try:
|
||||
phase = params["phase"]
|
||||
except (KeyError, TypeError):
|
||||
pass
|
||||
|
||||
try:
|
||||
uninstall = bool(params["uninstall"])
|
||||
except (KeyError, TypeError):
|
||||
pass
|
||||
|
||||
init_arg = PackageInstalls.phase_to_attr(phase)
|
||||
if uninstall:
|
||||
init_arg = init_arg + '_uninst'
|
||||
|
||||
init_args[init_arg] = init_args.get(init_arg, []) + [pkg_name]
|
||||
return PackageInstalls(**init_args)
|
||||
|
||||
def __init__(self, **phase_installs):
|
||||
for phase, pkgs in phase_installs.items():
|
||||
setattr(self, phase, pkgs)
|
||||
|
||||
|
||||
class PackageInstallsController(object):
|
||||
|
||||
def __init__(self, path='/usr/share/package-installs'):
|
||||
self.path = path
|
||||
|
||||
def package_installs(self):
|
||||
for yaml_path in os.listdir(self.path):
|
||||
full_path = os.path.join(self.path, yaml_path)
|
||||
if full_path.endswith('.yaml'):
|
||||
pi = PackageInstalls.from_yaml_path(full_path)
|
||||
elif full_path.endswith('.json'):
|
||||
pi = PackageInstalls.from_json_path(full_path)
|
||||
else:
|
||||
print("No decoder known for %s, skipping" % full_path)
|
||||
continue
|
||||
yield (yaml_path[:-5], pi)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Install or uninstall packages for a specific phase based"
|
||||
" on package-installs files.")
|
||||
parser.add_argument('--phase',
|
||||
parser.add_argument('--phase', required=True,
|
||||
help="Install phase to filter on. Valid options are"
|
||||
" 'install.d' or pre-install.d")
|
||||
parser.add_argument('--uninstall', action="store_true",
|
||||
help="Only show packages to uninstall. By default only"
|
||||
" packages to install are shown")
|
||||
args, extra = parser.parse_known_args()
|
||||
parser.add_argument('-n', '--noop', action="store_true",
|
||||
help="Don't actually install, just print the command")
|
||||
parser.add_argument('infile', help="File to process")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.phase:
|
||||
print("Please specify an install phase.")
|
||||
sys.exit(1)
|
||||
packages = json.load(open(args.infile))
|
||||
if args.uninstall:
|
||||
install = "uninstall"
|
||||
else:
|
||||
install = "install"
|
||||
|
||||
pi_c = PackageInstallsController()
|
||||
pkgs = []
|
||||
for element, pi in pi_c.package_installs():
|
||||
installs_attr = PackageInstalls.phase_to_attr(args.phase)
|
||||
if args.uninstall:
|
||||
installs_attr += '_uninst'
|
||||
pkgs = list()
|
||||
if args.phase in packages and install in packages[args.phase]:
|
||||
install_packages = packages[args.phase][install]
|
||||
else:
|
||||
print("Nothing to %s" % install)
|
||||
sys.exit(0)
|
||||
|
||||
for (pkg, element) in install_packages:
|
||||
print("%sing %s from %s" % (install, pkg, element))
|
||||
pkg_map_args = ["pkg-map", "--missing-ok", "--element", element]
|
||||
pkg_map_args.append(pkg)
|
||||
|
||||
try:
|
||||
phase_installs = getattr(pi, installs_attr)
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
for pkg in phase_installs:
|
||||
print("Installing %s from %s" % (phase_installs, element))
|
||||
pkg_map_args = ["pkg-map", "--missing-ok", "--element", element]
|
||||
pkg_map_args += phase_installs
|
||||
|
||||
try:
|
||||
map_output = subprocess.check_output(pkg_map_args)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if e.returncode == 1:
|
||||
map_output = subprocess.check_output(
|
||||
pkg_map_args, stderr=subprocess.STDOUT)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if e.returncode == 1:
|
||||
if args.noop:
|
||||
pkgs.append(pkg)
|
||||
continue
|
||||
else:
|
||||
print("pkg-map failed with error %s" % e.output)
|
||||
sys.exit(1)
|
||||
elif e.returncode == 2:
|
||||
pkgs += phase_installs
|
||||
continue
|
||||
pkgs += map_output.strip().split('\n')
|
||||
elif e.returncode == 2:
|
||||
pkgs.append(pkg)
|
||||
continue
|
||||
pkgs.extend(map_output.strip().split('\n'))
|
||||
|
||||
install_args = ["install-packages"]
|
||||
if args.uninstall:
|
||||
install_args.append("-e")
|
||||
install_args.extend(pkgs)
|
||||
install_args.extend(list(set(pkgs)))
|
||||
|
||||
try:
|
||||
subprocess.check_output(install_args)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("install failed with error %s" % e.output)
|
||||
sys.exit(1)
|
||||
if args.noop:
|
||||
print(" ".join(install_args))
|
||||
else:
|
||||
try:
|
||||
subprocess.check_output(install_args)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("install failed with error %s" % e.output)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@ -1,14 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -eux
|
||||
set -o pipefail
|
||||
|
||||
dest_dir="$TMP_MOUNT_PATH/usr/share/package-installs/"
|
||||
sudo mkdir -p $dest_dir
|
||||
|
||||
for ELEMENT in $IMAGE_ELEMENT ; do
|
||||
for DIR in ${ELEMENTS_PATH//:/ }; do
|
||||
if [ -f "$DIR/$ELEMENT/package-installs.yaml" ]; then
|
||||
sudo cp $DIR/$ELEMENT/package-installs.yaml $dest_dir/$ELEMENT.yaml
|
||||
fi
|
||||
done
|
||||
done
|
||||
5
elements/package-installs/extra-data.d/99-squash-package-install
Executable file
5
elements/package-installs/extra-data.d/99-squash-package-install
Executable file
@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -eux
|
||||
set -o pipefail
|
||||
|
||||
sudo $(dirname $0)/../bin/package-installs-squash --elements="$IMAGE_ELEMENT" --path=$ELEMENTS_PATH $TMP_MOUNT_PATH/tmp/package-installs.json
|
||||
@ -4,4 +4,4 @@ set -eux
|
||||
set -o pipefail
|
||||
|
||||
package-installs -d $(dirname $0)
|
||||
package-installs-v2 --phase install.d
|
||||
package-installs-v2 --phase install.d /tmp/package-installs.json
|
||||
|
||||
@ -4,4 +4,4 @@ set -eux
|
||||
set -o pipefail
|
||||
|
||||
package-uninstalls -d $(dirname $0)
|
||||
package-installs-v2 --phase install.d --uninstall
|
||||
package-installs-v2 --phase install.d --uninstall /tmp/package-installs.json
|
||||
|
||||
@ -4,4 +4,4 @@ set -eux
|
||||
set -o pipefail
|
||||
|
||||
package-installs -d $(dirname $0)
|
||||
package-installs-v2 --phase post-install.d
|
||||
package-installs-v2 --phase post-install.d /tmp/package-installs.json
|
||||
|
||||
@ -4,4 +4,4 @@ set -eux
|
||||
set -o pipefail
|
||||
|
||||
package-uninstalls -d $(dirname $0)
|
||||
package-installs-v2 --phase post-install.d --uninstall
|
||||
package-installs-v2 --phase post-install.d --uninstall /tmp/package-installs.json
|
||||
|
||||
@ -4,4 +4,4 @@ set -eux
|
||||
set -o pipefail
|
||||
|
||||
package-installs -d $(dirname $0)
|
||||
package-installs-v2 --phase pre-install.d
|
||||
package-installs-v2 --phase pre-install.d /tmp/package-installs.json
|
||||
|
||||
@ -4,4 +4,4 @@ set -eux
|
||||
set -o pipefail
|
||||
|
||||
package-uninstalls -d $(dirname $0)
|
||||
package-installs-v2 --phase pre-install.d --uninstall
|
||||
package-installs-v2 --phase pre-install.d --uninstall /tmp/package-installs.json
|
||||
|
||||
Loading…
Reference in New Issue
Block a user