Add script to split the secret hieradata file

Although this only needs to be done once, it's ugly enough of a task
that it should have a script, and that script should be reviewed.

The script can be run locally on a machine without a hieradata and it
will create a local directory with all of the keys and none of the data.

It will also print out the total mapping and a report on which keys are
duplicated across hosts after group de-duplication is taken in to
account.

Change-Id: Id0af46abb8f4167bd91e95cd7a6cd2e35e5b0a94

tools/remap_manifest.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a utility intended to split a single common.yaml file from
+# a hieradata dir into a common.yaml / group/$::group.yaml / fqdn/$::fqdn.yaml
+# structure.
+# Values associated with the default node go into common.yaml
+# Values associated with a node that has a group and is specified with a
+# regex go into the group
+# Values associated with a node that has a group but specifies a specific host
+# will go into the fqdn file if they do not also appead in a group-related
+# host.
+#
+# This utility is not intended to solve all problems and is not intended to be
+# run on the same data twice. It's a one-off helper script, and it's
+# potentially destructure. So be prepared to have a backup of your common.yaml
+# file you can revert to after you inspect the output if it got something wrong
+
+import collections
+import pprint
+import yaml
+import os
+
+
+groups = {}
+fqdns = {}
+common = []
+
+current = None
+current_keys = None
+
+manifest_path = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', 'manifests/site.pp'))
+
+with open(manifest_path, 'r') as manifest:
+    for line in manifest:
+        if line.startswith('#'):
+            continue
+        if 'node default' in line:
+            current_keys = common
+            continue
+        elif line.startswith('node'):
+            current_keys = list()
+            current = dict(keys=current_keys)
+            if '/' in line:
+                name = line.split('/')[1]
+                groups[name] = current
+            else:
+                name = line.split("'")[1]
+                fqdns[name] = current
+            continue
+        if '$group' in line:
+            name = line.split('"')[1]
+            current['group'] = name
+        if 'hiera' in line:
+            key = line.split("'")[1]
+            if key not in common:
+                current_keys.append(key)
+
+
+new_groups = {}
+for value in groups.values():
+    new_groups[value['group']] = dict(keys=value['keys'])
+groups = new_groups
+
+# Trim group duplicates to just be in the group
+for key, value in fqdns.items():
+    if 'group' in value:
+        new_keys = []
+        for possible_key in value['keys']:
+            if possible_key not in groups[value['group']]['keys']:
+                new_keys.append(possible_key)
+        value['keys'] = new_keys
+
+# Print the values so that the person running can verify what's going on
+pprint.pprint(dict(common=common, groups=groups, fqdns=fqdns))
+
+
+def write_values(reverse_map, target, input_dict, source_vaues, root):
+    outdir = os.path.join(root, target)
+    if not os.path.exists(outdir):
+        os.makedirs(outdir)
+    for key, value in input_dict.items():
+        output_dict = {}
+        for name in value['keys']:
+            reverse_map[name].append(dict(target=target, key=key))
+            output_dict[name] = source_values[name]
+        with open(os.path.join(outdir, '%s.yaml' % key), 'w') as outfile:
+            outfile.write(yaml.dump(output_dict, default_flow_style=False))
+    return reverse_map
+
+
+def write_common_values(input_dict, source_values, root):
+    outdir = root
+    if not os.path.exists(outdir):
+        os.makedirs(outdir)
+    output_dict = {}
+    for name in input_dict:
+        output_dict[name] = source_values[name]
+    with open(os.path.join(outdir, 'common.yaml'), 'w') as outfile:
+        outfile.write(yaml.dump(output_dict, default_flow_style=False))
+
+
+if os.path.exists('/etc/puppet/hieradata/production/common.yaml'):
+    source_values = yaml.load(
+        open('/etc/puppet/hieradata/production/common.yaml'))
+    root = '/etc/puppet/hieradata/production'
+else:
+    def get_default():
+        return "Default data"
+    source_values = collections.defaultdict(get_default)
+    root = 'testoutput'
+
+
+write_common_values(common, source_values, root)
+reverse_map = collections.defaultdict(list)
+reverse_map = write_values(reverse_map, 'fqdn', fqdns, source_values, root)
+reverse_map = write_values(reverse_map, 'group', groups, source_values, root)
+
+
+for key, value in reverse_map.items():
+    if len(value) > 1:
+        print "Key %s duplicated in %r" % (key, value)