Add a script to make platform-specific versions of jobs

This automatically creates platform-specific versions of jobs
with special tags.  The changes to existing files are minor
cleanup from running the script.

Change-Id: I818b240b32610a00d717e021494d2e4d8e518c32

tools/ruamellib.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import ruamel.yaml
+
+
+def none_representer(dumper, data):
+    return dumper.represent_scalar('tag:yaml.org,2002:null', 'null')
+
+
+class YAML(object):
+    def __init__(self):
+        """Wrap construction of ruamel yaml object."""
+        self.yaml = ruamel.yaml.YAML()
+        self.yaml.allow_duplicate_keys = True
+        self.yaml.representer.add_representer(type(None), none_representer)
+        self.yaml.indent(mapping=2, sequence=4, offset=2)
+
+    def load(self, stream):
+        return self.yaml.load(stream)
+
+    def tr(self, x):
+        newlines = []
+        for line in x.split('\n'):
+            if '#' in line:
+                newlines.append(line)
+            else:
+                newlines.append(line[2:])
+        x = '\n'.join(newlines)
+        x = re.sub(r'([^\n])\n-', r'\1\n\n-', x)
+        return x
+
+    def dump(self, data, *args, **kwargs):
+        if isinstance(data, list):
+            kwargs['transform'] = self.tr
+        self.yaml.dump(data, *args, **kwargs)
+
+
+_yaml = YAML()
+
+
+def load(*args, **kwargs):
+    return _yaml.load(*args, **kwargs)
+
+
+def dump(*args, **kwargs):
+    return _yaml.dump(*args, **kwargs)

tools/update-test-platforms.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+#
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Update job definitions for multi-platform jobs, and make sure every
+# in-repo test job appears in a project definition.  This script
+# re-writes the files in zuul-tests.d.  It should be run from the root
+# of the repo.
+
+import os
+
+from ruamel.yaml.comments import CommentedMap
+
+import ruamellib
+
+PLATFORMS = [
+    'centos-7',
+    'debian-stable',
+    'fedora-latest',
+    'opensuse-15',
+    'opensuse-tumbleweed',
+    'ubuntu-bionic',
+    'ubuntu-trusty',
+    'ubuntu-xenial',
+]
+
+
+def get_nodeset(platform, multinode):
+    if not multinode:
+        return platform
+    d = CommentedMap()
+    # There are nodesets for fedora-latest and debian-stable
+    # but no labels, and there are no nodesets for fedora-29
+    # and debian-stretch. So, map between these.
+    if platform == 'fedora-latest':
+        platform = 'fedora-29'
+    elif platform == 'debian-stable':
+        platform = 'debian-stretch'
+    d['nodes'] = [
+        CommentedMap([('name', 'primary'), ('label', platform)]),
+        CommentedMap([('name', 'secondary'), ('label', platform)]),
+    ]
+    d['groups'] = [
+        CommentedMap([('name', 'switch'), ('nodes', ['primary'])]),
+        CommentedMap([('name', 'peers'), ('nodes', ['secondary'])]),
+    ]
+    return d
+
+
+def handle_file(fn):
+    yaml = ruamellib.YAML()
+    data = yaml.load(open(fn))
+    outdata = []
+    outprojects = []
+    joblist = []
+    for obj in data:
+        if 'job' in obj:
+            job = obj['job']
+            if 'auto-generated' in job.get('tags', []):
+                continue
+            outdata.append(obj)
+            tags = job.get('tags', [])
+            all_platforms = False
+            if 'all-platforms-multinode' in tags:
+                multinode = True
+                all_platforms = True
+            elif 'all-platforms' in tags:
+                all_platforms = True
+                multinode = False
+            if all_platforms:
+                for platform in PLATFORMS:
+                    ojob = CommentedMap()
+                    ojob['name'] = job['name'] + '-' + platform
+                    desc = job['description'].split('\n')[0]
+                    ojob['description'] = desc + ' on ' \
+                        + platform
+                    ojob['parent'] = job['name']
+                    ojob['tags'] = 'auto-generated'
+                    ojob['nodeset'] = get_nodeset(platform, multinode)
+                    outdata.append({'job': ojob})
+                    joblist.append(ojob['name'])
+            else:
+                joblist.append(job['name'])
+        elif 'project' in obj:
+            outprojects.append(obj)
+        else:
+            outdata.append(obj)
+    # We control the last project stanza
+    outdata.extend(outprojects)
+    project = outprojects[-1]['project']
+    project['check']['jobs'] = joblist
+    project['gate']['jobs'] = joblist
+    with open(fn, 'w') as f:
+        yaml.dump(outdata, stream=f)
+
+
+def main():
+    for f in os.listdir('zuul-tests.d'):
+        if not f.endswith('.yaml'):
+            continue
+        if f == 'project.yaml':
+            continue
+        handle_file(os.path.join('zuul-tests.d', f))
+
+
+if __name__ == "__main__":
+    main()

zuul-tests.d/container-roles-jobs.yaml

@@ -1,5 +1,3 @@
-# Jobs which test roles listed in container-roles.rst.
-
 - job:
     name: zuul-jobs-test-registry
     description: |
@@ -16,8 +14,7 @@
       - roles/run-buildset-registry/.*
       - roles/use-buildset-registry/.*
       - test-playbooks/registry/.*
-    run:
+    run: test-playbooks/registry/test-registry.yaml
-      test-playbooks/registry/test-registry.yaml
     nodeset:
       nodes:
         - name: intermediate-registry
@@ -31,8 +28,7 @@
 
 - project:
     check:
-      jobs:
+      jobs: &id001
         - zuul-jobs-test-registry
     gate:
-      jobs:
+      jobs: *id001
-        - zuul-jobs-test-registry

zuul-tests.d/general-roles-jobs.yaml

@@ -1,5 +1,3 @@
-# Jobs which test roles listed in general-roles.rst
-
 - job:
     name: zuul-jobs-test-upload-git-mirror
     description: |
@@ -13,8 +11,7 @@
 
 - project:
     check:
-      jobs:
+      jobs: &id001
         - zuul-jobs-test-upload-git-mirror
     gate:
-      jobs:
+      jobs: *id001
-        - zuul-jobs-test-upload-git-mirror

zuul-tests.d/jobs.yaml

@@ -1,5 +1,3 @@
-# Test jobs which apply to all parts of this repo.
-
 - job:
     name: zuul-jobs-tox-linters
     parent: tox-linters
@@ -16,8 +14,7 @@
 
 - project:
     check:
-      jobs:
+      jobs: &id001
-        - zuul-jobs-test-install-nodejs
+        - zuul-jobs-tox-linters
     gate:
-      jobs:
+      jobs: *id001
-        - zuul-jobs-test-install-nodejs

zuul-tests.d/js-roles-jobs.yaml

@@ -1,12 +1,9 @@
-# Jobs which test roles listed in js-roles.rst
-
 - job:
     name: zuul-jobs-test-install-nodejs
     description: Test the install-nodejs role
     files:
       - roles/install-nodejs/.*
-    run:
+    run: test-playbooks/simple-role-test.yaml
-      test-playbooks/simple-role-test.yaml
     vars:
       role_name: install-nodejs
 
@@ -14,8 +11,7 @@
 
 - project:
     check:
-      jobs:
+      jobs: &id001
         - zuul-jobs-test-install-nodejs
     gate:
-      jobs:
+      jobs: *id001
-        - zuul-jobs-test-install-nodejs

zuul-tests.d/puppet-roles-jobs.yaml

@@ -1,12 +1,9 @@
-# Jobs which test roles listed in puppet-roles.rst
-
 - job:
     name: zuul-jobs-test-install-pdk-dependencies
     description: Test the install-pdk-dependencies role
     files:
       - roles/install-pdk-dependencies/.*
-    run:
+    run: test-playbooks/simple-role-test.yaml
-      test-playbooks/simple-role-test.yaml
     vars:
       role_name: install-pdk-dependencies
 
@@ -14,8 +11,7 @@
 
 - project:
     check:
-      jobs:
+      jobs: &id001
         - zuul-jobs-test-install-pdk-dependencies
     gate:
-      jobs:
+      jobs: *id001
-        - zuul-jobs-test-install-pdk-dependencies