diff --git a/tools/ruamellib.py b/tools/ruamellib.py new file mode 100755 index 000000000..947134108 --- /dev/null +++ b/tools/ruamellib.py @@ -0,0 +1,61 @@ +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import ruamel.yaml + + +def none_representer(dumper, data): + return dumper.represent_scalar('tag:yaml.org,2002:null', 'null') + + +class YAML(object): + def __init__(self): + """Wrap construction of ruamel yaml object.""" + self.yaml = ruamel.yaml.YAML() + self.yaml.allow_duplicate_keys = True + self.yaml.representer.add_representer(type(None), none_representer) + self.yaml.indent(mapping=2, sequence=4, offset=2) + + def load(self, stream): + return self.yaml.load(stream) + + def tr(self, x): + newlines = [] + for line in x.split('\n'): + if '#' in line: + newlines.append(line) + else: + newlines.append(line[2:]) + x = '\n'.join(newlines) + x = re.sub(r'([^\n])\n-', r'\1\n\n-', x) + return x + + def dump(self, data, *args, **kwargs): + if isinstance(data, list): + kwargs['transform'] = self.tr + self.yaml.dump(data, *args, **kwargs) + + +_yaml = YAML() + + +def load(*args, **kwargs): + return _yaml.load(*args, **kwargs) + + +def dump(*args, **kwargs): + return _yaml.dump(*args, **kwargs) diff --git a/tools/update-test-platforms.py b/tools/update-test-platforms.py new file mode 100755 index 000000000..5d15147ba --- /dev/null +++ b/tools/update-test-platforms.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Update job definitions for multi-platform jobs, and make sure every +# in-repo test job appears in a project definition. This script +# re-writes the files in zuul-tests.d. It should be run from the root +# of the repo. + +import os + +from ruamel.yaml.comments import CommentedMap + +import ruamellib + +PLATFORMS = [ + 'centos-7', + 'debian-stable', + 'fedora-latest', + 'opensuse-15', + 'opensuse-tumbleweed', + 'ubuntu-bionic', + 'ubuntu-trusty', + 'ubuntu-xenial', +] + + +def get_nodeset(platform, multinode): + if not multinode: + return platform + d = CommentedMap() + # There are nodesets for fedora-latest and debian-stable + # but no labels, and there are no nodesets for fedora-29 + # and debian-stretch. So, map between these. + if platform == 'fedora-latest': + platform = 'fedora-29' + elif platform == 'debian-stable': + platform = 'debian-stretch' + d['nodes'] = [ + CommentedMap([('name', 'primary'), ('label', platform)]), + CommentedMap([('name', 'secondary'), ('label', platform)]), + ] + d['groups'] = [ + CommentedMap([('name', 'switch'), ('nodes', ['primary'])]), + CommentedMap([('name', 'peers'), ('nodes', ['secondary'])]), + ] + return d + + +def handle_file(fn): + yaml = ruamellib.YAML() + data = yaml.load(open(fn)) + outdata = [] + outprojects = [] + joblist = [] + for obj in data: + if 'job' in obj: + job = obj['job'] + if 'auto-generated' in job.get('tags', []): + continue + outdata.append(obj) + tags = job.get('tags', []) + all_platforms = False + if 'all-platforms-multinode' in tags: + multinode = True + all_platforms = True + elif 'all-platforms' in tags: + all_platforms = True + multinode = False + if all_platforms: + for platform in PLATFORMS: + ojob = CommentedMap() + ojob['name'] = job['name'] + '-' + platform + desc = job['description'].split('\n')[0] + ojob['description'] = desc + ' on ' \ + + platform + ojob['parent'] = job['name'] + ojob['tags'] = 'auto-generated' + ojob['nodeset'] = get_nodeset(platform, multinode) + outdata.append({'job': ojob}) + joblist.append(ojob['name']) + else: + joblist.append(job['name']) + elif 'project' in obj: + outprojects.append(obj) + else: + outdata.append(obj) + # We control the last project stanza + outdata.extend(outprojects) + project = outprojects[-1]['project'] + project['check']['jobs'] = joblist + project['gate']['jobs'] = joblist + with open(fn, 'w') as f: + yaml.dump(outdata, stream=f) + + +def main(): + for f in os.listdir('zuul-tests.d'): + if not f.endswith('.yaml'): + continue + if f == 'project.yaml': + continue + handle_file(os.path.join('zuul-tests.d', f)) + + +if __name__ == "__main__": + main() diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml index 3ec5865cf..729be52c3 100644 --- a/zuul-tests.d/container-roles-jobs.yaml +++ b/zuul-tests.d/container-roles-jobs.yaml @@ -1,5 +1,3 @@ -# Jobs which test roles listed in container-roles.rst. - - job: name: zuul-jobs-test-registry description: | @@ -16,8 +14,7 @@ - roles/run-buildset-registry/.* - roles/use-buildset-registry/.* - test-playbooks/registry/.* - run: - test-playbooks/registry/test-registry.yaml + run: test-playbooks/registry/test-registry.yaml nodeset: nodes: - name: intermediate-registry @@ -31,8 +28,7 @@ - project: check: - jobs: + jobs: &id001 - zuul-jobs-test-registry gate: - jobs: - - zuul-jobs-test-registry + jobs: *id001 diff --git a/zuul-tests.d/general-roles-jobs.yaml b/zuul-tests.d/general-roles-jobs.yaml index f56f75c26..2ee83f531 100644 --- a/zuul-tests.d/general-roles-jobs.yaml +++ b/zuul-tests.d/general-roles-jobs.yaml @@ -1,5 +1,3 @@ -# Jobs which test roles listed in general-roles.rst - - job: name: zuul-jobs-test-upload-git-mirror description: | @@ -13,8 +11,7 @@ - project: check: - jobs: + jobs: &id001 - zuul-jobs-test-upload-git-mirror gate: - jobs: - - zuul-jobs-test-upload-git-mirror + jobs: *id001 diff --git a/zuul-tests.d/jobs.yaml b/zuul-tests.d/jobs.yaml index f880e6c1b..c37569b35 100644 --- a/zuul-tests.d/jobs.yaml +++ b/zuul-tests.d/jobs.yaml @@ -1,5 +1,3 @@ -# Test jobs which apply to all parts of this repo. - - job: name: zuul-jobs-tox-linters parent: tox-linters @@ -16,8 +14,7 @@ - project: check: - jobs: - - zuul-jobs-test-install-nodejs + jobs: &id001 + - zuul-jobs-tox-linters gate: - jobs: - - zuul-jobs-test-install-nodejs + jobs: *id001 diff --git a/zuul-tests.d/js-roles-jobs.yaml b/zuul-tests.d/js-roles-jobs.yaml index 25a49fc1b..a6f80955f 100644 --- a/zuul-tests.d/js-roles-jobs.yaml +++ b/zuul-tests.d/js-roles-jobs.yaml @@ -1,12 +1,9 @@ -# Jobs which test roles listed in js-roles.rst - - job: name: zuul-jobs-test-install-nodejs description: Test the install-nodejs role files: - roles/install-nodejs/.* - run: - test-playbooks/simple-role-test.yaml + run: test-playbooks/simple-role-test.yaml vars: role_name: install-nodejs @@ -14,8 +11,7 @@ - project: check: - jobs: + jobs: &id001 - zuul-jobs-test-install-nodejs gate: - jobs: - - zuul-jobs-test-install-nodejs + jobs: *id001 diff --git a/zuul-tests.d/puppet-roles-jobs.yaml b/zuul-tests.d/puppet-roles-jobs.yaml index 6c8d28c63..4724b33b9 100644 --- a/zuul-tests.d/puppet-roles-jobs.yaml +++ b/zuul-tests.d/puppet-roles-jobs.yaml @@ -1,12 +1,9 @@ -# Jobs which test roles listed in puppet-roles.rst - - job: name: zuul-jobs-test-install-pdk-dependencies description: Test the install-pdk-dependencies role files: - roles/install-pdk-dependencies/.* - run: - test-playbooks/simple-role-test.yaml + run: test-playbooks/simple-role-test.yaml vars: role_name: install-pdk-dependencies @@ -14,8 +11,7 @@ - project: check: - jobs: + jobs: &id001 - zuul-jobs-test-install-pdk-dependencies gate: - jobs: - - zuul-jobs-test-install-pdk-dependencies + jobs: *id001