From f999949aedec7d01c5837649d0d9535a3723c4c3 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Sat, 16 Jul 2022 11:44:39 -0700 Subject: [PATCH] Add support for freezing the job graph This adds a "job-graph" subcommand which uses the freeze-jobs api to return information about what jobs may be run. This also adds a new formatter, "dot". The only command supporting this formatter is job-graph. To see it in action, try: zuul-client --format dot --zuul-url https://zuul.opendev.org job-graph --tenant openstack --pipeline check --project opendev/system-config --branch master | xdot - Finally, this also adds "json" as an acceptable alias to "JSON" when specifying the output format since that is a widely used convention. Change-Id: I9adc3ab87bfa11432ae621b65dd94189bb17e42c --- doc/source/commands.rst | 21 +++++++++ .../notes/job-graph-7d9c6194dd60b06d.yaml | 5 +++ tests/unit/test_api.py | 32 ++++++++++++++ tests/unit/test_cmd.py | 22 ++++++++++ zuulclient/api/__init__.py | 12 +++++ zuulclient/cmd/__init__.py | 31 ++++++++++++- zuulclient/utils/formatters.py | 44 +++++++++++++++++++ 7 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/job-graph-7d9c6194dd60b06d.yaml diff --git a/doc/source/commands.rst b/doc/source/commands.rst index 1d32be2..648b65b 100644 --- a/doc/source/commands.rst +++ b/doc/source/commands.rst @@ -164,6 +164,27 @@ Note that zero values for ``oldrev`` and ``newrev`` can indicate branch creation and deletion; the source code of Zuul is the best reference for these more advanced operations. +Job-graph +^^^^^^^^^ + +Display the set of jobs that would be triggered in a project's +pipeline. This will show the complete set of jobs that Zuul will +consider running if an item for the given project and branch were +enqueued into the specified pipeline. Information about job +dependencies (soft and hard) is also included. The actual set of jobs +run for a given change or ref may be less than what is output by this +command if some jobs have non-matching file matchers. + +This command supports the ``dot`` output format. When used, the +output may be supplied to graphviz in order to render a graphical view +of the job graph. + +.. program-output:: zuul-client job-graph --help + +Example:: + + zuul-client job-graph --tenant mytenant --pipeline check --project org/project --branch master + zuul-client --format dot job-graph --tenant mytenant --pipeline check --project org/project --branch master | xdot Promote ^^^^^^^ diff --git a/releasenotes/notes/job-graph-7d9c6194dd60b06d.yaml b/releasenotes/notes/job-graph-7d9c6194dd60b06d.yaml new file mode 100644 index 0000000..a2af707 --- /dev/null +++ b/releasenotes/notes/job-graph-7d9c6194dd60b06d.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Support for freezing and displaying the job graph has been added + via the ``zuul-client job-graph`` subcommand. diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 6662998..583c37a 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -456,3 +456,35 @@ GuS6/ewjS+arA1Iyeg/IxmECAwEAAQ== client.session.get.assert_any_call( 'https://fake.zuul/api/tenant/tenant1/build/a1a1a1a1') self.assertEqual(fakejson, ahl) + + def test_freeze_jobs(self): + """Test freeze-jobs endpoint""" + client = ZuulRESTClient(url='https://fake.zuul/') + # test status checks + self._test_status_check( + client, 'get', client.freeze_jobs, + 'tenant1', 'check', 'project1', 'master') + + fakejson = [ + { + "dependencies": [], + "name": "zuul-build-image" + }, + { + "dependencies": [ + { + "name": "zuul-build-image", + "soft": False + } + ], + "name": "zuul-quick-start" + }, + ] + req = FakeRequestResponse(200, fakejson) + client.session.get = MagicMock(return_value=req) + client.info_ = {} + graph = client.freeze_jobs('tenant1', 'check', 'project1', 'master') + client.session.get.assert_any_call( + 'https://fake.zuul/api/tenant/tenant1/pipeline/check/' + 'project/project1/branch/master/freeze-jobs') + self.assertEqual(fakejson, graph) diff --git a/tests/unit/test_cmd.py b/tests/unit/test_cmd.py index f5b319a..81dd635 100644 --- a/tests/unit/test_cmd.py +++ b/tests/unit/test_cmd.py @@ -627,3 +627,25 @@ verify_ssl=True""" session.get.assert_any_call( 'https://fake.zuul/api/tenant/tenant1/build/a1a1a1a1') self.assertEqual(0, exit_code) + + def test_job_graph(self): + """Test job-graph subcommand""" + ZC = ZuulClient() + with patch('requests.Session') as mock_sesh: + session = mock_sesh.return_value + session.get = MagicMock( + side_effect=mock_get( + MagicMock(return_value=FakeRequestResponse(200, [])) + ) + ) + exit_code = ZC._main( + ['--zuul-url', 'https://fake.zuul', 'job-graph', + '--tenant', 'tenant1', + '--pipeline', 'check', + '--project', 'project1', + '--branch', 'master']) + session.get.assert_any_call( + 'https://fake.zuul/api/tenant/tenant1/pipeline/check/' + 'project/project1/branch/master/freeze-jobs', + ) + self.assertEqual(0, exit_code) diff --git a/zuulclient/api/__init__.py b/zuulclient/api/__init__.py index d48a5c8..0263c59 100644 --- a/zuulclient/api/__init__.py +++ b/zuulclient/api/__init__.py @@ -293,3 +293,15 @@ class ZuulRESTClient(object): except Exception as e: build_info['inventory'] = {'error': str(e)} return build_info + + def freeze_jobs(self, tenant, pipeline, project, branch): + suffix = (f'pipeline/{pipeline}/project/{project}/' + f'branch/{branch}/freeze-jobs') + if self.info.get("tenant"): + self._check_scope(tenant) + else: + suffix = f'tenant/{tenant}/{suffix}' + url = urllib.parse.urljoin(self.base_url, suffix) + req = self.session.get(url) + self._check_request_status(req) + return req.json() diff --git a/zuulclient/cmd/__init__.py b/zuulclient/cmd/__init__.py index 735365e..6defc59 100644 --- a/zuulclient/cmd/__init__.py +++ b/zuulclient/cmd/__init__.py @@ -86,7 +86,8 @@ class ZuulClient(): action='store_false', help='Do not verify SSL connection to Zuul ' '(Defaults to False)') - parser.add_argument('--format', choices=['JSON', 'text'], + parser.add_argument('--format', + choices=['JSON', 'json', 'text', 'dot'], default='text', required=False, help='The output format, when applicable') self.createCommandParsers(parser) @@ -107,6 +108,7 @@ class ZuulClient(): self.add_encrypt_subparser(subparsers) self.add_builds_list_subparser(subparsers) self.add_build_info_subparser(subparsers) + self.add_job_graph_subparser(subparsers) return subparsers @@ -143,10 +145,12 @@ class ZuulClient(): @property def formatter(self): - if self.args.format == 'JSON': + if self.args.format.lower() == 'json': return formatters.JSONFormatter elif self.args.format == 'text': return formatters.PrettyTableFormatter + elif self.args.format == 'dot': + return formatters.DotFormatter else: raise Exception('Unsupported formatter: %s' % self.args.format) @@ -787,6 +791,29 @@ class ZuulClient(): return True + def add_job_graph_subparser(self, subparsers): + cmd_job_graph = subparsers.add_parser( + 'job-graph', help='Freeze and display a job graph') + cmd_job_graph.add_argument( + '--tenant', help='tenant name', required=False, default='') + cmd_job_graph.add_argument('--pipeline', help='pipeline name', + required=True) + cmd_job_graph.add_argument('--project', help='project name', + required=True) + cmd_job_graph.add_argument('--branch', help='branch name', + required=True) + cmd_job_graph.set_defaults(func=self.job_graph) + self.cmd_job_graph = cmd_job_graph + + def job_graph(self): + client = self.get_client() + self._check_tenant_scope(client) + graph = client.freeze_jobs(self.tenant(), self.args.pipeline, + self.args.project, self.args.branch) + formatted_result = self.formatter('JobGraph')(graph) + print(formatted_result) + return True + def main(): ZuulClient().main() diff --git a/zuulclient/utils/formatters.py b/zuulclient/utils/formatters.py index ccc9104..2cb508d 100644 --- a/zuulclient/utils/formatters.py +++ b/zuulclient/utils/formatters.py @@ -63,6 +63,9 @@ class BaseFormatter: def formatBuildSets(self, data): raise NotImplementedError + def formatJobGraph(self, data): + raise NotImplementedError + class JSONFormatter(BaseFormatter): def __call__(self, data) -> str: @@ -263,3 +266,44 @@ class PrettyTableFormatter(BaseFormatter): def formatJobResource(self, data) -> str: return data.get('name', 'N/A') + + def formatJobGraph(self, data) -> str: + table = prettytable.PrettyTable( + field_names=['Job', 'Dependencies'] + ) + table.align = 'l' + for job in data: + deps = [] + for dep in job.get('dependencies', []): + d = dep['name'] + if dep['soft']: + d += ' (soft)' + deps.append(d) + table.add_row([ + job.get('name', 'N/A'), + ', '.join(deps), + ]) + return str(table) + + +class DotFormatter(BaseFormatter): + """Format for graphviz""" + + def formatJobGraph(self, data) -> str: + ret = 'digraph job_graph {\n' + ret += ' rankdir=LR;\n' + ret += ' node [shape=box];\n' + for job in data: + name = job['name'] + deps = job.get('dependencies', []) + if deps: + for dep in deps: + if dep['soft']: + soft = ' [style=dashed dir=back]' + else: + soft = ' [dir=back]' + ret += f""" "{dep['name']}" -> "{name}"{soft};\n""" + else: + ret += f' "{name}";\n' + ret += '}\n' + return ret