Add support for freezing the job graph

This adds a "job-graph" subcommand which uses the freeze-jobs api
to return information about what jobs may be run.

This also adds a new formatter, "dot".  The only command supporting
this formatter is job-graph.  To see it in action, try:

  zuul-client --format dot --zuul-url https://zuul.opendev.org job-graph --tenant openstack --pipeline check --project opendev/system-config --branch master | xdot -

Finally, this also adds "json" as an acceptable alias to "JSON" when
specifying the output format since that is a widely used convention.

Change-Id: I9adc3ab87bfa11432ae621b65dd94189bb17e42c

doc/source/commands.rst

@@ -164,6 +164,27 @@ Note that zero values for ``oldrev`` and ``newrev`` can indicate
 branch creation and deletion; the source code of Zuul is the best reference
 for these more advanced operations.
 
+Job-graph
+^^^^^^^^^
+
+Display the set of jobs that would be triggered in a project's
+pipeline.  This will show the complete set of jobs that Zuul will
+consider running if an item for the given project and branch were
+enqueued into the specified pipeline.  Information about job
+dependencies (soft and hard) is also included.  The actual set of jobs
+run for a given change or ref may be less than what is output by this
+command if some jobs have non-matching file matchers.
+
+This command supports the ``dot`` output format.  When used, the
+output may be supplied to graphviz in order to render a graphical view
+of the job graph.
+
+.. program-output:: zuul-client job-graph --help
+
+Example::
+
+  zuul-client job-graph --tenant mytenant --pipeline check --project org/project --branch master
+  zuul-client --format dot job-graph --tenant mytenant --pipeline check --project org/project --branch master | xdot
 
 Promote
 ^^^^^^^

releasenotes/notes/job-graph-7d9c6194dd60b06d.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Support for freezing and displaying the job graph has been added
+    via the ``zuul-client job-graph`` subcommand.

tests/unit/test_api.py

@@ -456,3 +456,35 @@ GuS6/ewjS+arA1Iyeg/IxmECAwEAAQ==
         client.session.get.assert_any_call(
             'https://fake.zuul/api/tenant/tenant1/build/a1a1a1a1')
         self.assertEqual(fakejson, ahl)
+
+    def test_freeze_jobs(self):
+        """Test freeze-jobs endpoint"""
+        client = ZuulRESTClient(url='https://fake.zuul/')
+        # test status checks
+        self._test_status_check(
+            client, 'get', client.freeze_jobs,
+            'tenant1', 'check', 'project1', 'master')
+
+        fakejson = [
+            {
+                "dependencies": [],
+                "name": "zuul-build-image"
+            },
+            {
+                "dependencies": [
+                    {
+                        "name": "zuul-build-image",
+                        "soft": False
+                    }
+                ],
+                "name": "zuul-quick-start"
+            },
+        ]
+        req = FakeRequestResponse(200, fakejson)
+        client.session.get = MagicMock(return_value=req)
+        client.info_ = {}
+        graph = client.freeze_jobs('tenant1', 'check', 'project1', 'master')
+        client.session.get.assert_any_call(
+            'https://fake.zuul/api/tenant/tenant1/pipeline/check/'
+            'project/project1/branch/master/freeze-jobs')
+        self.assertEqual(fakejson, graph)

tests/unit/test_cmd.py

@@ -627,3 +627,25 @@ verify_ssl=True"""
             session.get.assert_any_call(
                 'https://fake.zuul/api/tenant/tenant1/build/a1a1a1a1')
             self.assertEqual(0, exit_code)
+
+    def test_job_graph(self):
+        """Test job-graph subcommand"""
+        ZC = ZuulClient()
+        with patch('requests.Session') as mock_sesh:
+            session = mock_sesh.return_value
+            session.get = MagicMock(
+                side_effect=mock_get(
+                    MagicMock(return_value=FakeRequestResponse(200, []))
+                )
+            )
+            exit_code = ZC._main(
+                ['--zuul-url', 'https://fake.zuul', 'job-graph',
+                 '--tenant', 'tenant1',
+                 '--pipeline', 'check',
+                 '--project', 'project1',
+                 '--branch', 'master'])
+            session.get.assert_any_call(
+                'https://fake.zuul/api/tenant/tenant1/pipeline/check/'
+                'project/project1/branch/master/freeze-jobs',
+            )
+            self.assertEqual(0, exit_code)

zuulclient/api/__init__.py

@@ -293,3 +293,15 @@ class ZuulRESTClient(object):
         except Exception as e:
             build_info['inventory'] = {'error': str(e)}
         return build_info
+
+    def freeze_jobs(self, tenant, pipeline, project, branch):
+        suffix = (f'pipeline/{pipeline}/project/{project}/'
+                  f'branch/{branch}/freeze-jobs')
+        if self.info.get("tenant"):
+            self._check_scope(tenant)
+        else:
+            suffix = f'tenant/{tenant}/{suffix}'
+        url = urllib.parse.urljoin(self.base_url, suffix)
+        req = self.session.get(url)
+        self._check_request_status(req)
+        return req.json()

zuulclient/cmd/__init__.py

@@ -86,7 +86,8 @@ class ZuulClient():
                             action='store_false',
                             help='Do not verify SSL connection to Zuul '
                                  '(Defaults to False)')
-        parser.add_argument('--format', choices=['JSON', 'text'],
+        parser.add_argument('--format',
+                            choices=['JSON', 'json', 'text', 'dot'],
                             default='text', required=False,
                             help='The output format, when applicable')
         self.createCommandParsers(parser)
@@ -107,6 +108,7 @@ class ZuulClient():
         self.add_encrypt_subparser(subparsers)
         self.add_builds_list_subparser(subparsers)
         self.add_build_info_subparser(subparsers)
+        self.add_job_graph_subparser(subparsers)
 
         return subparsers
 
@@ -143,10 +145,12 @@ class ZuulClient():
 
     @property
     def formatter(self):
-        if self.args.format == 'JSON':
+        if self.args.format.lower() == 'json':
             return formatters.JSONFormatter
         elif self.args.format == 'text':
             return formatters.PrettyTableFormatter
+        elif self.args.format == 'dot':
+            return formatters.DotFormatter
         else:
             raise Exception('Unsupported formatter: %s' % self.args.format)
 
@@ -787,6 +791,29 @@ class ZuulClient():
 
         return True
 
+    def add_job_graph_subparser(self, subparsers):
+        cmd_job_graph = subparsers.add_parser(
+            'job-graph', help='Freeze and display a job graph')
+        cmd_job_graph.add_argument(
+            '--tenant', help='tenant name', required=False, default='')
+        cmd_job_graph.add_argument('--pipeline', help='pipeline name',
+                                   required=True)
+        cmd_job_graph.add_argument('--project', help='project name',
+                                   required=True)
+        cmd_job_graph.add_argument('--branch', help='branch name',
+                                   required=True)
+        cmd_job_graph.set_defaults(func=self.job_graph)
+        self.cmd_job_graph = cmd_job_graph
+
+    def job_graph(self):
+        client = self.get_client()
+        self._check_tenant_scope(client)
+        graph = client.freeze_jobs(self.tenant(), self.args.pipeline,
+                                   self.args.project, self.args.branch)
+        formatted_result = self.formatter('JobGraph')(graph)
+        print(formatted_result)
+        return True
+
 
 def main():
     ZuulClient().main()

zuulclient/utils/formatters.py

@@ -63,6 +63,9 @@ class BaseFormatter:
     def formatBuildSets(self, data):
         raise NotImplementedError
 
+    def formatJobGraph(self, data):
+        raise NotImplementedError
+
 
 class JSONFormatter(BaseFormatter):
     def __call__(self, data) -> str:
@@ -263,3 +266,44 @@ class PrettyTableFormatter(BaseFormatter):
 
     def formatJobResource(self, data) -> str:
         return data.get('name', 'N/A')
+
+    def formatJobGraph(self, data) -> str:
+        table = prettytable.PrettyTable(
+            field_names=['Job', 'Dependencies']
+        )
+        table.align = 'l'
+        for job in data:
+            deps = []
+            for dep in job.get('dependencies', []):
+                d = dep['name']
+                if dep['soft']:
+                    d += ' (soft)'
+                deps.append(d)
+            table.add_row([
+                job.get('name', 'N/A'),
+                ', '.join(deps),
+            ])
+        return str(table)
+
+
+class DotFormatter(BaseFormatter):
+    """Format for graphviz"""
+
+    def formatJobGraph(self, data) -> str:
+        ret = 'digraph job_graph {\n'
+        ret += '  rankdir=LR;\n'
+        ret += '  node [shape=box];\n'
+        for job in data:
+            name = job['name']
+            deps = job.get('dependencies', [])
+            if deps:
+                for dep in deps:
+                    if dep['soft']:
+                        soft = ' [style=dashed dir=back]'
+                    else:
+                        soft = ' [dir=back]'
+                    ret += f"""  "{dep['name']}" -> "{name}"{soft};\n"""
+            else:
+                ret += f'  "{name}";\n'
+        ret += '}\n'
+        return ret