First iteration of callback using API

- Remove mockdata.py (using the callback is better)
- Add first iteration of callback
  Note: This will eventually be moved, it's here for simplicity
- Add test playbook/role to exercise the callback

Note: The callback will be moved to ara-plugins

Change-Id: I8f590be4cfafd4714f40f4165e2973cb803b8756

README.rst

@@ -24,19 +24,18 @@ This is python3 only right now.
 **TL;DR**: Using tox is convenient for the time being::
 
   # Use the source Luke
-  git clone https://github.com/dmsimard/ara-django
-  cd ara-django
+  git clone https://github.com/openstack/ara-server
+  cd ara-server
 
   # Install tox
   pip install tox # (or the tox python library from your distro packages)
 
+  # Create data from a test playbook and callback
+  tox -e ansible-playbook
+
   # Run test server -> http://127.0.0.1:8000/api/v1/
   tox -e runserver
 
-  # Create mock data
-  source .tox/runserver/bin/activate
-  python standalone/mockdata.py
-
   # Run actual tests or get coverage
   tox -e pep8
   tox -e py35
@@ -45,12 +44,14 @@ This is python3 only right now.
   # Build docs
   tox -e docs
 
+See the ``hacking`` directory for testing resources.
+
 Contributors
 ============
 
 See contributors on GitHub_.
 
-.. _GitHub: https://github.com/dmsimard/ara-django/graphs/contributors
+.. _GitHub: https://github.com/openstack/ara-server/graphs/contributors
 
 Copyright
 =========

hacking/callback_plugins/ara.py

@@ -0,0 +1,262 @@
+#  Copyright (c) 2018 Red Hat, Inc.
+#
+#  This file is part of ARA: Ansible Run Analysis.
+#
+#  ARA is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import (absolute_import, division, print_function)
+
+import datetime
+import json
+import logging
+import os
+import six
+
+from ansible import __version__ as ansible_version
+from ansible.plugins.callback import CallbackBase
+# To retrieve Ansible CLI options
+try:
+    from __main__ import cli
+except ImportError:
+    cli = None
+
+try:
+    # Default to offline API client
+    from django import setup as django_setup
+    from django.core.management import execute_from_command_line
+    from django.test import Client as offline_client
+
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ara.settings')
+
+    # Automatically create the database and run migrations (is there a better way?)
+    execute_from_command_line(['django', 'migrate'])
+
+    # Set up the things Django needs
+    django_setup()
+
+except ImportError:
+    # Default to online API client
+    # TODO
+    pass
+
+
+class AraOfflineClient(object):
+    def __init__(self):
+        self.log = logging.getLogger('ara.client.offline')
+        self.client = offline_client()
+
+    def _request(self, method, endpoint, **kwargs):
+        func = getattr(self.client, method)
+        response = func(
+            endpoint,
+            json.dumps(kwargs),
+            content_type='application/json'
+        )
+
+        self.log.debug('HTTP {status}: {method} on {endpoint}'.format(
+            status=response.status_code,
+            method=method,
+            endpoint=endpoint
+        ))
+
+        if response.status_code not in [200, 201]:
+            self.log.error(
+                'Failed to {method} on {endpoint}: {content}'.format(
+                    method=method,
+                    endpoint=endpoint,
+                    content=kwargs
+                )
+            )
+            self.log.fatal(response.content)
+
+        return response.json()
+
+    def get(self, endpoint, **kwargs):
+        return self._request('get', endpoint, **kwargs)
+
+    def patch(self, endpoint, **kwargs):
+        return self._request('patch', endpoint, **kwargs)
+
+    def post(self, endpoint, **kwargs):
+        return self._request('post', endpoint, **kwargs)
+
+    def put(self, endpoint, **kwargs):
+        return self._request('put', endpoint, **kwargs)
+
+    def delete(self, endpoint, **kwargs):
+        return self._request('delete', endpoint, **kwargs)
+
+
+class CallbackModule(CallbackBase):
+    """
+    Saves data from an Ansible run into a database
+    """
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'awesome'
+    CALLBACK_NAME = 'ara'
+
+    def __init__(self):
+        super(CallbackModule, self).__init__()
+        self.log = logging.getLogger('ara.callback')
+
+        # TODO: logic for picking between offline and online client
+        self.client = AraOfflineClient()
+
+        self.result = None
+        self.task = None
+        self.play = None
+        self.playbook = None
+        self.stats = None
+        self.loop_items = []
+
+        if cli:
+            self._options = cli.options
+        else:
+            self._options = None
+
+    def v2_playbook_on_start(self, playbook):
+        self.log.debug('v2_playbook_on_start')
+
+        path = os.path.abspath(playbook._file_name)
+        if self._options is not None:
+            parameters = self._options.__dict__.copy()
+        else:
+            parameters = {}
+
+        # Create the playbook
+        self.playbook = self.client.post(
+            '/api/v1/playbooks/',
+            ansible_version=ansible_version,
+            parameters=parameters,
+            file=dict(
+                path=path,
+                content=self._read_file(path)
+            )
+        )
+
+        # Record all the files involved in the playbook
+        self._load_files(playbook._loader._FILE_CACHE.keys())
+
+        return self.playbook
+
+    def v2_playbook_on_play_start(self, play):
+        self.log.debug('v2_playbook_on_play_start')
+        self._end_task()
+        self._end_play()
+
+        # Record all the files involved in the play
+        self._load_files(play._loader._FILE_CACHE.keys())
+
+        # Create the play
+        self.play = self.client.post(
+            '/api/v1/plays/',
+            name=play.name,
+            playbook=self.playbook['id']
+        )
+
+        return self.play
+
+    def v2_playbook_on_task_start(self, task, is_conditional, handler=False):
+        self.log.debug('v2_playbook_on_task_start')
+        self._end_task()
+
+        pathspec = task.get_path()
+        if pathspec:
+            path, lineno = pathspec.split(':', 1)
+            lineno = int(lineno)
+        else:
+            # Task doesn't have a path, default to "something"
+            path = self.playbook['path']
+            lineno = 1
+
+        # Ensure this task's file was added to the playbook -- files that are
+        # dynamically included do not show up in the playbook or play context
+        self._load_files([path])
+
+        # Find the task file (is there a better way?)
+        task_file = self.playbook['file']['id']
+        for file in self.playbook['files']:
+            if file['path'] == path:
+                task_file = file['id']
+                break
+
+        self.task = self.client.post(
+            '/api/v1/tasks/',
+            name=task.get_name(),
+            action=task.action,
+            play=self.play['id'],
+            playbook=self.playbook['id'],
+            file=task_file,
+            tags=task._attributes['tags'],
+            lineno=lineno,
+            handler=handler
+        )
+
+        return self.task
+
+    def v2_playbook_on_stats(self, stats):
+        self.log.debug('v2_playbook_on_stats')
+
+        self._end_task()
+        self._end_play()
+        self._end_playbook()
+
+    def _end_task(self):
+        if self.task is not None:
+            self.client.patch(
+                '/api/v1/tasks/%s/' % self.task['id'],
+                completed=True,
+                ended=datetime.datetime.now().isoformat()
+            )
+            self.task = None
+            self.loop_items = []
+
+    def _end_play(self):
+        if self.play is not None:
+            self.client.patch(
+                '/api/v1/plays/%s/' % self.play['id'],
+                completed=True,
+                ended=datetime.datetime.now().isoformat()
+            )
+            self.play = None
+
+    def _end_playbook(self):
+        self.playbook = self.client.patch(
+            '/api/v1/playbooks/%s/' % self.playbook['id'],
+            completed=True,
+            ended=datetime.datetime.now().isoformat()
+        )
+
+    def _load_files(self, files):
+        self.log.debug('Loading %s file(s)...' % len(files))
+        playbook_files = [file['path'] for file in self.playbook['files']]
+        for file in files:
+            if file not in playbook_files:
+                self.client.post(
+                    '/api/v1/playbooks/%s/files/' % self.playbook['id'],
+                    path=file,
+                    content=self._read_file(file)
+                )
+
+    def _read_file(self, path):
+        try:
+            with open(path, 'r') as fd:
+                content = fd.read()
+        except IOError as e:
+            self.log.error("Unable to open {0} for reading: {1}".format(
+                path, six.text_type(e)
+            ))
+            content = """ARA was not able to read this file successfully.
+                      Refer to the logs for more information"""
+        return content

hacking/roles/hacking/defaults/main.yml

@@ -0,0 +1,19 @@
+---
+#  Copyright (c) 2018 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
+
+filename: /tmp/ara-hacking

hacking/roles/hacking/handlers/main.yml

@@ -0,0 +1,22 @@
+---
+#  Copyright (c) 2018 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
+
+- name: Create a file
+  template:
+    src: hacking.j2
+    dest: "{{ filename }}"

hacking/roles/hacking/meta/main.yml

@@ -0,0 +1,42 @@
+---
+#  Copyright (c) 2018 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
+
+galaxy_info:
+  author: OpenStack community
+  description: Hacking
+  company: Red Hat
+  license: GPL v3
+  min_ansible_version: 2.5
+  platforms:
+   - name: EL
+     versions:
+      - all
+   - name: Fedora
+     versions:
+      - all
+   - name: Ubuntu
+     versions:
+      - all
+   - name: Debian
+     versions:
+      - all
+  galaxy_tags:
+    - installer
+    - application
+    - system
+dependencies: []

hacking/roles/hacking/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+#  Copyright (c) 2018 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
+
+- name: echo something
+  command: echo something
+  register: echo
+
+- name: debug something
+  debug:
+    var: echo
+
+- name: Ensure a file is absent
+  file:
+    path: "{{ filename }}"
+    state: absent
+  notify:
+    - create a file

hacking/roles/hacking/templates/hacking.j2

@@ -0,0 +1 @@
+Hello world

hacking/test-playbook.yml

@@ -0,0 +1,22 @@
+- name: A test play
+  hosts: localhost
+  tasks:
+    - name: Include a role
+      include_role:
+        name: hacking
+
+- name: Another test play
+  hosts: localhost
+  gather_facts: no
+  pre_tasks:
+    - name: Pre task
+      debug:
+        msg: Task from a pre-task
+  tasks:
+    - name: Task
+      debug:
+        msg: Task from a task
+  post_tasks:
+    - name: Post Task
+      debug:
+        msg: Task from a post-task

standalone/mockdata.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-#  Copyright (c) 2018 Red Hat, Inc.
-#
-#  This file is part of ARA Records Ansible.
-#
-#  ARA is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  ARA is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
-
-# Creates mock data offline leveraging the API
-import django
-import json
-import os
-import sys
-
-parent_directory = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-sys.path.append(parent_directory)
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ara.settings')
-django.setup()
-
-from django.test import Client
-
-
-def post(endpoint, data):
-    client = Client()
-    print("Posting to %s..." % endpoint)
-    obj = client.post(endpoint,
-                      json.dumps(data),
-                      content_type="application/json")
-    print("HTTP %s" % obj.status_code)
-    print("Got: %s" % json.dumps(obj.json(), indent=2))
-    print("#" * 40)
-    return obj.json()
-
-
-playbook = post(
-    '/api/v1/playbooks/',
-    {
-        'started': '2016-05-06T17:20:25.749489-04:00',
-        'ansible_version': '2.3.4',
-        'completed': False,
-        'parameters': {'foo': 'bar'}
-    }
-)
-
-playbook_with_files = post(
-    '/api/v1/playbooks/',
-    {
-        'started': '2016-05-06T17:20:25.749489-04:00',
-        'ansible_version': '2.3.4',
-        'completed': False,
-        'parameters': {'foo': 'bar'},
-        'files': [
-            {'path': '/tmp/1/playbook.yml', 'content': '# playbook'},
-            {'path': '/tmp/2/playbook.yml', 'content': '# playbook'}
-        ],
-    }
-)
-
-play = post(
-    '/api/v1/plays/',
-    {
-        'started': '2016-05-06T17:20:25.749489-04:00',
-        'name': 'Test play',
-        'playbook': playbook['id']
-    }
-)

tox.ini

@@ -33,6 +33,14 @@ commands =
 setenv =
   DJANGO_DEBUG=1
 
+[testenv:ansible-playbook]
+deps = ansible
+commands =
+  ansible-playbook {toxinidir}/hacking/test-playbook.yml
+setenv =
+  DJANGO_DEBUG=1
+  DJANGO_LOG_LEVEL=DEBUG
+
 [testenv:cover]
 commands =
     coverage erase