diff --git a/.gitreview b/.gitreview index d9010ee..d4317c4 100644 --- a/.gitreview +++ b/.gitreview @@ -1,4 +1,4 @@ [gerrit] host=review.openstack.org port=29418 -project=openstack-infra/puppet-log_processor.git +project=openstack-infra/log_processor.git diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000..f428e7d --- /dev/null +++ b/.testr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} OS_TEST_TIMEOUT=60 ${PYTHON:-python} -m subunit.run discover -t ./ log_processor/tests/ $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..e69de29 diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..0d2cd5b --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = -W +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Logprocessor.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Logprocessor.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Logprocessor" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Logprocessor" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..d33a8a3 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,235 @@ +# -*- coding: utf-8 -*- +# +# Log processor documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 8 14:44:26 2012. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os, datetime + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../..')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ 'sphinxcontrib.programoutput' ] +#extensions = ['sphinx.ext.intersphinx'] +#intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'log_processor' +copyright = u'%s, OpenStack' % datetime.date.today().year + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Logprocessordoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'log_processor.tex', u'Log Processor Documentation', + u'OpenStack', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'log_processor', u'Log Processor Documentation', + [u'OpenStack'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'log_processor', u'Log Processor Documentation', + u'OpenStack', 'log_processor', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..f5975d1 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,24 @@ +Log Processor +============= + +Log Processor is a system for subscribing to a Jenkins ZMQ publisher +and acts as a Gearman server. When a Jenkins build finishes, Log Processor +Client will create Gearman jobs to fetch the build logs from a web server +and the Log Processor Workers will send those logs to a Logstash instance +running the tcp input plugin. + +Contents: + +.. toctree:: + :maxdepth: 2 + + installation + operation + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/installation.rst b/doc/source/installation.rst new file mode 100644 index 0000000..9d92fe2 --- /dev/null +++ b/doc/source/installation.rst @@ -0,0 +1,35 @@ +:title: Installation + +Installation +============ + +Log_processor consists of a client daemon and multiple worker daemons. The +client daemon subscribes to ZeroMQ on a Jenkins master listening for build +completes and then dispatches jobs to the worker daemons using gearman. + +External Requirements +--------------------- + +Jenkins +~~~~~~~ + +You should have a Jenkins server running with the `ZMQ Event Publisher +`_ +plugin installed (it is available in the Jenkins Update Center). Be +sure that the machine where you plan to run Log Processor can connect to +the ZMQ port specified by the plugin on your Jenkins master(s). + +Logstash +~~~~~~~~ + +You should have Logstash running with the tcp input plugin. Be sure that +the machine where you plan to run Log Processor can connect to the tcp input +port specified in the logstash configuration. + +Install Log Processor +--------------------- + +To install directly from git:: + + git clone git://git.openstack.org/openstack-infra/log_processor.git /path/to/log_processor + pip install /path/to/log_processor diff --git a/doc/source/operation.rst b/doc/source/operation.rst new file mode 100644 index 0000000..df47f35 --- /dev/null +++ b/doc/source/operation.rst @@ -0,0 +1,17 @@ +.. _operation: + +Command Line Tools +================== + +Usage +----- + +log-processor-client +^^^^^^^^^^^^^^^^^^^^ +.. program-output:: log-processor-client --help + :nostderr: + +log-processor-worker +^^^^^^^^^^^^^^^^^^^^ +.. program-output:: log-processor-worker --help + :nostderr: diff --git a/log_processor/__init__.py b/log_processor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/log_processor/client.py b/log_processor/client.py index 4957fe7..ad946a9 100644 --- a/log_processor/client.py +++ b/log_processor/client.py @@ -63,8 +63,11 @@ class EventProcessor(threading.Thread): self.socket.setsockopt(zmq.SUBSCRIBE, event_filter) self.socket.connect(self.zmq_address) + def _get_event(self): + return self.socket.recv().decode('utf-8') + def _read_event(self): - string = self.socket.recv().decode('utf-8') + string = self._get_event() event = json.loads(string.split(None, 1)[1]) logging.debug("Jenkins event received: " + json.dumps(event)) for fileopts in self.files: diff --git a/log_processor/tests/__init__.py b/log_processor/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/log_processor/tests/fixtures/client-config.yaml b/log_processor/tests/fixtures/client-config.yaml new file mode 100644 index 0000000..16a7f47 --- /dev/null +++ b/log_processor/tests/fixtures/client-config.yaml @@ -0,0 +1,9 @@ +source-url: http://fake + +zmq-publishers: + - tcp://fake + +source-files: + - name: console.html + tags: + - console diff --git a/log_processor/tests/test_client.py b/log_processor/tests/test_client.py new file mode 100644 index 0000000..13d0681 --- /dev/null +++ b/log_processor/tests/test_client.py @@ -0,0 +1,64 @@ +#!/usr/bin/python2 +# +# Copyright 2015 Hewlett-Packard Enterprise Development, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import os +import Queue +import testtools +import yaml + +from log_processor import client, tests + +class FakeEventProcessor(client.EventProcessor): + def _connect_zmq(self): + self.events = Queue.Queue() + self.events.put(u'onFinalized {"name": "fake-job", "build": {}}') + self.events.put(u'onFinalized {"name": "fake-job2", "build": {}}') + + def _get_event(self): + return self.events.get(block=False) + +class ClientTestCase(testtools.TestCase): + def setUp(self): + super(ClientTestCase, self).setUp() + + config_file = os.path.join(os.path.dirname(tests.__file__), + 'fixtures', 'client-config.yaml') + with open(config_file, 'r') as config_stream: + self.config = yaml.load(config_stream) + + zmq_publishers = self.config.get('zmq-publishers', []) + gearman_client = mock.Mock() + files = self.config.get('source-files', []) + source_url = self.config.get('source-url', None) + + for zmq_publisher in zmq_publishers: + self.log_processor = FakeEventProcessor(zmq_publisher, + gearman_client, files, source_url) + + def test_client_get(self): + event0 = self.log_processor._get_event() + self.assertEqual(event0, + u'onFinalized {"name": "fake-job", "build": {}}') + + event1 = self.log_processor._get_event() + self.assertEqual(event1, + u'onFinalized {"name": "fake-job2", "build": {}}') + + def test_client_read(self): + self.log_processor._read_event() + job_count = self.log_processor.gearman_client.submitJob.call_count + self.assertEqual(job_count, 1) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3e205c1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +PyYAML +gear +python-daemon +pyzmq +statsd diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..364d213 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,30 @@ +[metadata] +name = log_processor +summary = Gearman server and workers for fetching logs over HTTP and injecting them into Logstash. +description-file = + README.rst +author = OpenStack Infrastructure Team +author-email = openstack-infra@lists.openstack.org +home-page = http://docs.openstack.org/infra/system-config/ +classifier = + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 2.6 + +[pbr] +warnerrors = True + +[entry_points] +console_scripts = + log-processor-client = log_processor.client:main + log-processor-worker = log_processor.worker:main + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4e7501e --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +setuptools.setup( + setup_requires=['pbr>=1.3'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..665099c --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,7 @@ +# log_processor uses hacking for its dependencies +hacking>=0.10.2,<0.11 +coverage +sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 +sphinxcontrib-programoutput +testrepository>=0.0.13 +mock diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..e91b8b3 --- /dev/null +++ b/tox.ini @@ -0,0 +1,33 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = pep8, py27 + +[testenv] +setenv = VIRTUAL_ENV={envdir} +usedevelop = True +install_command = pip install {opts} {packages} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + python setup.py testr --slowest --testr-args='{posargs}' + +[testenv:pep8] +commands = flake8 log_processor + +[testenv:cover] +commands = + python setup.py testr --coverage + +[testenv:docs] +commands = + python setup.py build_sphinx + +[testenv:venv] +commands = {posargs} + +[flake8] +ignore = E123,E125,H +select = H231,F +show-source = True +exclude = .venv,.tox,dist,doc,build,*.egg