Add lodgeit to puppet

Will automatically install paste.drizzle.org and paste.openstack.org onto a server

Change-Id: Ia2c1e37892f3ae8e3d4034e38ddfaa01c6a92a54

manifests/site.pp

@@ -194,6 +194,20 @@ node "docs.openstack.org" {
   include doc_server
 }
 
+node "paste.openstack.org" {
+  include openstack_server
+  include lodgeit
+  lodgeit::site { "openstack":
+    port => "5000",
+    image => "header-bg2.png"
+  }
+
+  lodgeit::site { "drizzle":
+    port => "5001"
+  }
+
+}
+
 node "devstack-oneiric.template.openstack.org" {
   include openstack_template
   include devstack_host

modules/lodgeit/files/database.py

@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+"""
+    lodgeit.database
+    ~~~~~~~~~~~~~~~~
+
+    Database fun :)
+
+    :copyright: 2007-2008 by Armin Ronacher, Christopher Grebs.
+    :license: BSD
+"""
+import time
+import difflib
+from datetime import datetime
+from werkzeug import cached_property
+from sqlalchemy import MetaData, Integer, Text, DateTime, ForeignKey, \
+     String, Boolean, Table, Column, select, and_, func
+from sqlalchemy.orm import scoped_session, create_session, backref, relation
+from sqlalchemy.orm.scoping import ScopedSession
+from lodgeit import local
+from lodgeit.utils import generate_paste_hash
+from lodgeit.lib.highlighting import highlight, preview_highlight, LANGUAGES
+
+from sqlalchemy.orm import mapper as sqla_mapper
+
+def session_mapper(scoped_session):
+    def mapper(cls, *arg, **kw):
+        cls.query = scoped_session.query_property()
+        return sqla_mapper(cls, *arg, **kw)
+    return mapper
+
+session = scoped_session(lambda: create_session(local.application.engine),
+    scopefunc=local._local_manager.get_ident)
+
+metadata = MetaData()
+
+pastes = Table('pastes', metadata,
+    Column('paste_id', Integer, primary_key=True),
+    Column('code', Text),
+    Column('parent_id', Integer, ForeignKey('pastes.paste_id'),
+           nullable=True),
+    Column('pub_date', DateTime),
+    Column('language', String(30)),
+    Column('user_hash', String(40), nullable=True),
+    Column('handled', Boolean, nullable=False),
+    Column('private_id', String(40), unique=True, nullable=True)
+)
+
+
+class Paste(object):
+    """Represents a paste."""
+
+    def __init__(self, code, language, parent=None, user_hash=None,
+                 private=False):
+        if language not in LANGUAGES:
+            language = 'text'
+        self.code = u'\n'.join(code.splitlines())
+        self.language = language
+        if isinstance(parent, Paste):
+            self.parent = parent
+        elif parent is not None:
+            self.parent_id = parent
+        self.pub_date = datetime.now()
+        self.handled = False
+        self.user_hash = user_hash
+        self.private = private
+
+    @staticmethod
+    def get(identifier):
+        """Return the paste for an identifier.  Private pastes must be loaded
+        with their unique hash and public with the paste id.
+        """
+        if isinstance(identifier, basestring) and not identifier.isdigit():
+            return Paste.query.filter(Paste.private_id == identifier).first()
+        return Paste.query.filter(
+            (Paste.paste_id == int(identifier)) &
+            (Paste.private_id == None)
+        ).first()
+
+    @staticmethod
+    def find_all():
+        """Return a query for all public pastes ordered by the id in reverse
+        order.
+        """
+        return Paste.query.filter(Paste.private_id == None) \
+                          .order_by(Paste.paste_id.desc())
+
+    @staticmethod
+    def count():
+        """Count all pastes."""
+        s = select([func.count(pastes.c.paste_id)])
+        return session.execute(s).fetchone()[0]
+
+    @staticmethod
+    def resolve_root(identifier):
+        """Find the root paste for a paste tree."""
+        paste = Paste.get(identifier)
+        if paste is None:
+            return
+        while paste.parent_id is not None:
+            paste = paste.parent
+        return paste
+
+    @staticmethod
+    def fetch_replies():
+        """Get the new replies for the ower of a request and flag them
+        as handled.
+        """
+        s = select([pastes.c.paste_id],
+            Paste.user_hash == local.request.user_hash
+        )
+
+        paste_list = Paste.query.filter(and_(
+            Paste.parent_id.in_(s),
+            Paste.handled == False,
+            Paste.user_hash != local.request.user_hash,
+        )).order_by(pastes.c.paste_id.desc()).all()
+
+        to_mark = [p.paste_id for p in paste_list]
+        session.execute(pastes.update(pastes.c.paste_id.in_(to_mark),
+                                      values={'handled': True}))
+        return paste_list
+
+    def _get_private(self):
+        return self.private_id is not None
+
+    def _set_private(self, value):
+        if not value:
+            self.private_id = None
+            return
+        if self.private_id is None:
+            while 1:
+                self.private_id = generate_paste_hash()
+                paste = Paste.query.filter(Paste.private_id ==
+                                           self.private_id).first()
+                if paste is None:
+                    break
+    private = property(_get_private, _set_private, doc='''
+        The private status of the paste.  If the paste is private it gets
+        a unique hash as identifier, otherwise an integer.
+    ''')
+    del _get_private, _set_private
+
+    @property
+    def identifier(self):
+        """The paste identifier.  This is a string, the same the `get`
+        method accepts.
+        """
+        if self.private:
+            return self.private_id
+        return str(self.paste_id)
+
+    @property
+    def url(self):
+        """The URL to the paste."""
+        return '/show/%s/' % self.identifier
+
+    def compare_to(self, other, context_lines=4, template=False):
+        """Compare the paste with another paste."""
+        udiff = u'\n'.join(difflib.unified_diff(
+            self.code.splitlines(),
+            other.code.splitlines(),
+            fromfile='Paste #%s' % self.identifier,
+            tofile='Paste #%s' % other.identifier,
+            lineterm='',
+            n=context_lines
+        ))
+        if template:
+            from lodgeit.lib.diff import prepare_udiff
+            diff, info = prepare_udiff(udiff)
+            return diff and diff[0] or None
+        return udiff
+
+    @cached_property
+    def parsed_code(self):
+        """The paste as rendered code."""
+        return highlight(self.code, self.language)
+
+    def to_xmlrpc_dict(self):
+        """Convert the paste into a dict for XMLRCP."""
+        return {
+            'paste_id':         self.paste_id,
+            'code':             self.code,
+            'parsed_code':      self.parsed_code,
+            'pub_date':         int(time.mktime(self.pub_date.timetuple())),
+            'language':         self.language,
+            'parent_id':        self.parent_id,
+            'url':              self.url
+        }
+
+    def render_preview(self, num=5):
+        """Render a preview for this paste."""
+        return preview_highlight(self.code, self.language, num)
+
+mapper= session_mapper(session)
+
+mapper(Paste, pastes, properties={
+    'children': relation(Paste,
+        primaryjoin=pastes.c.parent_id==pastes.c.paste_id,
+        cascade='all',
+        backref=backref('parent', remote_side=[pastes.c.paste_id])
+    )
+})

modules/lodgeit/manifests/init.pp

@@ -0,0 +1,52 @@
+class lodgeit {
+  $packages = [ "nginx",
+                "python-imaging",
+                "python-pip",
+                "python-jinja2",
+                "python-pybabel",
+                "python-werkzeug",
+                "python-simplejson",
+                "python-pygments",
+                "mercurial",
+                "drizzle",
+                "python-mysqldb" ]
+
+  package { $packages: ensure => latest }
+
+  package { 'SQLAlchemy':
+    provider => pip,
+    ensure => present,
+    require => Package[python-pip]
+  }
+
+  file { '/srv/lodgeit':
+    ensure => directory
+  }
+
+  service { 'drizzle':
+    ensure => running,
+    hasrestart => true
+  }
+
+
+# if we already have the mercurial repo the pull updates
+
+  exec { "update_lodgeit":
+    command => "hg pull /tmp/lodgeit-main",
+    path => "/bin:/usr/bin",
+    onlyif => "test -d /tmp/lodgeit-main"
+  }
+
+# otherwise get a new clone of it
+
+  exec { "get_lodgeit":
+    command => "hg clone http://dev.pocoo.org/hg/lodgeit-main /tmp/lodgeit-main",
+    path => "/bin:/usr/bin",
+    onlyif => "test ! -d /tmp/lodgeit-main"
+  }
+
+  service { 'nginx':
+    ensure => running,
+    hasrestart => true
+  }
+}

modules/lodgeit/manifests/site.pp

@@ -0,0 +1,66 @@
+define lodgeit::site($port, $image="") {
+
+  file { "/etc/nginx/sites-available/${name}":
+    ensure => 'present',
+    content => template("lodgeit/nginx.erb"),
+    replace => 'true',
+    require => Package[nginx]
+  }
+
+  file { "/etc/nginx/sites-enabled/${name}":
+    ensure => link,
+    target => "/etc/nginx/sites-available/${name}",
+    require => Package[nginx]
+  }
+
+  file { "/etc/init/${name}-paste.conf":
+    ensure => 'present',
+    content => template("lodgeit/upstart.erb"),
+    replace => 'true',
+    require => Package[nginx]
+  }
+
+  file { "/srv/lodgeit/${name}":
+    ensure => directory,
+    recurse => true,
+    source => "/tmp/lodgeit-main"
+  }
+
+  if $image != '' {
+    file { "/srv/lodgeit/${name}/lodgeit/static/${image}":
+      ensure => present,
+      source => "puppet:///lodgeit/${image}"
+    }
+  }
+
+# Database file needs replacing to be compatible with SQLAlchemy 0.7
+
+  file { "/srv/lodgeit/${name}/lodgeit/database.py":
+    replace => true,
+    source => 'puppet:///modules/lodgeit/database.py'
+  }
+
+  file { "/srv/lodgeit/${name}/manage.py":
+    mode => 755,
+    replace => true,
+    content => template("lodgeit/manage.py.erb")
+  }
+
+  file { "/srv/lodgeit/${name}/lodgeit/views/layout.html":
+    replace => true,
+    content => template("lodgeit/layout.html.erb")
+  }
+
+  exec { "create_database_${name}":
+    command => "drizzle --user=root -e \"create database if not exists ${name};\"",
+    path => "/bin:/usr/bin",
+    require => Service["drizzle"]
+  }
+
+  service { "${name}-paste":
+    provider => upstart,
+    ensure => running,
+    require => [Service["drizzle", "nginx"], Exec["create_database_${name}"]]
+  }
+
+}

modules/lodgeit/templates/layout.html.erb

@@ -0,0 +1,91 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+  "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+  <head>
+    <title>{{ page_title|e }} | LodgeIt!</title>
+    <link rel="stylesheet" href="/static/style.css" type="text/css">
+    <link rel="stylesheet" href="/static/print.css" type="text/css" media="print">
+    <script type="text/javascript" src="/static/jquery.js"></script>
+    <script type="text/javascript" src="/static/cookie.js"></script>
+    <script type="text/javascript" src="/static/lodgeit.js"></script>
+    {%- if css %}
+    <style type="text/css">
+      {{ css|e }}
+    </style>
+    {%- endif %}
+  </head>
+  <body>
+    <div class="page">
+      <div id="header">
+        <h1>
+          <% if has_variable?("image") then %>
+          <img src="/static/<%= image %>" style="padding-bottom: 10px; margin-left: 10px;" alt="<%= name.capitalize %> Pastebin" />
+          <% else %>
+          <%= name.capitalize %> Pastebin
+          <% end %>
+        </h1></div>
+      <ul id="navigation">
+      {%- for href, id, caption in [
+        ('/', 'new', _('New')),
+        ('/all/', 'all', _('All')),
+        ('/about/', 'about', _('About')),
+        ('/help/', 'help', '?')
+      ] %}
+        <li{% if active_page == id %} class="active"{% endif %}>
+          <a href="{{ href|e }}">{{ caption|e }}</a>
+        </li>
+      {%- endfor %}
+      </ul>
+      {# <ul id="languages">
+      {% for lang, name in i18n_languages %}
+        <li{% if request.locale.language == lang %} class="active"{% endif %}>
+          <a href="{{ '/language/%s'|format(lang) }}"><img alt="{{ lang }}" src="{{ '/static/flags/%s.png'|format(lang) }}"></a>
+        </li>
+      {% endfor %}
+      </ul> #}
+      <div class="content">
+        <h2>{{ page_title|e }}</h2>
+        {%- if new_replies %}
+          <div class="notification">
+            <h3>{% trans %}Someone Replied To Your Paste{% endtrans %}</h3>
+            {% for paste in new_replies %}
+            <p>{% trans date=paste.pub_date|datetimeformat, parent=paste.parent.paste_id,
+                        paste=paste.paste_id, paste_url=paste.url|e, parent_url=paste.parent.url|e %}
+              on {{ date }} someone replied to your paste 
+              <a href="{{ parent_url }}">#{{ parent }}</a>,
+              in paste <a href="{{ paste_url }}">#{{ paste }}</a>. Click here to
+              <a href="/compare/{{ paste }}/{{ parent }}/">compare
+              those two pastes</a>.{% endtrans %}
+            </p>
+            {% endfor %}
+            <p><a href="javascript:LodgeIt.hideNotification()">{% trans %}hide this notification{% endtrans %}</a></p>
+          </div>
+        {% elif request.first_visit %}
+          <div class="notification">
+            <h3>{% trans %}Welcome On LodgeIt{% endtrans %}</h3>
+            <p>{%- trans -%}
+              Welcome to the LodgeIt pastebin. In order to use the notification feature
+              a 31 day cookie with an unique ID was created for you. The lodgeit database
+              does not store any information about you, it's just used for an advanced
+              pastebin experience :-). Read more on the <a href="/about/#privacy">about
+              lodgeit</a> page. Have fun :-){%- endtrans -%}
+            </p>
+            <p><a href="javascript:LodgeIt.hideNotification()">{% trans %}hide this notification{% endtrans %}</a></p>
+          </div>
+        {% endif -%}
+        {% block body %}{% endblock -%}
+        <div class="footer"></div>
+      </div>
+    </div>
+    <script type="text/javascript">
+      var _gaq = _gaq || [];
+      _gaq.push(['_setAccount', 'UA-27485426-1']);
+      _gaq.push(['_trackPageview']);
+      (function() {
+        var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+        ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+        var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+      })();
+    </script>
+  </body>
+</html>

modules/lodgeit/templates/manage.py.erb

@@ -0,0 +1,38 @@
+import os
+
+from werkzeug import script, create_environ, run_wsgi_app
+from werkzeug.serving import run_simple
+
+from lodgeit import local
+from lodgeit.application import make_app
+from lodgeit.database import session
+
+#dburi = 'sqlite:////tmp/lodgeit.db'
+dburi = 'drizzle://127.0.0.1:4427/<%= name %>'
+
+SECRET_KEY = 'no secret key'
+
+
+def run_app(app, path='/'):
+    env = create_environ(path, SECRET_KEY)
+    return run_wsgi_app(app, env)
+
+action_runserver = script.make_runserver(
+    lambda: make_app(dburi, SECRET_KEY),
+    use_reloader=True)
+
+action_shell = script.make_shell(
+    lambda: {
+        'app': make_app(dburi, SECRET_KEY, False, True),
+        'local': local,
+        'session': session,
+        'run_app': run_app
+    },
+    ('\nWelcome to the interactive shell environment of LodgeIt!\n'
+     '\n'
+     'You can use the following predefined objects: app, local, session.\n'
+     'To run the application (creates a request) use *run_app*.')
+)
+
+if __name__ == '__main__':
+    script.run()

modules/lodgeit/templates/nginx.erb

@@ -0,0 +1,10 @@
+server {
+  listen 80;
+  server_name paste.<%= name %>.org;
+  root /srv/lodgeit/<%= name %>;
+
+  location / {
+    proxy_pass http://localhost:<%= port %>/;
+  }
+}
+

modules/lodgeit/templates/upstart.erb

@@ -0,0 +1,8 @@
+description "<%= name %> Lodgeit server"
+author      "Andrew Hutchings <andrew@linuxjedi.co.uk>"
+
+start on (local-filesystems and net-device-up)
+stop on runlevel [!2345]
+
+exec python /srv/lodgeit/<%= name %>/manage.py runserver -h 127.0.0.1 -p <%= port %>
+