Remove puppetdb / puppetboard server

Currently puppetdb and puppetboard have been broken for some time (+1
year) and with ubuntu precise becoming EOL it is prime for deleting.

This leaves openstack-infra with a gap in reporting for non-root
users.  As such, as proposal is in the works to maybe use ARA.

Change-Id: Ifc73a2dba3b37ebe790a29c0daa948d6bad0aa33
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

doc/source/puppet.rst

@@ -31,9 +31,7 @@ Puppet Driving Ansible Driving Puppet
 
 In OpenStack Infra, there are ansible playbooks that drive the running of
 ``puppet apply`` on all of the hosts in the inventory. That process first
-copies appropriate ``hiera`` data files to each host, and when it is done
+copies appropriate ``hiera`` data files to each host.
-it copies back the JSON report of the puppet run and submits it to
-``puppetdb``.
 
 The cron jobs, current configuration files and more can be done with ``puppet
 apply`` but first some bootstrapping needs to be done.

doc/source/running-your-own.rst

@@ -39,8 +39,7 @@ Initial setup
 =============
 
 #. Manually boot a machine or VM with 2G+ of ram to be the puppetmaster.
-   Average memory consumption is between 1GB-1.5GB with random peaks around
+   Average memory consumption is between 1GB-1.5GB.
-   2GB for puppetdb and ruby processes.
 
 #. Clone the CI config repository and adjust it as necessary. Avoiding forks
    and overriding the default config from Infra is a good practice to
@@ -127,8 +126,6 @@ The minimum set of things to port across is:
 
 * The puppetmaster definition in site.pp
 
-* The puppetdb definition in site.pp
-
 Then follow the :ref:`puppet-master` instructions for bringing up a
 puppetmaster, replacing openstack_project with your project name.
 You'll need to populate hiera at the end with the minimum set of keys:
@@ -141,23 +138,7 @@ Copy in your cloud credentials to /root/ci-launch - e.g. to
 Stage 2
 ~~~~~~~
 
-Migrate:
+N/A
-
-* modules/openstack_project/manifests/puppetdb.pp
-
-Then start up your puppet db with puppet board (see :file:`launch/README`
-for full details)::
-
-    sudo su -
-    cd /opt/system-config/production/launch
-    . /root/ci-launch/
-    export FQDN=servername.project.example.com
-    puppet cert generate $FQDN
-    ./launch-node.py $FQDN --server puppetmaster.project.example.com
-
-* This will chug for a while.
-
-* Run the DNS update commands [nb: install your DNS API by hand at the moment]
 
 Stage 3 - gerrit
 ~~~~~~~~~~~~~~~~

hiera/common.yaml

@@ -450,7 +450,6 @@ cacti_hosts:
 - pbx.openstack.org
 - planet01.openstack.org
 - proposal.slave.openstack.org
-- puppetdb.openstack.org
 - puppetmaster.openstack.org
 - mirror01.dfw.rax.openstack.org
 - mirror01.ord.rax.openstack.org

install_puppet.sh

@@ -194,13 +194,11 @@ function setup_puppet_ubuntu {
         fi
         PUPPET_VERSION=3.*
         puppetpkg=puppet
-        PUPPETDB_VERSION=2.*
         FACTER_VERSION=2.*
     elif [ "$PUPPET_VERSION" == "4" ] ; then
         puppet_deb=puppetlabs-release-pc1-${lsbdistcodename}.deb
         puppetpkg=puppet-agent
         PUPPET_VERSION=4.*
-        PUPPETDB_VERSION=4.*
         FACTER_VERSION=3.*
     else
         echo "Unsupported puppet version ${PUPPET_VERSION}"
@@ -212,10 +210,6 @@ Package: puppet puppet-common puppetmaster puppetmaster-common puppetmaster-pass
 Pin: version $PUPPET_VERSION
 Pin-Priority: 501
 
-Package: puppetdb puppetdb-terminus
-Pin: version $PUPPETDB_VERSION
-Pin-Priority: 501
-
 Package: facter
 Pin: version $FACTER_VERSION
 Pin-Priority: 501

manifests/site.pp

@@ -166,7 +166,6 @@ node 'puppetmaster.openstack.org' {
   class { 'openstack_project::puppetmaster':
     root_rsa_key                               => hiera('puppetmaster_root_rsa_key'),
     puppetmaster_clouds                        => hiera('puppetmaster_clouds'),
-    puppetdb                                   => false,
     enable_mqtt                                => true,
     mqtt_password                              => hiera('mqtt_service_user_password'),
     mqtt_ca_cert_contents                      => hiera('mosquitto_tls_ca_file'),
@@ -189,28 +188,6 @@ node 'puppetmaster.openstack.org' {
   }
 }
 
-# Node-OS: trusty
-node 'puppetdb.openstack.org' {
-  $open_ports = [8081, 80]
-  class { 'openstack_project::server':
-    iptables_public_tcp_ports => $open_ports,
-    sysadmins                 => hiera('sysadmins', []),
-  }
-  include openstack_project::puppetdb
-}
-
-# Node-OS: trusty
-node 'puppetdb01.openstack.org' {
-  $open_ports = [8081, 80]
-  class { 'openstack_project::server':
-    iptables_public_tcp_ports => $open_ports,
-    sysadmins                 => hiera('sysadmins', []),
-  }
-  class { 'openstack_project::puppetdb':
-    version => '4.0.2-1puppetlabs1',
-  }
-}
-
 # Node-OS: trusty
 node 'graphite.openstack.org' {
   $statsd_hosts = ['git.openstack.org',

modules.env

@@ -72,7 +72,6 @@ SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-java_ks"]="1.3.1"
 SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-mysql"]="3.11.0"
 SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-ntp"]="3.2.1"
 SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-postgresql"]="3.4.2"
-SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-puppetdb"]="3.0.1"
 SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-rabbitmq"]="5.2.3"
 SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-stdlib"]="4.17.0"
 SOURCE_MODULES["https://github.com/rafaelfelix/puppet-pear"]="1.0.3"
@@ -84,7 +83,6 @@ SOURCE_MODULES["https://github.com/voxpupuli/puppet-alternatives"]="0.3.0"
 SOURCE_MODULES["https://github.com/voxpupuli/puppet-archive"]="v0.5.1"
 SOURCE_MODULES["https://github.com/voxpupuli/puppet-git_resource"]="0.3.0"
 SOURCE_MODULES["https://github.com/voxpupuli/puppet-nodejs"]="1.2.0"
-SOURCE_MODULES["https://github.com/voxpupuli/puppet-puppetboard"]="2.4.0"
 
 
 # Add modules that should be part of the openstack-infra integration test here

modules/openstack_project/lib/puppet/reports/puppetdb_file.rb

@@ -1,130 +0,0 @@
-# File-writing code is from the store report processor in puppet's master branch.
-# The rest of the code is from the puppetdb report processor in puppetdb's 2.3.x branch.
-require 'puppet'
-
-Puppet::Reports.register_report(:puppetdb_file) do
-  desc <<-DESC
-  Save report information to a file for sending to PuppetDB via the REST API
-later.  Reports are serialized to JSON format and may then submitted to puppetdb.
-  DESC
-
-  # Process the report by formatting it into a PuppetDB 'store report'
-  # written to disk to be submitted to PuppetDB later.
-  #
-  # @return [void]
-  def process
-    dir = File.join(Puppet[:reportdir], host)
-    if ! Puppet::FileSystem.exist?(dir)
-      FileUtils.mkdir_p(dir)
-      FileUtils.chmod_R(0750, dir)
-    end
-    now = Time.now.gmtime
-    name = %w{year month day hour min}.collect do |method|
-      "%02d" % now.send(method).to_s
-    end.join("") + "_puppetdb.json"
-    file = File.join(dir, name)
-    begin
-      Puppet::Util.replace_file(file, 0640) do |fh|
-        fh.print({ "command" => "store report", "version" => 3, "payload" => report_to_hash }.to_json)
-      end
-    rescue => detail
-       Puppet.log_exception(detail, "Could not write report for #{host} at #{file}: #{detail}")
-    end
-
-    nil
-  end
-
-  # Convert `self` (an instance of `Puppet::Transaction::Report`) to a hash
-  # suitable for sending over the wire to PuppetDB
-  #
-  # @return Hash[<String, Object>]
-  # @api private
-  def report_to_hash
-    if environment.nil?
-      raise Puppet::Error, "Environment is nil, unable to submit report. This may be due a bug with Puppet. Ensure you are running the latest revision, see PUP-2508 for more details."
-    end
-
-    {
-      "certname"              => host,
-      "puppet-version"        => puppet_version,
-      "report-format"         => report_format,
-      "configuration-version" => configuration_version.to_s,
-      "start-time"            => time.iso8601(9),
-      "end-time"              => (time + run_duration).iso8601(9),
-      "resource-events"       => build_events_list,
-      "environment"           => environment,
-      "transaction-uuid"      => transaction_uuid,
-      "status"                => status,
-    }
-  end
-
-  # Build a resource-events array from both evaluated and skipped resources.
-  #
-  # @return Array[Hash]
-  # @api private
-  def build_events_list
-    resource_statuses.inject([]) do |events, status_entry|
-      _, status = *status_entry
-      if ! (status.events.empty?)
-        events.concat(status.events.map { |event| event_to_hash(status, event) })
-      elsif status.skipped
-        events.concat([fabricate_event(status, "skipped")])
-      end
-      events
-    end
-  end
-
-  # Calculate run duration.
-  #
-  # @return Number
-  # @api private
-  def run_duration
-    if metrics["time"] and metrics["time"]["total"]
-      metrics["time"]["total"]
-    else
-      0
-    end
-  end
-
-  # Convert an instance of `Puppet::Transaction::Event` to a hash
-  # suitable for sending over the wire to PuppetDB
-  #
-  # @return Hash[<String, Object>]
-  # @api private
-  def event_to_hash(resource_status, event)
-    {
-      "status"           => event.status,
-      "timestamp"        => event.time.iso8601(9),
-      "resource-type"    => resource_status.resource_type,
-      "resource-title"   => resource_status.title.to_s,
-      "property"         => event.property,
-      "new-value"        => event.desired_value,
-      "old-value"        => event.previous_value,
-      "message"          => event.message,
-      "file"             => resource_status.file,
-      "line"             => resource_status.line,
-      "containment-path" => resource_status.containment_path,
-    }
-  end
-
-  # Given an instance of `Puppet::Resource::Status` and a status
-  # string, this method fabricates a PuppetDB event object with the
-  # provided `"status"`.
-  #
-  # @api private
-  def fabricate_event(resource_status, event_status)
-    {
-      "status"           => event_status,
-      "timestamp"        => resource_status.time.iso8601(9),
-      "resource-type"    => resource_status.resource_type,
-      "resource-title"   => resource_status.title.to_s,
-      "property"         => nil,
-      "new-value"        => nil,
-      "old-value"        => nil,
-      "message"          => nil,
-      "file"             => resource_status.file,
-      "line"             => resource_status.line,
-      "containment-path" => resource_status.containment_path,
-    }
-  end
-end

modules/openstack_project/manifests/puppetboard.pp

@@ -1,51 +0,0 @@
-# Class to configure puppetboard on a node.
-# This will only work on the puppetdb server for now
-class openstack_project::puppetboard(
-  $basedir = $::puppetboard::params::basedir,
-  $user    = $::puppetboard::params::user,
-  $group   = $::puppetboard::params::group,
-  $port    = '80',
-) inherits ::puppetboard::params {
-
-  include ::httpd
-
-  class { '::httpd::mod::wsgi': }
-
-  class { '::puppetboard':
-    unresponsive => '1.5',
-    enable_query => 'False', # This being a python false
-    git_source   => 'https://github.com/voxpupuli/puppetboard',
-    revision     => '3042e22a1b4dfc0e3b7f3850c77da5a9398a8a52',
-  }
-
-  $docroot = "${basedir}/puppetboard"
-
-  # Template Uses:
-  # - $basedir
-  #
-  file { "${docroot}/wsgi.py":
-    ensure  => present,
-    content => template('puppetboard/wsgi.py.erb'),
-    owner   => $user,
-    group   => $group,
-    require => User[$user],
-  }
-
-  # Template Uses:
-  # - $docroot
-  # - $user
-  # - $group
-  # - $port
-  #
-  ::httpd::vhost { $::fqdn:
-    port     => 80,
-    docroot  => $docroot,
-    priority => '50',
-    template => 'openstack_project/puppetboard/puppetboard.vhost.erb',
-    require  => [
-      User[$user],
-      Group[$group],
-    ],
-  }
-
-}

modules/openstack_project/manifests/puppetdb.pp

@@ -1,61 +0,0 @@
-# == Class: openstack_project::puppetdb
-#
-class openstack_project::puppetdb (
-  $puppetboard = true,
-  $version = '2.3.8-1puppetlabs1',
-) {
-
-  # The puppetlabs postgres module does not manage the postgres user
-  # and group for us. Create them here to ensure concat can create
-  # dirs and files owned by this user and group.
-  user { 'postgres':
-    ensure  => present,
-    gid     => 'postgres',
-    system  => true,
-    require => Group['postgres'],
-  }
-
-  group { 'postgres':
-    ensure => present,
-    system => true,
-  }
-
-  class { 'puppetdb::database::postgresql':
-    require         => [User['postgres'],
-      Class['openstack_project::server'],],
-  }
-
-  class { '::puppetdb::server':
-    database_host      => 'localhost',
-    ssl_listen_address => '0.0.0.0', # works for ipv6 too
-    java_args          => { '-Xmx' => '512m', '-Xms' => '256m' },
-    puppetdb_version   => $version,
-    require            => [ User['postgres'],
-      Class['puppetdb::database::postgresql'],],
-  }
-
-  if $puppetboard {
-    class { 'openstack_project::puppetboard': }
-  }
-
-  if versioncmp($version, '2.3.8') > 0 {
-    file { '/etc/puppetdb/':
-       ensure => directory,
-      before  => Class['::puppetdb::server'],
-    }
-    file { '/etc/puppetdb/conf.d/':
-       ensure => directory,
-      before  => Class['::puppetdb::server'],
-    }
-    apt::source { 'puppetlabs-pc1':
-      location => 'http://apt.puppetlabs.com',
-      repos    => 'PC1',
-      key      => {
-        'id'     =>'47B320EB4C7C375AA9DAE1A01054B7A24BD6EC30',
-        'server' => 'pgp.mit.edu',
-      },
-      before   => Class['::puppetdb::server'],
-    }
-  }
-
-}

modules/openstack_project/manifests/puppetmaster.pp

@@ -3,8 +3,6 @@
 class openstack_project::puppetmaster (
   $puppetmaster_clouds,
   $root_rsa_key = 'xxx',
-  $puppetdb = true,
-  $puppetdb_server = 'puppetdb.openstack.org',
   $puppetmaster_update_cron_interval = { min     => '*/15',
                                          hour    => '*',
                                          day     => '*',
@@ -269,17 +267,6 @@ class openstack_project::puppetmaster (
     require => File['/root/signing.gnupg'],
   }
 
-# Enable puppetdb
-
-  if $puppetdb {
-    class { 'puppetdb::master::config':
-      puppetdb_server              => $puppetdb_server,
-      puppet_service_name          => 'apache2',
-      puppetdb_soft_write_failure  => true,
-      manage_storeconfigs          => false,
-    }
-  }
-
   # Ansible mgmt
   # TODO: Put this into its own class, maybe called bastion::ansible or something
 

modules/openstack_project/templates/puppetboard/puppetboard.vhost.erb

@@ -1,16 +0,0 @@
-<VirtualHost *:<%= @port %>>
-    ServerName <%= @fqdn %>
-
-    WSGIDaemonProcess puppetboard user=<%= @user %> group=<%= @group %> threads=5
-    WSGIScriptAlias / <%= @docroot %>/wsgi.py
-
-    <Directory <%= @docroot %>>
-        WSGIProcessGroup <%= @group %>
-        WSGIApplicationGroup %{GLOBAL}
-        Order deny,allow
-        Allow from all
-        <IfVersion >= 2.4>
-            Require all granted
-        </IfVersion>
-    </Directory>
-</VirtualHost>

tools/puppetmaster_bootstrap.sh

@@ -17,7 +17,6 @@ node default {
     sysadmins    => hiera('sysadmins', []),
     version      => '3.6.',
     ca_server    => 'ci-puppetmaster.openstack.org',
-    puppetdb     => false,
   }
 }
 EOF