Remove puppetdb / puppetboard server
Currently puppetdb and puppetboard have been broken for some time (+1 year) and with ubuntu precise becoming EOL it is prime for deleting. This leaves openstack-infra with a gap in reporting for non-root users. As such, as proposal is in the works to maybe use ARA. Change-Id: Ifc73a2dba3b37ebe790a29c0daa948d6bad0aa33 Signed-off-by: Paul Belanger <pabelanger@redhat.com>
This commit is contained in:
Paul Belanger
parent
7c37d2bffc
commit
a3a2a84d8a
@ -31,9 +31,7 @@ Puppet Driving Ansible Driving Puppet
|
||||
|
||||
In OpenStack Infra, there are ansible playbooks that drive the running of
|
||||
``puppet apply`` on all of the hosts in the inventory. That process first
|
||||
copies appropriate ``hiera`` data files to each host, and when it is done
|
||||
it copies back the JSON report of the puppet run and submits it to
|
||||
``puppetdb``.
|
||||
copies appropriate ``hiera`` data files to each host.
|
||||
|
||||
The cron jobs, current configuration files and more can be done with ``puppet
|
||||
apply`` but first some bootstrapping needs to be done.
|
||||
|
||||
@ -39,8 +39,7 @@ Initial setup
|
||||
=============
|
||||
|
||||
#. Manually boot a machine or VM with 2G+ of ram to be the puppetmaster.
|
||||
Average memory consumption is between 1GB-1.5GB with random peaks around
|
||||
2GB for puppetdb and ruby processes.
|
||||
Average memory consumption is between 1GB-1.5GB.
|
||||
|
||||
#. Clone the CI config repository and adjust it as necessary. Avoiding forks
|
||||
and overriding the default config from Infra is a good practice to
|
||||
@ -127,8 +126,6 @@ The minimum set of things to port across is:
|
||||
|
||||
* The puppetmaster definition in site.pp
|
||||
|
||||
* The puppetdb definition in site.pp
|
||||
|
||||
Then follow the :ref:`puppet-master` instructions for bringing up a
|
||||
puppetmaster, replacing openstack_project with your project name.
|
||||
You'll need to populate hiera at the end with the minimum set of keys:
|
||||
@ -141,23 +138,7 @@ Copy in your cloud credentials to /root/ci-launch - e.g. to
|
||||
Stage 2
|
||||
~~~~~~~
|
||||
|
||||
Migrate:
|
||||
|
||||
* modules/openstack_project/manifests/puppetdb.pp
|
||||
|
||||
Then start up your puppet db with puppet board (see :file:`launch/README`
|
||||
for full details)::
|
||||
|
||||
sudo su -
|
||||
cd /opt/system-config/production/launch
|
||||
. /root/ci-launch/
|
||||
export FQDN=servername.project.example.com
|
||||
puppet cert generate $FQDN
|
||||
./launch-node.py $FQDN --server puppetmaster.project.example.com
|
||||
|
||||
* This will chug for a while.
|
||||
|
||||
* Run the DNS update commands [nb: install your DNS API by hand at the moment]
|
||||
N/A
|
||||
|
||||
Stage 3 - gerrit
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
@ -450,7 +450,6 @@ cacti_hosts:
|
||||
- pbx.openstack.org
|
||||
- planet01.openstack.org
|
||||
- proposal.slave.openstack.org
|
||||
- puppetdb.openstack.org
|
||||
- puppetmaster.openstack.org
|
||||
- mirror01.dfw.rax.openstack.org
|
||||
- mirror01.ord.rax.openstack.org
|
||||
|
||||
@ -194,13 +194,11 @@ function setup_puppet_ubuntu {
|
||||
fi
|
||||
PUPPET_VERSION=3.*
|
||||
puppetpkg=puppet
|
||||
PUPPETDB_VERSION=2.*
|
||||
FACTER_VERSION=2.*
|
||||
elif [ "$PUPPET_VERSION" == "4" ] ; then
|
||||
puppet_deb=puppetlabs-release-pc1-${lsbdistcodename}.deb
|
||||
puppetpkg=puppet-agent
|
||||
PUPPET_VERSION=4.*
|
||||
PUPPETDB_VERSION=4.*
|
||||
FACTER_VERSION=3.*
|
||||
else
|
||||
echo "Unsupported puppet version ${PUPPET_VERSION}"
|
||||
@ -212,10 +210,6 @@ Package: puppet puppet-common puppetmaster puppetmaster-common puppetmaster-pass
|
||||
Pin: version $PUPPET_VERSION
|
||||
Pin-Priority: 501
|
||||
|
||||
Package: puppetdb puppetdb-terminus
|
||||
Pin: version $PUPPETDB_VERSION
|
||||
Pin-Priority: 501
|
||||
|
||||
Package: facter
|
||||
Pin: version $FACTER_VERSION
|
||||
Pin-Priority: 501
|
||||
|
||||
@ -166,7 +166,6 @@ node 'puppetmaster.openstack.org' {
|
||||
class { 'openstack_project::puppetmaster':
|
||||
root_rsa_key => hiera('puppetmaster_root_rsa_key'),
|
||||
puppetmaster_clouds => hiera('puppetmaster_clouds'),
|
||||
puppetdb => false,
|
||||
enable_mqtt => true,
|
||||
mqtt_password => hiera('mqtt_service_user_password'),
|
||||
mqtt_ca_cert_contents => hiera('mosquitto_tls_ca_file'),
|
||||
@ -189,28 +188,6 @@ node 'puppetmaster.openstack.org' {
|
||||
}
|
||||
}
|
||||
|
||||
# Node-OS: trusty
|
||||
node 'puppetdb.openstack.org' {
|
||||
$open_ports = [8081, 80]
|
||||
class { 'openstack_project::server':
|
||||
iptables_public_tcp_ports => $open_ports,
|
||||
sysadmins => hiera('sysadmins', []),
|
||||
}
|
||||
include openstack_project::puppetdb
|
||||
}
|
||||
|
||||
# Node-OS: trusty
|
||||
node 'puppetdb01.openstack.org' {
|
||||
$open_ports = [8081, 80]
|
||||
class { 'openstack_project::server':
|
||||
iptables_public_tcp_ports => $open_ports,
|
||||
sysadmins => hiera('sysadmins', []),
|
||||
}
|
||||
class { 'openstack_project::puppetdb':
|
||||
version => '4.0.2-1puppetlabs1',
|
||||
}
|
||||
}
|
||||
|
||||
# Node-OS: trusty
|
||||
node 'graphite.openstack.org' {
|
||||
$statsd_hosts = ['git.openstack.org',
|
||||
|
||||
@ -72,7 +72,6 @@ SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-java_ks"]="1.3.1"
|
||||
SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-mysql"]="3.11.0"
|
||||
SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-ntp"]="3.2.1"
|
||||
SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-postgresql"]="3.4.2"
|
||||
SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-puppetdb"]="3.0.1"
|
||||
SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-rabbitmq"]="5.2.3"
|
||||
SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-stdlib"]="4.17.0"
|
||||
SOURCE_MODULES["https://github.com/rafaelfelix/puppet-pear"]="1.0.3"
|
||||
@ -84,7 +83,6 @@ SOURCE_MODULES["https://github.com/voxpupuli/puppet-alternatives"]="0.3.0"
|
||||
SOURCE_MODULES["https://github.com/voxpupuli/puppet-archive"]="v0.5.1"
|
||||
SOURCE_MODULES["https://github.com/voxpupuli/puppet-git_resource"]="0.3.0"
|
||||
SOURCE_MODULES["https://github.com/voxpupuli/puppet-nodejs"]="1.2.0"
|
||||
SOURCE_MODULES["https://github.com/voxpupuli/puppet-puppetboard"]="2.4.0"
|
||||
|
||||
|
||||
# Add modules that should be part of the openstack-infra integration test here
|
||||
|
||||
@ -1,130 +0,0 @@
|
||||
# File-writing code is from the store report processor in puppet's master branch.
|
||||
# The rest of the code is from the puppetdb report processor in puppetdb's 2.3.x branch.
|
||||
require 'puppet'
|
||||
|
||||
Puppet::Reports.register_report(:puppetdb_file) do
|
||||
desc <<-DESC
|
||||
Save report information to a file for sending to PuppetDB via the REST API
|
||||
later. Reports are serialized to JSON format and may then submitted to puppetdb.
|
||||
DESC
|
||||
|
||||
# Process the report by formatting it into a PuppetDB 'store report'
|
||||
# written to disk to be submitted to PuppetDB later.
|
||||
#
|
||||
# @return [void]
|
||||
def process
|
||||
dir = File.join(Puppet[:reportdir], host)
|
||||
if ! Puppet::FileSystem.exist?(dir)
|
||||
FileUtils.mkdir_p(dir)
|
||||
FileUtils.chmod_R(0750, dir)
|
||||
end
|
||||
now = Time.now.gmtime
|
||||
name = %w{year month day hour min}.collect do |method|
|
||||
"%02d" % now.send(method).to_s
|
||||
end.join("") + "_puppetdb.json"
|
||||
file = File.join(dir, name)
|
||||
begin
|
||||
Puppet::Util.replace_file(file, 0640) do |fh|
|
||||
fh.print({ "command" => "store report", "version" => 3, "payload" => report_to_hash }.to_json)
|
||||
end
|
||||
rescue => detail
|
||||
Puppet.log_exception(detail, "Could not write report for #{host} at #{file}: #{detail}")
|
||||
end
|
||||
|
||||
nil
|
||||
end
|
||||
|
||||
# Convert `self` (an instance of `Puppet::Transaction::Report`) to a hash
|
||||
# suitable for sending over the wire to PuppetDB
|
||||
#
|
||||
# @return Hash[<String, Object>]
|
||||
# @api private
|
||||
def report_to_hash
|
||||
if environment.nil?
|
||||
raise Puppet::Error, "Environment is nil, unable to submit report. This may be due a bug with Puppet. Ensure you are running the latest revision, see PUP-2508 for more details."
|
||||
end
|
||||
|
||||
{
|
||||
"certname" => host,
|
||||
"puppet-version" => puppet_version,
|
||||
"report-format" => report_format,
|
||||
"configuration-version" => configuration_version.to_s,
|
||||
"start-time" => time.iso8601(9),
|
||||
"end-time" => (time + run_duration).iso8601(9),
|
||||
"resource-events" => build_events_list,
|
||||
"environment" => environment,
|
||||
"transaction-uuid" => transaction_uuid,
|
||||
"status" => status,
|
||||
}
|
||||
end
|
||||
|
||||
# Build a resource-events array from both evaluated and skipped resources.
|
||||
#
|
||||
# @return Array[Hash]
|
||||
# @api private
|
||||
def build_events_list
|
||||
resource_statuses.inject([]) do |events, status_entry|
|
||||
_, status = *status_entry
|
||||
if ! (status.events.empty?)
|
||||
events.concat(status.events.map { |event| event_to_hash(status, event) })
|
||||
elsif status.skipped
|
||||
events.concat([fabricate_event(status, "skipped")])
|
||||
end
|
||||
events
|
||||
end
|
||||
end
|
||||
|
||||
# Calculate run duration.
|
||||
#
|
||||
# @return Number
|
||||
# @api private
|
||||
def run_duration
|
||||
if metrics["time"] and metrics["time"]["total"]
|
||||
metrics["time"]["total"]
|
||||
else
|
||||
0
|
||||
end
|
||||
end
|
||||
|
||||
# Convert an instance of `Puppet::Transaction::Event` to a hash
|
||||
# suitable for sending over the wire to PuppetDB
|
||||
#
|
||||
# @return Hash[<String, Object>]
|
||||
# @api private
|
||||
def event_to_hash(resource_status, event)
|
||||
{
|
||||
"status" => event.status,
|
||||
"timestamp" => event.time.iso8601(9),
|
||||
"resource-type" => resource_status.resource_type,
|
||||
"resource-title" => resource_status.title.to_s,
|
||||
"property" => event.property,
|
||||
"new-value" => event.desired_value,
|
||||
"old-value" => event.previous_value,
|
||||
"message" => event.message,
|
||||
"file" => resource_status.file,
|
||||
"line" => resource_status.line,
|
||||
"containment-path" => resource_status.containment_path,
|
||||
}
|
||||
end
|
||||
|
||||
# Given an instance of `Puppet::Resource::Status` and a status
|
||||
# string, this method fabricates a PuppetDB event object with the
|
||||
# provided `"status"`.
|
||||
#
|
||||
# @api private
|
||||
def fabricate_event(resource_status, event_status)
|
||||
{
|
||||
"status" => event_status,
|
||||
"timestamp" => resource_status.time.iso8601(9),
|
||||
"resource-type" => resource_status.resource_type,
|
||||
"resource-title" => resource_status.title.to_s,
|
||||
"property" => nil,
|
||||
"new-value" => nil,
|
||||
"old-value" => nil,
|
||||
"message" => nil,
|
||||
"file" => resource_status.file,
|
||||
"line" => resource_status.line,
|
||||
"containment-path" => resource_status.containment_path,
|
||||
}
|
||||
end
|
||||
end
|
||||
@ -1,51 +0,0 @@
|
||||
# Class to configure puppetboard on a node.
|
||||
# This will only work on the puppetdb server for now
|
||||
class openstack_project::puppetboard(
|
||||
$basedir = $::puppetboard::params::basedir,
|
||||
$user = $::puppetboard::params::user,
|
||||
$group = $::puppetboard::params::group,
|
||||
$port = '80',
|
||||
) inherits ::puppetboard::params {
|
||||
|
||||
include ::httpd
|
||||
|
||||
class { '::httpd::mod::wsgi': }
|
||||
|
||||
class { '::puppetboard':
|
||||
unresponsive => '1.5',
|
||||
enable_query => 'False', # This being a python false
|
||||
git_source => 'https://github.com/voxpupuli/puppetboard',
|
||||
revision => '3042e22a1b4dfc0e3b7f3850c77da5a9398a8a52',
|
||||
}
|
||||
|
||||
$docroot = "${basedir}/puppetboard"
|
||||
|
||||
# Template Uses:
|
||||
# - $basedir
|
||||
#
|
||||
file { "${docroot}/wsgi.py":
|
||||
ensure => present,
|
||||
content => template('puppetboard/wsgi.py.erb'),
|
||||
owner => $user,
|
||||
group => $group,
|
||||
require => User[$user],
|
||||
}
|
||||
|
||||
# Template Uses:
|
||||
# - $docroot
|
||||
# - $user
|
||||
# - $group
|
||||
# - $port
|
||||
#
|
||||
::httpd::vhost { $::fqdn:
|
||||
port => 80,
|
||||
docroot => $docroot,
|
||||
priority => '50',
|
||||
template => 'openstack_project/puppetboard/puppetboard.vhost.erb',
|
||||
require => [
|
||||
User[$user],
|
||||
Group[$group],
|
||||
],
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
# == Class: openstack_project::puppetdb
|
||||
#
|
||||
class openstack_project::puppetdb (
|
||||
$puppetboard = true,
|
||||
$version = '2.3.8-1puppetlabs1',
|
||||
) {
|
||||
|
||||
# The puppetlabs postgres module does not manage the postgres user
|
||||
# and group for us. Create them here to ensure concat can create
|
||||
# dirs and files owned by this user and group.
|
||||
user { 'postgres':
|
||||
ensure => present,
|
||||
gid => 'postgres',
|
||||
system => true,
|
||||
require => Group['postgres'],
|
||||
}
|
||||
|
||||
group { 'postgres':
|
||||
ensure => present,
|
||||
system => true,
|
||||
}
|
||||
|
||||
class { 'puppetdb::database::postgresql':
|
||||
require => [User['postgres'],
|
||||
Class['openstack_project::server'],],
|
||||
}
|
||||
|
||||
class { '::puppetdb::server':
|
||||
database_host => 'localhost',
|
||||
ssl_listen_address => '0.0.0.0', # works for ipv6 too
|
||||
java_args => { '-Xmx' => '512m', '-Xms' => '256m' },
|
||||
puppetdb_version => $version,
|
||||
require => [ User['postgres'],
|
||||
Class['puppetdb::database::postgresql'],],
|
||||
}
|
||||
|
||||
if $puppetboard {
|
||||
class { 'openstack_project::puppetboard': }
|
||||
}
|
||||
|
||||
if versioncmp($version, '2.3.8') > 0 {
|
||||
file { '/etc/puppetdb/':
|
||||
ensure => directory,
|
||||
before => Class['::puppetdb::server'],
|
||||
}
|
||||
file { '/etc/puppetdb/conf.d/':
|
||||
ensure => directory,
|
||||
before => Class['::puppetdb::server'],
|
||||
}
|
||||
apt::source { 'puppetlabs-pc1':
|
||||
location => 'http://apt.puppetlabs.com',
|
||||
repos => 'PC1',
|
||||
key => {
|
||||
'id' =>'47B320EB4C7C375AA9DAE1A01054B7A24BD6EC30',
|
||||
'server' => 'pgp.mit.edu',
|
||||
},
|
||||
before => Class['::puppetdb::server'],
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -3,8 +3,6 @@
|
||||
class openstack_project::puppetmaster (
|
||||
$puppetmaster_clouds,
|
||||
$root_rsa_key = 'xxx',
|
||||
$puppetdb = true,
|
||||
$puppetdb_server = 'puppetdb.openstack.org',
|
||||
$puppetmaster_update_cron_interval = { min => '*/15',
|
||||
hour => '*',
|
||||
day => '*',
|
||||
@ -269,17 +267,6 @@ class openstack_project::puppetmaster (
|
||||
require => File['/root/signing.gnupg'],
|
||||
}
|
||||
|
||||
# Enable puppetdb
|
||||
|
||||
if $puppetdb {
|
||||
class { 'puppetdb::master::config':
|
||||
puppetdb_server => $puppetdb_server,
|
||||
puppet_service_name => 'apache2',
|
||||
puppetdb_soft_write_failure => true,
|
||||
manage_storeconfigs => false,
|
||||
}
|
||||
}
|
||||
|
||||
# Ansible mgmt
|
||||
# TODO: Put this into its own class, maybe called bastion::ansible or something
|
||||
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
<VirtualHost *:<%= @port %>>
|
||||
ServerName <%= @fqdn %>
|
||||
|
||||
WSGIDaemonProcess puppetboard user=<%= @user %> group=<%= @group %> threads=5
|
||||
WSGIScriptAlias / <%= @docroot %>/wsgi.py
|
||||
|
||||
<Directory <%= @docroot %>>
|
||||
WSGIProcessGroup <%= @group %>
|
||||
WSGIApplicationGroup %{GLOBAL}
|
||||
Order deny,allow
|
||||
Allow from all
|
||||
<IfVersion >= 2.4>
|
||||
Require all granted
|
||||
</IfVersion>
|
||||
</Directory>
|
||||
</VirtualHost>
|
||||
@ -17,7 +17,6 @@ node default {
|
||||
sysadmins => hiera('sysadmins', []),
|
||||
version => '3.6.',
|
||||
ca_server => 'ci-puppetmaster.openstack.org',
|
||||
puppetdb => false,
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
Loading…
Reference in New Issue
Block a user