From a3a2a84d8a250e1af40ac692cac7d206d2e3279d Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Thu, 27 Jul 2017 11:51:02 -0400 Subject: [PATCH] Remove puppetdb / puppetboard server Currently puppetdb and puppetboard have been broken for some time (+1 year) and with ubuntu precise becoming EOL it is prime for deleting. This leaves openstack-infra with a gap in reporting for non-root users. As such, as proposal is in the works to maybe use ARA. Change-Id: Ifc73a2dba3b37ebe790a29c0daa948d6bad0aa33 Signed-off-by: Paul Belanger --- doc/source/puppet.rst | 4 +- doc/source/running-your-own.rst | 23 +--- hiera/common.yaml | 1 - install_puppet.sh | 6 - manifests/site.pp | 23 ---- modules.env | 2 - .../lib/puppet/reports/puppetdb_file.rb | 130 ------------------ .../manifests/puppetboard.pp | 51 ------- .../openstack_project/manifests/puppetdb.pp | 61 -------- .../manifests/puppetmaster.pp | 13 -- .../puppetboard/puppetboard.vhost.erb | 16 --- tools/puppetmaster_bootstrap.sh | 1 - 12 files changed, 3 insertions(+), 328 deletions(-) delete mode 100644 modules/openstack_project/lib/puppet/reports/puppetdb_file.rb delete mode 100644 modules/openstack_project/manifests/puppetboard.pp delete mode 100644 modules/openstack_project/manifests/puppetdb.pp delete mode 100644 modules/openstack_project/templates/puppetboard/puppetboard.vhost.erb diff --git a/doc/source/puppet.rst b/doc/source/puppet.rst index 3e07a78ba9..e23e87f7e2 100644 --- a/doc/source/puppet.rst +++ b/doc/source/puppet.rst @@ -31,9 +31,7 @@ Puppet Driving Ansible Driving Puppet In OpenStack Infra, there are ansible playbooks that drive the running of ``puppet apply`` on all of the hosts in the inventory. That process first -copies appropriate ``hiera`` data files to each host, and when it is done -it copies back the JSON report of the puppet run and submits it to -``puppetdb``. +copies appropriate ``hiera`` data files to each host. The cron jobs, current configuration files and more can be done with ``puppet apply`` but first some bootstrapping needs to be done. diff --git a/doc/source/running-your-own.rst b/doc/source/running-your-own.rst index 002b1abdd0..260293f74d 100644 --- a/doc/source/running-your-own.rst +++ b/doc/source/running-your-own.rst @@ -39,8 +39,7 @@ Initial setup ============= #. Manually boot a machine or VM with 2G+ of ram to be the puppetmaster. - Average memory consumption is between 1GB-1.5GB with random peaks around - 2GB for puppetdb and ruby processes. + Average memory consumption is between 1GB-1.5GB. #. Clone the CI config repository and adjust it as necessary. Avoiding forks and overriding the default config from Infra is a good practice to @@ -127,8 +126,6 @@ The minimum set of things to port across is: * The puppetmaster definition in site.pp -* The puppetdb definition in site.pp - Then follow the :ref:`puppet-master` instructions for bringing up a puppetmaster, replacing openstack_project with your project name. You'll need to populate hiera at the end with the minimum set of keys: @@ -141,23 +138,7 @@ Copy in your cloud credentials to /root/ci-launch - e.g. to Stage 2 ~~~~~~~ -Migrate: - -* modules/openstack_project/manifests/puppetdb.pp - -Then start up your puppet db with puppet board (see :file:`launch/README` -for full details):: - - sudo su - - cd /opt/system-config/production/launch - . /root/ci-launch/ - export FQDN=servername.project.example.com - puppet cert generate $FQDN - ./launch-node.py $FQDN --server puppetmaster.project.example.com - -* This will chug for a while. - -* Run the DNS update commands [nb: install your DNS API by hand at the moment] +N/A Stage 3 - gerrit ~~~~~~~~~~~~~~~~ diff --git a/hiera/common.yaml b/hiera/common.yaml index 38a243cb7f..822a5342fa 100644 --- a/hiera/common.yaml +++ b/hiera/common.yaml @@ -450,7 +450,6 @@ cacti_hosts: - pbx.openstack.org - planet01.openstack.org - proposal.slave.openstack.org -- puppetdb.openstack.org - puppetmaster.openstack.org - mirror01.dfw.rax.openstack.org - mirror01.ord.rax.openstack.org diff --git a/install_puppet.sh b/install_puppet.sh index f99388ebe9..e454c20a72 100755 --- a/install_puppet.sh +++ b/install_puppet.sh @@ -194,13 +194,11 @@ function setup_puppet_ubuntu { fi PUPPET_VERSION=3.* puppetpkg=puppet - PUPPETDB_VERSION=2.* FACTER_VERSION=2.* elif [ "$PUPPET_VERSION" == "4" ] ; then puppet_deb=puppetlabs-release-pc1-${lsbdistcodename}.deb puppetpkg=puppet-agent PUPPET_VERSION=4.* - PUPPETDB_VERSION=4.* FACTER_VERSION=3.* else echo "Unsupported puppet version ${PUPPET_VERSION}" @@ -212,10 +210,6 @@ Package: puppet puppet-common puppetmaster puppetmaster-common puppetmaster-pass Pin: version $PUPPET_VERSION Pin-Priority: 501 -Package: puppetdb puppetdb-terminus -Pin: version $PUPPETDB_VERSION -Pin-Priority: 501 - Package: facter Pin: version $FACTER_VERSION Pin-Priority: 501 diff --git a/manifests/site.pp b/manifests/site.pp index 5f31a6ae3e..bd96155fcb 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -166,7 +166,6 @@ node 'puppetmaster.openstack.org' { class { 'openstack_project::puppetmaster': root_rsa_key => hiera('puppetmaster_root_rsa_key'), puppetmaster_clouds => hiera('puppetmaster_clouds'), - puppetdb => false, enable_mqtt => true, mqtt_password => hiera('mqtt_service_user_password'), mqtt_ca_cert_contents => hiera('mosquitto_tls_ca_file'), @@ -189,28 +188,6 @@ node 'puppetmaster.openstack.org' { } } -# Node-OS: trusty -node 'puppetdb.openstack.org' { - $open_ports = [8081, 80] - class { 'openstack_project::server': - iptables_public_tcp_ports => $open_ports, - sysadmins => hiera('sysadmins', []), - } - include openstack_project::puppetdb -} - -# Node-OS: trusty -node 'puppetdb01.openstack.org' { - $open_ports = [8081, 80] - class { 'openstack_project::server': - iptables_public_tcp_ports => $open_ports, - sysadmins => hiera('sysadmins', []), - } - class { 'openstack_project::puppetdb': - version => '4.0.2-1puppetlabs1', - } -} - # Node-OS: trusty node 'graphite.openstack.org' { $statsd_hosts = ['git.openstack.org', diff --git a/modules.env b/modules.env index aed537a199..a85a5fa68d 100644 --- a/modules.env +++ b/modules.env @@ -72,7 +72,6 @@ SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-java_ks"]="1.3.1" SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-mysql"]="3.11.0" SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-ntp"]="3.2.1" SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-postgresql"]="3.4.2" -SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-puppetdb"]="3.0.1" SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-rabbitmq"]="5.2.3" SOURCE_MODULES["https://github.com/puppetlabs/puppetlabs-stdlib"]="4.17.0" SOURCE_MODULES["https://github.com/rafaelfelix/puppet-pear"]="1.0.3" @@ -84,7 +83,6 @@ SOURCE_MODULES["https://github.com/voxpupuli/puppet-alternatives"]="0.3.0" SOURCE_MODULES["https://github.com/voxpupuli/puppet-archive"]="v0.5.1" SOURCE_MODULES["https://github.com/voxpupuli/puppet-git_resource"]="0.3.0" SOURCE_MODULES["https://github.com/voxpupuli/puppet-nodejs"]="1.2.0" -SOURCE_MODULES["https://github.com/voxpupuli/puppet-puppetboard"]="2.4.0" # Add modules that should be part of the openstack-infra integration test here diff --git a/modules/openstack_project/lib/puppet/reports/puppetdb_file.rb b/modules/openstack_project/lib/puppet/reports/puppetdb_file.rb deleted file mode 100644 index 4c8b0bf847..0000000000 --- a/modules/openstack_project/lib/puppet/reports/puppetdb_file.rb +++ /dev/null @@ -1,130 +0,0 @@ -# File-writing code is from the store report processor in puppet's master branch. -# The rest of the code is from the puppetdb report processor in puppetdb's 2.3.x branch. -require 'puppet' - -Puppet::Reports.register_report(:puppetdb_file) do - desc <<-DESC - Save report information to a file for sending to PuppetDB via the REST API -later. Reports are serialized to JSON format and may then submitted to puppetdb. - DESC - - # Process the report by formatting it into a PuppetDB 'store report' - # written to disk to be submitted to PuppetDB later. - # - # @return [void] - def process - dir = File.join(Puppet[:reportdir], host) - if ! Puppet::FileSystem.exist?(dir) - FileUtils.mkdir_p(dir) - FileUtils.chmod_R(0750, dir) - end - now = Time.now.gmtime - name = %w{year month day hour min}.collect do |method| - "%02d" % now.send(method).to_s - end.join("") + "_puppetdb.json" - file = File.join(dir, name) - begin - Puppet::Util.replace_file(file, 0640) do |fh| - fh.print({ "command" => "store report", "version" => 3, "payload" => report_to_hash }.to_json) - end - rescue => detail - Puppet.log_exception(detail, "Could not write report for #{host} at #{file}: #{detail}") - end - - nil - end - - # Convert `self` (an instance of `Puppet::Transaction::Report`) to a hash - # suitable for sending over the wire to PuppetDB - # - # @return Hash[] - # @api private - def report_to_hash - if environment.nil? - raise Puppet::Error, "Environment is nil, unable to submit report. This may be due a bug with Puppet. Ensure you are running the latest revision, see PUP-2508 for more details." - end - - { - "certname" => host, - "puppet-version" => puppet_version, - "report-format" => report_format, - "configuration-version" => configuration_version.to_s, - "start-time" => time.iso8601(9), - "end-time" => (time + run_duration).iso8601(9), - "resource-events" => build_events_list, - "environment" => environment, - "transaction-uuid" => transaction_uuid, - "status" => status, - } - end - - # Build a resource-events array from both evaluated and skipped resources. - # - # @return Array[Hash] - # @api private - def build_events_list - resource_statuses.inject([]) do |events, status_entry| - _, status = *status_entry - if ! (status.events.empty?) - events.concat(status.events.map { |event| event_to_hash(status, event) }) - elsif status.skipped - events.concat([fabricate_event(status, "skipped")]) - end - events - end - end - - # Calculate run duration. - # - # @return Number - # @api private - def run_duration - if metrics["time"] and metrics["time"]["total"] - metrics["time"]["total"] - else - 0 - end - end - - # Convert an instance of `Puppet::Transaction::Event` to a hash - # suitable for sending over the wire to PuppetDB - # - # @return Hash[] - # @api private - def event_to_hash(resource_status, event) - { - "status" => event.status, - "timestamp" => event.time.iso8601(9), - "resource-type" => resource_status.resource_type, - "resource-title" => resource_status.title.to_s, - "property" => event.property, - "new-value" => event.desired_value, - "old-value" => event.previous_value, - "message" => event.message, - "file" => resource_status.file, - "line" => resource_status.line, - "containment-path" => resource_status.containment_path, - } - end - - # Given an instance of `Puppet::Resource::Status` and a status - # string, this method fabricates a PuppetDB event object with the - # provided `"status"`. - # - # @api private - def fabricate_event(resource_status, event_status) - { - "status" => event_status, - "timestamp" => resource_status.time.iso8601(9), - "resource-type" => resource_status.resource_type, - "resource-title" => resource_status.title.to_s, - "property" => nil, - "new-value" => nil, - "old-value" => nil, - "message" => nil, - "file" => resource_status.file, - "line" => resource_status.line, - "containment-path" => resource_status.containment_path, - } - end -end diff --git a/modules/openstack_project/manifests/puppetboard.pp b/modules/openstack_project/manifests/puppetboard.pp deleted file mode 100644 index a02ee76db9..0000000000 --- a/modules/openstack_project/manifests/puppetboard.pp +++ /dev/null @@ -1,51 +0,0 @@ -# Class to configure puppetboard on a node. -# This will only work on the puppetdb server for now -class openstack_project::puppetboard( - $basedir = $::puppetboard::params::basedir, - $user = $::puppetboard::params::user, - $group = $::puppetboard::params::group, - $port = '80', -) inherits ::puppetboard::params { - - include ::httpd - - class { '::httpd::mod::wsgi': } - - class { '::puppetboard': - unresponsive => '1.5', - enable_query => 'False', # This being a python false - git_source => 'https://github.com/voxpupuli/puppetboard', - revision => '3042e22a1b4dfc0e3b7f3850c77da5a9398a8a52', - } - - $docroot = "${basedir}/puppetboard" - - # Template Uses: - # - $basedir - # - file { "${docroot}/wsgi.py": - ensure => present, - content => template('puppetboard/wsgi.py.erb'), - owner => $user, - group => $group, - require => User[$user], - } - - # Template Uses: - # - $docroot - # - $user - # - $group - # - $port - # - ::httpd::vhost { $::fqdn: - port => 80, - docroot => $docroot, - priority => '50', - template => 'openstack_project/puppetboard/puppetboard.vhost.erb', - require => [ - User[$user], - Group[$group], - ], - } - -} diff --git a/modules/openstack_project/manifests/puppetdb.pp b/modules/openstack_project/manifests/puppetdb.pp deleted file mode 100644 index 926de2501f..0000000000 --- a/modules/openstack_project/manifests/puppetdb.pp +++ /dev/null @@ -1,61 +0,0 @@ -# == Class: openstack_project::puppetdb -# -class openstack_project::puppetdb ( - $puppetboard = true, - $version = '2.3.8-1puppetlabs1', -) { - - # The puppetlabs postgres module does not manage the postgres user - # and group for us. Create them here to ensure concat can create - # dirs and files owned by this user and group. - user { 'postgres': - ensure => present, - gid => 'postgres', - system => true, - require => Group['postgres'], - } - - group { 'postgres': - ensure => present, - system => true, - } - - class { 'puppetdb::database::postgresql': - require => [User['postgres'], - Class['openstack_project::server'],], - } - - class { '::puppetdb::server': - database_host => 'localhost', - ssl_listen_address => '0.0.0.0', # works for ipv6 too - java_args => { '-Xmx' => '512m', '-Xms' => '256m' }, - puppetdb_version => $version, - require => [ User['postgres'], - Class['puppetdb::database::postgresql'],], - } - - if $puppetboard { - class { 'openstack_project::puppetboard': } - } - - if versioncmp($version, '2.3.8') > 0 { - file { '/etc/puppetdb/': - ensure => directory, - before => Class['::puppetdb::server'], - } - file { '/etc/puppetdb/conf.d/': - ensure => directory, - before => Class['::puppetdb::server'], - } - apt::source { 'puppetlabs-pc1': - location => 'http://apt.puppetlabs.com', - repos => 'PC1', - key => { - 'id' =>'47B320EB4C7C375AA9DAE1A01054B7A24BD6EC30', - 'server' => 'pgp.mit.edu', - }, - before => Class['::puppetdb::server'], - } - } - -} diff --git a/modules/openstack_project/manifests/puppetmaster.pp b/modules/openstack_project/manifests/puppetmaster.pp index d4a44dc641..aad32cea0a 100644 --- a/modules/openstack_project/manifests/puppetmaster.pp +++ b/modules/openstack_project/manifests/puppetmaster.pp @@ -3,8 +3,6 @@ class openstack_project::puppetmaster ( $puppetmaster_clouds, $root_rsa_key = 'xxx', - $puppetdb = true, - $puppetdb_server = 'puppetdb.openstack.org', $puppetmaster_update_cron_interval = { min => '*/15', hour => '*', day => '*', @@ -269,17 +267,6 @@ class openstack_project::puppetmaster ( require => File['/root/signing.gnupg'], } -# Enable puppetdb - - if $puppetdb { - class { 'puppetdb::master::config': - puppetdb_server => $puppetdb_server, - puppet_service_name => 'apache2', - puppetdb_soft_write_failure => true, - manage_storeconfigs => false, - } - } - # Ansible mgmt # TODO: Put this into its own class, maybe called bastion::ansible or something diff --git a/modules/openstack_project/templates/puppetboard/puppetboard.vhost.erb b/modules/openstack_project/templates/puppetboard/puppetboard.vhost.erb deleted file mode 100644 index cd7d6be025..0000000000 --- a/modules/openstack_project/templates/puppetboard/puppetboard.vhost.erb +++ /dev/null @@ -1,16 +0,0 @@ -> - ServerName <%= @fqdn %> - - WSGIDaemonProcess puppetboard user=<%= @user %> group=<%= @group %> threads=5 - WSGIScriptAlias / <%= @docroot %>/wsgi.py - - > - WSGIProcessGroup <%= @group %> - WSGIApplicationGroup %{GLOBAL} - Order deny,allow - Allow from all - = 2.4> - Require all granted - - - diff --git a/tools/puppetmaster_bootstrap.sh b/tools/puppetmaster_bootstrap.sh index 7c96f2b871..f53d029782 100755 --- a/tools/puppetmaster_bootstrap.sh +++ b/tools/puppetmaster_bootstrap.sh @@ -17,7 +17,6 @@ node default { sysadmins => hiera('sysadmins', []), version => '3.6.', ca_server => 'ci-puppetmaster.openstack.org', - puppetdb => false, } } EOF