From c716240692f67eccace614fb35c77f0a30ac938a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 17 Aug 2018 06:34:50 -0500 Subject: [PATCH] Clean up puppetmaster puppet config handled by ansible We have a bunch of this handled now in ansible, so remove the old stuff. Remove puppetmaster group management files. It's confusing for there to be two files. Remove the old one. Remove mqtt config. This isn't really a thing currently, and we're eyeing running things from zuul anyway, so no need to port to ansible. Change-Id: I8b64d21eadcc4a08bd5e5440fc5f756ae5bcd46b --- doc/source/sysadmin.rst | 24 +- manifests/site.pp | 3 - .../files/puppetmaster/expand-groups.sh | 47 --- .../files/puppetmaster/groups.txt | 33 -- .../manifests/puppetmaster.pp | 281 ------------------ 5 files changed, 10 insertions(+), 378 deletions(-) delete mode 100644 modules/openstack_project/files/puppetmaster/expand-groups.sh delete mode 100644 modules/openstack_project/files/puppetmaster/groups.txt diff --git a/doc/source/sysadmin.rst b/doc/source/sysadmin.rst index d361f345a1..34bb521c19 100644 --- a/doc/source/sysadmin.rst +++ b/doc/source/sysadmin.rst @@ -107,7 +107,7 @@ Accessing Clouds ================ As an unprivileged user who is a member of the `admin` group on -puppetmaster, you can access any of the clouds with:: +bridge, you can access any of the clouds with:: export OS_CLIENT_CONFIG_FILE=/etc/openstack/all-clouds.yaml openstack --os-cloud --os-cloud-region @@ -328,7 +328,7 @@ Launching New Servers New servers are launched using the ``launch/launch-node.py`` tool from the git repository ``https://git.openstack.org/openstack-infra/system-config``. This -tool is run from a checkout on the puppetmaster - please see :cgit_file:`launch/README` +tool is run from a checkout on the bridge - please see :cgit_file:`launch/README` for detailed instructions. .. _disable-enable-puppet: @@ -354,14 +354,14 @@ to take. In the case of needing to disable the running of puppet on a node, it's a simple matter of adding an entry to the ansible inventory "disabled" group -in :cgit_file:`modules/openstack_project/files/puppetmaster/groups.txt`. The +in :cgit_file:`inventory/groups.yaml`. The disabled entry is an input to `ansible --list-hosts` so you can check your entry simply by running it with `ansible $hostlist --list-hosts` as root -on the puppetmaster host and ensuring that the list of hosts returned is as +on the bridge host and ensuring that the list of hosts returned is as expected. Globs, group names and server UUIDs should all be acceptable input. If you need to disable a host immediately without waiting for a patch to land -to `system-config`, there is a file on the puppetmaster host, +to `system-config`, there is a file on the bridge host, `/etc/ansible/hosts/emergency` that can be edited directly. `/etc/ansible/hosts/emergency` is a file that should normally be empty, but @@ -376,15 +376,10 @@ have that have more than one host with the same name (such as in the case of being in the midst of a migration) will show up as a group with the name of the hostname and the individual servers will be listed by UUID. -Because of the way static and dynamic inventories get merged by ansible, the -emergency file needs to stand alone. If you need to disable a group of servers -from OpenStack you need to not only add it to `disabled:children`, you need to -add an emtpy group into the emergency file too. - Disabling puppet via ansible inventory does not disable puppet from being able to be run directly on the host, it merely prevents ansible from attempting to run it. If you choose to run puppet manually on a host, take care -to ensure that it has not been disabled at the puppetmaster level first. +to ensure that it has not been disabled at the bridge level first. Examples -------- @@ -416,11 +411,12 @@ hosts. pypi To disable a staticly defined host that is not an OpenStack host, such as -the Infra cloud controller hosts, put the following in groups.txt. +the Infra cloud controller hosts, update the ``disabled`` entry in +groups.yaml with something like: :: - disabled controller.useast.openstack.org + disabled: inventory_hostname == 'controller.useast.openstack.org' .. _cinder: @@ -433,7 +429,7 @@ Adding a New Device If the main volume group doesn't have enough space for what you want to do, this is how you can add a new volume. -Log into puppetmaster.openstack.org and run:: +Log into bridge.openstack.org and run:: export OS_CLOUD=openstackci-rax export OS_REGION_NAME=DFW diff --git a/manifests/site.pp b/manifests/site.pp index 0df0349c45..919ecf0f55 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -193,9 +193,6 @@ node 'puppetmaster.openstack.org' { class { 'openstack_project::puppetmaster': root_rsa_key => hiera('puppetmaster_root_rsa_key'), puppetmaster_clouds => hiera('puppetmaster_clouds'), - enable_mqtt => true, - mqtt_password => hiera('mqtt_service_user_password'), - mqtt_ca_cert_contents => hiera('mosquitto_tls_ca_file'), } file { '/etc/openstack/limestone_cacert.pem': ensure => present, diff --git a/modules/openstack_project/files/puppetmaster/expand-groups.sh b/modules/openstack_project/files/puppetmaster/expand-groups.sh deleted file mode 100644 index 50af74c79c..0000000000 --- a/modules/openstack_project/files/puppetmaster/expand-groups.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Copyright 2016 IBM Corp -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -outdir=$(mktemp -d) -trap "rm -rf $outdir" EXIT - -outfile=$outdir/generated-groups -echo "# This file is autogenerated" > $outfile - -# Protect against leaky environment settings -unset OS_CLOUD -unset OS_REGION_NAME - -# Because we are generating "generated-groups", we need to work on an -# inventory without out it ... otherwise entries that have actually -# disappeared (according to /etc/ansible/hosts/openstack_inventory's -# live list) will keep coming back as "ansible --list" finds them in -# the generated-groups file. Later versions of ansible have a -# "inventory_ignore_patterns" that might help, we just copy & remove -# in a separate inventory -inv=${outdir}/hosts -mkdir ${inv} -cp /etc/ansible/hosts/* ${inv} -rm -f ${inv}/generated-groups* - -IFS=$'\n' -for line in $(> $outfile - ansible -i ${inv} "${pattern}" --list-hosts | egrep -v '^ +hosts \([0-9]+\):' >> $outfile -done - -cp $outfile /etc/ansible/hosts/generated-groups diff --git a/modules/openstack_project/files/puppetmaster/groups.txt b/modules/openstack_project/files/puppetmaster/groups.txt deleted file mode 100644 index c7c5dddd97..0000000000 --- a/modules/openstack_project/files/puppetmaster/groups.txt +++ /dev/null @@ -1,33 +0,0 @@ -afs afs*.*.openstack.org -afsadmin ~mirror-update\d+\.openstack\.org -afsdb afsdb*.openstack.org -cacti ~cacti\d+\.openstack\.org -ci-backup backup\d+.*\ci\.openstack\.org -disabled backup\d+.*\ci\.openstack\.org:wiki.openstack.org -eavesdrop ~eavesdrop\d*\.openstack\.org -elasticsearch ~elasticsearch0[1-7]\.openstack\.org -ethercalc ~ethercalc\d+\.openstack\.org -files ~files\d+\.openstack\.org -git-loadbalancer ~git(-fe\d+)?\.openstack\.org -git-server ~git\d+\.openstack\.org -logstash-worker ~logstash-worker\d+\.openstack\.org -mailman ~lists\d*\.openstack\.org:~lists\d*\.katacontainers\.io -nodepool nodepool*.openstack.org:nb*.openstack.org:nl*.openstack.org -review ~review\d+\.openstack\.org -review-dev ~review-dev\d*\.openstack\.org -subunit-worker ~subunit-worker\d+\.openstack\.org -survey ~survey\d+\.openstack\.org -translate ~translate\d+\.openstack\.org -translate-dev ~translate-dev\d*\.openstack\.org -wiki ~wiki\d+\.openstack\.org -wiki-dev ~wiki-dev\d+\.openstack\.org -zuul-scheduler ~zuul\d+\.openstack\.org -zuul-merger ~z[lm](static)?\d+\.openstack\.org -zuul-executor ~ze\d+\.openstack\.org -grafana ~grafana\d*\.openstack\.org -status ~status\d*\.openstack\.org -paste ~paste\d*\.openstack\.org -adns ~adns\d*\.openstack\.org -ns ~ns\d*\.openstack\.org -futureparser ~review-dev\d*\.openstack\.org:~graphite\d*\.openstack\.org:~groups\d*\.openstack\.org:~groups-dev\d*\.openstack\.org:~etherpad-dev\d*\.openstack\.org:~ask-staging\d*\.openstack\.org:~codesearch\d*\.openstack\.org -puppet !bridge.openstack.org diff --git a/modules/openstack_project/manifests/puppetmaster.pp b/modules/openstack_project/manifests/puppetmaster.pp index 7f135973c5..044bbc4f76 100644 --- a/modules/openstack_project/manifests/puppetmaster.pp +++ b/modules/openstack_project/manifests/puppetmaster.pp @@ -9,29 +9,9 @@ class openstack_project::puppetmaster ( month => '*', weekday => '*', }, - $enable_mqtt = false, - $mqtt_hostname = 'firehose.openstack.org', - $mqtt_port = 8883, - $mqtt_username = 'infra', - $mqtt_password = undef, - $mqtt_ca_cert_contents = undef, ) { include logrotate - class { '::ansible': - ansible_hostfile => '/etc/ansible/hosts', - retry_files_enabled => 'False', - ansible_version => '2.2.1.0', - } - - file { '/etc/ansible/hostfile': - ensure => present, - owner => 'root', - group => 'root', - mode => '0644', - require => Class['ansible'], - } - cron { 'updatecloudlauncher': ensure => absent, } @@ -47,86 +27,8 @@ class openstack_project::puppetmaster ( 'daily', 'notifempty', ], - require => Cron['updatepuppetmaster'], } - logrotate::file { 'updatecloudlaunchercron': - ensure => present, - log => '/var/log/puppet_run_cloud_launcher_cron.log', - options => ['compress', - 'copytruncate', - 'delaycompress', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - require => Cron['updatepuppetmaster'], - } - - cron { 'updatepuppetmaster': - ensure => absent, - } - - logrotate::file { 'updatepuppetmaster': - ensure => present, - log => '/var/log/puppet_run_all.log', - options => ['compress', - 'copytruncate', - 'delaycompress', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - require => Cron['updatepuppetmaster'], - } - - logrotate::file { 'updatepuppetmastercron': - ensure => present, - log => '/var/log/puppet_run_all_cron.log', - options => ['compress', - 'copytruncate', - 'delaycompress', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - require => Cron['updatepuppetmaster'], - } - - cron { 'deleteoldreports': - ensure => absent, - } - - cron { 'deleteoldreports-json': - ensure => absent, - } - - file { '/etc/puppet/hieradata': - ensure => directory, - group => 'puppet', - mode => '0750', - owner => 'puppet', - } - - file { '/etc/puppet/hieradata/production': - ensure => directory, - group => 'puppet', - mode => '0750', - owner => 'root', - recurse => true, - require => File['/etc/puppet/hieradata'], - } - - file { '/var/lib/puppet/reports': - ensure => directory, - owner => 'puppet', - group => 'puppet', - mode => '0750', - } - if ! defined(File['/root/.ssh']) { file { '/root/.ssh': ensure => directory, @@ -171,38 +73,6 @@ class openstack_project::puppetmaster ( content => template('openstack_project/puppetmaster/all-clouds.yaml.erb'), } -# For puppet master apache serving. - package { 'puppetmaster-passenger': - ensure => absent, - } - - file { '/etc/apache2/sites-available/puppetmaster.conf': - ensure => absent, - } - - file { '/etc/apache2/envvars': - ensure => absent, - } - -# For launch/launch-node.py. - $pip_packages = [ - 'shade', - 'python-openstackclient', - ] - package { $pip_packages: - ensure => latest, - provider => openstack_pip, - } - package { 'python-paramiko': - ensure => present, - } - # No longer needed with latest client libs - package { 'python-lxml': - ensure => absent, - } - package { 'libxslt1-dev': - ensure => absent, - } # For signing key management package { 'gnupg': @@ -234,157 +104,6 @@ class openstack_project::puppetmaster ( require => File['/root/signing.gnupg'], } - # Ansible mgmt - # TODO: Put this into its own class, maybe called bastion::ansible or something - - vcsrepo { '/opt/ansible': - ensure => latest, - provider => git, - revision => 'devel', - source => 'https://github.com/ansible/ansible', - } - - file { '/etc/ansible/hosts': - ensure => directory, - owner => 'root', - group => 'admin', - mode => '0755', - } - - file { '/etc/ansible/hosts/puppet': - ensure => absent, - } - - file { '/etc/ansible/hosts/openstack': - ensure => absent, - owner => 'root', - group => 'root', - mode => '0755', - require => Vcsrepo['/opt/ansible'], - } - - file { '/etc/ansible/hosts/openstack_inventory': - owner => 'root', - group => 'root', - mode => '0755', - source => '/opt/ansible/contrib/inventory/openstack_inventory.py', - replace => true, - require => [ - File['/etc/ansible/hosts/openstack'], - Vcsrepo['/opt/ansible'], - ], - } - - file { '/etc/ansible/hosts/static': - ensure => absent, - } - - file { '/etc/ansible/hosts/emergency': - ensure => present, - owner => 'root', - group => 'admin', - mode => '0664', - } - - file { '/etc/ansible/hosts/generated-groups': - ensure => present, - owner => 'root', - group => 'admin', - mode => '0664', - } - - file { '/etc/ansible/groups.txt': - owner => 'root', - group => 'root', - mode => '0444', - source => 'puppet:///modules/openstack_project/puppetmaster/groups.txt', - notify => Exec['expand_groups'], - } - - file { '/var/cache/ansible-inventory': - ensure => directory, - owner => 'root', - group => 'admin', - mode => '2775', - } - - file { '/var/cache/ansible-inventory/ansible-inventory.cache': - ensure => present, - owner => 'root', - group => 'admin', - mode => '0664', - } - - file { '/usr/local/bin/expand-groups.sh': - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/puppetmaster/expand-groups.sh', - notify => Exec['expand_groups'], - } - - cron { 'expandgroups': - ensure => absent, - } - - logrotate::file { 'expandgroups': - ensure => present, - log => '/var/log/expand_groups.log', - options => ['compress', - 'copytruncate', - 'delaycompress', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - require => Cron['expandgroups'], - } - - # Temporarily pin paho-mqtt to 1.2.3 since 1.3.0 won't support TLS on - # Trusty's Python 2.7. - if $enable_mqtt { - package {'paho-mqtt': - ensure => '1.2.3', - provider => openstack_pip, - require => Class['pip'], - } - - file { '/etc/mqtt_ca_cert.pem.crt': - ensure => present, - content => $mqtt_ca_cert_contents, - replace => true, - owner => 'root', - group => 'admin', - mode => '0555', - } - - file { '/etc/mqtt_client.yaml': - owner => 'root', - group => 'admin', - mode => '0664', - content => template('openstack_project/puppetmaster/mqtt_client.yaml.erb'), - } - - file { '/opt/ansible/lib/ansible/plugins/callback/mqtt.py': - ensure => absent, - } - - file { '/etc/ansible/callback_plugins/mqtt.py': - owner => 'root', - group => 'admin', - mode => '0664', - source => 'puppet:///modules/openstack_project/puppetmaster/mqtt.py', - require => File['/etc/ansible/callback_plugins'], - } - } - - exec { 'expand_groups': - command => 'expand-groups.sh', - path => '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - refreshonly => true, - } - # Certificate Authority for zuul services. file { '/etc/zuul-ca': ensure => directory,