Merge "Start and stop collectd containers while running workloads"

This commit is contained in:
Zuul 2021-11-23 06:46:22 +00:00 committed by Gerrit Code Review
commit 58ab0b3430
30 changed files with 384 additions and 0 deletions

View File

@ -0,0 +1,8 @@
---
- hosts: localhost
tasks:
- name: Check if graphite_host is empty in group_vars/all.yml
include_vars:
file: "group_vars/all.yml"
name: group_vars
failed_when: group_vars['graphite_host'] is none

View File

@ -0,0 +1,80 @@
---
#
# Playbook to start collectd on undercloud/overcloud
#
# Start on all undercloud/overcloud machines:
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml
#
# Or use tags:
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "undercloud"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "controller"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "networker"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "blockstorage"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "objectstorage"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "cephstorage"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "compute"
#
- hosts: Undercloud
remote_user: "{{ local_remote_user }}"
tasks:
- name: Start Collectd on undercloud
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: undercloud
- hosts: Controller
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on controller
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: controller
- hosts: Networker
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on networker
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: networker
- hosts: BlockStorage
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on blockstorage
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: blockstorage
- hosts: ObjectStorage
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on objectstorage
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: objectstorage
- hosts: CephStorage
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on cephstorage
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: cephstorage
- hosts: Compute
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on compute
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: compute

View File

@ -0,0 +1,122 @@
#
# Playbook to start collectd on undercloud/overcloud
#
# Start collectd on all undercloud/overcloud machines:
# ansible-playbook -i hosts.yaml install/start-collectd-container.yml
#
# Or use tags:
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "undercloud"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "controller"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "networker"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "blockstorage"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "objectstorage"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "cephstorage"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "compute"
#
- hosts: Undercloud
remote_user: "{{ local_remote_user }}"
vars:
config_type: undercloud
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: undercloud
- hosts: Controller
remote_user: "{{ host_remote_user }}"
vars:
config_type: controller
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: controller
- hosts: Networker
remote_user: "{{ host_remote_user }}"
vars:
config_type: networker
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: networker
- hosts: BlockStorage
remote_user: "{{ host_remote_user }}"
vars:
config_type: blockstorage
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: blockstorage
- hosts: ObjectStorage
remote_user: "{{ host_remote_user }}"
vars:
config_type: objectstorage
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: objectstorage
- hosts: CephStorage
remote_user: "{{ host_remote_user }}"
vars:
config_type: cephstorage
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: cephstorage
- hosts: Compute
remote_user: "{{ host_remote_user }}"
vars:
config_type: compute
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: compute

View File

@ -0,0 +1,11 @@
---
- name: set fact collectd_container
import_playbook: pre-collectd.yml
- name: Start containerized collectd (Stein and greater recommended)
import_playbook: start-collectd-container.yml
when: hostvars['undercloud']['collectd_container']
- name: Start collectd installed through RPMs
import_playbook: start-collectd-baremetal.yml
when: not hostvars['undercloud']['collectd_container']

View File

@ -3,10 +3,20 @@ browbeat:
cloud_name: openstack
rerun: 3
rerun_type: iteration
# This option enables starting collectd before running workloads,
# and stopping collectd after running workloads. It should be enabled
# when it is required to store collectd data only when workloads
# are running. Please install collectd by running the command
# "cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml" before
# setting this option to true.
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: true

View File

@ -7,10 +7,20 @@ browbeat:
# complete reruns after all workloads complete
# rerun_type: complete
rerun_type: iteration
# This option enables starting collectd before running workloads,
# and stopping collectd after running workloads. It should be enabled
# when it is required to store collectd data only when workloads
# are running. Please install collectd by running the command
# "cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml" before
# setting this option to true.
start_stop_collectd: false
ansible:
hosts: ansible/hosts.yml
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -182,6 +182,10 @@ def main():
" with browbeat_uuid {}".format(browbeat_uuid))
tools.common_logging(browbeat_uuid, logging_status=True)
if _config['browbeat']['start_stop_collectd'] and tools.check_collectd_config():
_logger.info("Starting collectd")
tools.start_collectd()
_logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads)))
# Iteration rerun_type pushes rerun logic down to the workload itself. This allows the workload
@ -191,6 +195,9 @@ def main():
elif _config["browbeat"]["rerun_type"] == "complete":
# Complete rerun_type, reruns after all workloads have been run.
run_complete(_config, _cli_args, result_dir_ts, _logger, tools)
if _config["browbeat"]["start_stop_collectd"] and tools.check_collectd_config():
_logger.info("Stopping collectd")
tools.stop_collectd()
if terminate:
_logger.info("Browbeat execution halting due to user intervention")
sys.exit(1)

View File

@ -19,6 +19,9 @@ mapping:
type: str
enum: ['iteration', 'complete']
required: True
start_stop_collectd:
type: bool
required: True
ansible:
required: True
type: map
@ -32,6 +35,15 @@ mapping:
logging_playbook:
type: str
required: True
start_collectd_playbook:
type: str
required: True
stop_collectd_playbook:
type: str
required: True
check_collectd_config_playbook:
type: str
required: True
ssh_config:
type: str
required: True

View File

@ -92,6 +92,46 @@ class Tools(object):
self.logger.info("Metadata about cloud has been gathered")
return True
def check_collectd_config(self):
ansible_cmd = \
'ansible-playbook {}' \
.format(self.config['ansible']['check_collectd_config_playbook'])
returncode = self.run_cmd(ansible_cmd)['rc']
if returncode > 0:
self.logger.warning("""graphite_host is empty in all.yml. Please fill it and run the command
(cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml)
in order to install collectd.""")
return False
else:
self.logger.info("Collectd config in all.yml validated")
return True
def start_collectd(self):
ansible_cmd = \
'ansible-playbook -i {} {}' \
.format(self.config['ansible']['hosts'],
self.config['ansible']['start_collectd_playbook'])
returncode = self.run_cmd(ansible_cmd)['rc']
if returncode > 0:
self.logger.warning("Collectd could not be started")
return False
else:
self.logger.info("Collectd started successfully")
return True
def stop_collectd(self):
ansible_cmd = \
'ansible-playbook -i {} {}' \
.format(self.config['ansible']['hosts'],
self.config['ansible']['stop_collectd_playbook'])
returncode = self.run_cmd(ansible_cmd)['rc']
if returncode > 0:
self.logger.warning("Collectd could not be stopped")
return False
else:
self.logger.info("Collectd stopped successfully")
return True
def common_logging(self, browbeat_uuid, logging_status):
os.putenv("ANSIBLE_SSH_ARGS", " -F {}".format(self.config['ansible']['ssh_config']))
ansible_cmd = \

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -4,10 +4,14 @@ browbeat:
cloud_name: openstack
rerun: 3
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -4,10 +4,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -6,10 +6,14 @@ browbeat:
cloud_name: openstack
rerun: 10
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -4,10 +4,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: quickstart
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -10,10 +10,14 @@ browbeat:
cloud_name: openstack
rerun: 10
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -10,10 +10,14 @@ browbeat:
cloud_name: openstack
rerun: 10
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -10,10 +10,14 @@ browbeat:
cloud_name: openstack
rerun: 2
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -10,10 +10,14 @@ browbeat:
cloud_name: openstack
rerun: 10
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -7,10 +7,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -7,10 +7,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -6,10 +6,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -8,10 +8,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -6,10 +6,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -7,10 +7,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -4,10 +4,14 @@ browbeat:
rerun: 1
rerun_type: complete
invalid_flag: invalid
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/site.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: browbeat-test
rerun: 1
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/site.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: browbeat-test
rerun: 1
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/site.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false