From 43900bc8b64bf6b41f1b600a7ceb3f7c0a88e2f1 Mon Sep 17 00:00:00 2001 From: Mathieu Rohon Date: Fri, 28 Apr 2017 12:21:50 +0200 Subject: [PATCH] Add skydive service This patch introduces the ansible materials to deploy the skydive service, that can be used to monitor and troubleshoot networking in an openstack deployment. Implements: blueprint skydive-service Co-Authored-By: Nicolas Bouron Signed-off-by: Mathieu Rohon Change-Id: I53051a1b0c85380416288e17040a398b6efb62c0 --- ansible/group_vars/all.yml | 7 ++- ansible/inventory/all-in-one | 12 ++++ ansible/inventory/multinode | 11 ++++ .../roles/haproxy/templates/haproxy.cfg.j2 | 17 +++++ ansible/roles/skydive/defaults/main.yml | 35 +++++++++++ ansible/roles/skydive/handlers/main.yml | 43 +++++++++++++ ansible/roles/skydive/meta/main.yml | 3 + ansible/roles/skydive/tasks/config.yml | 53 ++++++++++++++++ ansible/roles/skydive/tasks/deploy.yml | 7 +++ ansible/roles/skydive/tasks/main.yml | 2 + ansible/roles/skydive/tasks/precheck.yml | 34 ++++++++++ ansible/roles/skydive/tasks/pull.yml | 10 +++ ansible/roles/skydive/tasks/reconfigure.yml | 2 + ansible/roles/skydive/tasks/upgrade.yml | 5 ++ .../skydive/templates/skydive-agent.conf.j2 | 54 ++++++++++++++++ .../skydive/templates/skydive-agent.json.j2 | 18 ++++++ .../templates/skydive-analyzer.conf.j2 | 62 +++++++++++++++++++ .../templates/skydive-analyzer.json.j2 | 18 ++++++ ansible/site.yml | 11 ++++ etc/kolla/globals.yml | 1 + 20 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/skydive/defaults/main.yml create mode 100644 ansible/roles/skydive/handlers/main.yml create mode 100644 ansible/roles/skydive/meta/main.yml create mode 100644 ansible/roles/skydive/tasks/config.yml create mode 100644 ansible/roles/skydive/tasks/deploy.yml create mode 100644 ansible/roles/skydive/tasks/main.yml create mode 100644 ansible/roles/skydive/tasks/precheck.yml create mode 100644 ansible/roles/skydive/tasks/pull.yml create mode 100644 ansible/roles/skydive/tasks/reconfigure.yml create mode 100644 ansible/roles/skydive/tasks/upgrade.yml create mode 100644 ansible/roles/skydive/templates/skydive-agent.conf.j2 create mode 100644 ansible/roles/skydive/templates/skydive-agent.json.j2 create mode 100644 ansible/roles/skydive/templates/skydive-analyzer.conf.j2 create mode 100644 ansible/roles/skydive/templates/skydive-analyzer.json.j2 diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 896eb9de13..736678dfb8 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -249,6 +249,10 @@ zun_api_port: "9517" ovsdb_port: "6640" +skydive_analyzer_port: "8085" +skydive_agents_port: "8090" + + public_protocol: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}" internal_protocol: "http" admin_protocol: "http" @@ -362,6 +366,7 @@ enable_rally: "no" enable_sahara: "no" enable_searchlight: "no" enable_senlin: "no" +enable_skydive: "no" enable_solum: "no" enable_swift: "no" enable_tacker: "no" @@ -394,7 +399,7 @@ enable_destroy_images: "no" elasticsearch_address: "{{ kolla_internal_vip_address }}" elasticsearch_protocol: "{{ internal_protocol }}" -enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool else 'no' }}" +enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool or enable_skydive | bool else 'no' }}" enable_kibana: "{{ 'yes' if enable_central_logging | bool else 'no' }}" #################### diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index f0450c5cad..6c089fbcde 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -192,6 +192,9 @@ deployment [zun:children] control +[skydive:children] +monitoring + # Additional control implemented here. These groups allow you to control which # services run on which hosts at a per-service level. # @@ -542,3 +545,12 @@ zun [zun-compute:children] compute + +# Skydive +[skydive-analyzer:children] +skydive + +[skydive-agent:children] +compute +network + diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 7775bdcb68..c21426d505 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -210,6 +210,9 @@ deployment [zun:children] control +[skydive:children] +monitoring + # Additional control implemented here. These groups allow you to control which # services run on which hosts at a per-service level. # @@ -560,3 +563,11 @@ zun [zun-compute:children] compute + +# Skydive +[skydive-analyzer:children] +skydive + +[skydive-agent:children] +compute +network diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2 index f3aeff0a60..b6d62bd21b 100644 --- a/ansible/roles/haproxy/templates/haproxy.cfg.j2 +++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2 @@ -822,6 +822,22 @@ listen zun_api_external {% endif %} {% endif %} +{% if enable_skydive | bool %} +listen skydive_server + bind {{ kolla_internal_vip_address }}:{{ skydive_analyzer_port }} +{% for host in groups['skydive-analyzer'] %} + server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ skydive_analyzer_port }} check inter 2000 rise 2 fall 5 +{% endfor %} +{% if haproxy_enable_external_vip | bool %} + +listen skydive_server_external + bind {{ kolla_external_vip_address }}:{{ skydive_analyzer_port }} {{ tls_bind_info }} +{% for host in groups['skydive-analyzer'] %} + server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ skydive_analyzer_port }} check inter 2000 rise 2 fall 5 +{% endfor %} +{% endif %} +{% endif %} + # (NOTE): This defaults section deletes forwardfor as recommended by: # https://marc.info/?l=haproxy&m=141684110710132&w=1 @@ -852,3 +868,4 @@ listen mariadb {% endfor %} {% endif %} + diff --git a/ansible/roles/skydive/defaults/main.yml b/ansible/roles/skydive/defaults/main.yml new file mode 100644 index 0000000000..3d9b96d254 --- /dev/null +++ b/ansible/roles/skydive/defaults/main.yml @@ -0,0 +1,35 @@ +--- +project_name: "skydive" + +skydive_services: + skydive-analyzer: + container_name: skydive_analyzer + group: skydive-analyzer + enabled: true + image: "{{ skydive_analyzer_image_full }}" + volumes: + - "{{ node_config_directory }}/skydive-analyzer/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla/" + skydive-agent: + container_name: skydive_agent + group: skydive-agent + enabled: true + image: "{{ skydive_agent_image_full }}" + volumes: + - "{{ node_config_directory }}/skydive-agent/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "/var/run/openvswitch/db.sock:/var/run/openvswitch/db.sock:ro" + - "kolla_logs:/var/log/kolla/" + +#################### +# Docker +#################### +skydive_analyzer_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-skydive-analyzer" +skydive_analyzer_tag: "{{ openstack_release }}" +skydive_analyzer_image_full: "{{ skydive_analyzer_image }}:{{ skydive_analyzer_tag }}" + +skydive_agent_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-skydive-agent" +skydive_agent_tag: "{{ openstack_release }}" +skydive_agent_image_full: "{{ skydive_agent_image }}:{{ skydive_agent_tag }}" + diff --git a/ansible/roles/skydive/handlers/main.yml b/ansible/roles/skydive/handlers/main.yml new file mode 100644 index 0000000000..af6386e231 --- /dev/null +++ b/ansible/roles/skydive/handlers/main.yml @@ -0,0 +1,43 @@ +--- +- name: Restart skydive-analyzer container + vars: + service_name: "skydive-analyzer" + service: "{{ skydive_services[service_name] }}" + config_json: "{{ skydive_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + skydive_conf: "{{ skydive_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" + skydive_analyzer_container: "{{ check_skydive_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or skydive_conf.changed |bool + or skydive_analyzer_container.changed | bool + +- name: Restart skydive-agent container + vars: + service_name: "skydive-agent" + service: "{{ skydive_services[service_name] }}" + config_json: "{{ skydive_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + skydive_conf: "{{ skydive_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" + skydive_agent_container: "{{ check_skydive_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or skydive_conf.changed |bool + or skydive_agent_container.changed | bool + diff --git a/ansible/roles/skydive/meta/main.yml b/ansible/roles/skydive/meta/main.yml new file mode 100644 index 0000000000..6b4fff8fef --- /dev/null +++ b/ansible/roles/skydive/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: common } diff --git a/ansible/roles/skydive/tasks/config.yml b/ansible/roles/skydive/tasks/config.yml new file mode 100644 index 0000000000..0ebcce2ba4 --- /dev/null +++ b/ansible/roles/skydive/tasks/config.yml @@ -0,0 +1,53 @@ +--- +- name: Ensuring config directories exist + file: + path: "{{ node_config_directory }}/{{ item.key }}/config" + state: "directory" + recurse: yes + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ skydive_services }}" + +- name: Copying over default config.json files + template: + src: "{{ item.key }}.json.j2" + dest: "{{ node_config_directory }}/{{ item.key }}/config.json" + register: skydive_config_jsons + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ skydive_services }}" + notify: + - Restart skydive-analyzer container + - Restart skydive-agent container + +- name: Copying over skydive config file + template: + src: "{{ item.key }}.conf.j2" + dest: "{{ node_config_directory }}/{{ item.key }}/skydive.conf" + register: skydive_confs + when: + - item.value.enabled | bool + - inventory_hostname in groups[item.value.group] + with_dict: "{{ skydive_services }}" + notify: + - Restart skydive-analyzer container + - Restart skydive-agent container + +- name: Check skydive containers + kolla_docker: + action: "compare_container" + common_options: "{{ docker_common_options }}" + name: "{{ item.value.container_name }}" + image: "{{ item.value.image }}" + volumes: "{{ item.value.volumes }}" + register: check_skydive_containers + when: + - action != "config" + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ skydive_services }}" + notify: + - Restart skydive-analyzer container + - Restart skydive-agent container diff --git a/ansible/roles/skydive/tasks/deploy.yml b/ansible/roles/skydive/tasks/deploy.yml new file mode 100644 index 0000000000..17e717a8b0 --- /dev/null +++ b/ansible/roles/skydive/tasks/deploy.yml @@ -0,0 +1,7 @@ +--- +- include: config.yml + when: inventory_hostname in groups['skydive-analyzer'] or + inventory_hostname in groups['skydive-agent'] + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/skydive/tasks/main.yml b/ansible/roles/skydive/tasks/main.yml new file mode 100644 index 0000000000..b017e8b4ad --- /dev/null +++ b/ansible/roles/skydive/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ action }}.yml" diff --git a/ansible/roles/skydive/tasks/precheck.yml b/ansible/roles/skydive/tasks/precheck.yml new file mode 100644 index 0000000000..e67c612c6a --- /dev/null +++ b/ansible/roles/skydive/tasks/precheck.yml @@ -0,0 +1,34 @@ +--- +- name: Get container facts + kolla_container_facts: + name: + - skydive_analyzer + - skydive_agent + register: container_facts + +- name: Checking free port for Skydive Analyzer + vars: + skydive_analyzer: "{{ skydive_services['skydive-analyzer'] }}" + wait_for: + host: "{{ api_interface_address }}" + port: "{{ skydive_analyzer_port }}" + connect_timeout: 1 + state: stopped + when: + - container_facts['skydive_analyzer'] is not defined + - inventory_hostname in groups[skydive_analyzer.group] + - skydive_analyzer.enabled | bool + +- name: Checking free port for Skydive Agent + vars: + skydive_agent: "{{ skydive_services['skydive-agent'] }}" + wait_for: + host: "{{ api_interface_address }}" + port: "{{ skydive_agents_port }}" + connect_timeout: 1 + state: stopped + when: + - container_facts['skydive_agent'] is not defined + - inventory_hostname in groups[skydive_agent.group] + - skydive_agent.enabled | bool + diff --git a/ansible/roles/skydive/tasks/pull.yml b/ansible/roles/skydive/tasks/pull.yml new file mode 100644 index 0000000000..c2db429c78 --- /dev/null +++ b/ansible/roles/skydive/tasks/pull.yml @@ -0,0 +1,10 @@ +--- +- name: Pulling skydive images + kolla_docker: + action: "pull_image" + common_options: "{{ docker_common_options }}" + image: "{{ item.value.image }}" + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ skydive_services }}" diff --git a/ansible/roles/skydive/tasks/reconfigure.yml b/ansible/roles/skydive/tasks/reconfigure.yml new file mode 100644 index 0000000000..56169f9727 --- /dev/null +++ b/ansible/roles/skydive/tasks/reconfigure.yml @@ -0,0 +1,2 @@ +--- +- include: "deploy.yml" diff --git a/ansible/roles/skydive/tasks/upgrade.yml b/ansible/roles/skydive/tasks/upgrade.yml new file mode 100644 index 0000000000..dd26ecc34d --- /dev/null +++ b/ansible/roles/skydive/tasks/upgrade.yml @@ -0,0 +1,5 @@ +--- +- include: config.yml + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/skydive/templates/skydive-agent.conf.j2 b/ansible/roles/skydive/templates/skydive-agent.conf.j2 new file mode 100644 index 0000000000..8985b11b68 --- /dev/null +++ b/ansible/roles/skydive/templates/skydive-agent.conf.j2 @@ -0,0 +1,54 @@ +### Skydive agent config file + +auth: + type: keystone + analyzer_username: {{ openstack_auth['username'] }} + analyzer_password: {{ openstack_auth['password'] }} + +logging: + default: DEBUG + topology/probes: INFO + topology/graph: INFO + +openstack: + auth_url: {{ keystone_public_url }}/v3 + username: {{ openstack_auth['username'] }} + password: {{ openstack_auth['password'] }} + tenant_name: {{ openstack_auth['project_name'] }} + region_name: {{ openstack_region_name }} + domain_name: Default + +etcd: + servers: +{% if enable_etcd == "yes" %} +{% for host in groups['etcd'] %} + - http://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ etcd_client_port }} +{% endfor %} +{% else %} +{% for host in groups['skydive-analyzer'] %} + - http://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ etcd_client_port }} +{% endfor %} +{% endif %} + +analyzers: +{% for host in groups['skydive-analyzer'] %} + - {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ skydive_analyzer_port }} +{% endfor %} + +agent: + listen: {{ hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname]['api_interface']]['ipv4']['address'] }}:{{ skydive_agents_port }} + flow: + probes: + - gopacket +{% if neutron_plugin_agent == "openvswitch" %} + - ovssflow +{% endif %} + topology: + probes: + - netlink + - netns + - neutron +{% if neutron_plugin_agent == "openvswitch" %} + - ovsdb +{% endif %} + diff --git a/ansible/roles/skydive/templates/skydive-agent.json.j2 b/ansible/roles/skydive/templates/skydive-agent.json.j2 new file mode 100644 index 0000000000..1d4a0f4fea --- /dev/null +++ b/ansible/roles/skydive/templates/skydive-agent.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "skydive agent --conf /etc/skydive/skydive.conf --listen={{ api_interface_address }}:{{ skydive_agents_port }}", + "config_files": [ + { + "source": "{{ container_config_directory }}/skydive.conf", + "dest": "/etc/skydive/skydive.conf", + "owner": "skydive", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/skydive", + "owner": "skydive:skydive", + "recurse": true + } + ] +} diff --git a/ansible/roles/skydive/templates/skydive-analyzer.conf.j2 b/ansible/roles/skydive/templates/skydive-analyzer.conf.j2 new file mode 100644 index 0000000000..7cc7bd161d --- /dev/null +++ b/ansible/roles/skydive/templates/skydive-analyzer.conf.j2 @@ -0,0 +1,62 @@ +### Skydive analyzer config file + +auth: + type: keystone + +logging: + default: DEBUG + topology/probes: INFO + topology/graph: INFO + +openstack: + auth_url: {{ keystone_public_url }}/v3 + tenant_name: {{ openstack_auth['project_name'] }} + region_name: {{ openstack_region_name }} + domain_name: Default + +etcd: + client_timeout: 100 +{% if enable_etcd == "yes" %} + embeded: no + servers: +{% for host in groups['etcd'] %} + - http://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ etcd_client_port }} +{% endfor %} +{% else %} + embeded: yes + servers: +{% for host in groups['skydive-analyzer'] %} + - http://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ etcd_client_port }} +{% endfor %} + listen: {{ api_interface_address }}:{{ etcd_client_port }} +{% endif %} + +analyzer: + listen: {{ api_interface_address }}:{{ skydive_analyzer_port }} + flowtable_expire: 60 + flowtable_update: 5 + flowtable_agent_ratio: 0.5 + storage: + backend: elasticsearch +{% if groups['skydive-agent'] | length > 1 %} + topology: + fabric: +{% for interface in [network_interface, neutron_external_interface]|unique %} +{% set interfaces_loop = loop %} +{% for host in groups['skydive-agent'] %} + - TOR{{ interfaces_loop.index }}[Name=tor{{ interfaces_loop.index }}] -> TOR{{ interfaces_loop.index }}_PORT{{ loop.index }}[Name=port{{ loop.index }}, MTU=1500] + - TOR{{ interfaces_loop.index }}_PORT{{ loop.index }} -> *[Type=host,Name={{ hostvars[host]['ansible_hostname'] }}]/{{ interface }} +{% endfor %} +{% endfor %} +{% endif %} + +storage: + elasticsearch: + host: {{ elasticsearch_address }}:{{ elasticsearch_port }} + maxconns: 10 + retry: 60 + +graph: + embeded: true + client_timeout: 100 + diff --git a/ansible/roles/skydive/templates/skydive-analyzer.json.j2 b/ansible/roles/skydive/templates/skydive-analyzer.json.j2 new file mode 100644 index 0000000000..b773a272c9 --- /dev/null +++ b/ansible/roles/skydive/templates/skydive-analyzer.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "skydive analyzer --conf /etc/skydive/skydive.conf --listen={{ api_interface_address }}:{{ skydive_analyzer_port }}", + "config_files": [ + { + "source": "{{ container_config_directory }}/skydive.conf", + "dest": "/etc/skydive/skydive.conf", + "owner": "skydive", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/skydive", + "owner": "skydive:skydive", + "recurse": true + } + ] +} diff --git a/ansible/site.yml b/ansible/site.yml index ec2d43156b..f3c162efa2 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -634,3 +634,14 @@ - { role: zun, tags: zun, when: enable_zun | bool } + +- name: Apply role skydive + hosts: + - skydive-agent + - skydive-analyzer + serial: '{{ serial|default("0") }}' + roles: + - { role: skydive, + tags: skydive, + when: enable_skydive | bool } + diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index e699e8efd0..6f952b6c92 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -186,6 +186,7 @@ kolla_internal_vip_address: "10.10.10.254" #enable_sahara: "no" #enable_searchlight: "no" #enable_senlin: "no" +#enable_skydive: "no" #enable_solum: "no" #enable_swift: "no" #enable_telegraf: "no"