Add missing configuration files for ceilometer

Added pipeline.yaml, event_pipeline.yaml and event_definitions.yaml
based on sample files in OpenStack documentation

Edited haproxy.cfg for ceilometer support

Edited ceilometer-base dockerfile for missing dependency

Change-Id: I6ade05255e7e1aa7dbcffd026fad5869036d0d32
Closes-Bug: #1604004

ansible/roles/ceilometer/tasks/config.yml

@@ -39,3 +39,12 @@
     - "ceilometer-api"
     - "ceilometer-central"
     - "ceilometer-compute"
+
+- name: Copying over event and pipeline yaml for notification service
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ node_config_directory }}/ceilometer-notification/{{ item }}"
+  with_items:
+    - "event_definitions.yaml"
+    - "event_pipeline.yaml"
+    - "pipeline.yaml"

ansible/roles/ceilometer/templates/ceilometer-notification.json.j2

@@ -6,6 +6,24 @@
             "dest": "/etc/ceilometer/ceilometer.conf",
             "owner": "ceilometer",
             "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/event_definitions.yaml",
+            "dest": "/etc/ceilometer/event_definitions.yaml",
+            "owner": "ceilometer",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/event_pipeline.yaml",
+            "dest": "/etc/ceilometer/event_pipeline.yaml",
+            "owner": "ceilometer",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/pipeline.yaml",
+            "dest": "/etc/ceilometer/pipeline.yaml",
+            "owner": "ceilometer",
+            "perm": "0600"
         }
     ]
 }

ansible/roles/ceilometer/templates/event_definitions.yaml.j2

@@ -0,0 +1,553 @@
+---
+- event_type: compute.instance.*
+  traits: &instance_traits
+    tenant_id:
+      fields: payload.tenant_id
+    user_id:
+      fields: payload.user_id
+    instance_id:
+      fields: payload.instance_id
+    host:
+      fields: publisher_id.`split(., 1, 1)`
+    service:
+      fields: publisher_id.`split(., 0, -1)`
+    memory_mb:
+      type: int
+      fields: payload.memory_mb
+    disk_gb:
+      type: int
+      fields: payload.disk_gb
+    root_gb:
+      type: int
+      fields: payload.root_gb
+    ephemeral_gb:
+      type: int
+      fields: payload.ephemeral_gb
+    vcpus:
+      type: int
+      fields: payload.vcpus
+    instance_type_id:
+      type: int
+      fields: payload.instance_type_id
+    instance_type:
+      fields: payload.instance_type
+    state:
+      fields: payload.state
+    os_architecture:
+      fields: payload.image_meta.'org.openstack__1__architecture'
+    os_version:
+      fields: payload.image_meta.'org.openstack__1__os_version'
+    os_distro:
+      fields: payload.image_meta.'org.openstack__1__os_distro'
+    launched_at:
+      type: datetime
+      fields: payload.launched_at
+    deleted_at:
+      type: datetime
+      fields: payload.deleted_at
+- event_type: compute.instance.exists
+  traits:
+    <<: *instance_traits
+    audit_period_beginning:
+      type: datetime
+      fields: payload.audit_period_beginning
+    audit_period_ending:
+      type: datetime
+      fields: payload.audit_period_ending
+- event_type: ['volume.exists', 'volume.create.*', 'volume.delete.*', 'volume.resize.*', 'volume.attach.*', 'volume.detach.*', 'volume.update.*', 'snapshot.exists', 'snapshot.create.*', 'snapshot.delete.*', 'snapshot.update.*']
+  traits: &cinder_traits
+    user_id:
+      fields: payload.user_id
+    project_id:
+      fields: payload.tenant_id
+    availability_zone:
+      fields: payload.availability_zone
+    display_name:
+      fields: payload.display_name
+    replication_status:
+      fields: payload.replication_status
+    status:
+      fields: payload.status
+    created_at:
+      fields: payload.created_at
+- event_type: ['volume.exists', 'volume.create.*', 'volume.delete.*', 'volume.resize.*', 'volume.attach.*', 'volume.detach.*', 'volume.update.*']
+  traits:
+    <<: *cinder_traits
+    resource_id:
+      fields: payload.volume_id
+    host:
+      fields: payload.host
+    size:
+      fields: payload.size
+    type:
+      fields: payload.volume_type
+    replication_status:
+      fields: payload.replication_status
+- event_type: ['snapshot.exists', 'snapshot.create.*', 'snapshot.delete.*', 'snapshot.update.*']
+  traits:
+    <<: *cinder_traits
+    resource_id:
+      fields: payload.snapshot_id
+    volume_id:
+      fields: payload.volume_id
+- event_type: ['image_volume_cache.*']
+  traits:
+    image_id:
+      fields: payload.image_id
+    host:
+      fields: payload.host
+- event_type: ['image.update', 'image.upload', 'image.delete']
+  traits: &glance_crud
+    project_id:
+      fields: payload.owner
+    resource_id:
+      fields: payload.id
+    name:
+      fields: payload.name
+    status:
+      fields: payload.status
+    created_at:
+      fields: payload.created_at
+    user_id:
+      fields: payload.owner
+    deleted_at:
+      fields: payload.deleted_at
+    size:
+      fields: payload.size
+- event_type: image.send
+  traits: &glance_send
+    receiver_project:
+      fields: payload.receiver_tenant_id
+    receiver_user:
+      fields: payload.receiver_user_id
+    user_id:
+      fields: payload.owner_id
+    image_id:
+      fields: payload.image_id
+    destination_ip:
+      fields: payload.destination_ip
+    bytes_sent:
+      fields: payload.bytes_sent
+- event_type: orchestration.stack.*
+  traits: &orchestration_crud
+    project_id:
+      fields: payload.tenant_id
+    user_id:
+      fields: ['_context_trustor_user_id', '_context_user_id']
+    resource_id:
+      fields: payload.stack_identity
+- event_type: sahara.cluster.*
+  traits: &sahara_crud
+    project_id:
+      fields: payload.project_id
+    user_id:
+      fields: _context_user_id
+    resource_id:
+      fields: payload.cluster_id
+- event_type: sahara.cluster.health
+  traits: &sahara_health
+    <<: *sahara_crud
+    verification_id:
+      fields: payload.verification_id
+    health_check_status:
+      fields: payload.health_check_status
+    health_check_name:
+      fields: payload.health_check_name
+    health_check_description:
+      fields: payload.health_check_description
+    created_at:
+      type: datetime
+      fields: payload.created_at
+    updated_at:
+      type: datetime
+      fields: payload.updated_at
+- event_type: ['identity.user.*', 'identity.project.*', 'identity.group.*', 'identity.role.*', 'identity.OS-TRUST:trust.*',
+               'identity.region.*', 'identity.service.*', 'identity.endpoint.*', 'identity.policy.*']
+  traits: &identity_crud
+    resource_id:
+      fields: payload.resource_info
+    initiator_id:
+      fields: payload.initiator.id
+    project_id:
+      fields: payload.initiator.project_id
+    domain_id:
+      fields: payload.initiator.domain_id
+- event_type: identity.role_assignment.*
+  traits: &identity_role_assignment
+    role:
+      fields: payload.role
+    group:
+      fields: payload.group
+    domain:
+      fields: payload.domain
+    user:
+      fields: payload.user
+    project:
+      fields: payload.project
+- event_type: identity.authenticate
+  traits: &identity_authenticate
+    typeURI:
+      fields: payload.typeURI
+    id:
+      fields: payload.id
+    action:
+      fields: payload.action
+    eventType:
+      fields: payload.eventType
+    eventTime:
+      fields: payload.eventTime
+    outcome:
+      fields: payload.outcome
+    initiator_typeURI:
+      fields: payload.initiator.typeURI
+    initiator_id:
+      fields: payload.initiator.id
+    initiator_name:
+      fields: payload.initiator.name
+    initiator_host_agent:
+      fields: payload.initiator.host.agent
+    initiator_host_addr:
+      fields: payload.initiator.host.address
+    target_typeURI:
+      fields: payload.target.typeURI
+    target_id:
+      fields: payload.target.id
+    observer_typeURI:
+      fields: payload.observer.typeURI
+    observer_id:
+      fields: payload.observer.id
+- event_type: objectstore.http.request
+  traits: &objectstore_request
+    typeURI:
+      fields: payload.typeURI
+    id:
+      fields: payload.id
+    action:
+      fields: payload.action
+    eventType:
+      fields: payload.eventType
+    eventTime:
+      fields: payload.eventTime
+    outcome:
+      fields: payload.outcome
+    initiator_typeURI:
+      fields: payload.initiator.typeURI
+    initiator_id:
+      fields: payload.initiator.id
+    initiator_project_id:
+      fields: payload.initiator.project_id
+    target_typeURI:
+      fields: payload.target.typeURI
+    target_id:
+      fields: payload.target.id
+    target_action:
+      fields: payload.target.action
+    target_metadata_path:
+      fields: payload.target.metadata.path
+    target_metadata_version:
+      fields: payload.target.metadata.version
+    target_metadata_container:
+      fields: payload.target.metadata.container
+    target_metadata_object:
+      fields: payload.target.metadata.object
+    observer_id:
+      fields: payload.observer.id
+- event_type: magnetodb.table.*
+  traits: &kv_store
+    resource_id:
+      fields: payload.table_uuid
+    user_id:
+      fields: _context_user_id
+    project_id:
+      fields: _context_tenant
+- event_type: ['network.*', 'subnet.*', 'port.*', 'router.*', 'floatingip.*', 'pool.*', 'vip.*', 'member.*', 'health_monitor.*', 'healthmonitor.*', 'listener.*', 'loadbalancer.*', 'firewall.*', 'firewall_policy.*', 'firewall_rule.*', 'vpnservice.*', 'ipsecpolicy.*', 'ikepolicy.*', 'ipsec_site_connection.*']
+  traits: &network_traits
+    user_id:
+      fields: _context_user_id
+    project_id:
+      fields: _context_tenant_id
+- event_type: network.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.network.id', 'payload.id']
+- event_type: subnet.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.subnet.id', 'payload.id']
+- event_type: port.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.port.id', 'payload.id']
+- event_type: router.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.router.id', 'payload.id']
+- event_type: floatingip.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.floatingip.id', 'payload.id']
+- event_type: pool.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.pool.id', 'payload.id']
+- event_type: vip.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.vip.id', 'payload.id']
+- event_type: member.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.member.id', 'payload.id']
+- event_type: health_monitor.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.health_monitor.id', 'payload.id']
+- event_type: healthmonitor.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.healthmonitor.id', 'payload.id']
+- event_type: listener.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.listener.id', 'payload.id']
+- event_type: loadbalancer.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.loadbalancer.id', 'payload.id']
+- event_type: firewall.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.firewall.id', 'payload.id']
+- event_type: firewall_policy.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.firewall_policy.id', 'payload.id']
+- event_type: firewall_rule.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.firewall_rule.id', 'payload.id']
+- event_type: vpnservice.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.vpnservice.id', 'payload.id']
+- event_type: ipsecpolicy.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.ipsecpolicy.id', 'payload.id']
+- event_type: ikepolicy.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.ikepolicy.id', 'payload.id']
+- event_type: ipsec_site_connection.*
+  traits:
+    <<: *network_traits
+    resource_id:
+      fields: ['payload.ipsec_site_connection.id', 'payload.id']
+- event_type: '*http.*'
+  traits: &http_audit
+    project_id:
+      fields: payload.initiator.project_id
+    user_id:
+      fields: payload.initiator.id
+    typeURI:
+      fields: payload.typeURI
+    eventType:
+      fields: payload.eventType
+    action:
+      fields: payload.action
+    outcome:
+      fields: payload.outcome
+    id:
+      fields: payload.id
+    eventTime:
+      fields: payload.eventTime
+    requestPath:
+      fields: payload.requestPath
+    observer_id:
+      fields: payload.observer.id
+    target_id:
+      fields: payload.target.id
+    target_typeURI:
+      fields: payload.target.typeURI
+    target_name:
+      fields: payload.target.name
+    initiator_typeURI:
+      fields: payload.initiator.typeURI
+    initiator_id:
+      fields: payload.initiator.id
+    initiator_name:
+      fields: payload.initiator.name
+    initiator_host_address:
+      fields: payload.initiator.host.address
+- event_type: '*http.response'
+  traits:
+    <<: *http_audit
+    reason_code:
+      fields: payload.reason.reasonCode
+- event_type: ['dns.domain.create', 'dns.domain.update', 'dns.domain.delete']
+  traits: &dns_domain_traits
+    status:
+      fields: payload.status
+    retry:
+      fields: payload.retry
+    description:
+      fields: payload.description
+    expire:
+      fields: payload.expire
+    email:
+      fields: payload.email
+    ttl:
+      fields: payload.ttl
+    action:
+      fields: payload.action
+    name:
+      fields: payload.name
+    resource_id:
+      fields: payload.id
+    created_at:
+      fields: payload.created_at
+    updated_at:
+      fields: payload.updated_at
+    version:
+      fields: payload.version
+    parent_domain_id:
+      fields: parent_domain_id
+    serial:
+      fields: payload.serial
+- event_type: dns.domain.exists
+  traits:
+    <<: *dns_domain_traits
+    audit_period_beginning:
+      type: datetime
+      fields: payload.audit_period_beginning
+    audit_period_ending:
+      type: datetime
+      fields: payload.audit_period_ending
+- event_type: trove.*
+  traits: &trove_base_traits
+    state:
+      fields: payload.state_description
+    instance_type:
+      fields: payload.instance_type
+    user_id:
+      fields: payload.user_id
+    resource_id:
+      fields: payload.instance_id
+    instance_type_id:
+      fields: payload.instance_type_id
+    launched_at:
+      type: datetime
+      fields: payload.launched_at
+    instance_name:
+      fields: payload.instance_name
+    state:
+      fields: payload.state
+    nova_instance_id:
+      fields: payload.nova_instance_id
+    service_id:
+      fields: payload.service_id
+    created_at:
+      type: datetime
+      fields: payload.created_at
+    region:
+      fields: payload.region
+- event_type: ['trove.instance.create', 'trove.instance.modify_volume', 'trove.instance.modify_flavor', 'trove.instance.delete']
+  traits: &trove_common_traits
+    name:
+      fields: payload.name
+    availability_zone:
+      fields: payload.availability_zone
+    instance_size:
+      type: int
+      fields: payload.instance_size
+    volume_size:
+      type: int
+      fields: payload.volume_size
+    nova_volume_id:
+      fields: payload.nova_volume_id
+- event_type: trove.instance.create
+  traits:
+    <<: [*trove_base_traits, *trove_common_traits]
+- event_type: trove.instance.modify_volume
+  traits:
+    <<: [*trove_base_traits, *trove_common_traits]
+    old_volume_size:
+      type: int
+      fields: payload.old_volume_size
+    modify_at:
+      type: datetime
+      fields: payload.modify_at
+- event_type: trove.instance.modify_flavor
+  traits:
+    <<: [*trove_base_traits, *trove_common_traits]
+    old_instance_size:
+      type: int
+      fields: payload.old_instance_size
+    modify_at:
+      type: datetime
+      fields: payload.modify_at
+- event_type: trove.instance.delete
+  traits:
+    <<: [*trove_base_traits, *trove_common_traits]
+    deleted_at:
+      type: datetime
+      fields: payload.deleted_at
+- event_type: trove.instance.exists
+  traits:
+    <<: *trove_base_traits
+    display_name:
+      fields: payload.display_name
+    audit_period_beginning:
+      type: datetime
+      fields: payload.audit_period_beginning
+    audit_period_ending:
+      type: datetime
+      fields: payload.audit_period_ending
+- event_type: profiler.*
+  traits:
+    project:
+      fields: payload.project
+    service:
+      fields: payload.service
+    name:
+      fields: payload.name
+    base_id:
+      fields: payload.base_id
+    trace_id:
+      fields: payload.trace_id
+    parent_id:
+      fields: payload.parent_id
+    timestamp:
+      fields: payload.timestamp
+    host:
+      fields: payload.info.host
+    path:
+      fields: payload.info.request.path
+    query:
+      fields: payload.info.request.query
+    method:
+      fields: payload.info.request.method
+    scheme:
+      fields: payload.info.request.scheme
+    db.statement:
+      fields: payload.info.db.statement
+    db.params:
+      fields: payload.info.db.params

ansible/roles/ceilometer/templates/event_pipeline.yaml.j2

@@ -0,0 +1,13 @@
+---
+sources:
+    - name: event_source
+      events:
+          - "*"
+      sinks:
+          - event_sink
+sinks:
+    - name: event_sink
+      transformers:
+      triggers:
+      publishers:
+          - notifier://

ansible/roles/ceilometer/templates/pipeline.yaml.j2

@@ -0,0 +1,92 @@
+---
+sources:
+    - name: meter_source
+      interval: 600
+      meters:
+          - "*"
+      sinks:
+          - meter_sink
+    - name: cpu_source
+      interval: 600
+      meters:
+          - "cpu"
+      sinks:
+          - cpu_sink
+          - cpu_delta_sink
+    - name: disk_source
+      interval: 600
+      meters:
+          - "disk.read.bytes"
+          - "disk.read.requests"
+          - "disk.write.bytes"
+          - "disk.write.requests"
+          - "disk.device.read.bytes"
+          - "disk.device.read.requests"
+          - "disk.device.write.bytes"
+          - "disk.device.write.requests"
+      sinks:
+          - disk_sink
+    - name: network_source
+      interval: 600
+      meters:
+          - "network.incoming.bytes"
+          - "network.incoming.packets"
+          - "network.outgoing.bytes"
+          - "network.outgoing.packets"
+      sinks:
+          - network_sink
+sinks:
+    - name: meter_sink
+      transformers:
+      publishers:
+          - notifier://
+    - name: cpu_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                target:
+                    name: "cpu_util"
+                    unit: "%"
+                    type: "gauge"
+                    scale: "100.0 / (10**9 * (resource_metadata.cpu_number or 1))"
+      publishers:
+          - notifier://
+    - name: cpu_delta_sink
+      transformers:
+          - name: "delta"
+            parameters:
+                target:
+                    name: "cpu.delta"
+                growth_only: True
+      publishers:
+          - notifier://
+    - name: disk_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                    map_from:
+                        name: "(disk\\.device|disk)\\.(read|write)\\.(bytes|requests)"
+                        unit: "(B|request)"
+                target:
+                    map_to:
+                        name: "\\1.\\2.\\3.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          - notifier://
+    - name: network_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                   map_from:
+                       name: "network\\.(incoming|outgoing)\\.(bytes|packets)"
+                       unit: "(B|packet)"
+                target:
+                    map_to:
+                        name: "network.\\1.\\2.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          - notifier://

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -381,6 +381,22 @@ listen elasticsearch
 {% endfor %}
 {% endif %}
 
+{% if enable_ceilometer | bool %}
+listen ceilometer_api
+  bind {{ kolla_internal_vip_address }}:{{ ceilometer_api_port }}
+{% for host in groups['ceilometer-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ ceilometer_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% if haproxy_enable_external_vip | bool %}
+
+listen ceilometer_api_external
+  bind {{ kolla_external_vip_address }}:{{ ceilometer_api_port }} {{ tls_bind_info }}
+{% for host in groups['ceilometer-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ ceilometer_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
+{% endif %}
+
 # (NOTE): This defaults section deletes forwardfor as recomended by:
 #         https://marc.info/?l=haproxy&m=141684110710132&w=1
 

docker/ceilometer/ceilometer-base/Dockerfile.j2

@@ -14,6 +14,7 @@ RUN yum -y install \
 
 RUN apt-get -y install --no-install-recommends \
         ceilometer-common \
+        python-pymongo \
     && apt-get clean
 
     {% endif %}