Control plane test results 1000 nodes

Change-Id: I3390761ddafd7511ab9b0be2b28d833062ee5588
This commit is contained in:
David Burnazyan 2016-09-27 14:48:12 +03:00
parent 33b23b3362
commit b8f3e1ae57
34 changed files with 4594 additions and 1 deletions

View File

@ -371,3 +371,4 @@ Reports
Test plan execution reports:
* :ref:`openstack_control_plane_performance_report_200_nodes`
* :ref:`openstack_control_plane_performance_report_400_nodes`
* :ref:`openstack_control_plane_performance_report_1000_nodes`

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

View File

@ -0,0 +1,13 @@
{
"admin": {
"password": "password",
"tenant_name": "tenant",
"username": "user"
},
"auth_url": "http://1.2.3.4:5000/v2.0",
"region_name": "RegionOne",
"type": "ExistingCloud",
"endpoint_type": "internal",
"admin_port": 35357,
"https_insecure": true
}

View File

@ -0,0 +1,112 @@
<domain type='kvm' id='9'>
<name>vm2-729036-comp-disk-283</name>
<uuid>2afa875e-609b-487a-96e4-ed453c690d9b</uuid>
<memory unit='KiB'>4194304</memory>
<currentMemory unit='KiB'>4194304</currentMemory>
<vcpu placement='static' cpuset='4-5'>2</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='4'/>
<vcpupin vcpu='1' cpuset='5'/>
</cputune>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
<boot dev='network'/>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<pae/>
</features>
<cpu mode='host-passthrough'>
</cpu>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/usr/bin/kvm-spice</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/vm2-729036-comp-disk-283.qcow2'/>
<target dev='vda' bus='virtio'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</disk>
<controller type='usb' index='0'>
<alias name='usb0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'>
<alias name='pci.0'/>
</controller>
<controller type='pci' index='1' model='pci-bridge'>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</controller>
<controller type='ide' index='0'>
<alias name='ide0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:4c:bf:ef'/>
<source bridge='br-p1p1'/>
<target dev='vnet0'/>
<model type='virtio'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x01' function='0x0' multifunction='on'/>
</interface>
<interface type='bridge'>
<mac address='52:54:00:24:e5:59'/>
<source bridge='br-p1p2'/>
<target dev='vnet1'/>
<model type='virtio'/>
<alias name='net1'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x01' function='0x1'/>
</interface>
<interface type='bridge'>
<mac address='52:54:00:30:0c:bf'/>
<source bridge='br-p4p1'/>
<target dev='vnet2'/>
<model type='virtio'/>
<alias name='net2'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x04' function='0x0' multifunction='on'/>
</interface>
<interface type='bridge'>
<mac address='52:54:00:61:6a:55'/>
<source bridge='br-p4p2'/>
<target dev='vnet3'/>
<model type='virtio'/>
<alias name='net3'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x04' function='0x1'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/3'/>
<target port='0'/>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/3'>
<source path='/dev/pts/3'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='vnc' port='5900' autoport='yes' listen='0.0.0.0'>
<listen type='address' address='0.0.0.0'/>
</graphics>
<video>
<model type='cirrus' vram='9216' heads='1'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</video>
<memballoon model='virtio'>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</memballoon>
</devices>
<seclabel type='none'/>
</domain>

View File

@ -0,0 +1,5 @@
---
concurrency: 5
compute: 1000
gre_enabled: false
http_server_with_glance_images: "1.2.3.4"

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,14 @@
rally --plugin-paths /home/rally/plugins task start \
keystone.json \
--task-args '{"gre_enabled": true, "compute": 375, "concurrency": 5}' \
--out=keystone.html
rally --plugin-paths /home/rally/plugins task start \
boot_attach_live_migrate_and_delete_server_with_secgroups.json \
--task-args '{"gre_enabled": true, "compute": 375, "concurrency": 5}' \
--out=boot_attach_live_migrate_and_delete_server_with_secgroups.html
rally --plugin-paths /home/rally/plugins task start \
create-and-delete-image.json \
--task-args '{"gre_enabled": true, "compute": 375, "concurrency": 5, "http_server_with_glance_images": "1.2.3.4"}' \
--out=create-and-delete-image.html

View File

@ -0,0 +1,327 @@
.. _openstack_control_plane_performance_report_1000_nodes:
*********************************************************
OpenStack control plane performance report for 1000 nodes
*********************************************************
:Abstract:
This document includes OpenStack control plane performance test results for
1000 compute nodes cluster.
All tests have been performed regarding
:ref:`openstack_control_plane_performance_test_plan`
Environment description
=======================
Environment contains 4 types of servers:
- rally node
- controller node
- osd node
- hypervisor
- compute node
.. table:: Amount of servers each role
+------------+--------------+------------+
|Role |Servers count |Server Type |
+============+==============+============+
|rally |1 |bare-metal |
+------------+--------------+------------+
|controller |3 |bare-metal |
+------------+--------------+------------+
|osd |20 |bare-metal |
+------------+--------------+------------+
|compute |1000 |virtual |
+------------+--------------+------------+
Hardware configuration of each server
-------------------------------------
All bare-metal servers have the same configuration describing in table below
.. table:: Description of servers hardware
+-------+----------------+-------------------------------+
|server |vendor,model |HP,DL380 Gen9 |
+-------+----------------+-------------------------------+
|CPU |vendor,model |Intel,E5-2680 v3 |
| +----------------+-------------------------------+
| |processor_count |2 |
| +----------------+-------------------------------+
| |core_count |12 |
| +----------------+-------------------------------+
| |frequency_MHz |2500 |
+-------+----------------+-------------------------------+
|RAM |vendor,model |HP,752369-081 |
| +----------------+-------------------------------+
| |amount_MB |262144 |
+-------+----------------+-------------------------------+
|NETWORK|interface_name |p1p1 |
| +----------------+-------------------------------+
| |vendor,model |Intel,X710 Dual Port |
| +----------------+-------------------------------+
| |bandwidth |10G |
+-------+----------------+-------------------------------+
|STORAGE|dev_name |/dev/sda |
| +----------------+-------------------------------+
| |vendor,model | | raid10 - HP P840 |
| | | | 12 disks EH0600JEDHE |
| +----------------+-------------------------------+
| |SSD/HDD |HDD |
| +----------------+-------------------------------+
| |size | 3,6TB |
+-------+----------------+-------------------------------+
All virtual servers has the same configuration. QEMU-KVM used as hypervisor.
For datailed info about VM configuration look into sample
:download:`domain.xml <configs/domain.xml>`.
Network configuration of each server
------------------------------------
Lab network configuration you can find in scheme below:
.. image:: configs/Network_Scheme.png
:alt: Network Scheme of the environment
Here is the part of switch configuration for each switch port which connected to
p1p1 interface of a bare-metal server:
.. code:: bash
switchport mode trunk
switchport trunk native vlan 600
switchport trunk allowed vlan 600-602,630-649
spanning-tree port type edge trunk
spanning-tree bpduguard enable
no snmp trap link-status
Software configuration on servers with controller, osd and compute roles
------------------------------------------------------------------------
.. table:: Services on servers by role
+------------+----------------------------+
|Role |Service name |
+============+============================+
|controller || horizon |
| || keystone |
| || nova-api |
| || nava-scheduler |
| || nova-cert |
| || nova-conductor |
| || nova-consoleauth |
| || nova-consoleproxy |
| || cinder-api |
| || cinder-backup |
| || cinder-scheduler |
| || cinder-volume |
| || glance-api |
| || glance-glare |
| || glance-registry |
| || neutron-dhcp-agent |
| || neutron-l3-agent |
| || neutron-metadata-agent |
| || neutron-openvswitch-agent |
| || neutron-server |
| || heat-api |
| || heat-api-cfn |
| || heat-api-cloudwatch |
| || ceph-mon |
| || rados-gw |
| || memcached |
| || rabbitmq_server |
| || mysqld |
| || galera |
| || corosync |
| || pacemaker |
| || haproxy |
+------------+----------------------------+
|osd || ceph-osd |
+------------+----------------------------+
|compute || nova-compute |
| || neutron-l3-agent |
| || neutron-metadata-agent |
| || neutron-openvswitch-agent |
+------------+----------------------------+
.. table:: Software version on servers with controller, compute and compute-osd roles
+------------+-------------------+
|Software |Version |
+============+===================+
|OpenStack |Mitaka |
+------------+-------------------+
|Ceph |Hammer |
+------------+-------------------+
|Ubuntu |Ubuntu 14.04.3 LTS |
+------------+-------------------+
You can find outputs of some commands and /etc folder in the following archives:
| :download:`controller-1.tar.gz <configs/controller-1.tar.gz>`
| :download:`controller-2.tar.gz <configs/controller-2.tar.gz>`
| :download:`controller-3.tar.gz <configs/controller-3.tar.gz>`
| :download:`compute-1.tar.gz <configs/compute-1.tar.gz>`
| :download:`osd-1.tar.gz <configs/osd-1.tar.gz>`
Software configuration on servers with rally role
-------------------------------------------------
On this server should be installed Rally. How to do it you can find in
`Rally installation documentation`_
.. table:: Software version on server with rally role
+------------+-------------------+
|Software |Version |
+============+===================+
|Rally |0.4.0 |
+------------+-------------------+
|Ubuntu |Ubuntu 14.04.3 LTS |
+------------+-------------------+
Testing process
===============
.. table:: Some test parameters
+--------------------------------+--------+
|Name |Value |
+================================+========+
|Volume size to create in Cinder |1GB |
+--------------------------------+--------+
|Flavor to create VM from |m1.tiny |
+--------------------------------+--------+
|Image name to create VM from |cirros |
+--------------------------------+--------+
1. Create work directory on server with Rally role. In future we will call that directory as WORK_DIR
2. Create directory "plugins" in WORK_DIR and copy to that directory
:download:`nova_performance.py <../../test_plans/control_plane/plugins/nova_performance.py>` plugin.
3. Create directory "scenarios" in WORK_DIR and copy to that directory
:download:`boot_attach_live_migrate_and_delete_server_with_secgroups.json
<rally_scenarios/boot_attach_live_migrate_and_delete_server_with_secgroups.json>`,
:download:`create-and-delete-image.json <rally_scenarios/create-and-delete-image.json>`
and :download:`keystone.json <rally_scenarios/keystone.json>` scenarios.
4. Create deployment.json file in WORK_DIR and fill it with OpenStack environment info.
It should looks like this:
.. literalinclude:: configs/deployment.json
:language: bash
5. Create job-params.yaml file in WORK_DIR and fill it with scenarios info.
It should looks like this:
.. literalinclude:: configs/job-params.yaml
:language: bash
6. Perform tests:
.. literalinclude:: configs/run_test_script.sh
:language: bash
As a result of this part we got the following HTML file:
:download:`rally_report.html <configs/rally_report.html>`
Test results
============
All values in tables below are in seconds.
Cinder
------
+---------------+---------+----------+----------+---------+----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+===============+=========+==========+==========+=========+==========+
| create_volume | 2.58376 | 2.71106 | 2.56 | 3.03855 | 2.40338 |
+---------------+---------+----------+----------+---------+----------+
| delete_volume | 2.31383 | 2.4265 | 2.30166 | 2.65687 | 0.432665 |
+---------------+---------+----------+----------+---------+----------+
Glance
------
+--------------+----------+----------+----------+---------+-----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+==============+==========+==========+==========+=========+===========+
| create_image | 44.8271 | 60.351 | 42.9408 | 79.4237 | 21.3831 |
+--------------+----------+----------+----------+---------+-----------+
| delete_image | 1.53144 | 1.98743 | 1.54234 | 2.6857 | 0.601818 |
+--------------+----------+----------+----------+---------+-----------+
Keystone
--------
+-------------+-----------+----------+----------+----------+----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+=============+===========+==========+==========+==========+==========+
| keystone | 0.0758036 | 0.102582 | 0.069742 | 0.460225 | 0.048439 |
+-------------+-----------+----------+----------+----------+----------+
Neutron
-------
+--------------------------+----------+----------+----------+----------+----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+==========================+==========+==========+==========+==========+==========+
| create_20_rules | 4.40657 | 4.69323 | 4.38976 | 6.1356 | 3.72487 |
+--------------------------+----------+----------+----------+----------+----------+
| create_2_security_groups | 0.3527 | 0.402639 | 0.343685 | 1.07336 | 0.25683 |
+--------------------------+----------+----------+----------+----------+----------+
| delete_2_security_groups | 0.314032 | 0.395924 | 0.306359 | 0.635733 | 0.187815 |
+--------------------------+----------+----------+----------+----------+----------+
Nova
----
+----------------------+----------+----------+----------+-----------+-----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+======================+==========+==========+==========+===========+===========+
| attach_volume | 2.76634 | 2.93256 | 2.74051 | 4.9991 | 2.49794 |
+----------------------+----------+----------+----------+-----------+-----------+
| boot_server | 13.0862 | 14.1081 | 12.9067 | 17.4019 | 11.2524 |
+----------------------+----------+----------+----------+-----------+-----------+
| delete_server | 2.66759 | 2.61565 | 2.42714 | 301.85 | 2.32366 |
+----------------------+----------+----------+----------+-----------+-----------+
| detach_volume | 2.63999 | 2.79136 | 2.62008 | 3.24591 | 2.39884 |
+----------------------+----------+----------+----------+-----------+-----------+
| find_host_to_migrate | 1.19007 | 1.35736 | 1.19707 | 1.74729 | 0.927079 |
+----------------------+----------+----------+----------+-----------+-----------+
| live_migrate | 14.4896 | 15.3379 | 14.9336 | 28.0886 | 8.27224 |
+----------------------+----------+----------+----------+-----------+-----------+
Issues which have been found during the tests
=============================================
.. table:: Issues which have been found during the tests
+---------------------------------+-----------------------------+-------------+----------+
|Issue description |Root cause, Link | Link to bug | Is fixed |
+=================================+=============================+=============+==========+
|| Live migration failure. || Qemu and nova use the same || 1627476_ | Yes |
|| Port range intersection. || port range. || | |
+---------------------------------+-----------------------------+-------------+----------+
|| Live migration failure. || Root cause of the issue || 1628652_ | No |
|| In nova-compute logs: || haven't found yet || | |
|| "Unable to pre-create chardev || || | |
|| file console.log. No such file || || | |
|| or directory || || | |
+---------------------------------+-----------------------------+-------------+----------+
Test result from run where first bug was not fixed: :download:`rally_report.html <configs/rally_report_last.html>`
.. references:
.. _Rally installation documentation: https://rally.readthedocs.io/en/latest/install.html
.. _1628652: https://bugs.launchpad.net/mos/+bug/1628652
.. _1627476: https://bugs.launchpad.net/mos/+bug/1627476

View File

@ -0,0 +1,68 @@
{% set flavor_name = flavor_name or "m1.tiny" %}
{% set image_name = image_name or "^(cirros.*uec|TestVM)$" %}
{
"NovaPerformancePlugin.boot_attach_live_migrate_and_delete_server_with_secgroups": [
{
"args": {
"flavor": {
"name": "{{flavor_name}}"
},
"image": {
"name": "{{image_name}}"
},
"security_group_count": 2,
"rules_per_security_group": 10,
"boot_server_kwargs": { "auto_assign_nic" : true },
"create_volume_kwargs": {},
"volume_size": 1,
"disk_over_commit": false,
"min_sleep": 0,
"max_sleep": 0,
"block_migration": false,
"do_migration": true,
"do_delete": true
},
"runner": {
"type": "constant",
"times": {{ 5 * compute }},
"concurrency": {{concurrency}}
},
"context": {
"users": {
"tenants": 1,
"users_per_tenant": 10
},
"quotas": {
"neutron": {
"network": -1,
"security_group": -1,
"security_group_rule": -1,
"port": -1,
"subnet": -1,
"router": -1
},
"nova": {
"instances": -1,
"cores": -1,
"ram": -1,
"floating_ips": -1,
"security_groups": -1,
"security_group_rules": -1
},
"cinder": {
"volumes": -1,
"gigabytes": -1,
"snapshots": -1
}
},
"network": {
"start_cidr": "{{ "100.1.0.0/25" if gre_enabled else "1.0.0.0/25" }}",
"networks_per_tenant": 1
}
},
"sla": {
"failure_rate": { "max": 0 }
}
}
]
}

View File

@ -0,0 +1,25 @@
{
"GlanceImages.create_and_delete_image": [
{
"args": {
"image_location": "http://{{http_server_with_glance_images}}/images/glance_1g_image.raw",
"container_format": "bare",
"disk_format": "raw"
},
"runner": {
"type": "constant",
"times": 400,
"concurrency": 50
},
"context": {
"users": {
"tenants": 2,
"users_per_tenant": 3
}
},
"sla": {
"failure_rate": { "max": 0 }
}
}
]
}

View File

@ -0,0 +1,20 @@
{
"Authenticate.keystone": [
{
"runner": {
"type": "rps",
"times": 120000,
"rps": 180
},
"context": {
"users": {
"tenants": 10,
"users_per_tenant": 10
}
},
"sla": {
"failure_rate": { "max": 0 }
}
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

View File

@ -0,0 +1,13 @@
{
"admin": {
"password": "password",
"tenant_name": "tenant",
"username": "user"
},
"auth_url": "http://1.2.3.4:5000/v2.0",
"region_name": "RegionOne",
"type": "ExistingCloud",
"endpoint_type": "internal",
"admin_port": 35357,
"https_insecure": true
}

View File

@ -0,0 +1,112 @@
<domain type='kvm' id='9'>
<name>vm2-729036-comp-disk-283</name>
<uuid>2afa875e-609b-487a-96e4-ed453c690d9b</uuid>
<memory unit='KiB'>4194304</memory>
<currentMemory unit='KiB'>4194304</currentMemory>
<vcpu placement='static' cpuset='4-5'>2</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='4'/>
<vcpupin vcpu='1' cpuset='5'/>
</cputune>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
<boot dev='network'/>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<pae/>
</features>
<cpu mode='host-passthrough'>
</cpu>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/usr/bin/kvm-spice</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/vm2-729036-comp-disk-283.qcow2'/>
<target dev='vda' bus='virtio'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</disk>
<controller type='usb' index='0'>
<alias name='usb0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'>
<alias name='pci.0'/>
</controller>
<controller type='pci' index='1' model='pci-bridge'>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</controller>
<controller type='ide' index='0'>
<alias name='ide0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:4c:bf:ef'/>
<source bridge='br-p1p1'/>
<target dev='vnet0'/>
<model type='virtio'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x01' function='0x0' multifunction='on'/>
</interface>
<interface type='bridge'>
<mac address='52:54:00:24:e5:59'/>
<source bridge='br-p1p2'/>
<target dev='vnet1'/>
<model type='virtio'/>
<alias name='net1'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x01' function='0x1'/>
</interface>
<interface type='bridge'>
<mac address='52:54:00:30:0c:bf'/>
<source bridge='br-p4p1'/>
<target dev='vnet2'/>
<model type='virtio'/>
<alias name='net2'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x04' function='0x0' multifunction='on'/>
</interface>
<interface type='bridge'>
<mac address='52:54:00:61:6a:55'/>
<source bridge='br-p4p2'/>
<target dev='vnet3'/>
<model type='virtio'/>
<alias name='net3'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x04' function='0x1'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/3'/>
<target port='0'/>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/3'>
<source path='/dev/pts/3'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='vnc' port='5900' autoport='yes' listen='0.0.0.0'>
<listen type='address' address='0.0.0.0'/>
</graphics>
<video>
<model type='cirrus' vram='9216' heads='1'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</video>
<memballoon model='virtio'>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</memballoon>
</devices>
<seclabel type='none'/>
</domain>

View File

@ -0,0 +1,5 @@
---
concurrency: 5
compute: 1000
gre_enabled: false
http_server_with_glance_images: "1.2.3.4"

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,14 @@
rally --plugin-paths /home/rally/plugins task start \
keystone.json \
--task-args '{"gre_enabled": true, "compute": 375, "concurrency": 5}' \
--out=keystone.html
rally --plugin-paths /home/rally/plugins task start \
boot_attach_live_migrate_and_delete_server_with_secgroups.json \
--task-args '{"gre_enabled": true, "compute": 375, "concurrency": 5}' \
--out=boot_attach_live_migrate_and_delete_server_with_secgroups.html
rally --plugin-paths /home/rally/plugins task start \
create-and-delete-image.json \
--task-args '{"gre_enabled": true, "compute": 375, "concurrency": 5, "http_server_with_glance_images": "1.2.3.4"}' \
--out=create-and-delete-image.html

View File

@ -0,0 +1,327 @@
.. _openstack_control_plane_performance_report_1000_nodes:
*********************************************************
OpenStack control plane performance report for 1000 nodes
*********************************************************
:Abstract:
This document includes OpenStack control plane performance test results for
1000 compute nodes cluster.
All tests have been performed regarding
:ref:`openstack_control_plane_performance_test_plan`
Environment description
=======================
Environment contains 4 types of servers:
- rally node
- controller node
- osd node
- hypervisor
- compute node
.. table:: Amount of servers each role
+------------+--------------+------------+
|Role |Servers count |Server Type |
+============+==============+============+
|rally |1 |bare-metal |
+------------+--------------+------------+
|controller |3 |bare-metal |
+------------+--------------+------------+
|osd |20 |bare-metal |
+------------+--------------+------------+
|compute |1000 |virtual |
+------------+--------------+------------+
Hardware configuration of each server
-------------------------------------
All bare-metal servers have the same configuration describing in table below
.. table:: Description of servers hardware
+-------+----------------+-------------------------------+
|server |vendor,model |HP,DL380 Gen9 |
+-------+----------------+-------------------------------+
|CPU |vendor,model |Intel,E5-2680 v3 |
| +----------------+-------------------------------+
| |processor_count |2 |
| +----------------+-------------------------------+
| |core_count |12 |
| +----------------+-------------------------------+
| |frequency_MHz |2500 |
+-------+----------------+-------------------------------+
|RAM |vendor,model |HP,752369-081 |
| +----------------+-------------------------------+
| |amount_MB |262144 |
+-------+----------------+-------------------------------+
|NETWORK|interface_name |p1p1 |
| +----------------+-------------------------------+
| |vendor,model |Intel,X710 Dual Port |
| +----------------+-------------------------------+
| |bandwidth |10G |
+-------+----------------+-------------------------------+
|STORAGE|dev_name |/dev/sda |
| +----------------+-------------------------------+
| |vendor,model | | raid10 - HP P840 |
| | | | 12 disks EH0600JEDHE |
| +----------------+-------------------------------+
| |SSD/HDD |HDD |
| +----------------+-------------------------------+
| |size | 3,6TB |
+-------+----------------+-------------------------------+
All virtual servers has the same configuration. QEMU-KVM used as hypervisor.
For datailed info about VM configuration look into sample
:download:`domain.xml <configs/domain.xml>`.
Network configuration of each server
------------------------------------
Lab network configuration you can find in scheme below:
.. image:: configs/Network_Scheme.png
:alt: Network Scheme of the environment
Here is the part of switch configuration for each switch port which connected to
p1p1 interface of a bare-metal server:
.. code:: bash
switchport mode trunk
switchport trunk native vlan 600
switchport trunk allowed vlan 600-602,630-649
spanning-tree port type edge trunk
spanning-tree bpduguard enable
no snmp trap link-status
Software configuration on servers with controller, osd and compute roles
------------------------------------------------------------------------
.. table:: Services on servers by role
+------------+----------------------------+
|Role |Service name |
+============+============================+
|controller || horizon |
| || keystone |
| || nova-api |
| || nava-scheduler |
| || nova-cert |
| || nova-conductor |
| || nova-consoleauth |
| || nova-consoleproxy |
| || cinder-api |
| || cinder-backup |
| || cinder-scheduler |
| || cinder-volume |
| || glance-api |
| || glance-glare |
| || glance-registry |
| || neutron-dhcp-agent |
| || neutron-l3-agent |
| || neutron-metadata-agent |
| || neutron-openvswitch-agent |
| || neutron-server |
| || heat-api |
| || heat-api-cfn |
| || heat-api-cloudwatch |
| || ceph-mon |
| || rados-gw |
| || memcached |
| || rabbitmq_server |
| || mysqld |
| || galera |
| || corosync |
| || pacemaker |
| || haproxy |
+------------+----------------------------+
|osd || ceph-osd |
+------------+----------------------------+
|compute || nova-compute |
| || neutron-l3-agent |
| || neutron-metadata-agent |
| || neutron-openvswitch-agent |
+------------+----------------------------+
.. table:: Software version on servers with controller, compute and compute-osd roles
+------------+-------------------+
|Software |Version |
+============+===================+
|OpenStack |Mitaka |
+------------+-------------------+
|Ceph |Hammer |
+------------+-------------------+
|Ubuntu |Ubuntu 14.04.3 LTS |
+------------+-------------------+
You can find outputs of some commands and /etc folder in the following archives:
| :download:`controller-1.tar.gz <configs/controller-1.tar.gz>`
| :download:`controller-2.tar.gz <configs/controller-2.tar.gz>`
| :download:`controller-3.tar.gz <configs/controller-3.tar.gz>`
| :download:`compute-1.tar.gz <configs/compute-1.tar.gz>`
| :download:`osd-1.tar.gz <configs/osd-1.tar.gz>`
Software configuration on servers with rally role
-------------------------------------------------
On this server should be installed Rally. How to do it you can find in
`Rally installation documentation`_
.. table:: Software version on server with rally role
+------------+-------------------+
|Software |Version |
+============+===================+
|Rally |0.4.0 |
+------------+-------------------+
|Ubuntu |Ubuntu 14.04.3 LTS |
+------------+-------------------+
Testing process
===============
.. table:: Some test parameters
+--------------------------------+--------+
|Name |Value |
+================================+========+
|Volume size to create in Cinder |1GB |
+--------------------------------+--------+
|Flavor to create VM from |m1.tiny |
+--------------------------------+--------+
|Image name to create VM from |cirros |
+--------------------------------+--------+
1. Create work directory on server with Rally role. In future we will call that directory as WORK_DIR
2. Create directory "plugins" in WORK_DIR and copy to that directory
:download:`nova_performance.py <../../test_plans/control_plane/plugins/nova_performance.py>` plugin.
3. Create directory "scenarios" in WORK_DIR and copy to that directory
:download:`boot_attach_live_migrate_and_delete_server_with_secgroups.json
<rally_scenarios/boot_attach_live_migrate_and_delete_server_with_secgroups.json>`,
:download:`create-and-delete-image.json <rally_scenarios/create-and-delete-image.json>`
and :download:`keystone.json <rally_scenarios/keystone.json>` scenarios.
4. Create deployment.json file in WORK_DIR and fill it with OpenStack environment info.
It should looks like this:
.. literalinclude:: configs/deployment.json
:language: bash
5. Create job-params.yaml file in WORK_DIR and fill it with scenarios info.
It should looks like this:
.. literalinclude:: configs/job-params.yaml
:language: bash
6. Perform tests:
.. literalinclude:: configs/run_test_script.sh
:language: bash
As a result of this part we got the following HTML file:
:download:`rally_report.html <configs/rally_report.html>`
Test results
============
All values in tables below are in seconds.
Cinder
------
+---------------+---------+----------+----------+---------+----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+===============+=========+==========+==========+=========+==========+
| create_volume | 2.58376 | 2.71106 | 2.56 | 3.03855 | 2.40338 |
+---------------+---------+----------+----------+---------+----------+
| delete_volume | 2.31383 | 2.4265 | 2.30166 | 2.65687 | 0.432665 |
+---------------+---------+----------+----------+---------+----------+
Glance
------
+--------------+----------+----------+----------+---------+-----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+==============+==========+==========+==========+=========+===========+
| create_image | 44.8271 | 60.351 | 42.9408 | 79.4237 | 21.3831 |
+--------------+----------+----------+----------+---------+-----------+
| delete_image | 1.53144 | 1.98743 | 1.54234 | 2.6857 | 0.601818 |
+--------------+----------+----------+----------+---------+-----------+
Keystone
--------
+-------------+-----------+----------+----------+----------+----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+=============+===========+==========+==========+==========+==========+
| keystone | 0.0758036 | 0.102582 | 0.069742 | 0.460225 | 0.048439 |
+-------------+-----------+----------+----------+----------+----------+
Neutron
-------
+--------------------------+----------+----------+----------+----------+----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+==========================+==========+==========+==========+==========+==========+
| create_20_rules | 4.40657 | 4.69323 | 4.38976 | 6.1356 | 3.72487 |
+--------------------------+----------+----------+----------+----------+----------+
| create_2_security_groups | 0.3527 | 0.402639 | 0.343685 | 1.07336 | 0.25683 |
+--------------------------+----------+----------+----------+----------+----------+
| delete_2_security_groups | 0.314032 | 0.395924 | 0.306359 | 0.635733 | 0.187815 |
+--------------------------+----------+----------+----------+----------+----------+
Nova
----
+----------------------+----------+----------+----------+-----------+-----------+
| Operation | Mean | 90%ile | 50%ile | Max | Min |
+======================+==========+==========+==========+===========+===========+
| attach_volume | 2.76634 | 2.93256 | 2.74051 | 4.9991 | 2.49794 |
+----------------------+----------+----------+----------+-----------+-----------+
| boot_server | 13.0862 | 14.1081 | 12.9067 | 17.4019 | 11.2524 |
+----------------------+----------+----------+----------+-----------+-----------+
| delete_server | 2.66759 | 2.61565 | 2.42714 | 301.85 | 2.32366 |
+----------------------+----------+----------+----------+-----------+-----------+
| detach_volume | 2.63999 | 2.79136 | 2.62008 | 3.24591 | 2.39884 |
+----------------------+----------+----------+----------+-----------+-----------+
| find_host_to_migrate | 1.19007 | 1.35736 | 1.19707 | 1.74729 | 0.927079 |
+----------------------+----------+----------+----------+-----------+-----------+
| live_migrate | 14.4896 | 15.3379 | 14.9336 | 28.0886 | 8.27224 |
+----------------------+----------+----------+----------+-----------+-----------+
Issues which have been found during the tests
=============================================
.. table:: Issues which have been found during the tests
+---------------------------------+-----------------------------+-------------+----------+
|Issue description |Root cause, Link | Link to bug | Is fixed |
+=================================+=============================+=============+==========+
|| Live migration failure. || Qemu and nova use the same || 1627476_ | Yes |
|| Port range intersection. || port range. || | |
+---------------------------------+-----------------------------+-------------+----------+
|| Live migration failure. || Root cause of the issue || 1628652_ | No |
|| In nova-compute logs: || haven't found yet || | |
|| "Unable to pre-create chardev || || | |
|| file console.log. No such file || || | |
|| or directory || || | |
+---------------------------------+-----------------------------+-------------+----------+
Test result from run where first bug was not fixed: :download:`rally_report.html <configs/rally_report_last.html>`
.. references:
.. _Rally installation documentation: https://rally.readthedocs.io/en/latest/install.html
.. _1628652: https://bugs.launchpad.net/mos/+bug/1628652
.. _1627476: https://bugs.launchpad.net/mos/+bug/1627476

View File

@ -0,0 +1,68 @@
{% set flavor_name = flavor_name or "m1.tiny" %}
{% set image_name = image_name or "^(cirros.*uec|TestVM)$" %}
{
"NovaPerformancePlugin.boot_attach_live_migrate_and_delete_server_with_secgroups": [
{
"args": {
"flavor": {
"name": "{{flavor_name}}"
},
"image": {
"name": "{{image_name}}"
},
"security_group_count": 2,
"rules_per_security_group": 10,
"boot_server_kwargs": { "auto_assign_nic" : true },
"create_volume_kwargs": {},
"volume_size": 1,
"disk_over_commit": false,
"min_sleep": 0,
"max_sleep": 0,
"block_migration": false,
"do_migration": true,
"do_delete": true
},
"runner": {
"type": "constant",
"times": {{ 5 * compute }},
"concurrency": {{concurrency}}
},
"context": {
"users": {
"tenants": 1,
"users_per_tenant": 10
},
"quotas": {
"neutron": {
"network": -1,
"security_group": -1,
"security_group_rule": -1,
"port": -1,
"subnet": -1,
"router": -1
},
"nova": {
"instances": -1,
"cores": -1,
"ram": -1,
"floating_ips": -1,
"security_groups": -1,
"security_group_rules": -1
},
"cinder": {
"volumes": -1,
"gigabytes": -1,
"snapshots": -1
}
},
"network": {
"start_cidr": "{{ "100.1.0.0/25" if gre_enabled else "1.0.0.0/25" }}",
"networks_per_tenant": 1
}
},
"sla": {
"failure_rate": { "max": 0 }
}
}
]
}

View File

@ -0,0 +1,25 @@
{
"GlanceImages.create_and_delete_image": [
{
"args": {
"image_location": "http://{{http_server_with_glance_images}}/images/glance_1g_image.raw",
"container_format": "bare",
"disk_format": "raw"
},
"runner": {
"type": "constant",
"times": 400,
"concurrency": 50
},
"context": {
"users": {
"tenants": 2,
"users_per_tenant": 3
}
},
"sla": {
"failure_rate": { "max": 0 }
}
}
]
}

View File

@ -0,0 +1,20 @@
{
"Authenticate.keystone": [
{
"runner": {
"type": "rps",
"times": 120000,
"rps": 180
},
"context": {
"users": {
"tenants": 10,
"users_per_tenant": 10
}
},
"sla": {
"failure_rate": { "max": 0 }
}
}
]
}

View File

@ -1,5 +1,5 @@
[tox]
envlist = docs,py34,py27,pep8
envlist = docs,py27,pep8
minversion = 1.6
skipsdist = True