From 681fcc7ab091cae943fcc5225ffe61570618b881 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Wed, 20 Jan 2016 17:08:27 +0300 Subject: [PATCH] Add test plan for tenant networking Also fixes the test to handle `include` statements. Change-Id: I071bb09a535022c3ae79ddbadec4bd97fca055cb --- doc/source/test_plans/index.rst | 1 + .../test_plans/tenant_networking/plan.rst | 292 ++++++++++++++++++ .../test_plans/tenant_networking/shaker.rst | 114 +++++++ tests/test_titles.py | 5 +- 4 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 doc/source/test_plans/tenant_networking/plan.rst create mode 100644 doc/source/test_plans/tenant_networking/shaker.rst diff --git a/doc/source/test_plans/index.rst b/doc/source/test_plans/index.rst index 55b96b5..d4f3b37 100644 --- a/doc/source/test_plans/index.rst +++ b/doc/source/test_plans/index.rst @@ -12,3 +12,4 @@ Test Plans mq/plan db/plan provisioning/plan + tenant_networking/plan diff --git a/doc/source/test_plans/tenant_networking/plan.rst b/doc/source/test_plans/tenant_networking/plan.rst new file mode 100644 index 0000000..0d9596e --- /dev/null +++ b/doc/source/test_plans/tenant_networking/plan.rst @@ -0,0 +1,292 @@ +.. _openstack_tenant_networking_test_plan: + +===================================== +OpenStack Tenant Networking Test Plan +===================================== + +:status: **draft** +:version: 1.0 + +:Abstract: + + This document describes test plan for measuring performance of tenant + networking of the OpenStack cloud. + +:Conventions: + + - **Topology** is how instances are plugged into tenant network + + - **L2 topology** is tenant network topology when instances located in the + same L2 domain + + - **L3 east-west topology** is tenant network topology when instances + located in different L2 domains connected to the same Neutron router + + - **L3 north-south topology** is tenant network topology when instances + located in different L2 domains connected to different Neutron routers, + thus the traffic goes outside of the cloud and back + +Test Plan +========= + +This test plan covers base topologies (L2, L3 east-west and L3 north-south) +and collection of common network parameters like bandwidth and latency. + +Test Environment +---------------- + +Preparation +^^^^^^^^^^^ + +This test plan is performed against existing OpenStack cloud. + +Environment description +^^^^^^^^^^^^^^^^^^^^^^^ + +The environment description includes hardware specification of servers, +network parameters, operation system and OpenStack deployment characteristics. + +Hardware +~~~~~~~~ + +This section contains list of all types of hardware nodes. + ++-----------+-------+----------------------------------------------------+ +| Parameter | Value | Comments | ++-----------+-------+----------------------------------------------------+ +| model | | e.g. Supermicro X9SRD-F | ++-----------+-------+----------------------------------------------------+ +| CPU | | e.g. 6 x Intel(R) Xeon(R) CPU E5-2620 v2 @ 2.10GHz | ++-----------+-------+----------------------------------------------------+ +| role | | e.g. compute or network | ++-----------+-------+----------------------------------------------------+ + +Network +~~~~~~~ + +This section contains list of interfaces and network parameters. +For complicated cases this section may include topology diagram and switch +parameters. + ++------------------+-------+-------------------------+ +| Parameter | Value | Comments | ++------------------+-------+-------------------------+ +| network role | | e.g. provider or public | ++------------------+-------+-------------------------+ +| card model | | e.g. Intel | ++------------------+-------+-------------------------+ +| driver | | e.g. ixgbe | ++------------------+-------+-------------------------+ +| speed | | e.g. 10G or 1G | ++------------------+-------+-------------------------+ +| MTU | | e.g. 9000 | ++------------------+-------+-------------------------+ +| offloading modes | | e.g. default | ++------------------+-------+-------------------------+ + +Software +~~~~~~~~ + +This section describes installed software. + ++-----------------+-------+---------------------------+ +| Parameter | Value | Comments | ++-----------------+-------+---------------------------+ +| OS | | e.g. Ubuntu 14.04.3 | ++-----------------+-------+---------------------------+ +| OpenStack | | e.g. Liberty | ++-----------------+-------+---------------------------+ +| Hypervisor | | e.g. KVM | ++-----------------+-------+---------------------------+ +| Neutron plugin | | e.g. ML2 + OVS | ++-----------------+-------+---------------------------+ +| L2 segmentation | | e.g. VLAN or VxLAN or GRE | ++-----------------+-------+---------------------------+ +| virtual routers | | e.g. legacy or HA or DVR | ++-----------------+-------+---------------------------+ + +.. _openstack_tenant_networking_test_plan_l2: + +Test Case 1: L2 instance-to-instance performance +------------------------------------------------ + +Description +^^^^^^^^^^^ + +This test case is executed on a single pair of instances launched on different +compute nodes and plugged into the same tenant network. The measurement is +done during 60 seconds, average values are calculated. + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +======== =============== ================= ====================================== +Priority Value Measurement Units Description +======== =============== ================= ====================================== +1 Latency ms The network latency +1 TCP bandwidth Mbits/s TCP network bandwidth +2 UDP bandwidth packets per sec Number of UDP packets of 32 bytes size +2 TCP retransmits packets per sec Number of retransmitted TCP packets +======== =============== ================= ====================================== + + +.. _openstack_tenant_networking_test_plan_l2_concurrent: + +Test Case 2: L2 concurrent performance +-------------------------------------- + +Description +^^^^^^^^^^^ + +This test case is executed on pairs of instances. Every instance is deployed +on a different compute nodes (one instance per node). All instances are +plugged into the same tenant network. The measurement is run simultaneously +on different number of instances, starting with one pair and increasing in +geometric progression until all pairs are involved. On every iteration +the measurement is run during 60 seconds, then average numbers are stored. + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Test case result is series of following measurements done at different +levels of concurrency. The output may be shown in table form and/or as chart +showing dependency of parameters from concurrency. + +======== =============== ================= ====================================== +Priority Value Measurement Units Description +======== =============== ================= ====================================== +1 Latency ms The network latency +1 TCP bandwidth Mbits/s TCP network bandwidth +2 UDP bandwidth packets per sec Number of UDP packets of 32 bytes size +2 TCP retransmits packets per sec Number of retransmitted TCP packets +======== =============== ================= ====================================== + + +.. _openstack_tenant_networking_test_plan_l3_east_west: + +Test Case 3: L3 east-west instance-to-instance performance +---------------------------------------------------------- + +Description +^^^^^^^^^^^ + +This test case is executed on a single pair of instances launched on different +compute nodes and plugged into different tenant networks. Networks are +connected by a single router. The measurement is done during 60 seconds, +average values are calculated. + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +======== =============== ================= ====================================== +Priority Value Measurement Units Description +======== =============== ================= ====================================== +1 Latency ms The network latency +1 TCP bandwidth Mbits/s TCP network bandwidth +2 UDP bandwidth packets per sec Number of UDP packets of 32 bytes size +2 TCP retransmits packets per sec Number of retransmitted TCP packets +======== =============== ================= ====================================== + + +.. _openstack_tenant_networking_test_plan_l3_east_west_concurrent: + +Test Case 4: L3 east-west concurrent performance +------------------------------------------------ + +Description +^^^^^^^^^^^ + +This test case is executed on pairs of instances. Every instance is deployed +on a different compute nodes (one instance per node). All instances are +grouped by pairs, one member is plugged in tenant network A, the other into +tenant network B. Networks A and B are plugged into a single router. +The measurement is run simultaneously on different number of instances, +starting with one pair and increasing in geometric progression until all +pairs are involved. On every iteration the measurement is run during 60 +seconds, then average numbers are stored. + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Test case result is series of following measurements done at different +levels of concurrency. The output may be shown in table form and/or as chart +showing dependency of parameters from concurrency. + +======== =============== ================= ====================================== +Priority Value Measurement Units Description +======== =============== ================= ====================================== +1 Latency ms The network latency +1 TCP bandwidth Mbits/s TCP network bandwidth +2 UDP bandwidth packets per sec Number of UDP packets of 32 bytes size +2 TCP retransmits packets per sec Number of retransmitted TCP packets +======== =============== ================= ====================================== + + +.. _openstack_tenant_networking_test_plan_l3_north_south: + +Test Case 5: L3 north-south instance-to-instance performance +------------------------------------------------------------ + +Description +^^^^^^^^^^^ + +This test case is executed on a single pair of instances launched on different +compute nodes and plugged into different tenant networks. Each networks is +connected to own router, thus traffic goes via external network. The +destination instance is reached by floating IP. +The measurement is done during 60 seconds, average values are calculated. + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +======== =============== ================= ====================================== +Priority Value Measurement Units Description +======== =============== ================= ====================================== +1 Latency ms The network latency +1 TCP bandwidth Mbits/s TCP network bandwidth +2 UDP bandwidth packets per sec Number of UDP packets of 32 bytes size +2 TCP retransmits packets per sec Number of retransmitted TCP packets +======== =============== ================= ====================================== + + +.. _openstack_tenant_networking_test_plan_l3_north_south_concurrent: + +Test Case 6: L3 north-south concurrent performance +-------------------------------------------------- + +Description +^^^^^^^^^^^ + +This test case is executed on pairs of instances. Every instance is deployed +on a different compute nodes (one instance per node). All instances are +grouped by pairs, one member is plugged into tenant network A, the other into +tenant network B. Networks A and B are plugged into different routers. +Instances from network B are reached via floating IP. +The measurement is run simultaneously on different number of instances, +starting with one pair and increasing in geometric progression until all +pairs are involved. On every iteration the measurement is run during 60 +seconds, then average numbers are stored. + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Test case result is series of following measurements done at different +levels of concurrency. The output may be shown in table form and/or as chart +showing dependency of parameters from concurrency. + +======== =============== ================= ====================================== +Priority Value Measurement Units Description +======== =============== ================= ====================================== +1 Latency ms The network latency +1 TCP bandwidth Mbits/s TCP network bandwidth +2 UDP bandwidth packets per sec Number of UDP packets of 32 bytes size +2 TCP retransmits packets per sec Number of retransmitted TCP packets +======== =============== ================= ====================================== + + +Tools +===== + +This section contains tools that can be used to perform the test plan. + +.. include:: shaker.rst diff --git a/doc/source/test_plans/tenant_networking/shaker.rst b/doc/source/test_plans/tenant_networking/shaker.rst new file mode 100644 index 0000000..197e0c7 --- /dev/null +++ b/doc/source/test_plans/tenant_networking/shaker.rst @@ -0,0 +1,114 @@ +Shaker +------ + +This section describes how to perform +:ref:`openstack_tenant_networking_test_plan` with `Shaker`_ tool. + +Test environment preparation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To run tests you will need a machine located outside of the cloud. This machine +must be routable from OpenStack instances and need to have one open port +to accept connections from Shaker agents. See more details in +`Shaker deployment`_ guide. + +Shaker is distributed as Python package and available through PyPi +(https://pypi.python.org/pypi/pyshaker/). +It is recommended to be installed inside virtualenv. + +.. code:: + + $ virtualenv venv + $ . venv/bin/activate + $ pip install pyshaker + +The connection to OpenStack can be configured using standard ``openrc`` file. +(Refer to http://docs.openstack.org/cli-reference/content/cli_openrc.html on +how to retrieve it). Alternatively the configuration can be passed to Shaker +via set of CLI parameters ``--os-tenant-name``, ``--os-username``, +``--os-password``, ``--os-auth-url`` and ``--os-region-name``. Connection to +SSL endpoints is configured by parameters ``--os-cacert`` and +``--os-insecure``. Note that it is highly recommended to run Shaker with admin +user. Use of non-admin user is also possible though, see +`Running Shaker by non-admin user`_. + +Before starting tests a master image must be built. The process downloads +Ubuntu cloud image, installs all necessary packages and stores snapshot into +Glance. This snapshot is used by ``shaker`` to boot instances. + +.. code:: + + $ shaker-image-builder + +Running `shaker` tool starts a server that accepts connections from +agents located on instances. In order to work user needs to specify host +address and port number. The result may be stored in raw format (``--json``), +in ReST (``--book``) or as interactive HTML report (``--report``). + + +Test Case 1: L2 instance-to-instance performance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Test case specification**: :ref:`openstack_tenant_networking_test_plan_l2` + +**Execution**:: + + shaker --server-endpoint --scenario networking/perf_l2 --book + + +Test Case 2: L2 concurrent performance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Test case specification**: :ref:`openstack_tenant_networking_test_plan_l2_concurrent` + +**Execution**:: + + + shaker --server-endpoint --scenario networking/full_l2 --book + + +Test Case 3: L3 east-west instance-to-instance performance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Test case specification**: :ref:`openstack_tenant_networking_test_plan_l3_east_west` + +**Execution**:: + + shaker --server-endpoint --scenario networking/perf_l3_east_west --book + + +Test Case 4: L3 east-west concurrent performance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Test case specification**: :ref:`openstack_tenant_networking_test_plan_l3_east_west_concurrent` + +**Execution**:: + + shaker --server-endpoint --scenario networking/full_l3_east_west --book + + +Test Case 5: L3 north-south instance-to-instance performance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Test case specification**: :ref:`openstack_tenant_networking_test_plan_l3_north_south` + +**Execution**:: + + shaker --server-endpoint --scenario networking/perf_l3_north_south --book + + +Test Case 6: L3 north_south concurrent performance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Test case specification**: :ref:`openstack_tenant_networking_test_plan_l3_north_south_concurrent` + +**Execution**:: + + shaker --server-endpoint --scenario networking/full_l3_north_south --book + + +.. references: + +.. _Shaker: http://pyshaker.readthedocs.org/en/latest/index.html +.. _Shaker deployment: http://pyshaker.readthedocs.org/en/latest/installation.html#openstack-deployment +.. _Running Shaker by non-admin user: http://pyshaker.readthedocs.org/en/latest/installation.html#running-shaker-by-non-admin-user \ No newline at end of file diff --git a/tests/test_titles.py b/tests/test_titles.py index 5692cc6..92483b9 100644 --- a/tests/test_titles.py +++ b/tests/test_titles.py @@ -11,6 +11,7 @@ # under the License. import glob +import os import re import docutils.core @@ -192,12 +193,14 @@ class TestTitles(testtools.TestCase): test_plan_tmpl = docutils.core.publish_doctree(template) template_titles = self._get_titles(test_plan_tmpl) - files = glob.glob("doc/source/test_plans/*/*.rst") + files = glob.glob("doc/source/test_plans/*/plan.rst") + files = [os.path.abspath(filename) for filename in files] for filename in files: with open(filename) as f: data = f.read() + os.chdir(os.path.dirname(filename)) test_plan = docutils.core.publish_doctree(data) self._check_titles(filename, template_titles,