From 6f350c38f13c59cd7d50357102bf29b2313b8ab0 Mon Sep 17 00:00:00 2001
From: Mikhail Dubov <mdubov@mirantis.com>
Date: Fri, 24 Apr 2015 13:50:51 +0300
Subject: [PATCH] Add streaming algorithms and SLA to check for outliers

In the common.streaming_algorithms module, we add the base for the streaming
algorithms that take values from the input stream and compute certain
quantities needed for the benchmark results processing. These algorithms
should use constant memory and be able to scale.

Two streaming algorithms for computing mean and standard deviation are implemented.

There is also a new "max_outliers" SLA that checks for the maximum number of
outliers based on the mean and standard deviation of the durations, computed
using these new streaming algorithms. By default, the outliers are detected
only when the total number of iterations reaches 10 (can be configured).

Example:
  3.1 4.2 3.6 4.5 2.8 3.3 4.1 3.8 4.3 2.9 10.2 11.2 3.4

  has 2 outliers (10.2 and 11.2), so:
    {"outliers": {"max": 2}} -> PASS
    {"outliers": {"max": 1}} -> FAIL

Bonus:
  * Add gate testing of different SLAs with the Dummy scenario
  * Add samples for all SLAs
  * Fix detailed message for max_avg_duration

Change-Id: I7c4f77c418c7b61f71b43216110fa4c7aaccc2f5
---
 rally-jobs/rally.yaml                         |  6 ++++++
 samples/tasks/sla/create-and-delete-user.json | 10 +++++++---
 samples/tasks/sla/create-and-delete-user.yaml |  7 ++++++-
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/rally-jobs/rally.yaml b/rally-jobs/rally.yaml
index 3a5c1855..69d44a8a 100755
--- a/rally-jobs/rally.yaml
+++ b/rally-jobs/rally.yaml
@@ -307,6 +307,12 @@
       sla:
         failure_rate:
           max: 0
+        max_seconds_per_iteration: 1.0
+        max_avg_duration: 0.5
+        outliers:
+          max: 1
+          min_iterations: 10
+          sigmas: 10
 
     -
       args:
diff --git a/samples/tasks/sla/create-and-delete-user.json b/samples/tasks/sla/create-and-delete-user.json
index a7ed7d14..38d42755 100644
--- a/samples/tasks/sla/create-and-delete-user.json
+++ b/samples/tasks/sla/create-and-delete-user.json
@@ -10,9 +10,13 @@
                 "concurrency": 10
             },
             "sla": {
-                "max_seconds_per_iteration": 4,
-                "failure_rate": {
-                    "max": 1
+                "max_seconds_per_iteration": 4.0,
+                "failure_rate": {"max": 1},
+                "max_avg_duration": 3.0,
+                "outliers": {
+                    "max": 1,
+                    "min_iterations": 10,
+                    "sigmas": 10
                 }
             }
         }
diff --git a/samples/tasks/sla/create-and-delete-user.yaml b/samples/tasks/sla/create-and-delete-user.yaml
index c34d0d16..4d32ece4 100644
--- a/samples/tasks/sla/create-and-delete-user.yaml
+++ b/samples/tasks/sla/create-and-delete-user.yaml
@@ -8,6 +8,11 @@
         times: 100
         concurrency: 10
       sla:
-        max_seconds_per_iteration: 4
+        max_seconds_per_iteration: 4.0
         failure_rate:
           max: 1
+        max_avg_duration: 3.0
+        outliers:
+          max: 1
+          min_iterations: 10
+          sigmas: 10