From f84ec2ce07cc9695d922e2b474895dae1fb5f29b Mon Sep 17 00:00:00 2001 From: Alexandr Nevenchannyy Date: Thu, 23 Jun 2016 19:43:20 +0300 Subject: [PATCH] Add reliability test results This commit add part of reliability testing results. Scope of this commit is testing Nova API under several factors. Change-Id: Id3cb644ccf4bd315846399e6ac40a446297787f3 --- doc/source/test_plans/reliability/plan.rst | 38 +- .../reliability/plan_conventions.rst | 36 + doc/source/test_results/index.rst | 1 + .../reliability/images/Network_Scheme.png | Bin 0 -> 20176 bytes doc/source/test_results/reliability/index.rst | 650 +++++++++++++ .../reliability/rally_plugins/scrappy.conf | 16 + .../reliability/rally_plugins/scrappy.py | 115 +++ .../reliability/rally_plugins/scrappy.sh | 116 +++ .../reliability/rally_plugins/scrappy_host.sh | 132 +++ ...om_controller_freeze_keystone_150_sec.json | 37 + ...m_controller_freeze_memcached_150_sec.json | 37 + ...om_controller_freeze_nova-api_150_sec.json | 37 + .../random_controller_kill_mysqld.json | 38 + .../random_controller_kill_rabbitmq.json | 38 + .../random_controller_reboot_factor.json | 38 + ...om_controller_freeze_keystone_150_sec.html | 856 ++++++++++++++++++ ...m_controller_freeze_memcached_150_sec.html | 856 ++++++++++++++++++ ...om_controller_freeze_nova_api_150_sec.html | 856 ++++++++++++++++++ .../random_controller_kill_mysqld.html | 856 ++++++++++++++++++ .../random_controller_kill_rabbitmq.html | 856 ++++++++++++++++++ .../reboot_random_controller.html | 856 ++++++++++++++++++ 21 files changed, 6428 insertions(+), 37 deletions(-) create mode 100644 doc/source/test_plans/reliability/plan_conventions.rst create mode 100644 doc/source/test_results/reliability/images/Network_Scheme.png create mode 100644 doc/source/test_results/reliability/index.rst create mode 100644 doc/source/test_results/reliability/rally_plugins/scrappy.conf create mode 100644 doc/source/test_results/reliability/rally_plugins/scrappy.py create mode 100755 doc/source/test_results/reliability/rally_plugins/scrappy.sh create mode 100755 doc/source/test_results/reliability/rally_plugins/scrappy_host.sh create mode 100644 doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.json create mode 100644 doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.json create mode 100644 doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_nova-api_150_sec.json create mode 100644 doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.json create mode 100644 doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.json create mode 100644 doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_reboot_factor.json create mode 100644 raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.html create mode 100644 raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.html create mode 100644 raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_nova_api_150_sec.html create mode 100644 raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.html create mode 100644 raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.html create mode 100644 raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/reboot_random_controller.html diff --git a/doc/source/test_plans/reliability/plan.rst b/doc/source/test_plans/reliability/plan.rst index 8c4bd02..3b778d7 100644 --- a/doc/source/test_plans/reliability/plan.rst +++ b/doc/source/test_plans/reliability/plan.rst @@ -14,43 +14,7 @@ OpenStack reliability testing :Conventions: -- **OpenStack cluster:** consists of server nodes with deployed and fully - operational OpenStack environment in high-availability configuration. - -- **Fault-injection operation:** represents common types of failures which can - occur in production environment: service-hang, service-crash, - network-partition, network-flapping, and node-crash. - -- **Service-hang:** faults are injected into specified OpenStack service by - sending -SIGSTOP and -SIGCONT POSIX signals. - -- **Service-crash:** faults are injected by sending -SIGKILL signal into - specified OpenStack service. - -- **Node-crash:** faults are injected to an OpenStack cluster by rebooting - or shutting down a server node. - -- **Network-partition:** faults are injected by inserting iptables rules to - OpenStack cluster nodes to a corresponding service that should be - network-partitioned. - -- **Network-flapping:** faults are injected into OpenStack cluster nodes by - inserting/deleting iptables rules on the fly which will affect - corresponding service that should be tested. - -- **Factor:** consists of a set of atomic fault-injection operations. For - example: reboot-random-controller, reboot-random-rabbitmq. - -- **Test plan:** contains two elements: test scenario - execution graph and fault-injection factors. - -- **SLA**: Service-level agreement - -- **Testing-cycles**: number of test cycles of each factor - -- **Inf**: assumes infinite time to auto-healing of cluster - after fault-factor injection. - +.. include:: plan_conventions.rst Test Plan ========= diff --git a/doc/source/test_plans/reliability/plan_conventions.rst b/doc/source/test_plans/reliability/plan_conventions.rst new file mode 100644 index 0000000..41ef51f --- /dev/null +++ b/doc/source/test_plans/reliability/plan_conventions.rst @@ -0,0 +1,36 @@ +- **OpenStack cluster:** consists of server nodes with deployed and fully + operational OpenStack environment in high-availability configuration. + +- **Fault-injection operation:** represents common types of failures which can + occur in production environment: service-hang, service-crash, + network-partition, network-flapping, and node-crash. + +- **Service-hang:** faults are injected into specified OpenStack service by + sending -SIGSTOP and -SIGCONT POSIX signals. + +- **Service-crash:** faults are injected by sending -SIGKILL signal into + specified OpenStack service. + +- **Node-crash:** faults are injected to an OpenStack cluster by rebooting + or shutting down a server node. + +- **Network-partition:** faults are injected by inserting iptables rules to + OpenStack cluster nodes to a corresponding service that should be + network-partitioned. + +- **Network-flapping:** faults are injected into OpenStack cluster nodes by + inserting/deleting iptables rules on the fly which will affect + corresponding service that should be tested. + +- **Factor:** consists of a set of atomic fault-injection operations. For + example: reboot-random-controller, reboot-random-rabbitmq. + +- **Test plan:** contains two elements: test scenario + execution graph and fault-injection factors. + +- **SLA**: Service-level agreement + +- **Testing-cycles**: number of test cycles of each factor + +- **Inf**: assumes infinite time to auto-healing of cluster + after fault-factor injection. diff --git a/doc/source/test_results/index.rst b/doc/source/test_results/index.rst index 3affdb6..7005b37 100644 --- a/doc/source/test_results/index.rst +++ b/doc/source/test_results/index.rst @@ -20,3 +20,4 @@ Test Results hardware_features/index provisioning/index 1000_nodes/index + reliability/index diff --git a/doc/source/test_results/reliability/images/Network_Scheme.png b/doc/source/test_results/reliability/images/Network_Scheme.png new file mode 100644 index 0000000000000000000000000000000000000000..b3bdca4a97f53fbcc681be3b25caf642de784525 GIT binary patch literal 20176 zcmd74c{r7A+c&(Lg@g);vP>0aN@X5aLNX*mnW9MMc^(=N2}ww%Bq1TC%w$TEj3M)o zOqntd@%~oN_x$l}-}ijm`wiRnzH7Vh>*~^Co#%1x$G-ojL*NBfg-+e^j!+Wt=^Tf4>>ErWD<2?@bUI}fYrtyo8-&TuyIa#XdaAh+j z2B%Hi6BEQo418 z0mT2+-ypZoY11>8pwLihr?JPW>4ayCGmDFDZEaKe3&gH6#-m4%=6kJv`1cn--dkvg ze%RelQ*vi|+IGi|9qQVfWTrk|6`-c3c6aBco$TX%b}`LSOuyw4p+Q_xNy+ZWk z{{LE#+Z7)TA4f)hs;>4~S>ZP*lA)0FUdzkQ&aS?{`5EIAF{f7{At9;hONS_2rWvE7 zqj|MnH#9Y^%w?I~ym|KDYhLdX!Lz=jqvL5z%(ZLRaDOhr%G}yoTK5Ym3GTs33Zd@q z?)MAsf9ret|NYrqwcOvZ-`{BVSKS_`eFMMB{ivFnnx1EFUXx+#l0BViP|Pz%Us)G@ z;s#q+SuP4gj3c~e*REa9GY8a`mzOydLiFC7MYu~l{J;F zokdGxp*burUYMFnQ9To%zNvcV?c28n1!Dd5+wb;IO&!|5Kah4{cG=I*FF!w@g(jsl z%OSt(v`x>?;bE8KZ}al<@Z~}J+cFJKL_#mUulR_aGPlF`SC=9qBd2;x4@bVydHv(v zjov)Fq!%wZ`T1?umS(~as~r?~{yp|RT~}Avy1eY36dU>I_$I*5|Wa; z4@x|VifZfXDsmic%Dz_J(cZ2cC-(l?vuDk1v$M0TEG*p0Pp+Dn%#UZ4zWrg@uaIR> zOulgEi7VCf3DQ1NreB|&kK&uJEFnxE#j+mmx6eO$^5msUm&)HYe)#YKm$$I6@a4;w zj*ct0Wn_-oSXpJHrKM$LaH-zDd@SFAa@7q19(7jF{wOWwQ)Ry~kS8q&L=P z`^xS*I5-%VcC5NO2mE&( z5VKlb?Dc7DXgG58XdvDd80fY*$vUHc=FHug7%W}Po67?fd4+|!c)fq`HEOq`&q3B^ z|MV2|fdf%|`YV5i&Oc9C7;R3mva-V6?4hSWvR6r6oi4ma!ufam{e8R#CEb6Hjs_a( zSXhid5xhnskrX)}p)4K}&-m`CI1&BT(0e&KE$!ywWVfiOC@O7rVM5~q`DN?u91cuF&Sz~K3^X%se9qV z8_(rGfBx80`aRFg#C>O`qzn%X2-yu(;zFqo=oPy=xw$QO7rN+Pxngf`&%mdrs5$cM z*RS8dzxLa5a&ZNo)X>m4B;~m@*)1*eRY;@s5#>ic0YP`${BI`5~V* zJG6s=fq_Dosb9mxhq=hF;DeH4f`W8JVXF?|qes(1cEyXksA7v?+1J{lP4!Sm;>b=MMQ9g*I>ULV+g7r)=1k}rTQ~fF3HEIdj34V zP+e6OR&#y-{%HK*JsDQr1((#+Y!-h1uskKs&tL!fb82=r`R-9u;x3bVZ?4MbigO)5 zu9AGwNjtxo+?2SyWiK8ZYTF*<0G%)b!T; zI}bm9R>+MJ>h0H>U#dt+N#U+m9}H4|)Smlm5IOVb&#Tl_TBm)|vDTq(=g*%<6)w*X z2j&h|haTp;{JFMvPqboEV}KN=MeA$zix&&r=f}`#sCVs}>dHsAn%fABGd(4vcd(5A&W4VOxma+Aq8&Fq>Nu5YePCm;Wo@>?lGBGjRwC*XM)XNtyy7ChMcbOG`{lY-M#-K~8S%%r1I*deqaobLa4+Il>e%47AZWw6qv9 z8**(7FJJyqR3w2R=Sv$?7a0|mm6a709^P5(ahi1G#EEw~IhyE%=p6F$rl?_x#MiH1 zKYBzzuv3>%Qap9Ki;jekvgs|^ho>8fA#^r~!RZRx#0RY7i5uT^baa+hR_yb;GC6{7 znwjOifB&PiGv?`2>#n@)q9T?G0ikF;UENhQ@`VLwsf3N`{tEnblt}+cT2#q4DhA6R zZ~dwVBO)W^6%@$BYdnjJWHvKUg%&b5KOZOUvmSilG&?){Mbu5EJa@RHgajUtr>Ccx z@D+@W84bI^PrLY{boKS)A|m!3JScVjGrFojtzqKDG&}<*7nip0-!Exsbe8&*;(3xr zP?@?PEZx@E^25R!D+Bj*|9dqzH`}q~e)w?e)F}pMSHm|~%i6QAwPorD(XtoJu=(og z=^ZGVMM*xDKEU{r_XxQK~E=?n?B>WnA4$kruTe(_3~`XzJ9gDHo+Bd z*-lNnJsKMyWre4SQg!|F^PIM}Hho|Y7BKmI6vN)Vv~3QbLJuW8B)^`$erRZx{yVQ;y?XTM zQE>3CnQLsvk4xTc;xsWak(Msqb6C}~^Fwg;U@)tAYg-#5aW4ZyYe&bce>B^%c$|YV1I7Flm+{?o9dt$u!Dr?ga>bmY{jg*|PDCoR#x=FBej&STj0cn-LY+<0(zX_R$`v|U}jhzWO? z;zpOqob}dX3P-=K|AhiH+zn_HM){r9|Gaw#D8Y2f^z=0LufLyPzYlAx$m!EA^JCu| z8ygcE0VG%ijknIo5SAjztis0~jTH#1Ekn0$bh!)@1sJKNMN@s>cBw7k6@5ORuiKwL zU%N6t?srXu!xs>s?QpFv%Z+TEiHQkU7me4pA}kC+-zO9(h}0Dwot>gAs%`tGX}Wbv zEw#NJ8}`vV`VYFc-P1hl$Ak_mwZ3|Ev~+CYT*s7FsrTPw$B%PxaPS+J6kV^OH%`PB zuq9mgr46m-i0ArNQ`0}=%FWGP=sdwJV#_vj^V+q+^}nlo_U!5G=)lN&Z$I=UWMX!C zt)Za-lVbSi&(B}JT)BMNs{M_uhDP|3@afZwzkj^@@#BZ5l%aycjb9kqH@^Aq|0KN| zIFq`125mN|;C?AOIbML)(pyo$*Q!G>wE-YH+uLc?rk#BUmr$?RncH_Cv@|jK-qUk% z-@b^44{2FhC55KrXMjvust-xHUcGT+7?_)${!v_< z7h0~aZs*8|8G0yRl=zKrEd9G3w6t0=;e&&N$ulY=qi%5KTY_|{^F&8qU*E0+rzAWU z9@Jcq7j^hlSGS9(sji-l{{H+qI~~jBY(hUxK_dPSUNkspIotV>S>yW zhP^+3URP7A&&kONIYhUo!uhFxRYQY-pkVg<_wPS^U?8^G^~m-M>rxO0_O7Gn5sAcd zKTM!Xf)#o7g>N?Ds=3wTCCnPw#KpxK8KY$HZZAvSoj~3`4Z%hg{(!4iJw?kOZ*O58 zj^}bLs5#1VCt*ahc}7XKx~=l{G1SR~aM`gqRRxNpyu8(wl`oQ$yYuWg#KbzPtA{Zf zfO@zxn{I7Tl7HeZRF-vj!GY7(ERyb;BKe}E046_i^s>P(fH7cI;JT`+G(@?t<%&m+ z9HA}YcCP|qnIBH@nRHGc*I>ixXc2m*eB(1ET_^`4Yi<`SN9r6jiP;#k#sWBO@c~?G=rUVX7&Znrm1G zpiA41b+>=rtEr_$SU_OJH6m7y-Oa<}#`Wu{LyE#`@+zyS90#7r?ezwVZfwgoGy=B7iRS?%k&a1SqJeR&m<_R6?3O^C)UejePWftQTO{?b~H&rI>4H z*~8EV@R-3B3vM=Z^F^h8%X_PTXm zNy)t7nQXVWwt+zjqvZ6wBmjO~zX%_T=2p++w{PEORny-za+7xQ*$I|vKYb3l%>4Xi zaDpwMo1*qF$bv#(U?55h)p^f$$3Fu9K$-jAy?bb^(nCW-=%Y5Ez4&WJr{f7Xdy3rf zX0mi)`z+zV2qQAW;88$6_bP}z#!L$n9z&E{@BI2@YRq}emo{Xu{P5wzayvaX+O-Nw zf~PhZ`RPCX6+6&NfHm2D1*mo3UJooN0GwdES0Rd0TU(xuIfy;b*AFpZgGYm z4Gn!`V=L&gz85q#Ti%x54-K8l@7~mRNG9*D&F$JDcMlH;Sfw6|G(_~JjlQxi=uW^u zc>1$=B*5U?w{NFnIL3DI?Y&qLy8(RdpX6jG5(_M`W5+!-q~2l=@`MJkdh`Chq~plL zgapGBeyALl!{6cnx->3c1es-D*i6JYTA1j(d-rbO{^al9zpwuFaTNQDE#oOwI_$jT zmJgxW$p~e__&6QBg@H^!D^v-@F+} z%O0kPQ3keU^zMhg#pn9^1R)d3dlhyrE?KFmZ?dzEvUPx;!OeAZt>Yxz-WC-VL79P` z=;}&-%ENUrr_u5-sSNV%w-H;wS3NkW+>xZclCKDUY09MfT#w#Xst-CK?y0nXe z0<7*JW?O!~o3io)uv~v5Y{=7i2?|CFnSANYwGm~Z@i#(O&i?QL@05s8I!gg%v!=Gz z11q(AJ)d~NSO0=7i?*ibZ{bIK6G6F9`%AIr5a*yL{`=1tVPSM6 zMQv@yiZ>8b{HPcK0qd)(&dSOfH9kL!@r(Y8Ck9DSlP98jaOc)LA-T%uI3K-u@j{?3 zFebTDZL_S3%C3qxXl585kowZo(z;81HvX&~~m(m9ZK-;Awn(e0VrnxOi{T zW^QJNoe6fT@>F;yQ4iHXmiv2k$h*|+Z!8q$FS;}a9XgRZ!jpepw5UyX#KS!iyc z$CjV(*W~#Q_K3}b4S9wlA}Z<{)63*!uC5%A@?d!m?p;xlky@87p)(Y_&tq{r(Q4>O z&`);m+zDYC!|8CCBCe{Spa762VR<*%OOJ zV|gPZTPHemK}8_XiDf|TX+e=+ExUu~t&E!k^NZp)MCs_ey5>#4Xs@fYLS3?^=o}EZ z`VneDeS$Q`&}Woo%xRnT_0=iX`lo(Sq-|H`M)6@YgVn=BL---%xYD>*Vt9?C81Vc& z^k%%@*})+QGN7`u1e(L2q1r$vV0c3eaGPE!;~rJthrYg*IX9a#4NFTbe+WSc#5xYF zcmVy{+rI@C!Y#+g_du$GiaUcg1J(ul!Kd@)S#q+UnbCM_#>e%NbaJu@ItEGzuW}7bo(LZ3LA<=; z?sI0SDCnIlEA9~Vu<`JL+U+5@^O6#8FhFy2^MLc6ob;n}qs=rlG~IIbU&~T3V6H+*DcL!WB8;C<#SdT3Ho?q+pb?R70ZSGb~Y_ zQ{mL|m>Z$T5?xySWI0s*jt*NE zMvAT23laxIb-oSRV`NcLQWjW8HWp@UHGF%_6`n#bxu$oWPURy;t@kX zTx$J4#E1r`savjWlTFp1K6wwUU?@=}0zw|l7^@a@*pU!SN~C5Hx!`Jle_d?LmN->L zgQ16yAMagQos@G`0p6qXNMZAZZ6swU*DHlKINGeR1jYJdyK42RqkF&fp2W0NC}Im! zjLU3vdg^{hHN~ZaK`>lTTaHnUIN-mP@O)d;w*Lu-bA)oI7>9f}z0*D)dWx+Si6E^& z8$kt;^|&jxG5U|8Fhm^(3&+x6#vC1^FQKp#z|&5p+9b^Yvul5}vOk^hrjd#N=cb%Q z0V|w+9 zXZP;i>WqJ)0ma&~4Ee`+WJ1X(JxuWhcnLg4GR~f8&#w6Vd1Un{xM+}4vbXg7nzl0c zp7Os6gwvY;Wo4hE(b4VO_r0?)?#0WOpDHW+MAnB?OiWD;@@!?7J|rgIuDQOhUn>$a z{OeaMSe#{>}X~?ku6zoo`Ko&`IC|`kn_cc@JmiMtyw0 zNIY*w4`GU78n-5iX8F}o`~2Pqx4fwg5$8T zumC55%U}nZ(%j=%J0dFj^T`eth164tW2Am^j$+ldXz-s{PqM}&% zWamYoR#)IF7@V1zs!XQ^6IimUswyY}&@Sy99p~_Y zpsa=Q_ZAocQ^6Fdx1VvP1tn2eZ^V2+V+r62mJhyN^PgFO;9#H(_?b}4ii-5|OdT9% zvFXJQe#Be_f`yQB?8J$1QsQgf+|0ob;3xU^$(U}|)|24xPy#OL=qP%&w6^}5oa}*q z)zrkUSDKlLzitB(~uAt~vF_KgOQ?gK;*4Qli4s~D2de^*vk+UWVGKhEL0UJ0wFTfGJ9~6Y|*$4HPq{nw!O3Hir&k&wHMIz>+tA__r0r&@P z44|?HkRtBklBVWMwFD`XFMENV$wUtlA+THAuhMQ%BC@T($yu^|dlUQi8W;JUe&o@`4NnS2QPQ4%T@f zfgP3vigwcll9Y+9Fo z`T5aH0n-Z;x}u`_P{pC=IVpAZBSAr-dU2{3glNl_yS~1@z|iUG z>80-TU%I-AEG+g?o`sn}N`xB&+~+l0y!^${$>|2_K{6X@NDN}+Z*6Rh7IiR%HUG$w(Tzm4d>$K#vQCLblQMQjP#b4+SUPqpq0Uv^Z9sdQ}4pV zTT`3$A~$VhCMNb{P$GT7*7Gzvnl9xY4-XH_UkV}wBI*iS01pGyH-HCL0uK@B9b=B( zXsA~PUxWQeQUqG49;9Pz9d#6&NrQL^LlA?52ykMY54tD=Remeb49ud+r*h9~(Qj*O z8TakO&nNogi!dmv6J$_eXU|sQ3Q?KRCDAfwfY5-8A^qcuW@cuvbpT=J$J?;zQDI?d zvCxmaF`W$zdayf`Ra8nR`Zuv}>0gh)4zF!))@ZrpS)y7zLV@oGu16!1ryvjTCpkHe zxrT7MMBpA{Wyw(lpggo#KK+868@vqfaTC-P5QC5r2$lzPj|dOHCcspgf{TQtG=7PW z(vOsUG2-##x9{Es>ZZZ!mZ3O$^;!~2|^t4vyYdaqMc+ebqTy}hNPYOTQ3%+0@i`-ap7MiN)J zN8PqF+~NCaX(4#K^nJYT{dWbHjq%Wzh|-Pq!nn9NOr^-S`E}kTX0{i>Q9Gih+FqSy5lV8?gG6 zb=MKg*0M5Ls64&BrGp>9^iuL2h8M<~X{HTj^L)IZ=E8FTPB_t>?@(P31i zC;NSJauo&|QaFJ5hzP*|*OArN_X45ECb;n}Rup?&PHt;VUHmDl4)91kfWUE4Q&ZDT zn>JA-TEX_(f8tYDSM124xA>2yri;&6f&MTMuu+=La%^(z4P^`seX8#713|_eqX|Oz z(ByGKzs2(Utj#DO`H_}33LhL^Bld}(@@q2s(ZB<1Mlo;5tnp=D5xb zz|qG~M`MBU#UA#<(iyeqZv?W)$$1^T4F#^Ht-X8KF4j|)I@OQRke+8`bf>AJ{8?G! zZ0r?3x@_4npo`iIw2b`A2~z6o*S{@JF1bKOdi?k?@?MgM#l)T}bGzt=80nlpPZeIn zxnkNUqT02>VSrC%2=X&xngKa0c=^)5et1xXkCvii<9_uZsI+l*z{_?^=V~?7XCn>q z^l3(74&03Ro*6alPcc;Gz)6sFu~jJ7@N+}bvvrO=52v1qtR1o!VHr4yelYn`h7G%e z4m7wt#L&jZhOuPA)z^-tS^y0@znNhtP(}5ilfL4bL!SsF%Dc2mDk^`wwh+WP)>ZEZ z2mh^{ha^i&LWD~4%D&kAyJ{r%uU4N}pfq({)5+U^Z)($~-;OFL+#~!Gu zxmg)N*PRI_<_u|6pb>Yr$uf5lRU3EukB|d-rbk z&mUOsC~JgdE+GwrPXQJJdfUA5U|De@RaH;E9-}2*zI2kb|V*8B~4x#O@p5_9r0mN~-1<20nzdvxRpFo~NMG7e7f(9s)Q_ z)#OR24-E()8!=#g5FW@fsqY1ZtZKp!2O8D-)3CaaHgWMpJVhTpaDd#PFh5|OeZ@{J zyR$9ypd^U`dtLp=-hPoy=th-HY=k*^csB0Ypt4GeiVqbz_v{IW*9&1BOdL|cBSlX7 zKw;PEKG+Awb@-e;gCD8)3%!8J4``{!7sV79C7(p+un+vg#WJ>j*o9Sa`>meM<*akY zV6CoYrtfYFGb2}YF%56lIYLX}yT!waB{xVu3GM@adjFi0oH)q7%-ZH2zLSPUb zM05V+OF*jcpozrt=87|O>Ase z2&;HkR~LMU=AIsd%a`3CjklP{8AAzzJVBOL0U{84f`H5-Y(^WQ1g95jS1)81B>3GHB73iCl{hcFA6+ zo!tSjE6!@Ty6{5QYe`jy_cF)!AMdMFZta@_7`URN6Hsu!zH-Pu(EG1Uk!;>w>)!x& z{pE<*!Eit)!PTDCKYIgH2x2IV@&rg4&apok0(kM0b^-hhY&sNHA)wsCMQGripB(o` zfVp5>Kh4Ksi<>u*H(KS9yn$_>4$-HGl|X@b?}0Kq31~W?NUl&N3xPjjj{jNq^t}M+ zMIzl%q*`ME6~GqS;ORfOckgv$<2$tA{9$2Xn4J!Gb_fhn3$oZLAQQ&UUXFSD{rfy( z46M1|t_m>l8ysWd6g_dm0XP;Vjs>av_U%8JYe^XF3guW4^`)=FFgP%W1Ily}WfFrN zHZ}ceG-c^NYfs=yP_g%L_F-14#7nrrvcQ9mYZV2a0F%UmAQFcN6vne(f_3MI3D6R_ zE3y=VOw{1+w`Jad5aV_Eohk6TcgDrC#ogAachnH#1FS>?#*IWrLy&+^fK+B2a*xnU zpp!k=e_}70KH(D}655!V!6s1G(BLTfX8KN2Qqu44u@avRDRFV66I|ik0my=zfBW{W z*kzyq7^bqS3MjN5X{9HL0_^MqaE74S!(GuU^)7z&s0lmUO(Gs~9336(vknkC{56y) zI3Cz>WV!D7poG5>XuN)&Z9k|EEeRfMifWv_lT&h|S>=Z($}dYxeay^8&?g!B^hUhWHhCWCf_VX;YZuViQE%a^ADMt3M^Xlb`mEE@R#3q~#Itx|7K z8YfX#Kq4L(B!|n;tjQ5ej0NmKB(4_6+d}2ov7(0#B|!OrimUom_#EsWxM9G0c#N^| zQ%EBaR3#4Dz@Nfh$WTBm!=QlBjX0VY6k)ivN~)@p$^8cp93UIE_*SGh;P|5D3T zAQR7vi@h8jb71M>n-pp2Xm^rVsqCufmx>v~HY_HP6bCyyi1PZ%JU(a~*#~s1moK>w zA5MYnE$y-J_1m{eG-_ZFYzQ8nSK!pBFcz8}#K#c+ez#|19$_RtF|) zbf(193C{`cjxe(dZlI?}ikEjBG#RNo-BNE!k|HKP%=bL|=jrJl{pH)hKLNzvydk?0 z5QacR0g{Nm@Pu8^8_umybqqT8D{w&lJ%g_*4&NFrT;#O*<$>xxJQrKF@p zMMdGd1-hyN6BCJoKVQVhAN-fN#%?IzuZ0UjR|0s1sk6F(#(tJ!*FlLqm{@hDs6r@3 zdn}mu?}tPY#bgoxWGO}lKGT^i?7m2!Mk^zLu^-e3a)|r`COTHujLgjVu&^0aX~oPZ^)y-zAFJ8h9HKf4qVYOdvB{JBU_gTbo2Cs%)~xCAN`8Biv9~}Ef7^{Z2)MM zs5{OA#QhNXd|gP8@)8sv2#_Ao(qJG$0vv?$3W*2K1c(d1q@t?I$ICmrxOn(w*`D3F zx60rP2SX761{E|Derj9(5Ya$r^%6>zB%x?!gjplN00RAR{}e>*qg}g#p{dYQ!F~o% z7o!P5R8W8?LP_tw`h^R4h?&T{XHKDgaFC~jVUg=?qbJxb$eLp-zj*Nzp|Qn@&Zx(a zxAw0O-s%zR>z&7c%CRHqL+1ZrwFd=_0VX*LGPyZ8!eP#-tM`wNE&~|>2qC>}4RQzi z50YM)4^#xP$5`OS^9x{wVO1d6Q1_srM5C57baMkTkAi(461XbUmWVJio;x}bun|mF ztO*vAf+!Xz1$2oZ;3WhHT3b<>P;vm?(X)u`>~|{qA26R(($YF5Bm@C*60#VC4%9iC zQW3PxerzygmabnHh9v?jgq?Bc&Yj=C5y!=ZfW+v#m|GS~_|{cc8UvvU4s+D8%R-n6 z@L$nTi?Pwi%~4Mh|Y4$UJx}rrdnfx0ijhWT8qSr5P#=EUAR}ca+qVdtLsU3+^eDJS-5Vpac7mJs%m2h#u*`?g4ggqesIJJNgo6rvtvR{5>Cdb#$m`$OAQkMO5+R|BAcJZ$ zG9M8aJPV^^>*m`yfD=?j??g<1Xrd+}BSEYEAJh!@%>`G(0vNsga42C(3JCI>B4-oC zQCJ*=;;(qHAj8roD*ZBhCCPC(Ldj;Dd$Xf1udp)$Wbe7tkPonVZUb$E01fQAF{?ip z01P?p^xqQQ|MQWzJ|Z{yloXl|VCyY|q-W1$TDTmy;D)wrr`hr>7oLfak&R7n`i^(o zWHjB~Uq2y7^VYWckuXf~ErjXhK7XG79S+j{Z+N!v|KgI&LqB*!`~?d0>~TRr2YwJ( zuXY+I2gerTVXe*TS1%xmIc%_iA8hUvdEfrn{Hpiz_gDky>5aCX6_7W%eN$Is2y zOD|p>B@h}c{$-%o7Jf&I6UD@?V3QuO97%`5i3SOyrVCkma_S7ydnhY);6ix)276hIOBDFh~X4KNmTeJAeE@AZyR~6e7Oh zg_r|zr@C8P5lcm;qLh5Zj;N`ti{uCn{9%F*!Bt1nnjY3U6h8W>cVUcoqpH~UKkZW0_6Rn>1$lnbm^ z05yP=5kup&k1e|ju^U^3-4_roH}`Zlb4*=kTG}~9MKeoF1dV`QfP(;>dwZ6lTw-kdHZnah(y{a0=nHXe&zK$k1VS%S#9@OI284i+5EL?oI8|w=Xdum?+bFe~$m*IS)8qAtLfKO(TUH(I!YekjUi} z6q-lpFr+V}X+qC|qJgv{^msDS3nizTQ6;MJ649hYq|*oh3@)LAb?WJfZ=VDg zARj0s>j{t0Q*epK0^#|N{`GT^ zmr&@JvA@#Oapa1YgyR%nYimyk2s~>Z1x%FNz26sUjmNREdS+%v{75?~DPLt}p@b^) zZo&zK*G&%H*yC{S4PZSCYAAy6z>zhAZbCl92wCNjq&w&DD#es$5Y&IZ8jhv`4hu8a zwYM*UZXlZ80+tvaeujco!u8FyylYI(*sb`^ifD0y^f8J>HKeDvQ!}x7BJ>6UQ>iF2 zJe-516~GikM{Jqt!U`dhN98k*+%2P`E0_)C-r%K5%;hgnV zuL?y2SpzmJ=SI1g+{b>of1eUOnbNQ6OboyTDD>2}2qjcPxps{bSSt?Q!RFz&RA8f( zz{Bf!Z-JhOuJk4+Csy1A8|-MHT!F`8W>%IBeBS9Bzb)JreqTns5S&V!#+6)@K?@#7ijZ(^dNbkdPb{zz49-?2jts1JRt zas@-;+t;tZ+wVXUfIWo=hx|2wGC4Dhm=GZ+pF9uE-Y7MjHL zp|I*fM2w(n6hC`*OEm@c1{b`&y!`pFcd4w+jtvt}1^AV1G$#@O`aRvq=r;?^x+CW% z;{SWNm2o;EA37^ymWqlLBr|*+{*b=MkWW6Q4n`-=d}IJLARSS;R1ygTeKsZ_7+W(7=9ysKFf`NDBmMZ)bNICpvM84tW}MSCIQp zA*|!W!)-HxW`ya~(@V&vj($dnmZ~tJY z1b#suzwz}+<2)}UYk7*~moF(71=-jLM8}o6n=I8EOM|r^ILFj#l4b(E*{986}BmB^*t^>@>kpE0gOt`2`NRU9BoZic@QBYGuwMs`_rlyW` z=4Jt70p;U-Bt|t(tIw=upk9$k#YlnJh(2n^qNbwq9qh5W8FR(IpkM`$2I)mHs}*Eo z;Y}bHAk2(&8c%hOP`gl9ASRmp;swH0Fpc?nc@NOi-ZE;3yNc7Sg98J$EIy!pQ1PML zf{@@N;AaHLQk75h(w2Xb6+>__1RFbxvIx)D>tJP-_Bnp$~%jVv<+c#a0%yir!+CN4511G_2?+PiRX$@tu&Q$_?XiNU*S=!1=FMify%>1hQ)vFE({{79yX6 zgMi4=#H}BPZ~zenq`mp3=+e4ZjP1Qzp=SpW2}CQ*dHa@KY>Wc^J~chvrRW>9w{ENp z`CkY|b8uY4Pl5V`L4~HWTDq}X3MtZSX}SzSbpR|+DIbV*xNxX>&@axZsbytmO4$u; z4H?v@vUPOC1bun$o&rWe^$^fB4K?-L!h-kw*d=6e(AI!%U`U|9d_eE(^!PS)#B zw2}|t;1huQaN3B1i0cCY9?(((K!D+aA+|BU^zO%RI8wGEu{cmA82uHBC_`PIy!3k-)rH)amKGcu#5F_y!$v4tcxYA?QL$G`@uDmygwhGEgkqE&U8P|W|TPS^jPPiY;xhvQ?+S-VO zWCK3pL2vLLKMqLQ-P40p)@nG-tmnL=8wU~^b}^ZIyD2&|Y*5X0!t^Tz1x0E` z299Pxw+z4&M@S>6>iNaA$~0aQ^2*uJD}dlobLA|D{~f0X?C^0JK;7j5W1`scY%e0< zucPxXHS_^q203$q{0f3q*ps*e3L?H8ANu?EZ}QKRN)peLv?f3d?WC1+7t zFe~ta%ZpfMppFB|9|8mLFao~_&p;bPeh2PCxz}%BruxZRoA@o+TW?Hmgop$q>X{HI zBDt0YzJZt(nZ|%mhWwa8(?SjfZIm*KjJplUqS|-NDG_q0d9a$CD^rO<(jF%VvdKvV z2ZwjSifHN6!h5RiXznEeEIH0jMdIkx_3JMn2iq#1L@{Hb9z38P$bS<8sV9_G{1mh~ zcmi2K?r6+$A%yT6pf%JK1re4mdM;=sauR)QhCWzGujN0K(URPL zMsh^BgN;&&>Jj^BLhG5WlD-0|j1hliO-`kpRoeQTy}*ynL-7P};9ft+FI`?sWk%N* zdnSv`CR|DrPKSyQQ*SqvEbX0&HF>4Wz{m(S8)$=trW%CpJ6*=un0Yq6d`ffLuv5NX zG3h96KAf~iaqDA_IwaTkiAY$6+9728u;>05Rcr-ET`aEFh=IqkP~wX*}92jD?5FJODX_@uSC ziena6g_)6CobC2}_T<7PA5uBWf&M9uU*Cb!!#o&ZQ%!Ne=55p{M2ZP65Ar?B zbE6uZ^w0;OI-XZi(bv(rU^rW`^E(vKr*)=ID(5-rXIB40PE<`f;oh4ZshTQ}7+tgZ zd)1U#RDy9Wq=(m9>19_hv~4E(zy9jWvys2Cuh<}~=L&uEdR4?~;u*4A{iU`R*{u@H z7F&^HS}`qcZR7Lq^CtMEIM_U_tgKq;?{RK_FXa!*4*oIM z>J>WInZCo6L6`({Y1FX*`M|$FTyt@j5VeJD{u`^#Q)kr?^a7W}hHd!%eIFy^@Qf>5 z$&<`=;0-WcA>$eeGsC}(K`sqr9Djmp4JG@^u}BqM{B@GBD&^nZOQ4>5w4}F5s$w$jOu68ycQ{H&2jo!vUV;BVR^FQadLQmDJj6 zVrHgD{h`Qfwbe{mzV~%Qd%K9FWOqx;&)Mb5$fYIxRt0yb*uOW(r^T>&u;HL4V=U45 zA>DorXAazfYJkFg)K1B2GO1qOWjZAH4Q7tueOg3Q*|aiofk*38D{5EB#aqH+gP zKh-@wJ(!>QdYLFcND;`jLqmd21K}Ji7wAO}+KY>qdaZ(TIN+F{pnKoqDpL1%%C4fw zy}j3)A`TCHMqkv-MfK%%PoA`Bjg|D!Qh)5{w=J}~xw{*x`n6Lr6V8S5Np5RP*=1MX zbWRMY`PB~}U|~U5Qc+a2E$sF#mBw)bWKO^s8G=mEVo?AH?2e3%;&d3EcWUYpoPRHL zf%byEhNEdfF@I)ft7Z=t6c!d6Ou1x1hS;k?4soi%xZ&Zs=!4%Wfa8TphOfVyL+v_k z*-l$*hZ7|*C%{U^ewR1Zvolebc>gVc1O*rfwusZDo(ut^gM)+I;c2C%hMs~ILc~d# zV>9l>81?{0MfmMcu1`j?ycL$JwYVphs|eZXY#f*cQVI2zr-IJ!@4Lt3u){bIn-5pa z%j^tmHWMx!pJKd@Y*}z7=9ZZdM@3%=M{mC lI6y<7XP^9E`G=J)mmSQPE$n`$L$xNHJF6;}NxFLHe*h}~2~q$6 literal 0 HcmV?d00001 diff --git a/doc/source/test_results/reliability/index.rst b/doc/source/test_results/reliability/index.rst new file mode 100644 index 0000000..62ab01c --- /dev/null +++ b/doc/source/test_results/reliability/index.rst @@ -0,0 +1,650 @@ +.. _reliability_testing_results: + +============================= +OpenStack reliability testing +============================= + +:status: draft +:version: 0 + +:Abstract: + This document describes an abstract methodology for OpenStack cluster + high-availability testing and analysis. OpenStack data plane testing + at this moment is out of scope but will be described in future. + +:Conventions: + +.. include:: ../../test_plans/reliability/plan_conventions.rst + + +Test results +============ + +Test environment +---------------- + +Software configuration on servers with OpenStack +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. table:: **Basic cluster configuration** + + +-------------------------+---------------------------------------------+ + |Name | Build-9.0.0-451 | + +-------------------------+---------------------------------------------+ + |OpenStack release | Mitaka on Ubuntu 14.04 | + +-------------------------+---------------------------------------------+ + |Total nodes | 6 nodes | + +-------------------------+---------------------------------------------+ + |Controller | 3 nodes | + +-------------------------+---------------------------------------------+ + |Compute, Ceph OSD | 3 nodes with KVM hypervisor | + +-------------------------+---------------------------------------------+ + |Network | Neutron with tunneling segmentation | + +-------------------------+---------------------------------------------+ + |Storage back ends | | Ceph RBD for volumes (Cinder) | + | | | Ceph RadosGW for objects (Swift API) | + | | | Ceph RBD for ephemeral volumes (Nova) | + | | | Ceph RBD for images (Glance) | + +-------------------------+---------------------------------------------+ + +Software configuration on servers with Rally role +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before you start configuring a server with Rally role, verify that Rally +is installed. For more information, see `Rally installation documentation`_. + +.. table:: **Software version of Rally server** + + +------------+-------------------+ + |Software |Version | + +============+===================+ + |Rally |0.4.0 | + +------------+-------------------+ + |Ubuntu |14.04.3 LTS | + +------------+-------------------+ + +Environment description +^^^^^^^^^^^^^^^^^^^^^^^ + +Hardware +~~~~~~~~ + +.. table:: **Description of server hardware** + + +--------+-----------------+------------------------+------------------------+ + |SERVER |name | | 728997-comp-disk-228 | 729017-comp-disk-255 | + | | | | 728998-comp-disk-227 | | + | | | | 728999-comp-disk-226 | | + | | | | 729000-comp-disk-225 | | + | | | | 729001-comp-disk-224 | | + | | | | 729002-comp-disk-223 | | + | +-----------------+------------------------+------------------------+ + | |role | | controller | Rally | + | | | | controller | | + | | | | controller | | + | | | | compute, ceph-osd | | + | | | | compute, ceph-osd | | + | | | | compute, ceph-osd | | + | +-----------------+------------------------+------------------------+ + | |vendor, model |HP, DL380 Gen9 |HP, DL380 Gen9 | + | +-----------------+------------------------+------------------------+ + | |operating_system | | 3.13.0-87-generic | | 3.13.0-87-generic | + | | | | Ubuntu-trusty | | Ubuntu-trusty | + | | | | x86_64 | | x86_64 | + +--------+-----------------+------------------------+------------------------+ + |CPU |vendor, model |Intel, E5-2680 v3 |Intel, E5-2680 v3 | + | +-----------------+------------------------+------------------------+ + | |processor_count |2 |2 | + | +-----------------+------------------------+------------------------+ + | |core_count |12 |12 | + | +-----------------+------------------------+------------------------+ + | |frequency_MHz |2500 |2500 | + +--------+-----------------+------------------------+------------------------+ + |RAM |vendor, model |HP, 752369-081 |HP, 752369-081 | + | +-----------------+------------------------+------------------------+ + | |amount_MB |262144 |262144 | + +--------+-----------------+------------------------+------------------------+ + |NETWORK |interface_name |p1p1 |p1p1 | + | +-----------------+------------------------+------------------------+ + | |vendor, model |Intel, X710 Dual Port |Intel, X710 Dual Port | + | +-----------------+------------------------+------------------------+ + | |bandwidth |10 Gbit |10 Gbit | + +--------+-----------------+------------------------+------------------------+ + |STORAGE |dev_name |/dev/sda |/dev/sda | + | +-----------------+------------------------+------------------------+ + | |vendor, model | | raid10 - HP P840 | | raid10 - HP P840 | + | | | | 12 disks EH0600JEDHE | | 12 disks EH0600JEDHE | + | +-----------------+------------------------+------------------------+ + | |SSD/HDD |HDD |HDD | + | +-----------------+------------------------+------------------------+ + | |size | 3,6 TB | 3,6 TB | + +--------+-----------------+------------------------+------------------------+ + +Software +~~~~~~~~ + +.. table:: **Services on servers by role** + + +------------+----------------------------+ + |Role |Service name | + +============+============================+ + |controller || horizon | + | || keystone | + | || nova-api | + | || nava-scheduler | + | || nova-cert | + | || nova-conductor | + | || nova-consoleauth | + | || nova-consoleproxy | + | || cinder-api | + | || cinder-backup | + | || cinder-scheduler | + | || cinder-volume | + | || glance-api | + | || glance-glare | + | || glance-registry | + | || neutron-dhcp-agent | + | || neutron-l3-agent | + | || neutron-metadata-agent | + | || neutron-openvswitch-agent | + | || neutron-server | + | || heat-api | + | || heat-api-cfn | + | || heat-api-cloudwatch | + | || ceph-mon | + | || rados-gw | + | || heat-engine | + | || memcached | + | || rabbitmq_server | + | || mysqld | + | || galera | + | || corosync | + | || pacemaker | + | || haproxy | + +------------+----------------------------+ + |compute-osd || nova-compute | + | || neutron-l3-agent | + | || neutron-metadata-agent | + | || neutron-openvswitch-agent | + | || ceph-osd | + +------------+----------------------------+ + + +High availability cluster architecture +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Controller nodes: + +.. image:: https://docs.mirantis.com/openstack/fuel/fuel-8.0/_images/logical-diagram-controller.svg + :height: 700px + :width: 600px + :alt: Mirantis reference HA architecture + +Compute nodes: + +.. image:: https://docs.mirantis.com/openstack/fuel/fuel-8.0/_images/logical-diagram-compute.svg + :height: 250px + :width: 350px + :alt: Mirantis reference HA architecture + + +Networking +~~~~~~~~~~ + +All servers have the similar network configuration: + +.. image:: images/Network_Scheme.png + :alt: Network Scheme of the environment + +The following example shows a part of a switch configuration for each switch +port that is connected to ens1f0 interface of a server: + +.. code:: bash + + switchport mode trunk + switchport trunk native vlan 600 + switchport trunk allowed vlan 600-602,630-649 + spanning-tree port type edge trunk + spanning-tree bpduguard enable + no snmp trap link-status + + +Factors description +------------------- + +- **reboot-random-controller:** consists of a node-crash fault injection on a + random OpenStack controller node. + +- **sigkill-random-rabbitmq:** consists of a service-crash fault injection on + a random slave RabbitMQ messaging node. + +- **sigkill-random-mysql:** consists of a service-crash fault injection on a + random MySQL node. + +- **freeze-random-nova-api:** consists of a service-hang fault injection to + all nova-api process on a random controller node for a 150 seconds period. + +- **freeze-random-memcached:** consists of a service-hang fault injection to + the memcached service on a random controller node for a 150 seconds period. + +- **freeze-random-keystone:** consists of a service-hang fault injection to + the keystone (public and admin endpoints) service on a random controller + node for a 150 seconds period. + + +Testing process +=============== + +Use the following VM parameters for testing purposes: + +.. table:: **Test parameters** + + +--------------------------------+--------+ + |Name |Value | + +================================+========+ + |Flavor to create VM from |m1.tiny | + +--------------------------------+--------+ + |Image name to create VM from |cirros | + +--------------------------------+--------+ + +#. Create a work directory on a server with Rally role. + In this documentation, we name this directory ``WORK_DIR``. The path + example: ``/data/rally``. + +#. Create a directory ``plugins`` in ``WORK_DIR`` and copy the + :download:`scrappy.py ` plugin into that directory. + +#. Download the bash framework :download:`scrappy.sh ` + and :download:`scrappy.conf ` to + ``WORK_DIR/plugins``. + +#. Modify the ``scrappy.conf`` file with appropriate values. For example: + + .. literalinclude:: rally_plugins/scrappy.conf + :language: bash + +#. Create a ``scenarios`` directory in ``WORK_DIR`` and copy all Rally + scenarios with factors that you are planning to test to that directory. + For example: + :download:`random_controller_reboot_factor.json + `. + +#. Create a ``deployment.json`` file in the ``WORK_DIR`` and fill it with + your OpenStack environment info. It should looks like this: + + .. code:: json + + { + "admin": { + "password": "password", + "tenant_name": "tenant", + "username": "user" + }, + "auth_url": "http://1.2.3.4:5000/v2.0", + "region_name": "RegionOne", + "type": "ExistingCloud", + "endpoint_type": "internal", + "admin_port": 35357, + "https_insecure": true + } + +#. Prepare for tests: + + .. code:: bash + + ${WORK_DIR:?} + DEPLOYMENT_NAME="$(uuidgen)" + DEPLOYMENT_CONFIG="${WORK_DIR}/deployment.json" + rally deployment create --filename $(DEPLOYMENT_CONFIG) --name $(DEPLOYMENT_NAME) + +#. Create a ``/root/scrappy`` directory on every node in your OpenStack + environment and copy :download:`scrappy_host.sh ` + to that directory. + +#. Perform tests: + + .. code:: bash + + PLUGIN_PATH="${WORK_DIR}/plugins" + SCENARIOS="random_controller_reboot_factor.json" + for scenario in SCENARIOS; do + rally --plugin-paths ${PLUGINS_PATH} task start --tag ${scenario} ${WORK_DR}/scenarios/${scenario} + done + task_list="$(rally task list --uuids-only)" + rally task report --tasks ${task_list} --out=${WORK_DIR}/rally_report.html + +Once these steps are done, you get an HTML file with Rally test results. + + +Test case 1: NovaServers.boot_and_delete_server +----------------------------------------------- + +**Description** + +This Rally scenario boots and deletes virtual instances with injected fault +factors using OpenStack Nova API. + +**Service-level agreement** + +=================== ======== +Parameter Value +=================== ======== +MTTR (sec) <=240 +Failure rate (%) <=95 +Auto-healing Yes +=================== ======== + +**Parameters** + +=================== ======== +Parameter Value +=================== ======== +Runner constant +Concurrency 5 +Times 100 +Injection-iteration 20 +Testing-cycles 5 +=================== ======== + +**List of reliability metrics** + +======== ============== ================= ================================================= +Priority Value Measurement Units Description +======== ============== ================= ================================================= +1 SLA Boolean Service-level agreement result +2 Auto-healing Boolean Is cluster auto-healed after fault-injection +3 Failure rate Percents Test iteration failure ratio +4 MTTR (auto) Seconds Automatic mean time to repair +5 MTTR (manual) Seconds Manual mean time to repair, if Auto MTTR is Inf. +======== ============== ================= ================================================= + +Test case 1 results +------------------- + +reboot-random-controller +~~~~~~~~~~~~~~~~~~~~~~~~ + +**Rally scenario used during factor testing:** + +.. literalinclude:: rally_scenarios/NovaServers/boot_and_delete_server/random_controller_reboot_factor.json + :language: bash + +**Factor testing results:** + +.. table:: **Full description of cyclic execution results** + + +--------+-----------+-----------------+--------------+-------------------------+ + | Cycles | MTTR(sec) | Failure rate(%) | Auto-healing | Performance degradation | + +--------+-----------+-----------------+--------------+-------------------------+ + | 1 | 4.31 | 2 | Yes | Yes, up to 148.52 sec. | + +--------+-----------+-----------------+--------------+-------------------------+ + | 2 | 19.88 | 14 | Yes | Yes, up to 150.946 sec. | + +--------+-----------+-----------------+--------------+-------------------------+ + | 3 | 7.31 | 8 | Yes | Yes, up to 124.593 sec. | + +--------+-----------+-----------------+--------------+-------------------------+ + | 4 | 95.07 | 9 | Yes | Yes, up to 240.893 | + +--------+-----------+-----------------+--------------+-------------------------+ + | 5 | Inf. | 80.00 | No | Inf. | + +--------+-----------+-----------------+--------------+-------------------------+ + +**Rally report:** :download:`reboot_random_controller.html <../../../../raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/reboot_random_controller.html>` + +.. table:: **Testing results summary** + + +---------------+-----------------+------------------+ + | Value | MTTR(sec) | Failure rate | + +---------------+-----------------+------------------+ + | Min | 4.31 | 2 | + +---------------+-----------------+------------------+ + | Max | 95.07 | 80 | + +---------------+-----------------+------------------+ + | SLA | Yes | No | + +---------------+-----------------+------------------+ + +**Detailed results description** + +This factor affects OpenStack cluster operation on every run. +Auto-healing works, but may take a long time. In our testing results, the +cluster was recovered on the fifth testing cycle only, after Rally had +completed testing with the error status. Therefore, the performance degradation +is very significant during cluster recovering. + +sigkill-random-rabbitmq +~~~~~~~~~~~~~~~~~~~~~~~ + +**Rally scenario used during factor testing:** + +.. literalinclude:: rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.json + :language: bash + +**Factor testing results:** + +.. table:: **Full description of cyclic execution results** + + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | Cycles | MTTR(sec) | Failure rate(%) | Auto-healing | Performance degradation | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 1 | 0 | 0 | Yes | Yes, up to 12.266 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 2 | 0 | 0 | Yes | Yes, up to 15.775 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 3 | 98.52 | 1 | Yes | Yes, up to 145.115 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 4 | 0 | 0 | Yes | No | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 5 | 0 | 0 | Yes | Yes, up to 65.926 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + +**Rally report:** :download:`random_controller_kill_rabbitmq.html <../../../../raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.html>` + +.. table:: **Testing results summary** + + +--------+-----------+--------------+ + | Value | MTTR(sec) | Failure rate | + +--------+-----------+--------------+ + | Min | 0 | 0 | + +--------+-----------+--------------+ + | Max | 98.52 | 1 | + +--------+-----------+--------------+ + | SLA | Yes | Yes | + +--------+-----------+--------------+ + +**Detailed results description** + +This factor may affect OpenStack cluster operation. +Auto-healing works fine. +Performance degradation is significant during cluster recovering. + +sigkill-random-mysql +~~~~~~~~~~~~~~~~~~~~ + +**Rally scenario used during factor testing:** + +.. literalinclude:: rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.json + :language: bash + +**Factor testing results:** + +.. table:: **Full description of cyclic execution results** + + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | Cycles | MTTR(sec) | Failure rate(%) | Auto-healing | Performance degradation | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 1 | 2.31 | 0 | Yes | Yes, up to 12.928 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 2 | 0 | 0 | Yes | Yes, up to 11.156 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 3 | 0 | 1 | Yes | Yes, up to 13.592 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 4 | 0 | 0 | Yes | Yes, up to 11.864 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 5 | 0 | 0 | Yes | Yes, up to 12.715 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + +**Rally report:** :download:`random_controller_kill_mysqld.html <../../../../raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.html>` + +.. table:: **Testing results summary** + + +--------+-----------+--------------+ + | Value | MTTR(sec) | Failure rate | + +--------+-----------+--------------+ + | Min | 0 | 0 | + +--------+-----------+--------------+ + | Max | 2.31 | 1 | + +--------+-----------+--------------+ + | SLA | Yes | Yes | + +--------+-----------+--------------+ + +**Detailed results description** + +This factor may affect OpenStack cluster operation. +Auto-healing works fine. +Performance degradation is not significant. + + +freeze-random-nova-api +~~~~~~~~~~~~~~~~~~~~~~ + +**Rally scenario used during factor testing:** + +.. literalinclude:: rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_nova-api_150_sec.json + :language: bash + +**Factor testing results:** + +.. table:: **Full description of cyclic execution results** + + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | Cycles | MTTR(sec) | Failure rate(%) | Auto-healing | Performance degradation | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 1 | 0 | 0 | Yes | Yes, up to 156.935 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 2 | 0 | 0 | Yes | Yes, up to 155.085 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 3 | 0 | 0 | Yes | Yes, up to 156.93 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 4 | 0 | 0 | Yes | Yes, up to 156.782 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 5 | 150.55 | 1 | Yes | Yes, up to 154.741 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + +**Rally report:** :download:`random_controller_freeze_nova_api_150_sec.html <../../../../raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_nova_api_150_sec.html>` + +.. table:: **Testing results summary** + + +--------+-----------+--------------+ + | Value | MTTR(sec) | Failure rate | + +--------+-----------+--------------+ + | Min | 0 | 0 | + +--------+-----------+--------------+ + | Max | 150.55 | 1 | + +--------+-----------+--------------+ + | SLA | Yes | Yes | + +--------+-----------+--------------+ + +**Detailed results description** + +This factor affects OpenStack cluster operation. +Auto-healing does not work. Cluster operation was recovered +only after sending SIGCONT POSIX signal to all freezed nova-api +processes. Performance degradation is determined by the factor duration time. +This behaviour is not normal for an HA OpenStack configuration +and should be investigated. + + +freeze-random-memcached +~~~~~~~~~~~~~~~~~~~~~~~ + +**Rally scenario used during factor testing:** + +.. literalinclude:: rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.json + :language: bash + +**Factor testing results:** + +.. table:: **Full description of cyclic execution results** + + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | Cycles | MTTR(sec) | Failure rate(%) | Auto-healing | Performance degradation | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 1 | 0 | 0 | Yes | Yes, up to 26.679 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 2 | 0 | 0 | Yes | Yes, up to 23.726 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 3 | 0 | 0 | Yes | Yes, up to 21.893 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 4 | 0 | 0 | Yes | Yes, up to 22.796 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + | 5 | 0 | 0 | Yes | Yes, up to 27.737 sec. | + +--------------------+----------------+---------------------+------------------+-----------------------------+ + +**Rally report:** :download:`random_controller_freeze_memcached_150_sec.html <../../../../raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.html>` + +.. table:: **Testing results summary** + + +--------+-----------+--------------+ + | Value | MTTR(sec) | Failure rate | + +--------+-----------+--------------+ + | Min | 0 | 0 | + +--------+-----------+--------------+ + | Max | 0 | 0 | + +--------+-----------+--------------+ + | SLA | Yes | Yes | + +--------+-----------+--------------+ + +**Detailed results description** + +This factor does not affect an OpenStack cluster operations. +During the factor testing, a small performance degradation is observed. + +freeze-random-keystone +~~~~~~~~~~~~~~~~~~~~~~ + +**Rally scenario used during factor testing:** + +.. literalinclude:: rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.json + :language: bash + +**Factor testing results:** + +.. table:: **Full description of cyclic execution results** + + +--------+-------------+-----------------+--------------+-------------------------+ + | Cycles | MTTR(sec) | Failure rate(%) | Auto-healing | Performance degradation | + +--------+-------------+-----------------+--------------+-------------------------+ + | 1 | 97.19 | 7 | Yes | No | + +--------+-------------+-----------------+--------------+-------------------------+ + | 2 | 93.87 | 6 | Yes | No | + +--------+-------------+-----------------+--------------+-------------------------+ + | 3 | 92.12 | 8 | Yes | No | + +--------+-------------+-----------------+--------------+-------------------------+ + | 4 | 94.51 | 6 | Yes | No | + +--------+-------------+-----------------+--------------+-------------------------+ + | 5 | 98.37 | 7 | Yes | No | + +--------+-------------+-----------------+--------------+-------------------------+ + +**Rally report:** :download:`random_controller_freeze_keystone_150_sec.html <../../../../raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.html>` + +.. table:: **Testing results summary** + + +--------+-----------+--------------+ + | Value | MTTR(sec) | Failure rate | + +--------+-----------+--------------+ + | Min | 92.12 | 6 | + +--------+-----------+--------------+ + | Max | 98.37 | 8 | + +--------+-----------+--------------+ + | SLA | Yes | No | + +--------+-----------+--------------+ + +**Detailed results description** + +This factor affects an OpenStack cluster operations. +After the keystone processes freeze on controllers, the HA +logic needs approximately 95 seconds to recover service operation. +After recovering, performance degradation is not observed but +only at small concurrency. This behaviour is not normal +for an HA OpenStack configuration and should be investigated in future. + +.. references: +.. _Rally installation documentation: https://rally.readthedocs.io/en/latest/install.html \ No newline at end of file diff --git a/doc/source/test_results/reliability/rally_plugins/scrappy.conf b/doc/source/test_results/reliability/rally_plugins/scrappy.conf new file mode 100644 index 0000000..8c4a088 --- /dev/null +++ b/doc/source/test_results/reliability/rally_plugins/scrappy.conf @@ -0,0 +1,16 @@ +# SSH credentials +SSH_LOGIN="root" +SSH_PASS="r00tme" + +# Controller nodes +CONTROLLERS[0]="10.44.0.7" +CONTROLLERS[1]="10.44.0.6" +CONTROLLERS[2]="10.44.0.5" + +# Compute nodes +COMPUTES[0]="10.44.0.3" +COMPUTES[1]="10.44.0.4" +COMPUTES[2]="10.44.0.8" + +#Scrappy base path +SCRAPPY_BASE="/root/scrappy" diff --git a/doc/source/test_results/reliability/rally_plugins/scrappy.py b/doc/source/test_results/reliability/rally_plugins/scrappy.py new file mode 100644 index 0000000..8e2fc84 --- /dev/null +++ b/doc/source/test_results/reliability/rally_plugins/scrappy.py @@ -0,0 +1,115 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Rully scrappy plugin +This is pluging was designed for OpenStack +reliability testing. +""" + +from rally.common.i18n import _ +from rally import consts +from rally.task import sla +import os +from rally.common import logging +from rally.common import streaming_algorithms as streaming + + +LOG = logging.getLogger(__name__) + +class MttrCalculation(): + def __init__(self): + self.min_timestamp = streaming.MinComputation() + self.max_timestamp = streaming.MaxComputation() + self.mttr = 0 + self.last_error_duration = 0 + self.last_iteration = None + + def add(self, iteration): + if iteration["error"]: + # Store duration of last error iteration + if self.max_timestamp.result() < iteration["timestamp"]: + self.last_error_duration = iteration["duration"] + + self.min_timestamp.add(iteration["timestamp"]) + self.max_timestamp.add(iteration["timestamp"]) + LOG.info("TIMESTAMP: %s" % iteration["timestamp"]) + + self.last_iteration = iteration + + def result(self): + self.mttr = round(self.max_timestamp.result() - + self.min_timestamp.result() + + self.last_error_duration, 2) + # SLA Context don't have information about iterations count, + # so assume that if last iteration completed with error, + # that cluster was not auto-healed + if self.last_iteration["error"]: + self.mttr = "Inf." + return(self.mttr) + + +@sla.configure(name="scrappy") +class Scrappy(sla.SLA): + """Scrappy events.""" + CONFIG_SCHEMA = { + "type": "object", + "$schema": consts.JSON_SCHEMA, + "properties": { + "on_iter": {"type": "number"}, + "execute": {"type": "string"}, + "cycle": {"type": "number"} + } + } + + def __init__(self, criterion_value): + super(Scrappy, self).__init__(criterion_value) + self.on_iter = self.criterion_value.get("on_iter", None) + self.execute = self.criterion_value.get("execute", None) + self.cycle = self.criterion_value.get("cycle", 0) + self.errors = 0 + self.total = 0 + self.error_rate = 0.0 + self.mttr = MttrCalculation() + + def add_iteration(self, iteration): + self.total += 1 + if iteration["error"]: + self.errors += 1 + + self.mttr.add(iteration) + + """Start iteration event""" + if self.on_iter == self.total: + LOG.info("Scrappy testing cycle: ITER: %s" % self.cycle) + LOG.info("Scrappy executing: %s" % self.on_iter) + os.system(self.execute) + + self.error_rate = self.errors * 100.0 / self.total + self.success = self.error_rate <= 5 + return self.success + + def merge(self, other): + self.total += other.total + self.errors += other.errors + if self.total: + self.error_rate = self.errors * 100.0 / self.total + self.success = self.error_rate <= 5 + return self.success + + def details(self): + return (_("Scrappy failure rate %.2f%% MTTR %s seconds - %s") % + (self.error_rate, self.mttr.result(), self.status())) diff --git a/doc/source/test_results/reliability/rally_plugins/scrappy.sh b/doc/source/test_results/reliability/rally_plugins/scrappy.sh new file mode 100755 index 0000000..fdbc33a --- /dev/null +++ b/doc/source/test_results/reliability/rally_plugins/scrappy.sh @@ -0,0 +1,116 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +#!/bin/bash -xe + +# source credentionals +if [ -f /data/rally/rally_plugins/scrappy/scrappy.conf ]; +then + . /data/rally/rally_plugins/scrappy/scrappy.conf +else + exit -1 +fi + +# +# Function exetute command over ssh +# Login & password stored in scrappy.conf +# +function ssh_exec() { + local ssh_node=$1 + local ssh_cmd=$2 + local ssh_options='-oConnectTimeout=5 -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null -oRSAAuthentication=no' + echo "sshpass -p ${SSH_PASS} ssh ${ssh_options} ${SSH_LOGIN}@${ssh_node} ${ssh_cmd}" + local ssh_result=`sshpass -p ${SSH_PASS} ssh ${ssh_options} ${SSH_LOGIN}@${ssh_node} ${ssh_cmd}` + echo "$ssh_result" +} + +# +# Function return random controller node from Fuel cluster +# +function get_random_controller() { + local random_controller=${CONTROLLERS[$RANDOM % ${#CONTROLLERS[@]}]} + echo $random_controller +} + +# +# Function return random compute node from Fuel cluster +# +function get_random_compute() { + local random_compute=${COMPUTES[$RANDOM % ${#COMPUTES[@]}]} + echo $random_compute +} + +# +# Factors +# +function random_controller_kill_rabbitmq() { + local action=$1 + local controller_node=$(get_random_controller) + local result=`ssh_exec ${controller_node} "${SCRAPPY_BASE}/scrappy_host.sh send_signal rabbitmq_server -KILL"` + echo "$result" +} + +function random_controller_freeze_process_random_interval() { + local process_name=$1 + local interval=$2 + local controller_node=$(get_random_controller) + local result=`ssh_exec ${controller_node} "${SCRAPPY_BASE}/scrappy_host.sh freeze_process_random_interval ${process_name} ${interval}"` + echo "$result" +} + +function random_controller_freeze_process_fixed_interval() { + local process_name=$1 + local interval=$2 + local controller_node=$(get_random_controller) + local result=`ssh_exec ${controller_node} "${SCRAPPY_BASE}/scrappy_host.sh freeze_process_fixed_interval ${process_name} ${interval}"` + echo "$result" +} + +function random_controller_reboot() { + local controller_node=$(get_random_controller) + local result=`ssh_exec ${controller_node} "${SCRAPPY_BASE}/scrappy_host.sh reboot_node"` + echo "$result" +} + +function usage() { + echo "usage" + echo "TODO" +} + +# +# Main +# +function main() { + local factor=$1 + case ${factor} in + random_controller_kill_rabbitmq) + random_controller_kill_rabbitmq $2 + ;; + random_controller_freeze_process_random_interval) + random_controller_freeze_process_random_interval $2 $3 + ;; + random_controller_freeze_process_fixed_interval) + random_controller_freeze_process_fixed_interval $2 $3 + ;; + random_controller_reboot) + random_controller_reboot + ;; + *) + usage + ;; + esac +} + +main "$@" diff --git a/doc/source/test_results/reliability/rally_plugins/scrappy_host.sh b/doc/source/test_results/reliability/rally_plugins/scrappy_host.sh new file mode 100755 index 0000000..80919ee --- /dev/null +++ b/doc/source/test_results/reliability/rally_plugins/scrappy_host.sh @@ -0,0 +1,132 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +#!/bin/bash -xe + +LOG_FILE="/var/log/scrappy.log" + +# +# Logging function +# +function log() { + echo "`date -u` scrappy_host: $1" >> ${LOG_FILE} +} + +# +# This is function send specified signal +# to all processes with given name +# +function send_signal() { + local process_name=$1 + local signal=$2 + local pids=`ps -ef | grep $process_name | grep -v grep | grep -v scrappy_host | awk '{print $2}'` + for each_pid in ${pids}; + do + log "sending signal: ${signal} to ${process_name} with pid:$each_pid" + kill ${signal} ${each_pid} + done +} + +# +# This is function control services +# +function service_control() { + local service_name=$1 + local service_action=$2 + log "service control: $service_name action: $service_action" + service $service_name $service_action +} + +# +# This is function reboot node +# +function reboot_node() { + log "reboot" + shutdown -r now +} + +# +# This factor freeze specifid process +# +function freeze_process_random_interval { + local process_name=$1 + local max_interval=$2 + local interval=$(( ($RANDOM % ${max_interval}) + 1)) + log "freeze_process_random_interval: freezing process ${process_name} freeze interval ${interval}" + send_signal ${process_name} '-STOP' + sleep ${interval} + log "freeze_process_random_interval: unfreezing process ${process_name}" + send_signal ${process_name} '-CONT' +} + +# +# This factor freeze specifid process +# +function freeze_process_fixed_interval { + local process_name=$1 + local interval=$2 + log "freeze_process_fixed_interval: freezing process ${process_name} freeze interval ${interval}" + send_signal ${process_name} '-STOP' + sleep ${interval} + log "freeze_process_fixed_interval: unfreezing process ${process_name}" + send_signal ${process_name} '-CONT' +} + +# +# Show usage +# +function usage() { + echo "scrappy_host usage:" + echo "scrappy_host commands:" + echo -e "\t send_signal process_name signal" + echo -e "\t service_control service_name action" + echo -e "\t freeze_process_random_interval process max_interval" + echo -e "\t freeze_process_fixed_interval process interval" + echo -e "\t reboot_node" +} + +# +# main +# +function main() { + local command=$1 + case $command in + send_signal) + send_signal $2 $3 + ;; + service_control) + service_control $2 $3 + ;; + reboot_node) + reboot_node + ;; + freeze_process_random_interval) + set +xe + freeze_process_random_interval $2 $3 & + set -xe + ;; + freeze_process_fixed_interval) + set +xe + freeze_process_fixed_interval $2 $3 & + set -xe + ;; + *) + usage + exit -1 + ;; + esac +} + +main "$@" diff --git a/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.json b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.json new file mode 100644 index 0000000..d190347 --- /dev/null +++ b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.json @@ -0,0 +1,37 @@ +{% set flavor_name = flavor_name or "m1.tiny" %} +{% set image_name = image_name or "^(cirros.*uec|TestVM)$" %} +{ + "NovaServers.boot_and_delete_server": [ + {% for i in range (0, 5, 1) %} + { + "args": { + "flavor": { + "name": "{{flavor_name}}" + }, + "image": { + "name": "{{image_name}}" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + }, + "sla": { + "scrappy": { + "on_iter": 20, + "execute": "/bin/bash /data/rally/rally_plugins/scrappy/scrappy.sh random_controller_freeze_process_fixed_interval keystone 150", + "cycle": {{i}} + }, + } + }, + {% endfor %} + ] +} diff --git a/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.json b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.json new file mode 100644 index 0000000..980e23a --- /dev/null +++ b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.json @@ -0,0 +1,37 @@ +{% set flavor_name = flavor_name or "m1.tiny" %} +{% set image_name = image_name or "^(cirros.*uec|TestVM)$" %} +{ + "NovaServers.boot_and_delete_server": [ + {% for i in range (0, 5, 1) %} + { + "args": { + "flavor": { + "name": "{{flavor_name}}" + }, + "image": { + "name": "{{image_name}}" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + }, + "sla": { + "scrappy": { + "on_iter": 20, + "execute": "/bin/bash /data/rally/rally_plugins/scrappy/scrappy.sh random_controller_freeze_process_fixed_interval memcached 150", + "cycle": {{i}} + }, + } + }, + {% endfor %} + ] +} diff --git a/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_nova-api_150_sec.json b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_nova-api_150_sec.json new file mode 100644 index 0000000..553f7f8 --- /dev/null +++ b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_freeze_nova-api_150_sec.json @@ -0,0 +1,37 @@ +{% set flavor_name = flavor_name or "m1.tiny" %} +{% set image_name = image_name or "^(cirros.*uec|TestVM)$" %} +{ + "NovaServers.boot_and_delete_server": [ + {% for i in range (0, 5, 1) %} + { + "args": { + "flavor": { + "name": "{{flavor_name}}" + }, + "image": { + "name": "{{image_name}}" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 15 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + }, + "sla": { + "scrappy": { + "on_iter": 20, + "execute": "/bin/bash /data/rally/rally_plugins/scrappy/scrappy.sh random_controller_freeze_process_fixed_interval nova-api 150", + "cycle": {{i}} + }, + } + }, + {% endfor %} + ] +} diff --git a/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.json b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.json new file mode 100644 index 0000000..94ac35a --- /dev/null +++ b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.json @@ -0,0 +1,38 @@ +{% set flavor_name = flavor_name or "m1.tiny" %} +{% set image_name = image_name or "^(cirros.*uec|TestVM)$" %} +{ + "NovaServers.boot_and_delete_server": [ + {% for i in range (0, 5, 1) %} + { + + "args": { + "flavor": { + "name": "{{flavor_name}}" + }, + "image": { + "name": "{{image_name}}" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + }, + "sla": { + "scrappy": { + "on_iter": 20, + "execute": "/bin/bash /data/rally/rally_plugins/scrappy/scrappy.sh send_signal /usr/sbin/mysqld -KILL", + "cycle": {{i}} + } + } + }, + {% endfor %} + ] +} diff --git a/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.json b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.json new file mode 100644 index 0000000..a237b02 --- /dev/null +++ b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.json @@ -0,0 +1,38 @@ +{% set flavor_name = flavor_name or "m1.tiny" %} +{% set image_name = image_name or "^(cirros.*uec|TestVM)$" %} +{ + "NovaServers.boot_and_delete_server": [ + {% for i in range (0, 5, 1) %} + { + + "args": { + "flavor": { + "name": "{{flavor_name}}" + }, + "image": { + "name": "{{image_name}}" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + }, + "sla": { + "scrappy": { + "on_iter": 20, + "execute": "/bin/bash /data/rally/rally_plugins/scrappy/scrappy.sh random_controller_kill_rabbitmq", + "cycle": {{i}} + } + } + }, + {% endfor %} + ] +} diff --git a/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_reboot_factor.json b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_reboot_factor.json new file mode 100644 index 0000000..9fff00c --- /dev/null +++ b/doc/source/test_results/reliability/rally_scenarios/NovaServers/boot_and_delete_server/random_controller_reboot_factor.json @@ -0,0 +1,38 @@ +{% set flavor_name = flavor_name or "m1.tiny" %} +{% set image_name = image_name or "^(cirros.*uec|TestVM)$" %} +{ + "NovaServers.boot_and_delete_server": [ + {% for i in range (0, 5, 1) %} + { + + "args": { + "flavor": { + "name": "{{flavor_name}}" + }, + "image": { + "name": "{{image_name}}" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + }, + "sla": { + "scrappy": { + "on_iter": 20, + "execute": "/bin/bash /data/rally/rally_plugins/scrappy/scrappy.sh random_controller_reboot", + "cycle": {{i}} + } + } + }, + {% endfor %} + ] +} diff --git a/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.html b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.html new file mode 100644 index 0000000..87a0460 --- /dev/null +++ b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_keystone_150_sec.html @@ -0,0 +1,856 @@ + + + + + + Rally | Rally Task Report + + + + + + + + + + + + + +
+
+ Rally  + task results +
+
+ +
+ + + + +
+
+ + +
+
+ + +
+
+ +
+ +
+

Task overview

+ + + + + + + + + +
+ Scenario + + + + + + Load duration (s) + + + + + + Full duration (s) + + + + + + Iterations + + + + + + Runner + + + + + + Errors + + + + + + Success (SLA) + + + + +
{{sc.ref}} + {{sc.load_duration | number:3}} + {{sc.full_duration | number:3}} + {{sc.iterations_count}} + {{sc.runner}} + {{sc.errors.length}} + + + +
+
+ +
+

Input file

+
{{source}}
+
+ +
+

{{scenario.cls}}.{{scenario.name}} ({{scenario.full_duration | number:3}}s)

+
    +
  • +
    {{t.name}}
    +
  • +
    +
+
+ + + + + + + + + + +
+ +
+
+ + +
+ + + + \ No newline at end of file diff --git a/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.html b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.html new file mode 100644 index 0000000..4415cf6 --- /dev/null +++ b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_memcached_150_sec.html @@ -0,0 +1,856 @@ + + + + + + Rally | Rally Task Report + + + + + + + + + + + + + +
+
+ Rally  + task results +
+
+ +
+ + + + +
+
+ + +
+
+ + +
+
+ +
+ +
+

Task overview

+ + + + + + + + + +
+ Scenario + + + + + + Load duration (s) + + + + + + Full duration (s) + + + + + + Iterations + + + + + + Runner + + + + + + Errors + + + + + + Success (SLA) + + + + +
{{sc.ref}} + {{sc.load_duration | number:3}} + {{sc.full_duration | number:3}} + {{sc.iterations_count}} + {{sc.runner}} + {{sc.errors.length}} + + + +
+
+ +
+

Input file

+
{{source}}
+
+ +
+

{{scenario.cls}}.{{scenario.name}} ({{scenario.full_duration | number:3}}s)

+
    +
  • +
    {{t.name}}
    +
  • +
    +
+
+ + + + + + + + + + +
+ +
+
+ + +
+ + + + \ No newline at end of file diff --git a/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_nova_api_150_sec.html b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_nova_api_150_sec.html new file mode 100644 index 0000000..2ca5a9e --- /dev/null +++ b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_freeze_nova_api_150_sec.html @@ -0,0 +1,856 @@ + + + + + + Rally | Rally Task Report + + + + + + + + + + + + + +
+
+ Rally  + task results +
+
+ +
+ + + + +
+
+ + +
+
+ + +
+
+ +
+ +
+

Task overview

+ + + + + + + + + +
+ Scenario + + + + + + Load duration (s) + + + + + + Full duration (s) + + + + + + Iterations + + + + + + Runner + + + + + + Errors + + + + + + Success (SLA) + + + + +
{{sc.ref}} + {{sc.load_duration | number:3}} + {{sc.full_duration | number:3}} + {{sc.iterations_count}} + {{sc.runner}} + {{sc.errors.length}} + + + +
+
+ +
+

Input file

+
{{source}}
+
+ +
+

{{scenario.cls}}.{{scenario.name}} ({{scenario.full_duration | number:3}}s)

+
    +
  • +
    {{t.name}}
    +
  • +
    +
+
+ + + + + + + + + + +
+ +
+
+ + +
+ + + + \ No newline at end of file diff --git a/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.html b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.html new file mode 100644 index 0000000..a29d27f --- /dev/null +++ b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_mysqld.html @@ -0,0 +1,856 @@ + + + + + + Rally | Rally Task Report + + + + + + + + + + + + + +
+
+ Rally  + task results +
+
+ +
+ + + + +
+
+ + +
+
+ + +
+
+ +
+ +
+

Task overview

+ + + + + + + + + +
+ Scenario + + + + + + Load duration (s) + + + + + + Full duration (s) + + + + + + Iterations + + + + + + Runner + + + + + + Errors + + + + + + Success (SLA) + + + + +
{{sc.ref}} + {{sc.load_duration | number:3}} + {{sc.full_duration | number:3}} + {{sc.iterations_count}} + {{sc.runner}} + {{sc.errors.length}} + + + +
+
+ +
+

Input file

+
{{source}}
+
+ +
+

{{scenario.cls}}.{{scenario.name}} ({{scenario.full_duration | number:3}}s)

+
    +
  • +
    {{t.name}}
    +
  • +
    +
+
+ + + + + + + + + + +
+ +
+
+ + +
+ + + + \ No newline at end of file diff --git a/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.html b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.html new file mode 100644 index 0000000..f26c775 --- /dev/null +++ b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/random_controller_kill_rabbitmq.html @@ -0,0 +1,856 @@ + + + + + + Rally | Rally Task Report + + + + + + + + + + + + + +
+
+ Rally  + task results +
+
+ +
+ + + + +
+
+ + +
+
+ + +
+
+ +
+ +
+

Task overview

+ + + + + + + + + +
+ Scenario + + + + + + Load duration (s) + + + + + + Full duration (s) + + + + + + Iterations + + + + + + Runner + + + + + + Errors + + + + + + Success (SLA) + + + + +
{{sc.ref}} + {{sc.load_duration | number:3}} + {{sc.full_duration | number:3}} + {{sc.iterations_count}} + {{sc.runner}} + {{sc.errors.length}} + + + +
+
+ +
+

Input file

+
{{source}}
+
+ +
+

{{scenario.cls}}.{{scenario.name}} ({{scenario.full_duration | number:3}}s)

+
    +
  • +
    {{t.name}}
    +
  • +
    +
+
+ + + + + + + + + + +
+ +
+
+ + +
+ + + + \ No newline at end of file diff --git a/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/reboot_random_controller.html b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/reboot_random_controller.html new file mode 100644 index 0000000..55ff4f6 --- /dev/null +++ b/raw_results/reliability/rally_results/NovaServers/boot_and_delete_server/reboot_random_controller.html @@ -0,0 +1,856 @@ + + + + + + Rally | Rally Task Report + + + + + + + + + + + + + +
+
+ Rally  + task results +
+
+ +
+ + + + +
+
+ + +
+
+ + +
+
+ +
+ +
+

Task overview

+ + + + + + + + + +
+ Scenario + + + + + + Load duration (s) + + + + + + Full duration (s) + + + + + + Iterations + + + + + + Runner + + + + + + Errors + + + + + + Success (SLA) + + + + +
{{sc.ref}} + {{sc.load_duration | number:3}} + {{sc.full_duration | number:3}} + {{sc.iterations_count}} + {{sc.runner}} + {{sc.errors.length}} + + + +
+
+ +
+

Input file

+
{{source}}
+
+ +
+

{{scenario.cls}}.{{scenario.name}} ({{scenario.full_duration | number:3}}s)

+
    +
  • +
    {{t.name}}
    +
  • +
    +
+
+ + + + + + + + + + +
+ +
+
+ + +
+ + + + \ No newline at end of file