From e1ddea22ea0c12c61c5cf74da87248b806573c87 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Thu, 15 Sep 2016 12:34:36 +0300 Subject: [PATCH] New version of reliability test plan The new version defines MTTR, service downtime and operation degradation metrics. It is targeted to be executed with Rally and os-faults library. Change-Id: I31f74a41b1b2e725986e4593fde92768f0237aa4 --- doc/source/test_plans/reliability/index.rst | 14 + .../reliability/{ => version_1}/plan.rst | 0 .../{ => version_1}/plan_conventions.rst | 0 .../reliability/{ => version_1}/template.rst | 0 .../code/rally_plugins/fault_injection.py | 49 ++++ .../reliability/version_2/hypothesis.png | Bin 0 -> 40539 bytes .../test_plans/reliability/version_2/plan.rst | 264 ++++++++++++++++++ 7 files changed, 327 insertions(+) create mode 100644 doc/source/test_plans/reliability/index.rst rename doc/source/test_plans/reliability/{ => version_1}/plan.rst (100%) rename doc/source/test_plans/reliability/{ => version_1}/plan_conventions.rst (100%) rename doc/source/test_plans/reliability/{ => version_1}/template.rst (100%) create mode 100644 doc/source/test_plans/reliability/version_2/code/rally_plugins/fault_injection.py create mode 100644 doc/source/test_plans/reliability/version_2/hypothesis.png create mode 100644 doc/source/test_plans/reliability/version_2/plan.rst diff --git a/doc/source/test_plans/reliability/index.rst b/doc/source/test_plans/reliability/index.rst new file mode 100644 index 0000000..6110ebb --- /dev/null +++ b/doc/source/test_plans/reliability/index.rst @@ -0,0 +1,14 @@ +.. raw:: pdf + + PageBreak oneColumn + +============================= +OpenStack reliability testing +============================= + +.. toctree:: + :maxdepth: 3 + :glob: + + */plan + */index diff --git a/doc/source/test_plans/reliability/plan.rst b/doc/source/test_plans/reliability/version_1/plan.rst similarity index 100% rename from doc/source/test_plans/reliability/plan.rst rename to doc/source/test_plans/reliability/version_1/plan.rst diff --git a/doc/source/test_plans/reliability/plan_conventions.rst b/doc/source/test_plans/reliability/version_1/plan_conventions.rst similarity index 100% rename from doc/source/test_plans/reliability/plan_conventions.rst rename to doc/source/test_plans/reliability/version_1/plan_conventions.rst diff --git a/doc/source/test_plans/reliability/template.rst b/doc/source/test_plans/reliability/version_1/template.rst similarity index 100% rename from doc/source/test_plans/reliability/template.rst rename to doc/source/test_plans/reliability/version_1/template.rst diff --git a/doc/source/test_plans/reliability/version_2/code/rally_plugins/fault_injection.py b/doc/source/test_plans/reliability/version_2/code/rally_plugins/fault_injection.py new file mode 100644 index 0000000..a0a6477 --- /dev/null +++ b/doc/source/test_plans/reliability/version_2/code/rally_plugins/fault_injection.py @@ -0,0 +1,49 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os_faults + +from rally.common import logging +from rally import consts +from rally.task import hook + +LOG = logging.getLogger(__name__) + + +@hook.configure(name="fault_injection") +class FaultInjectionHook(hook.Hook): + """Performs fault injection.""" + + CONFIG_SCHEMA = { + "type": "object", + "$schema": consts.JSON_SCHEMA, + "properties": { + "action": {"type": "string"}, + }, + "required": [ + "action", + ], + "additionalProperties": False, + } + + def run(self): + LOG.debug("Injecting fault: %s", self.config["action"]) + injector = os_faults.connect() + + try: + os_faults.human_api(injector, self.config["action"]) + self.set_status(consts.HookStatus.SUCCESS) + except Exception as e: + self.set_status(consts.HookStatus.FAILED) + self.set_error(exception_name=type(e), + description='Fault injection failure', + details=str(e)) diff --git a/doc/source/test_plans/reliability/version_2/hypothesis.png b/doc/source/test_plans/reliability/version_2/hypothesis.png new file mode 100644 index 0000000000000000000000000000000000000000..56023935dde1cf0fa18eef0388218e35f56c0f2a GIT binary patch literal 40539 zcmeGE1yq#VA3uyDpr9N;L_k17Is{aZ9zc{Bx;qY{QUcNqq97vaP|{sPjFQrdgmg1V zcMTyi49(r+IiC8RbJu&v#m|# zuQm{vzYk8VmcD-eMRK-fm?CeV9D~oiyN|wp{6-zd#AT&l#&)!< z@9SIUa;pKhm+`n@jBQzZVT(Pf2<-wLPHt{R85x;yq-v^!v8v=r*od~vrikf_|EhrTOGq_hvSHq~HCA0b!giH6@zfx$9Pjxa)) ztKhOO|7uVLIahgI_?Z;AXoQK!0!6vqM0Fp=qt2wcxtWuT3qI@~L_!yxm^d~*fRBSC z_59)({$Z=+=?A-yGZ(_>A)|<+D_5>)X=r4$w=2jz@UHXs+g(;Kw?P(nd)d{j3lbkd=Mr&9jsA*VfSG*+JF%s0HhZZFbs0!#987?xksjvJ9iF-ypR&EnFTx?8x z+36zsHUe3&z3sR#KW{lv?IL~B=tv-Xo3p$ZBL8kKevt96z3Q}wVc zFjuUO_qXoOF?fIXe?`h*vIJ>sYkTwNjmDEFi}5OSVa`)N80hwi)O`3IXb=lRx4kQd ziyPG-AYq~ay`8A9ShZPistBX^T00f3Phe>?7fpROeJV;S*)&dnyAoMxZ|br!{%zd~ zzwEtI+E}GMLd0otkY=pRD$2xt=KRStfn@Ky+ve6S7Trnti26eXA}Zc!7UFBSq+Sw` z4j1Z|IuB{<=KCD&C5X8d?`=#Jv_-KakK%7WFpM*HjUUq_6pJ$+{>c6Ff zIIBoVDLB>1+nAW_i5fR#?Vh{1LB-Sf;`&2t^m5;QIx*M5dY>Z!i|*^hw1S<@KUNOz z#a+MMl>s^MXhe(p?E5lga))?R-6lOoOCE=S%gCIaeMo#=SPdMoGKm+=eYEdFUYLPP=)RIyBy_8LR~x%~)8nckf$zVFR?2l^uh1gd zCHA4Y`9zAW6j4X+vybW3P%^Mdb^{m?EVshL`=SL#|vPed<_wJ0W@dVNm|2ZxYmO$yI! zwKtv6*g=MUef*ok`lz;bd$f51|I+)h0bCr$=?{3e0c_HSCJ0><1QPp%bWMVXx|55v zv{@-AWi6ZE@Y`J5@+56TK?apd@0nsvzef*}cn>mUA{2>jM*Vb92zZ=~39l0_Mlt zQep7pzki~IWJx{-Y-gO5dyJNl;nqdte#iViy6PA5kueDG}1o9Oz?Nu`Mv&G{a z39F9%*9k>*gnw9v znbee(edT+CGPOC}U8z1hJsK*v(QQ3?()li(gie%0Lc)YuE*ug=@g(xpqz%@n%J2TQ5QW-8P1X-MgluKvgy_Xn$-ENmyLwXp+9a$PtL zzz;3Z;zidF%gO|x{V-NVgXt+8|9qwX8*QiY7l0Pa>2Y>M-#Q$?I|*i=)Dc09klA~q;@|*f{{`huy|kGxLC7PIq*e2fsDd?zD}Ss1loc0u!6Omy+IwpI ztwd&FVPSQ$u7bx3V<~* zkN@Pu{{aq&#_&DYN6TqIzo>Rz$p9Rnz@jH5;m+eXfN)&jC z#4$_Ic64wW@*7zD$MlWQ1tqBmY8p)7B+y2!m^MiUdl^JPT?tD8zPXynOAmnh=!66~U^@Cr*apgbX^d{!db19b z0%%T|c`Hd*vY0Mr%CDi?b=_bSeT3PJYn@X^nABN^qMxM0*@dZZ;_Si~lqXio|%s@%N1 zJcBtpQD-$fJG(lJdZc|p^-KW4_py9C#(60Uv7fjoE-tRsjEkhPKEV9ndte=sorNV2 zG@6)Pfj~NlrxutP1(^Eraii?qTyAFO%;8i`hORGqJE388Vik*tMP5$&|I-Bb_A5g} zW9|v*n3xo&-{-S!s`IZjdErv%nM}}V)cE-6a&LP1l`5%3Q;u+O6dh2&&)tqwqbul! z0yLAeySqE!cs+?1+_81Cs7wFx!Hx^)Idxktv}lvKq@+S}b`=gu$%bKf6O$BJRFt7- z0vP{i`Xdr8#k_!ME2yXp52#8O)ofH3blXTA%##z-2*~p_96J^{Mx;QztoOgqcy2XQ zsn!Iwehy%!H`k?&_<;BP*L&F60&KB6xd2eK;pz=7Kq6-6xxAA8j&cKSaqAv7U-Frw zol<-Hv~UzqR-inr%tc5J|E(*6BM$!LmVi?aX3CL*=cG80tBOn#aT0hs^4?Pt;9SEb zqsI|Ze`6)3?|ElNN-ZKPo_@;!RNZBB(wNHF`E5pKrj7B=kgn;F_sOHjj~^S9J_&mh z5_0$1^rs6o+6UV5!WjXXf(pVojCzrk!bGUn5pX_K*^)>q{7(YPnga;aa~#TwACyT- zGxEPM`+s&AGh;m4VCLJl4)X<&@b49WB}4y*+Wgs-{vS(i{=e=q%>Mtu{{S8Qcgp&I z7hDRkz1RxMc-gZV>64w*vuGqrJAxkK+`&`1yiv2Mqokzt5))1jF$fF{EN=neaCb4$ ze%Ol?@NYP+$`ydOQ=t2Kc8p3&YxCPmjRZeQH<0DH1O&=PWR0r$KSuScXtnO`eJG>Y)$~}>eVmQ88_~~8VoH5_2)B|7HUsClU`3sUcDe~ zoxc84DlsuJp?B#e5eMS9ww-DtF+kVW$YSH74_+Hpay{ZZpz z)ln8jl~J#I>|4a&kW5canp#M2tlNu^ra~h+Reg5_dgf zv#?FSw$`XMg28^!>nbHByd=Ge5R}^+>`%#;HMO<0EDyF9s*krj=sIq*SOK~T#G#k7 zb8~kc=jB1Uq3*l_{@9M$|1^St!CNiW_b?x{-f8#s!F&T4_(ftH5Ae5Wpe>*y`jk8xun{ZuWq!WKY@v z1vx=fM&R7U=pJufS%Yl&^;z#53Cd{Ep08h@d9clxcfPwVuF^FEq&aO9lZ`!rMunyB zWT4W0A{xtmCf`rQSDOX|OPk4B4}j4WqNV_z_&YCLc|t5F9+)YZ(s&N%^NZSM{gK#b zwfnOnhpA@=3@67+lE}(>G|xI9O^d4f`ubfi{1)AV^KDT!CiWxm?-l42!4Is2`S?_= ztgJ-6_n$?%y~nEO;s6ktZ_PHt3kwIyRmIcUQ`|F#k%D&Pd7x1$!b(a?%DgZe2?#KG zl`F>`LztMEhbn9hDXw1S09r8-<;~vjGha|A;y|tWC!tDuuzVBDr#KVO{K>2eAgA9w ztF`RQFeujryaVKg|3xLOG)*B*HAy)A#S7eI@4b%Ip#t=LlU<;F&7RyH;T7=X+< zq>{&KQEbZCbGp^<(19u3yg&_nA;!V;+PFS@-z;o^q6Eu|;nszhlxP4&BaVem!dnkG zLUt;Mdw?Q1)~r10p^r%MU{j$Ls&us_83tXtCt zT(F7_k;ZPx044FY=}z{6l@n*7Ckbw>>YHPY5V}8v8)h6IbQegp zeE<{vR24noD!y4Vm)$WFZs;J%<9k>fiAO4))}1V-2*ZNR=;*5x9!tr*J>Q+dEQe*W zD#m^kB^`MutnRszjW9LsL&!7K?K_+ZHDRkGkK}D#R_OLe$VD6}W-LyEV;5p&!-=iW z(b$xP?Qwo?ildp}rxo66`8VOQ?d{%(nv^a?BM=c-@5M>s`ER6nWhp0em3ug4!^GmO zKfe^H#KpO+@EcwNP(AA~pC>jWBP-6dIMP{u+cSIIF~8rNX7Nu>FtaF1s{3Qabb;hE zGNM)}NgvBu@eMbK9Rn@n3xZF(5)VNHaC&dZ}lHD zSuFV0HHQpUVXgsW#D|U;E?W0+I(9N#99%-sbRvJlf`#}K=4;zKJLsdSGZU<@wKdmj zzI6Q1JG&2LU)8xGGBe=*JiX zVZZwYm%mA^`v)vmFEXrzo41DBskZWgqrmNiFP0O0XOK8{85&BzPc~Xnr2UDdcZfLP zdfr2JEu;tI50|B`wpN@()8Vq#MQ6Y`H8eF}@_NJy549TP`DWJb&pOfxUrz7Wi16Gi z?0s+c*iJS7LU3uFn(>l<=+9YvUYmR-M%N_Y4=Aj5s(8=33J{db@fL5;9j5es33r2l z<%Fk-0S}Ic39Ro{uyiEP4tlV}-FG*WAs_gAZ(i>oT0GY>^*%<8w1LE)Vw|4FM8d zoaqu!hNyUr@+41IRAE_DCmU|LJ|}Oui+h2<{0LGci>;3TFfQb&EFb>N0tk?a62b93 z*G4|90&Y@gYL6^Vo_TtVohAQ}_$`^7Tc80Kg!|a9J6tidO?p81TT4<~m(m@pb}7(7 zm=po(gdN^$(4t}z5)2CHu|k>UMX+*uQXqz)0=;Kh0qs@i-aUT>MMW2^K3r5esR69o z@nlBur_u&$w%>gBXM#MJ(}M)Cq>PFR^~oQPi<9$*0>M|m&y*Iil@r_>i$oP!iTe{H zFlx%oH=h24R9Jft42W*AQ9c%qc~1KiT&1GY<(B~10?Xrpp83qdl^D1vI!~TFQJ#a| zI@+J(Af_9I>KgUwGx%Ox-*cvkj<2fEd3#w<`f)M=Ik3=vcvX~Itk>njud8FNTT@hj zc1ZYjk%!`$Eue?!a5&q<#drufJS143q~nAtDB2);BzAySdF#m6gqSAWcf zArfa`8KajZ&-8(oUvSq1xcDABi}|JvzPw$(So&5~k#TVz9JW!khI^Fx`DKOhgDR~N z$=X{v;hHGwyqp#Sic{s3Z;dayz|;!p^@oc>4f&y=zhEVgFkh^{zXBFqz31Lq{>P8P z#_ltLhiHUf5iq93CJl9Ta!#9j23TGxeWg7T1@GRy6W#3iEwzAEnw1j-ps-yKSy)MO>h5#uf#`1X3Ih@#y)X|C z2;hQje>JPj?c)HvpHFMpQ+9}PCa zoX6@&Lq{IKf%eWUmz#b1SBLDpH!cLs=_0)k(whxu^^<;QJ|_)ln6uS7U-R24yMlEme+Tz-61<1jC6#6t?)WteVVh%f!K;S6OR*VysxWorA zAS){?0FQ-teSrri+93`10WkVcOC_ba3CK#x-i|Tt0EeECm}npW@R8p^hpB+y>9J@-#gi_Y zlkK>M-6IGU5Cp9Ta;yNpO}J(DVh#=R3Zf3P0iX@pge@&B40(MGVFAbJ+^kAp0qg`g z7h>ghbdQx{f&U8>Q@PHt$4{PEk5$;(sWKOW@e(n;x`?G{{vL(;bSXZ2`heYZ`#<%JdS7>D<_=4`Mzl6+u=pnw%7 zzMk^a>7s!M;n(~ToKQIkli~TR7V1uc^T`J|T`#N&s&6C;*$?YmBo8g4Oue^cPI}Ic zBYaPHQc;Fehq10eB-+UZPh_z=p5MZHd%o>1g5Z~HX}}k=0r>=AQ%193%O8OCMJ{yM zLaX^nh)kg$({SbkriTg$&4~IQf0O`B*1)uV}PGMO9aQt8%HXx(Z5z8|Gw5cL`h-V&X0HD>TVk1Ss3{3vqzzRWqua^tyRN2@#YU7I)9(OSDCZ;+D^yN*=B5Rh4!z(2z}7)F(@RKY{IfS zcx#ltef#z!=L8_MGG^Q<;=(W2rni5=vdmR5YY)NL_gSR+50>seZof(= z;`jj6GSBsLoos)HKDGIfp{=ro6`1&A^mBrh6;>{*~5j`+=v z6RQgTeu6Ov{nm>|zkhT=KTz;5p%O}f9DDpucM^iUbVEWzLBrTg*at;=%q# z&F$)g#XT_3h7&}~lb`g2yLnmP2>DBt;k@=mIVGh~V3mTfP1Nt9k|E^)p{a2L0@6cY zK}pA^%*v_op_io9BBi~}KUIE^2*KWTtf5^W5J}Pi8mb0x9DVIBTxn}d8O{migmhpAZH`xI0>3-ETNHH@D|v~pqky1IID(%525N|yJvca+w}-`B zOXxBin`=LF#PwnDd$RruP2mbnVXrN_?ec7IT(qBV0Soj49>MpY3w0sib27-u{lb!`Lq2M83 zwgX7~6p-ZI#Hr();Ss=~={4Dp1gNF@J@mC2Ac}BfeSuV9;adY*0-$LG zy~jLRWv(7y0-Cww*qQb^>%9!Rkg9neYK-S~P<>}KHFLIEv3OzqWf{i@hHE#(WXd$|EX%NUPb$g>W zyq%XUJwANU#&RO-$y#A;9R>5&ST5FufFIecn}D%Hfbc(XW%=E^YuMNt76JhJZk!O$ zkHt{4ZQ5X7X=&!tYsv&(Wngfx0UhZ9A?=@ciPcxYlOiCd$pU?<3KGT!!aF*7DMl1u zAJhK8K0n#d59|Y|&n@#CqqDR{kSi8{4d>hh7({HdR+F!8S0l5yZns|%4aC-p??0Y* zH?D+#{Yp;aAP5sCAgQUT0eHz|Q+k&M%o`8{mA7v=so2|`LNFXp6YzFl1yf{(5gZSI z80>_AdAq1ZK=$4NBSfPh{@1;=Z`E_*V!IK3fSs@@J+pxmXZ$a>`N3zFq%0miT0TS} z5IbiNqS#f30j-^FETjqgcf;lRl)r@j6D~;O(JH50008IFoEGP$_z@-AdXRz;9i0Se-i_{`& z`yqIYT^(8nf*#c5?Z$>gW*GCXSV9v7tYFxDsx8p&KwVu0;SwN@Vu5rOr2GQV*Z-*w zB!FgRjLQQB(7ZgQo9q`r)&EULOH)dFNk-I1Ic2&2R(R~M$+&w%fW_h9d<^>;tu7-2 z>&v5VGNFYW+i`~ep_*aE|9{mFsQ)WHhD|?(ym!{wkD|mj~ZXm zq=oGqQXum*S?^Or?>>DF8><8Y#*jZA0gcZl5PbfYk>a_n3@#u49XJUV?i?R`L;Has zjc%c}-*@o?a^prds#w-k6$IA)8S(>-KG2rjtU$-&wDdlJI{@U+y72t_Bd-wawXpuZ zn82~g+V9h!J-6G~?^^bzc_MQEJw4+}aHyn|EUL||TRaHFuyXgUhYO2~)`NNKqrlx8 z(*)RCiip|Y?gbXH0UW(L)zE-l8^Br61$HqArRvvV3QZeKfF|Ak^=lq_X)x#uKc#{> z_X0#q&1OQ2^z^- zr7x!Yhe>h2E9ZB3YtW8DoXKeTl=p%}SI*8wf(^vZ_xnE*umGEBfNL!WY0%2tEAJaU z2|U4%Fz+C^mX_9FtqKSz1LDeZ(0;f6aFv_K=%X5}%|@dc_&8 z6n}!4kP5eMOI~4TrtMgk*oI{;1^1;Sdwd+m&HxwOI)H>kTccnw>r)R9aVod=XH!Ya z14P(%vQ{5p0<5vbTl-zQxwW-&-`hX7%+`=QCdSA3$=f)pmD#5NJ{p-NOXUh=YzT;m z9F%>?%E~f;x|%-&a5jPPM9Ak*9c%#z-CGaQdiXFK2uPW}6Fq;;w}Pp^bxn-oK>Fc7=Y24(zPCUnX{WoO%zgFju z5@Moa#lMKA7hNB*E4%p%T1Y5r)O;0%dlcteuNIXg zAj!0En6#e5k-7$Us~GftA}H3%Hz-h`*KW)F3Ma>knK(%|ehb}VFXp8ok@L-rF2DXI zuJU5>3_c->8b}f4J%Aph^p|D=exirp!n8VR0?E$HF==;HpLU@fRYOPJ!fdC*rvaAv z`(J}*PNstXK*7PT+nhRKa7J@x9#dcaKzo~G^VOKMO0H#P?hq4jS4`3hcb-ZASYs zhxj*thOD!;!Wl5jO~DA$xEN?);HjStD71f#bN?}%G<>|p3YoGo`R1((&8@om_EVS8 zDBTUrQSaaLCmtY_1s_j&kf*4q7{$V)U(z(>@kt{->pl$0y|8##y|YUGTUNGELSJ89 z)N?SC8*$Udk=g77#^RB1qzT0ypsr-UC+J6hc>8}OnI2kH zGNIYabz-v2x0m<*TDTX~`=ATt^@szLN}ZU8+Sx4Ht7LQ%nLUUfuDP@9=F1kMR+kOI zADSsQ505rT$czmTT4J{-0CXuVn5?8N2FVmO$iL>3#qs!LsfS($E=9(r77*T+DG1jf z(1yR{qA1{sQS{W>I@D9Q*gXLB_s>Iu@<9oDG4RO&d5en%;Ztn949lT`i#C+!)zRKA z;Iya&H2Dx9c?^PJ4|;b%wEzgK=*chn5*3Z>^YAX~Y`3eKeyeG`QD^0y->kEIm?0L0EOsT*`fLIf6MI~n?j#XUkQ2K~O`Q*l zH`r5CHXZ7J2q|_C>)&Ey?SVNbhOb0e7hOq-K8N!>^G}R&0@TWQ)3*5; zG=)a1WP(e7XxtSSoi*40ch?otivhII#_W|WDKdjB!Nb7t6o;*5ThSahip-NYPy0`> zPbtoYUjFBn6a5iEGQk)Sz%VjO%E+*m{E+sWd}BxU9VqpBA#UIbQv`U0mMKf@o9HXsgg zLmQ(Qx>)PE0d6NdMUT^=Y!bNx(-*kTz5 zt$l4e0AY<=qpA`I4_;m%?(g7hCosnh z!ype2vL#snmL(toK`p3y0(8LyaSz8X)H%EhmHk1CR$9Tm8ZPDDjH+`LIEu|b^o3w>bjN( zwvnyqHXO!)O%LizOIg|3&?}N{fie7J{-=SXtIExmFG=8GX4`A?L^AN)?tTNhLe4GA z>!3&cp17k~#%>-HNr{fu+UW_QqxN6B80SUD3&$%&JZDw!-e@(Bc`0U{DpJ#NYkgB=-|P4d+Bj!pkpR zO&4?SJa|Yo?t64Qc_BYq2dW;GWrH|5OuUI5d6#4Y{y#@vIY5V_g;efTh)hSGChtMm zumdFfVL|YrEqc}R#L?o#u*w^a&9CHKF><%z{bKF9q{(?pya|MVjH4xEqN`7lkf?+FG0ltI{`KAW$$l2C%z!V11Vtk>Vy=Z_nwGr{56PpET(T=ik^_Ci zwgPERVyx%yt;pU_xt<6oiq4C0QHwirQMi%7)r<7tVEXeCo)c9xH*sdKy|d#HW{hc7 zOdk6P$>w)W;c|`%w@I0Gt`asnk8`;ifH$M}%sFA*lJxI-JSE5_`))tx&@rhm2LkIg zzYc0_6HXIKei{)xaL{_cV5S=d$x^H{g-3D4q=Og&IVG-7auCzc2l+cv_n&kw;bBvu z3edw%20IWLEQ*oDTtlgYqw6aHfl{VR>92o+*54tZ|G2ZpVsP(d-bf_(%r&qd5vsu+ zW1TX)8^iQsHh%c?%}At2-XAvyqU|lNEoFTdBjlFi0*6VQ%hS0uVJgcm?ZP0O_6DEk zRt2Ow>>`v5vH9v-eQ7C!swH3J|aR3Yve2d?yj%7umU04Z!Ta;A$ zXQUZG+_`>90ukm;_*O6Gmg0(G^L1A=xFAMnO7k-~>dCKf#=CdqN zFdioQU)q!dV4I1Gu_V?J+X8ahhtmC0(0o|Z)k4B`GgK{Af&#F7Tp2mtyJJVG;}7&wPhl|D zb(lceHwh8%-%dfZ1M$jQN{w@L^hRtpm{xDea^5LfkQ6gA-(dT(eHHIjMQhWFcwD;> zNP^T!oCqj<`}X#5Y$^ji?qq8eFK2%M-VyM5Zu;KJ{M4$h7Dyv*bP+tGy(sW2OQtbw z(r!%uM3JOs`yhza&&OF&KVI%=kNPRy5O`exp#&a*0+lgKxgAfr4LdQz#*qc1k$R&U zmZ(h1XXdQ8(dBt+HYmhc0eYnU)r=2cVkiooT`>kK$eo$`?md%!r^e`nxs&Y`9>@Vr>v&YxdU2|fCwRU0gC<~ zyg((meNd&(K)p;(cXNzWTQIWlM?};$%1H*sVtTPn6dDKRf(;HgQ=h|Gm7SaorLjyq zf2>HaEni(LzMF#a&hb23@^h;ZCBliVc={IX16#3>j3TKSxPU9btswlIGLkjIvp{;i zKn2_K2cQ~W2kpX#HGoO-Y);>^>bz#PX~M}QeK=3P!f^J?OXpL(8NnzaNac-7RB&UR5WUxIS*OH1qo1k^PqoE1%9yN$Tn-9PQmT8gk_VB?(MKu%8M@ z3spXj+OgthaZ*UPD`v(tWAY#U8p`aFwdhUKp9)kWJ%pJSPf=sKV8bTD)NNyJEZD9P zGu1QL!Dq3G-_Xl{A8aVSLc;7^I_g+zSfkayWcE%Hk@vU zj-mER*nE#>49WG#S`*;goE&wCzWc6)3pSDa8CT2m)pK+uu(~WWb|oVBy{3E(!LG>< zqOM8`s+bsG7M4MubzM_vjVtu!g}W!zLmnPLm#i;|0xRql@y_NcoNJ{HlXdL9(r`A` zu&M7KkkqXI`m`MPGu!T^*NaRFC^&4W?j|J&rHCGbfjlh zX8FLvg`O$TLkU=A<&IP&Z3@9su0QUtB#Mq6{oT!^nJPO|kT}*E9`^qc@=r>}cohI`7{xG`Y46S&@^3A5}*wvKo}bL|u2!N|~6lij94OtLM2P)B<^17_b9T<@?;6&t{(%qJH&g zaxcVm-#2j&(;#$GcPs;D5zUWX?v6Y2@Vl^_05jZ)ktjmemFv(CFhNBFZqv37Qq`)( zv8lcF>FC(uR2<-S>_(;TyC$4%kxn5zQhkQwl%~88S~`njj{6O#1uJJJb{=SLtM;oa zy*E~R{Pu>x3kEFKZG_XiQnWh@;NFtL#io2RG6xDovAni@5x>ZP2~t#*h|kYpS_zUY zlw2gDl%H!oa`mfmjj?N}vr5G*CieSa4%%Cv^u!*EK*r~`V&0AWR3e<`2qc9|`AbrK zhA|N3?e$lZ2U!A+j|QfC7(jeCG>VY;S%&JqFr%sCNA-nO#LN0k5J^gU;NA}n7~+jC z)7E&Al?glLT+xu6Lhp@bH-|L%`mAKv=eo+*H+%vs2eHc>Y2;P3J8Wp$Kra|-vGgiR zInlc&{NSJ-IZ+Kw_1idd^{=L&Qe>9h_Z@jU(1O8h%~yWzcT#|WzAsd&UZgRFjp}7* z*WY;yHd)2JE01P4?Zs@W>8~8!Sly`c$>utBM{7@!o(Ffk`h_+G$Qi8HrT7okG~}Q@ zU!D<+nwGNgL`5@O)n+j$ZeuvLk*1rd)i`gz&q>Fq;k>+Qa$$W!I)3pRsQh!kJl1 zDRn~;Uk9;w5(u$SnLtudDlqsiA1Uq~@6eu|YUX%PyxXmrItbSXT0MWNP`8qPJmO+S znpE|{6VrZjvKteA4VyvILea5@?KFEE=(-gC*0@^Nji-`vV4~Uk?XYeY=o}~*SJ-1% zD|6d>Fsx~`LJGEHhb`^x^{0n@ne`K^E$<{I;+z%hO>dsA>ej>FcpTRelh za>cu*HMe@tV2bn-%s%YVV`#EyvhPF(-_b1Vfx>4>zSx#9v6E`^R_`+iW^Jpke&)i; z8}9nz^KujWUX$2YABu-pTMlIDA9(KjfZLIZ=HouVip!BwjN&y~75bVO-pKe>AFtJ~!8?q_hx_b>2 z(uY1n+RBNVny`+S1U|vO(k~EI-@C;YFf+GW!w-Bv!d;iuPE-%C2`r-+stqdld?F&f z&-{-2BC_Nd&?iEpHAu`Z%vvs7{`~p*n<|Z>{*yIsi29?6O)e15!s=mkZsX}7X5V@w z8$n+L_HAYjr`jOuz+M!fdM~tCnVBk#rhivXI}r(f&np=_sNH|ROE7cVVfs0rK3qis z?iO%}!LX{JA&Pr5LHsev>tg9VLX{h)MfPKCV7mr)X~UuQ)CSlGX;b)gY&h2r2e?sln1>z@B61W~JDP*J$D zS^6+gz)~&eRMKmSj$V7KZp%$5`LcXE3rm2Sv~HGS3{x5-A(M>MOTvfuZ|?ZDxM~V-u&Z}b_ zyl`Mr#HqW}*v(chh=f(B!To6=eiyGEqmD|Va??yazYMKPPJh-YW@2U?!5H+e?ua;L@xP7d^gwh~kYB&a zdugHloEHYS+yo^)E(eZ`;zmz4zkSI3PL$mF?#8B?T4Z#et_kO%&jXoY zhMFO+1)sgAj7-wU33Ic%B=>xyIhl7XZEz)ch_Bt27QMjW!!kU#(6LZyHIQ{`IcbEy z^X~RGuk$Lt_ja&zf3;iOasLZeWj2q=wl;;*t4%zp#<#JtufY3*8V3f*LE2SHL*t5U z=rw?xQF)+=i=J}oV4_g;^V0d z4sJ5hFT|h3a5nI6U3wlx_U-JKT$SJv+wuKTU6JvRWYS-gZX{zumdx5;yd=nk*5zDya*Z|e`kii~OB3FJRRDk#7_8km`r3EYUUF@82@ z`M#9OROX51I>fJgZxB^dC=?kH}1djO;ME~tBl`X@b}nhQeNt$KIl?^5W~%W z&L;5jiKmB30@|bfs(vqC}(1mLrs3@)!E9U`KW#<{=Col`+MHWQLqO|7w})#AZNam zs49higzT7Znn2F>)r6K#4*nYyBxJ1-s|U(#6M+L;C24p&Vgto{C? z|K@pzj!VnUAyji9VkYdpe+z6sA;5Y6?ddWS!2r^LVT)ZmsibK@Qga94TI07`&3;Lb zBdbzv?&mzL$qsTUerty#isydu(_NY(g|Lg{4|cP%gIPhu-X`R0s<^&es<4YffQ?e< zhND@z!gEqa+?_=GCXJ&{c%Qz{9NzM)a%5_E1aIUzvoD337rdR?^)>a)M>JO0r@n@H zz3G*MYsk=;cPS(yw>xOSt1&Q1Z*LTQR!(5Swd`Xe!{@hHRM~-Rf6`P9w)};N{9L?K z&YMyT7cDouEx08Qyw) z;8;K98)ciTs)+OG%d3{|h%Po4J%ovj;X`8wqDnNducg5|NJ^Xeae(iSH#s>uQf8%@ z-tBO({di<2F-6Hi@ja20DNZ7eQcbs26}3CK&Np_5@r#6w8r7TZ#V?{|`FT}n z@~8Z$Ogn8pPnn)gJ*Aux@%~7ERVyF1uOKTIa@S!-dMhTGU+#g-*(np(l;70B`@;~v z_DDo-?&}gzkBzFYi=8(ykxA{FKUiWl9)bt6KJC8#?X*v>WPbYugJQZ8Y33sLRFK;B zY#Q%_JcBq#;|=yRjNhR~&k;t<&D?6F*Ig0{eR!Vtb<+dmwd>;GHBGkfj?#lJNyw?3 z9uII#78{WfHB=mX_oU3ambxLt#~Po_ydX?7u6gjl!~JWSTMs06(`n4MMI;Ol58<=h z^tMxptS46m>e>0^B8-S+xM(`?aU;Hn>eE6(S=$>oO7$mS`klU|I7|v&O3KP^6Dxqo zMM2vWGKpG-iynrhh*7y8JA4=yb6nNh&=-0$H!Se@jb4~sXe49Z^XKFK>7$nY`I`N~ zL$d_35wRup-c=t9o{q7NH8#%cFRnG!)X)wjMKV5y8{D0n}q+70QdZF|X8VLyru)Xd4qz}eepvxkKlj6a$K^act8=3g? z4c9Jzvcq|)h~^TyV1Q%R{6?kmbI2!K99b#vj5D0+n`$RBLD@Gxe0xL#Pq^EBKI5Ux zH~Q<|%FwUmQEeQe;^IvTqECM5tt#y#)X;j;b){oEkf{G7SVNS9)Kf_c-d6XXF!j}!%+1ZM6&jVGrw>frVwW)S z{@jyHj@m5LhfM*6Wz%c#YbVuQtK#H&e0laq9hARLTTUORN2q85Jx~J! znpz?hqi={KsMP!W?%-92caeL~pW`G+g0uw=h`C6khKtnGrO#g%al{AipLDKD^7mE- zzcf#J5*x19z2zUR%il-g@Zwmi!*#1}bF?c2oWD)?X&t8pcmHeKCa9}l8Yx3wtJWAl zXd6&ns@rveg24N0Dcg@)hU4@54py%(kj9=#o*4z$m?%ct1iVuC66Yj*Qd~2i*;ijD zP-tb<|6ItP#2YiQcT;j;@^wU<*;o=T&E?OOWa)mD-pL1#?{wX%ZRgQ>6Q2$tG81yo{nke$VZxLzy^7=3-($R`GQCK zk1JXusJPEo2KuQTDngwAVv3LFXEhBXq=m7iQ$6T19W$^wN}cs>$Mx% zlOgl)yt4dd^*ZPL4}}Y@Zc}YyuBWwnRk}qmlKI6hjpFf%=Hxtm{;hSs?xz%IM+~?2 zY^JxFr6P&n@7uKotV@ke);TadFOM7nd#5c*Pp3#fT_C&5_L}=0>2sm+a~)Sc*)Tl6 z`ic1pA)4UIIcIz+@?Sk^Z>*QM>&z%{a zOYnP=q@1nKe-MDdt*;?$(#5BDAaD)Bp*}jN_uOgg(^t>uODLRn9Qg}M)Sq}6s84FI z5ROw4ir3=HuW-`!%7$g}?|uK$|3So4s~=H?S8iL-Dl;Ew5d8k~cgk#;5K3>b$;r_c zVO%397kW)S%PCpHCm|&zB}v?)WMC>YBSR1H^X^2U2=G3_R}sw*Z?Ab3*e8m`JsrN8 zP-;Qty*6IecU`cvG0v1N3RbL|(7;A5?`Tzg(oanX35so~h$KM1JwiqvA#WY7>so#* z5jYaVAk21y5CacIi*(C53~ zU$gz1F3(V6T+`1-AV_=AOx5>vP`-gzYSKwnz-{gQEdj0iIBvPU70o-yuCI)#Ug3rM z{0ZvnlsUOkUtU=qT<*H__+}D+?z1JYEt&oS&Hm+ex~{e;o!`;ofcODNhdnUG5=9sSuJ zxtmd{Uuo`sjl67L_2xBWk&Z`f<&>S-=Hw%3+;90&w|ywMtq*9rUK0}W8FTrXP&_vL ze^vI@QB_9Wzwh3FfKq~Vw@3>}r?hl;NJw}0CZ#(>N(t%iMnc#iEh3GCba&sy`~J=t zXPiIIy>|?UTiNVqKhIio&Q)`MKT^mu&7527c*8IV+Qlx_T-|uK1=mnoW0eYx?7?`R z*+RR~Xto`VO zqo~4|wjOWW%rV(WDc1rE6GZ z)m*uDbeVT^+$xCK5{mE@(5s%mkbU^JHdVT|;2~WLRL(4hCk|HgHZ5aow41ngT8BUQ z+DAl~Md2SuTTj&-j!|ulYE9CPiaZDc>4biuVGyOFqN2Vw%wiLW`SdB=9FIN%1V#*-F-Oc(4u6i`GBs6NN9Ia?aT>O$qAA9GjL)ORA3vfO z`}jmv)_(kWyEi6wK?js7q_A*a^tD)UY+ZT(i`M?^bB0rDEdR4!!KcwKBfmW0VLo+F zNd#}l-@C>4N9>#HG^1u}O*hf7UGcZQEe?%xd>k2lIh~YDr&bX71*8al8Fe{>#x|S;<_T$s=tIX6MTdpDUZez)6?Z`pY@c+YIo=>1YNK*p z<6oLbXeU=9Tt4l}a@MIsQpW7!iMJ9bBNLc_@cxMZOckbQWJEOM*n484iNFK>1g&d& zad}Q$9czc7$k(d1;c>amT)6x`A2Gx-buL+Dbb)s!qE5zbN?3n`~%1>(&OB-GlqD^Kv zY}4pJeOeRkrSic@Wv%B-^Kd!}=;MtI4Ml+ph?zMc3@Zn@1!^CWt`(q+u13Gv*%xH* zX5EwH;T3p0Umad7t@&Lf1O~P|L8m-u=Q%L-zlmWsGF;+ej&Ad9_MGl?44kizl2>54 zuC|jA&H`ZzYo-9rYZ-Pf#}C(@s97=&bicl`AR#MnC^#eHKV@uh-$NQ$UX~{pw0xCt za({E+<-V2pYfg2l)aaFwv?|kdeL#}`&5@o)3SN!ld>yHG~g#2 z@YZ{2WEOfM?u2hb`Ce>9o*|@i&QIYnv}|T~WRk~piBQ@ePMIVH<`T*+>4{OP&uX=0 zWpFdB#2DE{aWA7hS?e%R!9d$Ckf^QjxibD=6m2&3cwBEVt^Di&>QxVx6=ZpQ7rPWC#lDR2vJG0hy7^BWsYEZNJlhcS7 zFE_Wg;1=0lVeYM2AU_)vm!XM^i+cgdrQN&7l8-o$f-`#G;d5aR3PTe#OH*jQ?r*(7 ztwRodQiRCEZSABD$YOKqf#CaHqh*ZTa*%~#+Z>|#$I&&_Z2v}t#3&^w@E2`gC zX)`y6k!MxAmuL1b8I7onQSHbGOIi_rh(Dzw;#XCY9{x-nc^`#veLhzn7U|j@l8?jZ zFqAs$#e$A5*pG;k`Sc_g4TK*c(&GH!?>8r!7>B#^2dq#3x;o)eQusFwq_J_MV1_?u zQj-e#+3ezVK=OiSA7PfWSY0;k51QQjAuq*Ucjw>CRz(zk$!(tFSN==>at+pWCd}O2 zpqQ_no_cO6$96$i!qbYwy=+aZFLLefLayKP)DoM0ve4+}3I!`Q*0wwx<9O&~ zwO=yc%QLoc-vMutaWDMkK_=Uw$Pr9U`#{^AvgRBqYAALbog_F@FPE*9!M(OUWwt&j zCAP6=aa-Z%>jyvEK^r--p%XITH`8Qlwt8_jV-d>pV|RgxjHP_f*=2Gv;ASsWN&N1* z`9MT4Q_!bj^CBrz9+QIg>TavhOh2>E?*3jC6G*Dh&TmhxFN{WkbSDO=0i>}EO&XX< zwefcZbP8coQ0psgZO!=e17|9m5hle)_U)yB2QrXIHgFG$YgN#RIFNu49D`N`DJVNe z5)l#knJq-JSMSD=6+dY}%c6kr8k0Qv{n9Hn`(d{YQ&aOxwVr=FAiqz`|(qHP4Eh z@#A@)SDhi*Dg*|9=*mPgrh@Imin-(Y5I3cs8Y1>dQuQF|j0&TTs&{n|3`8Sa(?k4I z0!3I6!lAyC&;ZD59VXll5YIylrzM?Qy@j8=`;(lt2lwUl*aB zBhU%yItyvG6RKT{FIcRNR@6>#d1sUUp5Y7P;gz1~=#L+QhmwEs`D=9h_9XR`3$@r* zGsl3f1mq00bhNhqBwhJO|J_Yjv@CU_qoW<0gvq?dR>w(8_FZDAOaR_Gs*)4 z3EmD<(`mPtt3i{WQpD>VB!$H@tgA%PEPWa%Z%HcnUCaVNca zIe6arKu^MhzY~;0p;MWSK8zB7%f0hQJyC!Y9)^)s@X%=y*x}0;8;opV4kQjTIbX7A zMdb!uUnmu%r3o?pVGIVh;8J5W&OB!1`@ZhquVR|HhcDWF&p3YG9!H^nMRd*%M2vF1 zGI6=88OQkI>*c(mcy|#+KqsSs_7mt4l{LtvC4$`o4s($(4qN{Y*q!z7Uz1z^=zssq z#UU!r0giE`eyOQqlb ziDiG{B_I(cW&>GfF}l~U_xIN_`Ak&GjEoSNZZ4*!cjg!1o0c0FpCp{%sJX#s{MlR= zY0I7bZfkC71OCTmp%-y0qHlo z>QwS>b8=SwT3h87R?*kHRoENbVD3*wo|)|Y{_*yHIru7PwY;&Hf|2*r?quQVgtt(5 z1QiNKqLkQ)*j#gx?}b6J_Aw2W7-R8|m@KVRaDt3dV5>93_7VkCCKQMG-J(|^dO(8N zM@+@AU)w`ywVeHYD=YcV6kIa5oS_fdFxGMeBjNgF4BDlHMbC+V%m zTrfX=NIWgPw8Zb7HX3pvee=$x)Cr?pZ)c^`-XPE6E>7fKUcrKvM2PDyj!cI_4QoNt z@5nW!WYy%PU=;Xo19%wXfX_8 z?|t@N_38QUu_o-EMKrFhHh%K`rO5qqOKc+fOi;=q(Uf^DD?O3hBQb&2Co!7(>YZk` zwPkJe+L@E@JuKXlf?{>7eKp=+`FXQL&g(n5)Xn>0Somm==dC%;8=2&(C!gu}>uNrw zzv|Xy>J%1s7CGCFXc~?RL3*`APEP&FVg#4qQ~dMhF!~9ygLZoZ?A^0%S=}G8ZvOUB zk{`|-e`+ZynMzNoeERs2WU1M0hgQU9iu$E|yxmJB;=3Crjl@&SB}BwNjNLU^>`xY% zSKrL4O;KhVD>HABl-3Lw^-|mRxEI`hypExNF;iE3;$qPJR`@}X2rp%hLZC3C|04yz zNWW=Zf}!EH-}-5e|CxqU;PE+rb(Ypb5t8e72Iw9_CSI@Aq-&t^SA=G3jm3Tov&~c- zQQvg;y+NMIYKqK%%8D%BxWaLit{CUB>G01r7zI#V&eCeDLsjV87fUk9h*ij+YHR# z1ep!1*IXmj_#+;J>bn>l2C$#HbKRI=t5CxPN7*wQh%=ihHcQWuaY2q)k|Cw3&SDlx zJJpdfi9q@m+LyFGAv>yOHe)m(Wg)|p=%xzSs!XIXq;trTNA{D^ybtxIGWf=*FLhK+ zri48!hWnLFl+REufF<(KUy#YD#A1CKHPNkuM&X7oM-Ve{AqoDm_S}ztM}M7DwsJ|P z>?O7Wa@ot9{Dg#@q&L~LIaeY_PAp*7Y1Hhzmby10@Ftea=G!!pzkkc8@m{r&gV^tr z`}@?i9(y=)j_uwpn=W8t8IxNrhPYuB5%og6w?yL@)9St@8{~z$cUn2c!5oXAE)Z2A9it(kP}URgfHEZd4yM+92n-ksWXw>BDMarXQE%MLem)F z)ED_<997!Jw?Fnr9iL&ID&6tndn>h_DbC0%FjuadDi*ryBY{w8yJlC*@@U9D!# zk)Tn+OK|o6Jx9B~isg)|h#?7!W7sf#>fK_y`RNo2OW<=`kZiZX*M)CN2!MxL2ZBzt)7O7XHdrIxck&KPt38mT02^1B=bK;EhLxdV_A? z#;Q1XfCiH6hu^{kZ2Y>JE~>uJyB;h8ozm2psS@+U>VDPDcYl3YD5BOWSyF>=udoM3$3o8D8BU6_Y|WRUmg(1H>Cq zQEPPuFGU^(&f9$ooXRz0b{7{I@;N!_94iz^u({LIT?bQ`CYG(mt8zXRR5HO~>JvBU zM9r|>MCzaepU6!&+Q2p4$S=34I9+G36xW+@2?3n%`4%CewH=L)mueH~uINp)eH=oW z)Qgi0sS^PVc?_nt$JJDl3^%G0C#JMo3G(^#H#O!swj^YBxB=g@WG>=l1jSAGw;F?Y zog>N~&}J?O`4*%!eLwfUpk1W*>!cpKz^7WqnfF-`s?j4a*Esd3TIn2kYywI{1 zjHRpH+7Oz>FN?A@K}UpVJDW@@Vu&!ITaSabW>c9kDoC?&bcQ!Rn;$XXnUUd&NybvT zKD1%b$Y#ha7F((*yP(!sky&UWN%Zw6yxFCt6&!A=@>J-*GWR|<^J=KIh{6Ljfjp+H zoE-9y9o~0ehy4?iDmyry{TqIb2!1?tbULz` zRbN^fk_)lDIZy1@-FOmTqoN*5nsAl5GnGVSRiU4tGBszPM7g!%emOa@b~)#erlLOV zT?wX&Qco=jp5x(3exd26RA#>S!Uk@NyuQR;*mB?G^dVuhBwL|57E#O4D! z{mW*U@j&hH+6yHaaiI^!nZpw29(`*vmOA2CxRIT&P!Nu^5{nJPC^ zZ{wdY=(gL=aIoq%L~~k1($R6KhD{%vUOA_SZLY>invwBQv)@9V2laB!j0Oq zNKSup9zQQy-K+WKa#ELB=WQqtf9shf z{32fESscg$idX~ji~0RL`4_Al^Y6T8;ocMxrQ*tj}XpZpfH7FbysPAm;I$V0vw6IWgORmmnk zv(P#ufKAStXB*3k5X*|9o3(PeuwW2&hgL#UX_&%aWT$t8K`yAh4N+3sI-IkuDGFb8 z17{yNR@E}3@cTRG9J{%9vlIdPWgWA%Zn*lfONaNqn(fUTfKz{aB1B%OmgK<&Np%Djc0 z5VCaWzI@5c+t}&$HmGU-Y>Q^gVGK53E6w~pg-K55(~j-FY@~?e)v+o!F%SOhjZZl( zb062dTHSxFvfv*DkB-7PU5nPT++(_b5UuTsN=Og}#6^rtzpFpKIqIMItzKTz&tthe zONTadx}B2qx8P9lw#tye(|b@ovhV$__T7T32Z57fBB;bsM`(d0!`z|NU-Ae^eeJ|x zX{>mH~){&7WsV=|Y?DI7g<9sW#&IXX6y z!UW{bU;DD;V3JK>#0>r_j$oFB8W7sj9()Vm46~5QOdP~4&i9FKlDy1(Ic}&__#3j0 z-hFhG;Ow~;uJQ>=jZ=|$o4GP4kZLw&ILKwXl=MlIJ6*hi)VhwNoMF2e7hGX55Pr*iUmuy$=;d;iQOnBfEU|dBY+NkO?ZEvkq>4RH z^7pVpnSIa4sMf(ak!=?_n~PP3M3?J(>oZlw1Wo0PyThKACNxy`XJjHGP8v;)VkV4r z*Y9rX9N4y=J>$5x%@}UpNnbW5x|^t2-)D7c7ucVyG2cHvKAj9@$|HWM7`_fR)3LKh z#`X`$$Q%1$BsrR{hoFdK8J#_)8&4!D=b?wFBa~qct*Nt7Xd7sL8 zk9Y5mCUkI}QzLHCZLSh(}_s8}{`H!OYzvsom za$XdyPF4J@&b4y^PvRgYB4yr)2xmvuvFsjK5fwo8(PE(}MaLR7?vLFP^t%>p^+-d$&hm&n*DurDa>f6U5L+N8 zsVpTGdBz!Gl1@%2eH}|R9v`AJaWv0lrp#saj*bb7u603Se<@JOO_00Bjos|5}81z44sSh#e_puUacTHc#UGy%%&X|8}pfc>|i1<6#hxuk| z>-zVe3cc=|ls)%TrBR2DxWXEx|AY=sODl<46xcd#Z`<{FOm&elxxQ6;pUq$8>R1kc z*gE9bbrctmS)!p?H+Y>P>FK(kIX8$B4vUNfE6`Kd_Pnn#w(G8Q&bSfZze^>o6PE3* zo0Ce7zVas(AtyJXqA#F%?UbvHfkirIz#0@}*4o+0%8~WtHU%k9YKttM!#p3o&YlJB zl2?FH zpKKWjaO;8rJ;ipaq#04w*OwOK{d=#dZwDQdBqB+&%u=!1%u)}cJUrFic+(g^uuC<{ zXl9?!S>$Xni8Z$lsjKj9$wxH3l>D;p`WLv@1h;FEz1Im-27Gk!vUWf$o+Wc)su-03DOEIqNogOewFN_7N{5Xy(MmX?Ae zqtk}?sTKHl+QL8ZLCk7oF)?a-aq&Lq+h1zU>5q(rXKq273IXa%seqHXK9? z93aVN#_GQP>sN4WIJ6XwN_0IsK*?iEiDk|r$K%%$POI=mqlDI~LRVwI-rG$G|EnsU z)h-yL0y^jC^Y_Q4O6e6i7AL<4a~$JiQ?XjbW?}bRU63BY&2SoH+djPHb$YAM=)T9% zlCeH@#QJn%qSvlK60LZ;-Zkx&DJdQ*2HtegN{vrrQ#~-Teb3o?Ac1*YA&IuiU}-oB zGh})^8#MT(L1Hl6`LOM#UTIj7wf8ZZ-+ranhQyD~ye2Tsa3Y7V7|Hd$GQ#YWaI}7i zW^t|-YV3H+XSYHtwSUjJm@x_1B8{{Sp^8t&!MU;B?5P9T#19@XBjR%_v8~mmQ)YZ6 zORS@$HfQ7A{7Q;pRlZ#3Q!R(;M2>)~G*nuij z<1bvvy1|@oC8#i!y!`PdYdYysO1*Z8*sEal>ZBg`ZU412)~!hrq3a?pL*T>>jKcGr z=ijn_DBf6#h43}M#^fV>?}}CACP>ZR{e!&RxgOyv-JP6Yw4Ke zhW8A}tk$`Gj=wz9^4pvpFMn5SZ~ZV`&O{a{upjU2@lW2q)YkY*o7RWg-HlDHE!)tB zD1x&QkT;~w1fT!jTyIUZ6orVg24eCRmqWI|CJ@Pi>kad7ekwfMUMGhW@hWF4iB$l- zlmw6n+dVsj&O2LZDviAoi$9*((eB=QnIe!;V7w@ZJ^B9q#P`c%xrInVLWZpz{x<=Q zi2Z^AyRY-2#Kk5PQjubF@y0B6iomJQ>%7y-gMs_IkwPeo)jJoux1lcT&)!9|XLJzeD&AF9kydTgH4|}d1pwQ!2 zj1FzkGHoiRsC0Q%p^RZsiI7`zTw10!tN5-KH&JWGDqlXw-A`ryW*kIXhaBmUC`ru} zem~;F)8v#qazAy=mq1lNp? zt&{Yk#-z`^?TX&G@j^=^e$1@RxzixqcB8VXB zZzypev!wE^dEg+UE5CFKQ3hGqBS7;EJLxl}dY8E~XEecrQ% zIaq_8`5R%JKef`D?{0TF{c^;DVS$L|DQ_GcjEC@-r?j9(MmiX$p#;$z@`|b6d6ZyA zh(}0vFlw9H>E3#4R2Z8~!ozdNC-4?ws&vSOGadH%O<=j6`B?+K0<51rDP?+M0@7Lb zF3-6Q4Czn?ZHzrh*PHb#dA9RW1q@MwB47;Df)UOw6TzKx8v4#ol4S2NgfM+JI%uz1{r-<#J$+`4jTA>;8^9 z2%Pm8CCR21k@IssxPa3*xfq0rD)}tN7=Fm^A1VyJr4G@(*-*LF5{(I7!)EHGyF1DG zn@fnz^t;uhB%q&!k?)F7rg_&`#cp*fCfm0&Q|>&Ye= zk7$U+`rvT|(vyfo{aPucZ>BmYCy4&HD%sytS*T|hJ;g(4Dc^z=9~gyyd~Ejb_nhex zWx(DsVyf7&`#vA;P03Z*)WSQ!(u9~Sv@ItGRX<5~qS>wVUT?(1ZAAk6bk=zZk7@A! zjdghVuhdkYjKFUSl0|7OdY8Ate!7d(Q3Nqw+j}f;ZI*;}gndUyu{&ac2xi+s?2=X` z>uo(aQb|2gl!>=E@4*Vy)0{~l6OdT@_U+w9yDFWIH!p{3Z+j!pSbeE0dRu!S&r-4? zCZ$|rqfMcRX{}Cdr?Vu#Lk73(# z4~Wb;R9G8n<&>(t{tu z0bxX7K%=b2fl*L?+aeOJ5`n3AwcxR9L3aZoE09ct_-{E&c{o*cG>w;z9z42js%nxU z`QH$Pyk(@tX$lJ_6<<`Opb`A6(p4x@?Ta8fBaVo0>gn*?xBUGxY}wc+B4^1AAq*?+ z!PHbz+X`P*L91L-^trQU2>kBJc7Oj; zm#C`31oof*{nOd0%|4o$;3t#kIgN_St{VgTdP%7^2>4iFpsM2AxA~JkD)-9ud>maS8=G6B=;kNYBo@;-6C13j%t8toR?=~g(v$P!f_VR_Nrh>ro|sI~E{{<) z_s2h9uAhDuUtYjNl1@Nydufz|djeMXec(YY@L%1Xe7H>QXAR z63K8N&LdGi6Lsstefr5AvSfrm-g!LSc2fRc+_(o>wsnSCyr83l`Wx`ZLMxYw`Ljoax)IkafCJ@LAFP7o4 zW>0tZC~LEv!THmyCgFbak2*IhGKa!(BTG_b1iO~c;i&BIt|2kkbDrph*7cb*rouSf z^kLOh+-IS=w92fctj3#TUuERvdH`AVcz$F2B`lN%>Wi|B)phdBSO}6_kC?>|6?s6{ zKSGM9#W$d!FkqOreiEddDguF!70F77smuGm@H{75?)HN8LoX2xk3^?qyBh2!JnQT+ zcITgHQ$u`_z7WBfAomDkNU)a@xDSY5$b04WpRyYu^I~*i!Qcq&J1t;2qCB@p9esX_ zSk{g%SpPz10b+~+>x9-~87sN@F9{1%f95jk2YLMw1?!ca`R^C&nHo2t5}TbkqsFZF*{A_dyCUUdyD=KYE^yFWFk?1x$HAly!Dky0 zt+!57uAS#=dYS2xZ5iXM&#S5`U!+w@J(7u?WWGM=ZoA|U1>!T&bS-jeqPBm&F!kQ- z%k{^J7y`YzmDK`c^|H;MTW<(Ok64bXt0&Uiyfr$2cQ5(U&~Wd~60!UUrQ*|>e=(Ao zfT^P#uiXu{DlfJ-asuraXgh!hW1AQ2nE$cG5BO^o;I>HRE+rrB+N4`I#xgkD>caAS z=#u&4R;140;jy{X>v4VkpC76n=k_Mx!d?@WU)y!YZwJ8n?ZwR?UiD_BVb+{o!R7|{ zzvkRi6|a47WK2?22M34s?gXqsggt6z9qbP|{)|uyL}VF9W#H4E7iUPhozMzlgk`+0 z&+9nx^2_~9up1&sJyah#Q3N6J9^sJEcpr(h90?tv*JPJcV%VvxBo6j73^Itw1pP#h zXo+O}W_bpoY6?aBMHdnz0KpRV(`V?vdf35|Xlq5+ez`Y{lnSG!un;{>`%DXqe>@}@PD_2U-|Dv0 z_Mq}?3Cjyoi+ zM3mH92%^T+b;lmBLbPlN34}C2{C8lIYzQAOpboLf72gmJQTF$dGk36Vw@{9brphc{ zLT@r)b2`oL)+OU9-xtWjBN47oq5Q7#pyI^@4y9c<=s^2bisHJJs5gIi*NEg|dF9rNv%8z)s0S-;`M~Jde{9-1 z5V`lQU_eKkNY6)t#cMt7&s^gcJ!6(jhGpYCIDXdy0;$=o6?8xv;iJk|Wrk-_�LR z%>&)3T>fnaTBfI8vzzK?UaAaRA&EdW49+gT=2L)Z(~k7*@zmF zQB7<@UgPqNCS%g{$S6N$_8t}1!vyR3(q_+Utlt&w;RJ>+2{=o+{KTrykZRp(;M-y3C3$Rf?MgJ*?y1rt;EqFTH75FR zb2u*dXQVP|nP&S&e4@!){Y;Y2JZ$Yh%Tk3^vkSKHc!NHJ6xh8f!HujI=6RuR&lsgU%m_^RiWJ)Hp=4K|s zY{IB;5|@`JVR^^<_bdBikx?>p4IBZxTa(8=zHPf|6`4RnTRl)Gn&0gd6Nc})a=3oYr$(~FSe&gMW-vao=}`pP(Dz0KL`u?L>fIbyQmr- zKeN4K`x;cLA2;YSop(Hc2BgKl3siogcmfe0NYCY})@BBUz5itV?%b*?vU#Dli_B=n zzIXLpix|Y=mHY6dYm*nSCPv@;W^}le#O8|EiUgmbi^t(%{0w5< zp5Lg}mN4r)|9oQAd~6DUG&6xOW4D_28Xg)vkzp5$a=k5X1l6`Pbi02NAPxs4H1h9n z&KG6^LwFd$%!XtlhB6zvv97Pu@}E#SY7obL6iDr${`@PNqw5a}4vBfZ`2}90~U?0*zulEqpR?O!~dez1((sn_5XpC-J(E z04+cyiXdwJs=EKQzyaj=rSpQK!teLsq%S9%iC=djE0KC*Tmo#H{WLJhlq9O4=)aaEsRhnaY1ITQ}bf{^J;DZU6*oAz5o?MLG6*!2n*3~_5zNg`!**ZjQ{=2&3f@XU|;;B z8FmO*NiW>ro6vPu_tridFK%Q85!hA^pZe^hvy^%MaI^|mj3C#MQNv6#yp27h&so2| zWZZ)f9QhC26z1SxpxK$%q!}{)-Xk0fqJ&mQ3uOGLUf7&TFL{5JieKX>)>gCF_!%9kKT{m~e52~V2b!RDRE z8HRdxEBRy7fwTqGs6K=wS$*DEQtY&lOr|8h#3cv5qjP+ZND6<3+t+T}1BLmU(@GLo z2{gxzzE_C?9!|1Ni{|c{+HPorGO*iC)h2J_2>Ym+#u8#@HpVXnpBiq z3XGGn>i*Mw;89hOl3^g!{r90GVULTKVJ8h-$>J9j6ggy`xsDh|M^AwPKAZmXzuj$t z&a=n!=dYS)PAt~zjMFZhWfGowU;mHa81DW{t;QZilw;>1)kP^|UR6jw$!Y>ExA}iw__6E0 zk{%mGJ>z!*Gm5dZBIp0Ma>_=9v{h=53clrkI{9(dj>-Tn2bT1o2Ec^^YYK55@L*xw z>^%clna=w^9_jI+|8Ex-3t|1g-Z=Bve_I;&nIyI{g6^XYKmLFLp|K7o?03zz-}&Hw zd38m_zD*7YXyO-@Cs1|K&zN;=C)Ui3Akw zNFW(*+!u+bU1bafS%`c1U!Kx`OyGP90bzwaPJcm z7PbO#R^31!GGXTv0H!pa+DrWR`Ex^5X6XX#b?S5A z$3i1!9ACZ9w!`+oJ6(XaCr4NQzdrPU`IsH<>nm8IS^n9^M^qFtk|UCH;MCF4ad>#h zFyX|VQGOYrft%iq6wLT8wYN8aCo4n`^MCWPKei3}{L=5&Qt`(8{rzPW6o%`C=;)rT zgEaF)-qr=o+8O<#RltFAJYJKnvznyW3Pi`m3~OxkIKKe8qQ!P}RePWfA-x(K*ZF<1 zYK8Ywj-p)CZjY>Zy{|$LT7dxY?Qpzq>GD&m>UAY%ggV0el-kdH1h){}a*Zko)`lKb--cYBYIo`y~>m1+b@s!F6b;sGy+1<<7YiQJ+2Q<{T9a28<8^ zA^X4Gp`NGX(%2LfanJA=%^U*2f2utW3=1b`Kz+q~kPwWDiOFpn7aq=#o}uzTmQDW| z_M6vZpA<;!M}We#+iTiQ~mc zo-PE?xGWr&+PpXc;^^R%Py>SZ-_Elqx1XWx{wn@4h}O{m$~-<~%DN9oGhh=FM*=32 z*#QqA{6rJJroQn3D!oAT9Zf!4kSLAK2pQ7$aDT(=cU|{h;z z3_zny0ht$cW$29U4Vu2f(1@NTHUmg#!IgJWke9^79Mny;O1zvRc zThHOEL96@ps^yz4YlXUm=V9b?k zG68k_FaWaWvYMcsKKNgjnUSPtD4Is^-b!||G5SbZHs(klYxnU$C*l6d?(;GCGE*dg z95AlFgXEX^%*<$&JSi$JF0N^RP=MtLG?J@8CCuf`4GkmXGeDke#(dRLEvux|570+d z_6z)(yiN#sjOwC*1ITVR@Vu$Hndn@@%aG%bEw?Wq20gm5iK*$|ot-EEQZQW`NI1N> z;MpxbZgc(9U`L`^p&tPhR&$kdgu|B9v(rcc@#6=OM8=rAy}Nr2V5&XbDI0Wh&dwZ< z2zhY4K?WWJkmS371hWc&73}({XA4NbYF0G^&=Eo*Z!Q2~jsZ>i9-taLS0Y|kc+}Vv zf9AG}1k{ohK(p%wj^nsL->iR=|LxmKMeBYyz#lwzDX{N6fNH!1h^PwjNC(?gW}P4q zL6^bz6ls*=muWJEVNt{aV%&H8MvH^OSeM z2?^P~0}q1<8jz67@;MNt4Fq%`Vt+L+!y4bi?sAA%3_opVL9jtT213Hs0aqNNPq~|9{Z}* z>+)>c$O~*MRv2|=Lz;l(f&R_UOui&gdm`w%seIl5NM5fQ7{ZH-i!0>!F-FJpq_2Pp z5;w5u)`dfwoK}DA+A)p-8ODWdn>k>zPhk`Rw@&Av;R|}gv1xcc4~?ZHBpgB4{ilnV z#l*xi`CLA9{O7&_Fh*EPO6nL8y2AEi&teBp+Enm2RZV7U%n#4b`oyt>lH2XZzKAiX zeM1N1g+G~^`^gxvL0^J)C=7(QM?IS&5bn<#06B&dgbL6JEPJ$z>c1)s1g3khfglos zai+NXx)=OCN4m2ephATd7QO`GC$U`t{IJ&R+TKUtU^X!pT; z1t&Y(nVKgMbgwed&$0>%eSn?^3Jn8HfmI!zoN&61Xk@1`YgHQdf%aiH{eiAtq7L5@ z2920~b+VBMfRLMu?E>^l*_k#^9f1Az1HTl-p`xN96!c`*t+NaS^+d+QpWlQ5E7^#_ z87t^*cG-CT{CT{MkFxS};4YQok@gB;1WhaT*jrsiq@bYa?dvlrR9_$kuYW@c47Lx=&xF$-R(3Bq!u^rUj9u#p57$bJMcNpK&zH zWoQJ^U>0~d6%7r|^!d-9^1&LqDlNcuf)(jMSI_M^dT8}^HUaD$_Cl+NbPA)!;ZZ*Y z$)njH3j%DUCxC-Go$u=W{QL$D{6aqn2#>A@0eAYC^tB9#k6@?&CbMPC^3f##)Y|g$ zeoP$eE9f=tCPylupZr90-Ai_5ef0?=-kK%JW1 zV}E`Z&k`5~D?mYuK9-0%EOq?Ht6~{7N+6;YTbqw_r_TFpra4H_v7m$f-oE%N)!GjV z%sMqlAWS?Ki9xW*flhrN!0A=GY}=Co5%^Fr3J^6|E!j+!xCte?iUaIO&|@G2Bgvl{ z^C-0kW#FSxGqzHosU7ccum1auRaNYyLf*Jw+BCfQkKs$J>lQ*lKtRd{>oO1;8u+tAkB+8Q;fyC5QWHz?S-R3)u?2R6`Oz7ITe}DmBtVA3IfLv0l4yarBmD6`HN$KXi^0fqgs_||8TFsH z`F7m}cBU&w!FRKBawgHHXJ-Bc_FN5w@@H;}|LfBUkLAL{!!dxKW(+2KmfRV9Ruf=y zRhAJ3;OIeMre-W7g9>Hm;z|bpUIqhu?M3yB?e6b#5-_tG1;Mcz-0tb=>6((rohzA| zzv(e3`h5XmQZMQP?m{RWd(v0}TmxvIi-Q_Ifcwk_0zjrf|HFzq1B5v_dRkhu#}UER ziT(>P0a8Gc%*)B4IB3~n1xlG|7$n@Z5KKaL7!bLCylFNa|GP6SFzBT!eCGOF^}a_N zn1N=Iav6^%J>#aXp>-^cHN)9;xF9%_a4Z8DHbA?sTE>AqLFqF zzy&V>WUnTT$`2rS(Oq;IpY?+Qr5b3X7k0j}o5fYg;2wVLY%>`}#UGF3HJDxLH#tH< zB(hv=;{(H1DpMDTqFYB`1fgMOMhCn$PFrB>9FIN>=#$^Uq{tB@Mu&X-m}fqe`p87A z@OzAXg-SU;R{El_2?#y{B-9tclx)^Nc^rzL{?AhAVQJ(pL}IG%J!!yxv^gag|A)}% zMO>9zwf7Im*61OkPAWyT5>060;g;B98Sio>l_mJZaO-0=C6QS4&2U5WzccU>6X_Do zs+%T-t#0~hVH~uzReDZ&mH)lbS>T&@897+&J9OmRfK&yl0kLTagamaA{0+lEvIH+( zK;#y@7$ZU-A;wjK5Df70M-+w(UTi5v5xX9CLEi(&ZaQxm1>{~Hd`Kp-y887(o0nRG z4DHuDFhd2Ryc57#o@pRMw4Z^?(Zc5%9o#@&Cz$GmoJU4JGXP0oV5f^ood{z5>hb~^ zMjPH|a>O?J;>C;W^SumG%#Zr4X+S*Rb*1+yA8Ble&vb=BsU=s5F?-SOt}Wnw_xGbA z2RrkU0t8v)#Z4<5F8d@{ta-s64;PoYW#5sv?C;2(OjH74E-x2-pt~C;5B!Yv30jq6e1;7{c*+F02(vM^dmDbd75QTmH z{adr3u&}hPO+O|k#+g40!0D6~6k-fJe2ETR&bBqc$RdApagg?&5RkaWfLnoC|F4;u z=H=%8{(ibw@2{i*L5$mBvCS6?74jtT^ak|Ue{L3_ zzf%t9*rNmNZp=+g-lm#cf#@+?Z7SUjxCMKYK=BV>@B*+DPFMQSz(Q3fjzT1L>0o+m zve513-0lqz58H|rgwhTB3zSk8%%FG#1TvD3rYnsq0mc;oZ%Yaa%;Br$hV8m%XJ;jq zmDBU{nV95)MSw^OKC(=qjfeq}T{-eJBm9x8*sG0NI9UuD?ds0o&DFJc4q#S)jgKcd z-k*ZG--eJ%US%a)2iIDfHuE>Y%vxt7h1@&(1_JAb4-Zd)3B}a>d>KH34sa=)U0ks3 zkb;Ol3G6^aO)U>F+?w@SW1^#F61}~>!Nh`pcw}T7tN}R`PDJ{*c6KH~JXt&P1b@Jn zp#VJ_A>T_!ot-Bzbz|*R^Kw^gNc^Dx?d8{_BPYHr#q@8*GPJoD2TT1tsEWX(00bK} zv7Gjapr9ag+E4-TPES1sTvaY&4KUa3CIzz+usBZvaB#EF-$0xurzkBgl@rcULPSJ7 zv42KNLZYUm6xaQ?3d}5uKt}@dh##NP6G%Q1=%xIo^K*9$42&&M^ulED?7u&J3?Uh) pfNS^vFT8-?=YIa>|KrCF35ef>$qVRgH-e8rWF-|PD&86g{XYy5wz~iT literal 0 HcmV?d00001 diff --git a/doc/source/test_plans/reliability/version_2/plan.rst b/doc/source/test_plans/reliability/version_2/plan.rst new file mode 100644 index 0000000..bf829d2 --- /dev/null +++ b/doc/source/test_plans/reliability/version_2/plan.rst @@ -0,0 +1,264 @@ +.. _reliability_testing: + +========================================== +OpenStack reliability testing. Version 2.0 +========================================== + +:status: **draft** +:version: 2.0 + +:Abstract: + + This test plan describes methodology for reliability testing of OpenStack. + +:Conventions: + + - **Recovery period** - the period of time after the fault when service + performance degrades + + - **MTTR** - mean time to recover service performance after the fault. + + - **Service Downtime** - the time when service was not available and number + of errors is more than defined by SLA. + + - **Operation Degradation** - the difference in operation performance + compared with performance when service operates normally. + + - **Fault injection** - the function that emulates failure in software or + hardware. + + - **Service hang** - fault that emulates hanging service by + sending `SIGSTOP` and `SIGCONT` POSIX signals to service process(es). + + - **Service crash** - fault that emulates abnormal program termination + by sending `SIGKILL` signal to service process(es). + + - **Node crash** - fault that emulates unexpected power outage of hardware. + + - **Network partition** - fault that result in connectivity loss between + service components running on different hardware nodes; used to toggle + split-brain conditions in HA service. + + - **Network flapping** - fault that emulates disconnection of network + interface on hardware node or switch. + + +Test Plan +========= + +Test Environment +---------------- + +Preparation +^^^^^^^^^^^ + +This test plan is executed against existing OpenStack cloud. + +Measurements can be done with the tool that: + * is able to inject faults into existing OpenStack cloud at specified moment + of execution; + * collects duration of single operations and errors; + * calculates metrics specified in the test plan (e.g. MTTR, Service Downtime). + + +Environment description +^^^^^^^^^^^^^^^^^^^^^^^ + +The environment description includes hardware specification of servers, +network parameters, operation system and OpenStack deployment characteristics. + +Hardware +~~~~~~~~ + +This section contains list of all types of hardware nodes. + ++-----------+-------+----------------------------------------------------+ +| Parameter | Value | Comments | ++-----------+-------+----------------------------------------------------+ +| model | | e.g. Supermicro X9SRD-F | ++-----------+-------+----------------------------------------------------+ +| CPU | | e.g. 6 x Intel(R) Xeon(R) CPU E5-2620 v2 @ 2.10GHz | ++-----------+-------+----------------------------------------------------+ +| role | | e.g. compute or network | ++-----------+-------+----------------------------------------------------+ + +Network +~~~~~~~ + +This section contains list of interfaces and network parameters. +For complicated cases this section may include topology diagram and switch +parameters. + ++------------------+-------+-------------------------+ +| Parameter | Value | Comments | ++------------------+-------+-------------------------+ +| network role | | e.g. provider or public | ++------------------+-------+-------------------------+ +| card model | | e.g. Intel | ++------------------+-------+-------------------------+ +| driver | | e.g. ixgbe | ++------------------+-------+-------------------------+ +| speed | | e.g. 10G or 1G | ++------------------+-------+-------------------------+ +| MTU | | e.g. 9000 | ++------------------+-------+-------------------------+ +| offloading modes | | e.g. default | ++------------------+-------+-------------------------+ + +Software +~~~~~~~~ + +This section describes installed software. + ++-----------------+-------+---------------------------+ +| Parameter | Value | Comments | ++-----------------+-------+---------------------------+ +| OS | | e.g. Ubuntu 14.04.3 | ++-----------------+-------+---------------------------+ +| OpenStack | | e.g. Liberty | ++-----------------+-------+---------------------------+ +| Hypervisor | | e.g. KVM | ++-----------------+-------+---------------------------+ +| Neutron plugin | | e.g. ML2 + OVS | ++-----------------+-------+---------------------------+ +| L2 segmentation | | e.g. VLAN or VxLAN or GRE | ++-----------------+-------+---------------------------+ +| virtual routers | | e.g. legacy or HA or DVR | ++-----------------+-------+---------------------------+ + + +Test Case: Reliability Metrics Calculation +------------------------------------------ + +Description +^^^^^^^^^^^ + +The test case is performed by running a specific OpenStack operation with +injected fault. Every test is executed several times to collect more reliable +statistical data. + + +Parameters +^^^^^^^^^^ + +The test case is configured with: + * OpenStack operation that is tested (e.g. *network creation*); + * fault that is injected into execution pipeline (e.g. *service restart*); + +Types of faults: + * Service-related: + + * restart - service is stopped gracefully and then started; + * kill - service is terminated abruptly by OS; + * unplug/plug - service network partitioning. + + * Node-related: + + * reboot - node is rebooted gracefully; + * reset - cold restart of the node with potential data loss; + * poweroff/poweron - node is switched off and on; + * connect/disconnect - node's network interface is flapped. + + + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A particular fault may affect operations in different ways. Operations +may fail with error and we can count such errors and estimate how long the +downtime was. Operations may degrade in performance and we can compare +performance with base numbers. Also we can estimate time while the +performance was degraded. + +If both errors and performance degradation are observed the image could +look like the following: + +.. image:: hypothesis.png + +Here the light blue line shows the mean operation duration, orange area is +where errors are observed and yellow where the performance is low. + +Overall the following metrics need to be collected: + +.. list-table:: + :header-rows: 1 + + * + - Priority + - Value + - Measurement Unit + - Description + * + - 1 + - Service downtime + - sec + - How long the service was not available and operations were in error + state. + * + - 1 + - MTTR + - sec + - How long does it takes to recover service performance after the failure. + * + - 1 + - Operation Degradation + - sec + - the mean of difference in operation performance during recovery period + and operation performance when service operates normally. + * + - 1 + - Operation Degradation Ratio + - sec + - the ratio between operation performance during recovery period and + operation performance when service operates normally. + +The final report may also contain one or more charts that show operation +behavior during the test. + + +Tools +===== + +Rally + os-faults +----------------- + +This test plan can be executed with `Rally`_ tool. Rally can report +duration of individual operations and report errors. Rally `Hooks` features +allows to call external code at specified moments of scenario execution. + +`os-faults`_ library provides a generic way to inject faults into OpenStack +cloud. It supports both service and node based operations. + +The integration between Rally and os-faults is implemented as Rally hooks +plugin: :download:`fault_injection.py ` + +Calculations +^^^^^^^^^^^^ + +Metrics calculations are based on raw data collected from Rally (Rally json +output). The raw data contains list of iterations with duration of each +iteration. If some operation failed the iteration contains error field. Also +raw data contains hook information, when it was started and its execution +status. + +The period of scenario execution before the hook is interpreted as the +baseline. It is used to measure operation's baseline mean and deviation. + +`Service downtime` is calculated as time interval between the first and the +last errors. The precision of calculation is average distance between the +last succeed operation and the first error, and the last error and the next +succeed operation. + +To find the recovery period we first calculate the mean duration of +consequent operations with sliding window. The period is treated as +`Recovery period` when mean operation duration is significantly more than +the mean operation duration in the baseline. `Operation degradation` is +calculated as difference between mean of operation duration during Recovery +period and the baseline's. `Operation ratio` is the ratio between mean of +operation duration during Recovery period and the baseline's. + + +.. references: + +.. _Rally: https://rally.readthedocs.io/ +.. _os-faults: https://os-faults.readthedocs.io/