From 150a0213c28ea92cfd1cbf761cb165c6bf8fd866 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Thu, 29 Dec 2016 17:12:18 +0400 Subject: [PATCH] Kubernetes pod recovery test plan and report Change-Id: I8db42b5ef403b191755c6a2c4771df6927c89e57 --- .../kubernetes_pod_recovery/code/Dockerfile | 7 ++ .../kubernetes_pod_recovery/code/svc-dpl.yaml | 20 +++++ .../kubernetes_pod_recovery/code/svc-svc.yaml | 13 +++ .../kubernetes_pod_recovery/plan.rst | 80 ++++++++++++++++++ .../kubernetes/pod_recovery/chart.png | Bin 0 -> 8641 bytes .../kubernetes/pod_recovery/index.rst | 56 ++++++++++++ 6 files changed, 176 insertions(+) create mode 100644 doc/source/test_plans/kubernetes_pod_recovery/code/Dockerfile create mode 100644 doc/source/test_plans/kubernetes_pod_recovery/code/svc-dpl.yaml create mode 100644 doc/source/test_plans/kubernetes_pod_recovery/code/svc-svc.yaml create mode 100644 doc/source/test_plans/kubernetes_pod_recovery/plan.rst create mode 100644 doc/source/test_results/container_cluster_systems/kubernetes/pod_recovery/chart.png create mode 100644 doc/source/test_results/container_cluster_systems/kubernetes/pod_recovery/index.rst diff --git a/doc/source/test_plans/kubernetes_pod_recovery/code/Dockerfile b/doc/source/test_plans/kubernetes_pod_recovery/code/Dockerfile new file mode 100644 index 0000000..f84ee64 --- /dev/null +++ b/doc/source/test_plans/kubernetes_pod_recovery/code/Dockerfile @@ -0,0 +1,7 @@ +FROM debian:latest +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get -y upgrade +RUN apt-get -y --no-install-recommends install python + +ENTRYPOINT ["python", "-m", "SimpleHTTPServer"] diff --git a/doc/source/test_plans/kubernetes_pod_recovery/code/svc-dpl.yaml b/doc/source/test_plans/kubernetes_pod_recovery/code/svc-dpl.yaml new file mode 100644 index 0000000..8d7cc0b --- /dev/null +++ b/doc/source/test_plans/kubernetes_pod_recovery/code/svc-dpl.yaml @@ -0,0 +1,20 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: svc +spec: + replicas: 1 + template: + metadata: + labels: + app: svc + annotations: + scheduler.alpha.kubernetes.io/affinity: '{"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": + {"nodeSelectorTerms": [{"matchExpressions": [{"key": "kubernetes.io/hostname", + "operator": "In", "values": ["node4", "node5", "node6"]}]}]}}}' + spec: + containers: + - name: svc + image: 127.0.0.1:31500/qa/svc + ports: + - containerPort: 8000 diff --git a/doc/source/test_plans/kubernetes_pod_recovery/code/svc-svc.yaml b/doc/source/test_plans/kubernetes_pod_recovery/code/svc-svc.yaml new file mode 100644 index 0000000..a8e8d4c --- /dev/null +++ b/doc/source/test_plans/kubernetes_pod_recovery/code/svc-svc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: svc +spec: + ports: + - nodePort: 32333 + port: 8000 + protocol: TCP + targetPort: 8000 + selector: + app: svc + type: NodePort diff --git a/doc/source/test_plans/kubernetes_pod_recovery/plan.rst b/doc/source/test_plans/kubernetes_pod_recovery/plan.rst new file mode 100644 index 0000000..b664103 --- /dev/null +++ b/doc/source/test_plans/kubernetes_pod_recovery/plan.rst @@ -0,0 +1,80 @@ +.. _Kubernetes_pod_recovery_test_plan: + +********************************* +Kubernetes POD recovery test plan +********************************* + +:status: **ready** +:version: 1.0 + +:Abstract: + + This test plan covers scenarios for Kuberneter POD recovery testing. + +Test Plan +========= + +In Kubernetes "classic" HTTP-based services can be represented with help of +replication controller and service. Replication controller is responsible +for keeping number of PODs constant and in restarts POD in case of failure. +If the POD is deleted, then replication controller creates a new one. +Depending on scheduler settings a new instance of POD can be scheduled +to a different node. + +Under service performance we mean the amount of work produced by a service, +for HTTP service this can be measured as number of requests per second. + +Test Environment +---------------- + +Preparation +^^^^^^^^^^^ + +The test plan is executed against Kubernetes deployed on bare-metal nodes. + +Environment description +^^^^^^^^^^^^^^^^^^^^^^^ + +The environment description includes hardware specification of servers, +network parameters, operation system and OpenStack deployment characteristics. + + +Test Case #1: POD restart +------------------------- + +Description +^^^^^^^^^^^ + +In this test case we investigate how POD rescheduling affects service +performance. + +Steps: + * Build Docker image and push it into local registry: + ``docker build -t localhost:31500/qa/svc code/`` + ``docker push localhost:31500/qa/svc + * Create Kubernetes Service and Deployments based on files + :download:`code/svc-dpl.yaml` and :download:`code/svc-svc.yaml` + * Measure service performance during 20 seconds: + ``wrk -c 20 -d 10 -t 20 -H "Connection: close" --latency --timeout 10s http://svc.ccp:8000/`` + * While measurement is running delete the pod via + ``kubectl delete pod -l app=svc`` + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. table:: list of test metrics to be collected during this test + + +-------------------------+---------------------------------------------+ + | Parameter | Description | + +=========================+=============================================+ + | RPS_BASE | Base RPS value | + +-------------------------+---------------------------------------------+ + | RPS_WHILE_RESCHEDULING | RPS measured during rescheduling | + +-------------------------+---------------------------------------------+ + + +Reports +======= + +Test plan execution reports: + * :ref:`Kubernetes_pod_recovery_test_report` diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/pod_recovery/chart.png b/doc/source/test_results/container_cluster_systems/kubernetes/pod_recovery/chart.png new file mode 100644 index 0000000000000000000000000000000000000000..50f78e5efa18e82d7f3f3b158bae87a5795aa4be GIT binary patch literal 8641 zcmd5?cRZEt|G!B{!YO1EMRwVdQ3#n8ii|QJ>)7*HMcG95OaocR-seb0ndd?Fietnv zva)hA%d9RNUEe)kpWDH~w1f5b*R=5p8gc1-$ zkVQ%e&Ky3sq68nuoUf~Blai8-Olp1xztX!X-gCKQZ{gzp(8(OKgxlMh^E;b5nVZ9% zf3kO3IaV(NL2QtU!ZmG=)TJ>W2kOm(cdHn~kI~cVWNLw06q!V)J;};Dw6hN65v zZir@(wEIAyTC7D<9xIn~Y=P?BCAqgJXmcAHij`9@w<@$>BPh^HE7pHTbf)!G-wOrB zC28A8+nCx3*1f&veL3Z7Cie-0ailDrJ!^e?Q{4~>CEeToO6fQR=_FyL!GsH+Y7s!O zH*z58CWkfzy@ByS{yn5LkepQ@8T3Nr`f*5W_?b+CgwwRLon65P>o+eHJogZ@f%xyS01)Jk1%qK=Nv=0YCQI$6xB=QRzlZaAy#c;fP^huh-d zY6A)Dpl!wD$B*}xYgZ(hX&|6@+wxdSOw4I=V%3{B*Vfl7ZHJteN2|e^yuO+T z4Uch7_aJ|T?yWcVNy3)1v$GavW*h0-U0{ab(e+0MD-rV=+0Gq_ms|2{YHFsYrW~;? zdVKNCVQ1Z4T~VtojO=T>IAqZEOvNzT3tU`Ltg`5;s;aE4SxF`s#FnYB&C*x9f|j!W zlyla|<6*<&<0sDwjFwqFAHdPT)O&}Ahf7LI!kc4aVuXZ+@9XO;G+|fOQYDJ#+8(;O zecM@|3ut&C5=jU>E80bGY(>-=tpu_9daQSf*MEI~a%pLaQOwq3t(8k0J~}u)F2F99 zQedf4J633w99&jeX)I)@Afm3SuD+B+NA?80Jr(xpllJpg>Y;ZK#6ud?*zmxiDHtX_ z#{~LZ?#;Y^auSst z7TjmOgc=eAgN`FsB5ai*sBdwienMH!Pk}Bsday)lYyCWGF(%Y5%LM*p(b+$hnrhya3Rj3^vabhp8ffHn5*j@%VV_( ze0pHLGZ4$w_u4*m3Y|{>*QJp0w zJ>zJ9j@579(baWxrgESD?miO*NoMGUu;%y15~o>Y=Nw^iTaLRFeGLI5?d|OmX`N|O z17<>YV>O~&T$byzol<;=6?gL*wJ=adG8Vy;-rb z)G!rwb$&s?3A7Kw)wR6OnxBtvdfWw)%TT-N(}>KnI)41P5i-GVyD4U5NlXAkcp^Mw zoo;={TO+#^jXsP)z}Vk$#Io&Agf;kQtUn6-kCn-gNeEqx_;n^j7+gi=lh% z*w%!fpO=@Hwe>=Yd3#s7G-BZsH|&bTXaDvY9zH&`n>X89TU#+0B%htukMPbV>g*tcW z9UJUn&f|v9vz;km7h4tI^U}1pw?7L@>FgJ7wkWkq=44OK+DwO6jzl~mfT}`C>m?lL zy3#5~o!ZN+dIgfw+I~T9QVo# z_CSqSR8#=03H=y&;>_V{l;4JJ=G(WeiN4h(B|<@sFg(sVesX?1xxT~0!}GGF6D?0e z_5{GVq?DAR!hmb(P-<$bfu7#b*q9w(k>AlFSS7A>50k;7hZd!tU8z?9MV;YPD;lzm zi;KgyLPF3kLOzaG zJF9c{f*tGZ-%l1ztVG!w){eo2l3q%&K-(*@+-nUhTT*FVANx1G zPyp9!umt``ESU_{i*B_*5VYvYWKindCrh z=&B?_yvPt8OzIT!EJ@tC2EE`BI5_1dmLek7f}nyN%=hIPew87An}2|zQc%9JAT#qS z3khUEkjbzpDLGyI0f-mZAE=mLZ$IUxnet7DTMAY%x0tUuy{@R}$F33uG{^^0 zBz0QAJdX~xGW-sAva+%wBO{|Zln$r4WPupb!0-sx@2?#m>>UAu3TKvb+gY3G&DIpp z^7}SqIZ|ma>$*`yj9DHShhoE{tNl14!eJ)_%|dF3R*q^PLao*=j)-95M7nJ&FI zWSLQbMEZSgAj$Arc#HiI#aL>DSSq*7@Tn*(D?_7YWM%y}7y7_Fs)GX?Cjl}^q;(}2 zdChD3?e?}l=gRQioyR#U1F6*Go@3FKnv{}~0vH=HwFNX1VDs!=Zgyddj_qaldc;)I z2~%BAv%bEE00e-vnZ;ztT6EA=j_|W{X{OuS*%?;Y>H+LJB|cgN)Zkqs&o`x|>yk{= z5SydIBu1Zo(a57ZUD~kMuN6wzq5Z90#@xcf!ViGiTNv&4yQ6%~)Vf7FElM}} z9#oEu3=b>3MxhR3=3Cw$FhS&J1>1@qI<1-n=G7WR=f}mc8~Z5cK?cSLWI9SpA6`Yt z43CYWK*?e`Pr{1z3m(b6u%xe_$iTxK5Kswf;}Y~GZXh;xnr2QLa%n^8S-0AVO2h_f;a7ue)0t#Mgn4(7Zjxz0^2k5+Mx}LhbO1d% zI%1YrOJuJzPrYDgGCLv%}B7D0ie#$ z&$F8dCAGD+0U7TG1p7U{c-F(OL8=J?s4gWEi0-SJQWQ(X79BdvO8?rmM+3N-o{93c zou+weBvAXenmwW*VyfSOuzuAj(B?fy?7uN7e?x~KdzpW{>}eAM3XX9Y6!2Ybc8HQG z4pSZ!FbSPBA|RIxn7zAyshBvisMV7R2atjLs2TyJ^^X(ytLOS(U50$ANi|s% z9E>?zE8XH`3=s=ok7L{Sg~1Ydv!v8jU||=s5JZ*HtcSE; zC@7e6Fj&{%r&Up@o-KzyCsXATJrrwCyc~^Oc2GFY0|gVnv!K6=%-n`9>$s=H#rjxj z3aE;3d6#i9CdRBS)`OA|DhwC?-9#Gf9(%@yXAh6W!Sd3Pr7Rnt{F5ec0G<9TTpl4# z&;H#aA}!-`b93(+`{?wRsXXT_xL0hlzrE^HNC+KFZ|wnvV}|oW8IfpO-qM-K`~@6= ztN=7TpG5Z$#*h#;RB4|vvIM;3`j_rkcr&4sr@J&(D}Ld^3{M_gWKHdxw<;y27r42{ z?)T`rYi6t-l~84q4Aa_Jo#SKQ=Y>s_*_b@RG^N4BwH` z(Q&oP5z9y0KGhsnTT?UcfsM#RK9=YaNs`nN;Rl4W+i50rrBH2}N0`}Q>yBQa_#FU* zc$Z~B+|kt~<9FzNiSCmK5Ow^#yzPmA3u2R!hI)FO;&gU>NTHueO7%E3X@%XRPN~7h z)w;*W;iW@fMhupn$yrYtAHu55;Rk{O6)m|8z4L3ec;3KK2-(brTZK4v`W-#dB@|o-2dD)UXuT z-Jr__FxD7E8Fc*|_O=m#ts6IJSO9R}D2cqL5P-M#Vy_Ogj{oQ}XPUEP)?ap)4& zX6l1&Z#0n2Iku%Msug$$+wAJnfOWeX4s1F|Tm^x`>({TP5#PSl_81OQ9X!had)EYT#j`<3v{)>iJnnz6;| z?z*Zf2|DsZgX$6xh4Av)^k%7Zb91xDBqw{l(V|Bh?gIgI_kpC>2Kec# zsCq9M8QJzUGeqtJtm?c=|G^(NtGk9@q3}_Pe>p9SQ8TIFH z?>fhpdR;q>wD+JZ4HqFxX-5u zpjG)C;^Vbr%bQ*tM&dOA1pYSH!HMUsO?ohYg3uK1|2idL8GQ-JnI?-2_ou=pS$s^+ zMi&`2Q8%Kdj?QUA#1!=tTpEte@E{6UMlfcDCY!?z(R`!?-Vu{qt$f{_elsl3dfQv5 z<-*+JH@YRt39uvRB-A+TC7RBiZtP{OcNh}v`c?IUXzdS}??%LsWS~o!1 z9rw^LU%t%De7Lnd*4Ni3mSVRwT;44~1j(`E?&iqsOg|s7uOFLqNL0%LhSaz}rUI~scH>HJAj7M&D&h#Pl%7#=c9$Sf|#hiJ7>0jWRG?*C(dv($@W@n4h12#d)?2n2)|bkes2m#t|eGO!&Ym z7^p7rCv6=aA7Cs%sQ8Q9O%b!x*QPiIJ(3s#zw{0{-gpKjBG)KX;5By^w&w1z@Vay$Z0|2bv;qZIya82!g({Yd0X690Q- zL5d3A=)tAS@q6#%sCb}bPbUP~r<|HyvbniUikn=bs)#ZPFFgon;V&pKsSIm%vE$wK zo^n#Lu{m$dYhJ8;vrQEFe zuw|fbwtv$-(Np2SCFXxj5`PDJ8E$18ySxl=9tJSvbjI)Y5f9b>l)JO7o?_;~^|$_< ziO2N!@>YNkGUd$=cJcK650Cmj)@frlwmVtMY*Z_bZ0vz#a#JUtOmA47c=97I_(iAr zH(83cPq&)Po*jSJgny7siaG-$R+>~H)bGJ>=>0F$KJ|i z-pTNtf*Q(k1N1tkrr3qPy!G{UyrBVTHPfDeuVX%L?j#u>FHo?ao;$qAy0sovLHhRJ z`vTQjSr%=v+#ZpR3OBJU# z2aGA7To0q=zp0_YBJ1l7uvJ|>qSzZe<_FU4A`=lXvWhIZz|KE?O8mTa1800xlaur1 z5eduo-o%jr{@YJAh0Gr9k$U`=^1sNkva_?()6;9&06+;!JU%}Dko6eE_DJ!NB@Q!p zGN_S+lEoWj{ldb+pv!RBbVk#50*ykU#_(3wqH;u0p|{LIf{#6+qerBGHGl+csgLq; zcxUQ7+mR$<6%*W%ENasdPX8f_RZ~?Jfc|xiiEq8JLcXreMe5Vha!BOh#vUG!6YQ(Y z%08JC2P+$WKbkm*pr*tT0OVurFL7RF74)7t`MNkDiv=^%laf3^x~Fx~%;3Pr+PdxI zM`d+&n&?}vZitGDYo=e-Q>dw}m1e<2`R!xdE42V>IY_Xx&_QfQfA*<>)|SzGAVwVP z={Zdig#Uh^tpI`xKngjChUW7hd=j&SiM3pBxj+Il`-Mo?uq`T6uPCz z-t?;8Ac2l0W9FPC(>6CZ`+YN7*GMn?OM8rM)zw|q^p%VI2Lw|vX(r*bZ4vb>4Vv^o zVYz<}p@7u;Luft<-oRy+L6_LbLA?)rKu`r!10?-nCl27svu#3xBq2q@N57$o@x}Am znyMG6f2sB#Wo74OelZF4Ux_(cZk}XY9kT5iq+7Yhb{n9>W6&)E{htz7um@VrLYFnF*``U?&B zcIGma%vT_PBHAZf)nm(%krdBc&FXAzZ0?s@s9(Ew&Bx~ebQj&0hFup2FPrLuyhUEV z5j3Cj^78D&MgWZL)w^#10~xtR<&TR5jhejr%-#KUOkQrT?^f*u@Li^%pron|z~%!x z9CWb2bbtc`6;?B{ga!dvZ*MO!ObPt=-9QoC$=2B1BK_HZpECS=+XsxsBPe*(Ea)4Z zo<1(zj9ndd=~b5&1bBr>y~4rH>a!7Dh{NNFOLlJ(i0XQJdH|L}r$?S0=G&(7bf$=7 zJgf{3dPO>CoHqGPK@>JJqIjN;JT$n|0+(c7IwYcwLtVp`biGWKsE+SVZFHGLy-xmoM4J z<)K0;-!4-@I`VHavLR~InO-ZXEpM5Lo;Rw9$JOw$8mZ+f4q@Tax#bfuD{Jdy&~yo| zG7t!B?in5R#uZeC1`puuY;4*)Iu=o#Fpe9b_b4kWT0YlL4!x73rU(jX_#6!5*sXsH zATT9G3Ls#50#H0F_I`UKAX^1UkWjz>rmzqWf~#!J3>k;daw;kze<&gK#7hkpr8>I0 zk&NPSfCH#XE3p(r{n&Acu4>m$!2A-kw3i4EgVk>%_EAh%Vs4(203PkdSI`b$9IM@mR$>X+l5^;++8|+DMeWR{ zfOWSSC|o+2*CeOL(1}#=K__i~w