openstack/ironic
Code Issues Proposed changes

Implements baremetal inspect abort

Browse Source 
This patch implements the feature of baremetal inspect abort.

For a node in inspect wait state, an abort request initiates the process of
introspection abort.

Adds transision from ``inspect wait`` to ``inspect failed`` with event
``abort`` to the state machine.

Adds an ``abort`` interface to ``InspectInterface``, which raises
UnsupportedDriverExtension by default.

Implements the ``abort`` interface for ironic inspector inspect interface.

API microversion is bumped for backwards compatibility.

Closes-Bug: #10602
Story: #1703089
Task: #10602

Change-Id: I02a39ed08c7c64b36c6ef05678f0af6d8324c5e2
This commit is contained in:
Kaifeng Wang 2017-10-18 16:04:42 +08:00
parent 30b494ad46
commit bce7f11785
15 changed files with 552 additions and 352 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/source/contributor/webapi-version-history.rst
View File

@ -2,6 +2,11 @@
REST API Version History REST API Version History
======================== ========================
1.41 (Rocky, master)
--------------------
Added support to abort inspection of a node in the ``inspect wait`` state.
1.40 (Rocky, master) 1.40 (Rocky, master)
--------------------- ---------------------

518
doc/source/images/states.svg
View File

@ -4,138 +4,138 @@
<!-- Generated by graphviz version 2.38.0 (20140413.2041) <!-- Generated by graphviz version 2.38.0 (20140413.2041)
--> -->
<!-- Title: Ironic states Pages: 1 --> <!-- Title: Ironic states Pages: 1 -->
<svg width="2609pt" height="851pt" <svg width="2609pt" height="855pt"
viewBox="0.00 0.00 2609.29 851.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> viewBox="0.00 0.00 2609.29 855.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 847)"> <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 851)">
<title>Ironic states</title> <title>Ironic states</title>
<polygon fill="white" stroke="none" points="-4,4 -4,-847 2605.29,-847 2605.29,4 -4,4"/> <polygon fill="white" stroke="none" points="-4,4 -4,-851 2605.29,-851 2605.29,4 -4,4"/>
<!-- enroll --> <!-- enroll -->
<g id="node1" class="node"><title>enroll</title> <g id="node1" class="node"><title>enroll</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="27" cy="-233" rx="27" ry="18"/> <ellipse fill="none" stroke="black" stroke-width="1.7" cx="27" cy="-237" rx="27" ry="18"/>
<text text-anchor="middle" x="27" y="-230.2" font-family="Times,serif" font-size="11.00">enroll</text> <text text-anchor="middle" x="27" y="-234.2" font-family="Times,serif" font-size="11.00">enroll</text>
</g> </g>
<!-- verifying --> <!-- verifying -->
<g id="node2" class="node"><title>verifying</title> <g id="node2" class="node"><title>verifying</title>
<ellipse fill="none" stroke="black" cx="208.675" cy="-233" rx="33.8507" ry="18"/> <ellipse fill="none" stroke="black" cx="208.675" cy="-237" rx="33.8507" ry="18"/>
<text text-anchor="middle" x="208.675" y="-230.2" font-family="Times,serif" font-size="11.00" fill="gray">verifying</text> <text text-anchor="middle" x="208.675" y="-234.2" font-family="Times,serif" font-size="11.00" fill="gray">verifying</text>
</g> </g>
<!-- enroll&#45;&gt;verifying --> <!-- enroll&#45;&gt;verifying -->
<g id="edge1" class="edge"><title>enroll&#45;&gt;verifying</title> <g id="edge1" class="edge"><title>enroll&#45;&gt;verifying</title>
<path fill="none" stroke="black" d="M54.1263,-233C83.0573,-233 130.15,-233 164.565,-233"/> <path fill="none" stroke="black" d="M54.1263,-237C83.0573,-237 130.15,-237 164.565,-237"/>
<polygon fill="black" stroke="black" points="164.805,-236.5 174.805,-233 164.805,-229.5 164.805,-236.5"/> <polygon fill="black" stroke="black" points="164.805,-240.5 174.805,-237 164.805,-233.5 164.805,-240.5"/>
<text text-anchor="middle" x="114.5" y="-236.4" font-family="Times,serif" font-size="12.00">manage (via API)</text> <text text-anchor="middle" x="114.5" y="-240.4" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g> </g>
<!-- verifying&#45;&gt;enroll --> <!-- verifying&#45;&gt;enroll -->
<g id="edge17" class="edge"><title>verifying&#45;&gt;enroll</title> <g id="edge17" class="edge"><title>verifying&#45;&gt;enroll</title>
<path fill="none" stroke="black" d="M181.26,-222.345C173.566,-219.736 165.059,-217.316 157,-216 119.716,-209.913 109.138,-209.075 72,-216 67.7433,-216.794 63.368,-217.985 59.1046,-219.375"/> <path fill="none" stroke="black" d="M181.26,-226.345C173.566,-223.736 165.059,-221.316 157,-220 119.716,-213.913 109.138,-213.075 72,-220 67.7433,-220.794 63.368,-221.985 59.1046,-223.375"/>
<polygon fill="black" stroke="black" points="57.6647,-216.175 49.4683,-222.888 60.0622,-222.751 57.6647,-216.175"/> <polygon fill="black" stroke="black" points="57.6647,-220.175 49.4683,-226.888 60.0622,-226.751 57.6647,-220.175"/>
<text text-anchor="middle" x="114.5" y="-219.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="114.5" y="-223.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- manageable --> <!-- manageable -->
<g id="node3" class="node"><title>manageable</title> <g id="node3" class="node"><title>manageable</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="344.443" cy="-233" rx="42.1875" ry="18"/> <ellipse fill="none" stroke="black" stroke-width="1.7" cx="344.443" cy="-237" rx="42.1875" ry="18"/>
<text text-anchor="middle" x="344.443" y="-230.2" font-family="Times,serif" font-size="11.00">manageable</text> <text text-anchor="middle" x="344.443" y="-234.2" font-family="Times,serif" font-size="11.00">manageable</text>
</g> </g>
<!-- verifying&#45;&gt;manageable --> <!-- verifying&#45;&gt;manageable -->
<g id="edge16" class="edge"><title>verifying&#45;&gt;manageable</title> <g id="edge16" class="edge"><title>verifying&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M242.398,-233C257.259,-233 275.195,-233 291.805,-233"/> <path fill="none" stroke="black" d="M242.398,-237C257.259,-237 275.195,-237 291.805,-237"/>
<polygon fill="black" stroke="black" points="291.937,-236.5 301.937,-233 291.937,-229.5 291.937,-236.5"/> <polygon fill="black" stroke="black" points="291.937,-240.5 301.937,-237 291.937,-233.5 291.937,-240.5"/>
<text text-anchor="middle" x="272.35" y="-236.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text> <text text-anchor="middle" x="272.35" y="-240.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g> </g>
<!-- cleaning --> <!-- cleaning -->
<g id="node4" class="node"><title>cleaning</title> <g id="node4" class="node"><title>cleaning</title>
<ellipse fill="none" stroke="black" cx="549.037" cy="-286" rx="32.4445" ry="18"/> <ellipse fill="none" stroke="black" cx="549.037" cy="-290" rx="32.4445" ry="18"/>
<text text-anchor="middle" x="549.037" y="-283.2" font-family="Times,serif" font-size="11.00" fill="gray">cleaning</text> <text text-anchor="middle" x="549.037" y="-287.2" font-family="Times,serif" font-size="11.00" fill="gray">cleaning</text>
</g> </g>
<!-- manageable&#45;&gt;cleaning --> <!-- manageable&#45;&gt;cleaning -->
<g id="edge2" class="edge"><title>manageable&#45;&gt;cleaning</title> <g id="edge2" class="edge"><title>manageable&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M359.543,-249.899C370.458,-261.604 386.688,-276.249 404.537,-283 437.687,-295.537 478.297,-295.069 507.845,-292.178"/> <path fill="none" stroke="black" d="M359.543,-253.899C370.458,-265.604 386.688,-280.249 404.537,-287 437.687,-299.537 478.297,-299.069 507.845,-296.178"/>
<polygon fill="black" stroke="black" points="508.252,-295.654 517.805,-291.071 507.479,-288.697 508.252,-295.654"/> <polygon fill="black" stroke="black" points="508.252,-299.654 517.805,-295.071 507.479,-292.697 508.252,-299.654"/>
<text text-anchor="middle" x="446.537" y="-296.4" font-family="Times,serif" font-size="12.00">provide (via API)</text> <text text-anchor="middle" x="446.537" y="-300.4" font-family="Times,serif" font-size="12.00">provide (via API)</text>
</g> </g>
<!-- manageable&#45;&gt;cleaning --> <!-- manageable&#45;&gt;cleaning -->
<g id="edge3" class="edge"><title>manageable&#45;&gt;cleaning</title> <g id="edge3" class="edge"><title>manageable&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M381.682,-241.647C389.232,-243.431 397.142,-245.288 404.537,-247 441.827,-255.634 451.682,-255.667 488.537,-266 496.052,-268.107 504.019,-270.6 511.597,-273.099"/> <path fill="none" stroke="black" d="M381.682,-245.647C389.232,-247.431 397.142,-249.288 404.537,-251 441.827,-259.634 451.682,-259.667 488.537,-270 496.052,-272.107 504.019,-274.6 511.597,-277.099"/>
<polygon fill="black" stroke="black" points="510.67,-276.48 521.264,-276.355 512.904,-269.846 510.67,-276.48"/> <polygon fill="black" stroke="black" points="510.67,-280.48 521.264,-280.355 512.904,-273.846 510.67,-280.48"/>
<text text-anchor="middle" x="446.537" y="-269.4" font-family="Times,serif" font-size="12.00">clean (via API)</text> <text text-anchor="middle" x="446.537" y="-273.4" font-family="Times,serif" font-size="12.00">clean (via API)</text>
</g> </g>
<!-- inspecting --> <!-- inspecting -->
<g id="node5" class="node"><title>inspecting</title> <g id="node5" class="node"><title>inspecting</title>
<ellipse fill="none" stroke="black" cx="549.037" cy="-98" rx="37.0671" ry="18"/> <ellipse fill="none" stroke="black" cx="549.037" cy="-102" rx="37.0671" ry="18"/>
<text text-anchor="middle" x="549.037" y="-95.2" font-family="Times,serif" font-size="11.00" fill="gray">inspecting</text> <text text-anchor="middle" x="549.037" y="-99.2" font-family="Times,serif" font-size="11.00" fill="gray">inspecting</text>
</g> </g>
<!-- manageable&#45;&gt;inspecting --> <!-- manageable&#45;&gt;inspecting -->
<g id="edge4" class="edge"><title>manageable&#45;&gt;inspecting</title> <g id="edge4" class="edge"><title>manageable&#45;&gt;inspecting</title>
<path fill="none" stroke="black" d="M351.504,-214.864C359.877,-192.636 377.133,-155.415 404.537,-135 408.707,-131.894 464.841,-117.955 505.643,-108.104"/> <path fill="none" stroke="black" d="M351.504,-218.864C359.877,-196.636 377.133,-159.415 404.537,-139 408.707,-135.894 464.841,-121.955 505.643,-112.104"/>
<polygon fill="black" stroke="black" points="506.482,-111.502 515.385,-105.759 504.844,-104.696 506.482,-111.502"/> <polygon fill="black" stroke="black" points="506.482,-115.502 515.385,-109.759 504.844,-108.696 506.482,-115.502"/>
<text text-anchor="middle" x="446.537" y="-138.4" font-family="Times,serif" font-size="12.00">inspect (via API)</text> <text text-anchor="middle" x="446.537" y="-142.4" font-family="Times,serif" font-size="12.00">inspect (via API)</text>
</g> </g>
<!-- adopting --> <!-- adopting -->
<g id="node6" class="node"><title>adopting</title> <g id="node6" class="node"><title>adopting</title>
<ellipse fill="none" stroke="black" cx="549.037" cy="-434" rx="32.4445" ry="18"/> <ellipse fill="none" stroke="black" cx="549.037" cy="-438" rx="32.4445" ry="18"/>
<text text-anchor="middle" x="549.037" y="-431.2" font-family="Times,serif" font-size="11.00" fill="gray">adopting</text> <text text-anchor="middle" x="549.037" y="-435.2" font-family="Times,serif" font-size="11.00" fill="gray">adopting</text>
</g> </g>
<!-- manageable&#45;&gt;adopting --> <!-- manageable&#45;&gt;adopting -->
<g id="edge5" class="edge"><title>manageable&#45;&gt;adopting</title> <g id="edge5" class="edge"><title>manageable&#45;&gt;adopting</title>
<path fill="none" stroke="black" d="M346.454,-251.026C349.374,-285.184 360.7,-359.451 404.537,-399 432.164,-423.924 474.573,-431.839 506.132,-434.018"/> <path fill="none" stroke="black" d="M346.454,-255.026C349.374,-289.184 360.7,-363.451 404.537,-403 432.164,-427.924 474.573,-435.839 506.132,-438.018"/>
<polygon fill="black" stroke="black" points="506.213,-437.527 516.382,-434.554 506.578,-430.537 506.213,-437.527"/> <polygon fill="black" stroke="black" points="506.213,-441.527 516.382,-438.554 506.578,-434.537 506.213,-441.527"/>
<text text-anchor="middle" x="446.537" y="-435.4" font-family="Times,serif" font-size="12.00">adopt (via API)</text> <text text-anchor="middle" x="446.537" y="-439.4" font-family="Times,serif" font-size="12.00">adopt (via API)</text>
</g> </g>
<!-- cleaning&#45;&gt;manageable --> <!-- cleaning&#45;&gt;manageable -->
<g id="edge30" class="edge"><title>cleaning&#45;&gt;manageable</title> <g id="edge30" class="edge"><title>cleaning&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M536.055,-269.182C525.345,-255.722 508.317,-237.844 488.537,-230 457.988,-217.885 420.811,-219.089 392.009,-223.083"/> <path fill="none" stroke="black" d="M536.055,-273.182C525.345,-259.722 508.317,-241.844 488.537,-234 457.988,-221.885 420.811,-223.089 392.009,-227.083"/>
<polygon fill="black" stroke="black" points="391.152,-219.674 381.804,-224.661 392.222,-226.592 391.152,-219.674"/> <polygon fill="black" stroke="black" points="391.152,-223.674 381.804,-228.661 392.222,-230.592 391.152,-223.674"/>
<text text-anchor="middle" x="446.537" y="-233.4" font-family="Times,serif" font-size="12.00" fill="gray">manage</text> <text text-anchor="middle" x="446.537" y="-237.4" font-family="Times,serif" font-size="12.00" fill="gray">manage</text>
</g> </g>
<!-- available --> <!-- available -->
<g id="node7" class="node"><title>available</title> <g id="node7" class="node"><title>available</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="755.232" cy="-457" rx="34.054" ry="18"/> <ellipse fill="none" stroke="black" stroke-width="1.7" cx="755.232" cy="-461" rx="34.054" ry="18"/>
<text text-anchor="middle" x="755.232" y="-454.2" font-family="Times,serif" font-size="11.00">available</text> <text text-anchor="middle" x="755.232" y="-458.2" font-family="Times,serif" font-size="11.00">available</text>
</g> </g>
<!-- cleaning&#45;&gt;available --> <!-- cleaning&#45;&gt;available -->
<g id="edge27" class="edge"><title>cleaning&#45;&gt;available</title> <g id="edge27" class="edge"><title>cleaning&#45;&gt;available</title>
<path fill="none" stroke="black" d="M562.458,-302.567C573.694,-316.818 591.239,-337.335 609.537,-352 643.392,-379.132 659.691,-374.154 694.537,-400 708.003,-409.988 721.629,-422.752 732.456,-433.669"/> <path fill="none" stroke="black" d="M562.458,-306.567C573.694,-320.818 591.239,-341.335 609.537,-356 643.392,-383.132 659.691,-378.154 694.537,-404 708.003,-413.988 721.629,-426.752 732.456,-437.669"/>
<polygon fill="black" stroke="black" points="729.986,-436.149 739.468,-440.875 735.003,-431.267 729.986,-436.149"/> <polygon fill="black" stroke="black" points="729.986,-440.149 739.468,-444.875 735.003,-435.267 729.986,-440.149"/>
<text text-anchor="middle" x="652.037" y="-403.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text> <text text-anchor="middle" x="652.037" y="-407.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g> </g>
<!-- clean failed --> <!-- clean failed -->
<g id="node17" class="node"><title>clean failed</title> <g id="node17" class="node"><title>clean failed</title>
<ellipse fill="none" stroke="black" cx="956.23" cy="-253" rx="41.4846" ry="18"/> <ellipse fill="none" stroke="black" cx="956.23" cy="-257" rx="41.4846" ry="18"/>
<text text-anchor="middle" x="956.23" y="-250.2" font-family="Times,serif" font-size="11.00" fill="red">clean failed</text> <text text-anchor="middle" x="956.23" y="-254.2" font-family="Times,serif" font-size="11.00" fill="red">clean failed</text>
</g> </g>
<!-- cleaning&#45;&gt;clean failed --> <!-- cleaning&#45;&gt;clean failed -->
<g id="edge28" class="edge"><title>cleaning&#45;&gt;clean failed</title> <g id="edge28" class="edge"><title>cleaning&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M581.627,-286.825C610.914,-287.395 655.693,-287.767 694.537,-286 782.539,-281.996 804.985,-282.191 891.927,-268 897.854,-267.033 904.05,-265.82 910.147,-264.506"/> <path fill="none" stroke="black" d="M581.627,-290.825C610.914,-291.395 655.693,-291.767 694.537,-290 782.539,-285.996 804.985,-286.191 891.927,-272 897.854,-271.033 904.05,-269.82 910.147,-268.506"/>
<polygon fill="black" stroke="black" points="911.268,-267.841 920.251,-262.223 909.725,-261.013 911.268,-267.841"/> <polygon fill="black" stroke="black" points="911.268,-271.841 920.251,-266.223 909.725,-265.013 911.268,-271.841"/>
<text text-anchor="middle" x="755.232" y="-288.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="755.232" y="-292.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- clean wait --> <!-- clean wait -->
<g id="node18" class="node"><title>clean wait</title> <g id="node18" class="node"><title>clean wait</title>
<ellipse fill="none" stroke="black" cx="755.232" cy="-246" rx="37.7689" ry="18"/> <ellipse fill="none" stroke="black" cx="755.232" cy="-250" rx="37.7689" ry="18"/>
<text text-anchor="middle" x="755.232" y="-243.2" font-family="Times,serif" font-size="11.00" fill="gray">clean wait</text> <text text-anchor="middle" x="755.232" y="-247.2" font-family="Times,serif" font-size="11.00" fill="gray">clean wait</text>
</g> </g>
<!-- cleaning&#45;&gt;clean wait --> <!-- cleaning&#45;&gt;clean wait -->
<g id="edge29" class="edge"><title>cleaning&#45;&gt;clean wait</title> <g id="edge29" class="edge"><title>cleaning&#45;&gt;clean wait</title>
<path fill="none" stroke="black" d="M579.355,-279.408C588.96,-277.298 599.692,-274.994 609.537,-273 642.885,-266.247 680.686,-259.215 709.287,-254.021"/> <path fill="none" stroke="black" d="M579.355,-283.408C588.96,-281.298 599.692,-278.994 609.537,-277 642.885,-270.247 680.686,-263.215 709.287,-258.021"/>
<polygon fill="black" stroke="black" points="710.165,-257.419 719.382,-252.194 708.919,-250.53 710.165,-257.419"/> <polygon fill="black" stroke="black" points="710.165,-261.419 719.382,-256.194 708.919,-254.53 710.165,-261.419"/>
<text text-anchor="middle" x="652.037" y="-276.4" font-family="Times,serif" font-size="12.00" fill="gray">wait</text> <text text-anchor="middle" x="652.037" y="-280.4" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
</g> </g>
<!-- inspecting&#45;&gt;manageable --> <!-- inspecting&#45;&gt;manageable -->
<g id="edge37" class="edge"><title>inspecting&#45;&gt;manageable</title> <g id="edge37" class="edge"><title>inspecting&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M517.555,-88.3444C486.468,-80.4713 438.156,-73.8882 404.537,-96 367.983,-120.042 353.94,-171.998 348.616,-204.567"/> <path fill="none" stroke="black" d="M517.555,-92.3444C486.468,-84.4713 438.156,-77.8882 404.537,-100 367.983,-124.042 353.94,-175.998 348.616,-208.567"/>
<polygon fill="black" stroke="black" points="345.103,-204.401 347.125,-214.801 352.03,-205.41 345.103,-204.401"/> <polygon fill="black" stroke="black" points="345.103,-208.401 347.125,-218.801 352.03,-209.41 345.103,-208.401"/>
<text text-anchor="middle" x="446.537" y="-99.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text> <text text-anchor="middle" x="446.537" y="-103.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g> </g>
<!-- inspect failed --> <!-- inspect failed -->
<g id="node19" class="node"><title>inspect failed</title> <g id="node19" class="node"><title>inspect failed</title>
<ellipse fill="none" stroke="black" cx="956.23" cy="-93" rx="46.1069" ry="18"/> <ellipse fill="none" stroke="black" cx="956.23" cy="-76" rx="46.1069" ry="18"/>
<text text-anchor="middle" x="956.23" y="-90.2" font-family="Times,serif" font-size="11.00" fill="red">inspect failed</text> <text text-anchor="middle" x="956.23" y="-73.2" font-family="Times,serif" font-size="11.00" fill="red">inspect failed</text>
</g> </g>
<!-- inspecting&#45;&gt;inspect failed --> <!-- inspecting&#45;&gt;inspect failed -->
<g id="edge38" class="edge"><title>inspecting&#45;&gt;inspect failed</title> <g id="edge38" class="edge"><title>inspecting&#45;&gt;inspect failed</title>
<path fill="none" stroke="black" d="M586.352,-97.5519C657.397,-96.6752 815.176,-94.7282 899.518,-93.6875"/> <path fill="none" stroke="black" d="M586.394,-102.893C649.051,-103.948 781.307,-104.054 891.927,-90 896.767,-89.3851 901.782,-88.5771 906.775,-87.659"/>
<polygon fill="black" stroke="black" points="899.721,-97.1853 909.677,-93.5621 899.635,-90.1859 899.721,-97.1853"/> <polygon fill="black" stroke="black" points="907.801,-91.024 916.929,-85.6465 906.44,-84.1576 907.801,-91.024"/>
<text text-anchor="middle" x="755.232" y="-99.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="755.232" y="-106.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- inspect wait --> <!-- inspect wait -->
<g id="node20" class="node"><title>inspect wait</title> <g id="node20" class="node"><title>inspect wait</title>
@ -144,362 +144,368 @@
</g> </g>
<!-- inspecting&#45;&gt;inspect wait --> <!-- inspecting&#45;&gt;inspect wait -->
<g id="edge39" class="edge"><title>inspecting&#45;&gt;inspect wait</title> <g id="edge39" class="edge"><title>inspecting&#45;&gt;inspect wait</title>
<path fill="none" stroke="black" d="M563.969,-81.4809C575.062,-69.4465 591.653,-53.7737 609.537,-45 625.929,-36.9587 669.104,-29.4821 704.055,-24.4452"/> <path fill="none" stroke="black" d="M564.269,-85.3671C575.396,-73.4368 591.894,-57.948 609.537,-49 626.012,-40.6446 669.636,-31.8943 704.709,-25.8292"/>
<polygon fill="black" stroke="black" points="704.716,-27.8866 714.131,-23.0267 703.741,-20.955 704.716,-27.8866"/> <polygon fill="black" stroke="black" points="705.538,-29.2387 714.811,-24.1146 704.367,-22.3374 705.538,-29.2387"/>
<text text-anchor="middle" x="652.037" y="-48.4" font-family="Times,serif" font-size="12.00" fill="gray">wait</text> <text text-anchor="middle" x="652.037" y="-52.4" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
</g> </g>
<!-- active --> <!-- active -->
<g id="node9" class="node"><title>active</title> <g id="node9" class="node"><title>active</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="1167.64" cy="-555" rx="27" ry="18"/> <ellipse fill="none" stroke="black" stroke-width="1.7" cx="1167.64" cy="-559" rx="27" ry="18"/>
<text text-anchor="middle" x="1167.64" y="-552.2" font-family="Times,serif" font-size="11.00">active</text> <text text-anchor="middle" x="1167.64" y="-556.2" font-family="Times,serif" font-size="11.00">active</text>
</g> </g>
<!-- adopting&#45;&gt;active --> <!-- adopting&#45;&gt;active -->
<g id="edge44" class="edge"><title>adopting&#45;&gt;active</title> <g id="edge45" class="edge"><title>adopting&#45;&gt;active</title>
<path fill="none" stroke="black" d="M581.489,-432.996C614.22,-432.006 666.965,-430.561 712.537,-430 750.485,-429.533 760.482,-423.825 797.927,-430 927.7,-451.399 1073.22,-512.489 1135.57,-540.566"/> <path fill="none" stroke="black" d="M581.489,-436.996C614.22,-436.006 666.965,-434.561 712.537,-434 750.485,-433.533 760.482,-427.825 797.927,-434 927.7,-455.399 1073.22,-516.489 1135.57,-544.566"/>
<polygon fill="black" stroke="black" points="1134.3,-543.831 1144.85,-544.778 1137.19,-537.457 1134.3,-543.831"/> <polygon fill="black" stroke="black" points="1134.3,-547.831 1144.85,-548.778 1137.19,-541.457 1134.3,-547.831"/>
<text text-anchor="middle" x="853.927" y="-453.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text> <text text-anchor="middle" x="853.927" y="-457.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g> </g>
<!-- adopt failed --> <!-- adopt failed -->
<g id="node21" class="node"><title>adopt failed</title> <g id="node21" class="node"><title>adopt failed</title>
<ellipse fill="none" stroke="black" cx="755.232" cy="-511" rx="41.4846" ry="18"/> <ellipse fill="none" stroke="black" cx="755.232" cy="-515" rx="41.4846" ry="18"/>
<text text-anchor="middle" x="755.232" y="-508.2" font-family="Times,serif" font-size="11.00" fill="red">adopt failed</text> <text text-anchor="middle" x="755.232" y="-512.2" font-family="Times,serif" font-size="11.00" fill="red">adopt failed</text>
</g> </g>
<!-- adopting&#45;&gt;adopt failed --> <!-- adopting&#45;&gt;adopt failed -->
<g id="edge45" class="edge"><title>adopting&#45;&gt;adopt failed</title> <g id="edge46" class="edge"><title>adopting&#45;&gt;adopt failed</title>
<path fill="none" stroke="black" d="M568.577,-448.632C582.662,-459.509 601.016,-473.109 609.537,-477 639.747,-490.793 676.151,-499.31 704.886,-504.371"/> <path fill="none" stroke="black" d="M568.577,-452.632C582.662,-463.509 601.016,-477.109 609.537,-481 639.747,-494.793 676.151,-503.31 704.886,-508.371"/>
<polygon fill="black" stroke="black" points="704.659,-507.882 715.1,-506.082 705.816,-500.978 704.659,-507.882"/> <polygon fill="black" stroke="black" points="704.659,-511.882 715.1,-510.082 705.816,-504.978 704.659,-511.882"/>
<text text-anchor="middle" x="652.037" y="-505.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="652.037" y="-509.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- available&#45;&gt;manageable --> <!-- available&#45;&gt;manageable -->
<g id="edge7" class="edge"><title>available&#45;&gt;manageable</title> <g id="edge7" class="edge"><title>available&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M726.023,-466.944C660.612,-488.018 497.13,-528.998 404.537,-449 376.12,-424.448 356.939,-313.941 349.238,-261.193"/> <path fill="none" stroke="black" d="M726.023,-470.944C660.612,-492.018 497.13,-532.998 404.537,-453 376.12,-428.448 356.939,-317.941 349.238,-265.193"/>
<polygon fill="black" stroke="black" points="352.684,-260.568 347.81,-251.161 345.754,-261.554 352.684,-260.568"/> <polygon fill="black" stroke="black" points="352.684,-264.568 347.81,-255.161 345.754,-265.554 352.684,-264.568"/>
<text text-anchor="middle" x="549.037" y="-499.4" font-family="Times,serif" font-size="12.00">manage (via API)</text> <text text-anchor="middle" x="549.037" y="-503.4" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g> </g>
<!-- deploying --> <!-- deploying -->
<g id="node8" class="node"><title>deploying</title> <g id="node8" class="node"><title>deploying</title>
<ellipse fill="none" stroke="black" cx="956.23" cy="-674" rx="35.4579" ry="18"/> <ellipse fill="none" stroke="black" cx="956.23" cy="-678" rx="35.4579" ry="18"/>
<text text-anchor="middle" x="956.23" y="-671.2" font-family="Times,serif" font-size="11.00" fill="gray">deploying</text> <text text-anchor="middle" x="956.23" y="-675.2" font-family="Times,serif" font-size="11.00" fill="gray">deploying</text>
</g> </g>
<!-- available&#45;&gt;deploying --> <!-- available&#45;&gt;deploying -->
<g id="edge6" class="edge"><title>available&#45;&gt;deploying</title> <g id="edge6" class="edge"><title>available&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M779.208,-470.106C785.558,-474.19 792.257,-478.956 797.927,-484 856.136,-535.775 911.05,-609.91 937.958,-648.482"/> <path fill="none" stroke="black" d="M779.208,-474.106C785.558,-478.19 792.257,-482.956 797.927,-488 856.136,-539.775 911.05,-613.91 937.958,-652.482"/>
<polygon fill="black" stroke="black" points="935.311,-650.808 943.877,-657.043 941.069,-646.827 935.311,-650.808"/> <polygon fill="black" stroke="black" points="935.311,-654.808 943.877,-661.043 941.069,-650.827 935.311,-654.808"/>
<text text-anchor="middle" x="853.927" y="-585.4" font-family="Times,serif" font-size="12.00">active (via API)</text> <text text-anchor="middle" x="853.927" y="-589.4" font-family="Times,serif" font-size="12.00">active (via API)</text>
</g> </g>
<!-- deploying&#45;&gt;active --> <!-- deploying&#45;&gt;active -->
<g id="edge20" class="edge"><title>deploying&#45;&gt;active</title> <g id="edge20" class="edge"><title>deploying&#45;&gt;active</title>
<path fill="none" stroke="black" d="M961.095,-655.96C967.906,-629.163 984.846,-579.678 1020.53,-558 1054.24,-537.527 1100.87,-540.808 1132.31,-546.546"/> <path fill="none" stroke="black" d="M960.849,-660.046C967.407,-632.827 984.101,-581.969 1020.53,-560 1054.39,-539.582 1101.19,-543.658 1132.61,-549.97"/>
<polygon fill="black" stroke="black" points="1131.86,-550.025 1142.35,-548.555 1133.24,-543.161 1131.86,-550.025"/> <polygon fill="black" stroke="black" points="1132.12,-553.445 1142.64,-552.164 1133.62,-546.607 1132.12,-553.445"/>
<text text-anchor="middle" x="1061.03" y="-561.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text> <text text-anchor="middle" x="1061.03" y="-563.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g> </g>
<!-- deploy failed --> <!-- deploy failed -->
<g id="node15" class="node"><title>deploy failed</title> <g id="node15" class="node"><title>deploy failed</title>
<ellipse fill="none" stroke="black" cx="1374.25" cy="-723" rx="44.498" ry="18"/> <ellipse fill="none" stroke="black" cx="1374.25" cy="-727" rx="44.498" ry="18"/>
<text text-anchor="middle" x="1374.25" y="-720.2" font-family="Times,serif" font-size="11.00" fill="red">deploy failed</text> <text text-anchor="middle" x="1374.25" y="-724.2" font-family="Times,serif" font-size="11.00" fill="red">deploy failed</text>
</g> </g>
<!-- deploying&#45;&gt;deploy failed --> <!-- deploying&#45;&gt;deploy failed -->
<g id="edge18" class="edge"><title>deploying&#45;&gt;deploy failed</title> <g id="edge18" class="edge"><title>deploying&#45;&gt;deploy failed</title>
<path fill="none" stroke="black" d="M966.444,-691.258C976.924,-708.883 995.833,-735.27 1020.53,-747 1123.58,-795.935 1262.91,-761.165 1331.53,-738.438"/> <path fill="none" stroke="black" d="M966.444,-695.258C976.924,-712.883 995.833,-739.27 1020.53,-751 1123.58,-799.935 1262.91,-765.165 1331.53,-742.438"/>
<polygon fill="black" stroke="black" points="1332.78,-741.71 1341.13,-735.184 1330.53,-735.08 1332.78,-741.71"/> <polygon fill="black" stroke="black" points="1332.78,-745.71 1341.13,-739.184 1330.53,-739.08 1332.78,-745.71"/>
<text text-anchor="middle" x="1167.64" y="-774.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="1167.64" y="-778.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- wait call&#45;back --> <!-- wait call&#45;back -->
<g id="node16" class="node"><title>wait call&#45;back</title> <g id="node16" class="node"><title>wait call&#45;back</title>
<ellipse fill="none" stroke="black" cx="1167.64" cy="-723" rx="48.2143" ry="18"/> <ellipse fill="none" stroke="black" cx="1167.64" cy="-727" rx="48.2143" ry="18"/>
<text text-anchor="middle" x="1167.64" y="-720.2" font-family="Times,serif" font-size="11.00" fill="gray">wait call&#45;back</text> <text text-anchor="middle" x="1167.64" y="-724.2" font-family="Times,serif" font-size="11.00" fill="gray">wait call&#45;back</text>
</g> </g>
<!-- deploying&#45;&gt;wait call&#45;back --> <!-- deploying&#45;&gt;wait call&#45;back -->
<g id="edge19" class="edge"><title>deploying&#45;&gt;wait call&#45;back</title> <g id="edge19" class="edge"><title>deploying&#45;&gt;wait call&#45;back</title>
<path fill="none" stroke="black" d="M972.756,-689.955C984.658,-700.992 1002.12,-714.772 1020.53,-721 1049.75,-730.882 1084.16,-731.829 1112.25,-730.099"/> <path fill="none" stroke="black" d="M972.756,-693.955C984.658,-704.992 1002.12,-718.772 1020.53,-725 1049.75,-734.882 1084.16,-735.829 1112.25,-734.099"/>
<polygon fill="black" stroke="black" points="1112.58,-733.585 1122.3,-729.352 1112.06,-726.604 1112.58,-733.585"/> <polygon fill="black" stroke="black" points="1112.58,-737.585 1122.3,-733.352 1112.06,-730.604 1112.58,-737.585"/>
<text text-anchor="middle" x="1061.03" y="-733.4" font-family="Times,serif" font-size="12.00" fill="gray">wait</text> <text text-anchor="middle" x="1061.03" y="-737.4" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
</g> </g>
<!-- active&#45;&gt;deploying --> <!-- active&#45;&gt;deploying -->
<g id="edge8" class="edge"><title>active&#45;&gt;deploying</title> <g id="edge8" class="edge"><title>active&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1142.64,-562.015C1101.35,-574.092 1021.19,-597.569 1020.53,-598 1001.04,-610.808 984.407,-631.423 973.091,-647.954"/> <path fill="none" stroke="black" d="M1142.29,-565.768C1102.09,-576.997 1025.73,-598.588 1020.53,-602 1001.04,-614.808 984.407,-635.423 973.091,-651.954"/>
<polygon fill="black" stroke="black" points="969.946,-646.365 967.367,-656.641 975.791,-650.217 969.946,-646.365"/> <polygon fill="black" stroke="black" points="969.946,-650.365 967.367,-660.641 975.791,-654.217 969.946,-650.365"/>
<text text-anchor="middle" x="1061.03" y="-601.4" font-family="Times,serif" font-size="12.00">rebuild (via API)</text> <text text-anchor="middle" x="1061.03" y="-605.4" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
</g> </g>
<!-- deleting --> <!-- deleting -->
<g id="node10" class="node"><title>deleting</title> <g id="node10" class="node"><title>deleting</title>
<ellipse fill="none" stroke="black" cx="2398.02" cy="-686" rx="31.0408" ry="18"/> <ellipse fill="none" stroke="black" cx="2398.02" cy="-690" rx="31.0408" ry="18"/>
<text text-anchor="middle" x="2398.02" y="-683.2" font-family="Times,serif" font-size="11.00" fill="gray">deleting</text> <text text-anchor="middle" x="2398.02" y="-687.2" font-family="Times,serif" font-size="11.00" fill="gray">deleting</text>
</g> </g>
<!-- active&#45;&gt;deleting --> <!-- active&#45;&gt;deleting -->
<g id="edge9" class="edge"><title>active&#45;&gt;deleting</title> <g id="edge9" class="edge"><title>active&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1188.2,-567.071C1231.19,-592.846 1338,-652.725 1436.75,-675 1526.73,-695.299 2174.51,-692.944 2266.75,-694 2303.2,-694.417 2312.43,-696.99 2348.75,-694 2351.83,-693.747 2355,-693.404 2358.19,-693.003"/> <path fill="none" stroke="black" d="M1188.2,-571.071C1231.19,-596.846 1338,-656.725 1436.75,-679 1526.73,-699.299 2174.51,-696.944 2266.75,-698 2303.2,-698.417 2312.43,-700.99 2348.75,-698 2351.83,-697.747 2355,-697.404 2358.19,-697.003"/>
<polygon fill="black" stroke="black" points="2358.88,-696.441 2368.28,-691.559 2357.88,-689.511 2358.88,-696.441"/> <polygon fill="black" stroke="black" points="2358.88,-700.441 2368.28,-695.559 2357.88,-693.511 2358.88,-700.441"/>
<text text-anchor="middle" x="1767.53" y="-694.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text> <text text-anchor="middle" x="1767.53" y="-698.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g> </g>
<!-- rescuing --> <!-- rescuing -->
<g id="node11" class="node"><title>rescuing</title> <g id="node11" class="node"><title>rescuing</title>
<ellipse fill="none" stroke="black" cx="1374.25" cy="-554" rx="32.4445" ry="18"/> <ellipse fill="none" stroke="black" cx="1374.25" cy="-556" rx="32.4445" ry="18"/>
<text text-anchor="middle" x="1374.25" y="-551.2" font-family="Times,serif" font-size="11.00" fill="gray">rescuing</text> <text text-anchor="middle" x="1374.25" y="-553.2" font-family="Times,serif" font-size="11.00" fill="gray">rescuing</text>
</g> </g>
<!-- active&#45;&gt;rescuing --> <!-- active&#45;&gt;rescuing -->
<g id="edge10" class="edge"><title>active&#45;&gt;rescuing</title> <g id="edge10" class="edge"><title>active&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M1194.97,-554.871C1229.52,-554.702 1290.56,-554.404 1331.46,-554.204"/> <path fill="none" stroke="black" d="M1194.97,-558.614C1229.52,-558.107 1290.56,-557.212 1331.46,-556.613"/>
<polygon fill="black" stroke="black" points="1331.67,-557.703 1341.65,-554.154 1331.63,-550.703 1331.67,-557.703"/> <polygon fill="black" stroke="black" points="1331.7,-560.11 1341.65,-556.463 1331.6,-553.11 1331.7,-560.11"/>
<text text-anchor="middle" x="1272.75" y="-557.4" font-family="Times,serif" font-size="12.00">rescue (via API)</text> <text text-anchor="middle" x="1272.75" y="-561.4" font-family="Times,serif" font-size="12.00">rescue (via API)</text>
</g> </g>
<!-- deleting&#45;&gt;cleaning --> <!-- deleting&#45;&gt;cleaning -->
<g id="edge36" class="edge"><title>deleting&#45;&gt;cleaning</title> <g id="edge36" class="edge"><title>deleting&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M2381.76,-670.467C2354.15,-641.411 2296.06,-575.723 2266.75,-509 2250,-470.853 2279.58,-445.028 2248.75,-417 1993.65,-185.069 1821.52,-341 1476.75,-341 754.232,-341 754.232,-341 754.232,-341 692.502,-341 623.776,-317.275 583.872,-300.989"/> <path fill="none" stroke="black" d="M2381.96,-674.272C2354.68,-644.879 2297.11,-578.548 2266.75,-512 2251.81,-479.236 2275.59,-457.018 2248.75,-433 1991.45,-202.67 1822.08,-345 1476.75,-345 754.232,-345 754.232,-345 754.232,-345 692.502,-345 623.776,-321.275 583.872,-304.989"/>
<polygon fill="black" stroke="black" points="585.186,-297.745 574.609,-297.13 582.494,-304.207 585.186,-297.745"/> <polygon fill="black" stroke="black" points="585.186,-301.745 574.609,-301.13 582.494,-308.207 585.186,-301.745"/>
<text text-anchor="middle" x="1475.75" y="-344.4" font-family="Times,serif" font-size="12.00" fill="gray">clean</text> <text text-anchor="middle" x="1475.75" y="-348.4" font-family="Times,serif" font-size="12.00" fill="gray">clean</text>
</g> </g>
<!-- error --> <!-- error -->
<g id="node12" class="node"><title>error</title> <g id="node12" class="node"><title>error</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="2574.29" cy="-722" rx="27" ry="18"/> <ellipse fill="none" stroke="black" stroke-width="1.7" cx="2574.29" cy="-726" rx="27" ry="18"/>
<text text-anchor="middle" x="2574.29" y="-719.2" font-family="Times,serif" font-size="11.00" fill="red">error</text> <text text-anchor="middle" x="2574.29" y="-723.2" font-family="Times,serif" font-size="11.00" fill="red">error</text>
</g> </g>
<!-- deleting&#45;&gt;error --> <!-- deleting&#45;&gt;error -->
<g id="edge35" class="edge"><title>deleting&#45;&gt;error</title> <g id="edge35" class="edge"><title>deleting&#45;&gt;error</title>
<path fill="none" stroke="black" d="M2427.24,-679.396C2454.35,-674.41 2495.98,-670.361 2529.29,-683 2538.21,-686.383 2546.48,-692.492 2553.39,-698.887"/> <path fill="none" stroke="black" d="M2427.24,-683.396C2454.35,-678.41 2495.98,-674.361 2529.29,-687 2538.21,-690.383 2546.48,-696.492 2553.39,-702.887"/>
<polygon fill="black" stroke="black" points="2551.3,-701.748 2560.83,-706.378 2556.26,-696.816 2551.3,-701.748"/> <polygon fill="black" stroke="black" points="2551.3,-705.748 2560.83,-710.378 2556.26,-700.816 2551.3,-705.748"/>
<text text-anchor="middle" x="2488.29" y="-686.4" font-family="Times,serif" font-size="12.00" fill="gray">error</text> <text text-anchor="middle" x="2488.29" y="-690.4" font-family="Times,serif" font-size="12.00" fill="gray">error</text>
</g> </g>
<!-- rescue --> <!-- rescue -->
<g id="node13" class="node"><title>rescue</title> <g id="node13" class="node"><title>rescue</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="1668.53" cy="-638" rx="27.824" ry="18"/> <ellipse fill="none" stroke="black" stroke-width="1.7" cx="1668.53" cy="-642" rx="27.824" ry="18"/>
<text text-anchor="middle" x="1668.53" y="-635.2" font-family="Times,serif" font-size="11.00">rescue</text> <text text-anchor="middle" x="1668.53" y="-639.2" font-family="Times,serif" font-size="11.00">rescue</text>
</g> </g>
<!-- rescuing&#45;&gt;rescue --> <!-- rescuing&#45;&gt;rescue -->
<g id="edge48" class="edge"><title>rescuing&#45;&gt;rescue</title> <g id="edge49" class="edge"><title>rescuing&#45;&gt;rescue</title>
<path fill="none" stroke="black" d="M1394.85,-568.16C1406.51,-575.964 1421.88,-585.214 1436.75,-591 1502.5,-616.598 1584.46,-629.041 1630.91,-634.434"/> <path fill="none" stroke="black" d="M1393.93,-570.387C1405.64,-578.743 1421.38,-588.81 1436.75,-595 1502.19,-621.37 1584.26,-633.545 1630.81,-638.68"/>
<polygon fill="black" stroke="black" points="1630.77,-637.939 1641.09,-635.568 1631.54,-630.982 1630.77,-637.939"/> <polygon fill="black" stroke="black" points="1630.71,-642.188 1641.02,-639.755 1631.44,-635.227 1630.71,-642.188"/>
<text text-anchor="middle" x="1475.75" y="-616.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text> <text text-anchor="middle" x="1475.75" y="-621.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g> </g>
<!-- rescue wait --> <!-- rescue wait -->
<g id="node22" class="node"><title>rescue wait</title> <g id="node22" class="node"><title>rescue wait</title>
<ellipse fill="none" stroke="black" cx="1573.64" cy="-523" rx="40.7822" ry="18"/> <ellipse fill="none" stroke="black" cx="1573.64" cy="-527" rx="40.7822" ry="18"/>
<text text-anchor="middle" x="1573.64" y="-520.2" font-family="Times,serif" font-size="11.00" fill="gray">rescue wait</text> <text text-anchor="middle" x="1573.64" y="-524.2" font-family="Times,serif" font-size="11.00" fill="gray">rescue wait</text>
</g> </g>
<!-- rescuing&#45;&gt;rescue wait --> <!-- rescuing&#45;&gt;rescue wait -->
<g id="edge49" class="edge"><title>rescuing&#45;&gt;rescue wait</title> <g id="edge50" class="edge"><title>rescuing&#45;&gt;rescue wait</title>
<path fill="none" stroke="black" d="M1405.66,-549.224C1437.39,-544.241 1487.59,-536.357 1524.67,-530.533"/> <path fill="none" stroke="black" d="M1406.09,-551.469C1437.8,-546.809 1487.64,-539.488 1524.54,-534.066"/>
<polygon fill="black" stroke="black" points="1525.42,-533.959 1534.75,-528.949 1524.33,-527.043 1525.42,-533.959"/> <polygon fill="black" stroke="black" points="1525.19,-537.508 1534.58,-532.591 1524.17,-530.582 1525.19,-537.508"/>
<text text-anchor="middle" x="1475.75" y="-546.4" font-family="Times,serif" font-size="12.00" fill="gray">wait</text> <text text-anchor="middle" x="1475.75" y="-549.4" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
</g> </g>
<!-- rescue failed --> <!-- rescue failed -->
<g id="node23" class="node"><title>rescue failed</title> <g id="node23" class="node"><title>rescue failed</title>
<ellipse fill="none" stroke="black" cx="1767.53" cy="-511" rx="44.498" ry="18"/> <ellipse fill="none" stroke="black" cx="1767.53" cy="-515" rx="44.498" ry="18"/>
<text text-anchor="middle" x="1767.53" y="-508.2" font-family="Times,serif" font-size="11.00" fill="red">rescue failed</text> <text text-anchor="middle" x="1767.53" y="-512.2" font-family="Times,serif" font-size="11.00" fill="red">rescue failed</text>
</g> </g>
<!-- rescuing&#45;&gt;rescue failed --> <!-- rescuing&#45;&gt;rescue failed -->
<g id="edge50" class="edge"><title>rescuing&#45;&gt;rescue failed</title> <g id="edge51" class="edge"><title>rescuing&#45;&gt;rescue failed</title>
<path fill="none" stroke="black" d="M1406.17,-557.61C1415.94,-558.591 1426.78,-559.515 1436.75,-560 1555.95,-565.801 1589.64,-572.303 1704.53,-540 1713.6,-537.449 1723,-533.695 1731.66,-529.729"/> <path fill="none" stroke="black" d="M1405.84,-560.174C1415.69,-561.332 1426.66,-562.428 1436.75,-563 1555.87,-569.758 1589.67,-576.294 1704.53,-544 1713.6,-541.449 1723,-537.695 1731.66,-533.729"/>
<polygon fill="black" stroke="black" points="1733.21,-532.868 1740.71,-525.388 1730.18,-526.557 1733.21,-532.868"/> <polygon fill="black" stroke="black" points="1733.21,-536.868 1740.71,-529.388 1730.18,-530.557 1733.21,-536.868"/>
<text text-anchor="middle" x="1573.64" y="-567.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="1573.64" y="-571.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- error&#45;&gt;deploying --> <!-- error&#45;&gt;deploying -->
<g id="edge11" class="edge"><title>error&#45;&gt;deploying</title> <g id="edge11" class="edge"><title>error&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M2560.15,-737.474C2533.22,-767.337 2468.43,-830 2399.02,-830 1166.64,-830 1166.64,-830 1166.64,-830 1096.77,-830 1074.52,-816.342 1020.53,-772 997.081,-752.739 979.397,-722.455 968.762,-700.649"/> <path fill="none" stroke="black" d="M2560.15,-741.474C2533.22,-771.337 2468.43,-834 2399.02,-834 1166.64,-834 1166.64,-834 1166.64,-834 1096.77,-834 1074.52,-820.342 1020.53,-776 997.081,-756.739 979.397,-726.455 968.762,-704.649"/>
<polygon fill="black" stroke="black" points="971.926,-699.152 964.508,-691.588 965.59,-702.127 971.926,-699.152"/> <polygon fill="black" stroke="black" points="971.926,-703.152 964.508,-695.588 965.59,-706.127 971.926,-703.152"/>
<text text-anchor="middle" x="1767.53" y="-833.4" font-family="Times,serif" font-size="12.00">rebuild (via API)</text> <text text-anchor="middle" x="1767.53" y="-837.4" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
</g> </g>
<!-- error&#45;&gt;deleting --> <!-- error&#45;&gt;deleting -->
<g id="edge12" class="edge"><title>error&#45;&gt;deleting</title> <g id="edge12" class="edge"><title>error&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M2547.55,-718.349C2521.97,-714.523 2481.69,-707.966 2447.29,-700 2443.48,-699.117 2439.52,-698.114 2435.59,-697.06"/> <path fill="none" stroke="black" d="M2547.55,-722.349C2521.97,-718.523 2481.69,-711.966 2447.29,-704 2443.48,-703.117 2439.52,-702.114 2435.59,-701.06"/>
<polygon fill="black" stroke="black" points="2436.52,-693.686 2425.95,-694.374 2434.64,-700.43 2436.52,-693.686"/> <polygon fill="black" stroke="black" points="2436.52,-697.686 2425.95,-698.374 2434.64,-704.43 2436.52,-697.686"/>
<text text-anchor="middle" x="2488.29" y="-718.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text> <text text-anchor="middle" x="2488.29" y="-722.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g> </g>
<!-- rescue&#45;&gt;deleting --> <!-- rescue&#45;&gt;deleting -->
<g id="edge14" class="edge"><title>rescue&#45;&gt;deleting</title> <g id="edge14" class="edge"><title>rescue&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1696.28,-639.765C1806.35,-647.028 2224.63,-674.625 2356.76,-683.344"/> <path fill="none" stroke="black" d="M1696.28,-643.765C1806.35,-651.028 2224.63,-678.625 2356.76,-687.344"/>
<polygon fill="black" stroke="black" points="2356.65,-686.844 2366.86,-684.01 2357.11,-679.859 2356.65,-686.844"/> <polygon fill="black" stroke="black" points="2356.65,-690.844 2366.86,-688.01 2357.11,-683.859 2356.65,-690.844"/>
<text text-anchor="middle" x="1979.53" y="-664.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text> <text text-anchor="middle" x="1979.53" y="-668.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g> </g>
<!-- rescue&#45;&gt;rescuing --> <!-- rescue&#45;&gt;rescuing -->
<g id="edge13" class="edge"><title>rescue&#45;&gt;rescuing</title> <g id="edge13" class="edge"><title>rescue&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M1643.31,-645.61C1598.14,-658.211 1499.91,-677.922 1436.75,-634 1418.72,-621.467 1431.86,-605.607 1418.75,-588 1414.84,-582.754 1409.92,-577.888 1404.81,-573.573"/> <path fill="none" stroke="black" d="M1643.2,-649.349C1598.28,-661.361 1501.02,-679.917 1436.75,-638 1420.75,-627.572 1429.71,-614.628 1418.75,-599 1413.57,-591.627 1407.05,-584.42 1400.65,-578.078"/>
<polygon fill="black" stroke="black" points="1406.78,-570.667 1396.74,-567.27 1402.47,-576.184 1406.78,-570.667"/> <polygon fill="black" stroke="black" points="1402.93,-575.418 1393.27,-571.072 1398.11,-580.495 1402.93,-575.418"/>
<text text-anchor="middle" x="1475.75" y="-663.4" font-family="Times,serif" font-size="12.00">rescue (via API)</text> <text text-anchor="middle" x="1475.75" y="-665.4" font-family="Times,serif" font-size="12.00">rescue (via API)</text>
</g> </g>
<!-- unrescuing --> <!-- unrescuing -->
<g id="node14" class="node"><title>unrescuing</title> <g id="node14" class="node"><title>unrescuing</title>
<ellipse fill="none" stroke="black" cx="1979.53" cy="-473" rx="39.1741" ry="18"/> <ellipse fill="none" stroke="black" cx="1979.53" cy="-476" rx="39.1741" ry="18"/>
<text text-anchor="middle" x="1979.53" y="-470.2" font-family="Times,serif" font-size="11.00" fill="gray">unrescuing</text> <text text-anchor="middle" x="1979.53" y="-473.2" font-family="Times,serif" font-size="11.00" fill="gray">unrescuing</text>
</g> </g>
<!-- rescue&#45;&gt;unrescuing --> <!-- rescue&#45;&gt;unrescuing -->
<g id="edge15" class="edge"><title>rescue&#45;&gt;unrescuing</title> <g id="edge15" class="edge"><title>rescue&#45;&gt;unrescuing</title>
<path fill="none" stroke="black" d="M1690.87,-627.307C1734.41,-605.398 1836.45,-553.544 1920.53,-507 1928.93,-502.35 1937.92,-497.178 1946.25,-492.3"/> <path fill="none" stroke="black" d="M1690.86,-631.201C1734.38,-609.079 1836.39,-556.752 1920.53,-510 1928.92,-505.337 1937.91,-500.161 1946.24,-495.282"/>
<polygon fill="black" stroke="black" points="1948.13,-495.252 1954.97,-487.161 1944.58,-489.22 1948.13,-495.252"/> <polygon fill="black" stroke="black" points="1948.12,-498.234 1954.96,-490.144 1944.57,-492.202 1948.12,-498.234"/>
<text text-anchor="middle" x="1767.53" y="-614.4" font-family="Times,serif" font-size="12.00">unrescue (via API)</text> <text text-anchor="middle" x="1767.53" y="-618.4" font-family="Times,serif" font-size="12.00">unrescue (via API)</text>
</g> </g>
<!-- unrescuing&#45;&gt;active --> <!-- unrescuing&#45;&gt;active -->
<g id="edge58" class="edge"><title>unrescuing&#45;&gt;active</title> <g id="edge59" class="edge"><title>unrescuing&#45;&gt;active</title>
<path fill="none" stroke="black" d="M1952.92,-459.812C1921.01,-444.131 1864.35,-418.736 1812.53,-408 1648.96,-374.115 1595.85,-363.143 1436.75,-414 1342.62,-444.085 1242.24,-505.962 1195.49,-536.758"/> <path fill="none" stroke="black" d="M1952.29,-462.999C1920.22,-447.83 1863.87,-423.491 1812.53,-413 1648.88,-379.562 1595.84,-367.145 1436.75,-418 1342.62,-448.085 1242.24,-509.962 1195.49,-540.758"/>
<polygon fill="black" stroke="black" points="1193.53,-533.859 1187.13,-542.308 1197.4,-539.69 1193.53,-533.859"/> <polygon fill="black" stroke="black" points="1193.53,-537.859 1187.13,-546.308 1197.4,-543.69 1193.53,-537.859"/>
<text text-anchor="middle" x="1573.64" y="-390.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text> <text text-anchor="middle" x="1573.64" y="-395.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g> </g>
<!-- unrescue failed --> <!-- unrescue failed -->
<g id="node24" class="node"><title>unrescue failed</title> <g id="node24" class="node"><title>unrescue failed</title>
<ellipse fill="none" stroke="black" cx="2197.64" cy="-444" rx="51.2277" ry="18"/> <ellipse fill="none" stroke="black" cx="2197.64" cy="-460" rx="51.2277" ry="18"/>
<text text-anchor="middle" x="2197.64" y="-441.2" font-family="Times,serif" font-size="11.00" fill="red">unrescue failed</text> <text text-anchor="middle" x="2197.64" y="-457.2" font-family="Times,serif" font-size="11.00" fill="red">unrescue failed</text>
</g> </g>
<!-- unrescuing&#45;&gt;unrescue failed --> <!-- unrescuing&#45;&gt;unrescue failed -->
<g id="edge59" class="edge"><title>unrescuing&#45;&gt;unrescue failed</title> <g id="edge60" class="edge"><title>unrescuing&#45;&gt;unrescue failed</title>
<path fill="none" stroke="black" d="M2017.19,-468.081C2050.68,-463.586 2100.75,-456.868 2139.49,-451.669"/> <path fill="none" stroke="black" d="M2018.17,-473.213C2050.89,-470.791 2098.87,-467.239 2136.84,-464.427"/>
<polygon fill="black" stroke="black" points="2140.14,-455.113 2149.59,-450.314 2139.21,-448.175 2140.14,-455.113"/> <polygon fill="black" stroke="black" points="2137.55,-467.885 2147.26,-463.656 2137.03,-460.904 2137.55,-467.885"/>
<text text-anchor="middle" x="2083.53" y="-467.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="2083.53" y="-474.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- deploy failed&#45;&gt;deploying --> <!-- deploy failed&#45;&gt;deploying -->
<g id="edge24" class="edge"><title>deploy failed&#45;&gt;deploying</title> <g id="edge24" class="edge"><title>deploy failed&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1338.95,-712.026C1307.33,-702.452 1258.9,-689.158 1215.75,-683 1141.3,-672.375 1053.63,-671.799 1001.89,-672.672"/> <path fill="none" stroke="black" d="M1338.95,-716.026C1307.33,-706.452 1258.9,-693.158 1215.75,-687 1141.3,-676.375 1053.63,-675.799 1001.89,-676.672"/>
<polygon fill="black" stroke="black" points="1001.66,-669.176 991.726,-672.87 1001.79,-676.175 1001.66,-669.176"/> <polygon fill="black" stroke="black" points="1001.66,-673.176 991.726,-676.87 1001.79,-680.175 1001.66,-673.176"/>
<text text-anchor="middle" x="1167.64" y="-686.4" font-family="Times,serif" font-size="12.00">rebuild (via API)</text> <text text-anchor="middle" x="1167.64" y="-690.4" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
</g> </g>
<!-- deploy failed&#45;&gt;deploying --> <!-- deploy failed&#45;&gt;deploying -->
<g id="edge25" class="edge"><title>deploy failed&#45;&gt;deploying</title> <g id="edge25" class="edge"><title>deploy failed&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1353.19,-707.136C1341.63,-698.649 1326.5,-688.613 1311.75,-682 1271.51,-663.966 1259.48,-662.601 1215.75,-657 1140.2,-647.325 1051.49,-657.887 1000.08,-666.147"/> <path fill="none" stroke="black" d="M1353.19,-711.136C1341.63,-702.649 1326.5,-692.613 1311.75,-686 1271.51,-667.966 1259.48,-666.601 1215.75,-661 1140.2,-651.325 1051.49,-661.887 1000.08,-670.147"/>
<polygon fill="black" stroke="black" points="999.297,-662.729 990.002,-667.815 1000.44,-669.635 999.297,-662.729"/> <polygon fill="black" stroke="black" points="999.297,-666.729 990.002,-671.815 1000.44,-673.635 999.297,-666.729"/>
<text text-anchor="middle" x="1167.64" y="-660.4" font-family="Times,serif" font-size="12.00">active (via API)</text> <text text-anchor="middle" x="1167.64" y="-664.4" font-family="Times,serif" font-size="12.00">active (via API)</text>
</g> </g>
<!-- deploy failed&#45;&gt;deleting --> <!-- deploy failed&#45;&gt;deleting -->
<g id="edge26" class="edge"><title>deploy failed&#45;&gt;deleting</title> <g id="edge26" class="edge"><title>deploy failed&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1418.69,-721.813C1497.28,-719.664 1668.29,-714.983 1812.53,-711 1860.53,-709.675 1872.54,-709.774 1920.53,-708 2082.06,-702.028 2274.29,-692.373 2356.49,-688.123"/> <path fill="none" stroke="black" d="M1418.69,-725.813C1497.28,-723.664 1668.29,-718.983 1812.53,-715 1860.53,-713.675 1872.54,-713.774 1920.53,-712 2082.06,-706.028 2274.29,-696.373 2356.49,-692.123"/>
<polygon fill="black" stroke="black" points="2356.84,-691.609 2366.65,-687.596 2356.48,-684.619 2356.84,-691.609"/> <polygon fill="black" stroke="black" points="2356.84,-695.609 2366.65,-691.596 2356.48,-688.619 2356.84,-695.609"/>
<text text-anchor="middle" x="1875.53" y="-713.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text> <text text-anchor="middle" x="1875.53" y="-717.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g> </g>
<!-- wait call&#45;back&#45;&gt;deploying --> <!-- wait call&#45;back&#45;&gt;deploying -->
<g id="edge21" class="edge"><title>wait call&#45;back&#45;&gt;deploying</title> <g id="edge21" class="edge"><title>wait call&#45;back&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1129.94,-711.739C1120.7,-709.058 1110.79,-706.308 1101.53,-704 1067.59,-695.533 1028.75,-687.554 999.918,-681.956"/> <path fill="none" stroke="black" d="M1129.94,-715.739C1120.7,-713.058 1110.79,-710.308 1101.53,-708 1067.59,-699.533 1028.75,-691.554 999.918,-685.956"/>
<polygon fill="black" stroke="black" points="1000.26,-678.457 989.777,-680.005 998.936,-685.331 1000.26,-678.457"/> <polygon fill="black" stroke="black" points="1000.26,-682.457 989.777,-684.005 998.936,-689.331 1000.26,-682.457"/>
<text text-anchor="middle" x="1061.03" y="-707.4" font-family="Times,serif" font-size="12.00" fill="gray">resume</text> <text text-anchor="middle" x="1061.03" y="-711.4" font-family="Times,serif" font-size="12.00" fill="gray">resume</text>
</g> </g>
<!-- wait call&#45;back&#45;&gt;deleting --> <!-- wait call&#45;back&#45;&gt;deleting -->
<g id="edge23" class="edge"><title>wait call&#45;back&#45;&gt;deleting</title> <g id="edge23" class="edge"><title>wait call&#45;back&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1198.4,-737.04C1237.57,-754.205 1309,-781 1373.25,-781 1373.25,-781 1373.25,-781 1669.53,-781 1973.71,-781 2049.15,-749.526 2348.75,-697 2352.16,-696.403 2355.69,-695.722 2359.21,-694.999"/> <path fill="none" stroke="black" d="M1198.4,-741.04C1237.57,-758.205 1309,-785 1373.25,-785 1373.25,-785 1373.25,-785 1669.53,-785 1973.71,-785 2049.15,-753.526 2348.75,-701 2352.16,-700.403 2355.69,-699.722 2359.21,-698.999"/>
<polygon fill="black" stroke="black" points="2360.01,-698.408 2369.04,-692.88 2358.53,-691.565 2360.01,-698.408"/> <polygon fill="black" stroke="black" points="2360.01,-702.408 2369.04,-696.88 2358.53,-695.565 2360.01,-702.408"/>
<text text-anchor="middle" x="1767.53" y="-783.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text> <text text-anchor="middle" x="1767.53" y="-787.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g> </g>
<!-- wait call&#45;back&#45;&gt;deploy failed --> <!-- wait call&#45;back&#45;&gt;deploy failed -->
<g id="edge22" class="edge"><title>wait call&#45;back&#45;&gt;deploy failed</title> <g id="edge22" class="edge"><title>wait call&#45;back&#45;&gt;deploy failed</title>
<path fill="none" stroke="black" d="M1215.84,-723C1246.72,-723 1287.15,-723 1319.31,-723"/> <path fill="none" stroke="black" d="M1215.84,-727C1246.72,-727 1287.15,-727 1319.31,-727"/>
<polygon fill="black" stroke="black" points="1319.43,-726.5 1329.43,-723 1319.43,-719.5 1319.43,-726.5"/> <polygon fill="black" stroke="black" points="1319.43,-730.5 1329.43,-727 1319.43,-723.5 1319.43,-730.5"/>
<text text-anchor="middle" x="1272.75" y="-726.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="1272.75" y="-730.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- clean failed&#45;&gt;manageable --> <!-- clean failed&#45;&gt;manageable -->
<g id="edge34" class="edge"><title>clean failed&#45;&gt;manageable</title> <g id="edge34" class="edge"><title>clean failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M929.475,-239.134C918.26,-233.755 904.767,-228.143 891.927,-225 851.265,-215.045 839.745,-220.941 797.927,-219 623.273,-210.894 577.797,-193.529 404.537,-217 399.338,-217.704 393.947,-218.732 388.632,-219.932"/> <path fill="none" stroke="black" d="M929.475,-243.134C918.26,-237.755 904.767,-232.143 891.927,-229 851.265,-219.045 839.745,-224.941 797.927,-223 623.273,-214.894 577.797,-197.529 404.537,-221 399.338,-221.704 393.947,-222.732 388.632,-223.932"/>
<polygon fill="black" stroke="black" points="387.444,-216.62 378.573,-222.412 389.12,-223.416 387.444,-216.62"/> <polygon fill="black" stroke="black" points="387.444,-220.62 378.573,-226.412 389.12,-227.416 387.444,-220.62"/>
<text text-anchor="middle" x="652.037" y="-216.4" font-family="Times,serif" font-size="12.00">manage (via API)</text> <text text-anchor="middle" x="652.037" y="-220.4" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g> </g>
<!-- clean wait&#45;&gt;cleaning --> <!-- clean wait&#45;&gt;cleaning -->
<g id="edge33" class="edge"><title>clean wait&#45;&gt;cleaning</title> <g id="edge33" class="edge"><title>clean wait&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M721.076,-237.942C691.098,-232.2 646.179,-227.488 609.537,-240 595.752,-244.707 582.682,-254.048 572.322,-263.089"/> <path fill="none" stroke="black" d="M721.076,-241.942C691.098,-236.2 646.179,-231.488 609.537,-244 595.752,-248.707 582.682,-258.048 572.322,-267.089"/>
<polygon fill="black" stroke="black" points="569.949,-260.517 564.96,-269.864 574.689,-265.668 569.949,-260.517"/> <polygon fill="black" stroke="black" points="569.949,-264.517 564.96,-273.864 574.689,-269.668 569.949,-264.517"/>
<text text-anchor="middle" x="652.037" y="-243.4" font-family="Times,serif" font-size="12.00" fill="gray">resume</text> <text text-anchor="middle" x="652.037" y="-247.4" font-family="Times,serif" font-size="12.00" fill="gray">resume</text>
</g> </g>
<!-- clean wait&#45;&gt;clean failed --> <!-- clean wait&#45;&gt;clean failed -->
<g id="edge31" class="edge"><title>clean wait&#45;&gt;clean failed</title> <g id="edge31" class="edge"><title>clean wait&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M793.165,-247.98C800.695,-248.349 808.565,-248.709 815.927,-249 845.273,-250.159 878.109,-251.118 904.528,-251.806"/> <path fill="none" stroke="black" d="M793.165,-251.98C800.695,-252.349 808.565,-252.709 815.927,-253 845.273,-254.159 878.109,-255.118 904.528,-255.806"/>
<polygon fill="black" stroke="black" points="904.6,-255.309 914.686,-252.066 904.779,-248.311 904.6,-255.309"/> <polygon fill="black" stroke="black" points="904.6,-259.309 914.686,-256.066 904.779,-252.311 904.6,-259.309"/>
<text text-anchor="middle" x="853.927" y="-254.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="853.927" y="-258.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- clean wait&#45;&gt;clean failed --> <!-- clean wait&#45;&gt;clean failed -->
<g id="edge32" class="edge"><title>clean wait&#45;&gt;clean failed</title> <g id="edge32" class="edge"><title>clean wait&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M788.212,-237.07C797.112,-234.977 806.833,-233.052 815.927,-232 849.481,-228.118 858.582,-226.613 891.927,-232 899.723,-233.259 907.859,-235.338 915.599,-237.711"/> <path fill="none" stroke="black" d="M788.212,-241.07C797.112,-238.977 806.833,-237.052 815.927,-236 849.481,-232.118 858.582,-230.613 891.927,-236 899.723,-237.259 907.859,-239.338 915.599,-241.711"/>
<polygon fill="black" stroke="black" points="914.544,-241.048 925.137,-240.833 916.722,-234.395 914.544,-241.048"/> <polygon fill="black" stroke="black" points="914.544,-245.048 925.137,-244.833 916.722,-238.395 914.544,-245.048"/>
<text text-anchor="middle" x="853.927" y="-235.4" font-family="Times,serif" font-size="12.00">abort (via API)</text> <text text-anchor="middle" x="853.927" y="-239.4" font-family="Times,serif" font-size="12.00">abort (via API)</text>
</g> </g>
<!-- inspect failed&#45;&gt;manageable --> <!-- inspect failed&#45;&gt;manageable -->
<g id="edge40" class="edge"><title>inspect failed&#45;&gt;manageable</title> <g id="edge40" class="edge"><title>inspect failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M911.165,-97.2979C816.144,-107.546 585.898,-137.675 404.537,-204 396.437,-206.962 387.977,-210.698 380.074,-214.494"/> <path fill="none" stroke="black" d="M920.607,-87.7177C911.35,-90.6363 901.309,-93.6109 891.927,-96 813.207,-116.046 791.832,-113.365 712.537,-131 574.8,-161.632 537.055,-159.537 404.537,-208 396.437,-210.962 387.977,-214.698 380.074,-218.494"/>
<polygon fill="black" stroke="black" points="378.514,-211.361 371.106,-218.936 381.62,-217.634 378.514,-211.361"/> <polygon fill="black" stroke="black" points="378.514,-215.361 371.106,-222.936 381.62,-221.634 378.514,-215.361"/>
<text text-anchor="middle" x="652.037" y="-149.4" font-family="Times,serif" font-size="12.00">manage (via API)</text> <text text-anchor="middle" x="652.037" y="-155.4" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g> </g>
<!-- inspect failed&#45;&gt;inspecting --> <!-- inspect failed&#45;&gt;inspecting -->
<g id="edge41" class="edge"><title>inspect failed&#45;&gt;inspecting</title> <g id="edge41" class="edge"><title>inspect failed&#45;&gt;inspecting</title>
<path fill="none" stroke="black" d="M910.157,-90.4617C878.716,-88.8216 835.779,-86.8418 797.927,-86 759.985,-85.1562 750.466,-84.7126 712.537,-86 666.68,-87.5565 655.289,-89.5232 609.537,-93 605.115,-93.336 600.509,-93.6994 595.905,-94.0716"/> <path fill="none" stroke="black" d="M910.04,-78.389C861.291,-81.0172 781.392,-85.472 712.537,-90 673.312,-92.5795 628.79,-95.8944 596.28,-98.3863"/>
<polygon fill="black" stroke="black" points="595.578,-90.5865 585.898,-94.8932 596.151,-97.563 595.578,-90.5865"/> <polygon fill="black" stroke="black" points="595.87,-94.9074 586.169,-99.1645 596.408,-101.887 595.87,-94.9074"/>
<text text-anchor="middle" x="755.232" y="-89.4" font-family="Times,serif" font-size="12.00">inspect (via API)</text> <text text-anchor="middle" x="755.232" y="-93.4" font-family="Times,serif" font-size="12.00">inspect (via API)</text>
</g> </g>
<!-- inspect wait&#45;&gt;manageable --> <!-- inspect wait&#45;&gt;manageable -->
<g id="edge42" class="edge"><title>inspect wait&#45;&gt;manageable</title> <g id="edge42" class="edge"><title>inspect wait&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M714.456,-12.2744C642.961,-4.36884 492.051,1.09013 404.537,-77 367.231,-110.289 353.367,-169.626 348.293,-204.739"/> <path fill="none" stroke="black" d="M714.208,-12.8694C642.652,-6.06784 492.105,-2.86119 404.537,-81 367.231,-114.289 353.367,-173.626 348.293,-208.739"/>
<polygon fill="black" stroke="black" points="344.804,-204.435 346.985,-214.803 351.745,-205.337 344.804,-204.435"/> <polygon fill="black" stroke="black" points="344.804,-208.435 346.985,-218.803 351.745,-209.337 344.804,-208.435"/>
<text text-anchor="middle" x="549.037" y="-26.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text> <text text-anchor="middle" x="549.037" y="-30.4" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g> </g>
<!-- inspect wait&#45;&gt;inspect failed --> <!-- inspect wait&#45;&gt;inspect failed -->
<g id="edge43" class="edge"><title>inspect wait&#45;&gt;inspect failed</title> <g id="edge43" class="edge"><title>inspect wait&#45;&gt;inspect failed</title>
<path fill="none" stroke="black" d="M797.48,-21.3299C825.163,-24.7203 861.915,-31.504 891.927,-45 905.56,-51.1302 918.995,-60.6725 929.956,-69.6393"/> <path fill="none" stroke="black" d="M794.746,-24.8927C822.183,-30.2627 859.749,-38.5546 891.927,-49 900.095,-51.6514 908.667,-54.9424 916.771,-58.3047"/>
<polygon fill="black" stroke="black" points="927.931,-72.5116 937.819,-76.318 932.463,-67.1765 927.931,-72.5116"/> <polygon fill="black" stroke="black" points="915.447,-61.545 926.019,-62.2505 918.195,-55.1066 915.447,-61.545"/>
<text text-anchor="middle" x="853.927" y="-48.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text> <text text-anchor="middle" x="853.927" y="-52.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- inspect wait&#45;&gt;inspect failed -->
<g id="edge44" class="edge"><title>inspect wait&#45;&gt;inspect failed</title>
<path fill="none" stroke="black" d="M791.951,-8.80514C820.179,-3.36397 860.004,0.178118 891.927,-13 910.324,-20.5943 926.187,-36.4115 937.432,-50.2964"/>
<polygon fill="black" stroke="black" points="934.948,-52.8076 943.818,-58.6012 940.497,-48.5405 934.948,-52.8076"/>
<text text-anchor="middle" x="853.927" y="-16.4" font-family="Times,serif" font-size="12.00">abort (via API)</text>
</g> </g>
<!-- adopt failed&#45;&gt;manageable --> <!-- adopt failed&#45;&gt;manageable -->
<g id="edge47" class="edge"><title>adopt failed&#45;&gt;manageable</title> <g id="edge48" class="edge"><title>adopt failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M727.614,-497.184C722.123,-493.451 716.741,-489.029 712.537,-484 699.249,-468.105 709.749,-455.066 694.537,-441 665.334,-413.996 643.124,-434.307 609.537,-413 599.715,-406.769 601.023,-400.733 591.537,-394 517.241,-341.261 475.997,-366.522 404.537,-310 386.806,-295.976 371.43,-275.504 360.811,-259.194"/> <path fill="none" stroke="black" d="M727.614,-501.184C722.123,-497.451 716.741,-493.029 712.537,-488 699.249,-472.105 709.749,-459.066 694.537,-445 665.334,-417.996 643.124,-438.307 609.537,-417 599.715,-410.769 601.023,-404.733 591.537,-398 517.241,-345.261 475.997,-370.522 404.537,-314 386.806,-299.976 371.43,-279.504 360.811,-263.194"/>
<polygon fill="black" stroke="black" points="363.706,-257.224 355.415,-250.628 357.783,-260.955 363.706,-257.224"/> <polygon fill="black" stroke="black" points="363.706,-261.224 355.415,-254.628 357.783,-264.955 363.706,-261.224"/>
<text text-anchor="middle" x="549.037" y="-397.4" font-family="Times,serif" font-size="12.00">manage (via API)</text> <text text-anchor="middle" x="549.037" y="-401.4" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g> </g>
<!-- adopt failed&#45;&gt;adopting --> <!-- adopt failed&#45;&gt;adopting -->
<g id="edge46" class="edge"><title>adopt failed&#45;&gt;adopting</title> <g id="edge47" class="edge"><title>adopt failed&#45;&gt;adopting</title>
<path fill="none" stroke="black" d="M728.923,-496.649C723.246,-492.889 717.454,-488.59 712.537,-484 702.79,-474.902 705.886,-466.999 694.537,-460 677.654,-449.587 627.891,-442.226 591.467,-438.054"/> <path fill="none" stroke="black" d="M728.923,-500.649C723.246,-496.889 717.454,-492.59 712.537,-488 702.79,-478.902 705.886,-470.999 694.537,-464 677.654,-453.587 627.891,-446.226 591.467,-442.054"/>
<polygon fill="black" stroke="black" points="591.438,-434.53 581.115,-436.912 590.67,-441.487 591.438,-434.53"/> <polygon fill="black" stroke="black" points="591.438,-438.53 581.115,-440.912 590.67,-445.487 591.438,-438.53"/>
<text text-anchor="middle" x="652.037" y="-463.4" font-family="Times,serif" font-size="12.00">adopt (via API)</text> <text text-anchor="middle" x="652.037" y="-467.4" font-family="Times,serif" font-size="12.00">adopt (via API)</text>
</g> </g>
<!-- rescue wait&#45;&gt;deleting --> <!-- rescue wait&#45;&gt;deleting -->
<g id="edge54" class="edge"><title>rescue wait&#45;&gt;deleting</title> <g id="edge55" class="edge"><title>rescue wait&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1607.54,-533.303C1615.67,-535.669 1624.38,-538.057 1632.53,-540 1711.24,-558.758 2270.01,-653.373 2348.75,-672 2352.56,-672.901 2356.52,-673.915 2360.45,-674.977"/> <path fill="none" stroke="black" d="M1607.54,-537.303C1615.67,-539.669 1624.38,-542.057 1632.53,-544 1711.24,-562.758 2270.01,-657.373 2348.75,-676 2352.56,-676.901 2356.52,-677.915 2360.45,-678.977"/>
<polygon fill="black" stroke="black" points="2359.52,-678.349 2370.09,-677.672 2361.4,-671.608 2359.52,-678.349"/> <polygon fill="black" stroke="black" points="2359.52,-682.349 2370.09,-681.672 2361.4,-675.608 2359.52,-682.349"/>
<text text-anchor="middle" x="1979.53" y="-614.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text> <text text-anchor="middle" x="1979.53" y="-618.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g> </g>
<!-- rescue wait&#45;&gt;rescuing --> <!-- rescue wait&#45;&gt;rescuing -->
<g id="edge51" class="edge"><title>rescue wait&#45;&gt;rescuing</title> <g id="edge52" class="edge"><title>rescue wait&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M1537.28,-514.509C1509.42,-509.334 1469.88,-505.379 1436.75,-515 1424.03,-518.69 1411.44,-525.885 1400.98,-533.1"/> <path fill="none" stroke="black" d="M1537.61,-518.274C1509.75,-512.851 1470.07,-508.536 1436.75,-518 1424.2,-521.563 1411.71,-528.494 1401.29,-535.472"/>
<polygon fill="black" stroke="black" points="1398.78,-530.375 1392.75,-539.089 1402.9,-536.035 1398.78,-530.375"/> <polygon fill="black" stroke="black" points="1399.23,-532.645 1393.07,-541.271 1403.26,-538.364 1399.23,-532.645"/>
<text text-anchor="middle" x="1475.75" y="-518.4" font-family="Times,serif" font-size="12.00" fill="gray">resume</text> <text text-anchor="middle" x="1475.75" y="-521.4" font-family="Times,serif" font-size="12.00" fill="gray">resume</text>
</g>
<!-- rescue wait&#45;&gt;rescue failed -->
<g id="edge52" class="edge"><title>rescue wait&#45;&gt;rescue failed</title>
<path fill="none" stroke="black" d="M1614.51,-523.873C1640.24,-524.07 1674.42,-523.646 1704.53,-521 1708.45,-520.655 1712.49,-520.209 1716.56,-519.697"/>
<polygon fill="black" stroke="black" points="1717.18,-523.143 1726.61,-518.308 1716.23,-516.209 1717.18,-523.143"/>
<text text-anchor="middle" x="1668.53" y="-526.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g> </g>
<!-- rescue wait&#45;&gt;rescue failed --> <!-- rescue wait&#45;&gt;rescue failed -->
<g id="edge53" class="edge"><title>rescue wait&#45;&gt;rescue failed</title> <g id="edge53" class="edge"><title>rescue wait&#45;&gt;rescue failed</title>
<path fill="none" stroke="black" d="M1604.53,-511.19C1613.4,-508.236 1623.23,-505.487 1632.53,-504 1659.82,-499.636 1690.6,-500.821 1715.77,-503.348"/> <path fill="none" stroke="black" d="M1614.51,-527.873C1640.24,-528.07 1674.42,-527.646 1704.53,-525 1708.45,-524.655 1712.49,-524.209 1716.56,-523.697"/>
<polygon fill="black" stroke="black" points="1715.51,-506.841 1725.83,-504.459 1716.28,-499.883 1715.51,-506.841"/> <polygon fill="black" stroke="black" points="1717.18,-527.143 1726.61,-522.308 1716.23,-520.209 1717.18,-527.143"/>
<text text-anchor="middle" x="1668.53" y="-507.4" font-family="Times,serif" font-size="12.00">abort (via API)</text> <text text-anchor="middle" x="1668.53" y="-530.4" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- rescue wait&#45;&gt;rescue failed -->
<g id="edge54" class="edge"><title>rescue wait&#45;&gt;rescue failed</title>
<path fill="none" stroke="black" d="M1604.53,-515.19C1613.4,-512.236 1623.23,-509.487 1632.53,-508 1659.82,-503.636 1690.6,-504.821 1715.77,-507.348"/>
<polygon fill="black" stroke="black" points="1715.51,-510.841 1725.83,-508.459 1716.28,-503.883 1715.51,-510.841"/>
<text text-anchor="middle" x="1668.53" y="-511.4" font-family="Times,serif" font-size="12.00">abort (via API)</text>
</g> </g>
<!-- rescue failed&#45;&gt;deleting --> <!-- rescue failed&#45;&gt;deleting -->
<g id="edge57" class="edge"><title>rescue failed&#45;&gt;deleting</title> <g id="edge58" class="edge"><title>rescue failed&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1811.94,-508.266C1931.91,-501.602 2264.67,-489.417 2348.75,-553 2381.35,-577.651 2391.97,-626.44 2395.4,-657.578"/> <path fill="none" stroke="black" d="M1811.51,-512.199C1931.12,-505.29 2264.6,-492.32 2348.75,-556 2381.56,-580.826 2392.11,-629.985 2395.48,-661.36"/>
<polygon fill="black" stroke="black" points="2391.94,-658.15 2396.34,-667.787 2398.91,-657.508 2391.94,-658.15"/> <polygon fill="black" stroke="black" points="2392.02,-661.997 2396.4,-671.648 2399,-661.376 2392.02,-661.997"/>
<text text-anchor="middle" x="2083.53" y="-508.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text> <text text-anchor="middle" x="2083.53" y="-511.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g> </g>
<!-- rescue failed&#45;&gt;rescuing --> <!-- rescue failed&#45;&gt;rescuing -->
<g id="edge55" class="edge"><title>rescue failed&#45;&gt;rescuing</title> <g id="edge56" class="edge"><title>rescue failed&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M1729.7,-501.286C1683.65,-490.364 1602.41,-475.12 1532.75,-483 1489.25,-487.92 1476.18,-485.999 1436.75,-505 1423.02,-511.611 1409.58,-521.766 1398.78,-531.12"/> <path fill="none" stroke="black" d="M1729.71,-505.399C1683.68,-494.593 1602.44,-479.466 1532.75,-487 1489.32,-491.694 1476.28,-489.434 1436.75,-508 1423.07,-514.423 1409.63,-524.364 1398.82,-533.536"/>
<polygon fill="black" stroke="black" points="1396.18,-528.758 1391.08,-538.045 1400.86,-533.964 1396.18,-528.758"/> <polygon fill="black" stroke="black" points="1396.3,-531.094 1391.11,-540.33 1400.93,-536.346 1396.3,-531.094"/>
<text text-anchor="middle" x="1573.64" y="-486.4" font-family="Times,serif" font-size="12.00">rescue (via API)</text> <text text-anchor="middle" x="1573.64" y="-490.4" font-family="Times,serif" font-size="12.00">rescue (via API)</text>
</g> </g>
<!-- rescue failed&#45;&gt;unrescuing --> <!-- rescue failed&#45;&gt;unrescuing -->
<g id="edge56" class="edge"><title>rescue failed&#45;&gt;unrescuing</title> <g id="edge57" class="edge"><title>rescue failed&#45;&gt;unrescuing</title>
<path fill="none" stroke="black" d="M1800.34,-498.783C1809.92,-495.496 1820.55,-492.237 1830.53,-490 1863.34,-482.645 1901.01,-478.376 1930.13,-475.955"/> <path fill="none" stroke="black" d="M1799.66,-502.402C1809.4,-498.88 1820.29,-495.372 1830.53,-493 1863.29,-485.409 1900.95,-481.144 1930.09,-478.784"/>
<polygon fill="black" stroke="black" points="1930.75,-479.417 1940.44,-475.145 1930.2,-472.439 1930.75,-479.417"/> <polygon fill="black" stroke="black" points="1930.7,-482.247 1940.41,-477.998 1930.17,-475.267 1930.7,-482.247"/>
<text text-anchor="middle" x="1875.53" y="-493.4" font-family="Times,serif" font-size="12.00">unrescue (via API)</text> <text text-anchor="middle" x="1875.53" y="-496.4" font-family="Times,serif" font-size="12.00">unrescue (via API)</text>
</g> </g>
<!-- unrescue failed&#45;&gt;deleting --> <!-- unrescue failed&#45;&gt;deleting -->
<g id="edge62" class="edge"><title>unrescue failed&#45;&gt;deleting</title> <g id="edge63" class="edge"><title>unrescue failed&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M2248.11,-447.572C2281.04,-452.559 2322.85,-464.339 2348.75,-492 2371.14,-515.906 2386.75,-610 2393.41,-657.803"/> <path fill="none" stroke="black" d="M2248.88,-458.967C2281.5,-461.04 2322.66,-469.253 2348.75,-495 2372.24,-518.169 2387.38,-613.595 2393.68,-661.824"/>
<polygon fill="black" stroke="black" points="2389.96,-658.365 2394.77,-667.804 2396.89,-657.424 2389.96,-658.365"/> <polygon fill="black" stroke="black" points="2390.22,-662.424 2394.95,-671.905 2397.17,-661.544 2390.22,-662.424"/>
<text text-anchor="middle" x="2307.75" y="-495.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text> <text text-anchor="middle" x="2307.75" y="-498.4" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g> </g>
<!-- unrescue failed&#45;&gt;rescuing --> <!-- unrescue failed&#45;&gt;rescuing -->
<g id="edge60" class="edge"><title>unrescue failed&#45;&gt;rescuing</title> <g id="edge61" class="edge"><title>unrescue failed&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M2146.17,-443.498C2094.13,-443.191 2010.62,-443.281 1938.53,-446 1715.16,-454.425 1635.78,-378.265 1436.75,-480 1416.29,-490.456 1400.06,-511.035 1389.42,-527.843"/> <path fill="none" stroke="black" d="M2147.71,-456.123C2112.39,-453.524 2063.55,-450.342 2020.53,-449 1760.73,-440.894 1668.86,-367.033 1436.75,-484 1416.68,-494.112 1400.51,-513.952 1389.79,-530.249"/>
<polygon fill="black" stroke="black" points="1386.25,-526.33 1384.1,-536.704 1392.25,-529.936 1386.25,-526.33"/> <polygon fill="black" stroke="black" points="1386.75,-528.519 1384.41,-538.854 1392.68,-532.231 1386.75,-528.519"/>
<text text-anchor="middle" x="1767.53" y="-445.4" font-family="Times,serif" font-size="12.00">rescue (via API)</text> <text text-anchor="middle" x="1767.53" y="-432.4" font-family="Times,serif" font-size="12.00">rescue (via API)</text>
</g> </g>
<!-- unrescue failed&#45;&gt;unrescuing --> <!-- unrescue failed&#45;&gt;unrescuing -->
<g id="edge61" class="edge"><title>unrescue failed&#45;&gt;unrescuing</title> <g id="edge62" class="edge"><title>unrescue failed&#45;&gt;unrescuing</title>
<path fill="none" stroke="black" d="M2154.15,-434.38C2121.8,-428.83 2076.52,-424.913 2038.53,-436 2027.58,-439.195 2016.73,-445.154 2007.43,-451.37"/> <path fill="none" stroke="black" d="M2154.85,-450.106C2122.68,-444.096 2077.31,-439.13 2038.53,-448 2029.91,-449.971 2021.08,-453.396 2013,-457.187"/>
<polygon fill="black" stroke="black" points="2005.16,-448.687 1999.03,-457.328 2009.21,-454.396 2005.16,-448.687"/> <polygon fill="black" stroke="black" points="2011.28,-454.134 2003.9,-461.743 2014.41,-460.393 2011.28,-454.134"/>
<text text-anchor="middle" x="2083.53" y="-439.4" font-family="Times,serif" font-size="12.00">unrescue (via API)</text> <text text-anchor="middle" x="2083.53" y="-451.4" font-family="Times,serif" font-size="12.00">unrescue (via API)</text>
</g> </g>
</g> </g>
</svg> </svg>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 39 KiB

After

Width:  |  Height:  |  Size: 39 KiB

100
ironic/api/controllers/v1/node.py
View File

@ -555,6 +555,54 @@ class NodeStatesController(rest.RestController):
url_args = '/'.join([node_ident, 'states']) url_args = '/'.join([node_ident, 'states'])
pecan.response.location = link.build_url('nodes', url_args) pecan.response.location = link.build_url('nodes', url_args)
def _do_provision_action(self, rpc_node, target, configdrive=None,
clean_steps=None, rescue_password=None):
topic = pecan.request.rpcapi.get_topic_for(rpc_node)
# Note that there is a race condition. The node state(s) could change
# by the time the RPC call is made and the TaskManager manager gets a
# lock.
if target in (ir_states.ACTIVE, ir_states.REBUILD):
rebuild = (target == ir_states.REBUILD)
pecan.request.rpcapi.do_node_deploy(context=pecan.request.context,
node_id=rpc_node.uuid,
rebuild=rebuild,
configdrive=configdrive,
topic=topic)
elif (target == ir_states.VERBS['unrescue']):
pecan.request.rpcapi.do_node_unrescue(
pecan.request.context, rpc_node.uuid, topic)
elif (target == ir_states.VERBS['rescue']):
if not (rescue_password and rescue_password.strip()):
msg = (_('A non-empty "rescue_password" is required when '
'setting target provision state to %s') %
ir_states.VERBS['rescue'])
raise wsme.exc.ClientSideError(
msg, status_code=http_client.BAD_REQUEST)
pecan.request.rpcapi.do_node_rescue(
pecan.request.context, rpc_node.uuid, rescue_password, topic)
elif target == ir_states.DELETED:
pecan.request.rpcapi.do_node_tear_down(
pecan.request.context, rpc_node.uuid, topic)
elif target == ir_states.VERBS['inspect']:
pecan.request.rpcapi.inspect_hardware(
pecan.request.context, rpc_node.uuid, topic=topic)
elif target == ir_states.VERBS['clean']:
if not clean_steps:
msg = (_('"clean_steps" is required when setting target '
'provision state to %s') % ir_states.VERBS['clean'])
raise wsme.exc.ClientSideError(
msg, status_code=http_client.BAD_REQUEST)
_check_clean_steps(clean_steps)
pecan.request.rpcapi.do_node_clean(
pecan.request.context, rpc_node.uuid, clean_steps, topic)
elif target in PROVISION_ACTION_STATES:
pecan.request.rpcapi.do_provisioning_action(
pecan.request.context, rpc_node.uuid, target, topic)
else:
msg = (_('The requested action "%(action)s" could not be '
'understood.') % {'action': target})
raise exception.InvalidStateRequested(message=msg)
@METRICS.timer('NodeStatesController.provision') @METRICS.timer('NodeStatesController.provision')
@expose.expose(None, types.uuid_or_name, wtypes.text, @expose.expose(None, types.uuid_or_name, wtypes.text,
wtypes.text, types.jsontype, wtypes.text, wtypes.text, types.jsontype, wtypes.text,
@ -614,7 +662,6 @@ class NodeStatesController(rest.RestController):
api_utils.check_allow_management_verbs(target) api_utils.check_allow_management_verbs(target)
rpc_node = api_utils.get_rpc_node(node_ident) rpc_node = api_utils.get_rpc_node(node_ident)
topic = pecan.request.rpcapi.get_topic_for(rpc_node)
if (target in (ir_states.ACTIVE, ir_states.REBUILD) if (target in (ir_states.ACTIVE, ir_states.REBUILD)
and rpc_node.maintenance): and rpc_node.maintenance):
@ -655,50 +702,13 @@ class NodeStatesController(rest.RestController):
raise wsme.exc.ClientSideError( raise wsme.exc.ClientSideError(
msg, status_code=http_client.BAD_REQUEST) msg, status_code=http_client.BAD_REQUEST)
# Note that there is a race condition. The node state(s) could change if (rpc_node.provision_state == ir_states.INSPECTWAIT and
# by the time the RPC call is made and the TaskManager manager gets a target == ir_states.VERBS['abort']):
# lock. if not api_utils.allow_inspect_abort():
if target in (ir_states.ACTIVE, ir_states.REBUILD): raise exception.NotAcceptable()
rebuild = (target == ir_states.REBUILD)
pecan.request.rpcapi.do_node_deploy(context=pecan.request.context, self._do_provision_action(rpc_node, target, configdrive, clean_steps,
node_id=rpc_node.uuid, rescue_password)
rebuild=rebuild,
configdrive=configdrive,
topic=topic)
elif (target == ir_states.VERBS['unrescue']):
pecan.request.rpcapi.do_node_unrescue(
pecan.request.context, rpc_node.uuid, topic)
elif (target == ir_states.VERBS['rescue']):
if not (rescue_password and rescue_password.strip()):
msg = (_('A non-empty "rescue_password" is required when '
'setting target provision state to %s') %
ir_states.VERBS['rescue'])
raise wsme.exc.ClientSideError(
msg, status_code=http_client.BAD_REQUEST)
pecan.request.rpcapi.do_node_rescue(
pecan.request.context, rpc_node.uuid, rescue_password, topic)
elif target == ir_states.DELETED:
pecan.request.rpcapi.do_node_tear_down(
pecan.request.context, rpc_node.uuid, topic)
elif target == ir_states.VERBS['inspect']:
pecan.request.rpcapi.inspect_hardware(
pecan.request.context, rpc_node.uuid, topic=topic)
elif target == ir_states.VERBS['clean']:
if not clean_steps:
msg = (_('"clean_steps" is required when setting target '
'provision state to %s') % ir_states.VERBS['clean'])
raise wsme.exc.ClientSideError(
msg, status_code=http_client.BAD_REQUEST)
_check_clean_steps(clean_steps)
pecan.request.rpcapi.do_node_clean(
pecan.request.context, rpc_node.uuid, clean_steps, topic)
elif target in PROVISION_ACTION_STATES:
pecan.request.rpcapi.do_provisioning_action(
pecan.request.context, rpc_node.uuid, target, topic)
else:
msg = (_('The requested action "%(action)s" could not be '
'understood.') % {'action': target})
raise exception.InvalidStateRequested(message=msg)
# Set the HTTP Location Header # Set the HTTP Location Header
url_args = '/'.join([node_ident, 'states']) url_args = '/'.join([node_ident, 'states'])

8
ironic/api/controllers/v1/utils.py
View File

@ -740,6 +740,14 @@ def allow_inspect_wait_state():
return pecan.request.version.minor >= versions.MINOR_39_INSPECT_WAIT return pecan.request.version.minor >= versions.MINOR_39_INSPECT_WAIT
def allow_inspect_abort():
"""Check if inspection abort is allowed.
Version 1.41 of the API added support for inspection abort
"""
return pecan.request.version.minor >= versions.MINOR_41_INSPECTION_ABORT
def handle_post_port_like_extra_vif(p_dict): def handle_post_port_like_extra_vif(p_dict):
"""Handle a Post request that sets .extra['vif_port_id']. """Handle a Post request that sets .extra['vif_port_id'].

4
ironic/api/controllers/v1/versions.py
View File

@ -78,6 +78,7 @@ BASE_VERSION = 1
# v1.39: Add inspect wait provision state. # v1.39: Add inspect wait provision state.
# v1.40: Add bios.properties. # v1.40: Add bios.properties.
# Add bios_interface to the node object. # Add bios_interface to the node object.
# v1.41: Add inspection abort support.
MINOR_0_JUNO = 0 MINOR_0_JUNO = 0
MINOR_1_INITIAL_VERSION = 1 MINOR_1_INITIAL_VERSION = 1
@ -120,6 +121,7 @@ MINOR_37_NODE_TRAITS = 37
MINOR_38_RESCUE_INTERFACE = 38 MINOR_38_RESCUE_INTERFACE = 38
MINOR_39_INSPECT_WAIT = 39 MINOR_39_INSPECT_WAIT = 39
MINOR_40_BIOS_INTERFACE = 40 MINOR_40_BIOS_INTERFACE = 40
MINOR_41_INSPECTION_ABORT = 41
# When adding another version, update: # When adding another version, update:
# - MINOR_MAX_VERSION # - MINOR_MAX_VERSION
@ -127,7 +129,7 @@ MINOR_40_BIOS_INTERFACE = 40
# explanation of what changed in the new version # explanation of what changed in the new version
# - common/release_mappings.py, RELEASE_MAPPING['master']['api'] # - common/release_mappings.py, RELEASE_MAPPING['master']['api']
MINOR_MAX_VERSION = MINOR_40_BIOS_INTERFACE MINOR_MAX_VERSION = MINOR_41_INSPECTION_ABORT
# String representations of the minor and maximum versions # String representations of the minor and maximum versions
_MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_1_INITIAL_VERSION) _MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_1_INITIAL_VERSION)

2
ironic/common/release_mappings.py
View File

@ -100,7 +100,7 @@ RELEASE_MAPPING = {
} }
}, },
'master': { 'master': {
'api': '1.40', 'api': '1.41',
'rpc': '1.44', 'rpc': '1.44',
'objects': { 'objects': {
'Node': ['1.25'], 'Node': ['1.25'],

3
ironic/common/states.py
View File

@ -413,6 +413,9 @@ machine.add_transition(INSPECTWAIT, MANAGEABLE, 'done')
# Inspection failed. # Inspection failed.
machine.add_transition(INSPECTWAIT, INSPECTFAIL, 'fail') machine.add_transition(INSPECTWAIT, INSPECTFAIL, 'fail')
# Inspection is aborted.
machine.add_transition(INSPECTWAIT, INSPECTFAIL, 'abort')
# Move the node to manageable state for any other # Move the node to manageable state for any other
# action. # action.
machine.add_transition(INSPECTFAIL, MANAGEABLE, 'manage') machine.add_transition(INSPECTFAIL, MANAGEABLE, 'manage')

126
ironic/conductor/manager.py
View File

@ -1396,7 +1396,8 @@ class ConductorManager(base_manager.BaseConductorManager):
@messaging.expected_exceptions(exception.NoFreeConductorWorker, @messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeLocked, exception.NodeLocked,
exception.InvalidParameterValue, exception.InvalidParameterValue,
exception.InvalidStateRequested) exception.InvalidStateRequested,
exception.UnsupportedDriverExtension)
def do_provisioning_action(self, context, node_id, action): def do_provisioning_action(self, context, node_id, action):
"""RPC method to initiate certain provisioning state transitions. """RPC method to initiate certain provisioning state transitions.
@ -1443,52 +1444,11 @@ class ConductorManager(base_manager.BaseConductorManager):
err_handler=utils.provisioning_error_handler) err_handler=utils.provisioning_error_handler)
return return
if (action == states.VERBS['abort'] if (action == states.VERBS['abort'] and
and node.provision_state == states.CLEANWAIT): node.provision_state in (states.CLEANWAIT,
states.RESCUEWAIT,
# Check if the clean step is abortable; if so abort it. states.INSPECTWAIT)):
# Otherwise, indicate in that clean step, that cleaning self._do_abort(task)
# should be aborted after that step is done.
if (node.clean_step
and not node.clean_step.get('abortable')):
LOG.info('The current clean step "%(clean_step)s" for '
'node %(node)s is not abortable. Adding a '
'flag to abort the cleaning after the clean '
'step is completed.',
{'clean_step': node.clean_step['step'],
'node': node.uuid})
clean_step = node.clean_step
if not clean_step.get('abort_after'):
clean_step['abort_after'] = True
node.clean_step = clean_step
node.save()
return
LOG.debug('Aborting the cleaning operation during clean step '
'"%(step)s" for node %(node)s in provision state '
'"%(prov)s".',
{'node': node.uuid,
'prov': node.provision_state,
'step': node.clean_step.get('step')})
target_state = None
if node.target_provision_state == states.MANAGEABLE:
target_state = states.MANAGEABLE
task.process_event(
'abort',
callback=self._spawn_worker,
call_args=(self._do_node_clean_abort, task),
err_handler=utils.provisioning_error_handler,
target_state=target_state)
return
if (action == states.VERBS['abort']
and node.provision_state == states.RESCUEWAIT):
utils.remove_node_rescue_password(node, save=True)
task.process_event(
'abort',
callback=self._spawn_worker,
call_args=(self._do_node_rescue_abort, task),
err_handler=utils.provisioning_error_handler)
return return
try: try:
@ -1498,6 +1458,78 @@ class ConductorManager(base_manager.BaseConductorManager):
action=action, node=node.uuid, action=action, node=node.uuid,
state=node.provision_state) state=node.provision_state)
def _do_abort(self, task):
"""Handle node abort for certain states."""
node = task.node
if node.provision_state == states.CLEANWAIT:
# Check if the clean step is abortable; if so abort it.
# Otherwise, indicate in that clean step, that cleaning
# should be aborted after that step is done.
if (node.clean_step and not
node.clean_step.get('abortable')):
LOG.info('The current clean step "%(clean_step)s" for '
'node %(node)s is not abortable. Adding a '
'flag to abort the cleaning after the clean '
'step is completed.',
{'clean_step': node.clean_step['step'],
'node': node.uuid})
clean_step = node.clean_step
if not clean_step.get('abort_after'):
clean_step['abort_after'] = True
node.clean_step = clean_step
node.save()
return
LOG.debug('Aborting the cleaning operation during clean step '
'"%(step)s" for node %(node)s in provision state '
'"%(prov)s".',
{'node': node.uuid,
'prov': node.provision_state,
'step': node.clean_step.get('step')})
target_state = None
if node.target_provision_state == states.MANAGEABLE:
target_state = states.MANAGEABLE
task.process_event(
'abort',
callback=self._spawn_worker,
call_args=(self._do_node_clean_abort, task),
err_handler=utils.provisioning_error_handler,
target_state=target_state)
return
if node.provision_state == states.RESCUEWAIT:
utils.remove_node_rescue_password(node, save=True)
task.process_event(
'abort',
callback=self._spawn_worker,
call_args=(self._do_node_rescue_abort, task),
err_handler=utils.provisioning_error_handler)
return
if node.provision_state == states.INSPECTWAIT:
try:
task.driver.inspect.abort(task)
except exception.UnsupportedDriverExtension:
with excutils.save_and_reraise_exception():
intf_name = task.driver.inspect.__class__.__name__
LOG.error('Inspect interface %(intf)s does not '
'support abort operation when aborting '
'inspection of node %(node)s',
{'intf': intf_name, 'node': node.uuid})
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.exception('Error in aborting the inspection of '
'node %(node)s', {'node': node.uuid})
node.last_error = _('Failed to abort inspection. '
'Error: %s') % e
node.save()
node.last_error = _('Inspection was aborted by request.')
task.process_event('abort')
LOG.info('Successfully aborted inspection of node %(node)s',
{'node': node.uuid})
return
@METRICS.timer('ConductorManager._sync_power_states') @METRICS.timer('ConductorManager._sync_power_states')
@periodics.periodic(spacing=CONF.conductor.sync_power_state_interval) @periodics.periodic(spacing=CONF.conductor.sync_power_state_interval)
def _sync_power_states(self, context): def _sync_power_states(self, context):

16
ironic/drivers/base.py
View File

@ -923,6 +923,22 @@ class InspectInterface(BaseInterface):
or None. or None.
""" """
def abort(self, task):
"""Abort asynchronized hardware inspection.
Abort an ongoing hardware introspection, this is only used for
asynchronize based inspect interface.
NOTE: This interface is called with node exclusive lock held, the
interface implementation is expected to be a quick processing.
:param task: a task from TaskManager.
:raises: UnsupportedDriverExtension, if the method is not implemented
by specific inspect interface.
"""
raise exception.UnsupportedDriverExtension(
driver=task.node.driver, extension='abort')
class BIOSInterface(BaseInterface): class BIOSInterface(BaseInterface):
interface_type = 'bios' interface_type = 'bios'

12
ironic/drivers/modules/inspector.py
View File

@ -34,7 +34,7 @@ LOG = logging.getLogger(__name__)
client = importutils.try_import('ironic_inspector_client') client = importutils.try_import('ironic_inspector_client')
INSPECTOR_API_VERSION = (1, 0) INSPECTOR_API_VERSION = (1, 3)
_INSPECTOR_SESSION = None _INSPECTOR_SESSION = None
@ -130,6 +130,16 @@ class Inspector(base.InspectInterface):
eventlet.spawn_n(_start_inspection, task.node.uuid, task.context) eventlet.spawn_n(_start_inspection, task.node.uuid, task.context)
return states.INSPECTWAIT return states.INSPECTWAIT
def abort(self, task):
"""Abort hardware inspection.
:param task: a task from TaskManager.
"""
node_uuid = task.node.uuid
LOG.debug('Aborting inspection for node %(uuid)s using '
'ironic-inspector', {'uuid': node_uuid})
_get_client(task.context).abort(node_uuid)
@periodics.periodic(spacing=CONF.inspector.status_check_period, @periodics.periodic(spacing=CONF.inspector.status_check_period,
enabled=CONF.inspector.enabled) enabled=CONF.inspector.enabled)
def _periodic_check_result(self, manager, context): def _periodic_check_result(self, manager, context):

18
ironic/tests/unit/api/controllers/v1/test_node.py
View File

@ -4128,6 +4128,24 @@ class TestPut(test_api_base.BaseApiTest):
obj_fields.NotificationLevel.ERROR, obj_fields.NotificationLevel.ERROR,
obj_fields.NotificationStatus.ERROR)]) obj_fields.NotificationStatus.ERROR)])
def test_inspect_abort_raises_before_1_41(self):
self.node.provision_state = states.INSPECTWAIT
self.node.save()
ret = self.put_json('/nodes/%s/states/provision' % self.node.uuid,
{'target': states.VERBS['abort']},
headers={api_base.Version.string: "1.40"},
expect_errors=True)
self.assertEqual(http_client.NOT_ACCEPTABLE, ret.status_code)
@mock.patch.object(rpcapi.ConductorAPI, 'do_provisioning_action')
def test_inspect_abort_accepted_after_1_41(self, mock_provision):
self.node.provision_state = states.INSPECTWAIT
self.node.save()
ret = self.put_json('/nodes/%s/states/provision' % self.node.uuid,
{'target': states.VERBS['abort']},
headers={api_base.Version.string: "1.41"})
self.assertEqual(http_client.ACCEPTED, ret.status_code)
class TestCheckCleanSteps(base.TestCase): class TestCheckCleanSteps(base.TestCase):
def test__check_clean_steps_not_list(self): def test__check_clean_steps_not_list(self):

7
ironic/tests/unit/api/controllers/v1/test_utils.py
View File

@ -514,6 +514,13 @@ class TestApiUtils(base.TestCase):
mock_request.version.minor = 37 mock_request.version.minor = 37
self.assertFalse(utils.allow_rescue_interface()) self.assertFalse(utils.allow_rescue_interface())
@mock.patch.object(pecan, 'request', spec_set=['version'])
def test_allow_inspect_abort(self, mock_request):
mock_request.version.minor = 41
self.assertTrue(utils.allow_inspect_abort())
mock_request.version.minor = 40
self.assertFalse(utils.allow_inspect_abort())
class TestNodeIdent(base.TestCase): class TestNodeIdent(base.TestCase):

61
ironic/tests/unit/conductor/test_manager.py
View File

@ -7580,3 +7580,64 @@ class NodeTraitsTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
def test_remove_node_traits_node_trait_not_found(self): def test_remove_node_traits_node_trait_not_found(self):
self._test_remove_node_traits_exception(exception.NodeTraitNotFound) self._test_remove_node_traits_exception(exception.NodeTraitNotFound)
@mgr_utils.mock_record_keepalive
class DoNodeInspectAbortTestCase(mgr_utils.CommonMixIn,
mgr_utils.ServiceSetUpMixin,
db_base.DbTestCase):
@mock.patch.object(manager, 'LOG')
@mock.patch('ironic.drivers.modules.fake.FakeInspect.abort')
@mock.patch('ironic.conductor.task_manager.acquire', autospec=True)
def test_do_inspect_abort_interface_not_support(self, mock_acquire,
mock_abort, mock_log):
node = obj_utils.create_test_node(self.context,
driver='fake-hardware',
provision_state=states.INSPECTWAIT)
task = task_manager.TaskManager(self.context, node.uuid)
mock_acquire.side_effect = self._get_acquire_side_effect(task)
mock_abort.side_effect = exception.UnsupportedDriverExtension(
driver='fake', extension='inspect')
self._start_service()
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_provisioning_action,
self.context, task.node.uuid,
"abort")
self.assertEqual(exception.UnsupportedDriverExtension,
exc.exc_info[0])
self.assertTrue(mock_log.error.called)
@mock.patch.object(manager, 'LOG')
@mock.patch('ironic.drivers.modules.fake.FakeInspect.abort')
@mock.patch('ironic.conductor.task_manager.acquire', autospec=True)
def test_do_inspect_abort_interface_return_failed(self, mock_acquire,
mock_abort, mock_log):
mock_abort.side_effect = exception.IronicException('Oops')
self._start_service()
node = obj_utils.create_test_node(self.context,
driver='fake-hardware',
provision_state=states.INSPECTWAIT)
task = task_manager.TaskManager(self.context, node.uuid)
mock_acquire.side_effect = self._get_acquire_side_effect(task)
self.assertRaises(exception.IronicException,
self.service.do_provisioning_action,
self.context, task.node.uuid,
"abort")
node.refresh()
self.assertTrue(mock_log.exception.called)
self.assertIn('Failed to abort inspection.', node.last_error)
@mock.patch('ironic.drivers.modules.fake.FakeInspect.abort')
@mock.patch('ironic.conductor.task_manager.acquire', autospec=True)
def test_do_inspect_abort_succeeded(self, mock_acquire, mock_abort):
self._start_service()
node = obj_utils.create_test_node(self.context,
driver='fake-hardware',
provision_state=states.INSPECTWAIT)
task = task_manager.TaskManager(self.context, node.uuid)
mock_acquire.side_effect = self._get_acquire_side_effect(task)
self.service.do_provisioning_action(self.context, task.node.uuid,
"abort")
node.refresh()
self.assertEqual('inspect failed', node.provision_state)
self.assertIn('Inspection was aborted', node.last_error)

16
ironic/tests/unit/drivers/modules/test_inspector.py
View File

@ -66,7 +66,7 @@ class GetClientTestCase(db_base.DbTestCase):
super(GetClientTestCase, self).setUp() super(GetClientTestCase, self).setUp()
# NOTE(pas-ha) force-reset global inspector session object # NOTE(pas-ha) force-reset global inspector session object
inspector._INSPECTOR_SESSION = None inspector._INSPECTOR_SESSION = None
self.api_version = (1, 0) self.api_version = (1, 3)
self.context = context.RequestContext(global_request_id='global') self.context = context.RequestContext(global_request_id='global')
def test__get_client(self, mock_init, mock_session, mock_auth, def test__get_client(self, mock_init, mock_session, mock_auth,
@ -216,3 +216,17 @@ class CheckStatusTestCase(BaseTestCase):
mock_get.assert_called_once_with(self.node.uuid) mock_get.assert_called_once_with(self.node.uuid)
self.task.process_event.assert_called_once_with('fail') self.task.process_event.assert_called_once_with('fail')
self.assertIn('boom', self.node.last_error) self.assertIn('boom', self.node.last_error)
@mock.patch('ironic.drivers.modules.inspector._get_client', autospec=True)
class InspectHardwareAbortTestCase(BaseTestCase):
def test_abort_ok(self, mock_client):
mock_abort = mock_client.return_value.abort
self.driver.inspect.abort(self.task)
mock_abort.assert_called_once_with(self.node.uuid)
def test_abort_error(self, mock_client):
mock_abort = mock_client.return_value.abort
mock_abort.side_effect = RuntimeError('boom')
self.assertRaises(RuntimeError, self.driver.inspect.abort, self.task)
mock_abort.assert_called_once_with(self.node.uuid)

8
releasenotes/notes/add-inspection-abort-a187e6e5c1f6311d.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- |
Adds support to abort the inspection of a node in the ``inspect wait``
state, as long as this operation is supported by the inspect interface in
use. Starting from API 1.41, the node in the ``inspect wait`` state
accepts ``abort`` provisioning verb to initiate the abort process.
This feature is supported by the ``inspector`` inspect interface.