From 9715d5e3e7de0e073de5f227f487dd4d69f230a3 Mon Sep 17 00:00:00 2001
From: Stan Lagun <slagun@mirantis.com>
Date: Thu, 22 Oct 2015 23:10:33 +0300
Subject: [PATCH] [Kubernetes] Bunch of issues regarding scaling was fixed

1. There was scaleGatewayUp but no Down action
2. Gracefully remove node from etcd upon down-scale of nodes/gateways
3. Log to user which gateways were created/deleted
4. Deployment either hanged or worked incorrectly when downscaling and then
     upscaling something again

Also sets minimum gateway node count to 1 until gateway-less mode be fully
functional again

Change-Id: Ie3255f70b8a3fdfadab119a579c00433938495c8
Closes-Bug: #1509074
---
 .../package/Classes/KubernetesCluster.yaml    | 20 ++++++++++--
 .../Classes/KubernetesGatewayNode.yaml        | 18 +++++++++++
 .../package/Classes/KubernetesMinionNode.yaml | 14 +++++---
 .../package/Classes/KubernetesNode.yaml       | 12 ++++++-
 .../Resources/EtcdRemoveMember.template       | 32 +++++++++++++++++++
 .../package/Resources/RemoveMinion.template   |  2 +-
 .../Resources/scripts/removeEtcdMember.sh     |  4 +++
 7 files changed, 93 insertions(+), 9 deletions(-)
 create mode 100644 Kubernetes/KubernetesCluster/package/Resources/EtcdRemoveMember.template
 create mode 100644 Kubernetes/KubernetesCluster/package/Resources/scripts/removeEtcdMember.sh

diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml
index 6213e2f..39a066a 100644
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml
@@ -39,12 +39,13 @@ Properties:
     Usage: InOut
 
   gatewayCount:
-    Contract: $.int().notNull()
+    Contract: $.int().notNull().check($ > 0)
     Usage: InOut
 
   gatewayNodes:
     Contract:
       - $.class(KubernetesGatewayNode).notNull()
+      - 1
 
   dockerRegistry:
     Contract: $.string()
@@ -119,13 +120,16 @@ Methods:
               - $.gatewayNodes.take($.gatewayCount).pselect($.deployInstance())
 
           - $.masterNode.setupEtcd()
+          - $.minionNodes.skip($.nodeCount).select($.removeFromCluster())
+          - $.gatewayNodes.skip($.gatewayCount).select($.removeFromCluster())
+
           - $.minionNodes.take($.nodeCount).select($.setupEtcd())
           - $.gatewayNodes.take($.gatewayCount).select($.setupEtcd())
 
           - $.masterNode.setupNode()
+
           - Parallel:
               - $.minionNodes.take($.nodeCount).pselect($.setupNode())
-              - $.minionNodes.skip($.nodeCount).pselect($.removeFromCluster())
               - $.gatewayNodes.take($.gatewayCount).pselect($.setupNode())
 
           - $._environment.stack.push()
@@ -487,6 +491,18 @@ Methods:
           - $._environment.reporter.report($this, 'The maximum number of nodes has been reached')
 
 
+  scaleGatewaysDown:
+    Usage: Action
+    Body:
+      - If: $.gatewayCount > 1
+        Then:
+          - $._environment.reporter.report($this, 'Removing gateway node')
+          - $.gatewayCount: $.gatewayCount - 1
+          - $.deploy()
+        Else:
+          - $._environment.reporter.report($this, 'No gateway nodes that can be removed')
+
+
   exportConfig:
     Usage: Action
     Body:
diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesGatewayNode.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesGatewayNode.yaml
index 6a296ec..7046f9a 100644
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesGatewayNode.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesGatewayNode.yaml
@@ -69,5 +69,23 @@ Methods:
                 masterIp => $._cluster.masterNode.getIp()
               ))
           - $.instance.agent.call($template, $resources)
+          - $._environment.reporter.report(
+                $this, 'Gateway {0} is now available at {1}'.format($.instance.name, $.getIp(true)))
           - $.setAttr(nodeConfigured, true)
 
+  removeFromCluster:
+    Body:
+      - If: $.getAttr(nodeConfigured, false)
+        Then:
+          - $._environment.reporter.report($this, 'Deleting Gateway Node')
+          - $resources: new(sys:Resources)
+          - $template: $resources.yaml('EtcdRemoveMember.template').bind(dict(
+                nodeId => $.getIp()
+              ))
+          - $.instance.agent.call($template, $resources)
+          - $.instance.releaseResources()
+          - $._environment.reporter.report($this, 'Gateway {0} at {1} was deleted'.format($.instance.name, $.getIp(true)))
+          - $.setAttr(nodeConfigured, false)
+          - $.setAttr(etcdConfigured, false)
+          - $.setAttr(instanceDeployed, false)
+
diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml
index 2f14d77..33d23f7 100644
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml
@@ -89,10 +89,7 @@ Methods:
           - $.instance.agent.call($template, $resources)
           - $.setAttr(nodeConfigured, true)
           - $msg: 'cAdvisor monitoring for Node {0} is now available at http://{1}:4194'
-          - $ip: $.getIp()
-          - If: $.exposeCAdvisor
-            Then:
-              - $ip: coalesce($.instance.floatingIpAddress, $.getIp())
+          - $ip: $.getIp($.exposeCAdvisor)
           - $._environment.reporter.report($this, $msg.format($.instance.name, $ip))
 
 
@@ -102,10 +99,17 @@ Methods:
         Then:
           - $._environment.reporter.report($this, 'Deleting Kubernetes Minion')
           - $resources: new(sys:Resources)
+          - $template: $resources.yaml('EtcdRemoveMember.template').bind(dict(
+                nodeId => $.getIp()
+              ))
+          - $.instance.agent.call($template, $resources)
           - $template: $resources.yaml('RemoveMinion.template').bind(dict(
                 nodeId => $.getIp()
               ))
           - $._cluster.masterNode.instance.agent.call($template, $resources)
           - $.instance.releaseResources()
-          - $._environment.reporter.report($this, 'Node {0} deleted'.format($.instance.name))
+          - $._environment.reporter.report($this, 'Node {0} was deleted'.format($.instance.name))
           - $.setAttr(nodeConfigured, false)
+          - $.setAttr(etcdConfigured, false)
+          - $.setAttr(instanceDeployed, false)
+
diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesNode.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesNode.yaml
index e003d41..f2f050b 100644
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesNode.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesNode.yaml
@@ -26,8 +26,17 @@ Properties:
 
 Methods:
   getIp:
+    Arguments:
+      - preferFloatingIp:
+          Contract: $.bool().notNull()
+          Default: false
     Body:
-      Return: $.instance.ipAddresses[0]
+      - $ip: $.instance.ipAddresses[0]
+      - If: $preferFloatingIp
+        Then:
+          - Return: coalesce($.instance.floatingIpAddress, $ip)
+        Else:
+          - Return: $.instance.ipAddresses[0]
 
 
   deployInstance:
@@ -37,3 +46,4 @@ Methods:
 
   setupEtcd:
   setupNode:
+  removeFromCluster:
diff --git a/Kubernetes/KubernetesCluster/package/Resources/EtcdRemoveMember.template b/Kubernetes/KubernetesCluster/package/Resources/EtcdRemoveMember.template
new file mode 100644
index 0000000..d6cebf1
--- /dev/null
+++ b/Kubernetes/KubernetesCluster/package/Resources/EtcdRemoveMember.template
@@ -0,0 +1,32 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+FormatVersion: 2.0.0
+Version: 1.0.0
+Name: Remove etcd member
+
+Parameters:
+  nodeId: $nodeId
+
+Body: |
+  removeMember(args.nodeId)
+
+Scripts:
+  removeMember:
+    Type: Application
+    Version: 1.0.0
+    EntryPoint: removeEtcdMember.sh
+    Files: []
+    Options:
+      captureStdout: false
+      captureStderr: false
+      verifyExitcode: false
diff --git a/Kubernetes/KubernetesCluster/package/Resources/RemoveMinion.template b/Kubernetes/KubernetesCluster/package/Resources/RemoveMinion.template
index c68f0a5..8b157b6 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/RemoveMinion.template
+++ b/Kubernetes/KubernetesCluster/package/Resources/RemoveMinion.template
@@ -18,7 +18,7 @@ Parameters:
   nodeId: $nodeId
 
 Body: |
-  return removeMinion('{0}'.format(args.nodeId)).stdout
+  removeMinion(args.nodeId)
 
 Scripts:
   removeMinion:
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/removeEtcdMember.sh b/Kubernetes/KubernetesCluster/package/Resources/scripts/removeEtcdMember.sh
new file mode 100644
index 0000000..b453cc5
--- /dev/null
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/removeEtcdMember.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+NODE_ID=$(/opt/bin/etcdctl member list | grep $1 | cut -d':' -f1)
+/opt/bin/etcdctl member remove $NODE_ID