do not rekey when vault server pods need upgrade

Changes in the upgrade procedure cause vault server pods to require
restart in order to update to new server version.  The work for restart
pods is performed in another commit.

Defer a request for vault rekey until the server pods match the expected
version.  The rekey procedure will not proceed if vault pods are being
restarted, and so we should not start a rekey when it is anticipated
that vault pods will be restarted.

Test Plan:
PASS  bashate
PASS  unit test
PASS  vault sanity master branch, rekey
PASS  simplex upgrade (manual server pod restart)
PASS  duplex 2+1 (vault ha, 3 replicas) application-update

Story: 2011073
Task: 50814

Change-Id: I91334d0577148c1e3f7bc674ab2a3edfaced1d1c
Signed-off-by: Michel Thebeau <Michel.Thebeau@windriver.com>

helm-charts/custom/vault-manager-helm/vault-manager-helm/vault-manager/templates/vault-init.yaml

@@ -13,6 +13,7 @@ data:
     VAULT_NAME={{ .Values.vault.name }}
     VAULT_FN={{ .Values.vault.fullname }}
     HA_REPLICAS={{ .Values.server.ha.replicas }}
+    VAULT_VERSION={{ .Values.server.version }}
 
     # Set the domain for resolving pod names
     DOMAIN="${VAULT_NS}.pod.cluster.local"
@@ -1993,6 +1994,40 @@ data:
         return 0
     }
 
+    # Check the vault server pods' metadata label "vault-version",
+    # and assert that all servers are running the expected version
+    # which is coded in vault-manager values.yaml server.version
+    function allServersCurrent {
+        local jdata
+        local podcount
+        local i
+        local poddata
+        local name
+        local version
+
+        jdata="$( kubectl get pods -n "$VAULT_NS" -o json )"
+        podcount="$( echo "$jdata" | jq ".items | length" )"
+
+        for i in $( seq 0 $((podcount -1 )) ); do
+            poddata="$( echo "$jdata" | jq ".items[$i]" )"
+            name="$( echo "$poddata" | jq -r ".metadata.name" )"
+            if ! [[ "$name" =~ ^${VAULT_FN}-[0-9]$ ]]; then
+                # this is not a vault server pod
+                continue
+            fi
+
+            version="$( echo "$poddata" \
+                | jq -r '.metadata.labels["vault-version"]' )"
+            if [ "$version" != "$VAULT_VERSION" ]; then
+                log $INFO "Vault server pod $name is version $version"
+                return 1
+            fi
+
+            log $DEBUG "Vault server pod $name is version $version"
+        done
+        return 0
+    }
+
     # Test the status of rekey procedure 'started' during pre-rekey
     # tests for procedure progress selection (sharing a single vaultAPI
     # call to GET /sys/rekey/init
@@ -2094,6 +2129,7 @@ data:
         local pods
         local sealed
         local response
+        local apiversion
 
         # the first milestone to be created is cluster-rekey-request;
         # the last milestone to be deleted is cluster-rekey-audit;
@@ -2131,7 +2167,15 @@ data:
             return 1
         fi
 
-        # The above three tests are based on output of kubectl get pods
+        # progress a rekey if all server pods are running the expected
+        # server version
+        if ! allServersCurrent; then
+            log $INFO "Rekey: wait for vault servers to be updated" \
+                "to the current version $VAULT_VERSION"
+            return 1
+        fi
+
+        # The above four tests are based on output of kubectl get pods
         # command. Doublecheck with REST API call to each server
         pods="$( getVaultPods | grep "^$VAULT_FN" | awk '{print $2}' )"
         for pod in $pods; do
@@ -2147,6 +2191,11 @@ data:
                 log $ERROR "$pod is sealed during rekey"
                 return 1
             fi
+            apiversion="$( echo "$response" | jq -r '.version' )"
+            if [ "$apiversion" != "$VAULT_VERSION" ]; then
+                log $ERROR "$pod is not version $VAULT_VERSION"
+                return 1
+            fi
         done
 
         assertServersConform

helm-charts/custom/vault-manager-helm/vault-manager-helm/vault-manager/values.yaml

@@ -11,6 +11,7 @@ vault:
   fullname: sva-vault
 
 server:
+  version: 1.14.0
   ha:
     replicas: 1