Update static nodes in Zookeeper on config change

Config changes of existing nodes were not updated in Zookeeper, leading
to inconsistent states.

Changing a label or connection related attributes is now reflected
without the workaround of first removing and re-adding the node.

Change-Id: Ia6278512c2e81a720a957fe269d1ae0522860b8f

nodepool/driver/static/provider.py

@@ -122,6 +122,52 @@ class StaticNodeProvider(Provider):
             self.zk.storeNode(node)
             self.log.debug("Registered static node %s", node.hostname)
 
+    def updateNodeFromConfig(self, static_node):
+        '''
+        Update a static node in ZooKeeper according to config.
+
+        The node is only updated if one of the relevant config items
+        changed. Name changes of nodes are handled via the
+        register/deregister flow.
+
+        :param dict static_node: The node definition from the config file.
+        '''
+        host_keys = self.checkHost(static_node)
+        nodes = self.getRegisteredReadyNodes(static_node["name"])
+        new_attrs = (
+            static_node["labels"],
+            static_node["username"],
+            static_node["connection-port"],
+            static_node["connection-type"],
+            host_keys,
+        )
+
+        for node in nodes:
+            original_attrs = (node.type, node.username, node.connection_port,
+                              node.connection_type, node.host_keys)
+
+            if original_attrs == new_attrs:
+                continue
+
+            node.type = static_node["labels"]
+            node.username = static_node["username"]
+            node.connection_port = static_node["connection-port"]
+            node.connection_type = static_node["connection-type"]
+            nodeutils.set_node_ip(node)
+            node.host_keys = host_keys
+
+            try:
+                self.zk.lockNode(node, blocking=False)
+            except exceptions.ZKLockException:
+                self.log.warning("Unable to lock node %s for update", node.id)
+                continue
+
+            try:
+                self.zk.storeNode(node)
+                self.log.debug("Updated static node %s", node.hostname)
+            finally:
+                self.zk.unlockNode(node)
+
     def deregisterNode(self, count, node_name):
         '''
         Attempt to delete READY nodes.
@@ -192,6 +238,12 @@ class StaticNodeProvider(Provider):
                         self.log.exception("Couldn't deregister static node:")
                         continue
 
+                try:
+                    self.updateNodeFromConfig(node)
+                except Exception:
+                    self.log.exception("Couldn't update static node:")
+                    continue
+
                 self.static_nodes[node["name"]] = node
 
         # De-register nodes to synchronize with our configuration.

nodepool/tests/fixtures/static-update.yaml

@@ -0,0 +1,23 @@
+zookeeper-servers:
+  - host: {zookeeper_host}
+    port: {zookeeper_port}
+    chroot: {zookeeper_chroot}
+
+labels:
+  - name: fake-label
+  - name: fake-label2
+
+providers:
+  - name: static-provider
+    driver: static
+    pools:
+      - name: main
+        nodes:
+          - name: fake-host-1
+            labels:
+              - fake-label
+              - fake-label2
+            timeout: 13
+            connection-type: winrm
+            connection-port: 5986
+            username: admin

nodepool/tests/test_driver_static.py

@@ -155,6 +155,29 @@ class TestDriverStatic(tests.DBTestCase):
         nodes = self.waitForNodes('fake-label')
         self.assertEqual(len(nodes), 1)
 
+    def test_static_node_update(self):
+        '''
+        Test that updates a static node on config change.
+        '''
+        configfile = self.setup_config('static-basic.yaml')
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        pool.start()
+
+        self.log.debug("Waiting for initial node")
+        nodes = self.waitForNodes('fake-label')
+        self.assertEqual(len(nodes), 1)
+
+        self.log.debug("Waiting for new label")
+        self.replace_config(configfile, 'static-update.yaml')
+        nodes = self.waitForNodes('fake-label2')
+        self.assertEqual(len(nodes), 1)
+        self.assertIn('fake-label', nodes[0].type)
+        self.assertIn('fake-label2', nodes[0].type)
+        self.assertEqual(nodes[0].username, 'admin')
+        self.assertEqual(nodes[0].connection_port, 5986)
+        self.assertEqual(nodes[0].connection_type, 'winrm')
+        self.assertEqual(nodes[0].host_keys, None)
+
     def test_static_multilabel(self):
         configfile = self.setup_config('static-multilabel.yaml')
         pool = self.useNodepool(configfile, watermark_sleep=1)

releasenotes/notes/static-driver-node-update-02ee5536737b20b8.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    The static driver now updates labels and connection related attributes
+    in Zookeeper at startup and on config change. Changing the name of a node
+    will be handled via the registration/deregistration flow as before.