Require TLS

Require TLS Zookeeper connections before making the 4.0 release.

Change-Id: I69acdcec0deddfdd191f094f13627ec1618142af
Depends-On: https://review.opendev.org/776696

.gitignore

@@ -17,3 +17,4 @@ dist/
 venv/
 *~
 .*.swp
+tools/ca/

.zuul.yaml

@@ -347,6 +347,26 @@
         env-vars:
           DIB_SIMPLE_INIT_NETWORKMANAGER: '1'
 
+- job:
+    name: nodepool-tox-py36
+    description: |
+      Nodepool unit tests with ZooKeeper running
+    parent: tox-py36
+    pre-run: playbooks/nodepool-tox/pre.yaml
+    vars: &nodepool_tox_vars
+      tox_environment:
+        NODEPOOL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem
+        NODEPOOL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem
+        NODEPOOL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem
+
+- job:
+    name: nodepool-tox-py38
+    description: |
+      Nodepool unit tests with ZooKeeper running
+    parent: tox-py38
+    pre-run: playbooks/nodepool-tox/pre.yaml
+    vars: *nodepool_tox_vars
+
 - project:
     vars:
       release_python: python3
@@ -357,8 +377,8 @@
         - nodepool-build-image-siblings
         - zuul-tox-docs
         - tox-pep8
-        - tox-py36
-        - tox-py38
+        - nodepool-tox-py36
+        - nodepool-tox-py38
         - nodepool-zuul-functional:
             voting: false
         - nodepool-functional-openstack:
@@ -380,8 +400,8 @@
         - nodepool-upload-image
         - zuul-tox-docs
         - tox-pep8
-        - tox-py36
-        - tox-py38
+        - nodepool-tox-py36
+        - nodepool-tox-py38
         - nodepool-functional-openstack
         - nodepool-functional-openstack-src
         - nodepool-functional-k8s

bindep.txt

@@ -15,5 +15,6 @@ musl-dev [compile test platform:apk]
 python3-dev [compile test platform:dpkg]
 python3-devel [compile test platform:rpm]
 sudo
-zookeeperd [platform:dpkg test]
-zookeeper [platform:suse test]
+docker.io [test platform:dpkg]
+docker [test platform:fedora]
+docker-compose [test]

nodepool/config.py

@@ -101,7 +101,7 @@ class Config(ConfigValue):
 
         for server in zk_cfg:
             z = zk.ZooKeeperConnectionConfig(server['host'],
-                                             server.get('port', 2181),
+                                             server.get('port', 2281),
                                              server.get('chroot', None))
             name = z.host + '_' + str(z.port)
             self.zookeeper_servers[name] = z
@@ -358,7 +358,6 @@ def loadSecureConfig(config, secure_config_path, env=os.environ):
     if secure.get('zookeeper-servers', []):
         config.zookeeper_servers = {}
 
-    # TODO(Shrews): Support ZooKeeper auth
     config.setZooKeeperServers(secure.get('zookeeper-servers'))
     config.setSecureDiskimageEnv(
         secure.get('diskimages', []), secure_config_path)

nodepool/tests/__init__.py

@@ -61,16 +61,37 @@ class ZookeeperServerFixture(fixtures.Fixture):
         self.zookeeper_host = host
 
         if not port:
-            self.zookeeper_port = 2181
+            self.zookeeper_port = 2281
         else:
             self.zookeeper_port = int(port)
 
+        zk_ca = os.environ.get('NODEPOOL_ZK_CA', None)
+        if not zk_ca:
+            zk_ca = os.path.join(os.path.dirname(__file__),
+                                 '../../tools/ca/certs/cacert.pem')
+        self.zookeeper_ca = zk_ca
+        zk_cert = os.environ.get('NODEPOOL_ZK_CERT', None)
+        if not zk_cert:
+            zk_cert = os.path.join(os.path.dirname(__file__),
+                                   '../../tools/ca/certs/client.pem')
+        self.zookeeper_cert = zk_cert
+        zk_key = os.environ.get('NODEPOOL_ZK_KEY', None)
+        if not zk_key:
+            zk_key = os.path.join(os.path.dirname(__file__),
+                                  '../../tools/ca/keys/clientkey.pem')
+        self.zookeeper_key = zk_key
+
 
 class ChrootedKazooFixture(fixtures.Fixture):
-    def __init__(self, zookeeper_host, zookeeper_port):
+    def __init__(self, zookeeper_host, zookeeper_port, zookeeper_ca,
+                 zookeeper_cert, zookeeper_key):
         super(ChrootedKazooFixture, self).__init__()
-        self.zookeeper_host = zookeeper_host
-        self.zookeeper_port = zookeeper_port
+        self.zk_args = dict(
+            hosts='%s:%s' % (zookeeper_host, zookeeper_port),
+            use_ssl=True,
+            ca=zookeeper_ca,
+            certfile=zookeeper_cert,
+            keyfile=zookeeper_key)
 
     def _setUp(self):
         # Make sure the test chroot paths do not conflict
@@ -82,8 +103,7 @@ class ChrootedKazooFixture(fixtures.Fixture):
         self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
 
         # Ensure the chroot path exists and clean up any pre-existing znodes.
-        _tmp_client = kazoo.client.KazooClient(
-            hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
+        _tmp_client = kazoo.client.KazooClient(**self.zk_args)
         _tmp_client.start()
 
         if _tmp_client.exists(self.zookeeper_chroot):
@@ -98,8 +118,7 @@ class ChrootedKazooFixture(fixtures.Fixture):
     def _cleanup(self):
         '''Remove the chroot path.'''
         # Need a non-chroot'ed client to remove the chroot path
-        _tmp_client = kazoo.client.KazooClient(
-            hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
+        _tmp_client = kazoo.client.KazooClient(**self.zk_args)
         _tmp_client.start()
         _tmp_client.delete(self.zookeeper_chroot, recursive=True)
         _tmp_client.stop()
@@ -373,7 +392,10 @@ class DBTestCase(BaseTestCase):
                                      context_name=context_name,
                                      zookeeper_host=self.zookeeper_host,
                                      zookeeper_port=self.zookeeper_port,
-                                     zookeeper_chroot=self.zookeeper_chroot)
+                                     zookeeper_chroot=self.zookeeper_chroot,
+                                     zookeeper_ca=self.zookeeper_ca,
+                                     zookeeper_cert=self.zookeeper_cert,
+                                     zookeeper_key=self.zookeeper_key)
                 os.write(fd, data.encode('utf8'))
             os.close(fd)
         self._config_images_dir = images_dir
@@ -399,7 +421,10 @@ class DBTestCase(BaseTestCase):
             data = config.format(
                 zookeeper_host=self.zookeeper_host,
                 zookeeper_port=self.zookeeper_port,
-                zookeeper_chroot=self.zookeeper_chroot)
+                zookeeper_chroot=self.zookeeper_chroot,
+                zookeeper_ca=self.zookeeper_ca,
+                zookeeper_cert=self.zookeeper_cert,
+                zookeeper_key=self.zookeeper_key)
             os.write(fd, data.encode('utf8'))
         os.close(fd)
         return path
@@ -587,16 +612,26 @@ class DBTestCase(BaseTestCase):
         self.useFixture(f)
         self.zookeeper_host = f.zookeeper_host
         self.zookeeper_port = f.zookeeper_port
+        self.zookeeper_ca = f.zookeeper_ca
+        self.zookeeper_cert = f.zookeeper_cert
+        self.zookeeper_key = f.zookeeper_key
 
         kz_fxtr = self.useFixture(ChrootedKazooFixture(
             self.zookeeper_host,
-            self.zookeeper_port))
+            self.zookeeper_port,
+            self.zookeeper_ca,
+            self.zookeeper_cert,
+            self.zookeeper_key,
+        ))
         self.zookeeper_chroot = kz_fxtr.zookeeper_chroot
         self.zk = zk.ZooKeeper(enable_cache=False)
         host = zk.ZooKeeperConnectionConfig(
-            self.zookeeper_host, self.zookeeper_port, self.zookeeper_chroot
+            self.zookeeper_host, self.zookeeper_port, self.zookeeper_chroot,
         )
-        self.zk.connect([host])
+        self.zk.connect([host],
+                        tls_ca=self.zookeeper_ca,
+                        tls_cert=self.zookeeper_cert,
+                        tls_key=self.zookeeper_key)
         self.addCleanup(self.zk.disconnect)
 
     def printZKTree(self, node):

nodepool/tests/fixtures/azure.yaml

@@ -6,6 +6,11 @@ zookeeper-servers:
   - host: 127.0.0.1
     port: 2181
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: bionic
     min-ready: 1

nodepool/tests/fixtures/builder_2_diskimages.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label1
   - name: fake-label2

nodepool/tests/fixtures/cleanup-port.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/disabled_provider.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/diskimage_build_timeout.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels: []
 
 providers: []

nodepool/tests/fixtures/external_driver.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: test-label
     min-ready: 1

nodepool/tests/fixtures/functional/kubernetes/basic.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: kubernetes-namespace
     min-ready: 1

nodepool/tests/fixtures/functional/openshift/basic.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: openshift-project
     min-ready: 1

nodepool/tests/fixtures/functional/openshift/pods.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: openshift-pod
     min-ready: 1

nodepool/tests/fixtures/ignore_provider_quota_false.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
 

nodepool/tests/fixtures/ignore_provider_quota_true.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
 

nodepool/tests/fixtures/info_cmd_two_provider.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/info_cmd_two_provider_remove.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/integration_occ.yaml

@@ -6,6 +6,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/kubernetes.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: pod-fedora
   - name: kubernetes-namespace

nodepool/tests/fixtures/launcher_reg1.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/launcher_reg2.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/launcher_two_provider.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/launcher_two_provider_max_1.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/launcher_two_provider_remove.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/leaked_node.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/leaked_node_nodepool_id.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/multi_drivers.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: test-label
     min-ready: 1

nodepool/tests/fixtures/multiple_pools.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label1
     min-ready: 1

nodepool/tests/fixtures/multiproviders.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-static-label
     min-ready: 1

nodepool/tests/fixtures/node-host-key-checking.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node-network_cli.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_auto_floating_ip.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label1
     min-ready: 1

nodepool/tests/fixtures/node_az.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_az_change.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_boot_from_volume.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_cmd.yaml

@@ -6,6 +6,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label1
     min-ready: 1

nodepool/tests/fixtures/node_delete_error.yaml

@@ -8,6 +8,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_disabled_label.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_diskimage_fail.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_diskimage_formats.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label-default-format
     min-ready: 1

nodepool/tests/fixtures/node_diskimage_only.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels: []
 
 providers: []

nodepool/tests/fixtures/node_diskimage_parents.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-image-parent-1
     min-ready: 1

nodepool/tests/fixtures/node_diskimage_pause.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_flavor_name.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_image_upload_pause.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_ipv6.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label1
     min-ready: 1

nodepool/tests/fixtures/node_label_provider.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_launch_retry.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_lost_requests.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_many_labels.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label1
     min-ready: 1

nodepool/tests/fixtures/node_max_hold_age.yaml

@@ -9,6 +9,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_max_hold_age_2.yaml

@@ -9,6 +9,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 2

nodepool/tests/fixtures/node_max_hold_age_no_default.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_max_ready_age.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     max-ready-age: 2

nodepool/tests/fixtures/node_min_ready_capacity.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_net_name.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label1
     min-ready: 1

nodepool/tests/fixtures/node_no_min_ready.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_quota_cloud.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_quota_pool_cores.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_quota_pool_instances.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_quota_pool_ram.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_second_provider.yaml

@@ -6,6 +6,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_security_group.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_two_image.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_two_image_remove.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_two_provider.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_two_provider_remove.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_unmanaged_image.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_upload_fail.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 2

nodepool/tests/fixtures/node_upload_hook.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/node_vhd.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/node_vhd_and_qcow2.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 2

nodepool/tests/fixtures/openshift.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: pod-fedora
   - name: openshift-project

nodepool/tests/fixtures/openshiftpods.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: pod-fedora
 

nodepool/tests/fixtures/pause_declined_1.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/pause_declined_2.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 0

nodepool/tests/fixtures/secure_file_secure.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 diskimages:
   - name: fake-image
     env-vars:

nodepool/tests/fixtures/static-2-nodes-multilabel.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
   - name: fake-label2

nodepool/tests/fixtures/static-2-nodes.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
 

nodepool/tests/fixtures/static-basic.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
 

nodepool/tests/fixtures/static-multilabel.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
   - name: fake-label2

nodepool/tests/fixtures/static-multiname.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
   - name: other-label

nodepool/tests/fixtures/static-no-check.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
 

nodepool/tests/fixtures/static-parallel-increase.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
 

nodepool/tests/fixtures/static-python-path.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
 

nodepool/tests/fixtures/static-unresolvable.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
 

nodepool/tests/fixtures/static-update.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
   - name: fake-label2

nodepool/tests/fixtures/static.yaml

@@ -3,6 +3,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
   - name: fake-label2

nodepool/tests/fixtures/unmanaged_image_provider_id.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/unmanaged_image_provider_name.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label
     min-ready: 1

nodepool/tests/fixtures/wedge_test.yaml

@@ -7,6 +7,11 @@ zookeeper-servers:
     port: {zookeeper_port}
     chroot: {zookeeper_chroot}
 
+zookeeper-tls:
+  ca: {zookeeper_ca}
+  cert: {zookeeper_cert}
+  key: {zookeeper_key}
+
 labels:
   - name: fake-label1
     min-ready: 1

nodepool/tests/unit/test_driver_aws.py

@@ -80,6 +80,11 @@ class TestDriverAws(tests.DBTestCase):
             'port': self.zookeeper_port,
             'chroot': self.zookeeper_chroot,
         }
+        raw_config['zookeeper-tls'] = {
+            'ca': self.zookeeper_ca,
+            'cert': self.zookeeper_cert,
+            'key': self.zookeeper_key,
+        }
         raw_config['providers'][0]['pools'][0]['subnet-id'] = subnet_id
         raw_config['providers'][0]['pools'][0]['security-group-id'] = sg_id
         raw_config['providers'][0]['pools'][1]['subnet-id'] = subnet_id

nodepool/tests/unit/test_driver_azure.py

@@ -169,6 +169,11 @@ class TestDriverAzure(tests.DBTestCase):
             'port': self.zookeeper_port,
             'chroot': self.zookeeper_chroot,
         }
+        raw_config['zookeeper-tls'] = {
+            'ca': self.zookeeper_ca,
+            'cert': self.zookeeper_cert,
+            'key': self.zookeeper_key,
+        }
         with tempfile.NamedTemporaryFile() as tf:
             tf.write(yaml.safe_dump(
                 raw_config, default_flow_style=False).encode('utf-8'))

nodepool/tests/unit/test_driver_gce.py

@@ -248,6 +248,11 @@ class TestDriverGce(tests.DBTestCase):
             'port': self.zookeeper_port,
             'chroot': self.zookeeper_chroot,
         }
+        raw_config['zookeeper-tls'] = {
+            'ca': self.zookeeper_ca,
+            'cert': self.zookeeper_cert,
+            'key': self.zookeeper_key,
+        }
 
         with tempfile.NamedTemporaryFile() as tf:
             tf.write(yaml.safe_dump(

nodepool/zk.py

@@ -983,11 +983,16 @@ class ZooKeeper(object):
             hosts = buildZooKeeperHosts(host_list)
             args = dict(hosts=hosts,
                         read_only=read_only)
-            if tls_key:
-                args['use_ssl'] = True
-                args['keyfile'] = tls_key
-                args['certfile'] = tls_cert
-                args['ca'] = tls_ca
+
+            args['use_ssl'] = True
+            if not (tls_key and tls_cert and tls_ca):
+                raise Exception("A TLS ZooKeeper connection is required; "
+                                "please supply the zookeeper-tls "
+                                "config values.")
+
+            args['keyfile'] = tls_key
+            args['certfile'] = tls_cert
+            args['ca'] = tls_ca
             self.client = KazooClient(**args)
             self.client.add_listener(self._connection_listener)
             # Manually retry initial connection attempt

playbooks/nodepool-functional-container-openstack/pre.yaml

@@ -5,6 +5,8 @@
       bindep_dir: "{{ zuul.projects['opendev.org/zuul/nodepool'].src_dir }}"
     - role: test-setup
       zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/nodepool'].src_dir }}"
+    - role: ensure-zookeeper
+      zookeeper_use_tls: true
     - ensure-docker
     # Note: keep after ensure-docker
     - use-buildset-registry

playbooks/nodepool-functional-container-openstack/templates/docker-compose.yaml.j2

@@ -23,6 +23,8 @@ services:
       - /var/log/nodepool:/var/log/nodepool
       # devstack tls-proxy puts CA here that is referenced by cloud config
       - /opt/stack/data:/opt/stack/data:ro
+      # zookeeper certs
+      - /opt/zookeeper/ca:/opt/zookeeper/ca:ro
 
   nodepool-launcher:
     image: zuul/nodepool-launcher{{ nodepool_container_tag|default('') }}
@@ -41,3 +43,5 @@ services:
       - /var/log/nodepool:/var/log/nodepool
       # devstack tls-proxy puts CA here that is referenced by cloud config
       - /opt/stack/data:/opt/stack/data:ro
+      # zookeeper certs
+      - /opt/zookeeper/ca:/opt/zookeeper/ca:ro

playbooks/nodepool-functional-container-openstack/templates/nodepool.yaml.j2

@@ -3,7 +3,12 @@ images-dir: {{ NODEPOOL_DIB_BASE_PATH }}/images
 
 zookeeper-servers:
   - host: localhost
-    port: 2181
+    port: 2281
+
+zookeeper-tls:
+  ca: /opt/zookeeper/ca/certs/cacert.pem
+  cert: /opt/zookeeper/ca/certs/client.pem
+  key: /opt/zookeeper/ca/keys/clientkey.pem
 
 labels:
   - name: test-image