Better ssh and x509 keys extraction

For ssh keys, try gathering all values found under both "public_keys"
and "keys" fields under metadata. For x509 certificate keys,
try looking into metadata under both "meta" and "keys" fields and
finally userdata if no such information is retrieved earlier.

Change-Id: I43345344aa008187787e97d6296abfc181416cf3

cloudbaseinit/metadata/services/baseopenstackservice.py

@@ -62,9 +62,18 @@ class BaseOpenStackService(base.BaseMetadataService):
         return self._get_meta_data().get('hostname')
 
     def get_public_keys(self):
-        public_keys = self._get_meta_data().get('public_keys')
+        """Get a list of all unique public keys found among the metadata."""
-        if public_keys:
+        public_keys = []
-            return public_keys.values()
+        meta_data = self._get_meta_data()
+        public_keys_dict = meta_data.get('public_keys')
+        if public_keys_dict:
+            public_keys = list(public_keys_dict.values())
+        keys = meta_data.get("keys")
+        if keys:
+            for key_dict in keys:
+                if key_dict["type"] == "ssh":
+                    public_keys.append(key_dict["data"])
+        return list(set(public_keys))
 
     def get_network_details(self):
         network_config = self._get_meta_data().get('network_config')
@@ -94,35 +103,44 @@ class BaseOpenStackService(base.BaseMetadataService):
         return password
 
     def get_client_auth_certs(self):
-        cert_data = None
+        """Gather all unique certificates found among the metadata.
 
+        If there are no certificates under "meta" or "keys" field,
+        then try looking into user-data for this kind of information.
+        """
+        certs = []
         meta_data = self._get_meta_data()
-        meta = meta_data.get('meta')
 
+        meta = meta_data.get("meta")
         if meta:
-            i = 0
+            cert_data_list = []
+            idx = 0
             while True:
                 # Chunking is necessary as metadata items can be
-                # max. 255 chars long
+                # max. 255 chars long.
-                cert_chunk = meta.get('admin_cert%d' % i)
+                cert_chunk = meta.get("admin_cert%d" % idx)
                 if not cert_chunk:
                     break
-                if not cert_data:
+                cert_data_list.append(cert_chunk)
-                    cert_data = cert_chunk
+                idx += 1
-                else:
+            if cert_data_list:
-                    cert_data += cert_chunk
+                # It's a list of strings for sure.
-                i += 1
+                certs.append("".join(cert_data_list))
 
-        if not cert_data:
+        keys = meta_data.get("keys")
+        if keys:
+            for key_dict in keys:
+                if key_dict["type"] == "x509":
+                    certs.append(key_dict["data"])
 
+        if not certs:
             # Look if the user_data contains a PEM certificate
             try:
                 user_data = self.get_user_data()
                 if user_data.startswith(
                         x509constants.PEM_HEADER.encode()):
-                    cert_data = user_data
+                    certs.append(user_data)
             except base.NotExistingMetadataException:
                 LOG.debug("user_data metadata not present")
 
-        if cert_data:
+        return list(set((cert.strip() for cert in certs)))
-            return [cert_data]

cloudbaseinit/tests/metadata/fake_json_response.py

@@ -68,9 +68,64 @@ def get_fake_metadata_json(version):
                 "FV3og4Wyz5zipPVh8YpVev6dlg0tRWUrCtZF9"
                 "IODpCTrT3vsPRG3xz7CppR+vGi/1gLXHtJCRj"
                 "frHwkY6cXyhypNmkU99K/wMqSv30vsDwdnsQ1"
-                "q3YhLarMHB Generated by Nova\n",
+                "q3YhLarMHB Generated by Nova\n"
-            0: "windows"
         },
+        "keys": [
+            {
+                "data":
+                "ssh-rsa "
+                "AAAAB3NzaC1yc2EAAAADAQABAAABA"
+                "QDf7kQHq7zvBod3yIZs0tB/AOOZz5pab7qt/h"
+                "78VF7yi6qTsFdUnQxRue43R/75wa9EEyokgYR"
+                "LKIN+Jq2A5tXNMcK+rNOCzLJFtioAwEl+S6VL"
+                "G9jfkbUv++7zoSMOsanNmEDvG0B79MpyECFCl"
+                "th2DsdE4MQypify35U5ri5Qi7E6PEYAsU65LF"
+                "MG2boeCIB29BEooE6AgPr2DuJeJ+2uw+YScF9"
+                "FV3og4Wyz5zipPVh8YpVev6dlg0tRWUrCtZF9"
+                "IODpCTrT3vsPRG3xz7CppR+vGi/1gLXHtJCRj"
+                "frHwkY6cXyhypNmkU99K/wMqSv30vsDwdnsQ1"
+                "q3YhLarMHB Generated by Nova\n",
+                "type": "ssh",
+                "name": "key"
+            },
+            {
+                "data":
+                "-----BEGIN CERTIFICATE-----\n"
+                "MIIDNzCCAh+gAwIBAgIJAODgoAY83AxRMA0GCS"
+                "qGSIb3DQEBCwUAMCsxKTAnBgNV\nBAMTIGE3ZD"
+                "g3YTM1NzE1MDQzYzI4NDUwYTMzZGFiYWQwMDk2"
+                "MB4XDTE1MDUxMjE5\nNDEwNVoXDTI1MDUwOTE5"
+                "NDEwNVowKzEpMCcGA1UEAxMgYTdkODdhMzU3MT"
+                "UwNDNj\nMjg0NTBhMzNkYWJhZDAwOTYwggEiMA"
+                "0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB\nAQ"
+                "DAbRN9IRO0vEj6lgh5ZBAdpMR8mDymcP5M8uLT"
+                "CqvrFSuv2yB+rei3fJ7Rw4Tv\nHX/aKCMVjqAO"
+                "T3CwsO9UA+Q2BmbkCi/e8VpzHN8m8VwkyBLXNn"
+                "4ed6e+G5WOPVXH\n76RajfRkDq+WCo330SiBWt"
+                "QBsC0AP8l1M57XYxtowh/EweJDZ5bYIogPOnrP"
+                "XRSl\nylDKiJuAs74fOiDITdeXCGeiPTy2VFCi"
+                "LUV7DLpCRUV89tEpFlNVlD6yTmnE/Mrh\nA49s"
+                "m9PUMY1cI8Bl5f5B/CBQa+5lEQT72HMOuumFrX"
+                "MOD4jQIcNyRl4LKdam1wwA\nESu0s7Cz5LxvJc"
+                "d4EgUxwYrHAgMBAAGjXjBcMBMGA1UdJQQMMAoG"
+                "CCsGAQUFBwMC\nMEUGA1UdEQQ+MDygOgYKKwYB"
+                "BAGCNxQCA6AsDCphN2Q4N2EzNTcxNTA0M2MyOD"
+                "Q1\nMGEzM2RhYmFkMDA5NkBsb2NhbGhvc3QwDQ"
+                "YJKoZIhvcNAQELBQADggEBABpTP8mg\nOHl+Jp"
+                "3wcBVz5sdFUmPRCGJgPaG/rUgAHuC1UNNP3Oy0"
+                "VlDOIc3Yv0Wt3kLa081N\nUy7NIQjQAN29Pq3c"
+                "1Jsq0ucpzEroIBdwacCkB51lZ5CtzwlztK2KZ3"
+                "gWJVpraQwn\nt9ZZ2w+eKKt332+uXFVqNgkBjN"
+                "3LHeehFv1Vd0HQYPkuwWIrf7O0gpZP9MD3+GnZ"
+                "\nEMvBxOV5yFPbPFQSWX50YHijtEkKbXb/liEA"
+                "RccxZwl1aC2+IfaSrMh3JqWF/AQf\nlVtJYgGw"
+                "ixt/OUsA+oA5u6LuPSJhwFnrAN8UWrrOLlswcl"
+                "jQ2mfTambZp43AomqJ\n6N07zSahNAf/UqI=\n"
+                "-----END CERTIFICATE-----\n",
+                "type": "x509",
+                "name": "cert"
+            }
+        ],
         "network_config": {
             "content_path": "network",
             # This is not actually in the metadata json file,

cloudbaseinit/tests/metadata/services/test_baseopenstackservice.py

@@ -43,6 +43,8 @@ class TestBaseOpenStackService(unittest.TestCase):
         fake_metadata = fake_json_response.get_fake_metadata_json(date)
         self._fake_network_config = fake_metadata["network_config"]
         self._fake_content = self._fake_network_config["debian_config"]
+        self._fake_public_keys = fake_metadata["public_keys"]
+        self._fake_keys = fake_metadata["keys"]
         self._partial_test_get_network_details = functools.partial(
             self._test_get_network_details,
             network_config=self._fake_network_config,
@@ -96,10 +98,16 @@ class TestBaseOpenStackService(unittest.TestCase):
     @mock.patch(MODPATH +
                 ".BaseOpenStackService._get_meta_data")
     def test_get_public_keys(self, mock_get_meta_data):
+        mock_get_meta_data.return_value.get.side_effect = \
+            [self._fake_public_keys, self._fake_keys]
         response = self._service.get_public_keys()
         mock_get_meta_data.assert_called_once_with()
-        mock_get_meta_data().get.assert_called_once_with('public_keys')
+        mock_get_meta_data.return_value.get.assert_any_call("public_keys")
-        self.assertEqual(mock_get_meta_data().get().values(), response)
+        mock_get_meta_data.return_value.get.assert_any_call("keys")
+        values = (list(self._fake_public_keys.values()) +
+                  [key["data"] for key in self._fake_keys
+                   if key["type"] == "ssh"])
+        self.assertEqual(sorted(list(set(values))), sorted(response))
 
     @mock.patch(MODPATH +
                 ".BaseOpenStackService._get_meta_data")
@@ -135,18 +143,30 @@ class TestBaseOpenStackService(unittest.TestCase):
         mock_get_user_data.side_effect = [ret_value]
         response = self._service.get_client_auth_certs()
         mock_get_meta_data.assert_called_once_with()
-        if 'meta' in meta_data:
+        if isinstance(ret_value, bytes) and ret_value.startswith(
-            self.assertEqual([b'fake cert'], response)
+                x509constants.PEM_HEADER.encode()):
-        elif type(ret_value) is str and ret_value.startswith(
-                x509constants.PEM_HEADER):
             mock_get_user_data.assert_called_once_with()
             self.assertEqual([ret_value], response)
         elif ret_value is base.NotExistingMetadataException:
-            self.assertIsNone(response)
+            self.assertFalse(response)
+        else:
+            expected = []
+            expectation = {
+                "meta": 'fake cert',
+                "keys": [key["data"].strip() for key in self._fake_keys
+                         if key["type"] == "x509"]
+            }
+            for field, value in expectation.items():
+                if field in meta_data:
+                    expected.extend(value if isinstance(value, list)
+                                    else [value])
+            self.assertEqual(sorted(list(set(expected))), sorted(response))
 
     def test_get_client_auth_certs(self):
         self._test_get_client_auth_certs(
-            meta_data={'meta': {'admin_cert0': b'fake cert'}})
+            meta_data={'meta': {'admin_cert0': 'fake ',
+                                'admin_cert1': 'cert'},
+                       "keys": self._fake_keys})
 
     def test_get_client_auth_certs_no_cert_data(self):
         self._test_get_client_auth_certs(