diff --git a/rally/plugins/openstack/verification/tempest/manager.py b/rally/plugins/openstack/verification/tempest/manager.py
index d5840a01..d6edfc50 100644
--- a/rally/plugins/openstack/verification/tempest/manager.py
+++ b/rally/plugins/openstack/verification/tempest/manager.py
@@ -57,6 +57,9 @@ class TempestManager(testr.TestrLauncher):
         return config.read_configfile(self.configfile)
 
     def configure(self, extra_options=None):
+        if not os.path.isdir(os.path.dirname(self.configfile)):
+            os.makedirs(os.path.dirname(self.configfile))
+
         cm = config.TempestConfigfileManager(self.verifier.deployment)
         raw_configfile = cm.create(self.configfile, extra_options)
         return raw_configfile
@@ -68,13 +71,12 @@ class TempestManager(testr.TestrLauncher):
         with open(self.configfile, "w") as f:
             f.write(new_content)
 
-    def install_extension(self, source, version=None, extra=None):
+    def install_extension(self, source, version=None, extra_settings=None):
         """Install a Tempest plugin."""
-        if extra:
+        if extra_settings:
             raise NotImplementedError(
-                _LE("'%s' verifiers don't support extra options for "
-                    "extension installations.")
-                % self.get_name())
+                _LE("'%s' verifiers don't support extra installation settings "
+                    "for extensions.") % self.get_name())
         version = version or "master"
         egg = re.sub("\.git$", "", os.path.basename(source.strip("/")))
         full_source = "git+{0}@{1}#egg={2}".format(source, version, egg)
diff --git a/tests/unit/plugins/openstack/verification/tempest/test_manager.py b/tests/unit/plugins/openstack/verification/tempest/test_manager.py
index 42eae5d8..3cd50579 100644
--- a/tests/unit/plugins/openstack/verification/tempest/test_manager.py
+++ b/tests/unit/plugins/openstack/verification/tempest/test_manager.py
@@ -88,7 +88,8 @@ class TempestManagerTestCase(test.TestCase):
                                                         system_wide=True))
         e = self.assertRaises(NotImplementedError, tempest.install_extension,
                               None, None, {"key": "value"})
-        self.assertIn("verifiers don't support extra options", "%s" % e)
+        self.assertIn("verifiers don't support extra installation settings",
+                      "%s" % e)
 
         # case #1 system-wide installation
         source = "https://github.com/example/example"