config: add environment variable substitution

This change enables setting configuration values through
environment variables. This is useful to manage user defined
configuration, such as user password, in Kubernetes deployment.

Change-Id: Iafbb63ebbb388ef3038f45fd3a929c3e7e2dc343

nodepool/cmd/config_validator.py

@@ -11,11 +11,12 @@
 # under the License.
 
 import logging
+import os
 import voluptuous as v
 import yaml
 
 from nodepool.driver import ProviderConfig
-from nodepool.config import get_provider_config
+from nodepool.config import get_provider_config, substitute_env_vars
 
 log = logging.getLogger(__name__)
 
@@ -80,7 +81,7 @@ class ConfigValidator:
         }
         return v.Schema(top_level)
 
-    def validate(self):
+    def validate(self, env=os.environ):
         '''
         Validate a configuration file
 
@@ -92,11 +93,13 @@ class ConfigValidator:
 
         try:
             with open(self.config_file) as f:
-                config = yaml.safe_load(f)
+                config = yaml.safe_load(substitute_env_vars(f.read(), env))
         except Exception:
             log.exception('YAML parsing failed')
             return 1
 
+        self.config = config
+
         try:
             # validate the overall schema
             ConfigValidator.getSchema()(config)

nodepool/config.py

@@ -14,7 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import math
+import os
 import time
 import yaml
 
@@ -293,7 +295,16 @@ def get_provider_config(provider):
     return driver.getProviderConfig(provider)
 
 
-def openConfig(path):
+def substitute_env_vars(config_str, env):
+    return functools.reduce(
+        lambda config, env_item: config.replace(
+            "%(" + env_item[0] + ")", env_item[1]),
+        [(k, v) for k, v in env.items()
+         if k.startswith('NODEPOOL_')],
+        config_str)
+
+
+def openConfig(path, env):
     retry = 3
 
     # Since some nodepool code attempts to dynamically re-read its config
@@ -303,8 +314,7 @@ def openConfig(path):
     while True:
         try:
             with open(path) as f:
-                config = yaml.safe_load(f)
+                return yaml.safe_load(substitute_env_vars(f.read(), env))
-            break
         except IOError as e:
             if e.errno == 2:
                 retry = retry - 1
@@ -313,11 +323,10 @@ def openConfig(path):
                 raise e
             if retry == 0:
                 raise e
-    return config
 
 
-def loadConfig(config_path):
+def loadConfig(config_path, env=os.environ):
-    config = openConfig(config_path)
+    config = openConfig(config_path, env)
 
     # Call driver config reset now to clean global hooks like openstacksdk
     for driver in Drivers.drivers.values():
@@ -347,8 +356,8 @@ def loadConfig(config_path):
     return newconfig
 
 
-def loadSecureConfig(config, secure_config_path):
+def loadSecureConfig(config, secure_config_path, env=os.environ):
-    secure = openConfig(secure_config_path)
+    secure = openConfig(secure_config_path, env)
     if not secure:   # empty file
         return
 

nodepool/tests/fixtures/config_validate/good.yaml

@@ -2,7 +2,7 @@ elements-dir: /etc/nodepool/elements
 images-dir: /opt/nodepool_dib
 
 webapp:
-  port: 8005
+  port: %(NODEPOOL_PORT)
   listen_address: '0.0.0.0'
 
 zookeeper-servers:

nodepool/tests/unit/test_config_validator.py

@@ -28,8 +28,9 @@ class TestConfigValidation(tests.BaseTestCase):
                               'fixtures', 'config_validate', 'good.yaml')
 
         validator = ConfigValidator(config)
-        ret = validator.validate()
+        ret = validator.validate(dict(NODEPOOL_PORT="8005"))
         self.assertEqual(ret, 0)
+        self.assertEqual(validator.config['webapp']['port'], 8005)
 
     def test_yaml_error(self):
         config = os.path.join(os.path.dirname(tests.__file__),

releasenotes/notes/config-env-substitution-ef203b725fd05168.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Configuration value can be set from the envirnonment variables using the
+    `%(NODEPOOL_env_name)` syntax.