Merge pull request #12 from gokrokvertskhov/rebase#07222012

Multipart data handling - new request for rebased code

cloudbaseinit/plugins/windows/userdata-plugins/cloudconfig.py

@@ -0,0 +1,34 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from cloudbaseinit.openstack.common import log as logging
+
+LOG = logging.getLogger("cloudbaseinit")
+
+def get_plugin(parent_set):
+    return CloudConfigHandler(parent_set)
+    
+class CloudConfigHandler:
+    
+    def __init__(self, parent_set):
+        LOG.info("Cloud-config part handler is loaded.")
+        self.type = "text/cloud-config"
+        self.name = "Cloud-config userdata plugin"
+        return
+    
+    def process(self, part):
+        return
+    

cloudbaseinit/plugins/windows/userdata-plugins/heathandler.py

@@ -0,0 +1,47 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import re
+import tempfile
+import uuid
+import email
+import tempfile
+import os
+import errno
+from cloudbaseinit.openstack.common import log as logging
+from cloudbaseinit.osutils.factory import *
+from cloudbaseinit.plugins.windows.userdata import handle
+
+LOG = logging.getLogger("cloudbaseinit")
+
+def get_plugin(parent_set):
+    return HeatUserDataHandler(parent_set)
+
+class HeatUserDataHandler:
+
+    def __init__(self, parent_set):
+        LOG.info("Heat user data part handler is loaded.")
+        self.type = "text/x-cfninitdata"
+        self.name = "Heat userdata plugin"
+        return
+
+    def process(self, part):
+        #Only user-data part of Heat multipart data is supported. All other cfinitdata part will be skipped
+        if part.get_filename() == "cfn-userdata":
+            handle(part.get_payload())
+        return
+ 

cloudbaseinit/plugins/windows/userdata-plugins/parthandler.py

@@ -0,0 +1,67 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import imp
+import os
+from cloudbaseinit.openstack.common import log as logging
+
+LOG = logging.getLogger("cloudbaseinit")
+
+def get_plugin(parent_set):
+    return PartHandlerScriptHandler(parent_set)
+
+def load_from_file(filepath, function):
+    class_inst = None
+
+    mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
+
+    if file_ext.lower() == '.py':
+        py_mod = imp.load_source(mod_name, filepath)
+
+    elif file_ext.lower() == '.pyc':
+        py_mod = imp.load_compiled(mod_name, filepath)
+
+    if hasattr(py_mod, function):
+        callable = getattr(__import__(mod_name),function)
+  
+    return callable
+
+class PartHandlerScriptHandler:
+        
+    def __init__(self, parent_set):
+        LOG.info("Part-handler script part handler is loaded.")
+        self.type = "text/part-handler"
+        self.name = "Part-handler userdata plugin"
+        self.parent_set = parent_set
+        return
+    
+    def process(self, part):
+       handler_path = self.parent_set.path + "/part-handler/"+part.get_filename()
+       with open(handler_path, "wb") as f:
+            f.write(part.get_payload())
+        
+        
+       list_types = load_from_file(handler_path,"list_types")
+       handle_part = load_from_file(handler_path, "handle_part")
+       
+       if list_types is not None and handle_part is not None:
+           parts = list_types() 
+           for part in parts:
+               LOG.info("Installing new custom handler for type: %s", part)
+               self.parent_set.custom_handlers[part] = handle_part
+           self.parent_set.has_custom_handlers = True       
+       return
+   

cloudbaseinit/plugins/windows/userdata-plugins/shellscript.py

@@ -0,0 +1,78 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import re
+import tempfile
+import uuid
+import email
+import tempfile
+import os
+import errno
+
+from cloudbaseinit.openstack.common import log as logging
+from cloudbaseinit.osutils.factory import *
+
+LOG = logging.getLogger("cloudbaseinit")
+
+def get_plugin(parent_set):
+    return ShellScriptHandler(parent_set)
+
+class ShellScriptHandler:
+    
+    def __init__(self, parent_set):
+        LOG.info("Shell-script part handler is loaded.")
+        self.type = "text/x-shellscript"
+        self.name = "Shell-script userdata plugin"
+        return
+    
+    def process(self, part):
+        osutils = OSUtilsFactory().get_os_utils()
+
+        file_name = part.get_filename()
+        target_path = os.path.join(tempfile.gettempdir(), file_name)
+        
+        if file_name.endswith(".cmd"):
+            args = [target_path]
+            shell = True
+        elif file_name.endswith(".sh"):
+            args = ['bash.exe', target_path]
+            shell = False
+        elif file_name.endswith(".ps1"):
+            args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned',
+                    '-NonInteractive', target_path]
+            shell = False           
+        else:
+            # Unsupported
+            LOG.warning('Unsupported shell format')
+            return False
+
+        try:
+            with open(target_path, 'wb') as f:
+                f.write(part.get_payload())
+            (out, err, ret_val) = osutils.execute_process(args, shell)
+
+            LOG.info('User_data script ended with return code: %d' % ret_val)
+            LOG.debug('User_data stdout:\n%s' % out)
+            LOG.debug('User_data stderr:\n%s' % err)
+        except Exception, ex:
+            LOG.warning('An error occurred during user_data execution: \'%s\'' % ex)
+        finally:
+            if os.path.exists(target_path):
+                os.remove(target_path)
+
+        return False
+    

cloudbaseinit/plugins/windows/userdata.py

@@ -18,16 +18,41 @@ import os
 import re
 import tempfile
 import uuid
+import email
+import errno
 
 from cloudbaseinit.metadata.services import base as metadata_services_base
 from cloudbaseinit.openstack.common import log as logging
 from cloudbaseinit.osutils import factory as osutils_factory
 from cloudbaseinit.plugins import base
+from cloudbaseinit.plugins.windows.userdata_plugins import PluginSet
+from cloudbaseinit.openstack.common import cfg
+
+
+opts = [
+    cfg.StrOpt('user_data_folder', default='cloud-data',
+        help='Specifies a folder to store multipart data files.'),
+    ]
+
+
+CONF = cfg.CONF
+CONF.register_opts(opts)
 
 LOG = logging.getLogger(__name__)
 
 
-class UserDataPlugin(base.BasePlugin):
+class UserDataPlugin(base.BasePlugin): 
+    def __init__(self, cfg=CONF):
+        self.cfg = cfg
+        self.msg = None
+        self.plugin_set = PluginSet(self.get_plugin_path())
+        self.plugin_set.reload()
+        return
+
+    def get_plugin_path(self):
+        return os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
+                 "windows/userdata-plugins")
+    
     def execute(self, service):
         try:
             user_data = service.get_user_data('openstack')
@@ -37,57 +62,120 @@ class UserDataPlugin(base.BasePlugin):
         if not user_data:
             return (base.PLUGIN_EXECUTION_DONE, False)
 
-        LOG.debug('User data content:\n%s' % user_data)
+        self.process_userdata(user_data)
-
-        osutils = osutils_factory.OSUtilsFactory().get_os_utils()
-
-        target_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
-        if re.search(r'^rem cmd\s', user_data, re.I):
-            target_path += '.cmd'
-            args = [target_path]
-            shell = True
-        elif re.search(r'^#!', user_data, re.I):
-            target_path += '.sh'
-            args = ['bash.exe', target_path]
-            shell = False
-        elif re.search(r'^#ps1\s', user_data, re.I):
-            target_path += '.ps1'
-            args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned',
-                    '-NonInteractive', target_path]
-            shell = False
-        elif re.search(r'^#ps1_sysnative\s', user_data, re.I):
-            if os.path.isdir(os.path.expandvars('%windir%\\sysnative')):
-                target_path += '.ps1'
-                args = [os.path.expandvars('%windir%\\sysnative\\'
-                                           'WindowsPowerShell\\v1.0\\'
-                                           'powershell.exe'),
-                        '-ExecutionPolicy',
-                        'RemoteSigned', '-NonInteractive', target_path]
-                shell = False
-            else:
-                # Unable to validate sysnative presence
-                LOG.warning('Unable to validate sysnative folder presence. '
-                            'If Target OS is Server 2003, please ensure you '
-                            'have KB942589 installed')
-                return (base.PLUGIN_EXECUTION_DONE, False)
-        else:
-            # Unsupported
-            LOG.warning('Unsupported user_data format')
-            return (base.PLUGIN_EXECUTION_DONE, False)
-
-        try:
-            with open(target_path, 'wb') as f:
-                f.write(user_data)
-            (out, err, ret_val) = osutils.execute_process(args, shell)
-
-            LOG.info('User_data script ended with return code: %d' % ret_val)
-            LOG.debug('User_data stdout:\n%s' % out)
-            LOG.debug('User_data stderr:\n%s' % err)
-        except Exception, ex:
-            LOG.warning('An error occurred during user_data execution: \'%s\''
-                        % ex)
-        finally:
-            if os.path.exists(target_path):
-                os.remove(target_path)
-
         return (base.PLUGIN_EXECUTION_DONE, False)
+    
+    def process_userdata(self, user_data):
+        LOG.debug('User data content:\n%s' % user_data)
+        if user_data.startswith('Content-Type: multipart'):
+            for part in self.parse_MIME(user_data):
+                self.process_part(part)
+        else:
+            handle(user_data)
+        return
+        
+    def process_part(self, part):
+        part_handler = self.get_part_handler(part)
+        if part_handler is not None:
+            try:
+                self.begin_part_process_event(part)
+                LOG.info("Processing part %s filename: %s with handler: %s", part.get_content_type(), part.get_filename(), part_handler.name)
+                part_handler.process(part)
+                self.end_part_process_event(part)
+            except Exception,e:
+                LOG.error('Exception during multipart part handling: %s %s \n %s' , part.get_content_type(), part.get_filename(), e)
+        return
+    
+    def begin_part_process_event(self, part):
+        handler = self.get_custom_handler(part)
+        if handler is not None:
+            try:
+              handler("","__begin__", part.get_filename(), part.get_payload())
+            except Exception,e:
+                LOG.error("Exception occurred during custom handle script invocation (__begin__): %s ", e)
+        return
+    
+    def end_part_process_event(self, part):
+        handler = self.get_custom_handler(part)
+        if handler is not None:
+            try:
+              handler("","__end__", part.get_filename(), part.get_payload())
+            except Exception,e:
+              LOG.error("Exception occurred during custom handle script invocation (__end__): %s ", e)
+        return
+    
+    
+    def get_custom_handler(self, part):
+        if self.plugin_set.has_custom_handlers:
+            if part.get_content_type() in self.plugin_set.custom_handlers:
+                handler = self.plugin_set.custom_handlers[part.get_content_type()]
+                return handler
+        return None
+
+    def get_part_handler(self, part):
+        if part.get_content_type() in self.plugin_set.set:
+            handler = self.plugin_set.set[part.get_content_type()]
+            return handler
+        else:
+            return None
+    
+    def parse_MIME(self, user_data):
+        self.msg = email.message_from_string(user_data)
+        return self.msg.walk()
+
+    
+    
+def handle(self, user_data):
+    osutils = osutils_factory.OSUtilsFactory().get_os_utils()
+
+    target_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
+    if re.search(r'^rem cmd\s', user_data, re.I):
+        target_path += '.cmd'
+        args = [target_path]
+        shell = True
+    elif re.search(r'^#!', user_data, re.I):
+        target_path += '.sh'
+        args = ['bash.exe', target_path]
+        shell = False
+    elif re.search(r'^#ps1\s', user_data, re.I):
+        target_path += '.ps1'
+        args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned',
+                '-NonInteractive', target_path]
+        shell = False
+    elif re.search(r'^#ps1_sysnative\s', user_data, re.I):
+        if os.path.isdir(os.path.expandvars('%windir%\\sysnative')):
+            target_path += '.ps1'
+            args = [os.path.expandvars('%windir%\\sysnative\\'
+                                       'WindowsPowerShell\\v1.0\\'
+                                       'powershell.exe'),
+                    '-ExecutionPolicy',
+                    'RemoteSigned', '-NonInteractive', target_path]
+            shell = False
+        else:
+            # Unable to validate sysnative presence
+            LOG.warning('Unable to validate sysnative folder presence. '
+                        'If Target OS is Server 2003, please ensure you '
+                        'have KB942589 installed')
+            return (base.PLUGIN_EXECUTION_DONE, False)
+    else:
+        # Unsupported
+        LOG.warning('Unsupported user_data format')
+        return (base.PLUGIN_EXECUTION_DONE, False)
+
+    try:
+        with open(target_path, 'wb') as f:
+            f.write(user_data)
+        (out, err, ret_val) = osutils.execute_process(args, shell)
+
+        LOG.info('User_data script ended with return code: %d' % ret_val)
+        LOG.debug('User_data stdout:\n%s' % out)
+        LOG.debug('User_data stderr:\n%s' % err)
+    except Exception, ex:
+        LOG.warning('An error occurred during user_data execution: \'%s\''
+                    % ex)
+    finally:
+        if os.path.exists(target_path):
+            os.remove(target_path)
+
+    return (base.PLUGIN_EXECUTION_DONE, False)
+

cloudbaseinit/plugins/windows/userdata_plugins.py

@@ -0,0 +1,81 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import imp
+import os
+import sys, glob
+from cloudbaseinit.openstack.common import log as logging
+import traceback
+
+LOG = logging.getLogger(__name__)
+global builders
+
+def load_from_file(filepath, parent_set):
+    class_inst = None
+
+    mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
+
+    if file_ext.lower() == '.py':
+        py_mod = imp.load_source(mod_name, filepath)
+
+    elif file_ext.lower() == '.pyc':
+        py_mod = imp.load_compiled(mod_name, filepath)
+
+    if hasattr(py_mod, "get_plugin"):
+        callable = getattr(__import__(mod_name), "get_plugin")
+        class_inst = callable(parent_set)
+
+    return class_inst
+
+
+class PluginSet:
+
+    def __init__(self, path):
+       self.path = path
+       sys.path.append(self.path)
+       self.set = {}
+       self.has_custom_handlers = False
+       self.custom_handlers = {}
+               
+    def get_plugin(self, content_type, file_name):
+       return
+
+    def load(self):
+        files = glob.glob(self.path + '/*.py')
+            
+        if len(files) == 0:
+           LOG.debug("No user data plug-ins found in %s:", self.path)
+           return
+       
+        for file in files:
+              LOG.debug("Trying to load user data plug-in from file: %s", file)
+              try:
+                plugin = load_from_file(file, self)
+                if plugin is not None:
+                    LOG.info("Plugin '%s' loaded.", plugin.name)
+                    self.set[plugin.type] = plugin
+              except:
+                exc_type, exc_value, exc_traceback = sys.exc_info()
+                LOG.error('Can`t load plugin from the file %s. Skip it.', file)
+                LOG.debug(repr(traceback.format_exception(exc_type, exc_value,
+                                              exc_traceback)))
+
+
+    def reload(self):
+        self.set = {}
+        self.load()
+

cloudbaseinit/test/test.mime

@@ -0,0 +1,173 @@
+Content-Type: multipart/mixed; boundary="===============1598784645116016685=="
+MIME-Version: 1.0
+
+--===============1598784645116016685==
+Content-Type: text/cloud-config; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="cloud-config"
+
+runcmd:
+ - setenforce 0 > /dev/null 2>&1 || true
+
+user: ec2-user
+
+cloud_config_modules:
+ - locale
+ - set_hostname
+ - ssh
+ - timezone
+ - update_etc_hosts
+ - update_hostname
+ - runcmd
+
+# Capture all subprocess output into a logfile
+# Useful for troubleshooting cloud-init issues
+output: {all: '| tee -a /var/log/cloud-init-output.log'}
+
+--===============1598784645116016685==
+Content-Type: text/part-handler; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="part-handler.py"
+
+#part-handler
+
+import os
+import datetime
+
+
+def list_types():
+    return(["text/x-cfninitdata"])
+
+
+def handle_part(data, ctype, filename, payload):
+    if ctype == "__begin__":
+        try:
+            os.makedirs('/var/lib/heat-cfntools', 0700)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        return
+
+    if ctype == "__end__":
+        return
+
+    with open('/var/log/part-handler.log', 'a') as log:
+        timestamp = datetime.datetime.now()
+        log.write('%s filename:%s, ctype:%s\n' % (timestamp, filename, ctype))
+
+    if ctype == 'text/x-cfninitdata':
+        with open('/var/lib/heat-cfntools/%s' % filename, 'w') as f:
+            f.write(payload)
+
+        # TODO(sdake) hopefully temporary until users move to heat-cfntools-1.3
+        with open('/var/lib/cloud/data/%s' % filename, 'w') as f:
+            f.write(payload)
+
+--===============1598784645116016685==
+Content-Type: text/x-cfninitdata; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="cfn-userdata"
+
+#ps1
+Get-Date
+
+--===============1598784645116016685==
+Content-Type: text/x-shellscript; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="loguserdata.py"
+
+#!/usr/bin/env python
+
+import sys
+import os
+import subprocess
+import datetime
+import pkg_resources
+from distutils.version import LooseVersion
+import errno
+
+path = '/var/lib/heat-cfntools'
+
+
+def chk_ci_version():
+    v = LooseVersion(pkg_resources.get_distribution('cloud-init').version)
+    return v >= LooseVersion('0.6.0')
+
+
+def create_log(path):
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0600)
+    return os.fdopen(fd, 'w')
+
+
+def call(args, log):
+    log.write('%s\n' % ' '.join(args))
+    log.flush()
+    p = subprocess.Popen(args, stdout=log, stderr=log)
+    p.wait()
+    return p.returncode
+
+
+def main(log):
+
+    if not chk_ci_version():
+        # pre 0.6.0 - user data executed via cloudinit, not this helper
+        log.write('Unable to log provisioning, need a newer version of'
+                  ' cloud-init\n')
+        return -1
+
+    userdata_path = os.path.join(path, 'cfn-userdata')
+    os.chmod(userdata_path, 0700)
+
+    log.write('Provision began: %s\n' % datetime.datetime.now())
+    log.flush()
+    returncode = call([userdata_path], log)
+    log.write('Provision done: %s\n' % datetime.datetime.now())
+    if returncode:
+        return returncode
+
+
+if __name__ == '__main__':
+    with create_log('/var/log/heat-provision.log') as log:
+        returncode = main(log)
+        if returncode:
+            log.write('Provision failed')
+            sys.exit(returncode)
+
+    userdata_path = os.path.join(path, 'provision-finished')
+    with create_log(userdata_path) as log:
+        log.write('%s\n' % datetime.datetime.now())
+
+--===============1598784645116016685==
+Content-Type: text/x-cfninitdata; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="cfn-watch-server"
+
+http://67.207.197.36:8003
+--===============1598784645116016685==
+Content-Type: text/x-cfninitdata; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="cfn-metadata-server"
+
+http://67.207.197.36:8000
+--===============1598784645116016685==
+Content-Type: text/x-cfninitdata; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="cfn-boto-cfg"
+
+[Boto]
+debug = 0
+is_secure = 0
+https_validate_certificates = 1
+cfn_region_name = heat
+cfn_region_endpoint = 67.207.197.36
+cloudwatch_region_name = heat
+cloudwatch_region_endpoint = 67.207.197.36
+--===============1598784645116016685==--
+