From fe4e46aa1b887fbffcef1878e58a5039bd1ec738 Mon Sep 17 00:00:00 2001 From: Georgy Okrokvertskhov Date: Mon, 22 Jul 2013 15:35:27 -0700 Subject: [PATCH] 1. Changed UserDataPlugin class to handle multipart user data 2. Added PluginSet class which dynamically loads all available plugins in userdata-plugins subfolder 3. Added standard plugins to handle typical content types in multipart data --- .../windows/userdata-plugins/cloudconfig.py | 34 +++ .../windows/userdata-plugins/heathandler.py | 47 +++++ .../windows/userdata-plugins/parthandler.py | 67 ++++++ .../windows/userdata-plugins/shellscript.py | 78 +++++++ cloudbaseinit/plugins/windows/userdata.py | 196 +++++++++++++----- .../plugins/windows/userdata_plugins.py | 81 ++++++++ cloudbaseinit/test/test.mime | 173 ++++++++++++++++ 7 files changed, 622 insertions(+), 54 deletions(-) create mode 100644 cloudbaseinit/plugins/windows/userdata-plugins/cloudconfig.py create mode 100644 cloudbaseinit/plugins/windows/userdata-plugins/heathandler.py create mode 100644 cloudbaseinit/plugins/windows/userdata-plugins/parthandler.py create mode 100644 cloudbaseinit/plugins/windows/userdata-plugins/shellscript.py create mode 100644 cloudbaseinit/plugins/windows/userdata_plugins.py create mode 100644 cloudbaseinit/test/test.mime diff --git a/cloudbaseinit/plugins/windows/userdata-plugins/cloudconfig.py b/cloudbaseinit/plugins/windows/userdata-plugins/cloudconfig.py new file mode 100644 index 00000000..1ae87ca6 --- /dev/null +++ b/cloudbaseinit/plugins/windows/userdata-plugins/cloudconfig.py @@ -0,0 +1,34 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from cloudbaseinit.openstack.common import log as logging + +LOG = logging.getLogger("cloudbaseinit") + +def get_plugin(parent_set): + return CloudConfigHandler(parent_set) + +class CloudConfigHandler: + + def __init__(self, parent_set): + LOG.info("Cloud-config part handler is loaded.") + self.type = "text/cloud-config" + self.name = "Cloud-config userdata plugin" + return + + def process(self, part): + return + diff --git a/cloudbaseinit/plugins/windows/userdata-plugins/heathandler.py b/cloudbaseinit/plugins/windows/userdata-plugins/heathandler.py new file mode 100644 index 00000000..b6ce15cd --- /dev/null +++ b/cloudbaseinit/plugins/windows/userdata-plugins/heathandler.py @@ -0,0 +1,47 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +import tempfile +import uuid +import email +import tempfile +import os +import errno +from cloudbaseinit.openstack.common import log as logging +from cloudbaseinit.osutils.factory import * +from cloudbaseinit.plugins.windows.userdata import handle + +LOG = logging.getLogger("cloudbaseinit") + +def get_plugin(parent_set): + return HeatUserDataHandler(parent_set) + +class HeatUserDataHandler: + + def __init__(self, parent_set): + LOG.info("Heat user data part handler is loaded.") + self.type = "text/x-cfninitdata" + self.name = "Heat userdata plugin" + return + + def process(self, part): + #Only user-data part of Heat multipart data is supported. All other cfinitdata part will be skipped + if part.get_filename() == "cfn-userdata": + handle(part.get_payload()) + return + diff --git a/cloudbaseinit/plugins/windows/userdata-plugins/parthandler.py b/cloudbaseinit/plugins/windows/userdata-plugins/parthandler.py new file mode 100644 index 00000000..fede1bb8 --- /dev/null +++ b/cloudbaseinit/plugins/windows/userdata-plugins/parthandler.py @@ -0,0 +1,67 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import imp +import os +from cloudbaseinit.openstack.common import log as logging + +LOG = logging.getLogger("cloudbaseinit") + +def get_plugin(parent_set): + return PartHandlerScriptHandler(parent_set) + +def load_from_file(filepath, function): + class_inst = None + + mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) + + if file_ext.lower() == '.py': + py_mod = imp.load_source(mod_name, filepath) + + elif file_ext.lower() == '.pyc': + py_mod = imp.load_compiled(mod_name, filepath) + + if hasattr(py_mod, function): + callable = getattr(__import__(mod_name),function) + + return callable + +class PartHandlerScriptHandler: + + def __init__(self, parent_set): + LOG.info("Part-handler script part handler is loaded.") + self.type = "text/part-handler" + self.name = "Part-handler userdata plugin" + self.parent_set = parent_set + return + + def process(self, part): + handler_path = self.parent_set.path + "/part-handler/"+part.get_filename() + with open(handler_path, "wb") as f: + f.write(part.get_payload()) + + + list_types = load_from_file(handler_path,"list_types") + handle_part = load_from_file(handler_path, "handle_part") + + if list_types is not None and handle_part is not None: + parts = list_types() + for part in parts: + LOG.info("Installing new custom handler for type: %s", part) + self.parent_set.custom_handlers[part] = handle_part + self.parent_set.has_custom_handlers = True + return + \ No newline at end of file diff --git a/cloudbaseinit/plugins/windows/userdata-plugins/shellscript.py b/cloudbaseinit/plugins/windows/userdata-plugins/shellscript.py new file mode 100644 index 00000000..39478785 --- /dev/null +++ b/cloudbaseinit/plugins/windows/userdata-plugins/shellscript.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +import tempfile +import uuid +import email +import tempfile +import os +import errno + +from cloudbaseinit.openstack.common import log as logging +from cloudbaseinit.osutils.factory import * + +LOG = logging.getLogger("cloudbaseinit") + +def get_plugin(parent_set): + return ShellScriptHandler(parent_set) + +class ShellScriptHandler: + + def __init__(self, parent_set): + LOG.info("Shell-script part handler is loaded.") + self.type = "text/x-shellscript" + self.name = "Shell-script userdata plugin" + return + + def process(self, part): + osutils = OSUtilsFactory().get_os_utils() + + file_name = part.get_filename() + target_path = os.path.join(tempfile.gettempdir(), file_name) + + if file_name.endswith(".cmd"): + args = [target_path] + shell = True + elif file_name.endswith(".sh"): + args = ['bash.exe', target_path] + shell = False + elif file_name.endswith(".ps1"): + args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned', + '-NonInteractive', target_path] + shell = False + else: + # Unsupported + LOG.warning('Unsupported shell format') + return False + + try: + with open(target_path, 'wb') as f: + f.write(part.get_payload()) + (out, err, ret_val) = osutils.execute_process(args, shell) + + LOG.info('User_data script ended with return code: %d' % ret_val) + LOG.debug('User_data stdout:\n%s' % out) + LOG.debug('User_data stderr:\n%s' % err) + except Exception, ex: + LOG.warning('An error occurred during user_data execution: \'%s\'' % ex) + finally: + if os.path.exists(target_path): + os.remove(target_path) + + return False + \ No newline at end of file diff --git a/cloudbaseinit/plugins/windows/userdata.py b/cloudbaseinit/plugins/windows/userdata.py index 87907295..4c6d8e5e 100644 --- a/cloudbaseinit/plugins/windows/userdata.py +++ b/cloudbaseinit/plugins/windows/userdata.py @@ -18,16 +18,41 @@ import os import re import tempfile import uuid +import email +import errno from cloudbaseinit.metadata.services import base as metadata_services_base from cloudbaseinit.openstack.common import log as logging from cloudbaseinit.osutils import factory as osutils_factory from cloudbaseinit.plugins import base +from cloudbaseinit.plugins.windows.userdata_plugins import PluginSet +from cloudbaseinit.openstack.common import cfg + + +opts = [ + cfg.StrOpt('user_data_folder', default='cloud-data', + help='Specifies a folder to store multipart data files.'), + ] + + +CONF = cfg.CONF +CONF.register_opts(opts) LOG = logging.getLogger(__name__) -class UserDataPlugin(base.BasePlugin): +class UserDataPlugin(base.BasePlugin): + def __init__(self, cfg=CONF): + self.cfg = cfg + self.msg = None + self.plugin_set = PluginSet(self.get_plugin_path()) + self.plugin_set.reload() + return + + def get_plugin_path(self): + return os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), + "windows/userdata-plugins") + def execute(self, service): try: user_data = service.get_user_data('openstack') @@ -37,57 +62,120 @@ class UserDataPlugin(base.BasePlugin): if not user_data: return (base.PLUGIN_EXECUTION_DONE, False) - LOG.debug('User data content:\n%s' % user_data) - - osutils = osutils_factory.OSUtilsFactory().get_os_utils() - - target_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) - if re.search(r'^rem cmd\s', user_data, re.I): - target_path += '.cmd' - args = [target_path] - shell = True - elif re.search(r'^#!', user_data, re.I): - target_path += '.sh' - args = ['bash.exe', target_path] - shell = False - elif re.search(r'^#ps1\s', user_data, re.I): - target_path += '.ps1' - args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned', - '-NonInteractive', target_path] - shell = False - elif re.search(r'^#ps1_sysnative\s', user_data, re.I): - if os.path.isdir(os.path.expandvars('%windir%\\sysnative')): - target_path += '.ps1' - args = [os.path.expandvars('%windir%\\sysnative\\' - 'WindowsPowerShell\\v1.0\\' - 'powershell.exe'), - '-ExecutionPolicy', - 'RemoteSigned', '-NonInteractive', target_path] - shell = False - else: - # Unable to validate sysnative presence - LOG.warning('Unable to validate sysnative folder presence. ' - 'If Target OS is Server 2003, please ensure you ' - 'have KB942589 installed') - return (base.PLUGIN_EXECUTION_DONE, False) - else: - # Unsupported - LOG.warning('Unsupported user_data format') - return (base.PLUGIN_EXECUTION_DONE, False) - - try: - with open(target_path, 'wb') as f: - f.write(user_data) - (out, err, ret_val) = osutils.execute_process(args, shell) - - LOG.info('User_data script ended with return code: %d' % ret_val) - LOG.debug('User_data stdout:\n%s' % out) - LOG.debug('User_data stderr:\n%s' % err) - except Exception, ex: - LOG.warning('An error occurred during user_data execution: \'%s\'' - % ex) - finally: - if os.path.exists(target_path): - os.remove(target_path) - + self.process_userdata(user_data) return (base.PLUGIN_EXECUTION_DONE, False) + + def process_userdata(self, user_data): + LOG.debug('User data content:\n%s' % user_data) + if user_data.startswith('Content-Type: multipart'): + for part in self.parse_MIME(user_data): + self.process_part(part) + else: + handle(user_data) + return + + def process_part(self, part): + part_handler = self.get_part_handler(part) + if part_handler is not None: + try: + self.begin_part_process_event(part) + LOG.info("Processing part %s filename: %s with handler: %s", part.get_content_type(), part.get_filename(), part_handler.name) + part_handler.process(part) + self.end_part_process_event(part) + except Exception,e: + LOG.error('Exception during multipart part handling: %s %s \n %s' , part.get_content_type(), part.get_filename(), e) + return + + def begin_part_process_event(self, part): + handler = self.get_custom_handler(part) + if handler is not None: + try: + handler("","__begin__", part.get_filename(), part.get_payload()) + except Exception,e: + LOG.error("Exception occurred during custom handle script invocation (__begin__): %s ", e) + return + + def end_part_process_event(self, part): + handler = self.get_custom_handler(part) + if handler is not None: + try: + handler("","__end__", part.get_filename(), part.get_payload()) + except Exception,e: + LOG.error("Exception occurred during custom handle script invocation (__end__): %s ", e) + return + + + def get_custom_handler(self, part): + if self.plugin_set.has_custom_handlers: + if part.get_content_type() in self.plugin_set.custom_handlers: + handler = self.plugin_set.custom_handlers[part.get_content_type()] + return handler + return None + + def get_part_handler(self, part): + if part.get_content_type() in self.plugin_set.set: + handler = self.plugin_set.set[part.get_content_type()] + return handler + else: + return None + + def parse_MIME(self, user_data): + self.msg = email.message_from_string(user_data) + return self.msg.walk() + + + +def handle(self, user_data): + osutils = osutils_factory.OSUtilsFactory().get_os_utils() + + target_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) + if re.search(r'^rem cmd\s', user_data, re.I): + target_path += '.cmd' + args = [target_path] + shell = True + elif re.search(r'^#!', user_data, re.I): + target_path += '.sh' + args = ['bash.exe', target_path] + shell = False + elif re.search(r'^#ps1\s', user_data, re.I): + target_path += '.ps1' + args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned', + '-NonInteractive', target_path] + shell = False + elif re.search(r'^#ps1_sysnative\s', user_data, re.I): + if os.path.isdir(os.path.expandvars('%windir%\\sysnative')): + target_path += '.ps1' + args = [os.path.expandvars('%windir%\\sysnative\\' + 'WindowsPowerShell\\v1.0\\' + 'powershell.exe'), + '-ExecutionPolicy', + 'RemoteSigned', '-NonInteractive', target_path] + shell = False + else: + # Unable to validate sysnative presence + LOG.warning('Unable to validate sysnative folder presence. ' + 'If Target OS is Server 2003, please ensure you ' + 'have KB942589 installed') + return (base.PLUGIN_EXECUTION_DONE, False) + else: + # Unsupported + LOG.warning('Unsupported user_data format') + return (base.PLUGIN_EXECUTION_DONE, False) + + try: + with open(target_path, 'wb') as f: + f.write(user_data) + (out, err, ret_val) = osutils.execute_process(args, shell) + + LOG.info('User_data script ended with return code: %d' % ret_val) + LOG.debug('User_data stdout:\n%s' % out) + LOG.debug('User_data stderr:\n%s' % err) + except Exception, ex: + LOG.warning('An error occurred during user_data execution: \'%s\'' + % ex) + finally: + if os.path.exists(target_path): + os.remove(target_path) + + return (base.PLUGIN_EXECUTION_DONE, False) + diff --git a/cloudbaseinit/plugins/windows/userdata_plugins.py b/cloudbaseinit/plugins/windows/userdata_plugins.py new file mode 100644 index 00000000..3931147e --- /dev/null +++ b/cloudbaseinit/plugins/windows/userdata_plugins.py @@ -0,0 +1,81 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import imp +import os +import sys, glob +from cloudbaseinit.openstack.common import log as logging +import traceback + +LOG = logging.getLogger(__name__) +global builders + +def load_from_file(filepath, parent_set): + class_inst = None + + mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) + + if file_ext.lower() == '.py': + py_mod = imp.load_source(mod_name, filepath) + + elif file_ext.lower() == '.pyc': + py_mod = imp.load_compiled(mod_name, filepath) + + if hasattr(py_mod, "get_plugin"): + callable = getattr(__import__(mod_name), "get_plugin") + class_inst = callable(parent_set) + + return class_inst + + +class PluginSet: + + def __init__(self, path): + self.path = path + sys.path.append(self.path) + self.set = {} + self.has_custom_handlers = False + self.custom_handlers = {} + + def get_plugin(self, content_type, file_name): + return + + def load(self): + files = glob.glob(self.path + '/*.py') + + if len(files) == 0: + LOG.debug("No user data plug-ins found in %s:", self.path) + return + + for file in files: + LOG.debug("Trying to load user data plug-in from file: %s", file) + try: + plugin = load_from_file(file, self) + if plugin is not None: + LOG.info("Plugin '%s' loaded.", plugin.name) + self.set[plugin.type] = plugin + except: + exc_type, exc_value, exc_traceback = sys.exc_info() + LOG.error('Can`t load plugin from the file %s. Skip it.', file) + LOG.debug(repr(traceback.format_exception(exc_type, exc_value, + exc_traceback))) + + + def reload(self): + self.set = {} + self.load() + diff --git a/cloudbaseinit/test/test.mime b/cloudbaseinit/test/test.mime new file mode 100644 index 00000000..b21713d6 --- /dev/null +++ b/cloudbaseinit/test/test.mime @@ -0,0 +1,173 @@ +Content-Type: multipart/mixed; boundary="===============1598784645116016685==" +MIME-Version: 1.0 + +--===============1598784645116016685== +Content-Type: text/cloud-config; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="cloud-config" + +runcmd: + - setenforce 0 > /dev/null 2>&1 || true + +user: ec2-user + +cloud_config_modules: + - locale + - set_hostname + - ssh + - timezone + - update_etc_hosts + - update_hostname + - runcmd + +# Capture all subprocess output into a logfile +# Useful for troubleshooting cloud-init issues +output: {all: '| tee -a /var/log/cloud-init-output.log'} + +--===============1598784645116016685== +Content-Type: text/part-handler; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="part-handler.py" + +#part-handler + +import os +import datetime + + +def list_types(): + return(["text/x-cfninitdata"]) + + +def handle_part(data, ctype, filename, payload): + if ctype == "__begin__": + try: + os.makedirs('/var/lib/heat-cfntools', 0700) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return + + if ctype == "__end__": + return + + with open('/var/log/part-handler.log', 'a') as log: + timestamp = datetime.datetime.now() + log.write('%s filename:%s, ctype:%s\n' % (timestamp, filename, ctype)) + + if ctype == 'text/x-cfninitdata': + with open('/var/lib/heat-cfntools/%s' % filename, 'w') as f: + f.write(payload) + + # TODO(sdake) hopefully temporary until users move to heat-cfntools-1.3 + with open('/var/lib/cloud/data/%s' % filename, 'w') as f: + f.write(payload) + +--===============1598784645116016685== +Content-Type: text/x-cfninitdata; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="cfn-userdata" + +#ps1 +Get-Date + +--===============1598784645116016685== +Content-Type: text/x-shellscript; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="loguserdata.py" + +#!/usr/bin/env python + +import sys +import os +import subprocess +import datetime +import pkg_resources +from distutils.version import LooseVersion +import errno + +path = '/var/lib/heat-cfntools' + + +def chk_ci_version(): + v = LooseVersion(pkg_resources.get_distribution('cloud-init').version) + return v >= LooseVersion('0.6.0') + + +def create_log(path): + fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0600) + return os.fdopen(fd, 'w') + + +def call(args, log): + log.write('%s\n' % ' '.join(args)) + log.flush() + p = subprocess.Popen(args, stdout=log, stderr=log) + p.wait() + return p.returncode + + +def main(log): + + if not chk_ci_version(): + # pre 0.6.0 - user data executed via cloudinit, not this helper + log.write('Unable to log provisioning, need a newer version of' + ' cloud-init\n') + return -1 + + userdata_path = os.path.join(path, 'cfn-userdata') + os.chmod(userdata_path, 0700) + + log.write('Provision began: %s\n' % datetime.datetime.now()) + log.flush() + returncode = call([userdata_path], log) + log.write('Provision done: %s\n' % datetime.datetime.now()) + if returncode: + return returncode + + +if __name__ == '__main__': + with create_log('/var/log/heat-provision.log') as log: + returncode = main(log) + if returncode: + log.write('Provision failed') + sys.exit(returncode) + + userdata_path = os.path.join(path, 'provision-finished') + with create_log(userdata_path) as log: + log.write('%s\n' % datetime.datetime.now()) + +--===============1598784645116016685== +Content-Type: text/x-cfninitdata; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="cfn-watch-server" + +http://67.207.197.36:8003 +--===============1598784645116016685== +Content-Type: text/x-cfninitdata; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="cfn-metadata-server" + +http://67.207.197.36:8000 +--===============1598784645116016685== +Content-Type: text/x-cfninitdata; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="cfn-boto-cfg" + +[Boto] +debug = 0 +is_secure = 0 +https_validate_certificates = 1 +cfn_region_name = heat +cfn_region_endpoint = 67.207.197.36 +cloudwatch_region_name = heat +cloudwatch_region_endpoint = 67.207.197.36 +--===============1598784645116016685==-- +