x/cloudbase-init
Code Issues Proposed changes

Merge pull request #12 from gokrokvertskhov/rebase#07222012

Browse Source 
Multipart data handling - new request for rebased code
This commit is contained in:
Alessandro Pilotti 2013-08-12 15:46:04 -07:00
commit 3c8cd5b39f
7 changed files with 622 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
cloudbaseinit/plugins/windows/userdata-plugins/cloudconfig.py Normal file
View File

@ -0,0 +1,34 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cloudbaseinit.openstack.common import log as logging
LOG = logging.getLogger("cloudbaseinit")
def get_plugin(parent_set):
return CloudConfigHandler(parent_set)
class CloudConfigHandler:
def __init__(self, parent_set):
LOG.info("Cloud-config part handler is loaded.")
self.type = "text/cloud-config"
self.name = "Cloud-config userdata plugin"
return
def process(self, part):
return

47
cloudbaseinit/plugins/windows/userdata-plugins/heathandler.py Normal file
View File

@ -0,0 +1,47 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
import tempfile
import uuid
import email
import tempfile
import os
import errno
from cloudbaseinit.openstack.common import log as logging
from cloudbaseinit.osutils.factory import *
from cloudbaseinit.plugins.windows.userdata import handle
LOG = logging.getLogger("cloudbaseinit")
def get_plugin(parent_set):
return HeatUserDataHandler(parent_set)
class HeatUserDataHandler:
def __init__(self, parent_set):
LOG.info("Heat user data part handler is loaded.")
self.type = "text/x-cfninitdata"
self.name = "Heat userdata plugin"
return
def process(self, part):
#Only user-data part of Heat multipart data is supported. All other cfinitdata part will be skipped
if part.get_filename() == "cfn-userdata":
handle(part.get_payload())
return

67
cloudbaseinit/plugins/windows/userdata-plugins/parthandler.py Normal file
View File

@ -0,0 +1,67 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import imp
import os
from cloudbaseinit.openstack.common import log as logging
LOG = logging.getLogger("cloudbaseinit")
def get_plugin(parent_set):
return PartHandlerScriptHandler(parent_set)
def load_from_file(filepath, function):
class_inst = None
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
if file_ext.lower() == '.py':
py_mod = imp.load_source(mod_name, filepath)
elif file_ext.lower() == '.pyc':
py_mod = imp.load_compiled(mod_name, filepath)
if hasattr(py_mod, function):
callable = getattr(__import__(mod_name),function)
return callable
class PartHandlerScriptHandler:
def __init__(self, parent_set):
LOG.info("Part-handler script part handler is loaded.")
self.type = "text/part-handler"
self.name = "Part-handler userdata plugin"
self.parent_set = parent_set
return
def process(self, part):
handler_path = self.parent_set.path + "/part-handler/"+part.get_filename()
with open(handler_path, "wb") as f:
f.write(part.get_payload())
list_types = load_from_file(handler_path,"list_types")
handle_part = load_from_file(handler_path, "handle_part")
if list_types is not None and handle_part is not None:
parts = list_types()
for part in parts:
LOG.info("Installing new custom handler for type: %s", part)
self.parent_set.custom_handlers[part] = handle_part
self.parent_set.has_custom_handlers = True
return

78
cloudbaseinit/plugins/windows/userdata-plugins/shellscript.py Normal file
View File

@ -0,0 +1,78 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
import tempfile
import uuid
import email
import tempfile
import os
import errno
from cloudbaseinit.openstack.common import log as logging
from cloudbaseinit.osutils.factory import *
LOG = logging.getLogger("cloudbaseinit")
def get_plugin(parent_set):
return ShellScriptHandler(parent_set)
class ShellScriptHandler:
def __init__(self, parent_set):
LOG.info("Shell-script part handler is loaded.")
self.type = "text/x-shellscript"
self.name = "Shell-script userdata plugin"
return
def process(self, part):
osutils = OSUtilsFactory().get_os_utils()
file_name = part.get_filename()
target_path = os.path.join(tempfile.gettempdir(), file_name)
if file_name.endswith(".cmd"):
args = [target_path]
shell = True
elif file_name.endswith(".sh"):
args = ['bash.exe', target_path]
shell = False
elif file_name.endswith(".ps1"):
args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned',
'-NonInteractive', target_path]
shell = False
else:
# Unsupported
LOG.warning('Unsupported shell format')
return False
try:
with open(target_path, 'wb') as f:
f.write(part.get_payload())
(out, err, ret_val) = osutils.execute_process(args, shell)
LOG.info('User_data script ended with return code: %d' % ret_val)
LOG.debug('User_data stdout:\n%s' % out)
LOG.debug('User_data stderr:\n%s' % err)
except Exception, ex:
LOG.warning('An error occurred during user_data execution: \'%s\'' % ex)
finally:
if os.path.exists(target_path):
os.remove(target_path)
return False

196
cloudbaseinit/plugins/windows/userdata.py
View File

@ -18,16 +18,41 @@ import os
import re
import tempfile
import uuid
import email
import errno
from cloudbaseinit.metadata.services import base as metadata_services_base
from cloudbaseinit.openstack.common import log as logging
from cloudbaseinit.osutils import factory as osutils_factory
from cloudbaseinit.plugins import base
from cloudbaseinit.plugins.windows.userdata_plugins import PluginSet
from cloudbaseinit.openstack.common import cfg
opts = [
cfg.StrOpt('user_data_folder', default='cloud-data',
help='Specifies a folder to store multipart data files.'),
]
CONF = cfg.CONF
CONF.register_opts(opts)
LOG = logging.getLogger(__name__)
class UserDataPlugin(base.BasePlugin):
class UserDataPlugin(base.BasePlugin):
def __init__(self, cfg=CONF):
self.cfg = cfg
self.msg = None
self.plugin_set = PluginSet(self.get_plugin_path())
self.plugin_set.reload()
return
def get_plugin_path(self):
return os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
"windows/userdata-plugins")
def execute(self, service):
try:
user_data = service.get_user_data('openstack')
@ -37,57 +62,120 @@ class UserDataPlugin(base.BasePlugin):
if not user_data:
return (base.PLUGIN_EXECUTION_DONE, False)
LOG.debug('User data content:\n%s' % user_data)
osutils = osutils_factory.OSUtilsFactory().get_os_utils()
target_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
if re.search(r'^rem cmd\s', user_data, re.I):
target_path += '.cmd'
args = [target_path]
shell = True
elif re.search(r'^#!', user_data, re.I):
target_path += '.sh'
args = ['bash.exe', target_path]
shell = False
elif re.search(r'^#ps1\s', user_data, re.I):
target_path += '.ps1'
args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned',
'-NonInteractive', target_path]
shell = False
elif re.search(r'^#ps1_sysnative\s', user_data, re.I):
if os.path.isdir(os.path.expandvars('%windir%\\sysnative')):
target_path += '.ps1'
args = [os.path.expandvars('%windir%\\sysnative\\'
'WindowsPowerShell\\v1.0\\'
'powershell.exe'),
'-ExecutionPolicy',
'RemoteSigned', '-NonInteractive', target_path]
shell = False
else:
# Unable to validate sysnative presence
LOG.warning('Unable to validate sysnative folder presence. '
'If Target OS is Server 2003, please ensure you '
'have KB942589 installed')
return (base.PLUGIN_EXECUTION_DONE, False)
else:
# Unsupported
LOG.warning('Unsupported user_data format')
return (base.PLUGIN_EXECUTION_DONE, False)
try:
with open(target_path, 'wb') as f:
f.write(user_data)
(out, err, ret_val) = osutils.execute_process(args, shell)
LOG.info('User_data script ended with return code: %d' % ret_val)
LOG.debug('User_data stdout:\n%s' % out)
LOG.debug('User_data stderr:\n%s' % err)
except Exception, ex:
LOG.warning('An error occurred during user_data execution: \'%s\''
% ex)
finally:
if os.path.exists(target_path):
os.remove(target_path)
self.process_userdata(user_data)
return (base.PLUGIN_EXECUTION_DONE, False)
def process_userdata(self, user_data):
LOG.debug('User data content:\n%s' % user_data)
if user_data.startswith('Content-Type: multipart'):
for part in self.parse_MIME(user_data):
self.process_part(part)
else:
handle(user_data)
return
def process_part(self, part):
part_handler = self.get_part_handler(part)
if part_handler is not None:
try:
self.begin_part_process_event(part)
LOG.info("Processing part %s filename: %s with handler: %s", part.get_content_type(), part.get_filename(), part_handler.name)
part_handler.process(part)
self.end_part_process_event(part)
except Exception,e:
LOG.error('Exception during multipart part handling: %s %s \n %s' , part.get_content_type(), part.get_filename(), e)
return
def begin_part_process_event(self, part):
handler = self.get_custom_handler(part)
if handler is not None:
try:
handler("","__begin__", part.get_filename(), part.get_payload())
except Exception,e:
LOG.error("Exception occurred during custom handle script invocation (__begin__): %s ", e)
return
def end_part_process_event(self, part):
handler = self.get_custom_handler(part)
if handler is not None:
try:
handler("","__end__", part.get_filename(), part.get_payload())
except Exception,e:
LOG.error("Exception occurred during custom handle script invocation (__end__): %s ", e)
return
def get_custom_handler(self, part):
if self.plugin_set.has_custom_handlers:
if part.get_content_type() in self.plugin_set.custom_handlers:
handler = self.plugin_set.custom_handlers[part.get_content_type()]
return handler
return None
def get_part_handler(self, part):
if part.get_content_type() in self.plugin_set.set:
handler = self.plugin_set.set[part.get_content_type()]
return handler
else:
return None
def parse_MIME(self, user_data):
self.msg = email.message_from_string(user_data)
return self.msg.walk()
def handle(self, user_data):
osutils = osutils_factory.OSUtilsFactory().get_os_utils()
target_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
if re.search(r'^rem cmd\s', user_data, re.I):
target_path += '.cmd'
args = [target_path]
shell = True
elif re.search(r'^#!', user_data, re.I):
target_path += '.sh'
args = ['bash.exe', target_path]
shell = False
elif re.search(r'^#ps1\s', user_data, re.I):
target_path += '.ps1'
args = ['powershell.exe', '-ExecutionPolicy', 'RemoteSigned',
'-NonInteractive', target_path]
shell = False
elif re.search(r'^#ps1_sysnative\s', user_data, re.I):
if os.path.isdir(os.path.expandvars('%windir%\\sysnative')):
target_path += '.ps1'
args = [os.path.expandvars('%windir%\\sysnative\\'
'WindowsPowerShell\\v1.0\\'
'powershell.exe'),
'-ExecutionPolicy',
'RemoteSigned', '-NonInteractive', target_path]
shell = False
else:
# Unable to validate sysnative presence
LOG.warning('Unable to validate sysnative folder presence. '
'If Target OS is Server 2003, please ensure you '
'have KB942589 installed')
return (base.PLUGIN_EXECUTION_DONE, False)
else:
# Unsupported
LOG.warning('Unsupported user_data format')
return (base.PLUGIN_EXECUTION_DONE, False)
try:
with open(target_path, 'wb') as f:
f.write(user_data)
(out, err, ret_val) = osutils.execute_process(args, shell)
LOG.info('User_data script ended with return code: %d' % ret_val)
LOG.debug('User_data stdout:\n%s' % out)
LOG.debug('User_data stderr:\n%s' % err)
except Exception, ex:
LOG.warning('An error occurred during user_data execution: \'%s\''
% ex)
finally:
if os.path.exists(target_path):
os.remove(target_path)
return (base.PLUGIN_EXECUTION_DONE, False)

81
cloudbaseinit/plugins/windows/userdata_plugins.py Normal file
View File

@ -0,0 +1,81 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import imp
import os
import sys, glob
from cloudbaseinit.openstack.common import log as logging
import traceback
LOG = logging.getLogger(__name__)
global builders
def load_from_file(filepath, parent_set):
class_inst = None
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
if file_ext.lower() == '.py':
py_mod = imp.load_source(mod_name, filepath)
elif file_ext.lower() == '.pyc':
py_mod = imp.load_compiled(mod_name, filepath)
if hasattr(py_mod, "get_plugin"):
callable = getattr(__import__(mod_name), "get_plugin")
class_inst = callable(parent_set)
return class_inst
class PluginSet:
def __init__(self, path):
self.path = path
sys.path.append(self.path)
self.set = {}
self.has_custom_handlers = False
self.custom_handlers = {}
def get_plugin(self, content_type, file_name):
return
def load(self):
files = glob.glob(self.path + '/*.py')
if len(files) == 0:
LOG.debug("No user data plug-ins found in %s:", self.path)
return
for file in files:
LOG.debug("Trying to load user data plug-in from file: %s", file)
try:
plugin = load_from_file(file, self)
if plugin is not None:
LOG.info("Plugin '%s' loaded.", plugin.name)
self.set[plugin.type] = plugin
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOG.error('Can`t load plugin from the file %s. Skip it.', file)
LOG.debug(repr(traceback.format_exception(exc_type, exc_value,
exc_traceback)))
def reload(self):
self.set = {}
self.load()

173
cloudbaseinit/test/test.mime Normal file
View File

@ -0,0 +1,173 @@
Content-Type: multipart/mixed; boundary="===============1598784645116016685=="
MIME-Version: 1.0
--===============1598784645116016685==
Content-Type: text/cloud-config; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cloud-config"
runcmd:
- setenforce 0 > /dev/null 2>&1 || true
user: ec2-user
cloud_config_modules:
- locale
- set_hostname
- ssh
- timezone
- update_etc_hosts
- update_hostname
- runcmd
# Capture all subprocess output into a logfile
# Useful for troubleshooting cloud-init issues
output: {all: '| tee -a /var/log/cloud-init-output.log'}
--===============1598784645116016685==
Content-Type: text/part-handler; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="part-handler.py"
#part-handler
import os
import datetime
def list_types():
return(["text/x-cfninitdata"])
def handle_part(data, ctype, filename, payload):
if ctype == "__begin__":
try:
os.makedirs('/var/lib/heat-cfntools', 0700)
except OSError as e:
if e.errno != errno.EEXIST:
raise
return
if ctype == "__end__":
return
with open('/var/log/part-handler.log', 'a') as log:
timestamp = datetime.datetime.now()
log.write('%s filename:%s, ctype:%s\n' % (timestamp, filename, ctype))
if ctype == 'text/x-cfninitdata':
with open('/var/lib/heat-cfntools/%s' % filename, 'w') as f:
f.write(payload)
# TODO(sdake) hopefully temporary until users move to heat-cfntools-1.3
with open('/var/lib/cloud/data/%s' % filename, 'w') as f:
f.write(payload)
--===============1598784645116016685==
Content-Type: text/x-cfninitdata; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cfn-userdata"
#ps1
Get-Date
--===============1598784645116016685==
Content-Type: text/x-shellscript; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="loguserdata.py"
#!/usr/bin/env python
import sys
import os
import subprocess
import datetime
import pkg_resources
from distutils.version import LooseVersion
import errno
path = '/var/lib/heat-cfntools'
def chk_ci_version():
v = LooseVersion(pkg_resources.get_distribution('cloud-init').version)
return v >= LooseVersion('0.6.0')
def create_log(path):
fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0600)
return os.fdopen(fd, 'w')
def call(args, log):
log.write('%s\n' % ' '.join(args))
log.flush()
p = subprocess.Popen(args, stdout=log, stderr=log)
p.wait()
return p.returncode
def main(log):
if not chk_ci_version():
# pre 0.6.0 - user data executed via cloudinit, not this helper
log.write('Unable to log provisioning, need a newer version of'
' cloud-init\n')
return -1
userdata_path = os.path.join(path, 'cfn-userdata')
os.chmod(userdata_path, 0700)
log.write('Provision began: %s\n' % datetime.datetime.now())
log.flush()
returncode = call([userdata_path], log)
log.write('Provision done: %s\n' % datetime.datetime.now())
if returncode:
return returncode
if __name__ == '__main__':
with create_log('/var/log/heat-provision.log') as log:
returncode = main(log)
if returncode:
log.write('Provision failed')
sys.exit(returncode)
userdata_path = os.path.join(path, 'provision-finished')
with create_log(userdata_path) as log:
log.write('%s\n' % datetime.datetime.now())
--===============1598784645116016685==
Content-Type: text/x-cfninitdata; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cfn-watch-server"
http://67.207.197.36:8003
--===============1598784645116016685==
Content-Type: text/x-cfninitdata; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cfn-metadata-server"
http://67.207.197.36:8000
--===============1598784645116016685==
Content-Type: text/x-cfninitdata; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cfn-boto-cfg"
[Boto]
debug = 0
is_secure = 0
https_validate_certificates = 1
cfn_region_name = heat
cfn_region_endpoint = 67.207.197.36
cloudwatch_region_name = heat
cloudwatch_region_endpoint = 67.207.197.36
--===============1598784645116016685==--