From 9ed4705fd64f4d320a4c051c14f169b99f84ec8b Mon Sep 17 00:00:00 2001 From: Cosmin Poieana Date: Mon, 10 Aug 2015 19:38:51 +0300 Subject: [PATCH] Ironic config drive support Search through partitions (containing raw ISO bytes) and volumes when looking for a configuration drive. This commit implies the following: 1. New config options are used for choosing the possible config drive paths (`config_drive_locations`) and the types the service will search for (`config_drive_types`). The old options are still available and marked as deprecated. 2. The configdrive plugin was intensively refactored and size computation, parsing and ISO extraction bugs were fixed. The plugin will search in locations like cdrom, hard disks or partitions for metadata content or raw ISO bytes. Also, is using the `disk` windows utility for reading disks and listing partitions. 3. A new method, `get_volumes`, was added in osutils for listing all the volumes. 4. Removed dead code virtual_disk.py and disk.py (physical_disk.py) was remade from scratch. 5. The ability to handle partitions within a disk for reading purposes and related bugs fixed: a. Wrong INVALID_HANDLE_VALUE (-1 in Python isn't the unsigned -1 of C) b. Erroneous geometry computations in Py3 ("/" lead to float) c. Comparing string with bytes in Py3 d. High risk of IndexErrors because of the insufficient buffer reads relying on standard block sector sizes. Change-Id: Ic3a5ef1ee81c694e41fc7a22abe63b0154f51065 --- .../metadata/services/configdrive.py | 85 ++- .../metadata/services/osconfigdrive/base.py | 7 +- .../services/osconfigdrive/windows.py | 282 ++++---- cloudbaseinit/osutils/windows.py | 67 +- .../services/osconfigdrive/test_windows.py | 680 +++++++++--------- .../metadata/services/test_configdrive.py | 85 ++- cloudbaseinit/tests/osutils/test_windows.py | 118 ++- .../tests/utils/windows/test_disk.py | 362 ++++++---- .../tests/utils/windows/test_virtual_disk.py | 263 ------- cloudbaseinit/utils/windows/disk.py | 335 +++++++-- cloudbaseinit/utils/windows/virtual_disk.py | 151 ---- 11 files changed, 1305 insertions(+), 1130 deletions(-) delete mode 100644 cloudbaseinit/tests/utils/windows/test_virtual_disk.py delete mode 100644 cloudbaseinit/utils/windows/virtual_disk.py diff --git a/cloudbaseinit/metadata/services/configdrive.py b/cloudbaseinit/metadata/services/configdrive.py index 431944f5..84d9ba39 100644 --- a/cloudbaseinit/metadata/services/configdrive.py +++ b/cloudbaseinit/metadata/services/configdrive.py @@ -14,23 +14,47 @@ import os import shutil -import tempfile -import uuid from oslo_config import cfg from oslo_log import log as oslo_logging +from cloudbaseinit import exception from cloudbaseinit.metadata.services import base from cloudbaseinit.metadata.services import baseopenstackservice from cloudbaseinit.metadata.services.osconfigdrive import factory + +# Config Drive types and possible locations. +CD_TYPES = { + "vfat", # Visible device (with partition table). + "iso", # "Raw" format containing ISO bytes. +} +CD_LOCATIONS = { + # Look into optical units devices. Only an ISO format could + # be used here (vfat ignored). + "cdrom", + # Search through physical disks for raw ISO content or vfat filesystems + # containing configuration drive's content. + "hdd", + # Search through partitions for raw ISO content or through volumes + # containing configuration drive's content. + "partition", +} + opts = [ cfg.BoolOpt('config_drive_raw_hhd', default=True, - help='Look for an ISO config drive in raw HDDs'), + help='Look for an ISO config drive in raw HDDs', + deprecated_for_removal=True), cfg.BoolOpt('config_drive_cdrom', default=True, - help='Look for a config drive in the attached cdrom drives'), + help='Look for a config drive in the attached cdrom drives', + deprecated_for_removal=True), cfg.BoolOpt('config_drive_vfat', default=True, - help='Look for a config drive in VFAT filesystems.'), + help='Look for a config drive in VFAT filesystems', + deprecated_for_removal=True), + cfg.ListOpt('config_drive_types', default=list(CD_TYPES), + help='Supported formats of a configuration drive'), + cfg.ListOpt('config_drive_locations', default=list(CD_LOCATIONS), + help='Supported configuration drive locations'), ] CONF = cfg.CONF @@ -45,33 +69,52 @@ class ConfigDriveService(baseopenstackservice.BaseOpenStackService): super(ConfigDriveService, self).__init__() self._metadata_path = None + def _preprocess_options(self): + self._searched_types = set(CONF.config_drive_types) + self._searched_locations = set(CONF.config_drive_locations) + + # Deprecation backward compatibility. + if CONF.config_drive_raw_hhd: + self._searched_types.add("iso") + self._searched_locations.add("hdd") + if CONF.config_drive_cdrom: + self._searched_types.add("iso") + self._searched_locations.add("cdrom") + if CONF.config_drive_vfat: + self._searched_types.add("vfat") + self._searched_locations.add("hdd") + + # Check for invalid option values. + if self._searched_types | CD_TYPES != CD_TYPES: + raise exception.CloudbaseInitException( + "Invalid Config Drive types %s", self._searched_types) + if self._searched_locations | CD_LOCATIONS != CD_LOCATIONS: + raise exception.CloudbaseInitException( + "Invalid Config Drive locations %s", self._searched_locations) + def load(self): super(ConfigDriveService, self).load() - target_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) + self._preprocess_options() + self._mgr = factory.get_config_drive_manager() + found = self._mgr.get_config_drive_files( + searched_types=self._searched_types, + searched_locations=self._searched_locations) - mgr = factory.get_config_drive_manager() - found = mgr.get_config_drive_files( - target_path, - check_raw_hhd=CONF.config_drive_raw_hhd, - check_cdrom=CONF.config_drive_cdrom, - check_vfat=CONF.config_drive_vfat) if found: - self._metadata_path = target_path - LOG.debug('Metadata copied to folder: \'%s\'' % - self._metadata_path) + self._metadata_path = self._mgr.target_path + LOG.debug('Metadata copied to folder: %r', self._metadata_path) return found def _get_data(self, path): norm_path = os.path.normpath(os.path.join(self._metadata_path, path)) try: - with open(norm_path, 'rb') as f: - return f.read() + with open(norm_path, 'rb') as stream: + return stream.read() except IOError: raise base.NotExistingMetadataException() def cleanup(self): - if self._metadata_path: - LOG.debug('Deleting metadata folder: \'%s\'' % self._metadata_path) - shutil.rmtree(self._metadata_path, ignore_errors=True) - self._metadata_path = None + LOG.debug('Deleting metadata folder: %r', self._mgr.target_path) + shutil.rmtree(self._mgr.target_path, ignore_errors=True) + self._metadata_path = None diff --git a/cloudbaseinit/metadata/services/osconfigdrive/base.py b/cloudbaseinit/metadata/services/osconfigdrive/base.py index 85685eed..6f1e7d91 100644 --- a/cloudbaseinit/metadata/services/osconfigdrive/base.py +++ b/cloudbaseinit/metadata/services/osconfigdrive/base.py @@ -13,6 +13,7 @@ # under the License. import abc +import tempfile import six @@ -20,7 +21,9 @@ import six @six.add_metaclass(abc.ABCMeta) class BaseConfigDriveManager(object): + def __init__(self): + self.target_path = tempfile.mkdtemp() + @abc.abstractmethod - def get_config_drive_files(self, target_path, check_raw_hhd=True, - check_cdrom=True, check_vfat=True): + def get_config_drive_files(self, check_types=None, check_locations=None): pass diff --git a/cloudbaseinit/metadata/services/osconfigdrive/windows.py b/cloudbaseinit/metadata/services/osconfigdrive/windows.py index b91d7856..80124421 100644 --- a/cloudbaseinit/metadata/services/osconfigdrive/windows.py +++ b/cloudbaseinit/metadata/services/osconfigdrive/windows.py @@ -12,13 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. -import ctypes + +import itertools import os import shutil +import struct import tempfile import uuid -from ctypes import wintypes from oslo_config import cfg from oslo_log import log as oslo_logging @@ -28,6 +29,7 @@ from cloudbaseinit.osutils import factory as osutils_factory from cloudbaseinit.utils.windows import disk from cloudbaseinit.utils.windows import vfat + opts = [ cfg.StrOpt('bsdtar_path', default='bsdtar.exe', help='Path to "bsdtar", used to extract ISO ConfigDrive ' @@ -39,83 +41,73 @@ CONF.register_opts(opts) LOG = oslo_logging.getLogger(__name__) +CONFIG_DRIVE_LABEL = 'config-2' +MAX_SECTOR_SIZE = 4096 +# Absolute offset values and the ISO magic string. +OFFSET_BOOT_RECORD = 0x8000 +OFFSET_ISO_ID = OFFSET_BOOT_RECORD + 1 +ISO_ID = b'CD001' +# Little-endian unsigned short size values. +OFFSET_VOLUME_SIZE = OFFSET_BOOT_RECORD + 80 +OFFSET_BLOCK_SIZE = OFFSET_BOOT_RECORD + 128 +PEEK_SIZE = 2 + class WindowsConfigDriveManager(base.BaseConfigDriveManager): - def _get_config_drive_cdrom_mount_point(self): - osutils = osutils_factory.get_os_utils() + def __init__(self): + super(WindowsConfigDriveManager, self).__init__() + self._osutils = osutils_factory.get_os_utils() - for drive in osutils.get_cdrom_drives(): - label = osutils.get_volume_label(drive) - if label == "config-2" and \ - os.path.exists(os.path.join(drive, - 'openstack\\latest\\' - 'meta_data.json')): - return drive - return None + def _check_for_config_drive(self, drive): + label = self._osutils.get_volume_label(drive) + if label and label.lower() == CONFIG_DRIVE_LABEL and \ + os.path.exists(os.path.join(drive, + 'openstack\\latest\\' + 'meta_data.json')): + LOG.info('Config Drive found on %s', drive) + return True + return False - def _c_char_array_to_c_ushort(self, buf, offset): - low = ctypes.cast(buf[offset], - ctypes.POINTER(wintypes.WORD)).contents - high = ctypes.cast(buf[offset + 1], - ctypes.POINTER(wintypes.WORD)).contents - return (high.value << 8) + low.value - - def _get_iso_disk_size(self, phys_disk): - geom = phys_disk.get_geometry() - - if geom.MediaType != disk.Win32_DiskGeometry.FixedMedia: + def _get_iso_file_size(self, device): + if not device.fixed: return None - disk_size = geom.Cylinders * geom.TracksPerCylinder * \ - geom.SectorsPerTrack * geom.BytesPerSector - - boot_record_off = 0x8000 - id_off = 1 - volume_size_off = 80 - block_size_off = 128 - iso_id = b'CD001' - - offset = boot_record_off // geom.BytesPerSector * geom.BytesPerSector - bytes_to_read = geom.BytesPerSector - - if disk_size <= offset + bytes_to_read: + if not device.size > (OFFSET_BLOCK_SIZE + PEEK_SIZE): return None - phys_disk.seek(offset) - (buf, bytes_read) = phys_disk.read(bytes_to_read) - - buf_off = boot_record_off - offset + id_off - if iso_id != buf[buf_off: buf_off + len(iso_id)]: + off = device.seek(OFFSET_ISO_ID) + magic = device.read(len(ISO_ID), skip=OFFSET_ISO_ID - off) + if ISO_ID != magic: return None - buf_off = boot_record_off - offset + volume_size_off - num_blocks = self._c_char_array_to_c_ushort(buf, buf_off) + off = device.seek(OFFSET_VOLUME_SIZE) + volume_size_bytes = device.read(PEEK_SIZE, + skip=OFFSET_VOLUME_SIZE - off) + off = device.seek(OFFSET_BLOCK_SIZE) + block_size_bytes = device.read(PEEK_SIZE, + skip=OFFSET_BLOCK_SIZE - off) + volume_size = struct.unpack("".format(self.__class__.__name__, self._path) + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def _get_geometry(self): + """Get details about the disk size bounds.""" + geom = Win32_DiskGeometry() + bytes_returned = wintypes.DWORD() + ret_val = kernel32.DeviceIoControl( + self._handle, + winioctlcon.IOCTL_DISK_GET_DRIVE_GEOMETRY, + 0, + 0, + ctypes.byref(geom), + ctypes.sizeof(geom), + ctypes.byref(bytes_returned), + 0) + + if not ret_val: + raise exception.WindowsCloudbaseInitException( + "Cannot get disk geometry: %r") + + _sector_size = geom.BytesPerSector + _disk_size = (geom.Cylinders * geom.TracksPerCylinder * + geom.SectorsPerTrack * geom.BytesPerSector) + fixed = geom.MediaType == Win32_DiskGeometry.FixedMedia + return _sector_size, _disk_size, fixed + + def _seek(self, offset): + high = wintypes.DWORD(offset >> 32) + low = wintypes.DWORD(offset & 0xFFFFFFFF) + + ret_val = kernel32.SetFilePointer(self._handle, low, + ctypes.byref(high), + self.FILE_BEGIN) + if ret_val == self.INVALID_SET_FILE_POINTER: + raise exception.WindowsCloudbaseInitException( + "Seek error: %r") + + def _read(self, size): + buff = ctypes.create_string_buffer(size) + bytes_read = wintypes.DWORD() + ret_val = kernel32.ReadFile(self._handle, buff, size, + ctypes.byref(bytes_read), 0) + if not ret_val: + raise exception.WindowsCloudbaseInitException( + "Read exception: %r") + return buff.raw[:bytes_read.value] # all bytes without the null byte def open(self): - if self._handle: - self.close() - handle = kernel32.CreateFileW( ctypes.c_wchar_p(self._path), self.GENERIC_READ, @@ -62,49 +233,119 @@ class PhysicalDisk(object): self.FILE_ATTRIBUTE_READONLY, 0) if handle == self.INVALID_HANDLE_VALUE: - raise exception.CloudbaseInitException('Cannot open file') + raise exception.WindowsCloudbaseInitException( + 'Cannot open file: %r') self._handle = handle + self._sector_size, self._disk_size, self.fixed =\ + self._get_geometry() def close(self): - kernel32.CloseHandle(self._handle) - self._handle = 0 - self._geom = None - - def get_geometry(self): - if not self._geom: - geom = Win32_DiskGeometry() - bytes_returned = wintypes.DWORD() - ret_val = kernel32.DeviceIoControl( - self._handle, - self.IOCTL_DISK_GET_DRIVE_GEOMETRY, - 0, - 0, - ctypes.byref(geom), - ctypes.sizeof(geom), - ctypes.byref(bytes_returned), - 0) - if not ret_val: - raise exception.WindowsCloudbaseInitException( - "Cannot get disk geometry: %r") - self._geom = geom - return self._geom + if self._handle: + kernel32.CloseHandle(self._handle) + self._handle = None def seek(self, offset): - high = wintypes.DWORD(offset >> 32) - low = wintypes.DWORD(offset & 0xFFFFFFFF) + """Drive geometry safe seek. - ret_val = kernel32.SetFilePointer(self._handle, low, - ctypes.byref(high), - self.FILE_BEGIN) - if ret_val == self.INVALID_SET_FILE_POINTER: - raise exception.CloudbaseInitException("Seek error") + Seek for a given offset and return the valid set one. + """ + safe_offset = int(offset / self._sector_size) * self._sector_size + self._seek(safe_offset) + return safe_offset + + def read(self, size, skip=0): + """Drive geometry safe read. + + Read and extract exactly the requested content. + """ + # Compute a size to fit both of the bytes we need to skip and + # also the minimum read size. + total = size + skip + safe_size = ((int(total / self._sector_size) + + bool(total % self._sector_size)) * self._sector_size) + content = self._read(safe_size) + return content[skip:total] + + @abc.abstractmethod + def size(self): + """Returns the size in bytes of the actual opened device.""" + + +class Disk(BaseDevice): + """Disk class with seek/read support. + + It also has the capability of obtaining partition objects. + """ + + PARTITION_ENTRY_UNUSED = 0 + PARTITION_STYLE_MBR = 0 + PARTITION_STYLE_GPT = 1 + + def _get_layout(self): + layout = Win32_DRIVE_LAYOUT_INFORMATION_EX() + bytes_returned = wintypes.DWORD() + ret_val = kernel32.DeviceIoControl( + self._handle, + winioctlcon.IOCTL_DISK_GET_DRIVE_LAYOUT_EX, + 0, + 0, + ctypes.byref(layout), + ctypes.sizeof(layout), + ctypes.byref(bytes_returned), + 0) - def read(self, bytes_to_read): - buf = ctypes.create_string_buffer(bytes_to_read) - bytes_read = wintypes.DWORD() - ret_val = kernel32.ReadFile(self._handle, buf, bytes_to_read, - ctypes.byref(bytes_read), 0) if not ret_val: raise exception.WindowsCloudbaseInitException( - "Read exception: %r") - return buf, bytes_read.value + "Cannot get disk layout: %r") + return layout + + def _get_partition_indexes(self, layout): + partition_style = layout.PartitionStyle + if partition_style not in (self.PARTITION_STYLE_MBR, + self.PARTITION_STYLE_GPT): + raise exception.CloudbaseInitException( + "Invalid partition style %r" % partition_style) + # If is GPT, then the count reflects the actual number of partitions + # but if is MBR, then the number of partitions is a multiple of 4 + # and just the indexes for the used partitions must be saved. + partition_indexes = [] + if partition_style == self.PARTITION_STYLE_GPT: + partition_indexes.extend(range(layout.PartitionCount)) + else: + for idx in range(layout.PartitionCount): + if (layout.PartitionEntry[idx].Mbr.PartitionType != + self.PARTITION_ENTRY_UNUSED): + partition_indexes.append(idx) + return partition_indexes + + def partitions(self): + """Return a list of partition objects available on disk.""" + layout = self._get_layout() + partition_indexes = self._get_partition_indexes(layout) + # Create and return the partition objects containing their sizes. + partitions = [] + disk_index = re.search(r"(disk|drive)(\d+)", self._path, + re.I | re.M).group(2) + for partition_index in partition_indexes: + path = r'\\?\GLOBALROOT\Device\Harddisk{}\Partition{}'.format( + disk_index, partition_index + 1) + size = layout.PartitionEntry[partition_index].PartitionLength + partition = Partition(path, size) + partitions.append(partition) + return partitions + + @property + def size(self): + return self._disk_size + + +class Partition(BaseDevice): + """Partition class with seek/read support.""" + + def __init__(self, path, size): + super(Partition, self).__init__(path) + self._partition_size = size + + @property + def size(self): + return self._partition_size diff --git a/cloudbaseinit/utils/windows/virtual_disk.py b/cloudbaseinit/utils/windows/virtual_disk.py deleted file mode 100644 index df64333d..00000000 --- a/cloudbaseinit/utils/windows/virtual_disk.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2012 Cloudbase Solutions Srl -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import ctypes - -from ctypes import windll -from ctypes import wintypes - -from cloudbaseinit import exception - -kernel32 = windll.kernel32 -# VirtDisk.dll is available starting from Windows Server 2008 R2 / Windows7 -virtdisk = None - - -class Win32_GUID(ctypes.Structure): - _fields_ = [("Data1", wintypes.DWORD), - ("Data2", wintypes.WORD), - ("Data3", wintypes.WORD), - ("Data4", wintypes.BYTE * 8)] - - -def get_WIN32_VIRTUAL_STORAGE_TYPE_VENDOR_MICROSOFT(): - guid = Win32_GUID() - guid.Data1 = 0xec984aec - guid.Data2 = 0xa0f9 - guid.Data3 = 0x47e9 - ByteArray8 = wintypes.BYTE * 8 - guid.Data4 = ByteArray8(0x90, 0x1f, 0x71, 0x41, 0x5a, 0x66, 0x34, 0x5b) - return guid - - -class Win32_VIRTUAL_STORAGE_TYPE(ctypes.Structure): - _fields_ = [ - ('DeviceId', wintypes.DWORD), - ('VendorId', Win32_GUID) - ] - - -class VirtualDisk(object): - VIRTUAL_STORAGE_TYPE_DEVICE_ISO = 1 - VIRTUAL_DISK_ACCESS_ATTACH_RO = 0x10000 - VIRTUAL_DISK_ACCESS_READ = 0xd0000 - OPEN_VIRTUAL_DISK_FLAG_NONE = 0 - DETACH_VIRTUAL_DISK_FLAG_NONE = 0 - ATTACH_VIRTUAL_DISK_FLAG_READ_ONLY = 1 - ATTACH_VIRTUAL_DISK_FLAG_NO_DRIVE_LETTER = 2 - - def __init__(self, path): - self._path = path - self._handle = 0 - - def _load_virtdisk_dll(self): - global virtdisk - if not virtdisk: - virtdisk = windll.virtdisk - - def open(self): - if self._handle: - self.close() - - self._load_virtdisk_dll() - - vst = Win32_VIRTUAL_STORAGE_TYPE() - vst.DeviceId = self.VIRTUAL_STORAGE_TYPE_DEVICE_ISO - vst.VendorId = get_WIN32_VIRTUAL_STORAGE_TYPE_VENDOR_MICROSOFT() - - handle = wintypes.HANDLE() - ret_val = virtdisk.OpenVirtualDisk(ctypes.byref(vst), - ctypes.c_wchar_p(self._path), - self.VIRTUAL_DISK_ACCESS_ATTACH_RO | - self.VIRTUAL_DISK_ACCESS_READ, - self.OPEN_VIRTUAL_DISK_FLAG_NONE, 0, - ctypes.byref(handle)) - if ret_val: - raise exception.WindowsCloudbaseInitException( - "Cannot open virtual disk: %r", ret_val) - self._handle = handle - - def attach(self): - ret_val = virtdisk.AttachVirtualDisk( - self._handle, 0, self.ATTACH_VIRTUAL_DISK_FLAG_READ_ONLY, 0, 0, 0) - if ret_val: - raise exception.WindowsCloudbaseInitException( - "Cannot attach virtual disk: %r", ret_val) - - def detach(self): - ret_val = virtdisk.DetachVirtualDisk( - self._handle, self.DETACH_VIRTUAL_DISK_FLAG_NONE, 0) - if ret_val: - raise exception.WindowsCloudbaseInitException( - "Cannot detach virtual disk: %r", ret_val) - - def get_physical_path(self): - buf = ctypes.create_unicode_buffer(1024) - bufLen = wintypes.DWORD(ctypes.sizeof(buf)) - ret_val = virtdisk.GetVirtualDiskPhysicalPath(self._handle, - ctypes.byref(bufLen), - buf) - if ret_val: - raise exception.WindowsCloudbaseInitException( - "Cannot get virtual disk physical path: %r", ret_val) - return buf.value - - def get_cdrom_drive_mount_point(self): - - mount_point = None - - buf = ctypes.create_unicode_buffer(2048) - buf_len = kernel32.GetLogicalDriveStringsW( - ctypes.sizeof(buf) / ctypes.sizeof(wintypes.WCHAR), buf) - if not buf_len: - raise exception.WindowsCloudbaseInitException( - "Cannot enumerate logical devices: %r") - - cdrom_dev = self.get_physical_path().rsplit('\\')[-1].upper() - - i = 0 - while not mount_point and i < buf_len: - curr_drive = ctypes.wstring_at(ctypes.addressof(buf) + i * - ctypes.sizeof(wintypes.WCHAR))[:-1] - - dev = ctypes.create_unicode_buffer(2048) - ret_val = kernel32.QueryDosDeviceW(curr_drive, dev, - ctypes.sizeof(dev) / - ctypes.sizeof(wintypes.WCHAR)) - if not ret_val: - raise exception.WindowsCloudbaseInitException( - "Cannot query NT device: %r") - - if dev.value.rsplit('\\')[-1].upper() == cdrom_dev: - mount_point = curr_drive - else: - i += len(curr_drive) + 2 - - return mount_point - - def close(self): - kernel32.CloseHandle(self._handle) - self._handle = 0