From 63bd68cc2a87e961874eabf240be58912d6ec23a Mon Sep 17 00:00:00 2001 From: efedorova Date: Tue, 8 Oct 2013 20:09:29 +0400 Subject: [PATCH] Add initial Archiver --- Services/ad-manifest.yaml | 2 +- archiver.py | 91 +++++++++++++++++++++++++++++++++++++++ cmd/__init__.py | 13 ++++++ cmd/run.py | 28 ++++++++++++ consts.py | 19 ++++++++ manifest.py | 17 +++++++- parser.py | 51 ++++++++++++++-------- 7 files changed, 200 insertions(+), 21 deletions(-) create mode 100644 archiver.py create mode 100644 cmd/__init__.py create mode 100644 cmd/run.py create mode 100644 consts.py diff --git a/Services/ad-manifest.yaml b/Services/ad-manifest.yaml index fe3fd15..5eec1c1 100644 --- a/Services/ad-manifest.yaml +++ b/Services/ad-manifest.yaml @@ -19,7 +19,7 @@ workflows: - Common.xml heat_templates: - Windows.template + - Windows.template agent_templates: - SetPassword.template diff --git a/archiver.py b/archiver.py new file mode 100644 index 0000000..fba6868 --- /dev/null +++ b/archiver.py @@ -0,0 +1,91 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import os +import tarfile +import tempfile +import shutil +import logging as log + +from consts import DATA_TYPES + + +class Archiver(object): + def __init__(self, + ui_forms_target="service_forms", + workflows_target="workflows", + heat_templates_target="templates/cf", + agent_templates_target="templates/agent", + scripts_target="templates/agent/scripts"): + """ + ui_forms_target -- relative path for a ui_forms location in + resulted archive + workflows_target -- relative path for a desired workflow location in + resulted archive + heat_templates_target -- relative path for a desired heat templates + location in resulted archive + agent_templates_target -- relative path for a heat templates location + scripts_target -- relative path for a agent script location + """ + self.archive_structure = {"ui_forms": ui_forms_target, + "workflows": workflows_target, + "heat_templates": heat_templates_target, + "agent_templates": agent_templates_target, + "scripts": scripts_target + } + + def create(self, manifests, *types): + """ + manifests -- list of Manifest objects + *types - desired data types to be added to archive + """ + temp_dir = tempfile.mkdtemp() + for data_type in types: + if data_type not in DATA_TYPES: + raise Exception("Please, specify one of the supported data " + "types: {0}".format(DATA_TYPES)) + + for manifest in manifests: + if not manifest.enabled and not manifest.valid: + continue + if hasattr(manifest, data_type): + dst_directory = os.path.join(temp_dir, + self.archive_structure[ + data_type]) + if not os.path.exists(dst_directory): + os.makedirs(dst_directory) + + for file_path in getattr(manifest, data_type): + basedir, filename = os.path.split(file_path) + destination = os.path.join(dst_directory, + filename) + try: + shutil.copyfile(file_path, destination) + except IOError: + log.error("Unable to copy file " + "{0}".format(file)) + else: + log.info( + "Manifest for {0} service has no file definitions for " + "{1}").format(manifest.service_display_name, data_type) + + with tarfile.open("sample.tar", "w") as tar: + tar.add(temp_dir) + try: + shutil.rmtree(temp_dir, ignore_errors=True) + except Exception as e: + log.error("Unable to delete temp directory: {0}".format(e)) + + + + diff --git a/cmd/__init__.py b/cmd/__init__.py new file mode 100644 index 0000000..207fa15 --- /dev/null +++ b/cmd/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. \ No newline at end of file diff --git a/cmd/run.py b/cmd/run.py new file mode 100644 index 0000000..6e8301c --- /dev/null +++ b/cmd/run.py @@ -0,0 +1,28 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +from parser import ManifestParser +from archiver import Archiver + + +def main(): + parser = ManifestParser(os.path.join(os.path.dirname(__file__), os.pardir, + 'Services')) + manifests = parser.parse() + Archiver().create(manifests, "ui_forms", "heat_templates", "agent_templates") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/consts.py b/consts.py new file mode 100644 index 0000000..c744aee --- /dev/null +++ b/consts.py @@ -0,0 +1,19 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +UI_FORMS = "ui_forms" +WORKFLOWS = "workflows" +HEAT_TEMPLATES = "heat_templates" +AGENT_TEMPLATES = "agent_templates" +SCRIPTS = "scripts" +DATA_TYPES = [UI_FORMS, WORKFLOWS, HEAT_TEMPLATES, AGENT_TEMPLATES, SCRIPTS] \ No newline at end of file diff --git a/manifest.py b/manifest.py index d95c28d..a6410f3 100644 --- a/manifest.py +++ b/manifest.py @@ -1,6 +1,19 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + class Manifest(object): def __init__(self, initial_data): for key in initial_data: - setattr(self, key, initial_data[key]) - self.valid = True + setattr(self, key, initial_data[key]) \ No newline at end of file diff --git a/parser.py b/parser.py index 143c180..37c6d1f 100644 --- a/parser.py +++ b/parser.py @@ -1,7 +1,20 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import os import yaml import logging as log - from manifest import Manifest @@ -17,9 +30,11 @@ class ManifestParser(object): """ manifest_directory -- absolute path to the directory with manifests ui_forms_directory -- absolute or relative path to ui forms definitions - workflows_directory -- absolute or relative path to workflow definitions + workflows_directory -- absolute or relative path to workflow + definitions heat_templates_directory -- absolute or relative path to heat templates - agent_templates_directory --absolute or relative path to agent templates + agent_templates_directory --absolute or relative path to agent + templates scripts_directory -- absolute or relative path to scripts """ if not os.path.isabs(ui_forms_directory): @@ -41,7 +56,7 @@ class ManifestParser(object): self.manifest_directory = manifest_directory self.directory_mapping = {"ui_forms": ui_forms_directory, "workflows": workflows_directory, - "heat_templates_directory": + "heat_templates": heat_templates_directory, "agent_templates": agent_templates_directory, "scripts": scripts_directory @@ -60,21 +75,29 @@ class ManifestParser(object): try: with open(manifest_file) as stream: - service_manifest_descr = yaml.load(stream) + service_manifest_data = yaml.load(stream) except yaml.YAMLError, exc: log.warn("Failed to load manifest file. {0}. " "The reason: {1!s}".format(manifest_file, exc)) continue - for key, value in service_manifest_descr.iteritems(): - valid_file_info = True + valid_file_info = True + for key, value in service_manifest_data.iteritems(): directory_location = self.directory_mapping.get(key) if directory_location: + if not isinstance(value, list): + log.error("{0} section should represent a file" + " listing in manifest {1}" + "".format(directory_location, file)) + valid_file_info = False + continue for i, filename in enumerate(value): absolute_path = os.path.join(directory_location, filename) - service_manifest_descr[key][i] = absolute_path + + service_manifest_data[key][i] = absolute_path + if not os.path.exists(absolute_path): valid_file_info = False log.warning( @@ -83,15 +106,7 @@ class ManifestParser(object): file, absolute_path )) - service_manifest_descr["valid"] = valid_file_info + service_manifest_data["valid"] = valid_file_info - manifests.append(Manifest(service_manifest_descr)) + manifests.append(Manifest(service_manifest_data)) return manifests - - -def main(): - ManifestParser(os.path.join(os.path.dirname(__file__), 'Services')).parse() - - -if __name__ == "__main__": - main() \ No newline at end of file