From 41c4549b5ae602bff42b576e87d6ffefb3015433 Mon Sep 17 00:00:00 2001 From: Zhou Ya Date: Wed, 16 Nov 2016 11:36:49 +0800 Subject: [PATCH] add daisy automatic backup feature Change-Id: Iae95a8df713d6cf117dce27aa32ec2fdbb4ac78e --- code/daisy/daisy/api/v1/backup_restore.py | 108 +++++++++++++++++ code/daisy/daisy/api/v1/router.py | 12 ++ code/daisy/daisy/auto_backup/__init__.py | 0 code/daisy/daisy/auto_backup/manager.py | 123 ++++++++++++++++++++ code/daisy/daisy/cmd/api.py | 0 code/daisy/daisy/cmd/auto_backup.py | 74 ++++++++++++ code/daisy/daisy/cmd/manage.py | 2 + code/daisy/daisy/common/utils.py | 1 - code/daisy/daisy/registry/api/v1/hosts.py | 2 +- code/daisy/etc/daisy-auto-backup.conf | 21 ++++ code/daisy/setup.cfg | 1 + rpm/SPECS/daisy.spec | 10 ++ tools/daisy-utils/daisy-auto-backup.service | 15 +++ tools/setup/common/daisy_common_func.sh | 2 + tools/setup/install/install_interface.sh | 4 + 15 files changed, 373 insertions(+), 2 deletions(-) mode change 100755 => 100644 code/daisy/daisy/api/v1/backup_restore.py create mode 100644 code/daisy/daisy/auto_backup/__init__.py create mode 100644 code/daisy/daisy/auto_backup/manager.py mode change 100755 => 100644 code/daisy/daisy/cmd/api.py create mode 100644 code/daisy/daisy/cmd/auto_backup.py create mode 100644 code/daisy/etc/daisy-auto-backup.conf create mode 100644 tools/daisy-utils/daisy-auto-backup.service diff --git a/code/daisy/daisy/api/v1/backup_restore.py b/code/daisy/daisy/api/v1/backup_restore.py old mode 100755 new mode 100644 index 8d6f557b..c8575f17 --- a/code/daisy/daisy/api/v1/backup_restore.py +++ b/code/daisy/daisy/api/v1/backup_restore.py @@ -16,9 +16,11 @@ """ /hosts endpoint for Daisy v1 API """ +import time import datetime import os import subprocess +import ConfigParser from oslo_log import log as logging from webob.exc import HTTPBadRequest from webob.exc import HTTPForbidden @@ -71,6 +73,8 @@ class Controller(controller.BaseController): self.notifier = notifier.Notifier() registry.configure_registry_client() self.policy = policy.Enforcer() + self.config_path = '/home/daisy_install/daisy.conf' + self.auto_back_path = '/home/daisy_backup/auto/' def _enforce(self, req, action, target=None): """Authorize an action against our policies""" @@ -169,6 +173,87 @@ class Controller(controller.BaseController): daisy_cmn.run_scrip(scripts, msg='Backup file failed!') return {"backup_file": BACK_PATH + backup_file_name} + @utils.mutating + def auto_backup_config_detail(self, req): + """ + Auto backup daisy data.. + + :param req: The WSGI/Webob Request object + + :raises HTTPBadRequest if backup failed + """ + auto_backup_switch = "ON" + save_days = "10" + config = ConfigParser.ConfigParser() + config.read(self.config_path) + if "auto_backup_switch" in dict(config.items('DEFAULT')): + auto_backup_switch = config.get('DEFAULT', 'auto_backup_switch') + if "save_days" in dict(config.items('DEFAULT')): + save_days = config.get("DEFAULT", "save_days") + auto_backup_meta = { + 'auto_backup_switch': auto_backup_switch, + 'save_days': save_days + } + return {"auto_backup_meta": auto_backup_meta} + + @utils.mutating + def auto_backup_config_update(self, req, auto_backup_meta): + """ + Auto backup daisy data.. + + :param req: The WSGI/Webob Request object + + :raises HTTPBadRequest if backup failed + """ + auto_backup_switch = auto_backup_meta.get("auto_backup_switch", "ON") + save_days = auto_backup_meta.get("save_days", "10") + config = ConfigParser.ConfigParser() + config.read(self.config_path) + config.set("DEFAULT", "auto_backup_switch", auto_backup_switch) + if auto_backup_switch == "ON": + config.set("DEFAULT", "save_days", save_days) + config.write(open(self.config_path, "w")) + return {"auto_backup_meta": auto_backup_meta} + + @utils.mutating + def list_backup_file(self, req): + """ + Returns detailed information for all available backup files + + :param req: The WSGI/Webob Request object + :retval The response body is a mapping of the following form:: + + {'backup_files': [ + {'id': , + 'name': , + 'create_time': }, ... + ]} + """ + self._enforce(req, 'list_backup_file') + params = self._get_query_params(req) + try: + backup_files = [] + if not os.path.exists(self.auto_back_path): + return dict(backup_files=backup_files) + for item in os.listdir(self.auto_back_path): + path = os.path.join(self.auto_back_path, item) + if not os.path.isfile(path): + continue + backup_file = { + "id": item, + "file_name": item, + "create_time": os.path.getctime(path) + } + backup_files.append(backup_file) + backup_files.sort(key=lambda x: x['create_time'], reverse=True) + for backup_file in backup_files: + backup_file['create_time'] = time.strftime( + "%Y-%m-%d %H:%M:%S", + time.localtime(backup_file['create_time'])) + except exception.Invalid as e: + raise HTTPBadRequest(explanation=e.msg, request=req) + return dict(backup_files=backup_files) + @utils.mutating def restore(self, req, file_meta): """ @@ -258,9 +343,20 @@ class BackupRestoreDeserializer(wsgi.JSONRequestDeserializer): result['file_meta'] = utils.get_dict_meta(request) return result + def _auto_backup_deserialize(self, request): + result = {} + result['auto_backup_meta'] = utils.get_dict_meta(request) + return result + def backup(self, request): return {} + def auto_backup_config_detail(self, request): + return {} + + def auto_backup_config_update(self, request): + return self._auto_backup_deserialize(request) + def restore(self, request): return self._deserialize(request) @@ -285,6 +381,18 @@ class BackupRestoreSerializer(wsgi.JSONResponseSerializer): response.body = self.to_json(result) return response + def auto_backup_config_detail(self, response, result): + response.status = 201 + response.headers['Content-Type'] = 'application/json' + response.body = self.to_json(result) + return response + + def auto_backup_config_update(self, response, result): + response.status = 201 + response.headers['Content-Type'] = 'application/json' + response.body = self.to_json(result) + return response + def restore(self, response, result): response.status = 201 response.headers['Content-Type'] = 'application/json' diff --git a/code/daisy/daisy/api/v1/router.py b/code/daisy/daisy/api/v1/router.py index 3b53118b..a4382b08 100755 --- a/code/daisy/daisy/api/v1/router.py +++ b/code/daisy/daisy/api/v1/router.py @@ -525,6 +525,18 @@ class API(wsgi.Router): controller=backup_restore_resource, action='backup', conditions={'method': ['POST']}) + mapper.connect("/backup/backup_file_list", + controller=backup_restore_resource, + action="list_backup_file", + conditions={'method': ['GET']}) + mapper.connect("/auto_backup_config_detail", + controller=backup_restore_resource, + action='auto_backup_config_detail', + conditions={'method': ['POST']}) + mapper.connect("/auto_backup_config_update", + controller=backup_restore_resource, + action='auto_backup_config_update', + conditions={'method': ['POST']}) mapper.connect("/restore", controller=backup_restore_resource, action='restore', diff --git a/code/daisy/daisy/auto_backup/__init__.py b/code/daisy/daisy/auto_backup/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/code/daisy/daisy/auto_backup/manager.py b/code/daisy/daisy/auto_backup/manager.py new file mode 100644 index 00000000..7cf4a079 --- /dev/null +++ b/code/daisy/daisy/auto_backup/manager.py @@ -0,0 +1,123 @@ +# Copyright 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +/auto backup for tecs API +""" +import time +import os + +from oslo_config import cfg +from oslo_log import log as logging + +from daisy.common import exception +from daisyclient.v1.client import Client +import ConfigParser +import daisy.api.backends.tecs.common as tecs_cmn + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class AutoBackupManager(): + + def __init__(self, *args, **kwargs): + """Load auto backup options and initialization.""" + self.backup_hour = 23 + self.backup_min = 59 + self.config_path = '/home/daisy_install/daisy.conf' + self.auto_back_path = '/home/daisy_backup/auto/' + + def get_auto_backup_config(self): + auto_backup_switch = 'OFF' + save_days = 10 + local_hour = time.localtime().tm_hour + local_min = time.localtime().tm_min + if local_hour == self.backup_hour and local_min == self.backup_min: + config = ConfigParser.ConfigParser() + config.read(self.config_path) + if 'auto_backup_switch' in dict(config.items('DEFAULT')): + auto_backup_switch = config.get("DEFAULT", + "auto_backup_switch") + else: + auto_backup_switch = 'ON' + if "save_days" in dict(config.items('DEFAULT')): + save_days = config.getint("DEFAULT", "save_days") + return { + "auto_backup_switch": auto_backup_switch, + "save_days": save_days + } + + def clean_history_backup_file(self, save_days): + if not os.path.exists(self.auto_back_path): + return + try: + clean_scripts = [] + for item in os.listdir(self.auto_back_path): + path = os.path.join(self.auto_back_path, item) + if not os.path.isfile(path): + continue + save_seconds = save_days * 24 * 3600 + if int(time.time()) - int(os.path.getctime(path)) > \ + save_seconds: + clean_scripts.append('rm -rf {0}'.format(path)) + if clean_scripts: + tecs_cmn.run_scrip(clean_scripts, + msg='Delete Backup files failed!') + except Exception as e: + LOG.error("excute clean history backup file failed!%s", + e.message) + + def backup_daisy_system(self, daisy_client): + try: + backup = daisy_client.backup_restore.backup(**{}) + backup_file = getattr(backup, 'backup_file') + if not backup_file: + LOG.error("Auto backup daisy failed,file name is empty.") + return + backup_file_name = os.path.split(backup_file)[1] + # copy backup file to dest directory + scripts = [ + 'test -d {0}||mkdir -p {0}'.format(self.auto_back_path), + 'cp -rf {0} {1}'.format(backup_file, self.auto_back_path), + 'chmod 777 {0} {0}{1}'.format(self.auto_back_path, + backup_file_name), + 'rm -rf {0}'.format(backup_file) + ] + tecs_cmn.run_scrip(scripts, msg='Auto Backup file failed!') + except Exception as e: + LOG.error("backup daisy system failed!%s", e.message) + + @staticmethod + def auto_backup(): + try: + auto_backup_insl = AutoBackupManager() + auto_backup_config = auto_backup_insl.get_auto_backup_config() + if auto_backup_config['auto_backup_switch'] != 'ON': + return + # clean history backup file + auto_backup_insl.clean_history_backup_file( + auto_backup_config['save_days']) + # back up daisy + daisy_version = 1.0 + config_discoverd = ConfigParser.ConfigParser() + config_discoverd.read("/etc/daisy/daisy-api.conf") + bind_port = config_discoverd.get("DEFAULT", "bind_port") + daisy_endpoint = "http://127.0.0.1:" + bind_port + daisy_client = Client( + version=daisy_version, endpoint=daisy_endpoint) + auto_backup_insl.backup_daisy_system(daisy_client) + except exception.Invalid as e: + LOG.exception(e.message) diff --git a/code/daisy/daisy/cmd/api.py b/code/daisy/daisy/cmd/api.py old mode 100755 new mode 100644 diff --git a/code/daisy/daisy/cmd/auto_backup.py b/code/daisy/daisy/cmd/auto_backup.py new file mode 100644 index 00000000..ab82d13b --- /dev/null +++ b/code/daisy/daisy/cmd/auto_backup.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Reference implementation server for Daisy auto backup +""" + +import os +import sys +import eventlet +from oslo_config import cfg +from oslo_log import log as logging +from daisy.common import exception +from daisy.common import config +from daisy.openstack.common import loopingcall +from daisy.auto_backup.manager import AutoBackupManager +import six + +# Monkey patch socket and time +eventlet.patcher.monkey_patch(all=False, socket=True, time=True, thread=True) + +# If ../glance/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'daisy', '__init__.py')): + sys.path.insert(0, possible_topdir) + + +CONF = cfg.CONF +backup_opts = [ + cfg.StrOpt('auto_backup_interval', default=60, + help='Number of seconds between two ' + 'checkings to auto backup daisy'), +] +CONF.register_opts(backup_opts, group='auto_backup') +logging.register_options(CONF) + + +def fail(returncode, e): + sys.stderr.write("ERROR: %s\n" % six.text_type(e)) + + +def main(): + try: + config.parse_args() + logging.setup(CONF, 'daisy') + timer = loopingcall.FixedIntervalLoopingCall( + AutoBackupManager.auto_backup) + timer.start(float(CONF.auto_backup.auto_backup_interval)).wait() + except exception.WorkerCreationFailure as e: + fail(2, e) + except RuntimeError as e: + fail(1, e) + +if __name__ == '__main__': + main() diff --git a/code/daisy/daisy/cmd/manage.py b/code/daisy/daisy/cmd/manage.py index ea7c0e40..5eb1bd77 100755 --- a/code/daisy/daisy/cmd/manage.py +++ b/code/daisy/daisy/cmd/manage.py @@ -261,6 +261,8 @@ def main(): prog='daisy-manage')) cfg_files.extend(cfg.find_config_files(project='daisy', prog='daisy-orchestration')) + cfg_files.extend(cfg.find_config_files(project='daisy', + prog='daisy-auto-backup')) config.parse_args(default_config_files=cfg_files, usage="%(prog)s [options] ") logging.setup(CONF, 'daisy') diff --git a/code/daisy/daisy/common/utils.py b/code/daisy/daisy/common/utils.py index f5f3639a..fac945e9 100644 --- a/code/daisy/daisy/common/utils.py +++ b/code/daisy/daisy/common/utils.py @@ -50,7 +50,6 @@ from webob import exc from daisy.common import exception from daisy import i18n -# from providerclient.v1 import client as provider_client CONF = cfg.CONF diff --git a/code/daisy/daisy/registry/api/v1/hosts.py b/code/daisy/daisy/registry/api/v1/hosts.py index 23b81a8f..7cdaa067 100755 --- a/code/daisy/daisy/registry/api/v1/hosts.py +++ b/code/daisy/daisy/registry/api/v1/hosts.py @@ -45,7 +45,7 @@ DISPLAY_FIELDS_IN_INDEX = ['id', 'name', 'size', 'disk_format', 'container_format', 'checksum'] -SUPPORTED_FILTERS = ['name', 'status', 'id', 'cluster_id', +SUPPORTED_FILTERS = ['name', 'status', 'id', 'cluster_id', 'func_id', 'auto_scale', 'container_format', 'disk_format', 'changes-since', 'protected'] diff --git a/code/daisy/etc/daisy-auto-backup.conf b/code/daisy/etc/daisy-auto-backup.conf new file mode 100644 index 00000000..ab51af74 --- /dev/null +++ b/code/daisy/etc/daisy-auto-backup.conf @@ -0,0 +1,21 @@ +[DEFAULT] +# Show more verbose log output (sets INFO log level output) +#verbose = False +verbose = True + +# Show debugging output in logs (sets DEBUG log level output) +#debug = False + +# Log to this file. Make sure you do not set the same log file for both the API +# and registry servers! +# +# If `log_file` is omitted and `use_syslog` is false, then log messages are +# sent to stdout as a fallback. +log_file = /var/log/daisy/auto_backup.log + +# Backlog requests when creating socket +backlog = 4096 + +# interval second in auto scale +auto_backup_interval=60 + diff --git a/code/daisy/setup.cfg b/code/daisy/setup.cfg index 1eec0230..6f4a6f46 100755 --- a/code/daisy/setup.cfg +++ b/code/daisy/setup.cfg @@ -26,6 +26,7 @@ console_scripts = daisy-manage = daisy.cmd.manage:main daisy-registry = daisy.cmd.registry:main daisy-orchestration = daisy.cmd.orchestration:main + daisy-auto-backup = daisy.cmd.auto_backup:main oslo_config.opts = daisy.api = daisy.opts:list_api_opts daisy.registry = daisy.opts:list_registry_opts diff --git a/rpm/SPECS/daisy.spec b/rpm/SPECS/daisy.spec index 57a80f38..5613a625 100755 --- a/rpm/SPECS/daisy.spec +++ b/rpm/SPECS/daisy.spec @@ -21,6 +21,8 @@ Source5: daisy-api-dist.conf Source6: daisy-registry-dist.conf Source9: daisy-orchestration.service Source10: daisy-orchestration.conf +Source11: daisy-auto-backup.service +Source12: daisy-auto-backup.conf BuildArch: noarch BuildRequires: python2-devel @@ -152,6 +154,7 @@ install -d -m 755 %{buildroot}%{_sharedstatedir}/daisy/images # Config file install -p -D -m 640 etc/daisy-api.conf %{buildroot}%{_sysconfdir}/daisy/daisy-api.conf install -p -D -m 640 etc/daisy-orchestration.conf %{buildroot}%{_sysconfdir}/daisy/daisy-orchestration.conf +install -p -D -m 640 etc/daisy-auto-backup.conf %{buildroot}%{_sysconfdir}/daisy/daisy-auto-backup.conf install -p -D -m 644 %{SOURCE5} %{buildroot}%{_datadir}/daisy/daisy-api-dist.conf install -p -D -m 644 etc/daisy-api-paste.ini %{buildroot}%{_datadir}/daisy/daisy-api-dist-paste.ini install -p -D -m 644 etc/daisy-api-paste.ini %{buildroot}%{_sysconfdir}/daisy/daisy-api-paste.ini @@ -166,6 +169,7 @@ install -p -D -m 640 etc/policy.json %{buildroot}%{_sysconfdir}/daisy/policy.jso install -p -D -m 644 %{SOURCE1} %{buildroot}%{_unitdir}/daisy-api.service install -p -D -m 644 %{SOURCE2} %{buildroot}%{_unitdir}/daisy-registry.service install -p -D -m 644 %{SOURCE9} %{buildroot}%{_unitdir}/daisy-orchestration.service +install -p -D -m 644 %{SOURCE11} %{buildroot}%{_unitdir}/daisy-auto-backup.service # Logrotate config install -p -D -m 644 %{SOURCE4} %{buildroot}%{_sysconfdir}/logrotate.d/daisy @@ -197,17 +201,20 @@ exit 0 %systemd_post daisy-api.service %systemd_post daisy-registry.service %systemd_post daisy-orchestration.service +%systemd_post daisy-auto-backup.service %preun %systemd_preun daisy-api.service %systemd_preun daisy-registry.service %systemd_preun daisy-orchestration.service +%systemd_preun daisy-auto-backup.service %postun %systemd_postun_with_restart daisy-api.service %systemd_postun_with_restart daisy-registry.service %systemd_postun_with_restart daisy-orchestration.service +%systemd_postun_with_restart daisy-auto-backup.service if [ $1 -eq 0 ] ; then rm -rf /var/lib/daisy @@ -224,6 +231,7 @@ fi %{_bindir}/daisy-manage %{_bindir}/daisy-registry %{_bindir}/daisy-orchestration +%{_bindir}/daisy-auto-backup %{_datadir}/daisy/daisy-api-dist.conf %{_datadir}/daisy/daisy-registry-dist.conf @@ -233,6 +241,7 @@ fi %{_unitdir}/daisy-api.service %{_unitdir}/daisy-registry.service %{_unitdir}/daisy-orchestration.service +%{_unitdir}/daisy-auto-backup.service #%{_mandir}/man1/daisy*.1.gz @@ -242,6 +251,7 @@ fi %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/daisy-api.conf %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/daisy-registry.conf %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/daisy-orchestration.conf +%config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/daisy-auto-backup.conf %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/policy.json %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/logrotate.d/daisy %dir %attr(0755, daisy, daisy) %{_sharedstatedir}/daisy diff --git a/tools/daisy-utils/daisy-auto-backup.service b/tools/daisy-utils/daisy-auto-backup.service new file mode 100644 index 00000000..f6bae52f --- /dev/null +++ b/tools/daisy-utils/daisy-auto-backup.service @@ -0,0 +1,15 @@ +[Unit] +Description=auto backup Service (code-named Daisy) +After=syslog.target network.target + +[Service] +Type=simple +NotifyAccess=all +Restart=always +User=root +ExecStart=/usr/bin/daisy-auto-backup --config-file /etc/daisy/daisy-auto-backup.conf +PrivateTmp=false + +[Install] +WantedBy=multi-user.target + diff --git a/tools/setup/common/daisy_common_func.sh b/tools/setup/common/daisy_common_func.sh index a5e3e5c8..9f1c686d 100644 --- a/tools/setup/common/daisy_common_func.sh +++ b/tools/setup/common/daisy_common_func.sh @@ -432,6 +432,7 @@ function stop_service_all service_stop "openstack-ironic-discoverd" service_stop "openstack-keystone" service_stop "daisy-orchestration" + service_stop "daisy-auto-backup" } # start all the service automatically @@ -445,6 +446,7 @@ function start_service_all service_start "openstack-ironic-conductor" service_start "openstack-ironic-discoverd" service_start "daisy-orchestration" + service_start "daisy-auto-backup" } _DAISY_COMMON_FUNC_FILE="common_func.sh" diff --git a/tools/setup/install/install_interface.sh b/tools/setup/install/install_interface.sh index 6cf63a6f..c7e143e2 100644 --- a/tools/setup/install/install_interface.sh +++ b/tools/setup/install/install_interface.sh @@ -223,7 +223,11 @@ function all_install systemctl start daisy-orchestration.service [ "$?" -ne 0 ] && { write_install_log "Error:systemctl start daisy-orchestration.service failed"; exit 1; } + systemctl start daisy-auto-backup.service + [ "$?" -ne 0 ] && { write_install_log "Error:systemctl start daisy-auto-backup.service failed"; exit 1; } + systemctl enable daisy-orchestration.service >> $install_logfile 2>&1 + systemctl enable daisy-auto-backup.service >> $install_logfile 2>&1 systemctl enable openstack-ironic-discoverd.service >> $install_logfile 2>&1 #init daisy