add daisy automatic backup feature

Change-Id: Iae95a8df713d6cf117dce27aa32ec2fdbb4ac78e

code/daisy/daisy/api/v1/backup_restore.py

@@ -16,9 +16,11 @@
 """
 /hosts endpoint for Daisy v1 API
 """
+import time
 import datetime
 import os
 import subprocess
+import ConfigParser
 from oslo_log import log as logging
 from webob.exc import HTTPBadRequest
 from webob.exc import HTTPForbidden
@@ -71,6 +73,8 @@ class Controller(controller.BaseController):
         self.notifier = notifier.Notifier()
         registry.configure_registry_client()
         self.policy = policy.Enforcer()
+        self.config_path = '/home/daisy_install/daisy.conf'
+        self.auto_back_path = '/home/daisy_backup/auto/'
 
     def _enforce(self, req, action, target=None):
         """Authorize an action against our policies"""
@@ -169,6 +173,87 @@ class Controller(controller.BaseController):
         daisy_cmn.run_scrip(scripts, msg='Backup file failed!')
         return {"backup_file": BACK_PATH + backup_file_name}
 
+    @utils.mutating
+    def auto_backup_config_detail(self, req):
+        """
+        Auto backup daisy data..
+
+        :param req: The WSGI/Webob Request object
+
+        :raises HTTPBadRequest if backup failed
+        """
+        auto_backup_switch = "ON"
+        save_days = "10"
+        config = ConfigParser.ConfigParser()
+        config.read(self.config_path)
+        if "auto_backup_switch" in dict(config.items('DEFAULT')):
+            auto_backup_switch = config.get('DEFAULT', 'auto_backup_switch')
+        if "save_days" in dict(config.items('DEFAULT')):
+            save_days = config.get("DEFAULT", "save_days")
+        auto_backup_meta = {
+            'auto_backup_switch': auto_backup_switch,
+            'save_days': save_days
+        }
+        return {"auto_backup_meta": auto_backup_meta}
+
+    @utils.mutating
+    def auto_backup_config_update(self, req, auto_backup_meta):
+        """
+        Auto backup daisy data..
+
+        :param req: The WSGI/Webob Request object
+
+        :raises HTTPBadRequest if backup failed
+        """
+        auto_backup_switch = auto_backup_meta.get("auto_backup_switch", "ON")
+        save_days = auto_backup_meta.get("save_days", "10")
+        config = ConfigParser.ConfigParser()
+        config.read(self.config_path)
+        config.set("DEFAULT", "auto_backup_switch", auto_backup_switch)
+        if auto_backup_switch == "ON":
+            config.set("DEFAULT", "save_days", save_days)
+        config.write(open(self.config_path, "w"))
+        return {"auto_backup_meta": auto_backup_meta}
+
+    @utils.mutating
+    def list_backup_file(self, req):
+        """
+        Returns detailed information for all available backup files
+
+        :param req: The WSGI/Webob Request object
+        :retval The response body is a mapping of the following form::
+
+            {'backup_files': [
+                {'id': <ID>,
+                 'name': <NAME>,
+                 'create_time': <TIMESTAMP>}, ...
+            ]}
+        """
+        self._enforce(req, 'list_backup_file')
+        params = self._get_query_params(req)
+        try:
+            backup_files = []
+            if not os.path.exists(self.auto_back_path):
+                return dict(backup_files=backup_files)
+            for item in os.listdir(self.auto_back_path):
+                path = os.path.join(self.auto_back_path, item)
+                if not os.path.isfile(path):
+                    continue
+                backup_file = {
+                    "id": item,
+                    "file_name": item,
+                    "create_time": os.path.getctime(path)
+                }
+                backup_files.append(backup_file)
+            backup_files.sort(key=lambda x: x['create_time'], reverse=True)
+            for backup_file in backup_files:
+                backup_file['create_time'] = time.strftime(
+                    "%Y-%m-%d %H:%M:%S",
+                    time.localtime(backup_file['create_time']))
+        except exception.Invalid as e:
+            raise HTTPBadRequest(explanation=e.msg, request=req)
+        return dict(backup_files=backup_files)
+
     @utils.mutating
     def restore(self, req, file_meta):
         """
@@ -258,9 +343,20 @@ class BackupRestoreDeserializer(wsgi.JSONRequestDeserializer):
         result['file_meta'] = utils.get_dict_meta(request)
         return result
 
+    def _auto_backup_deserialize(self, request):
+        result = {}
+        result['auto_backup_meta'] = utils.get_dict_meta(request)
+        return result
+
     def backup(self, request):
         return {}
 
+    def auto_backup_config_detail(self, request):
+        return {}
+
+    def auto_backup_config_update(self, request):
+        return self._auto_backup_deserialize(request)
+
     def restore(self, request):
         return self._deserialize(request)
 
@@ -285,6 +381,18 @@ class BackupRestoreSerializer(wsgi.JSONResponseSerializer):
         response.body = self.to_json(result)
         return response
 
+    def auto_backup_config_detail(self, response, result):
+        response.status = 201
+        response.headers['Content-Type'] = 'application/json'
+        response.body = self.to_json(result)
+        return response
+
+    def auto_backup_config_update(self, response, result):
+        response.status = 201
+        response.headers['Content-Type'] = 'application/json'
+        response.body = self.to_json(result)
+        return response
+
     def restore(self, response, result):
         response.status = 201
         response.headers['Content-Type'] = 'application/json'

code/daisy/daisy/api/v1/router.py

@@ -525,6 +525,18 @@ class API(wsgi.Router):
                        controller=backup_restore_resource,
                        action='backup',
                        conditions={'method': ['POST']})
+        mapper.connect("/backup/backup_file_list",
+                       controller=backup_restore_resource,
+                       action="list_backup_file",
+                       conditions={'method': ['GET']})
+        mapper.connect("/auto_backup_config_detail",
+                       controller=backup_restore_resource,
+                       action='auto_backup_config_detail',
+                       conditions={'method': ['POST']})
+        mapper.connect("/auto_backup_config_update",
+                       controller=backup_restore_resource,
+                       action='auto_backup_config_update',
+                       conditions={'method': ['POST']})
         mapper.connect("/restore",
                        controller=backup_restore_resource,
                        action='restore',

code/daisy/daisy/auto_backup/manager.py

@@ -0,0 +1,123 @@
+# Copyright 2013 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+/auto backup for tecs API
+"""
+import time
+import os
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from daisy.common import exception
+from daisyclient.v1.client import Client
+import ConfigParser
+import daisy.api.backends.tecs.common as tecs_cmn
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+class AutoBackupManager():
+
+    def __init__(self, *args, **kwargs):
+        """Load auto backup options and initialization."""
+        self.backup_hour = 23
+        self.backup_min = 59
+        self.config_path = '/home/daisy_install/daisy.conf'
+        self.auto_back_path = '/home/daisy_backup/auto/'
+
+    def get_auto_backup_config(self):
+        auto_backup_switch = 'OFF'
+        save_days = 10
+        local_hour = time.localtime().tm_hour
+        local_min = time.localtime().tm_min
+        if local_hour == self.backup_hour and local_min == self.backup_min:
+            config = ConfigParser.ConfigParser()
+            config.read(self.config_path)
+            if 'auto_backup_switch' in dict(config.items('DEFAULT')):
+                auto_backup_switch = config.get("DEFAULT",
+                                                "auto_backup_switch")
+            else:
+                auto_backup_switch = 'ON'
+            if "save_days" in dict(config.items('DEFAULT')):
+                save_days = config.getint("DEFAULT", "save_days")
+        return {
+            "auto_backup_switch": auto_backup_switch,
+            "save_days": save_days
+        }
+
+    def clean_history_backup_file(self, save_days):
+        if not os.path.exists(self.auto_back_path):
+            return
+        try:
+            clean_scripts = []
+            for item in os.listdir(self.auto_back_path):
+                path = os.path.join(self.auto_back_path, item)
+                if not os.path.isfile(path):
+                    continue
+                save_seconds = save_days * 24 * 3600
+                if int(time.time()) - int(os.path.getctime(path)) > \
+                        save_seconds:
+                    clean_scripts.append('rm -rf {0}'.format(path))
+            if clean_scripts:
+                tecs_cmn.run_scrip(clean_scripts,
+                                   msg='Delete Backup files failed!')
+        except Exception as e:
+            LOG.error("excute clean history backup file failed!%s",
+                      e.message)
+
+    def backup_daisy_system(self, daisy_client):
+        try:
+            backup = daisy_client.backup_restore.backup(**{})
+            backup_file = getattr(backup, 'backup_file')
+            if not backup_file:
+                LOG.error("Auto backup daisy failed,file name is empty.")
+                return
+            backup_file_name = os.path.split(backup_file)[1]
+            # copy backup file to dest directory
+            scripts = [
+                'test -d {0}||mkdir -p {0}'.format(self.auto_back_path),
+                'cp -rf {0} {1}'.format(backup_file, self.auto_back_path),
+                'chmod 777 {0} {0}{1}'.format(self.auto_back_path,
+                                              backup_file_name),
+                'rm -rf {0}'.format(backup_file)
+            ]
+            tecs_cmn.run_scrip(scripts, msg='Auto Backup file failed!')
+        except Exception as e:
+            LOG.error("backup daisy system failed!%s", e.message)
+
+    @staticmethod
+    def auto_backup():
+        try:
+            auto_backup_insl = AutoBackupManager()
+            auto_backup_config = auto_backup_insl.get_auto_backup_config()
+            if auto_backup_config['auto_backup_switch'] != 'ON':
+                return
+            # clean history backup file
+            auto_backup_insl.clean_history_backup_file(
+                auto_backup_config['save_days'])
+            # back up daisy
+            daisy_version = 1.0
+            config_discoverd = ConfigParser.ConfigParser()
+            config_discoverd.read("/etc/daisy/daisy-api.conf")
+            bind_port = config_discoverd.get("DEFAULT", "bind_port")
+            daisy_endpoint = "http://127.0.0.1:" + bind_port
+            daisy_client = Client(
+                version=daisy_version, endpoint=daisy_endpoint)
+            auto_backup_insl.backup_daisy_system(daisy_client)
+        except exception.Invalid as e:
+            LOG.exception(e.message)

code/daisy/daisy/cmd/auto_backup.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Reference implementation server for Daisy auto backup
+"""
+
+import os
+import sys
+import eventlet
+from oslo_config import cfg
+from oslo_log import log as logging
+from daisy.common import exception
+from daisy.common import config
+from daisy.openstack.common import loopingcall
+from daisy.auto_backup.manager import AutoBackupManager
+import six
+
+# Monkey patch socket and time
+eventlet.patcher.monkey_patch(all=False, socket=True, time=True, thread=True)
+
+# If ../glance/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'daisy', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+
+CONF = cfg.CONF
+backup_opts = [
+    cfg.StrOpt('auto_backup_interval', default=60,
+               help='Number of seconds between two '
+                    'checkings to auto backup daisy'),
+]
+CONF.register_opts(backup_opts, group='auto_backup')
+logging.register_options(CONF)
+
+
+def fail(returncode, e):
+    sys.stderr.write("ERROR: %s\n" % six.text_type(e))
+
+
+def main():
+    try:
+        config.parse_args()
+        logging.setup(CONF, 'daisy')
+        timer = loopingcall.FixedIntervalLoopingCall(
+            AutoBackupManager.auto_backup)
+        timer.start(float(CONF.auto_backup.auto_backup_interval)).wait()
+    except exception.WorkerCreationFailure as e:
+        fail(2, e)
+    except RuntimeError as e:
+        fail(1, e)
+
+if __name__ == '__main__':
+    main()

code/daisy/daisy/cmd/manage.py

@@ -261,6 +261,8 @@ def main():
                                                prog='daisy-manage'))
         cfg_files.extend(cfg.find_config_files(project='daisy',
                                                prog='daisy-orchestration'))
+        cfg_files.extend(cfg.find_config_files(project='daisy',
+                                               prog='daisy-auto-backup'))
         config.parse_args(default_config_files=cfg_files,
                           usage="%(prog)s [options] <cmd>")
         logging.setup(CONF, 'daisy')

code/daisy/daisy/common/utils.py

@@ -50,7 +50,6 @@ from webob import exc
 
 from daisy.common import exception
 from daisy import i18n
-# from providerclient.v1 import client as provider_client
 
 CONF = cfg.CONF
 

code/daisy/daisy/registry/api/v1/hosts.py

@@ -45,7 +45,7 @@ DISPLAY_FIELDS_IN_INDEX = ['id', 'name', 'size',
                            'disk_format', 'container_format',
                            'checksum']
 
-SUPPORTED_FILTERS = ['name', 'status', 'id', 'cluster_id',
+SUPPORTED_FILTERS = ['name', 'status', 'id', 'cluster_id', 'func_id',
                      'auto_scale', 'container_format', 'disk_format',
 
                      'changes-since', 'protected']

code/daisy/etc/daisy-auto-backup.conf

@@ -0,0 +1,21 @@
+[DEFAULT]
+# Show more verbose log output (sets INFO log level output)
+#verbose = False
+verbose = True
+
+# Show debugging output in logs (sets DEBUG log level output)
+#debug = False
+
+# Log to this file. Make sure you do not set the same log file for both the API
+# and registry servers!
+#
+# If `log_file` is omitted and `use_syslog` is false, then log messages are
+# sent to stdout as a fallback.
+log_file = /var/log/daisy/auto_backup.log
+
+# Backlog requests when creating socket
+backlog = 4096
+
+# interval second in auto scale
+auto_backup_interval=60
+

code/daisy/setup.cfg

@@ -26,6 +26,7 @@ console_scripts =
 	daisy-manage = daisy.cmd.manage:main
 	daisy-registry = daisy.cmd.registry:main
 	daisy-orchestration = daisy.cmd.orchestration:main
+        daisy-auto-backup = daisy.cmd.auto_backup:main
 oslo_config.opts = 
 	daisy.api = daisy.opts:list_api_opts
 	daisy.registry = daisy.opts:list_registry_opts

rpm/SPECS/daisy.spec

@@ -21,6 +21,8 @@ Source5:          daisy-api-dist.conf
 Source6:          daisy-registry-dist.conf
 Source9:          daisy-orchestration.service
 Source10:         daisy-orchestration.conf
+Source11:         daisy-auto-backup.service
+Source12:         daisy-auto-backup.conf
 
 BuildArch:        noarch
 BuildRequires:    python2-devel
@@ -152,6 +154,7 @@ install -d -m 755 %{buildroot}%{_sharedstatedir}/daisy/images
 # Config file
 install -p -D -m 640 etc/daisy-api.conf %{buildroot}%{_sysconfdir}/daisy/daisy-api.conf
 install -p -D -m 640 etc/daisy-orchestration.conf %{buildroot}%{_sysconfdir}/daisy/daisy-orchestration.conf
+install -p -D -m 640 etc/daisy-auto-backup.conf %{buildroot}%{_sysconfdir}/daisy/daisy-auto-backup.conf
 install -p -D -m 644 %{SOURCE5} %{buildroot}%{_datadir}/daisy/daisy-api-dist.conf
 install -p -D -m 644 etc/daisy-api-paste.ini %{buildroot}%{_datadir}/daisy/daisy-api-dist-paste.ini
 install -p -D -m 644 etc/daisy-api-paste.ini %{buildroot}%{_sysconfdir}/daisy/daisy-api-paste.ini
@@ -166,6 +169,7 @@ install -p -D -m 640 etc/policy.json %{buildroot}%{_sysconfdir}/daisy/policy.jso
 install -p -D -m 644 %{SOURCE1} %{buildroot}%{_unitdir}/daisy-api.service
 install -p -D -m 644 %{SOURCE2} %{buildroot}%{_unitdir}/daisy-registry.service
 install -p -D -m 644 %{SOURCE9} %{buildroot}%{_unitdir}/daisy-orchestration.service
+install -p -D -m 644 %{SOURCE11} %{buildroot}%{_unitdir}/daisy-auto-backup.service
 
 # Logrotate config
 install -p -D -m 644 %{SOURCE4} %{buildroot}%{_sysconfdir}/logrotate.d/daisy
@@ -197,17 +201,20 @@ exit 0
 %systemd_post daisy-api.service
 %systemd_post daisy-registry.service
 %systemd_post daisy-orchestration.service
+%systemd_post daisy-auto-backup.service
 
 
 %preun
 %systemd_preun daisy-api.service
 %systemd_preun daisy-registry.service
 %systemd_preun daisy-orchestration.service
+%systemd_preun daisy-auto-backup.service
 
 %postun
 %systemd_postun_with_restart daisy-api.service
 %systemd_postun_with_restart daisy-registry.service
 %systemd_postun_with_restart daisy-orchestration.service
+%systemd_postun_with_restart daisy-auto-backup.service
 
 if [ $1 -eq 0 ] ; then
     rm -rf /var/lib/daisy
@@ -224,6 +231,7 @@ fi
 %{_bindir}/daisy-manage
 %{_bindir}/daisy-registry
 %{_bindir}/daisy-orchestration
+%{_bindir}/daisy-auto-backup
 
 %{_datadir}/daisy/daisy-api-dist.conf
 %{_datadir}/daisy/daisy-registry-dist.conf
@@ -233,6 +241,7 @@ fi
 %{_unitdir}/daisy-api.service
 %{_unitdir}/daisy-registry.service
 %{_unitdir}/daisy-orchestration.service
+%{_unitdir}/daisy-auto-backup.service
 
 
 #%{_mandir}/man1/daisy*.1.gz
@@ -242,6 +251,7 @@ fi
 %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/daisy-api.conf
 %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/daisy-registry.conf
 %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/daisy-orchestration.conf
+%config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/daisy-auto-backup.conf
 %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/daisy/policy.json
 %config(noreplace) %attr(-, root, daisy) %{_sysconfdir}/logrotate.d/daisy
 %dir %attr(0755, daisy, daisy) %{_sharedstatedir}/daisy

tools/daisy-utils/daisy-auto-backup.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=auto backup Service (code-named Daisy)
+After=syslog.target network.target
+
+[Service]
+Type=simple
+NotifyAccess=all
+Restart=always
+User=root
+ExecStart=/usr/bin/daisy-auto-backup --config-file /etc/daisy/daisy-auto-backup.conf
+PrivateTmp=false
+
+[Install]
+WantedBy=multi-user.target
+

tools/setup/common/daisy_common_func.sh

@@ -432,6 +432,7 @@ function stop_service_all
     service_stop  "openstack-ironic-discoverd"
     service_stop  "openstack-keystone"
     service_stop  "daisy-orchestration"
+    service_stop  "daisy-auto-backup"
 }
 
 # start all the service automatically
@@ -445,6 +446,7 @@ function start_service_all
     service_start  "openstack-ironic-conductor"
     service_start  "openstack-ironic-discoverd"
     service_start  "daisy-orchestration"
+    service_start  "daisy-auto-backup"
 }
 
 _DAISY_COMMON_FUNC_FILE="common_func.sh"

tools/setup/install/install_interface.sh

@@ -223,7 +223,11 @@ function all_install
     systemctl start daisy-orchestration.service
     [ "$?" -ne 0 ] && { write_install_log "Error:systemctl start daisy-orchestration.service failed"; exit 1; }
 
+    systemctl start daisy-auto-backup.service
+    [ "$?" -ne 0 ] && { write_install_log "Error:systemctl start daisy-auto-backup.service failed"; exit 1; }
+    
     systemctl enable daisy-orchestration.service >> $install_logfile 2>&1
+    systemctl enable daisy-auto-backup.service >> $install_logfile 2>&1
     systemctl enable openstack-ironic-discoverd.service >> $install_logfile 2>&1
 
     #init daisy