Merge "Add container action etcd api"

zun/db/etcd/api.py

@@ -84,6 +84,10 @@ def translate_etcd_result(etcd_result, model_type):
             ret = models.PciDevice(data)
         elif model_type == 'volume_mapping':
             ret = models.VolumeMapping(data)
+        elif model_type == 'container_action':
+            ret = models.ContainerAction(data)
+        elif model_type == 'container_action_event':
+            ret = models.ContainerActionEvent(data)
         else:
             raise exception.InvalidParameterValue(
                 _('The model_type value: %s is invalid.'), model_type)
@@ -947,3 +951,192 @@ class EtcdAPI(object):
             raise
 
         return translate_etcd_result(target, 'volume_mapping')
+
+    @lockutils.synchronized('etcd_action')
+    def action_start(self, context, values):
+        values['created_at'] = datetime.isoformat(timeutils.utcnow())
+        if not values.get('uuid'):
+            values['uuid'] = uuidutils.generate_uuid()
+
+        action = models.ContainerAction(values)
+        try:
+            action.save()
+        except Exception:
+            raise
+        return action
+
+    def _actions_get(self, context, container_uuid, filters=None):
+        action_path = '/container_actions/' + container_uuid
+
+        try:
+            res = getattr(self.client.read(action_path), 'children', None)
+        except etcd.EtcdKeyNotFound:
+            return []
+        except Exception as e:
+            LOG.error(
+                "Error occurred while reading from etcd server: %s",
+                six.text_type(e))
+            raise
+
+        actions = []
+        for c in res:
+            if c.value is not None:
+                actions.append(translate_etcd_result(c, 'container_action'))
+        filters = self._add_project_filters(context, filters)
+        filtered_actions = self._filter_resources(actions, filters)
+        sorted_actions = self._process_list_result(filtered_actions,
+                                                   sort_key='created_at')
+        # Actions need descending order of created_at.
+        sorted_actions.reverse()
+        return sorted_actions
+
+    def actions_get(self, context, container_uuid):
+        return self._actions_get(context, container_uuid)
+
+    def _action_get_by_request_id(self, context, container_uuid, request_id):
+        filters = {'request_id': request_id}
+        actions = self._actions_get(context, container_uuid, filters=filters)
+        if not actions:
+            return None
+        return actions[0]
+
+    def action_get_by_request_id(self, context, container_uuid, request_id):
+        return self._action_get_by_request_id(context, container_uuid,
+                                              request_id)
+
+    @lockutils.synchronized('etcd_action')
+    def action_event_start(self, context, values):
+        """Start an event on a container action."""
+        action = self._action_get_by_request_id(context,
+                                                values['container_uuid'],
+                                                values['request_id'])
+
+        # When zun-compute restarts, the request_id was different with
+        # request_id recorded in ContainerAction, so we can't get the original
+        # recode according to request_id. Try to get the last created action
+        # so that init_container can continue to finish the recovery action.
+        if not action and not context.project_id:
+            actions = self._actions_get(context, values['container_uuid'])
+            if not actions:
+                action = actions[0]
+
+        if not action:
+            raise exception.ContainerActionNotFound(
+                request_id=values['request_id'],
+                container_uuid=values['container_uuid'])
+
+        values['action_id'] = action['id']
+        values['action_uuid'] = action['uuid']
+
+        values['created_at'] = datetime.isoformat(timeutils.utcnow())
+        if not values.get('uuid'):
+            values['uuid'] = uuidutils.generate_uuid()
+
+        event = models.ContainerActionEvent(values)
+        try:
+            event.save()
+        except Exception:
+            raise
+        return event
+
+    def _action_events_get(self, context, action_uuid, filters=None):
+        event_path = '/container_actions_events/' + action_uuid
+
+        try:
+            res = getattr(self.client.read(event_path), 'children', None)
+        except etcd.EtcdKeyNotFound:
+            return []
+        except Exception as e:
+            LOG.error(
+                "Error occurred while reading from etcd server: %s",
+                six.text_type(e))
+            raise
+
+        events = []
+        for c in res:
+            if c.value is not None:
+                events.append(translate_etcd_result(
+                    c, 'container_action_event'))
+
+        filters = filters or {}
+        filtered_events = self._filter_resources(events, filters)
+        sorted_events = self._process_list_result(filtered_events,
+                                                  sort_key='created_at')
+        # Events need descending order of created_at.
+        sorted_events.reverse()
+        return sorted_events
+
+    def _get_event_by_name(self, context, action_uuid, event_name):
+        filters = {'event': event_name}
+        events = self._action_events_get(context, action_uuid, filters)
+        if not events:
+            return None
+        return events[0]
+
+    @lockutils.synchronized('etcd_action')
+    def action_event_finish(self, context, values):
+        """Finish an event on a container action."""
+        action = self._action_get_by_request_id(context,
+                                                values['container_uuid'],
+                                                values['request_id'])
+
+        # When zun-compute restarts, the request_id was different with
+        # request_id recorded in ContainerAction, so we can't get the original
+        # recode according to request_id. Try to get the last created action
+        # so that init_container can continue to finish the recovery action.
+        if not action and not context.project_id:
+            actions = self._actions_get(context, values['container_uuid'])
+            if not actions:
+                action = actions[0]
+
+        if not action:
+            raise exception.ContainerActionNotFound(
+                request_id=values['request_id'],
+                container_uuid=values['container_uuid'])
+
+        event = self._get_event_by_name(context, action['uuid'],
+                                        values['event'])
+
+        if not event:
+            raise exception.ContainerActionEventNotFound(
+                action_id=action['uuid'], event=values['event'])
+
+        try:
+            target_path = '/container_actions_events/{0}/{1}'.\
+                format(action['uuid'], event['uuid'])
+            target = self.client.read(target_path)
+            target_values = json.loads(target.value)
+            target_values.update(values)
+            target.value = json.dump_as_bytes(target_values)
+            self.client.update(target)
+        except etcd.EtcdKeyNotFound:
+            raise exception.ContainerActionEventNotFound(
+                action_id=action['uuid'], event=values['event'])
+        except Exception as e:
+            LOG.error('Error occurred while updating action event: %s',
+                      six.text_type(e))
+            raise
+
+        if values['result'].lower() == 'error':
+            try:
+                target_path = '/container_actions/{0}/{1}'.\
+                    format(action['container_uuid'], action['uuid'])
+                target = self.client.read(target_path)
+                target_values = json.loads(target.value)
+                target_values.update({'message': 'Error'})
+                target.value = json.dump_as_bytes(target_values)
+                self.client.update(target)
+            except etcd.EtcdKeyNotFound:
+                raise exception.ContainerActionNotFound(
+                    request_id=action['request_id'],
+                    container_uuid=action['container_uuid'])
+            except Exception as e:
+                LOG.error('Error occurred while updating action : %s',
+                          six.text_type(e))
+                raise
+
+        return event
+
+    def action_events_get(self, context, action_id):
+        events = self._action_events_get(context, action_id)
+        return events

zun/db/etcd/models.py

@@ -312,3 +312,53 @@ class VolumeMapping(Base):
     @classmethod
     def fields(cls):
         return cls._fields
+
+
+class ContainerAction(Base):
+    """Represents a container action.
+
+    The intention is that there will only be one of these pre user request. A
+    lookup by(container_uuid, request_id) should always return a single result.
+    """
+    _path = '/container_actions'
+
+    _fields = list(objects.ContainerAction.fields) + ['uuid']
+
+    def __init__(self, action_data):
+        self.path = ContainerAction.path(action_data['container_uuid'])
+        for f in ContainerAction.fields():
+            setattr(self, f, None)
+        self.id = 1
+        self.update(action_data)
+
+    @classmethod
+    def path(cls, container_uuid):
+        return cls._path + '/' + container_uuid
+
+    @classmethod
+    def fields(cls):
+        return cls._fields
+
+
+class ContainerActionEvent(Base):
+    """Track events that occur during an ContainerAction."""
+
+    _path = '/container_actions_events'
+
+    _fields = list(objects.ContainerActionEvent.fields) + ['action_uuid',
+                                                           'uuid']
+
+    def __init__(self, event_data):
+        self.path = ContainerActionEvent.path(event_data['action_uuid'])
+        for f in ContainerActionEvent.fields():
+            setattr(self, f, None)
+        self.id = 1
+        self.update(event_data)
+
+    @classmethod
+    def path(cls, action_uuid):
+        return cls._path + '/' + action_uuid
+
+    @classmethod
+    def fields(cls):
+        return cls._fields

zun/tests/unit/db/test_container_action.py

@@ -11,8 +11,11 @@
 #    under the License.
 
 """Tests for manipulating Container Actions via the DB API"""
-import datetime
+from datetime import datetime
-
+from datetime import timedelta
+import etcd
+from etcd import Client as etcd_client
+import mock
 from oslo_config import cfg
 from oslo_utils import timeutils
 from oslo_utils import uuidutils
@@ -20,26 +23,26 @@ from oslo_utils import uuidutils
 from zun.common import exception
 import zun.conf
 from zun.db import api as dbapi
+from zun.db.etcd.api import EtcdAPI as etcd_api
 from zun.tests.unit.db import base
 from zun.tests.unit.db import utils
+from zun.tests.unit.db.utils import FakeEtcdMultipleResult
+from zun.tests.unit.db.utils import FakeEtcdResult
 
 CONF = zun.conf.CONF
 
 
-class DbContainerActionTestCase(base.DbTestCase,
+class _DBContainerActionBase(base.DbTestCase,
-                                base.ModelsObjectComparatorMixin):
+                             base.ModelsObjectComparatorMixin):
+
     IGNORED_FIELDS = [
         'id',
         'created_at',
         'updated_at',
+        'details'
     ]
 
-    def setUp(self):
-        cfg.CONF.set_override('db_type', 'sql')
-        super(DbContainerActionTestCase, self).setUp()
-
     def _create_action_values(self, uuid, action='create_container'):
-
         utils.create_test_container(context=self.context,
                                     name='cont1',
                                     uuid=uuid)
@@ -60,8 +63,7 @@ class DbContainerActionTestCase(base.DbTestCase,
             'event': event,
             'container_uuid': uuid,
             'request_id': self.context.request_id,
-            'start_time': timeutils.utcnow(),
+            'start_time': timeutils.utcnow()
-            'details': 'fake-details',
         }
         if extra is not None:
             values.update(extra)
@@ -80,6 +82,13 @@ class DbContainerActionTestCase(base.DbTestCase,
         self._assertEqualObjects(event, events[0],
                                  ['container_uuid', 'request_id'])
 
+
+class DbContainerActionTestCase(_DBContainerActionBase):
+
+    def setUp(self):
+        cfg.CONF.set_override('db_type', 'sql')
+        super(DbContainerActionTestCase, self).setUp()
+
     def test_container_action_start(self):
         """Create a container action."""
         uuid = uuidutils.generate_uuid()
@@ -166,7 +175,7 @@ class DbContainerActionTestCase(base.DbTestCase,
                                  self._create_event_values(uuid))
 
         event_values = {
-            'finish_time': timeutils.utcnow() + datetime.timedelta(seconds=5),
+            'finish_time': timeutils.utcnow() + timedelta(seconds=5),
             'result': 'Success'
         }
 
@@ -182,7 +191,7 @@ class DbContainerActionTestCase(base.DbTestCase,
         uuid = uuidutils.generate_uuid()
 
         event_values = {
-            'finish_time': timeutils.utcnow() + datetime.timedelta(seconds=5),
+            'finish_time': timeutils.utcnow() + timedelta(seconds=5),
             'result': 'Success'
         }
         event_values = self._create_event_values(uuid, extra=event_values)
@@ -202,7 +211,7 @@ class DbContainerActionTestCase(base.DbTestCase,
         }
 
         extra2 = {
-            'created_at': timeutils.utcnow() + datetime.timedelta(seconds=5)
+            'created_at': timeutils.utcnow() + timedelta(seconds=5)
         }
 
         event_val1 = self._create_event_values(uuid1, 'fake1', extra=extra1)
@@ -219,3 +228,224 @@ class DbContainerActionTestCase(base.DbTestCase,
 
         self._assertEqualOrderedListOfObjects([event3, event2, event1], events,
                                               ['container_uuid', 'request_id'])
+
+
+class EtcdDbContainerActionTestCase(_DBContainerActionBase):
+
+    def setUp(self):
+        cfg.CONF.set_override('db_type', 'etcd')
+        super(EtcdDbContainerActionTestCase, self).setUp()
+
+    @mock.patch.object(etcd_client, 'read')
+    @mock.patch.object(etcd_client, 'write')
+    def test_container_action_start(self, mock_write, mock_read):
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        uuid = uuidutils.generate_uuid()
+        action_values = self._create_action_values(uuid)
+        action = dbapi.action_start(self.context, action_values)
+
+        ignored_keys = self.IGNORED_FIELDS + ['finish_time', 'uuid']
+
+        self._assertEqualObjects(action_values, action.as_dict(), ignored_keys)
+        mock_read.side_effect = lambda *args: FakeEtcdMultipleResult(
+            [action.as_dict()])
+        action['start_time'] = datetime.isoformat(action['start_time'])
+        self._assertActionSaved(action.as_dict(), uuid)
+
+    @mock.patch.object(etcd_client, 'read')
+    @mock.patch.object(etcd_client, 'write')
+    def test_container_actions_get_by_container(self, mock_write, mock_read):
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        uuid1 = uuidutils.generate_uuid()
+
+        expected = []
+
+        action_values = self._create_action_values(uuid1)
+        action = dbapi.action_start(self.context, action_values)
+        action['start_time'] = datetime.isoformat(action['start_time'])
+        expected.append(action)
+
+        action_values['action'] = 'test-action'
+        action = dbapi.action_start(self.context, action_values)
+        action['start_time'] = datetime.isoformat(action['start_time'])
+        expected.append(action)
+
+        # Create an other container action.
+        uuid2 = uuidutils.generate_uuid()
+        action_values = self._create_action_values(uuid2, 'test-action')
+        dbapi.action_start(self.context, action_values)
+
+        mock_read.side_effect = lambda *args: FakeEtcdMultipleResult(
+            expected)
+        actions = dbapi.actions_get(self.context, uuid1)
+        self._assertEqualListsOfObjects(expected, actions)
+
+    @mock.patch.object(etcd_client, 'read')
+    @mock.patch.object(etcd_client, 'write')
+    def test_container_action_get_by_container_and_request(self, mock_write,
+                                                           mock_read):
+        """Ensure we can get an action by container UUID and request_id"""
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        uuid1 = uuidutils.generate_uuid()
+
+        action_values = self._create_action_values(uuid1)
+        action = dbapi.action_start(self.context, action_values)
+        request_id = action_values['request_id']
+
+        # An other action using a different req id
+        action_values['action'] = 'test-action'
+        action_values['request_id'] = 'req-00000000-7522-4d99-7ff-111111111111'
+        dbapi.action_start(self.context, action_values)
+
+        mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([action])
+        action = dbapi.action_get_by_request_id(self.context, uuid1,
+                                                request_id)
+        self.assertEqual('create_container', action['action'])
+        self.assertEqual(self.context.request_id, action['request_id'])
+
+    @mock.patch.object(etcd_client, 'read')
+    @mock.patch.object(etcd_client, 'write')
+    @mock.patch.object(etcd_api, '_action_get_by_request_id')
+    def test_container_action_event_start(self, mock__action_get_by_request_id,
+                                          mock_write, mock_read):
+        """Create a container action event."""
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        uuid = uuidutils.generate_uuid()
+
+        action_values = self._create_action_values(uuid)
+        action = dbapi.action_start(self.context, action_values)
+
+        event_values = self._create_event_values(uuid)
+
+        mock__action_get_by_request_id.return_value = action
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        event = dbapi.action_event_start(self.context, event_values)
+
+        ignored_keys = self.IGNORED_FIELDS + ['finish_time', 'traceback',
+                                              'result', 'action_uuid',
+                                              'request_id', 'container_uuid',
+                                              'uuid']
+        self._assertEqualObjects(event_values, event, ignored_keys)
+
+        event['start_time'] = datetime.isoformat(event['start_time'])
+        mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([event])
+        self._assertActionEventSaved(event, action['uuid'])
+
+    @mock.patch.object(etcd_client, 'read')
+    @mock.patch.object(etcd_client, 'write')
+    def test_container_action_event_start_without_action(self, mock_write,
+                                                         mock_read):
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        uuid = uuidutils.generate_uuid()
+
+        event_values = self._create_event_values(uuid)
+        self.assertRaises(exception.ContainerActionNotFound,
+                          dbapi.action_event_start, self.context, event_values)
+
+    @mock.patch.object(etcd_client, 'read')
+    @mock.patch.object(etcd_client, 'write')
+    @mock.patch.object(etcd_client, 'update')
+    @mock.patch.object(etcd_api, '_action_get_by_request_id')
+    @mock.patch.object(etcd_api, '_get_event_by_name')
+    def test_container_action_event_finish_success(
+            self, mock_get_event_by_name, mock__action_get_by_request_id,
+            mock_update, mock_write, mock_read):
+        """Finish a container action event."""
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        uuid = uuidutils.generate_uuid()
+
+        action = dbapi.action_start(self.context,
+                                    self._create_action_values(uuid))
+
+        event_values = self._create_event_values(uuid)
+        event_values['action_uuid'] = action['uuid']
+
+        mock__action_get_by_request_id.return_value = action
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        event = dbapi.action_event_start(self.context, event_values)
+
+        event_values = {
+            'finish_time': timeutils.utcnow() + timedelta(seconds=5),
+            'result': 'Success'
+        }
+
+        event_values = self._create_event_values(uuid, extra=event_values)
+
+        mock__action_get_by_request_id.return_value = action
+        mock_get_event_by_name.return_value = event
+        mock_read.side_effect = lambda *args: FakeEtcdResult(event)
+        event = dbapi.action_event_finish(self.context, event_values)
+
+        event['start_time'] = datetime.isoformat(event['start_time'])
+        mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([event])
+        self._assertActionEventSaved(event, action['uuid'])
+
+        mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([action])
+        action = dbapi.action_get_by_request_id(self.context, uuid,
+                                                self.context.request_id)
+        self.assertNotEqual('Error', action['message'])
+
+    @mock.patch.object(etcd_client, 'read')
+    @mock.patch.object(etcd_client, 'write')
+    def test_container_action_event_finish_without_action(self, mock_write,
+                                                          mock_read):
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        uuid = uuidutils.generate_uuid()
+
+        event_values = {
+            'finish_time': timeutils.utcnow() + timedelta(seconds=5),
+            'result': 'Success'
+        }
+        event_values = self._create_event_values(uuid, extra=event_values)
+        self.assertRaises(exception.ContainerActionNotFound,
+                          dbapi.action_event_finish,
+                          self.context, event_values)
+
+    @mock.patch.object(etcd_client, 'read')
+    @mock.patch.object(etcd_client, 'write')
+    @mock.patch.object(etcd_api, '_action_get_by_request_id')
+    def test_container_action_events_get_in_order(
+            self, mock__action_get_by_request_id, mock_write, mock_read):
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        uuid1 = uuidutils.generate_uuid()
+
+        action = dbapi.action_start(self.context,
+                                    self._create_action_values(uuid1))
+
+        extra1 = {
+            'action_uuid': action['uuid'],
+            'created_at': timeutils.utcnow()
+        }
+
+        extra2 = {
+            'action_uuid': action['uuid'],
+            'created_at': timeutils.utcnow() + timedelta(seconds=5)
+        }
+
+        event_val1 = self._create_event_values(uuid1, 'fake1', extra=extra1)
+        event_val2 = self._create_event_values(uuid1, 'fake2', extra=extra1)
+        event_val3 = self._create_event_values(uuid1, 'fake3', extra=extra2)
+
+        mock__action_get_by_request_id.return_value = action
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        event1 = dbapi.action_event_start(self.context, event_val1)
+        event1['start_time'] = datetime.isoformat(event1['start_time'])
+
+        mock__action_get_by_request_id.return_value = action
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        event2 = dbapi.action_event_start(self.context, event_val2)
+        event2['start_time'] = datetime.isoformat(event2['start_time'])
+
+        mock__action_get_by_request_id.return_value = action
+        mock_read.side_effect = etcd.EtcdKeyNotFound
+        event3 = dbapi.action_event_start(self.context, event_val3)
+        event3['start_time'] = datetime.isoformat(event3['start_time'])
+
+        mock_read.side_effect = lambda *args: FakeEtcdMultipleResult(
+            [event1, event2, event3])
+        events = dbapi.action_events_get(self.context, action['uuid'])
+
+        self.assertEqual(3, len(events))
+
+        self._assertEqualOrderedListOfObjects([event3, event2, event1], events,
+                                              ['container_uuid', 'request_id'])