Add a new scheduler for the l3 HA

This patch updates all schedulers in order to support the
scheduling of HA routers. It also refactors and adds tests for
the auto scheduling part.

The schedulers aren't expected to work when creating a router
that's both distributed and highly available. Specific issues
will be reported as bugs and fixed in a future patch.

Partially-implements: blueprint l3-high-availability
Change-Id: I2f80f45adeffa1a4eebcb375a4c8326177e84e83
Co-Authored-By: Assaf Muller <amuller@redhat.com>

neutron/db/l3_agentschedulers_db.py

@@ -23,12 +23,14 @@ from sqlalchemy import func
 from sqlalchemy import orm
 from sqlalchemy.orm import exc
 from sqlalchemy.orm import joinedload
+from sqlalchemy import sql
 
 from neutron.common import constants
 from neutron.common import utils as n_utils
 from neutron import context as n_ctx
 from neutron.db import agents_db
 from neutron.db import agentschedulers_db
+from neutron.db import l3_attrs_db
 from neutron.db import model_base
 from neutron.extensions import l3agentscheduler
 from neutron import manager
@@ -114,7 +116,13 @@ class L3AgentSchedulerDbMixin(l3agentscheduler.L3AgentSchedulerPluginBase,
             context.session.query(RouterL3AgentBinding).
             join(agents_db.Agent).
             filter(agents_db.Agent.heartbeat_timestamp < cutoff,
-                   agents_db.Agent.admin_state_up))
+                   agents_db.Agent.admin_state_up).
+            outerjoin(l3_attrs_db.RouterExtraAttributes,
+                      l3_attrs_db.RouterExtraAttributes.router_id ==
+                      RouterL3AgentBinding.router_id).
+            filter(sa.or_(l3_attrs_db.RouterExtraAttributes.ha == sql.false(),
+                          l3_attrs_db.RouterExtraAttributes.ha == sql.null())))
+
         for binding in down_bindings:
             LOG.warn(_LW("Rescheduling router %(router)s from agent %(agent)s "
                          "because the agent did not report to the server in "

neutron/db/l3_hascheduler_db.py

@@ -0,0 +1,59 @@
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from sqlalchemy import func
+from sqlalchemy import sql
+
+from neutron.db import agents_db
+from neutron.db import l3_agentschedulers_db as l3_sch_db
+from neutron.db import l3_attrs_db
+from neutron.db import l3_db
+
+
+class L3_HA_scheduler_db_mixin(l3_sch_db.L3AgentSchedulerDbMixin):
+
+    def get_ha_routers_l3_agents_count(self, context):
+        """Return a map between HA routers and how many agents every
+        router is scheduled to.
+        """
+
+        # Postgres requires every column in the select to be present in
+        # the group by statement when using an aggregate function.
+        # One solution is to generate a subquery and join it with the desired
+        # columns.
+        binding_model = l3_sch_db.RouterL3AgentBinding
+        sub_query = (context.session.query(
+            binding_model.router_id,
+            func.count(binding_model.router_id).label('count')).
+            join(l3_attrs_db.RouterExtraAttributes,
+                 binding_model.router_id ==
+                 l3_attrs_db.RouterExtraAttributes.router_id).
+            join(l3_db.Router).
+            filter(l3_attrs_db.RouterExtraAttributes.ha == sql.true()).
+            group_by(binding_model.router_id).subquery())
+
+        query = (context.session.query(
+                 l3_db.Router.id, l3_db.Router.tenant_id, sub_query.c.count).
+                 join(sub_query))
+        return query
+
+    def get_l3_agents_ordered_by_num_routers(self, context, agent_ids):
+        query = (context.session.query(agents_db.Agent, func.count(
+            l3_sch_db.RouterL3AgentBinding.router_id).label('count')).
+            outerjoin(l3_sch_db.RouterL3AgentBinding).
+            group_by(agents_db.Agent.id).
+            filter(agents_db.Agent.id.in_(agent_ids)).
+            order_by('count'))
+
+        return [record[0] for record in query]

neutron/scheduler/l3_agent_scheduler.py

@@ -14,24 +14,34 @@
 #    under the License.
 
 import abc
+import itertools
 import random
 
+from oslo.config import cfg
 from oslo.db import exception as db_exc
 import six
 from sqlalchemy import sql
 
 from neutron.common import constants
+from neutron.common import utils
 from neutron.db import l3_agentschedulers_db
 from neutron.db import l3_db
+from neutron.db import l3_hamode_db
+from neutron.openstack.common.gettextutils import _LE
 from neutron.openstack.common import log as logging
 
 
 LOG = logging.getLogger(__name__)
+cfg.CONF.register_opts(l3_hamode_db.L3_HA_OPTS)
 
 
 @six.add_metaclass(abc.ABCMeta)
 class L3Scheduler(object):
 
+    def __init__(self):
+        self.min_ha_agents = cfg.CONF.min_l3_agents_per_router
+        self.max_ha_agents = cfg.CONF.max_l3_agents_per_router
+
     @abc.abstractmethod
     def schedule(self, plugin, context, router_id,
                  candidates=None, hints=None):
@@ -41,6 +51,15 @@ class L3Scheduler(object):
         """
         pass
 
+    def router_has_binding(self, context, router_id, l3_agent_id):
+        router_binding_model = l3_agentschedulers_db.RouterL3AgentBinding
+
+        query = context.session.query(router_binding_model)
+        query = query.filter(router_binding_model.router_id == router_id,
+                             router_binding_model.l3_agent_id == l3_agent_id)
+
+        return query.count() > 0
+
     def filter_unscheduled_routers(self, context, plugin, routers):
         """Filter from list of routers the ones that are not scheduled."""
         unscheduled_routers = []
@@ -126,7 +145,10 @@ class L3Scheduler(object):
         unscheduled_routers = self.get_routers_to_schedule(
             context, plugin, router_ids, exclude_distributed=True)
         if not unscheduled_routers:
-            return False
+            if utils.is_extension_supported(
+                    plugin, constants.L3_HA_MODE_EXT_ALIAS):
+                return self.schedule_ha_routers_to_additional_agent(
+                    plugin, context, l3_agent)
 
         target_routers = self.get_routers_can_schedule(
             context, plugin, unscheduled_routers, l3_agent)
@@ -135,7 +157,7 @@ class L3Scheduler(object):
                        ' on host %s'), host)
             return False
 
-        self.bind_routers(context, target_routers, l3_agent)
+        self.bind_routers(context, plugin, target_routers, l3_agent)
         return True
 
     def get_candidates(self, plugin, context, sync_router):
@@ -173,8 +195,15 @@ class L3Scheduler(object):
 
             return candidates
 
-    def bind_routers(self, context, routers, l3_agent):
+    def bind_routers(self, context, plugin, routers, l3_agent):
         for router in routers:
+            if router.get('ha'):
+                if not self.router_has_binding(context, router['id'],
+                                               l3_agent.id):
+                    self.create_ha_router_binding(
+                        plugin, context, router['id'],
+                        router['tenant_id'], l3_agent)
+            else:
                 self.bind_router(context, router['id'], l3_agent)
 
     def bind_router(self, context, router_id, chosen_agent):
@@ -222,6 +251,12 @@ class L3Scheduler(object):
         if router_distributed:
             for chosen_agent in candidates:
                 self.bind_router(context, router_id, chosen_agent)
+        elif sync_router.get('ha', False):
+            chosen_agents = self.bind_ha_router(plugin, context,
+                                                router_id, candidates)
+            if not chosen_agents:
+                return
+            chosen_agent = chosen_agents[-1]
         else:
             chosen_agent = self._choose_router_agent(
                 plugin, context, candidates)
@@ -233,6 +268,82 @@ class L3Scheduler(object):
         """Choose an agent from candidates based on a specific policy."""
         pass
 
+    @abc.abstractmethod
+    def _choose_router_agents_for_ha(self, plugin, context, candidates):
+        """Choose agents from candidates based on a specific policy."""
+        pass
+
+    def get_num_of_agents_for_ha(self, candidates_count):
+        return (min(self.max_ha_agents, candidates_count) if self.max_ha_agents
+                else candidates_count)
+
+    def enough_candidates_for_ha(self, candidates):
+        if not candidates or len(candidates) < self.min_ha_agents:
+            LOG.error(_LE("Not enough candidates, a HA router needs at least "
+                          "%s agents"), self.min_ha_agents)
+            return False
+        return True
+
+    def create_ha_router_binding(self, plugin, context, router_id, tenant_id,
+                                 agent):
+        """Creates and binds a new HA port for this agent."""
+        ha_network = plugin.get_ha_network(context, tenant_id)
+        port_binding = plugin.add_ha_port(context.elevated(), router_id,
+                                          ha_network.network.id, tenant_id)
+        port_binding.l3_agent_id = agent['id']
+        self.bind_router(context, router_id, agent)
+
+    def schedule_ha_routers_to_additional_agent(self, plugin, context, agent):
+        """Bind already scheduled routers to the agent.
+
+        Retrieve the number of agents per router and check if the router has
+        to be scheduled on the given agent if max_l3_agents_per_router
+        is not yet reached.
+        """
+
+        routers_agents = plugin.get_ha_routers_l3_agents_count(context)
+
+        scheduled = False
+        admin_ctx = context.elevated()
+        for router_id, tenant_id, agents in routers_agents:
+            max_agents_not_reached = (
+                not self.max_ha_agents or agents < self.max_ha_agents)
+            if max_agents_not_reached:
+                if not self.router_has_binding(admin_ctx, router_id, agent.id):
+                    self.create_ha_router_binding(plugin, admin_ctx,
+                                                  router_id, tenant_id,
+                                                  agent)
+                    scheduled = True
+
+        return scheduled
+
+    def bind_ha_router_to_agents(self, plugin, context, router_id,
+                                 chosen_agents):
+        port_bindings = plugin.get_ha_router_port_bindings(context,
+                                                           [router_id])
+        for port_binding, agent in itertools.izip(port_bindings,
+                                                  chosen_agents):
+            port_binding.l3_agent_id = agent.id
+            self.bind_router(context, router_id, agent)
+
+            LOG.debug('HA Router %(router_id)s is scheduled to L3 agent '
+                      '%(agent_id)s)',
+                      {'router_id': router_id, 'agent_id': agent.id})
+
+    def bind_ha_router(self, plugin, context, router_id, candidates):
+        """Bind a HA router to agents based on a specific policy."""
+
+        if not self.enough_candidates_for_ha(candidates):
+            return
+
+        chosen_agents = self._choose_router_agents_for_ha(
+            plugin, context, candidates)
+
+        self.bind_ha_router_to_agents(plugin, context, router_id,
+                                      chosen_agents)
+
+        return chosen_agents
+
 
 class ChanceScheduler(L3Scheduler):
     """Randomly allocate an L3 agent for a router."""
@@ -245,6 +356,10 @@ class ChanceScheduler(L3Scheduler):
     def _choose_router_agent(self, plugin, context, candidates):
         return random.choice(candidates)
 
+    def _choose_router_agents_for_ha(self, plugin, context, candidates):
+        num_agents = self.get_num_of_agents_for_ha(len(candidates))
+        return random.sample(candidates, num_agents)
+
 
 class LeastRoutersScheduler(L3Scheduler):
     """Allocate to an L3 agent with the least number of routers bound."""
@@ -259,3 +374,9 @@ class LeastRoutersScheduler(L3Scheduler):
         chosen_agent = plugin.get_l3_agent_with_min_routers(
             context, candidate_ids)
         return chosen_agent
+
+    def _choose_router_agents_for_ha(self, plugin, context, candidates):
+        num_agents = self.get_num_of_agents_for_ha(len(candidates))
+        ordered_agents = plugin.get_l3_agents_ordered_by_num_routers(
+            context, [candidate['id'] for candidate in candidates])
+        return ordered_agents[:num_agents]

neutron/services/l3_router/l3_router_plugin.py

@@ -27,6 +27,7 @@ from neutron.db import extraroute_db
 from neutron.db import l3_dvrscheduler_db
 from neutron.db import l3_gwmode_db
 from neutron.db import l3_hamode_db
+from neutron.db import l3_hascheduler_db
 from neutron.openstack.common import importutils
 from neutron.plugins.common import constants
 
@@ -35,7 +36,8 @@ class L3RouterPlugin(common_db_mixin.CommonDbMixin,
                      extraroute_db.ExtraRoute_db_mixin,
                      l3_gwmode_db.L3_NAT_db_mixin,
                      l3_dvrscheduler_db.L3_DVRsch_db_mixin,
-                     l3_hamode_db.L3_HA_NAT_db_mixin):
+                     l3_hamode_db.L3_HA_NAT_db_mixin,
+                     l3_hascheduler_db.L3_HA_scheduler_db_mixin):
 
     """Implementation of the Neutron L3 Router Service Plugin.
 

neutron/tests/unit/test_l3_schedulers.py

@@ -17,23 +17,26 @@
 # @author: Emilien Macchi, eNovance SAS
 
 import contextlib
+import datetime
 import uuid
 
 import mock
 from oslo.config import cfg
 from sqlalchemy.orm import query
 
-from neutron.api.v2 import attributes as attr
 from neutron.common import constants
 from neutron.common import topics
 from neutron import context as q_context
 from neutron.db import agents_db
 from neutron.db import common_db_mixin
+from neutron.db import db_base_plugin_v2 as db_v2
 from neutron.db import l3_agentschedulers_db
 from neutron.db import l3_db
 from neutron.db import l3_dvrscheduler_db
-from neutron.extensions import l3 as ext_l3
+from neutron.db import l3_hamode_db
+from neutron.db import l3_hascheduler_db
 from neutron import manager
+from neutron.openstack.common import importutils
 from neutron.openstack.common import timeutils
 from neutron.scheduler import l3_agent_scheduler
 from neutron.tests import base
@@ -62,6 +65,26 @@ SECOND_L3_AGENT = {
     'start_flag': True
 }
 
+HOST_3 = 'my_l3_host_3'
+THIRD_L3_AGENT = {
+    'binary': 'neutron-l3-agent',
+    'host': HOST_3,
+    'topic': topics.L3_AGENT,
+    'configurations': {},
+    'agent_type': constants.AGENT_TYPE_L3,
+    'start_flag': True
+}
+
+HOST_4 = 'my_l3_host_4'
+FOURTH_L3_AGENT = {
+    'binary': 'neutron-l3-agent',
+    'host': HOST_4,
+    'topic': topics.L3_AGENT,
+    'configurations': {},
+    'agent_type': constants.AGENT_TYPE_L3,
+    'start_flag': True
+}
+
 HOST_DVR = 'my_l3_host_dvr'
 DVR_L3_AGENT = {
     'binary': 'neutron-l3-agent',
@@ -82,9 +105,6 @@ DVR_SNAT_L3_AGENT = {
     'start_flag': True
 }
 
-DB_PLUGIN_KLASS = ('neutron.plugins.openvswitch.ovs_neutron_plugin.'
-                   'OVSNeutronPluginV2')
-
 
 class FakeL3Scheduler(l3_agent_scheduler.L3Scheduler):
 
@@ -94,6 +114,9 @@ class FakeL3Scheduler(l3_agent_scheduler.L3Scheduler):
     def _choose_router_agent(self):
         pass
 
+    def _choose_router_agents_for_ha(self):
+        pass
+
 
 class L3SchedulerBaseTestCase(base.BaseTestCase):
 
@@ -123,9 +146,11 @@ class L3SchedulerBaseTestCase(base.BaseTestCase):
         self.assertFalse(result)
 
     def test_auto_schedule_routers_no_unscheduled_routers(self):
+        type(self.plugin).supported_extension_aliases = (
+            mock.PropertyMock(return_value=[]))
         with mock.patch.object(self.scheduler,
                                'get_routers_to_schedule') as mock_routers:
-            mock_routers.return_value = None
+            mock_routers.return_value = []
             result = self.scheduler.auto_schedule_routers(
                 self.plugin, mock.ANY, mock.ANY, mock.ANY)
         self.assertTrue(self.plugin.get_enabled_agent_on_host.called)
@@ -218,55 +243,49 @@ class L3SchedulerBaseTestCase(base.BaseTestCase):
     def test_bind_routers_centralized(self):
         routers = [{'id': 'foo_router'}]
         with mock.patch.object(self.scheduler, 'bind_router') as mock_bind:
-            self.scheduler.bind_routers(mock.ANY, routers, mock.ANY)
+            self.scheduler.bind_routers(mock.ANY, mock.ANY, routers, mock.ANY)
         mock_bind.assert_called_once_with(mock.ANY, 'foo_router', mock.ANY)
 
+    def _test_bind_routers_ha(self, has_binding):
+        routers = [{'id': 'foo_router', 'ha': True, 'tenant_id': '42'}]
+        agent = agents_db.Agent(id='foo_agent')
+        with contextlib.nested(
+            mock.patch.object(self.scheduler, 'router_has_binding',
+                              return_value=has_binding),
+            mock.patch.object(self.scheduler, 'create_ha_router_binding')) as (
+                mock_has_binding, mock_bind):
+            self.scheduler.bind_routers(mock.ANY, mock.ANY, routers, agent)
+        mock_has_binding.assert_called_once_with(mock.ANY, 'foo_router',
+                                                 'foo_agent')
+        self.assertEqual(not has_binding, mock_bind.called)
 
-class L3SchedulerTestExtensionManager(object):
+    def test_bind_routers_ha_has_binding(self):
+        self._test_bind_routers_ha(has_binding=True)
 
-    def get_resources(self):
+    def test_bind_routers_ha_no_binding(self):
-        attr.RESOURCE_ATTRIBUTE_MAP.update(ext_l3.RESOURCE_ATTRIBUTE_MAP)
+        self._test_bind_routers_ha(has_binding=False)
-        l3_res = ext_l3.L3.get_resources()
-        return l3_res
-
-    def get_actions(self):
-        return []
-
-    def get_request_extensions(self):
-        return []
 
 
-class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
+class L3SchedulerBaseMixin(object):
-                          l3_db.L3_NAT_db_mixin,
-                          common_db_mixin.CommonDbMixin,
-                          test_db_plugin.NeutronDbPluginV2TestCase,
-                          test_l3_plugin.L3NatTestCaseMixin):
 
-    def setUp(self):
+    def _register_l3_agent(self, agent, plugin=None):
-        ext_mgr = L3SchedulerTestExtensionManager()
+        if not plugin:
-        super(L3SchedulerTestCase, self).setUp(plugin=DB_PLUGIN_KLASS,
+            plugin = self.plugin
-                                               ext_mgr=ext_mgr)
 
-        self.adminContext = q_context.get_admin_context()
-        self.plugin = manager.NeutronManager.get_plugin()
-        self._register_l3_agents()
-
-    def _register_l3_agents(self):
         callback = agents_db.AgentExtRpcCallback()
         callback.report_state(self.adminContext,
-                              agent_state={'agent_state': FIRST_L3_AGENT},
+                              agent_state={'agent_state': agent},
                               time=timeutils.strtime())
-        agent_db = self.plugin.get_agents_db(self.adminContext,
+        agent_db = plugin.get_agents_db(self.adminContext,
-                                             filters={'host': [HOST]})
+                                        filters={'host': [agent['host']]})
-        self.agent_id1 = agent_db[0].id
+        return agent_db[0]
-        self.agent1 = agent_db[0]
 
-        callback.report_state(self.adminContext,
+    def _register_l3_agents(self, plugin=None):
-                              agent_state={'agent_state': SECOND_L3_AGENT},
+        self.agent1 = self._register_l3_agent(FIRST_L3_AGENT, plugin)
-                              time=timeutils.strtime())
+        self.agent_id1 = self.agent1.id
-        agent_db = self.plugin.get_agents_db(self.adminContext,
+
-                                             filters={'host': [HOST]})
+        self.agent2 = self._register_l3_agent(SECOND_L3_AGENT, plugin)
-        self.agent_id2 = agent_db[0].id
+        self.agent_id2 = self.agent2.id
 
     def _register_l3_dvr_agents(self):
         callback = agents_db.AgentExtRpcCallback()
@@ -289,6 +308,13 @@ class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
         update = {'agent': {'admin_state_up': state}}
         self.plugin.update_agent(context, agent_id, update)
 
+    def _set_l3_agent_dead(self, agent_id):
+        update = {
+            'agent': {
+                'heartbeat_timestamp':
+                timeutils.utcnow() - datetime.timedelta(hours=1)}}
+        self.plugin.update_agent(self.adminContext, agent_id, update)
+
     @contextlib.contextmanager
     def router_with_ext_gw(self, name='router1', admin_state_up=True,
                            fmt=None, tenant_id=str(uuid.uuid4()),
@@ -308,6 +334,9 @@ class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
             router['router']['id'], subnet['subnet']['network_id'])
         self._delete('routers', router['router']['id'])
 
+
+class L3SchedulerTestBaseMixin(object):
+
     def _test_add_router_to_l3_agent(self,
                                      distributed=False,
                                      already_scheduled=False):
@@ -612,6 +641,30 @@ class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
             self.assertTrue(val)
 
 
+class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
+                          l3_db.L3_NAT_db_mixin,
+                          common_db_mixin.CommonDbMixin,
+                          test_db_plugin.NeutronDbPluginV2TestCase,
+                          test_l3_plugin.L3NatTestCaseMixin,
+                          L3SchedulerBaseMixin,
+                          L3SchedulerTestBaseMixin):
+
+    def setUp(self):
+        self.mock_rescheduling = False
+        ext_mgr = test_l3_plugin.L3TestExtensionManager()
+        plugin_str = ('neutron.tests.unit.test_l3_plugin.'
+                      'TestL3NatIntAgentSchedulingPlugin')
+        super(L3SchedulerTestCase, self).setUp(plugin=plugin_str,
+                                               ext_mgr=ext_mgr)
+
+        self.adminContext = q_context.get_admin_context()
+        self.plugin = manager.NeutronManager.get_plugin()
+        self.plugin.router_scheduler = importutils.import_object(
+            'neutron.scheduler.l3_agent_scheduler.ChanceScheduler'
+        )
+        self._register_l3_agents()
+
+
 class L3AgentChanceSchedulerTestCase(L3SchedulerTestCase):
 
     def test_random_scheduling(self):
@@ -642,14 +695,38 @@ class L3AgentChanceSchedulerTestCase(L3SchedulerTestCase):
 
         random_patch.stop()
 
+    def test_scheduler_auto_schedule_when_agent_added(self):
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id1, False)
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id2, False)
+
+        with self.subnet() as subnet:
+            self._set_net_external(subnet['subnet']['network_id'])
+            with self.router_with_ext_gw(name='r1', subnet=subnet) as r1:
+                agents = self.get_l3_agents_hosting_routers(
+                    self.adminContext, [r1['router']['id']],
+                    admin_state_up=True)
+                self.assertEqual(0, len(agents))
+
+                self._set_l3_agent_admin_state(self.adminContext,
+                                               self.agent_id1, True)
+                self.plugin.auto_schedule_routers(self.adminContext,
+                                                  FIRST_L3_AGENT['host'],
+                                                  [r1['router']['id']])
+
+                agents = self.get_l3_agents_hosting_routers(
+                    self.adminContext, [r1['router']['id']],
+                    admin_state_up=True)
+                self.assertEqual(FIRST_L3_AGENT['host'], agents[0]['host'])
+
 
 class L3AgentLeastRoutersSchedulerTestCase(L3SchedulerTestCase):
     def setUp(self):
-        cfg.CONF.set_override('router_scheduler_driver',
-                              'neutron.scheduler.l3_agent_scheduler.'
-                              'LeastRoutersScheduler')
-
         super(L3AgentLeastRoutersSchedulerTestCase, self).setUp()
+        self.plugin.router_scheduler = importutils.import_object(
+            'neutron.scheduler.l3_agent_scheduler.LeastRoutersScheduler'
+        )
 
     def test_scheduler(self):
         # disable one agent to force the scheduling to the only one.
@@ -955,3 +1032,244 @@ class L3DvrSchedulerTestCase(testlib_api.SqlTestCase,
         self.assertTrue(mock_delete.call_count)
         core_plugin.assert_called_once_with()
         l3_notifier.assert_called_once_with()
+
+
+class L3HAPlugin(db_v2.NeutronDbPluginV2,
+                 l3_hamode_db.L3_HA_NAT_db_mixin,
+                 l3_hascheduler_db.L3_HA_scheduler_db_mixin):
+    supported_extension_aliases = ["l3-ha"]
+
+
+class L3HATestCaseMixin(testlib_api.SqlTestCase,
+                        L3SchedulerBaseMixin,
+                        testlib_plugin.PluginSetupHelper):
+
+    def setUp(self):
+        super(L3HATestCaseMixin, self).setUp()
+
+        self.adminContext = q_context.get_admin_context()
+        self.plugin = L3HAPlugin()
+
+        self.setup_coreplugin('neutron.plugins.ml2.plugin.Ml2Plugin')
+        mock.patch.object(l3_hamode_db.L3_HA_NAT_db_mixin,
+                          '_notify_ha_interfaces_updated').start()
+
+        cfg.CONF.set_override('max_l3_agents_per_router', 0)
+        self.plugin.router_scheduler = importutils.import_object(
+            'neutron.scheduler.l3_agent_scheduler.ChanceScheduler'
+        )
+
+        self._register_l3_agents()
+
+    def _create_ha_router(self, ha=True, tenant_id='tenant1'):
+        router = {'name': 'router1', 'admin_state_up': True}
+        if ha is not None:
+            router['ha'] = ha
+        return self.plugin._create_router_db(self.adminContext,
+                                             router, tenant_id)
+
+
+class L3_HA_scheduler_db_mixinTestCase(L3HATestCaseMixin):
+
+    def _register_l3_agents(self, plugin=None):
+        super(L3_HA_scheduler_db_mixinTestCase,
+              self)._register_l3_agents(plugin=plugin)
+
+        self.agent3 = self._register_l3_agent(THIRD_L3_AGENT, plugin)
+        self.agent_id3 = self.agent3.id
+
+        self.agent4 = self._register_l3_agent(FOURTH_L3_AGENT, plugin)
+        self.agent_id4 = self.agent4.id
+
+    def test_get_ha_routers_l3_agents_count(self):
+        router1 = self._create_ha_router()
+        router2 = self._create_ha_router()
+        router3 = self._create_ha_router(ha=False)
+        self.plugin.schedule_router(self.adminContext, router1['id'])
+        self.plugin.schedule_router(self.adminContext, router2['id'])
+        self.plugin.schedule_router(self.adminContext, router3['id'])
+        result = self.plugin.get_ha_routers_l3_agents_count(
+            self.adminContext).all()
+
+        self.assertEqual(2, len(result))
+        self.assertIn((router1['id'], router1['tenant_id'], 4), result)
+        self.assertIn((router2['id'], router2['tenant_id'], 4), result)
+        self.assertNotIn((router3['id'], router3['tenant_id'], mock.ANY),
+                         result)
+
+    def test_get_ordered_l3_agents_by_num_routers(self):
+        router1 = self._create_ha_router()
+        router2 = self._create_ha_router()
+        router3 = self._create_ha_router(ha=False)
+        router4 = self._create_ha_router(ha=False)
+
+        # Agent 1 will host 0 routers, agent 2 will host 1, agent 3 will
+        # host 2, and agent 4 will host 3.
+        self.plugin.schedule_router(self.adminContext, router1['id'],
+                                    candidates=[self.agent2, self.agent4])
+        self.plugin.schedule_router(self.adminContext, router2['id'],
+                                    candidates=[self.agent3, self.agent4])
+        self.plugin.schedule_router(self.adminContext, router3['id'],
+                                    candidates=[self.agent3])
+        self.plugin.schedule_router(self.adminContext, router4['id'],
+                                    candidates=[self.agent4])
+
+        agent_ids = [self.agent_id1, self.agent_id2, self.agent_id3,
+                     self.agent_id4]
+        result = self.plugin.get_l3_agents_ordered_by_num_routers(
+            self.adminContext, agent_ids)
+
+        self.assertEqual(agent_ids, [record['id'] for record in result])
+
+
+class L3AgentSchedulerDbMixinTestCase(L3HATestCaseMixin):
+
+    def test_reschedule_ha_routers_from_down_agents(self):
+        router = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, router['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+        self._set_l3_agent_dead(self.agent_id1)
+        with mock.patch.object(self.plugin, 'reschedule_router') as reschedule:
+            self.plugin.reschedule_routers_from_down_agents()
+            self.assertFalse(reschedule.called)
+
+
+class L3HAChanceSchedulerTestCase(L3HATestCaseMixin):
+
+    def test_scheduler_with_ha_enabled(self):
+        router = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, router['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+
+        for agent in agents:
+            sync_data = self.plugin.get_ha_sync_data_for_host(
+                self.adminContext, router_ids=[router['id']],
+                host=agent.host)
+            self.assertEqual(1, len(sync_data))
+            interface = sync_data[0][constants.HA_INTERFACE_KEY]
+            self.assertIsNotNone(interface)
+
+    def test_auto_schedule(self):
+        router = self._create_ha_router()
+        self.plugin.auto_schedule_routers(
+            self.adminContext, self.agent1.host, None)
+        self.plugin.auto_schedule_routers(
+            self.adminContext, self.agent2.host, None)
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']])
+        self.assertEqual(2, len(agents))
+
+    def test_auto_schedule_specific_router_when_agent_added(self):
+        self._auto_schedule_when_agent_added(True)
+
+    def test_auto_schedule_all_routers_when_agent_added(self):
+        self._auto_schedule_when_agent_added(False)
+
+    def _auto_schedule_when_agent_added(self, specific_router):
+        router = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, router['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+        agent_ids = [agent['id'] for agent in agents]
+        self.assertIn(self.agent_id1, agent_ids)
+        self.assertIn(self.agent_id2, agent_ids)
+
+        agent = self._register_l3_agent(THIRD_L3_AGENT)
+        self.agent_id3 = agent.id
+        routers_to_auto_schedule = [router['id']] if specific_router else []
+        self.plugin.auto_schedule_routers(self.adminContext,
+                                          THIRD_L3_AGENT['host'],
+                                          routers_to_auto_schedule)
+
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']],
+            admin_state_up=True)
+        self.assertEqual(3, len(agents))
+
+        # Simulate agent restart to make sure we don't try to re-bind
+        self.plugin.auto_schedule_routers(self.adminContext,
+                                          THIRD_L3_AGENT['host'],
+                                          routers_to_auto_schedule)
+
+    def test_scheduler_with_ha_enabled_not_enough_agent(self):
+        r1 = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, r1['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [r1['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id2, False)
+
+        r2 = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, r2['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [r2['id']],
+            admin_state_up=True)
+        self.assertEqual(0, len(agents))
+
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id2, True)
+
+
+class L3HALeastRoutersSchedulerTestCase(L3HATestCaseMixin):
+
+    def _register_l3_agents(self, plugin=None):
+        super(L3HALeastRoutersSchedulerTestCase,
+              self)._register_l3_agents(plugin=plugin)
+
+        agent = self._register_l3_agent(THIRD_L3_AGENT, plugin)
+        self.agent_id3 = agent.id
+
+        agent = self._register_l3_agent(FOURTH_L3_AGENT, plugin)
+        self.agent_id4 = agent.id
+
+    def setUp(self):
+        super(L3HALeastRoutersSchedulerTestCase, self).setUp()
+        self.plugin.router_scheduler = importutils.import_object(
+            'neutron.scheduler.l3_agent_scheduler.LeastRoutersScheduler'
+        )
+
+    def test_scheduler(self):
+        cfg.CONF.set_override('max_l3_agents_per_router', 2)
+
+        # disable the third agent to be sure that the router will
+        # be scheduled of the two firsts
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id3, False)
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id4, False)
+
+        r1 = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, r1['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [r1['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+        agent_ids = [agent['id'] for agent in agents]
+        self.assertIn(self.agent_id1, agent_ids)
+        self.assertIn(self.agent_id2, agent_ids)
+
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id3, True)
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id4, True)
+
+        r2 = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, r2['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [r2['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+        agent_ids = [agent['id'] for agent in agents]
+        self.assertIn(self.agent_id3, agent_ids)
+        self.assertIn(self.agent_id4, agent_ids)