Add HA support to the l3 agent

* Add HA mixins used by RouterInfo and LNAT3Agent
* For HA routers: Internal, external and floating IP addresses are no
  longer configured by the agent. Instead the interfaces and addresses
  are passed to a keepalived configuration, which configures the
  addresses when the router transitions to the master state.
* Only the master instance of the router opens the metadata proxy.
  This happens due to keepalived notification scripts that are
  called upon state transitions.
* Extra routes are handled via keepalived virtual routes and are
  no longer configured by the agent.
* HA routers create a 'HA device' on a VRRP-traffic only HA-network.
* Functional testing: Add two new tests to the L3 agent:
    1) Translation of a router configuration to a keepalived
       configuration.
    2) HA specific events when creating a HA router - Assert that
       keepalived is up, etc.

Change-Id: I83f2a5d2af42164c42773b385ba7b00872eed54e
Implements: blueprint l3-high-availability
Co-Authored-By: Assaf Muller <amuller@redhat.com>

etc/l3_agent.ini

@@ -88,3 +88,15 @@
 #   DVR. This mode must be used for an L3 agent running on a centralized
 #   node (or in single-host deployments, e.g. devstack).
 # agent_mode = legacy
+
+# Location to store keepalived and all HA configurations
+# ha_confs_path = $state_path/ha_confs
+
+# VRRP authentication type AH/PASS
+# ha_vrrp_auth_type = PASS
+
+# VRRP authentication password
+# ha_vrrp_auth_password =
+
+# The advertisement interval in seconds
+# ha_vrrp_advert_int = 2

neutron/agent/l3_agent.py

@@ -25,6 +25,7 @@ from oslo.config import cfg
 import Queue
 
 from neutron.agent.common import config
+from neutron.agent import l3_ha_agent
 from neutron.agent.linux import external_process
 from neutron.agent.linux import interface
 from neutron.agent.linux import ip_lib
@@ -237,7 +238,7 @@ class LinkLocalAllocator(object):
             return f.readlines()
 
 
-class RouterInfo(object):
+class RouterInfo(l3_ha_agent.RouterMixin):
 
     def __init__(self, router_id, root_helper, use_namespaces, router,
                  use_ipv6=False):
@@ -265,6 +266,8 @@ class RouterInfo(object):
         self.rtr_fip_subnet = None
         self.dist_fip_count = 0
 
+        super(RouterInfo, self).__init__()
+
     @property
     def router(self):
         return self._router
@@ -430,7 +433,9 @@ class RouterProcessingQueue(object):
                 yield (rp, update)
 
 
-class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
+class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback,
+                 l3_ha_agent.AgentMixin,
+                 manager.Manager):
     """Manager for L3NatAgent
 
         API version history:
@@ -734,7 +739,14 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
             ri.iptables_manager.ipv4['nat'].add_rule(c, r)
         ri.iptables_manager.apply()
         self.process_router_add(ri)
+
+        if ri.is_ha:
+            self.process_ha_router_added(ri)
+
         if self.conf.enable_metadata_proxy:
+            if ri.is_ha:
+                self._add_keepalived_notifiers(ri)
+            else:
                 self._spawn_metadata_proxy(ri.router_id, ri.ns_name)
 
     def _router_removed(self, router_id):
@@ -743,6 +755,10 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
             LOG.warn(_("Info for router %s were not found. "
                        "Skipping router removal"), router_id)
             return
+
+        if ri.is_ha:
+            self.process_ha_router_removed(ri)
+
         ri.router['gw_port'] = None
         ri.router[l3_constants.INTERFACE_KEY] = []
         ri.router[l3_constants.FLOATINGIP_KEY] = []
@@ -757,7 +773,8 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
         del self.router_info[router_id]
         self._destroy_router_namespace(ri.ns_name)
 
-    def _spawn_metadata_proxy(self, router_id, ns_name):
+    def _get_metadata_proxy_callback(self, router_id):
+
         def callback(pid_file):
             metadata_proxy_socket = cfg.CONF.metadata_proxy_socket
             proxy_cmd = ['neutron-ns-metadata-proxy',
@@ -771,19 +788,22 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
                 router_id))
             return proxy_cmd
 
-        pm = external_process.ProcessManager(
+        return callback
+
+    def _get_metadata_proxy_process_manager(self, router_id, ns_name):
+        return external_process.ProcessManager(
             self.conf,
             router_id,
             self.root_helper,
             ns_name)
+
+    def _spawn_metadata_proxy(self, router_id, ns_name):
+        callback = self._get_metadata_proxy_callback(router_id)
+        pm = self._get_metadata_proxy_process_manager(router_id, ns_name)
         pm.enable(callback)
 
     def _destroy_metadata_proxy(self, router_id, ns_name):
-        pm = external_process.ProcessManager(
+        pm = self._get_metadata_proxy_process_manager(router_id, ns_name)
-            self.conf,
-            router_id,
-            self.root_helper,
-            ns_name)
         pm.disable()
 
     def _set_subnet_arp_info(self, ri, port):
@@ -885,10 +905,20 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
         if ex_gw_port_id:
             interface_name = self.get_external_device_name(ex_gw_port_id)
         if ex_gw_port:
+            def _gateway_ports_equal(port1, port2):
+                def _get_filtered_dict(d, ignore):
+                    return dict((k, v) for k, v in d.iteritems()
+                                if k not in ignore)
+
+                keys_to_ignore = set(['binding:host_id'])
+                port1_filtered = _get_filtered_dict(port1, keys_to_ignore)
+                port2_filtered = _get_filtered_dict(port2, keys_to_ignore)
+                return port1_filtered == port2_filtered
+
             self._set_subnet_info(ex_gw_port)
             if not ri.ex_gw_port:
                 self.external_gateway_added(ri, ex_gw_port, interface_name)
-            elif ex_gw_port != ri.ex_gw_port:
+            elif not _gateway_ports_equal(ex_gw_port, ri.ex_gw_port):
                 self.external_gateway_updated(ri, ex_gw_port, interface_name)
         elif not ex_gw_port and ri.ex_gw_port:
             self.external_gateway_removed(ri, ri.ex_gw_port, interface_name)
@@ -946,6 +976,12 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
         ri.snat_ports = snat_ports
         ri.enable_snat = ri.router.get('enable_snat')
 
+        if ri.is_ha:
+            if ri.ha_port:
+                ri.spawn_keepalived()
+            else:
+                ri.disable_keepalived()
+
     def _handle_router_snat_rules(self, ri, ex_gw_port, internal_cidrs,
                                   interface_name, action):
         # Remove all the rules
@@ -1048,6 +1084,10 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
     def _add_floating_ip(self, ri, fip, interface_name, device):
         fip_ip = fip['floating_ip_address']
         ip_cidr = str(fip_ip) + FLOATING_IP_CIDR_SUFFIX
+
+        if ri.is_ha:
+            self._add_vip(ri, ip_cidr, interface_name)
+        else:
             net = netaddr.IPNetwork(ip_cidr)
             try:
                 device.addr.add(net.version, ip_cidr, str(net.broadcast))
@@ -1070,6 +1110,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
             return l3_constants.FLOATINGIP_STATUS_ACTIVE
 
     def _remove_floating_ip(self, ri, device, ip_cidr):
+        if ri.is_ha:
+            self._remove_vip(ri, ip_cidr)
+        else:
             net = netaddr.IPNetwork(ip_cidr)
             device.addr.delete(net.version, ip_cidr)
             if ri.router['distributed']:
@@ -1253,6 +1296,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
         self._external_gateway_added(ri, ex_gw_port, interface_name,
                                      ri.ns_name, preserve_ips)
 
+        if ri.is_ha:
+            self._ha_external_gateway_added(ri, ex_gw_port, interface_name)
+
     def external_gateway_updated(self, ri, ex_gw_port, interface_name):
         preserve_ips = []
         if ri.router['distributed']:
@@ -1272,6 +1318,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
         self._external_gateway_added(ri, ex_gw_port, interface_name,
                                      ns_name, preserve_ips)
 
+        if ri.is_ha:
+            self._ha_external_gateway_updated(ri, ex_gw_port, interface_name)
+
     def _external_gateway_added(self, ri, ex_gw_port, interface_name,
                                 ns_name, preserve_ips):
         if not ip_lib.device_exists(interface_name,
@@ -1284,8 +1333,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
                              namespace=ns_name,
                              prefix=EXTERNAL_DEV_PREFIX)
 
-        self.driver.init_l3(interface_name, [ex_gw_port['ip_cidr']],
+        if not ri.is_ha:
-                            namespace=ns_name,
+            self.driver.init_l3(
+                interface_name, [ex_gw_port['ip_cidr']], namespace=ns_name,
                 gateway=ex_gw_port['subnet'].get('gateway_ip'),
                 extra_subnets=ex_gw_port.get('extra_subnets', []),
                 preserve_ips=preserve_ips)
@@ -1343,6 +1393,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
         else:
             ns_name = ri.ns_name
 
+        if ri.is_ha:
+            self._ha_external_gateway_removed(ri, interface_name)
+
         self.driver.unplug(interface_name,
                            bridge=self.conf.external_network_bridge,
                            namespace=ns_name,
@@ -1404,7 +1457,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
 
     def _internal_network_added(self, ns_name, network_id, port_id,
                                 internal_cidr, mac_address,
-                                interface_name, prefix):
+                                interface_name, prefix, is_ha=False):
         if not ip_lib.device_exists(interface_name,
                                     root_helper=self.root_helper,
                                     namespace=ns_name):
@@ -1412,10 +1465,12 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
                              namespace=ns_name,
                              prefix=prefix)
 
+        if not is_ha:
             self.driver.init_l3(interface_name, [internal_cidr],
                                 namespace=ns_name)
             ip_address = internal_cidr.split('/')[0]
-        self._send_gratuitous_arp_packet(ns_name, interface_name, ip_address)
+            self._send_gratuitous_arp_packet(ns_name, interface_name,
+                                             ip_address)
 
     def internal_network_added(self, ri, port):
         network_id = port['network_id']
@@ -1427,7 +1482,11 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
 
         self._internal_network_added(ri.ns_name, network_id, port_id,
                                      internal_cidr, mac_address,
-                                     interface_name, INTERNAL_DEV_PREFIX)
+                                     interface_name, INTERNAL_DEV_PREFIX,
+                                     ri.is_ha)
+
+        if ri.is_ha:
+            self._add_vip(ri, internal_cidr, interface_name)
 
         ex_gw_port = self._get_ex_gw_port(ri)
         if ri.router['distributed'] and ex_gw_port:
@@ -1475,6 +1534,8 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
         if ip_lib.device_exists(interface_name,
                                 root_helper=self.root_helper,
                                 namespace=ri.ns_name):
+            if ri.is_ha:
+                self._clear_vips(ri, interface_name)
             self.driver.unplug(interface_name, namespace=ri.ns_name,
                                prefix=INTERNAL_DEV_PREFIX)
 
@@ -1843,6 +1904,10 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
 
     def routes_updated(self, ri):
         new_routes = ri.router['routes']
+        if ri.is_ha:
+            self._process_virtual_routes(ri, new_routes)
+            return
+
         old_routes = ri.routes
         adds, removes = common_utils.diff_list_of_dict(old_routes,
                                                        new_routes)
@@ -1931,6 +1996,7 @@ class L3NATAgentWithStateReport(L3NATAgent):
 
 def _register_opts(conf):
     conf.register_opts(L3NATAgent.OPTS)
+    conf.register_opts(l3_ha_agent.OPTS)
     config.register_interface_driver_opts_helper(conf)
     config.register_use_namespaces_opts_helper(conf)
     config.register_agent_state_opts_helper(conf)

neutron/agent/l3_ha_agent.py

@@ -0,0 +1,232 @@
+# Copyright (c) 2014 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import shutil
+import signal
+
+from oslo.config import cfg
+
+from neutron.agent.linux import keepalived
+from neutron.common import constants as l3_constants
+from neutron.openstack.common.gettextutils import _LE
+from neutron.openstack.common import log as logging
+from neutron.openstack.common import periodic_task
+
+LOG = logging.getLogger(__name__)
+
+HA_DEV_PREFIX = 'ha-'
+
+OPTS = [
+    cfg.StrOpt('ha_confs_path',
+               default='$state_path/ha_confs',
+               help=_('Location to store keepalived/conntrackd '
+                      'config files')),
+    cfg.StrOpt('ha_vrrp_auth_type',
+               default='PASS',
+               help=_('VRRP authentication type AH/PASS')),
+    cfg.StrOpt('ha_vrrp_auth_password',
+               help=_('VRRP authentication password'),
+               secret=True),
+    cfg.IntOpt('ha_vrrp_advert_int',
+               default=2,
+               help=_('The advertisement interval in seconds')),
+]
+
+
+class RouterMixin(object):
+    def __init__(self):
+        self.ha_port = None
+        self.keepalived_manager = None
+
+    def _verify_ha(self):
+        if not self.is_ha:
+            raise ValueError(_('Router %s is not a HA router') %
+                             self.router_id)
+
+    @property
+    def is_ha(self):
+        return self.router is not None and self.router.get('ha')
+
+    @property
+    def ha_priority(self):
+        self._verify_ha()
+        return self.router is not None and self.router.get(
+            'priority', keepalived.HA_DEFAULT_PRIORITY)
+
+    @property
+    def ha_vr_id(self):
+        self._verify_ha()
+        return self.router is not None and self.router.get('ha_vr_id')
+
+    @property
+    def ha_state(self):
+        self._verify_ha()
+        ha_state_path = self.keepalived_manager._get_full_config_file_path(
+            'state')
+        try:
+            with open(ha_state_path, 'r') as f:
+                return f.read()
+        except (OSError, IOError):
+            LOG.debug('Error while reading HA state for %s', self.router_id)
+            return None
+
+    def spawn_keepalived(self):
+        self.keepalived_manager.spawn_or_restart()
+
+    def disable_keepalived(self):
+        self.keepalived_manager.disable()
+        conf_dir = self.keepalived_manager.get_conf_dir()
+        shutil.rmtree(conf_dir)
+
+
+class AgentMixin(object):
+    def __init__(self, host):
+        self._init_ha_conf_path()
+        super(AgentMixin, self).__init__(host)
+
+    def _init_ha_conf_path(self):
+        ha_full_path = os.path.dirname("/%s/" % self.conf.ha_confs_path)
+        if not os.path.isdir(ha_full_path):
+            os.makedirs(ha_full_path, 0o755)
+
+    def _init_keepalived_manager(self, ri):
+        ri.keepalived_manager = keepalived.KeepalivedManager(
+            ri.router['id'],
+            keepalived.KeepalivedConf(),
+            conf_path=self.conf.ha_confs_path,
+            namespace=ri.ns_name,
+            root_helper=self.root_helper)
+
+        config = ri.keepalived_manager.config
+
+        interface_name = self.get_ha_device_name(ri.ha_port['id'])
+        instance = keepalived.KeepalivedInstance(
+            'BACKUP', interface_name, ri.ha_vr_id, nopreempt=True,
+            advert_int=self.conf.ha_vrrp_advert_int, priority=ri.ha_priority)
+        instance.track_interfaces.append(interface_name)
+
+        if self.conf.ha_vrrp_auth_password:
+            # TODO(safchain): use oslo.config types when it will be available
+            # in order to check the validity of ha_vrrp_auth_type
+            instance.set_authentication(self.conf.ha_vrrp_auth_type,
+                                        self.conf.ha_vrrp_auth_password)
+
+        group = keepalived.KeepalivedGroup(ri.ha_vr_id)
+        group.add_instance(instance)
+
+        config.add_group(group)
+        config.add_instance(instance)
+
+    def process_ha_router_added(self, ri):
+        ha_port = ri.router.get(l3_constants.HA_INTERFACE_KEY)
+        if not ha_port:
+            LOG.error(_LE('Unable to process HA router %s without ha port'),
+                      ri.router_id)
+            return
+
+        self._set_subnet_info(ha_port)
+        self.ha_network_added(ri, ha_port['network_id'], ha_port['id'],
+                              ha_port['ip_cidr'], ha_port['mac_address'])
+        ri.ha_port = ha_port
+
+        self._init_keepalived_manager(ri)
+
+    def process_ha_router_removed(self, ri):
+        self.ha_network_removed(ri)
+
+    def get_ha_device_name(self, port_id):
+        return (HA_DEV_PREFIX + port_id)[:self.driver.DEV_NAME_LEN]
+
+    def ha_network_added(self, ri, network_id, port_id, internal_cidr,
+                         mac_address):
+        interface_name = self.get_ha_device_name(port_id)
+        self.driver.plug(network_id, port_id, interface_name, mac_address,
+                         namespace=ri.ns_name,
+                         prefix=HA_DEV_PREFIX)
+        self.driver.init_l3(interface_name, [internal_cidr],
+                            namespace=ri.ns_name)
+
+    def ha_network_removed(self, ri):
+        interface_name = self.get_ha_device_name(ri.ha_port['id'])
+        self.driver.unplug(interface_name, namespace=ri.ns_name,
+                           prefix=HA_DEV_PREFIX)
+        ri.ha_port = None
+
+    def _add_vip(self, ri, ip_cidr, interface):
+        instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
+        instance.add_vip(ip_cidr, interface)
+
+    def _remove_vip(self, ri, ip_cidr):
+        instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
+        instance.remove_vip_by_ip_address(ip_cidr)
+
+    def _clear_vips(self, ri, interface):
+        instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
+        instance.remove_vips_vroutes_by_interface(interface)
+
+    def _add_keepalived_notifiers(self, ri):
+        callback = self._get_metadata_proxy_callback(ri.router_id)
+        pm = self._get_metadata_proxy_process_manager(ri.router_id, ri.ns_name)
+        pid = pm.get_pid_file_name(ensure_pids_dir=True)
+        ri.keepalived_manager.add_notifier(
+            callback(pid), 'master', ri.ha_vr_id)
+        for state in ('backup', 'fault'):
+            ri.keepalived_manager.add_notifier(
+                ['kill', '-%s' % signal.SIGKILL,
+                 '$(cat ' + pid + ')'], state, ri.ha_vr_id)
+
+    def _ha_external_gateway_updated(self, ri, ex_gw_port, interface_name):
+        old_gateway_cidr = ri.ex_gw_port['ip_cidr']
+        self._remove_vip(ri, old_gateway_cidr)
+        self._ha_external_gateway_added(ri, ex_gw_port, interface_name)
+
+    def _add_default_gw_virtual_route(self, ri, ex_gw_port, interface_name):
+        gw_ip = ex_gw_port['subnet']['gateway_ip']
+        if gw_ip:
+            instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
+            instance.virtual_routes = (
+                [route for route in instance.virtual_routes
+                 if route.destination != '0.0.0.0/0'])
+            instance.virtual_routes.append(
+                keepalived.KeepalivedVirtualRoute(
+                    '0.0.0.0/0', gw_ip, interface_name))
+
+    def _ha_external_gateway_added(self, ri, ex_gw_port, interface_name):
+        self._add_vip(ri, ex_gw_port['ip_cidr'], interface_name)
+        self._add_default_gw_virtual_route(ri, ex_gw_port, interface_name)
+
+    def _ha_external_gateway_removed(self, ri, interface_name):
+        self._clear_vips(ri, interface_name)
+
+    def _process_virtual_routes(self, ri, new_routes):
+        instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
+
+        # Filter out all of the old routes while keeping only the default route
+        instance.virtual_routes = [route for route in instance.virtual_routes
+                                   if route.destination == '0.0.0.0/0']
+        for route in new_routes:
+            instance.virtual_routes.append(keepalived.KeepalivedVirtualRoute(
+                route['destination'],
+                route['nexthop']))
+
+    def get_ha_routers(self):
+        return (router for router in self.router_info.values() if router.is_ha)
+
+    @periodic_task.periodic_task
+    def _ensure_keepalived_alive(self, context):
+        # TODO(amuller): Use external_process.ProcessMonitor
+        for router in self.get_ha_routers():
+            router.keepalived_manager.revive()

neutron/agent/linux/keepalived.py

@@ -83,7 +83,7 @@ class KeepalivedGroup(object):
         self.ha_vr_id = ha_vr_id
         self.name = 'VG_%s' % ha_vr_id
         self.instance_names = set()
-        self.notifiers = {}
+        self.notifiers = []
 
     def add_instance(self, instance):
         self.instance_names.add(instance.name)
@@ -91,7 +91,7 @@ class KeepalivedGroup(object):
     def set_notify(self, state, path):
         if state not in VALID_NOTIFY_STATES:
             raise InvalidNotifyStateException(state=state)
-        self.notifiers[state] = path
+        self.notifiers.append((state, path))
 
     def build_config(self):
         return itertools.chain(['vrrp_sync_group %s {' % self.name,
@@ -99,7 +99,7 @@ class KeepalivedGroup(object):
                                ('        %s' % i for i in self.instance_names),
                                ['    }'],
                                ('    notify_%s "%s"' % (state, path)
-                                for state, path in self.notifiers.items()),
+                                for state, path in self.notifiers),
                                ['}'])
 
 
@@ -132,6 +132,9 @@ class KeepalivedInstance(object):
 
         self.authentication = (auth_type, password)
 
+    def add_vip(self, ip_cidr, interface_name):
+        self.vips.append(KeepalivedVipAddress(ip_cidr, interface_name))
+
     def remove_vips_vroutes_by_interface(self, interface_name):
         self.vips = [vip for vip in self.vips
                      if vip.interface_name != interface_name]

neutron/services/firewall/agents/varmour/varmour_router.py

@@ -26,6 +26,7 @@ from oslo.config import cfg
 
 from neutron.agent.common import config
 from neutron.agent import l3_agent
+from neutron.agent import l3_ha_agent
 from neutron.agent.linux import external_process
 from neutron.agent.linux import interface
 from neutron.agent.linux import ip_lib
@@ -332,6 +333,7 @@ class vArmourL3NATAgentWithStateReport(vArmourL3NATAgent,
 def main():
     conf = cfg.CONF
     conf.register_opts(vArmourL3NATAgent.OPTS)
+    conf.register_opts(l3_ha_agent.OPTS)
     config.register_interface_driver_opts_helper(conf)
     config.register_use_namespaces_opts_helper(conf)
     config.register_agent_state_opts_helper(conf)

neutron/tests/functional/agent/test_l3_agent.py

@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
+
 import mock
 from oslo.config import cfg
 
@@ -60,9 +62,10 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
 
         mock.patch.object(self.agent, '_send_gratuitous_arp_packet').start()
 
-    def manage_router(self):
+    def manage_router(self, enable_ha):
         router = test_l3_agent.prepare_router_data(enable_snat=True,
-                                                   enable_floating_ip=True)
+                                                   enable_floating_ip=True,
+                                                   enable_ha=enable_ha)
         self.addCleanup(self._delete_router, router['id'])
         ri = self._create_router(router)
         return ri
@@ -77,6 +80,13 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
     def _delete_router(self, router_id):
         self.agent._router_removed(router_id)
 
+    def _add_fip(self, router, fip_address, fixed_address='10.0.0.2'):
+        fip = {'id': _uuid(),
+               'port_id': _uuid(),
+               'floating_ip_address': fip_address,
+               'fixed_ip_address': fixed_address}
+        router.router[l3_constants.FLOATINGIP_KEY].append(fip)
+
     def _namespace_exists(self, router):
         ip = ip_lib.IPWrapper(self.root_helper, router.ns_name)
         return ip.netns.exists(router.ns_name)
@@ -97,21 +107,137 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
             expected_device['mac_address'],
             namespace, self.root_helper)
 
+    def get_expected_keepalive_configuration(self, router):
+        ha_confs_path = cfg.CONF.ha_confs_path
+        router_id = router.router_id
+        ha_device_name = self.agent.get_ha_device_name(router.ha_port['id'])
+        ha_device_cidr = router.ha_port['ip_cidr']
+        external_port = self.agent._get_ex_gw_port(router)
+        external_device_name = self.agent.get_external_device_name(
+            external_port['id'])
+        external_device_cidr = external_port['ip_cidr']
+        internal_port = router.router[l3_constants.INTERFACE_KEY][0]
+        internal_device_name = self.agent.get_internal_device_name(
+            internal_port['id'])
+        internal_device_cidr = internal_port['ip_cidr']
+        floating_ip_cidr = (
+            self.agent.get_floating_ips(router)[0]
+            ['floating_ip_address'] + l3_agent.FLOATING_IP_CIDR_SUFFIX)
+        default_gateway_ip = external_port['subnet'].get('gateway_ip')
+
+        return """vrrp_sync_group VG_1 {
+    group {
+        VR_1
+    }
+    notify_master "%(ha_confs_path)s/%(router_id)s/notify_master.sh"
+    notify_backup "%(ha_confs_path)s/%(router_id)s/notify_backup.sh"
+    notify_fault "%(ha_confs_path)s/%(router_id)s/notify_fault.sh"
+}
+vrrp_instance VR_1 {
+    state BACKUP
+    interface %(ha_device_name)s
+    virtual_router_id 1
+    priority 50
+    nopreempt
+    advert_int 2
+    track_interface {
+        %(ha_device_name)s
+    }
+    virtual_ipaddress {
+        %(floating_ip_cidr)s dev %(external_device_name)s
+    }
+    virtual_ipaddress_excluded {
+        %(external_device_cidr)s dev %(external_device_name)s
+        %(internal_device_cidr)s dev %(internal_device_name)s
+    }
+    virtual_routes {
+        0.0.0.0/0 via %(default_gateway_ip)s dev %(external_device_name)s
+    }
+}""" % {
+            'ha_confs_path': ha_confs_path,
+            'router_id': router_id,
+            'ha_device_name': ha_device_name,
+            'ha_device_cidr': ha_device_cidr,
+            'external_device_name': external_device_name,
+            'external_device_cidr': external_device_cidr,
+            'internal_device_name': internal_device_name,
+            'internal_device_cidr': internal_device_cidr,
+            'floating_ip_cidr': floating_ip_cidr,
+            'default_gateway_ip': default_gateway_ip
+        }
+
 
 class L3AgentTestCase(L3AgentTestFramework):
-    def test_router_lifecycle(self):
+    def test_legacy_router_lifecycle(self):
-        router = self.manage_router()
+        self._router_lifecycle(enable_ha=False)
+
+    def test_ha_router_lifecycle(self):
+        self._router_lifecycle(enable_ha=True)
+
+    def test_keepalived_configuration(self):
+        router = self.manage_router(enable_ha=True)
+        expected = self.get_expected_keepalive_configuration(router)
+
+        self.assertEqual(expected,
+                         router.keepalived_manager.config.get_config_str())
+
+        # Add a new FIP and change the GW IP address
+        router.router = copy.deepcopy(router.router)
+        existing_fip = '19.4.4.2'
+        new_fip = '19.4.4.3'
+        self._add_fip(router, new_fip)
+        router.router['gw_port']['subnet']['gateway_ip'] = '19.4.4.5'
+        router.router['gw_port']['fixed_ips'][0]['ip_address'] = '19.4.4.10'
+
+        self.agent.process_router(router)
+
+        # Get the updated configuration and assert that both FIPs are in,
+        # and that the GW IP address was updated.
+        new_config = router.keepalived_manager.config.get_config_str()
+        old_gw = '0.0.0.0/0 via 19.4.4.1'
+        new_gw = '0.0.0.0/0 via 19.4.4.5'
+        old_external_device_ip = '19.4.4.4'
+        new_external_device_ip = '19.4.4.10'
+        self.assertIn(existing_fip, new_config)
+        self.assertIn(new_fip, new_config)
+        self.assertNotIn(old_gw, new_config)
+        self.assertIn(new_gw, new_config)
+        self.assertNotIn(old_external_device_ip, new_config)
+        self.assertIn(new_external_device_ip, new_config)
+
+    def _router_lifecycle(self, enable_ha):
+        router = self.manage_router(enable_ha)
+
+        if enable_ha:
+            self.wait_until(lambda: router.ha_state == 'master')
+
+            # Keepalived notifies of a state transition when it starts,
+            # not when it ends. Thus, we have to wait until keepalived finishes
+            # configuring everything. We verify this by waiting until the last
+            # device has an IP address.
+            device = router.router[l3_constants.INTERFACE_KEY][-1]
+            self.wait_until(self.device_exists_with_ip_mac, device,
+                            self.agent.get_internal_device_name,
+                            router.ns_name)
 
         self.assertTrue(self._namespace_exists(router))
         self.assertTrue(self._metadata_proxy_exists(router))
         self._assert_internal_devices(router)
         self._assert_external_device(router)
         self._assert_gateway(router)
+        self._assert_floating_ips(router)
         self._assert_snat_chains(router)
         self._assert_floating_ip_chains(router)
 
+        if enable_ha:
+            self._assert_ha_device(router)
+            self.assertTrue(router.keepalived_manager.process.active)
+
         self._delete_router(router.router_id)
+
         self._assert_router_does_not_exist(router)
+        if enable_ha:
+            self.assertFalse(router.keepalived_manager.process.active)
 
     def _assert_internal_devices(self, router):
         internal_devices = router.router[l3_constants.INTERFACE_KEY]
@@ -138,6 +264,17 @@ class L3AgentTestCase(L3AgentTestFramework):
         expected_gateway = external_port['subnet']['gateway_ip']
         self.assertEqual(expected_gateway, existing_gateway)
 
+    def _assert_floating_ips(self, router):
+        floating_ips = router.router[l3_constants.FLOATINGIP_KEY]
+        self.assertTrue(len(floating_ips))
+        external_port = self.agent._get_ex_gw_port(router)
+        for fip in floating_ips:
+            self.assertTrue(ip_lib.device_exists_with_ip_mac(
+                self.agent.get_external_device_name(external_port['id']),
+                '%s/32' % fip['floating_ip_address'],
+                external_port['mac_address'],
+                router.ns_name, self.root_helper))
+
     def _assert_snat_chains(self, router):
         self.assertFalse(router.iptables_manager.is_chain_empty(
             'nat', 'snat'))
@@ -154,3 +291,8 @@ class L3AgentTestCase(L3AgentTestFramework):
         # so there's no need to check that explicitly.
         self.assertFalse(self._namespace_exists(router))
         self.assertFalse(self._metadata_proxy_exists(router))
+
+    def _assert_ha_device(self, router):
+        self.assertTrue(self.device_exists_with_ip_mac(
+            router.router[l3_constants.HA_INTERFACE_KEY],
+            self.agent.get_ha_device_name, router.ns_name))

neutron/tests/functional/base.py

@@ -14,11 +14,14 @@
 #    under the License.
 
 import os
+import time
 
 from neutron.tests import base
 
 
 SUDO_CMD = 'sudo -n'
+TIMEOUT = 60
+SLEEP_INTERVAL = 1
 
 
 class BaseSudoTestCase(base.BaseTestCase):
@@ -55,3 +58,8 @@ class BaseSudoTestCase(base.BaseTestCase):
     def check_sudo_enabled(self):
         if not self.sudo_enabled:
             self.skipTest('testing with sudo is not enabled')
+
+    def wait_until(self, predicate, *args, **kwargs):
+        with self.assert_max_execution_time(TIMEOUT):
+            while not predicate(*args, **kwargs):
+                time.sleep(SLEEP_INTERVAL)

neutron/tests/unit/services/firewall/agents/l3reference/test_firewall_l3_agent.py

@@ -24,6 +24,7 @@ from oslo.config import cfg
 
 from neutron.agent.common import config as agent_config
 from neutron.agent import l3_agent
+from neutron.agent import l3_ha_agent
 from neutron.agent.linux import ip_lib
 from neutron.common import config as base_config
 from neutron import context
@@ -58,6 +59,7 @@ class TestFwaasL3AgentRpcCallback(base.BaseTestCase):
         self.conf = cfg.ConfigOpts()
         self.conf.register_opts(base_config.core_opts)
         self.conf.register_opts(l3_agent.L3NATAgent.OPTS)
+        self.conf.register_opts(l3_ha_agent.OPTS)
         agent_config.register_use_namespaces_opts_helper(self.conf)
         agent_config.register_root_helper(self.conf)
         self.conf.root_helper = 'sudo'

neutron/tests/unit/services/firewall/agents/varmour/test_varmour_router.py

@@ -18,10 +18,10 @@
 
 
 import mock
-from oslo.config import cfg
 
 from neutron.agent.common import config as agent_config
 from neutron.agent import l3_agent
+from neutron.agent import l3_ha_agent
 from neutron.agent.linux import interface
 from neutron.common import config as base_config
 from neutron.common import constants as l3_constants
@@ -39,9 +39,10 @@ class TestVarmourRouter(base.BaseTestCase):
 
     def setUp(self):
         super(TestVarmourRouter, self).setUp()
-        self.conf = cfg.ConfigOpts()
+        self.conf = agent_config.setup_conf()
         self.conf.register_opts(base_config.core_opts)
         self.conf.register_opts(varmour_router.vArmourL3NATAgent.OPTS)
+        self.conf.register_opts(l3_ha_agent.OPTS)
         agent_config.register_interface_driver_opts_helper(self.conf)
         agent_config.register_use_namespaces_opts_helper(self.conf)
         agent_config.register_root_helper(self.conf)
@@ -63,6 +64,9 @@ class TestVarmourRouter(base.BaseTestCase):
             'neutron.agent.linux.external_process.ProcessManager')
         self.external_process = self.external_process_p.start()
 
+        self.makedirs_p = mock.patch('os.makedirs')
+        self.makedirs = self.makedirs_p.start()
+
         self.dvr_cls_p = mock.patch('neutron.agent.linux.interface.NullDriver')
         driver_cls = self.dvr_cls_p.start()
         self.mock_driver = mock.MagicMock()

neutron/tests/unit/services/firewall/drivers/varmour/test_varmour_fwaas.py

@@ -18,10 +18,10 @@
 
 
 import mock
-from oslo.config import cfg
 
 from neutron.agent.common import config as agent_config
 from neutron.agent import l3_agent
+from neutron.agent import l3_ha_agent
 from neutron.agent.linux import interface
 from neutron.common import config as base_config
 from neutron.common import constants as l3_constants
@@ -40,9 +40,10 @@ class TestBasicRouterOperations(base.BaseTestCase):
 
     def setUp(self):
         super(TestBasicRouterOperations, self).setUp()
-        self.conf = cfg.ConfigOpts()
+        self.conf = agent_config.setup_conf()
         self.conf.register_opts(base_config.core_opts)
         self.conf.register_opts(varmour_router.vArmourL3NATAgent.OPTS)
+        self.conf.register_opts(l3_ha_agent.OPTS)
         agent_config.register_interface_driver_opts_helper(self.conf)
         agent_config.register_use_namespaces_opts_helper(self.conf)
         agent_config.register_root_helper(self.conf)
@@ -64,6 +65,9 @@ class TestBasicRouterOperations(base.BaseTestCase):
             'neutron.agent.linux.external_process.ProcessManager')
         self.external_process = self.external_process_p.start()
 
+        self.makedirs_p = mock.patch('os.makedirs')
+        self.makedirs = self.makedirs_p.start()
+
         self.dvr_cls_p = mock.patch('neutron.agent.linux.interface.NullDriver')
         driver_cls = self.dvr_cls_p.start()
         self.mock_driver = mock.MagicMock()

neutron/tests/unit/services/vpn/test_vpn_agent.py

@@ -18,6 +18,7 @@ from oslo.config import cfg
 
 from neutron.agent.common import config as agent_config
 from neutron.agent import l3_agent
+from neutron.agent import l3_ha_agent
 from neutron.agent.linux import interface
 from neutron.common import config as base_config
 from neutron.openstack.common import uuidutils
@@ -48,6 +49,7 @@ class TestVPNAgent(base.BaseTestCase):
         self.conf = cfg.CONF
         self.conf.register_opts(base_config.core_opts)
         self.conf.register_opts(l3_agent.L3NATAgent.OPTS)
+        self.conf.register_opts(l3_ha_agent.OPTS)
         self.conf.register_opts(interface.OPTS)
         agent_config.register_interface_driver_opts_helper(self.conf)
         agent_config.register_use_namespaces_opts_helper(self.conf)

neutron/tests/unit/test_l3_agent.py

@@ -24,6 +24,7 @@ from testtools import matchers
 
 from neutron.agent.common import config as agent_config
 from neutron.agent import l3_agent
+from neutron.agent import l3_ha_agent
 from neutron.agent.linux import interface
 from neutron.common import config as base_config
 from neutron.common import constants as l3_constants
@@ -230,7 +231,7 @@ def router_append_interface(router, count=1, ip_version=4, ra_mode=None,
 
 
 def prepare_router_data(ip_version=4, enable_snat=None, num_internal_ports=1,
-                        enable_floating_ip=False):
+                        enable_floating_ip=False, enable_ha=False):
     if ip_version == 4:
         ip_addr = '19.4.4.4'
         cidr = '19.4.4.0/24'
@@ -267,6 +268,10 @@ def prepare_router_data(ip_version=4, enable_snat=None, num_internal_ports=1,
 
     router_append_interface(router, count=num_internal_ports,
                             ip_version=ip_version)
+    if enable_ha:
+        router['ha'] = True
+        router['ha_vr_id'] = 1
+        router[l3_constants.HA_INTERFACE_KEY] = get_ha_interface()
 
     if enable_snat is not None:
         router['enable_snat'] = enable_snat
@@ -277,6 +282,26 @@ def _get_subnet_id(port):
     return port['fixed_ips'][0]['subnet_id']
 
 
+def get_ha_interface():
+    return {'admin_state_up': True,
+            'device_id': _uuid(),
+            'device_owner': 'network:router_ha_interface',
+            'fixed_ips': [{'ip_address': '169.254.0.2',
+                           'subnet_id': _uuid()}],
+            'id': _uuid(),
+            'mac_address': '12:34:56:78:2b:5d',
+            'name': u'L3 HA Admin port 0',
+            'network_id': _uuid(),
+            'status': u'ACTIVE',
+            'subnet': {'cidr': '169.254.0.0/24',
+                       'gateway_ip': '169.254.0.1',
+                       'id': _uuid()},
+            'tenant_id': '',
+            'agent_id': _uuid(),
+            'agent_host': 'aaa',
+            'priority': 1}
+
+
 class TestBasicRouterOperations(base.BaseTestCase):
 
     def setUp(self):
@@ -284,6 +309,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
         self.conf = agent_config.setup_conf()
         self.conf.register_opts(base_config.core_opts)
         self.conf.register_opts(l3_agent.L3NATAgent.OPTS)
+        self.conf.register_opts(l3_ha_agent.OPTS)
         agent_config.register_interface_driver_opts_helper(self.conf)
         agent_config.register_use_namespaces_opts_helper(self.conf)
         agent_config.register_root_helper(self.conf)
@@ -291,12 +317,19 @@ class TestBasicRouterOperations(base.BaseTestCase):
         self.conf.set_override('router_id', 'fake_id')
         self.conf.set_override('interface_driver',
                                'neutron.agent.linux.interface.NullDriver')
+        self.conf.set_override('send_arp_for_ha', 1)
+        self.conf.set_override('state_path', '')
         self.conf.root_helper = 'sudo'
 
         self.device_exists_p = mock.patch(
             'neutron.agent.linux.ip_lib.device_exists')
         self.device_exists = self.device_exists_p.start()
 
+        mock.patch('neutron.agent.l3_ha_agent.AgentMixin'
+                   '._init_ha_conf_path').start()
+        mock.patch('neutron.agent.linux.keepalived.KeepalivedNotifierMixin'
+                   '._get_full_config_file_path').start()
+
         self.utils_exec_p = mock.patch(
             'neutron.agent.linux.utils.execute')
         self.utils_exec = self.utils_exec_p.start()
@@ -959,6 +992,60 @@ class TestBasicRouterOperations(base.BaseTestCase):
         self.assertFalse(agent.process_router_floating_ip_addresses.called)
         self.assertFalse(agent.process_router_floating_ip_nat_rules.called)
 
+    def test_ha_router_keepalived_config(self):
+        agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
+        router = prepare_router_data(enable_ha=True)
+        router['routes'] = [
+            {'destination': '8.8.8.8/32', 'nexthop': '35.4.0.10'},
+            {'destination': '8.8.4.4/32', 'nexthop': '35.4.0.11'}]
+        ri = l3_agent.RouterInfo(router['id'], self.conf.root_helper,
+                                 self.conf.use_namespaces, router=router)
+        ri.router = router
+        with contextlib.nested(mock.patch.object(agent,
+                                                 '_spawn_metadata_proxy'),
+                               mock.patch('neutron.agent.linux.'
+                                          'utils.replace_file'),
+                               mock.patch('neutron.agent.linux.'
+                                          'utils.execute'),
+                               mock.patch('os.makedirs')):
+            agent.process_ha_router_added(ri)
+            agent.process_router(ri)
+            config = ri.keepalived_manager.config
+            ha_iface = agent.get_ha_device_name(ri.ha_port['id'])
+            ex_iface = agent.get_external_device_name(ri.ex_gw_port['id'])
+            int_iface = agent.get_internal_device_name(
+                ri.internal_ports[0]['id'])
+
+            expected = """vrrp_sync_group VG_1 {
+    group {
+        VR_1
+    }
+}
+vrrp_instance VR_1 {
+    state BACKUP
+    interface %(ha_iface)s
+    virtual_router_id 1
+    priority 50
+    nopreempt
+    advert_int 2
+    track_interface {
+        %(ha_iface)s
+    }
+    virtual_ipaddress {
+        19.4.4.4/24 dev %(ex_iface)s
+    }
+    virtual_ipaddress_excluded {
+        35.4.0.4/24 dev %(int_iface)s
+    }
+    virtual_routes {
+        0.0.0.0/0 via 19.4.4.1 dev %(ex_iface)s
+        8.8.8.8/32 via 35.4.0.10
+        8.8.4.4/32 via 35.4.0.11
+    }
+}""" % {'ha_iface': ha_iface, 'ex_iface': ex_iface, 'int_iface': int_iface}
+
+            self.assertEqual(expected, config.get_config_str())
+
     @mock.patch('neutron.agent.linux.ip_lib.IPDevice')
     def _test_process_router_floating_ip_addresses_add(self, ri,
                                                        agent, IPDevice):
@@ -1047,6 +1134,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
 
         ri = mock.MagicMock()
         ri.router.get.return_value = []
+        type(ri).is_ha = mock.PropertyMock(return_value=False)
         ri.router['distributed'].__nonzero__ = lambda self: False
 
         agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
@@ -1081,7 +1169,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
         device.addr.list.return_value = [{'cidr': '15.1.2.3/32'}]
         ri = mock.MagicMock()
         ri.router['distributed'].__nonzero__ = lambda self: False
-
+        type(ri).is_ha = mock.PropertyMock(return_value=False)
         ri.router.get.return_value = [fip]
 
         agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
@@ -1126,6 +1214,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
             'fixed_ip_address': '192.168.0.2'
         }
         ri = mock.MagicMock()
+        type(ri).is_ha = mock.PropertyMock(return_value=False)
         ri.router.get.return_value = [fip]
         ri.router['distributed'].__nonzero__ = lambda self: False
 
@@ -1710,7 +1799,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
                 agent, '_spawn_metadata_proxy') as spawn_proxy:
                 agent._router_added(router_id, router)
                 if enableflag:
-                    spawn_proxy.assert_called_with(mock.ANY, mock.ANY)
+                    spawn_proxy.assert_called_with(router_id, mock.ANY)
                 else:
                     self.assertFalse(spawn_proxy.call_count)
                 agent._router_removed(router_id)
@@ -2146,6 +2235,7 @@ class TestL3AgentEventHandler(base.BaseTestCase):
     def setUp(self):
         super(TestL3AgentEventHandler, self).setUp()
         cfg.CONF.register_opts(l3_agent.L3NATAgent.OPTS)
+        cfg.CONF.register_opts(l3_ha_agent.OPTS)
         agent_config.register_interface_driver_opts_helper(cfg.CONF)
         agent_config.register_use_namespaces_opts_helper(cfg.CONF)
         cfg.CONF.set_override(
@@ -2194,12 +2284,12 @@ class TestL3AgentEventHandler(base.BaseTestCase):
         cfg.CONF.set_override('debug', True)
 
         self.external_process_p.stop()
-        ns = 'qrouter-' + router_id
+        ri = l3_agent.RouterInfo(router_id, None, True, None)
         try:
             with mock.patch(ip_class_path) as ip_mock:
-                self.agent._spawn_metadata_proxy(router_id, ns)
+                self.agent._spawn_metadata_proxy(ri.router_id, ri.ns_name)
                 ip_mock.assert_has_calls([
-                    mock.call('sudo', ns),
+                    mock.call('sudo', ri.ns_name),
                     mock.call().netns.execute([
                         'neutron-ns-metadata-proxy',
                         mock.ANY,