Merge "Adds ipset support for Security Groups"

etc/neutron/plugins/ml2/ml2_conf.ini

@@ -65,3 +65,7 @@
 # Controls if neutron security group is enabled or not.
 # It should be false when you use nova security group.
 # enable_security_group = True
+
+# Use ipset to speed-up the iptables security groups. Enabling ipset support
+# requires that ipset is installed on L2 agent node.
+# enable_ipset = True

etc/neutron/rootwrap.d/ipset-firewall.filters

@@ -0,0 +1,12 @@
+# neutron-rootwrap command filters for nodes on which neutron is
+# expected to control network
+#
+# This file should be owned by (and only-writeable by) the root user
+
+# format seems to be
+# cmd-name: filter-name, raw-command, user, args
+
+[Filters]
+# neutron/agent/linux/iptables_firewall.py
+#   "ipset", "-A", ...
+ipset: CommandFilter, ipset, root

neutron/agent/linux/ipset_manager.py

@@ -0,0 +1,75 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from neutron.agent.linux import utils as linux_utils
+from neutron.common import utils
+
+
+class IpsetManager(object):
+    """Wrapper for ipset."""
+
+    def __init__(self, execute=None, root_helper=None):
+        self.execute = execute or linux_utils.execute
+        self.root_helper = root_helper
+
+    @utils.synchronized('ipset', external=True)
+    def create_ipset_chain(self, chain_name, ethertype):
+        cmd = ['ipset', 'create', '-exist', chain_name, 'hash:ip', 'family',
+               self._get_ipset_chain_type(ethertype)]
+        self._apply(cmd)
+
+    @utils.synchronized('ipset', external=True)
+    def add_member_to_ipset_chain(self, chain_name, member_ip):
+        cmd = ['ipset', 'add', '-exist', chain_name, member_ip]
+        self._apply(cmd)
+
+    @utils.synchronized('ipset', external=True)
+    def refresh_ipset_chain_by_name(self, chain_name, member_ips, ethertype):
+        new_chain_name = chain_name + '-new'
+        chain_type = self._get_ipset_chain_type(ethertype)
+        process_input = ["create %s hash:ip family %s" % (new_chain_name,
+                                                          chain_type)]
+        for ip in member_ips:
+            process_input.append("add %s %s" % (new_chain_name, ip))
+
+        self._restore_ipset_chains(process_input)
+        self._swap_ipset_chains(new_chain_name, chain_name)
+        self._destroy_ipset_chain(new_chain_name)
+
+    @utils.synchronized('ipset', external=True)
+    def del_ipset_chain_member(self, chain_name, member_ip):
+        cmd = ['ipset', 'del', chain_name, member_ip]
+        self._apply(cmd)
+
+    @utils.synchronized('ipset', external=True)
+    def destroy_ipset_chain_by_name(self, chain_name):
+        self._destroy_ipset_chain(chain_name)
+
+    def _apply(self, cmd, input=None):
+        input = '\n'.join(input) if input else None
+        self.execute(cmd, root_helper=self.root_helper, process_input=input)
+
+    def _get_ipset_chain_type(self, ethertype):
+        return 'inet6' if ethertype == 'IPv6' else 'inet'
+
+    def _restore_ipset_chains(self, process_input):
+        cmd = ['ipset', 'restore', '-exist']
+        self._apply(cmd, process_input)
+
+    def _swap_ipset_chains(self, src_chain, dest_chain):
+        cmd = ['ipset', 'swap', src_chain, dest_chain]
+        self._apply(cmd)
+
+    def _destroy_ipset_chain(self, chain_name):
+        cmd = ['ipset', 'destroy', chain_name]
+        self._apply(cmd)

neutron/agent/linux/iptables_firewall.py

@@ -17,6 +17,7 @@ import netaddr
 from oslo.config import cfg
 
 from neutron.agent import firewall
+from neutron.agent.linux import ipset_manager
 from neutron.agent.linux import iptables_manager
 from neutron.common import constants
 from neutron.common import ipv6_utils
@@ -33,7 +34,12 @@ CHAIN_NAME_PREFIX = {INGRESS_DIRECTION: 'i',
                      SPOOF_FILTER: 's'}
 DIRECTION_IP_PREFIX = {'ingress': 'source_ip_prefix',
                        'egress': 'dest_ip_prefix'}
+IPSET_DIRECTION = {INGRESS_DIRECTION: 'src',
+                   EGRESS_DIRECTION: 'dst'}
 LINUX_DEV_LEN = 14
+IPSET_CHAIN_LEN = 20
+IPSET_CHANGE_BULK_THRESHOLD = 10
+IPSET_ADD_BULK_THRESHOLD = 5
 
 
 class IptablesFirewallDriver(firewall.FirewallDriver):
@@ -42,9 +48,13 @@ class IptablesFirewallDriver(firewall.FirewallDriver):
                           EGRESS_DIRECTION: 'physdev-in'}
 
     def __init__(self):
+        self.root_helper = cfg.CONF.AGENT.root_helper
         self.iptables = iptables_manager.IptablesManager(
-            root_helper=cfg.CONF.AGENT.root_helper,
+            root_helper=self.root_helper,
             use_ipv6=ipv6_utils.is_enabled())
+        # TODO(majopela, shihanzhang): refactor out ipset to a separate
+        # driver composed over this one
+        self.ipset = ipset_manager.IpsetManager(root_helper=self.root_helper)
         # list of port which has security group
         self.filtered_ports = {}
         self._add_fallback_chain_v4v6()
@@ -56,6 +66,8 @@ class IptablesFirewallDriver(firewall.FirewallDriver):
         # List of security group member ips for ports residing on this host
         self.sg_members = {}
         self.pre_sg_members = None
+        self.ipset_chains = {}
+        self.enable_ipset = cfg.CONF.SECURITYGROUP.enable_ipset
 
     @property
     def ports(self):
@@ -273,6 +285,9 @@ class IptablesFirewallDriver(firewall.FirewallDriver):
         for sg_id in sg_ids:
             for rule in self.sg_rules.get(sg_id, []):
                 if rule['direction'] == direction:
+                    if self.enable_ipset:
+                        port_rules.append(rule)
+                        continue
                     remote_group_id = rule.get('remote_group_id')
                     if not remote_group_id:
                         port_rules.append(rule)
@@ -288,11 +303,25 @@ class IptablesFirewallDriver(firewall.FirewallDriver):
                         port_rules.append(ip_rule)
         return port_rules
 
+    def _get_remote_sg_ids(self, port, direction):
+        sg_ids = port.get('security_groups', [])
+        remote_sg_ids = []
+        for sg_id in sg_ids:
+            remote_sg_ids.extend([rule['remote_group_id']
+                                  for rule in self.sg_rules.get(sg_id, []) if
+                                  rule['direction'] == direction
+                                  and rule.get('remote_group_id')])
+        return remote_sg_ids
+
     def _add_rule_by_security_group(self, port, direction):
         chain_name = self._port_chain_name(port, direction)
         # select rules for current direction
         security_group_rules = self._select_sgr_by_direction(port, direction)
         security_group_rules += self._select_sg_rules_for_port(port, direction)
+        if self.enable_ipset:
+            remote_sg_ids = self._get_remote_sg_ids(port, direction)
+            # update the corresponding ipset chain member
+            self._update_ipset_chain_member(remote_sg_ids)
         # split groups by ip version
         # for ipv4, iptables command is used
         # for ipv6, iptables6 command is used
@@ -315,11 +344,96 @@ class IptablesFirewallDriver(firewall.FirewallDriver):
                                      ipv4_iptables_rule,
                                      ipv6_iptables_rule)
 
+    def _get_cur_sg_member_ips(self, sg_id, ethertype):
+        return self.sg_members.get(sg_id, {}).get(ethertype, [])
+
+    def _get_pre_sg_member_ips(self, sg_id, ethertype):
+        return self.pre_sg_members.get(sg_id, {}).get(ethertype, [])
+
+    def _get_new_sg_member_ips(self, sg_id, ethertype):
+        add_member_ips = (set(self._get_cur_sg_member_ips(sg_id, ethertype)) -
+                          set(self._get_pre_sg_member_ips(sg_id, ethertype)))
+        return list(add_member_ips)
+
+    def _get_deleted_sg_member_ips(self, sg_id, ethertype):
+        del_member_ips = (set(self._get_pre_sg_member_ips(sg_id, ethertype)) -
+                          set(self._get_cur_sg_member_ips(sg_id, ethertype)))
+        return list(del_member_ips)
+
+    def _bulk_set_ips_to_chain(self, chain_name, member_ips, ethertype):
+        self.ipset.refresh_ipset_chain_by_name(chain_name, member_ips,
+                                               ethertype)
+        self.ipset_chains[chain_name] = member_ips
+
+    def _add_ips_to_ipset_chain(self, chain_name, add_ips):
+        for ip in add_ips:
+            if ip not in self.ipset_chains[chain_name]:
+                self.ipset.add_member_to_ipset_chain(chain_name, ip)
+                self.ipset_chains[chain_name].append(ip)
+
+    def _del_ips_from_ipset_chain(self, chain_name, del_ips):
+        if chain_name in self.ipset_chains:
+            for del_ip in del_ips:
+                if del_ip in self.ipset_chains[chain_name]:
+                    self.ipset.del_ipset_chain_member(chain_name, del_ip)
+                    self.ipset_chains[chain_name].remove(del_ip)
+
+    def _update_ipset_chain_member(self, security_group_ids):
+        for sg_id in security_group_ids or []:
+            for ethertype in ['IPv4', 'IPv6']:
+                add_ips = self._get_new_sg_member_ips(sg_id, ethertype)
+                del_ips = self._get_deleted_sg_member_ips(sg_id, ethertype)
+                cur_member_ips = self._get_cur_sg_member_ips(sg_id, ethertype)
+                chain_name = ethertype + sg_id[:IPSET_CHAIN_LEN]
+                if chain_name not in self.ipset_chains:
+                    self.ipset_chains[chain_name] = []
+                    self.ipset.create_ipset_chain(
+                        chain_name, ethertype)
+                    self._bulk_set_ips_to_chain(chain_name,
+                                                cur_member_ips, ethertype)
+                elif (len(add_ips) + len(del_ips)
+                      < IPSET_CHANGE_BULK_THRESHOLD):
+                    self._add_ips_to_ipset_chain(chain_name, add_ips)
+                    self._del_ips_from_ipset_chain(chain_name, del_ips)
+                else:
+                    self._bulk_set_ips_to_chain(chain_name,
+                                                cur_member_ips, ethertype)
+
+    def _generate_ipset_chain(self, sg_rule, remote_gid):
+        iptables_rules = []
+        args = self._protocol_arg(sg_rule.get('protocol'))
+        args += self._port_arg('sport',
+                               sg_rule.get('protocol'),
+                               sg_rule.get('source_port_range_min'),
+                               sg_rule.get('source_port_range_max'))
+        args += self._port_arg('dport',
+                               sg_rule.get('protocol'),
+                               sg_rule.get('port_range_min'),
+                               sg_rule.get('port_range_max'))
+        direction = sg_rule.get('direction')
+        ethertype = sg_rule.get('ethertype')
+        # the length of ipset chain name require less than 31
+        # characters
+        ipset_chain_name = (ethertype + remote_gid[:IPSET_CHAIN_LEN])
+        if ipset_chain_name in self.ipset_chains:
+            args += ['-m set', '--match-set',
+                     ipset_chain_name,
+                     IPSET_DIRECTION[direction]]
+            args += ['-j RETURN']
+            iptables_rules += [' '.join(args)]
+        return iptables_rules
+
     def _convert_sgr_to_iptables_rules(self, security_group_rules):
         iptables_rules = []
         self._drop_invalid_packets(iptables_rules)
         self._allow_established(iptables_rules)
         for rule in security_group_rules:
+            if self.enable_ipset:
+                remote_gid = rule.get('remote_group_id')
+                if remote_gid:
+                    iptables_rules.extend(
+                        self._generate_ipset_chain(rule, remote_gid))
+                    continue
             # These arguments MUST be in the format iptables-save will
             # display them: source/dest, protocol, sport, dport, target
             # Otherwise the iptables_manager code won't be able to find
@@ -422,6 +536,13 @@ class IptablesFirewallDriver(firewall.FirewallDriver):
         for remove_chain_id in need_removed_ipset_chains:
             if remove_chain_id in self.sg_members:
                 self.sg_members.pop(remove_chain_id, None)
+            if self.enable_ipset:
+                for ethertype in ['IPv4', 'IPv6']:
+                    removed_chain = (
+                        ethertype + remove_chain_id[:IPSET_CHAIN_LEN])
+                    if removed_chain in self.ipset_chains:
+                        self.ipset.destroy_ipset_chain_by_name(removed_chain)
+                        self.ipset_chains.pop(removed_chain, None)
 
         # Remove unused security group rules
         for remove_group_id in need_removed_security_groups:

neutron/agent/securitygroups_rpc.py

@@ -37,7 +37,11 @@ security_group_opts = [
         help=_(
             'Controls whether the neutron security group API is enabled '
             'in the server. It should be false when using no security '
-            'groups or using the nova security group API.'))
+            'groups or using the nova security group API.')),
+    cfg.BoolOpt(
+        'enable_ipset',
+        default=True,
+        help=_('Use ipset to speed-up the iptables based security groups.'))
 ]
 cfg.CONF.register_opts(security_group_opts, 'SECURITYGROUP')
 

neutron/tests/unit/test_iptables_firewall.py

@@ -20,6 +20,7 @@ from oslo.config import cfg
 
 from neutron.agent.common import config as a_cfg
 from neutron.agent.linux import iptables_firewall
+from neutron.agent import securitygroups_rpc as sg_cfg
 from neutron.common import constants
 from neutron.tests import base
 from neutron.tests.unit import test_api_v2
@@ -31,11 +32,16 @@ FAKE_PREFIX = {'IPv4': '10.0.0.0/24',
 FAKE_IP = {'IPv4': '10.0.0.1',
            'IPv6': 'fe80::1'}
 
+TEST_IP_RANGE = ['10.0.0.1', '10.0.0.2', '10.0.0.3', '10.0.0.4',
+                 '10.0.0.5', '10.0.0.6', '10.0.0.7', '10.0.0.8',
+                 '10.0.0.9', '10.0.0.10']
 
-class IptablesFirewallTestCase(base.BaseTestCase):
+
+class BaseIptablesFirewallTestCase(base.BaseTestCase):
     def setUp(self):
-        super(IptablesFirewallTestCase, self).setUp()
+        super(BaseIptablesFirewallTestCase, self).setUp()
         cfg.CONF.register_opts(a_cfg.ROOT_HELPER_OPTS, 'AGENT')
+        cfg.CONF.register_opts(sg_cfg.security_group_opts, 'SECURITYGROUP')
         self.utils_exec_p = mock.patch(
             'neutron.agent.linux.utils.execute')
         self.utils_exec = self.utils_exec_p.start()
@@ -52,6 +58,9 @@ class IptablesFirewallTestCase(base.BaseTestCase):
         self.firewall = iptables_firewall.IptablesFirewallDriver()
         self.firewall.iptables = self.iptables_inst
 
+
+class IptablesFirewallTestCase(BaseIptablesFirewallTestCase):
+
     def _fake_port(self):
         return {'device': 'tapfake_dev',
                 'mac_address': 'ff:ff:ff:ff:ff:ff',
@@ -1221,3 +1230,120 @@ class IptablesFirewallTestCase(base.BaseTestCase):
                  mock.call.add_rule('ofake_dev', '-j $sg-fallback'),
                  mock.call.add_rule('sg-chain', '-j ACCEPT')]
         self.v4filter_inst.assert_has_calls(calls)
+
+
+class IptablesFirewallEnhancedIpsetTestCase(BaseIptablesFirewallTestCase):
+    def setUp(self):
+        super(IptablesFirewallEnhancedIpsetTestCase, self).setUp()
+        self.firewall.ipset = mock.Mock()
+
+    def _fake_port(self):
+        return {'device': 'tapfake_dev',
+                'mac_address': 'ff:ff:ff:ff:ff:ff',
+                'fixed_ips': [FAKE_IP['IPv4'],
+                              FAKE_IP['IPv6']],
+                'security_groups': ['fake_sgid'],
+                'security_group_source_groups': ['fake_sgid']}
+
+    def _fake_sg_rule(self):
+        return {'fake_sgid': [
+            {'direction': 'ingress', 'remote_group_id': 'fake_sgid'}]}
+
+    def test_prepare_port_filter_with_default_sg(self):
+        self.firewall.sg_rules = self._fake_sg_rule()
+        self.firewall.sg_members = {'fake_sgid': {
+            'IPv4': ['10.0.0.1', '10.0.0.2'], 'IPv6': ['fe80::1']}}
+        self.firewall.pre_sg_members = {}
+        port = self._fake_port()
+        self.firewall.prepare_port_filter(port)
+        calls = [mock.call.create_ipset_chain('IPv4fake_sgid', 'IPv4'),
+                 mock.call.refresh_ipset_chain_by_name(
+                     'IPv4fake_sgid', ['10.0.0.1', '10.0.0.2'], 'IPv4'),
+                 mock.call.create_ipset_chain('IPv6fake_sgid', 'IPv6'),
+                 mock.call.refresh_ipset_chain_by_name(
+                     'IPv6fake_sgid', ['fe80::1'], 'IPv6')]
+
+        self.firewall.ipset.assert_has_calls(calls)
+
+    def test_prepare_port_filter_with_add_members_beyond_4(self):
+        self.firewall.sg_rules = self._fake_sg_rule()
+        self.firewall.sg_members = {'fake_sgid': {
+            'IPv4': TEST_IP_RANGE[:5],
+            'IPv6': ['fe80::1']}}
+        self.firewall.pre_sg_members = {}
+        port = self._fake_port()
+        self.firewall.prepare_port_filter(port)
+        calls = [mock.call.create_ipset_chain('IPv4fake_sgid', 'IPv4'),
+                 mock.call.refresh_ipset_chain_by_name(
+                     'IPv4fake_sgid', TEST_IP_RANGE[:5], 'IPv4'),
+                 mock.call.create_ipset_chain('IPv6fake_sgid', 'IPv6'),
+                 mock.call.refresh_ipset_chain_by_name(
+                     'IPv6fake_sgid', ['fe80::1'], 'IPv6')]
+
+        self.firewall.ipset.assert_has_calls(calls)
+
+    def test_prepare_port_filter_with_ipset_chain_exist(self):
+        self.firewall.sg_rules = self._fake_sg_rule()
+        self.firewall.ipset_chains = {'IPv4fake_sgid': ['10.0.0.2']}
+        self.firewall.sg_members = {'fake_sgid': {
+            'IPv4': TEST_IP_RANGE[:5],
+            'IPv6': ['fe80::1']}}
+        self.firewall.pre_sg_members = {'fake_sgid': {
+            'IPv4': ['10.0.0.2'],
+            'IPv6': ['fe80::1']}}
+        port = self._fake_port()
+        self.firewall.prepare_port_filter(port)
+        calls = [
+            mock.call.add_member_to_ipset_chain('IPv4fake_sgid', '10.0.0.1'),
+            mock.call.add_member_to_ipset_chain('IPv4fake_sgid', '10.0.0.3'),
+            mock.call.add_member_to_ipset_chain('IPv4fake_sgid', '10.0.0.4'),
+            mock.call.add_member_to_ipset_chain('IPv4fake_sgid', '10.0.0.5'),
+            mock.call.create_ipset_chain('IPv6fake_sgid', 'IPv6'),
+            mock.call.refresh_ipset_chain_by_name(
+                'IPv6fake_sgid', ['fe80::1'], 'IPv6')]
+
+        self.firewall.ipset.assert_has_calls(calls, True)
+
+    def test_prepare_port_filter_with_del_member(self):
+        self.firewall.sg_rules = self._fake_sg_rule()
+        self.firewall.ipset_chains = {'IPv4fake_sgid': ['10.0.0.2']}
+        self.firewall.sg_members = {'fake_sgid': {
+            'IPv4': [
+                '10.0.0.1', '10.0.0.3', '10.0.0.4', '10.0.0.5'],
+            'IPv6': ['fe80::1']}}
+        self.firewall.pre_sg_members = {'fake_sgid': {
+            'IPv4': ['10.0.0.2'],
+            'IPv6': ['fe80::1']}}
+        port = self._fake_port()
+        self.firewall.prepare_port_filter(port)
+        calls = [
+            mock.call.add_member_to_ipset_chain('IPv4fake_sgid', '10.0.0.1'),
+            mock.call.add_member_to_ipset_chain('IPv4fake_sgid', '10.0.0.3'),
+            mock.call.add_member_to_ipset_chain('IPv4fake_sgid', '10.0.0.4'),
+            mock.call.add_member_to_ipset_chain('IPv4fake_sgid', '10.0.0.5'),
+            mock.call.del_ipset_chain_member('IPv4fake_sgid', '10.0.0.2'),
+            mock.call.create_ipset_chain('IPv6fake_sgid', 'IPv6'),
+            mock.call.refresh_ipset_chain_by_name(
+                'IPv6fake_sgid', ['fe80::1'], 'IPv6')]
+
+        self.firewall.ipset.assert_has_calls(calls, True)
+
+    def test_prepare_port_filter_change_beyond_9(self):
+        self.firewall.sg_rules = self._fake_sg_rule()
+        self.firewall.ipset_chains = {'IPv4fake_sgid': TEST_IP_RANGE[5:]}
+        self.firewall.sg_members = {'fake_sgid': {
+            'IPv4': TEST_IP_RANGE[:5],
+            'IPv6': ['fe80::1']}}
+        self.firewall.pre_sg_members = {'fake_sgid': {
+            'IPv4': TEST_IP_RANGE[5:],
+            'IPv6': ['fe80::1']}}
+        port = self._fake_port()
+        self.firewall.prepare_port_filter(port)
+        calls = [
+            mock.call.refresh_ipset_chain_by_name('IPv4fake_sgid',
+                                                  TEST_IP_RANGE[:5], 'IPv4'),
+            mock.call.create_ipset_chain('IPv6fake_sgid', 'IPv6'),
+            mock.call.refresh_ipset_chain_by_name(
+                'IPv6fake_sgid', ['fe80::1'], 'IPv6')]
+
+        self.firewall.ipset.assert_has_calls(calls)

neutron/tests/unit/test_security_groups_rpc.py

@@ -1477,6 +1477,58 @@ CHAINS_2 = CHAINS_1 + '|i_port2|o_port2|s_port2'
 
 IPTABLES_ARG['chains'] = CHAINS_1
 
+IPSET_FILTER_1 = """# Generated by iptables_manager
+*filter
+:neutron-filter-top - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+[0:0] -A FORWARD -j neutron-filter-top
+[0:0] -A OUTPUT -j neutron-filter-top
+[0:0] -A neutron-filter-top -j %(bn)s-local
+[0:0] -A INPUT -j %(bn)s-INPUT
+[0:0] -A OUTPUT -j %(bn)s-OUTPUT
+[0:0] -A FORWARD -j %(bn)s-FORWARD
+[0:0] -A %(bn)s-sg-fallback -j DROP
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-INGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-INGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-i_port1
+[0:0] -A %(bn)s-i_port1 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-i_port1 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-i_port1 -s 10.0.0.2/32 -p udp -m udp --sport 67 --dport 68 \
+-j RETURN
+[0:0] -A %(bn)s-i_port1 -p tcp -m tcp --dport 22 -j RETURN
+[0:0] -A %(bn)s-i_port1 -m set --match-set IPv4security_group1 src -j \
+RETURN
+[0:0] -A %(bn)s-i_port1 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-o_port1
+[0:0] -A %(bn)s-INPUT %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-o_port1
+[0:0] -A %(bn)s-s_port1 -m mac --mac-source 12:34:56:78:9a:bc -s 10.0.0.3/32 \
+-j RETURN
+[0:0] -A %(bn)s-s_port1 -j DROP
+[0:0] -A %(bn)s-o_port1 -p udp -m udp --sport 68 --dport 67 -j RETURN
+[0:0] -A %(bn)s-o_port1 -j %(bn)s-s_port1
+[0:0] -A %(bn)s-o_port1 -p udp -m udp --sport 67 --dport 68 -j DROP
+[0:0] -A %(bn)s-o_port1 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-o_port1 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-o_port1 -j RETURN
+[0:0] -A %(bn)s-o_port1 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-sg-chain -j ACCEPT
+COMMIT
+# Completed by iptables_manager
+""" % IPTABLES_ARG
+
 IPTABLES_FILTER_1 = """# Generated by iptables_manager
 *filter
 :neutron-filter-top - [0:0]
@@ -1581,6 +1633,174 @@ COMMIT
 
 IPTABLES_ARG['chains'] = CHAINS_2
 
+IPSET_FILTER_2 = """# Generated by iptables_manager
+*filter
+:neutron-filter-top - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+[0:0] -A FORWARD -j neutron-filter-top
+[0:0] -A OUTPUT -j neutron-filter-top
+[0:0] -A neutron-filter-top -j %(bn)s-local
+[0:0] -A INPUT -j %(bn)s-INPUT
+[0:0] -A OUTPUT -j %(bn)s-OUTPUT
+[0:0] -A FORWARD -j %(bn)s-FORWARD
+[0:0] -A %(bn)s-sg-fallback -j DROP
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-INGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-INGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-i_port1
+[0:0] -A %(bn)s-i_port1 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-i_port1 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-i_port1 -s 10.0.0.2/32 -p udp -m udp --sport 67 --dport 68 \
+-j RETURN
+[0:0] -A %(bn)s-i_port1 -p tcp -m tcp --dport 22 -j RETURN
+[0:0] -A %(bn)s-i_port1 -m set --match-set IPv4security_group1 src -j \
+RETURN
+[0:0] -A %(bn)s-i_port1 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-o_port1
+[0:0] -A %(bn)s-INPUT %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-o_port1
+[0:0] -A %(bn)s-s_port1 -m mac --mac-source 12:34:56:78:9a:bc -s 10.0.0.3/32 \
+-j RETURN
+[0:0] -A %(bn)s-s_port1 -j DROP
+[0:0] -A %(bn)s-o_port1 -p udp -m udp --sport 68 --dport 67 -j RETURN
+[0:0] -A %(bn)s-o_port1 -j %(bn)s-s_port1
+[0:0] -A %(bn)s-o_port1 -p udp -m udp --sport 67 --dport 68 -j DROP
+[0:0] -A %(bn)s-o_port1 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-o_port1 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-o_port1 -j RETURN
+[0:0] -A %(bn)s-o_port1 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-INGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-INGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-i_port2
+[0:0] -A %(bn)s-i_port2 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-i_port2 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-i_port2 -s 10.0.0.2/32 -p udp -m udp --sport 67 --dport 68 \
+-j RETURN
+[0:0] -A %(bn)s-i_port2 -p tcp -m tcp --dport 22 -j RETURN
+[0:0] -A %(bn)s-i_port2 -m set --match-set IPv4security_group1 src -j \
+RETURN
+[0:0] -A %(bn)s-i_port2 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-EGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-o_port2
+[0:0] -A %(bn)s-INPUT %(physdev_mod)s --physdev-EGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-o_port2
+[0:0] -A %(bn)s-s_port2 -m mac --mac-source 12:34:56:78:9a:bd -s 10.0.0.4/32 \
+-j RETURN
+[0:0] -A %(bn)s-s_port2 -j DROP
+[0:0] -A %(bn)s-o_port2 -p udp -m udp --sport 68 --dport 67 -j RETURN
+[0:0] -A %(bn)s-o_port2 -j %(bn)s-s_port2
+[0:0] -A %(bn)s-o_port2 -p udp -m udp --sport 67 --dport 68 -j DROP
+[0:0] -A %(bn)s-o_port2 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-o_port2 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-o_port2 -j RETURN
+[0:0] -A %(bn)s-o_port2 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-sg-chain -j ACCEPT
+COMMIT
+# Completed by iptables_manager
+""" % IPTABLES_ARG
+
+IPSET_FILTER_2_3 = """# Generated by iptables_manager
+*filter
+:neutron-filter-top - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+[0:0] -A FORWARD -j neutron-filter-top
+[0:0] -A OUTPUT -j neutron-filter-top
+[0:0] -A neutron-filter-top -j %(bn)s-local
+[0:0] -A INPUT -j %(bn)s-INPUT
+[0:0] -A OUTPUT -j %(bn)s-OUTPUT
+[0:0] -A FORWARD -j %(bn)s-FORWARD
+[0:0] -A %(bn)s-sg-fallback -j DROP
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-INGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-INGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-i_port1
+[0:0] -A %(bn)s-i_port1 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-i_port1 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-i_port1 -s 10.0.0.2/32 -p udp -m udp --sport 67 --dport 68 \
+-j RETURN
+[0:0] -A %(bn)s-i_port1 -p tcp -m tcp --dport 22 -j RETURN
+[0:0] -A %(bn)s-i_port1 -m set --match-set IPv4security_group1 src -j \
+RETURN
+[0:0] -A %(bn)s-i_port1 -p icmp -j RETURN
+[0:0] -A %(bn)s-i_port1 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-o_port1
+[0:0] -A %(bn)s-INPUT %(physdev_mod)s --physdev-EGRESS tap_port1 \
+%(physdev_is_bridged)s -j %(bn)s-o_port1
+[0:0] -A %(bn)s-s_port1 -m mac --mac-source 12:34:56:78:9a:bc -s 10.0.0.3/32 \
+-j RETURN
+[0:0] -A %(bn)s-s_port1 -j DROP
+[0:0] -A %(bn)s-o_port1 -p udp -m udp --sport 68 --dport 67 -j RETURN
+[0:0] -A %(bn)s-o_port1 -j %(bn)s-s_port1
+[0:0] -A %(bn)s-o_port1 -p udp -m udp --sport 67 --dport 68 -j DROP
+[0:0] -A %(bn)s-o_port1 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-o_port1 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-o_port1 -j RETURN
+[0:0] -A %(bn)s-o_port1 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-INGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-INGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-i_port2
+[0:0] -A %(bn)s-i_port2 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-i_port2 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-i_port2 -s 10.0.0.2/32 -p udp -m udp --sport 67 --dport 68 \
+-j RETURN
+[0:0] -A %(bn)s-i_port2 -p tcp -m tcp --dport 22 -j RETURN
+[0:0] -A %(bn)s-i_port2 -m set --match-set IPv4security_group1 src -j \
+RETURN
+[0:0] -A %(bn)s-i_port2 -p icmp -j RETURN
+[0:0] -A %(bn)s-i_port2 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-sg-chain
+[0:0] -A %(bn)s-sg-chain %(physdev_mod)s --physdev-EGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-o_port2
+[0:0] -A %(bn)s-INPUT %(physdev_mod)s --physdev-EGRESS tap_port2 \
+%(physdev_is_bridged)s -j %(bn)s-o_port2
+[0:0] -A %(bn)s-s_port2 -m mac --mac-source 12:34:56:78:9a:bd -s 10.0.0.4/32 \
+-j RETURN
+[0:0] -A %(bn)s-s_port2 -j DROP
+[0:0] -A %(bn)s-o_port2 -p udp -m udp --sport 68 --dport 67 -j RETURN
+[0:0] -A %(bn)s-o_port2 -j %(bn)s-s_port2
+[0:0] -A %(bn)s-o_port2 -p udp -m udp --sport 67 --dport 68 -j DROP
+[0:0] -A %(bn)s-o_port2 -m state --state INVALID -j DROP
+[0:0] -A %(bn)s-o_port2 -m state --state RELATED,ESTABLISHED -j RETURN
+[0:0] -A %(bn)s-o_port2 -j RETURN
+[0:0] -A %(bn)s-o_port2 -j %(bn)s-sg-fallback
+[0:0] -A %(bn)s-sg-chain -j ACCEPT
+COMMIT
+# Completed by iptables_manager
+""" % IPTABLES_ARG
+
 IPTABLES_FILTER_2 = """# Generated by iptables_manager
 *filter
 :neutron-filter-top - [0:0]
@@ -2016,6 +2236,7 @@ class TestSecurityGroupAgentWithIptables(base.BaseTestCase):
             'lock_path',
             '$state_path/lock')
         set_firewall_driver(self.FIREWALL_DRIVER)
+        cfg.CONF.set_override('enable_ipset', False, group='SECURITYGROUP')
 
         self.agent = sg_rpc.SecurityGroupAgentRpcMixin()
         self.agent.context = None
@@ -2031,7 +2252,6 @@ class TestSecurityGroupAgentWithIptables(base.BaseTestCase):
 
         self.agent.init_firewall(
             defer_refresh_firewall=defer_refresh_firewall)
-
         self.iptables = self.agent.firewall.iptables
         # TODO(jlibosva) Get rid of mocking iptables execute and mock out
         # firewall instead
@@ -2307,6 +2527,58 @@ class TestSecurityGroupAgentEnhancedRpcWithIptables(
         self._verify_mock_calls()
 
 
+class TestSecurityGroupAgentEnhancedIpsetWithIptables(
+        TestSecurityGroupAgentEnhancedRpcWithIptables):
+    def setUp(self, defer_refresh_firewall=False):
+        super(TestSecurityGroupAgentEnhancedIpsetWithIptables, self).setUp(
+            defer_refresh_firewall)
+        self.agent.firewall.enable_ipset = True
+        self.ipset = self.agent.firewall.ipset
+        self.ipset_execute = mock.patch.object(self.ipset,
+                                               "execute").start()
+
+    def test_prepare_remove_port(self):
+        self.sg_info.return_value = self.devices_info1
+        self._replay_iptables(IPSET_FILTER_1, IPTABLES_FILTER_V6_1)
+        self._replay_iptables(IPTABLES_FILTER_EMPTY, IPTABLES_FILTER_V6_EMPTY)
+
+        self.agent.prepare_devices_filter(['tap_port1'])
+        self.agent.remove_devices_filter(['tap_port1'])
+
+        self._verify_mock_calls()
+
+    def test_security_group_member_updated(self):
+        self.sg_info.return_value = self.devices_info1
+        self._replay_iptables(IPSET_FILTER_1, IPTABLES_FILTER_V6_1)
+        self._replay_iptables(IPSET_FILTER_1, IPTABLES_FILTER_V6_1)
+        self._replay_iptables(IPSET_FILTER_2, IPTABLES_FILTER_V6_2)
+        self._replay_iptables(IPSET_FILTER_2, IPTABLES_FILTER_V6_2)
+        self._replay_iptables(IPSET_FILTER_1, IPTABLES_FILTER_V6_1)
+        self._replay_iptables(IPTABLES_FILTER_EMPTY, IPTABLES_FILTER_V6_EMPTY)
+
+        self.agent.prepare_devices_filter(['tap_port1'])
+        self.sg_info.return_value = self.devices_info2
+        self.agent.security_groups_member_updated(['security_group1'])
+        self.agent.prepare_devices_filter(['tap_port2'])
+        self.sg_info.return_value = self.devices_info1
+        self.agent.security_groups_member_updated(['security_group1'])
+        self.agent.remove_devices_filter(['tap_port2'])
+        self.agent.remove_devices_filter(['tap_port1'])
+
+        self._verify_mock_calls()
+
+    def test_security_group_rule_updated(self):
+        self.sg_info.return_value = self.devices_info2
+        self._replay_iptables(IPSET_FILTER_2, IPTABLES_FILTER_V6_2)
+        self._replay_iptables(IPSET_FILTER_2_3, IPTABLES_FILTER_V6_2)
+
+        self.agent.prepare_devices_filter(['tap_port1', 'tap_port3'])
+        self.sg_info.return_value = self.devices_info3
+        self.agent.security_groups_rule_updated(['security_group1'])
+
+        self._verify_mock_calls()
+
+
 class SGNotificationTestMixin():
     def test_security_group_rule_updated(self):
         name = 'webservers'

setup.cfg

@@ -38,6 +38,7 @@ data_files =
         etc/neutron/rootwrap.d/debug.filters
         etc/neutron/rootwrap.d/dhcp.filters
         etc/neutron/rootwrap.d/iptables-firewall.filters
+        etc/neutron/rootwrap.d/ipset-firewall.filters
         etc/neutron/rootwrap.d/l3.filters
         etc/neutron/rootwrap.d/lbaas-haproxy.filters
         etc/neutron/rootwrap.d/linuxbridge-plugin.filters