Merge "Allow IPv6 addresses/hostnames in StatsD target"

doc/manpages/proxy-server.conf.5

@@ -119,7 +119,9 @@ If set, log_udp_host will override log_address.
 .IP "\fBlog_udp_port\fR
 UDP log port, the default is 514.
 .IP \fBlog_statsd_host\fR = localhost
-log_statsd_*  enable StatsD logging.
+log_statsd_*  enable StatsD logging.  IPv4/IPv6 addresses and hostnames are
+supported.  If a hostname resolves to an IPv4 and IPv6 address, the IPv4
+address will be used.
 .IP \fBlog_statsd_port\fR
 The default is 8125.
 .IP \fBlog_statsd_default_sample_rate\fR

doc/source/admin_guide.rst

@@ -629,7 +629,11 @@ configuration entries (see the sample configuration files)::
     log_statsd_metric_prefix =                [empty-string]
 
 If `log_statsd_host` is not set, this feature is disabled.  The default values
-for the other settings are given above.
+for the other settings are given above.  The `log_statsd_host` can be a
+hostname, an IPv4 address, or an IPv6 address (not surrounded with brackets, as
+this is unnecessary since the port is specified separately).  If a hostname
+resolves to an IPv4 address, an IPv4 socket will be used to send StatsD UDP
+packets, even if the hostname would also resolve to an IPv6 address.
 
 .. _StatsD: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/
 .. _Graphite: http://graphite.wikidot.com/

doc/source/deployment_guide.rst

@@ -478,7 +478,11 @@ log_custom_handlers              None        Comma-separated list of functions t
                                              to setup custom log handlers.
 log_udp_host                                 Override log_address
 log_udp_port                     514         UDP log port
-log_statsd_host                  localhost   StatsD logging
+log_statsd_host                  localhost   StatsD logging; IPv4/IPv6
+                                             address or a hostname.  If a
+                                             hostname resolves to an IPv4 and IPv6
+                                             address, the IPv4 address will be
+                                             used.
 log_statsd_port                  8125
 log_statsd_default_sample_rate   1.0
 log_statsd_sample_rate_factor    1.0

swift/common/utils.py

@@ -1174,10 +1174,44 @@ class StatsdClient(object):
         self.set_prefix(tail_prefix)
         self._default_sample_rate = default_sample_rate
         self._sample_rate_factor = sample_rate_factor
-        self._target = (self._host, self._port)
         self.random = random
         self.logger = logger
 
+        # Determine if host is IPv4 or IPv6
+        addr_info = None
+        try:
+            addr_info = socket.getaddrinfo(host, port, socket.AF_INET)
+            self._sock_family = socket.AF_INET
+        except socket.gaierror:
+            try:
+                addr_info = socket.getaddrinfo(host, port, socket.AF_INET6)
+                self._sock_family = socket.AF_INET6
+            except socket.gaierror:
+                # Don't keep the server from starting from what could be a
+                # transient DNS failure.  Any hostname will get re-resolved as
+                # necessary in the .sendto() calls.
+                # However, we don't know if we're IPv4 or IPv6 in this case, so
+                # we assume legacy IPv4.
+                self._sock_family = socket.AF_INET
+
+        # NOTE: we use the original host value, not the DNS-resolved one
+        # because if host is a hostname, we don't want to cache the DNS
+        # resolution for the entire lifetime of this process.  Let standard
+        # name resolution caching take effect.  This should help operators use
+        # DNS trickery if they want.
+        if addr_info is not None:
+            # addr_info is a list of 5-tuples with the following structure:
+            #     (family, socktype, proto, canonname, sockaddr)
+            # where sockaddr is the only thing of interest to us, and we only
+            # use the first result.  We want to use the originally supplied
+            # host (see note above) and the remainder of the variable-length
+            # sockaddr: IPv4 has (address, port) while IPv6 has (address,
+            # port, flow info, scope id).
+            sockaddr = addr_info[0][-1]
+            self._target = (host,) + (sockaddr[1:])
+        else:
+            self._target = (host, port)
+
     def set_prefix(self, new_prefix):
         if new_prefix and self._base_prefix:
             self._prefix = '.'.join([self._base_prefix, new_prefix, ''])
@@ -1212,7 +1246,7 @@ class StatsdClient(object):
                         self._target, err)
 
     def _open_socket(self):
-        return socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        return socket.socket(self._sock_family, socket.SOCK_DGRAM)
 
     def update_stats(self, m_name, m_value, sample_rate=None):
         return self._send(m_name, m_value, 'c', sample_rate)

test/unit/common/test_utils.py

@@ -3652,6 +3652,95 @@ class TestStatsdLogging(unittest.TestCase):
         self.assertEqual(logger.logger.statsd_client._sample_rate_factor,
                          0.81)
 
+    def test_ipv4_or_ipv6_hostname_defaults_to_ipv4(self):
+        def stub_getaddrinfo_both_ipv4_and_ipv6(host, port, family, *rest):
+            if family == socket.AF_INET:
+                return [(socket.AF_INET, 'blah', 'blah', 'blah',
+                        ('127.0.0.1', int(port)))]
+            elif family == socket.AF_INET6:
+                # Implemented so an incorrectly ordered implementation (IPv6
+                # then IPv4) would realistically fail.
+                return [(socket.AF_INET6, 'blah', 'blah', 'blah',
+                        ('::1', int(port), 0, 0))]
+
+        with mock.patch.object(utils.socket, 'getaddrinfo',
+                               new=stub_getaddrinfo_both_ipv4_and_ipv6):
+            logger = utils.get_logger({
+                'log_statsd_host': 'localhost',
+                'log_statsd_port': '9876',
+            }, 'some-name', log_route='some-route')
+        statsd_client = logger.logger.statsd_client
+
+        self.assertEqual(statsd_client._sock_family, socket.AF_INET)
+        self.assertEqual(statsd_client._target, ('localhost', 9876))
+
+        got_sock = statsd_client._open_socket()
+        self.assertEqual(got_sock.family, socket.AF_INET)
+
+    def test_ipv4_instantiation_and_socket_creation(self):
+        logger = utils.get_logger({
+            'log_statsd_host': '127.0.0.1',
+            'log_statsd_port': '9876',
+        }, 'some-name', log_route='some-route')
+        statsd_client = logger.logger.statsd_client
+
+        self.assertEqual(statsd_client._sock_family, socket.AF_INET)
+        self.assertEqual(statsd_client._target, ('127.0.0.1', 9876))
+
+        got_sock = statsd_client._open_socket()
+        self.assertEqual(got_sock.family, socket.AF_INET)
+
+    def test_ipv6_instantiation_and_socket_creation(self):
+        # We have to check the given hostname or IP for IPv4/IPv6 on logger
+        # instantiation so we don't call getaddrinfo() too often and don't have
+        # to call bind() on our socket to detect IPv4/IPv6 on every send.
+        logger = utils.get_logger({
+            'log_statsd_host': '::1',
+            'log_statsd_port': '9876',
+        }, 'some-name', log_route='some-route')
+        statsd_client = logger.logger.statsd_client
+
+        self.assertEqual(statsd_client._sock_family, socket.AF_INET6)
+        self.assertEqual(statsd_client._target, ('::1', 9876, 0, 0))
+
+        got_sock = statsd_client._open_socket()
+        self.assertEqual(got_sock.family, socket.AF_INET6)
+
+    def test_bad_hostname_instantiation(self):
+        logger = utils.get_logger({
+            'log_statsd_host': 'i-am-not-a-hostname-or-ip',
+            'log_statsd_port': '9876',
+        }, 'some-name', log_route='some-route')
+        statsd_client = logger.logger.statsd_client
+
+        self.assertEqual(statsd_client._sock_family, socket.AF_INET)
+        self.assertEqual(statsd_client._target,
+                         ('i-am-not-a-hostname-or-ip', 9876))
+
+        got_sock = statsd_client._open_socket()
+        self.assertEqual(got_sock.family, socket.AF_INET)
+        # Maybe the DNS server gets fixed in a bit and it starts working... or
+        # maybe the DNS record hadn't propagated yet.  In any case, failed
+        # statsd sends will warn in the logs until the DNS failure or invalid
+        # IP address in the configuration is fixed.
+
+    def test_sending_ipv6(self):
+        logger = utils.get_logger({
+            'log_statsd_host': '::1',
+            'log_statsd_port': '9876',
+        }, 'some-name', log_route='some-route')
+        statsd_client = logger.logger.statsd_client
+
+        fl = FakeLogger()
+        statsd_client.logger = fl
+        mock_socket = MockUdpSocket()
+
+        statsd_client._open_socket = lambda *_: mock_socket
+        logger.increment('tunafish')
+        self.assertEqual(fl.get_lines_for_level('warning'), [])
+        self.assertEqual(mock_socket.sent,
+                         [(b'some-name.tunafish:1|c', ('::1', 9876, 0, 0))])
+
     def test_no_exception_when_cant_send_udp_packet(self):
         logger = utils.get_logger({'log_statsd_host': 'some.host.com'})
         statsd_client = logger.logger.statsd_client