Add NUMAClient
The new client returns information about a hosts's NUMA characteristics. Original-review: https://review.rdoproject.org/r/#/c/18742/ Change-Id: I623ebc171a77309ff52844cbb9fc61a8d81ba6c6
This commit is contained in:
Artom Lifshitz
parent
4dd073afdb
commit
2236646b3d
@ -68,3 +68,54 @@ class NovaConfigClient(SSHClient):
|
|||||||
config = configparser.ConfigParser()
|
config = configparser.ConfigParser()
|
||||||
config.readfp(StringIO(self._read_nova_conf()))
|
config.readfp(StringIO(self._read_nova_conf()))
|
||||||
return config.get(section, option)
|
return config.get(section, option)
|
||||||
|
|
||||||
|
|
||||||
|
class NUMAClient(SSHClient):
|
||||||
|
"""A client to get host NUMA information. `numactl` needs to be installed
|
||||||
|
in the environment or container(s).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get_host_topology(self):
|
||||||
|
"""Returns the host topology as a dict.
|
||||||
|
|
||||||
|
:return nodes: A dict of CPUs in each host NUMA node, keyed by host
|
||||||
|
node number, for example: {0: [1, 2],
|
||||||
|
1: [3, 4]}
|
||||||
|
"""
|
||||||
|
nodes = {}
|
||||||
|
numactl = self.execute('numactl -H', sudo=True)
|
||||||
|
for line in StringIO(numactl).readlines():
|
||||||
|
if 'node' in line and 'cpus' in line:
|
||||||
|
cpus = [int(cpu) for cpu in line.split(':')[1].split()]
|
||||||
|
node = int(line.split()[1])
|
||||||
|
nodes[node] = cpus
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
def get_num_cpus(self):
|
||||||
|
nodes = self.get_host_topology()
|
||||||
|
return sum([len(cpus) for cpus in nodes.values()])
|
||||||
|
|
||||||
|
def get_pagesize(self):
|
||||||
|
proc_meminfo = self.execute('cat /proc/meminfo')
|
||||||
|
for line in StringIO(proc_meminfo).readlines():
|
||||||
|
if line.startswith('Hugepagesize'):
|
||||||
|
return int(line.split(':')[1].split()[0])
|
||||||
|
|
||||||
|
def get_hugepages(self):
|
||||||
|
"""Returns a nested dict of number of total and free pages, keyed by
|
||||||
|
NUMA node. For example:
|
||||||
|
|
||||||
|
{0: {'total': 2000, 'free': 2000},
|
||||||
|
1: {'total': 2000, 'free': 0}}
|
||||||
|
"""
|
||||||
|
pages = {}
|
||||||
|
for node in self.get_host_topology():
|
||||||
|
meminfo = self.execute(
|
||||||
|
'cat /sys/devices/system/node/node%d/meminfo' % node)
|
||||||
|
for line in StringIO(meminfo).readlines():
|
||||||
|
if 'HugePages_Total' in line:
|
||||||
|
total = int(line.split(':')[1].lstrip())
|
||||||
|
if 'HugePages_Free' in line:
|
||||||
|
free = int(line.split(':')[1].lstrip())
|
||||||
|
pages[node] = {'total': total, 'free': free}
|
||||||
|
return pages
|
||||||
|
|||||||
@ -74,3 +74,170 @@ class ConfigClientTestCase(base.WhiteboxPluginTestCase):
|
|||||||
return_value=fake_config):
|
return_value=fake_config):
|
||||||
self.assertEqual(config_client.getopt('default', 'fake-key'),
|
self.assertEqual(config_client.getopt('default', 'fake-key'),
|
||||||
'fake-value')
|
'fake-value')
|
||||||
|
|
||||||
|
|
||||||
|
class NUMAClientTestCase(base.WhiteboxPluginTestCase):
|
||||||
|
|
||||||
|
fake_numactl_output = textwrap.dedent("""
|
||||||
|
available: 2 nodes (0-1)
|
||||||
|
node 0 cpus: 0 1 2
|
||||||
|
node 0 size: 7793 MB
|
||||||
|
node 0 free: 1640 MB
|
||||||
|
node 1 cpus: 3 4
|
||||||
|
node 1 size: 7875 MB
|
||||||
|
node 1 free: 4059 MB
|
||||||
|
node distances:
|
||||||
|
node 0 1
|
||||||
|
0: 10 20
|
||||||
|
1: 20 10""")
|
||||||
|
|
||||||
|
fake_proc_meminfo = textwrap.dedent("""
|
||||||
|
MemTotal: 16035320 kB
|
||||||
|
MemFree: 481884 kB
|
||||||
|
MemAvailable: 2489036 kB
|
||||||
|
Buffers: 4128 kB
|
||||||
|
Cached: 2246912 kB
|
||||||
|
SwapCached: 572 kB
|
||||||
|
Active: 5348564 kB
|
||||||
|
Inactive: 1375316 kB
|
||||||
|
Active(anon): 4165752 kB
|
||||||
|
Inactive(anon): 347028 kB
|
||||||
|
Active(file): 1182812 kB
|
||||||
|
Inactive(file): 1028288 kB
|
||||||
|
Unevictable: 112816 kB
|
||||||
|
Mlocked: 112816 kB
|
||||||
|
SwapTotal: 629756 kB
|
||||||
|
SwapFree: 625648 kB
|
||||||
|
Dirty: 68 kB
|
||||||
|
Writeback: 0 kB
|
||||||
|
AnonPages: 4429328 kB
|
||||||
|
Mapped: 329536 kB
|
||||||
|
Shmem: 4052 kB
|
||||||
|
KReclaimable: 165572 kB
|
||||||
|
Slab: 350468 kB
|
||||||
|
SReclaimable: 165572 kB
|
||||||
|
SUnreclaim: 184896 kB
|
||||||
|
KernelStack: 13616 kB
|
||||||
|
PageTables: 26564 kB
|
||||||
|
NFS_Unstable: 0 kB
|
||||||
|
Bounce: 0 kB
|
||||||
|
WritebackTmp: 0 kB
|
||||||
|
CommitLimit: 4551416 kB
|
||||||
|
Committed_AS: 10445028 kB
|
||||||
|
VmallocTotal: 34359738367 kB
|
||||||
|
VmallocUsed: 0 kB
|
||||||
|
VmallocChunk: 0 kB
|
||||||
|
Percpu: 6600 kB
|
||||||
|
HardwareCorrupted: 0 kB
|
||||||
|
AnonHugePages: 0 kB
|
||||||
|
ShmemHugePages: 0 kB
|
||||||
|
ShmemPmdMapped: 0 kB
|
||||||
|
CmaTotal: 0 kB
|
||||||
|
CmaFree: 0 kB
|
||||||
|
HugePages_Total: 4000
|
||||||
|
HugePages_Free: 4000
|
||||||
|
HugePages_Rsvd: 0
|
||||||
|
HugePages_Surp: 0
|
||||||
|
Hugepagesize: 2048 kB
|
||||||
|
Hugetlb: 8192000 kB
|
||||||
|
DirectMap4k: 391032 kB
|
||||||
|
DirectMap2M: 10749952 kB
|
||||||
|
DirectMap1G: 5242880 kB""")
|
||||||
|
|
||||||
|
fake_node0_meminfo = textwrap.dedent("""
|
||||||
|
Node 0 MemTotal: 7971444 kB
|
||||||
|
Node 0 MemFree: 138612 kB
|
||||||
|
Node 0 MemUsed: 7832832 kB
|
||||||
|
Node 0 Active: 2832660 kB
|
||||||
|
Node 0 Inactive: 586824 kB
|
||||||
|
Node 0 Active(anon): 2538412 kB
|
||||||
|
Node 0 Inactive(anon): 326172 kB
|
||||||
|
Node 0 Active(file): 294248 kB
|
||||||
|
Node 0 Inactive(file): 260652 kB
|
||||||
|
Node 0 Unevictable: 105392 kB
|
||||||
|
Node 0 Mlocked: 105392 kB
|
||||||
|
Node 0 Dirty: 136 kB
|
||||||
|
Node 0 Writeback: 0 kB
|
||||||
|
Node 0 FilePages: 589864 kB
|
||||||
|
Node 0 Mapped: 126096 kB
|
||||||
|
Node 0 AnonPages: 2855072 kB
|
||||||
|
Node 0 Shmem: 2024 kB
|
||||||
|
Node 0 KernelStack: 3720 kB
|
||||||
|
Node 0 PageTables: 13308 kB
|
||||||
|
Node 0 NFS_Unstable: 0 kB
|
||||||
|
Node 0 Bounce: 0 kB
|
||||||
|
Node 0 WritebackTmp: 0 kB
|
||||||
|
Node 0 KReclaimable: 54788 kB
|
||||||
|
Node 0 Slab: 142752 kB
|
||||||
|
Node 0 SReclaimable: 54788 kB
|
||||||
|
Node 0 SUnreclaim: 87964 kB
|
||||||
|
Node 0 AnonHugePages: 0 kB
|
||||||
|
Node 0 ShmemHugePages: 0 kB
|
||||||
|
Node 0 ShmemPmdMapped: 0 kB
|
||||||
|
Node 0 HugePages_Total: 4000
|
||||||
|
Node 0 HugePages_Free: 3000
|
||||||
|
Node 0 HugePages_Surp: 0""")
|
||||||
|
|
||||||
|
fake_node1_meminfo = textwrap.dedent("""
|
||||||
|
Node 1 MemTotal: 8063876 kB
|
||||||
|
Node 1 MemFree: 323536 kB
|
||||||
|
Node 1 MemUsed: 7740340 kB
|
||||||
|
Node 1 Active: 2520684 kB
|
||||||
|
Node 1 Inactive: 795228 kB
|
||||||
|
Node 1 Active(anon): 1632468 kB
|
||||||
|
Node 1 Inactive(anon): 20852 kB
|
||||||
|
Node 1 Active(file): 888216 kB
|
||||||
|
Node 1 Inactive(file): 774376 kB
|
||||||
|
Node 1 Unevictable: 7424 kB
|
||||||
|
Node 1 Mlocked: 7424 kB
|
||||||
|
Node 1 Dirty: 308 kB
|
||||||
|
Node 1 Writeback: 0 kB
|
||||||
|
Node 1 FilePages: 1668208 kB
|
||||||
|
Node 1 Mapped: 206436 kB
|
||||||
|
Node 1 AnonPages: 1579356 kB
|
||||||
|
Node 1 Shmem: 2052 kB
|
||||||
|
Node 1 KernelStack: 9960 kB
|
||||||
|
Node 1 PageTables: 13452 kB
|
||||||
|
Node 1 NFS_Unstable: 0 kB
|
||||||
|
Node 1 Bounce: 0 kB
|
||||||
|
Node 1 WritebackTmp: 0 kB
|
||||||
|
Node 1 KReclaimable: 112860 kB
|
||||||
|
Node 1 Slab: 210308 kB
|
||||||
|
Node 1 SReclaimable: 112860 kB
|
||||||
|
Node 1 SUnreclaim: 97448 kB
|
||||||
|
Node 1 AnonHugePages: 0 kB
|
||||||
|
Node 1 ShmemHugePages: 0 kB
|
||||||
|
Node 1 ShmemPmdMapped: 0 kB
|
||||||
|
Node 1 HugePages_Total: 2000
|
||||||
|
Node 1 HugePages_Free: 1000
|
||||||
|
Node 1 HugePages_Surp: 0""")
|
||||||
|
|
||||||
|
def test_get_hugepages(self):
|
||||||
|
numa_client = clients.NUMAClient('fake-host')
|
||||||
|
with mock.patch.object(numa_client, 'execute',
|
||||||
|
side_effect=(self.fake_numactl_output,
|
||||||
|
self.fake_node0_meminfo,
|
||||||
|
self.fake_node1_meminfo)):
|
||||||
|
self.assertEqual(numa_client.get_hugepages(),
|
||||||
|
{0: {'total': 4000, 'free': 3000},
|
||||||
|
1: {'total': 2000, 'free': 1000}})
|
||||||
|
|
||||||
|
def test_get_pagesize(self):
|
||||||
|
numa_client = clients.NUMAClient('fake-host')
|
||||||
|
with mock.patch.object(numa_client, 'execute',
|
||||||
|
return_value=self.fake_proc_meminfo):
|
||||||
|
self.assertEqual(numa_client.get_pagesize(), 2048)
|
||||||
|
|
||||||
|
def test_get_host_topology(self):
|
||||||
|
numa_client = clients.NUMAClient('fake-host')
|
||||||
|
with mock.patch.object(numa_client, 'execute',
|
||||||
|
return_value=self.fake_numactl_output):
|
||||||
|
self.assertEqual(numa_client.get_host_topology(),
|
||||||
|
{0: [0, 1, 2],
|
||||||
|
1: [3, 4]})
|
||||||
|
|
||||||
|
def test_get_num_cpus(self):
|
||||||
|
numa_client = clients.NUMAClient('fake-host')
|
||||||
|
with mock.patch.object(numa_client, 'execute',
|
||||||
|
return_value=self.fake_numactl_output):
|
||||||
|
self.assertEqual(5, numa_client.get_num_cpus())
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user