openstack/kolla-ansible
Code Issues Proposed changes

Merge "Add unit tests for Nova Cells filters"

Browse Source
This commit is contained in:
Zuul 2019-10-21 14:48:32 +00:00 committed by Gerrit Code Review
commit 44d998e506
5 changed files with 303 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
ansible/roles/nova-cell/filter_plugins/filters.py
View File

@ -1,5 +1,7 @@
#!/usr/bin/python
# Copyright (c) 2019 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -12,82 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from jinja2.exceptions import TemplateRuntimeError
import re
from kolla_ansible import nova_filters as filters
class FilterModule(object):
"""Nova cell filters."""
def filters(self):
return {
'extract_cell': self._extract_cell,
'namespace_haproxy_for_cell': self._namespace_haproxy_for_cell,
}
def _extract_cell(self, list_cells_cli_output, cell_name):
"""Extract cell settings from nova_manage CLI
This filter tries to extract the cell settings for the specified cell
from the output of the command:
nova-manage cell_v2 list_cells --verbose
If the cell is not registered, nothing is returned.
An example line from this command for a cell with no name looks like this:
| | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:password@1.2.3.4:5672 | mysql+pymysql://nova:password@1.2.3.4:3306/nova | False | # noqa
And for a cell with a name:
| cell1 | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:password@1.2.3.4:5672 | mysql+pymysql://nova:password@1.2.3.4:3306/nova | False | # noqa
"""
# NOTE(priteau): regexp doesn't support passwords containing spaces
p = re.compile(
r'\| +(?P<cell_name>[^ ]+)? +'
r'\| +(?!00000000-0000-0000-0000-000000000000)'
r'(?P<cell_uuid>[0-9a-f\-]+) +'
r'\| +(?P<cell_message_queue>[^ ]+) +'
r'\| +(?P<cell_database>[^ ]+) +'
r'\| +(?P<cell_disabled>[^ ]+) +'
r'\|$')
cells = []
for line in list_cells_cli_output['stdout_lines']:
match = p.match(line)
if match:
# If there is no cell name, we get None in the cell_name match
# group. Use an empty string to match the default cell.
match_cell_name = match.group('cell_name') or ""
if match_cell_name == cell_name:
cells.append(match.groupdict())
if len(cells) > 1:
raise TemplateRuntimeError(
"Cell: {} has duplicates. "
"Manual cleanup required.".format(cell_name))
return cells[0] if cells else None
def _namespace_haproxy_for_cell(self, services, cell_name):
"""Add namespacing to HAProxy configuration for a cell.
:param services: dict defining service configuration.
:param cell_name: name of the cell, or empty if cell has no name.
:returns: the services dict, with haproxy configuration modified to
provide namespacing between cells.
"""
def _namespace(name):
# Backwards compatibility - no cell name suffix for cells without a
# name.
return "{}_{}".format(name, cell_name) if cell_name else name
# Service name must be namespaced as haproxy-config uses this as the
# config file name.
services = {
_namespace(service_name): service
for service_name, service in services.items()
}
for service in services.values():
if service.get('haproxy'):
service['haproxy'] = {
_namespace(name): service['haproxy'][name]
for name in service['haproxy']
}
return services
return filters.get_filters()

93
kolla_ansible/nova_filters.py Normal file
View File

@ -0,0 +1,93 @@
# Copyright (c) 2019 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import jinja2
import re
def extract_cell(list_cells_cli_output, cell_name):
"""Extract cell settings from nova_manage CLI
This filter tries to extract the cell settings for the specified cell
from the output of the command:
nova-manage cell_v2 list_cells --verbose
If the cell is not registered, nothing is returned.
An example line from this command for a cell with no name looks like this:
| | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:password@1.2.3.4:5672 | mysql+pymysql://nova:password@1.2.3.4:3306/nova | False | # noqa
And for a cell with a name:
| cell1 | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:password@1.2.3.4:5672 | mysql+pymysql://nova:password@1.2.3.4:3306/nova | False | # noqa
"""
# NOTE(priteau): regexp doesn't support passwords containing spaces
p = re.compile(
r'\| +(?P<cell_name>[^ ]+)? +'
r'\| +(?!00000000-0000-0000-0000-000000000000)'
r'(?P<cell_uuid>[0-9a-f\-]+) +'
r'\| +(?P<cell_message_queue>[^ ]+) +'
r'\| +(?P<cell_database>[^ ]+) +'
r'\| +(?P<cell_disabled>[^ ]+) +'
r'\|$')
cells = []
for line in list_cells_cli_output['stdout_lines']:
match = p.match(line)
if match:
# If there is no cell name, we get None in the cell_name match
# group. Use an empty string to match the default cell.
match_cell_name = match.group('cell_name') or ""
if match_cell_name == cell_name:
cells.append(match.groupdict())
if len(cells) > 1:
raise jinja2.TemplateRuntimeError(
"Cell: {} has duplicates. "
"Manual cleanup required.".format(cell_name))
return cells[0] if cells else None
def namespace_haproxy_for_cell(services, cell_name):
"""Add namespacing to HAProxy configuration for a cell.
:param services: dict defining service configuration.
:param cell_name: name of the cell, or empty if cell has no name.
:returns: the services dict, with haproxy configuration modified to
provide namespacing between cells.
"""
def _namespace(name):
# Backwards compatibility - no cell name suffix for cells without a
# name.
return "{}_{}".format(name, cell_name) if cell_name else name
# Service name must be namespaced as haproxy-config uses this as the
# config file name.
services = {
_namespace(service_name): service
for service_name, service in services.items()
}
for service in services.values():
if service.get('haproxy'):
service['haproxy'] = {
_namespace(name): service['haproxy'][name]
for name in service['haproxy']
}
return services
def get_filters():
return {
"extract_cell": extract_cell,
"namespace_haproxy_for_cell": namespace_haproxy_for_cell,
}

7
kolla_ansible/tests/unit/data/nova_manage_cli_output_duplicate_cells Normal file
View File

@ -0,0 +1,7 @@
+----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+
| Name | UUID | Transport URL | Database Connection | Disabled |
+----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+
| cell0 | 00000000-0000-0000-0000-000000000000 | none:/// | mysql+pymysql://nova:gTEDJEJU24ZXJjhhEt6Xu4nP3fnvPAoQ5oOb8OnS@172.28.128.253:3306/nova_cell0 | False |
| cell0001 | 5af50f3f-82ed-47b2-9de8-7dd4eafa7648 | rabbit://openstack:0flZc3qqwczsisbaT94uCcArVNXEJnhhj4JAAAMI@172.28.128.32:5673/nova_cell0001 | mysql+pymysql://nova:mpyerUgpbQeXjaDV1nhsVDLWjsFSoCss6dHCXK7G@172.28.128.252:3305/nova_cell0001 | False |
| cell0001 | 5af50f3f-82ed-47b2-9de8-7dd4eafa7648 | rabbit://openstack:0flZc3qqwczsisbaT94uCcArVNXEJnhhj4JAAAMI@172.28.128.32:5673/nova_cell0001 | mysql+pymysql://nova:mpyerUgpbQeXjaDV1nhsVDLWjsFSoCss6dHCXK7G@172.28.128.252:3305/nova_cell0001 | False |
+----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+

8
kolla_ansible/tests/unit/data/nova_manage_cli_output_multiple_cells Normal file
View File

@ -0,0 +1,8 @@
+----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+
| Name | UUID | Transport URL | Database Connection | Disabled |
+----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+
| cell0 | 00000000-0000-0000-0000-000000000000 | none:/// | mysql+pymysql://nova:gTEDJEJU24ZXJjhhEt6Xu4nP3fnvPAoQ5oOb8OnS@172.28.128.253:3306/nova_cell0 | False |
| | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:94uCcArVNXEJnhhj4JAAAMI432h5j3k2bb3bbjkB@172.28.128.30:5672 | mysql+pymysql://nova:1nhsVDLWjsFSoCsda453bJBsdsbjkabf77sadsdD@172.28.128.252:3305/nova | False |
| cell0001 | 5af50f3f-82ed-47b2-9de8-7dd4eafa7648 | rabbit://openstack:0flZc3qqwczsisbaT94uCcArVNXEJnhhj4JAAAMI@172.28.128.32:5673/nova_cell0001 | mysql+pymysql://nova:mpyerUgpbQeXjaDV1nhsVDLWjsFSoCss6dHCXK7G@172.28.128.252:3305/nova_cell0001 | False |
| cell0002 | 087697bb-bfc2-471d-befb-0c0fcc7630e4 | rabbit://openstack:d9LaCxGrQX9Lla9aMZcS7fQ5xLH8S5HwHFrl6jdJ@172.28.128.31:5672/nova_cell0002 | mysql+pymysql://nova:Dj3f3l4yv2SuhbsJyv3ahGIwUMa9TKchw6EXtQfp@172.28.128.253:3306/nova_cell0002 | True |
+----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+

189
kolla_ansible/tests/unit/test_nova_filters.py Normal file
View File

@ -0,0 +1,189 @@
# Copyright (c) 2019 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
import jinja2
import os
from kolla_ansible import nova_filters as filters
class TestFilters(unittest.TestCase):
def _test_extract_cell(self, test_data, cell_name):
nova_manage_output = {}
with open(test_data, 'r') as f:
nova_manage_output['stdout_lines'] = f.readlines()
return filters.extract_cell(nova_manage_output, cell_name)
def test_extract_with_no_name(self):
test_data = os.path.join(os.path.dirname(
__file__), 'data', 'nova_manage_cli_output_multiple_cells')
actual = self._test_extract_cell(test_data, '')
expected = {'cell_name': None,
'cell_uuid': '68a3f49e-27ec-422f-9e2e-2a4e5dc8291b',
'cell_message_queue': 'rabbit://openstack:94uCcArVNXEJnhhj4JAAAMI432h5j3k2bb3bbjkB@172.28.128.30:5672', # noqa
'cell_database': 'mysql+pymysql://nova:1nhsVDLWjsFSoCsda453bJBsdsbjkabf77sadsdD@172.28.128.252:3305/nova', # noqa
'cell_disabled': 'False'}
self.assertDictEqual(expected, actual)
def test_extract_cell0001(self):
test_data = os.path.join(os.path.dirname(
__file__), 'data', 'nova_manage_cli_output_multiple_cells')
actual = self._test_extract_cell(test_data, 'cell0001')
expected = {'cell_name': 'cell0001',
'cell_uuid': '5af50f3f-82ed-47b2-9de8-7dd4eafa7648',
'cell_message_queue': 'rabbit://openstack:0flZc3qqwczsisbaT94uCcArVNXEJnhhj4JAAAMI@172.28.128.32:5673/nova_cell0001', # noqa
'cell_database': 'mysql+pymysql://nova:mpyerUgpbQeXjaDV1nhsVDLWjsFSoCss6dHCXK7G@172.28.128.252:3305/nova_cell0001', # noqa
'cell_disabled': 'False'}
self.assertDictEqual(expected, actual)
def test_extract_cell0002(self):
test_data = os.path.join(os.path.dirname(
__file__), 'data', 'nova_manage_cli_output_multiple_cells')
actual = self._test_extract_cell(test_data, 'cell0002')
expected = {'cell_name': 'cell0002',
'cell_uuid': '087697bb-bfc2-471d-befb-0c0fcc7630e4',
'cell_message_queue': 'rabbit://openstack:d9LaCxGrQX9Lla9aMZcS7fQ5xLH8S5HwHFrl6jdJ@172.28.128.31:5672/nova_cell0002', # noqa
'cell_database': 'mysql+pymysql://nova:Dj3f3l4yv2SuhbsJyv3ahGIwUMa9TKchw6EXtQfp@172.28.128.253:3306/nova_cell0002', # noqa
'cell_disabled': 'True'}
self.assertDictEqual(expected, actual)
def test_extract_missing_cell(self):
test_data = os.path.join(os.path.dirname(
__file__), 'data', 'nova_manage_cli_output_multiple_cells')
actual = self._test_extract_cell(test_data, 'cell0003')
self.assertIsNone(actual)
def test_extract_duplicate_cell(self):
test_data = os.path.join(os.path.dirname(
__file__), 'data', 'nova_manage_cli_output_duplicate_cells')
self.assertRaisesRegexp(jinja2.TemplateRuntimeError, 'duplicates',
self._test_extract_cell, test_data, 'cell0001')
def test_namespace_haproxy_for_cell_with_empty_name(self):
example_services = {
'nova-novncproxy': {
'group': 'some_group',
'enabled': True,
'haproxy': {
'nova_novncproxy': {
'enabled': True,
'mode': 'http',
'external': False,
'port': '1232',
'listen_port': '1233',
'backend_http_extra': ['timeout tunnel 1h'],
},
'nova_novncproxy_external': {
'enabled': True,
'mode': 'http',
'external': True,
'port': '1234',
'listen_port': '1235',
'backend_http_extra': ['timeout tunnel 1h'],
}
}
}
}
actual = filters.namespace_haproxy_for_cell(example_services, '')
# No change
self.assertDictEqual(example_services, actual)
def test_namespace_haproxy_for_cell_with_single_proxy(self):
example_services = {
'nova-novncproxy': {
'group': 'some_group',
'enabled': True,
'haproxy': {
'nova_novncproxy': {
'enabled': True,
'mode': 'http',
'external': False,
'port': '1232',
'listen_port': '1233',
'backend_http_extra': ['timeout tunnel 1h'],
},
'nova_novncproxy_external': {
'enabled': True,
'mode': 'http',
'external': True,
'port': '1234',
'listen_port': '1235',
'backend_http_extra': ['timeout tunnel 1h'],
}
}
}
}
actual = filters.namespace_haproxy_for_cell(
example_services, 'cell0001')
expected = {
'nova-novncproxy_cell0001': {
'group': 'some_group',
'enabled': True,
'haproxy': {
'nova_novncproxy_cell0001': {
'enabled': True,
'mode': 'http',
'external': False,
'port': '1232',
'listen_port': '1233',
'backend_http_extra': ['timeout tunnel 1h'],
},
'nova_novncproxy_external_cell0001': {
'enabled': True,
'mode': 'http',
'external': True,
'port': '1234',
'listen_port': '1235',
'backend_http_extra': ['timeout tunnel 1h'],
}
}
}
}
self.assertDictEqual(expected, actual)
def test_namespace_haproxy_for_cell_with_multiple_proxies(self):
example_services = {
'nova-novncproxy': {
'haproxy': {
'nova_novncproxy': {},
'nova_novncproxy_external': {}
}
},
'nova-spicehtml5proxy': {
'haproxy': {
'nova_spicehtml5proxy': {},
'nova_spicehtml5proxy_external': {}
}
}
}
actual = filters.namespace_haproxy_for_cell(
example_services, 'cell0002')
expected = {
'nova-novncproxy_cell0002': {
'haproxy': {
'nova_novncproxy_cell0002': {},
'nova_novncproxy_external_cell0002': {}
}
},
'nova-spicehtml5proxy_cell0002': {
'haproxy': {
'nova_spicehtml5proxy_cell0002': {},
'nova_spicehtml5proxy_external_cell0002': {}
}
}
}
self.assertDictEqual(expected, actual)