Sync helpers for 20.05

Change-Id: Idb46b1a924314a126367bfada8c7851f0b4ea21f

Makefile

@@ -1,5 +1,5 @@
 #!/usr/bin/make
-PYTHON := /usr/bin/env python
+PYTHON := /usr/bin/env python3
 
 lint:
 	@tox -e pep8

hooks/charmhelpers/contrib/network/ovs/ovn.py

@@ -140,7 +140,7 @@ class OVNClusterStatus(object):
         return self.leader == 'self'
 
 
-def cluster_status(target, schema=None, use_ovs_appctl=False):
+def cluster_status(target, schema=None, use_ovs_appctl=False, rundir=None):
     """Retrieve status information from clustered OVSDB.
 
     :param target: Usually one of 'ovsdb-server', 'ovnnb_db', 'ovnsb_db', can
@@ -151,6 +151,8 @@ def cluster_status(target, schema=None, use_ovs_appctl=False):
     :param use_ovs_appctl: The ``ovn-appctl`` command appeared in OVN 20.03,
                            set this to True to use ``ovs-appctl`` instead.
     :type use_ovs_appctl: bool
+    :param rundir: Override path to sockets
+    :type rundir: Optional[str]
     :returns: cluster status data object
     :rtype: OVNClusterStatus
     :raises: subprocess.CalledProcessError, KeyError, RuntimeError
@@ -164,8 +166,9 @@ def cluster_status(target, schema=None, use_ovs_appctl=False):
 
     status = {}
     k = ''
-    for line in ovn_appctl(target, 'cluster/status',
+    for line in ovn_appctl(target,
-                           schema or schema_map[target],
+                           ('cluster/status', schema or schema_map[target]),
+                           rundir=rundir,
                            use_ovs_appctl=use_ovs_appctl).splitlines():
         if k and line.startswith(' '):
             # there is no key which means this is a instance of a multi-line/
@@ -222,7 +225,7 @@ def is_northd_active():
     :rtype: bool
     """
     try:
-        for line in ovn_appctl('ovn-northd', 'status').splitlines():
+        for line in ovn_appctl('ovn-northd', ('status',)).splitlines():
             if line.startswith('Status:') and 'active' in line:
                 return True
     except subprocess.CalledProcessError:

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -92,6 +92,7 @@ DEFAULT_PGS_PER_OSD_TARGET = 100
 DEFAULT_POOL_WEIGHT = 10.0
 LEGACY_PG_COUNT = 200
 DEFAULT_MINIMUM_PGS = 2
+AUTOSCALER_DEFAULT_PGS = 32
 
 
 class OsdPostUpgradeError(Exception):
@@ -399,16 +400,28 @@ class ReplicatedPool(Pool):
 
     def create(self):
         if not pool_exists(self.service, self.name):
+            nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0
             # Create it
-            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create',
+            if nautilus_or_later:
-                   self.name, str(self.pg_num)]
+                cmd = [
+                    'ceph', '--id', self.service, 'osd', 'pool', 'create',
+                    '--pg-num-min={}'.format(
+                        min(AUTOSCALER_DEFAULT_PGS, self.pg_num)
+                    ),
+                    self.name, str(self.pg_num)
+                ]
+            else:
+                cmd = [
+                    'ceph', '--id', self.service, 'osd', 'pool', 'create',
+                    self.name, str(self.pg_num)
+                ]
+
             try:
                 check_call(cmd)
                 # Set the pool replica size
                 update_pool(client=self.service,
                             pool=self.name,
                             settings={'size': str(self.replicas)})
-                nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0
                 if nautilus_or_later:
                     # Ensure we set the expected pool ratio
                     update_pool(client=self.service,
@@ -466,10 +479,24 @@ class ErasurePool(Pool):
             k = int(erasure_profile['k'])
             m = int(erasure_profile['m'])
             pgs = self.get_pgs(k + m, self.percent_data)
+            nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0
             # Create it
-            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create',
+            if nautilus_or_later:
+                cmd = [
+                    'ceph', '--id', self.service, 'osd', 'pool', 'create',
+                    '--pg-num-min={}'.format(
+                        min(AUTOSCALER_DEFAULT_PGS, pgs)
+                    ),
                     self.name, str(pgs), str(pgs),
-                   'erasure', self.erasure_code_profile]
+                    'erasure', self.erasure_code_profile
+                ]
+            else:
+                cmd = [
+                    'ceph', '--id', self.service, 'osd', 'pool', 'create',
+                    self.name, str(pgs), str(pgs),
+                    'erasure', self.erasure_code_profile
+                ]
+
             try:
                 check_call(cmd)
                 try:
@@ -478,7 +505,6 @@ class ErasurePool(Pool):
                                           name=self.app_name)
                 except CalledProcessError:
                     log('Could not set app name for pool {}'.format(self.name, level=WARNING))
-                nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0
                 if nautilus_or_later:
                     # Ensure we set the expected pool ratio
                     update_pool(client=self.service,

requirements.txt

@@ -1,6 +1,12 @@
-# The order of packages is significant, because pip processes them in the order
+# This file is managed centrally by release-tools and should not be modified
-# of appearance. Changing the order has an impact on the overall integration
+# within individual charm repos.  See the 'global' dir contents for available
-# process, which may cause wedges in the gate later.
+# choices of *requirements.txt files for OpenStack Charms:
+#     https://github.com/openstack-charmers/release-tools
+#
+# TODO: Distill the func test requirements from the lint/unit test
+#       requirements.  They are intertwined.  Also, Zaza itself should specify
+#       all of its own requirements and if it doesn't, fix it there.
+#
 pbr>=1.8.0,<1.9.0
 simplejson>=2.2.0
 netifaces>=0.10.4

test-requirements.txt

@@ -1,14 +1,18 @@
-# The order of packages is significant, because pip processes them in the order
+# This file is managed centrally by release-tools and should not be modified
-# of appearance. Changing the order has an impact on the overall integration
+# within individual charm repos.  See the 'global' dir contents for available
-# process, which may cause wedges in the gate later.
+# choices of *requirements.txt files for OpenStack Charms:
+#     https://github.com/openstack-charmers/release-tools
+#
+# TODO: Distill the func test requirements from the lint/unit test
+#       requirements.  They are intertwined.  Also, Zaza itself should specify
+#       all of its own requirements and if it doesn't, fix it there.
+#
 charm-tools>=2.4.4
-coverage>=3.6
+requests>=2.18.4
 mock>=1.2
 flake8>=2.2.4,<=2.4.1
 stestr>=2.2.0
-requests>=2.18.4
+coverage>=4.5.2
-# NOTE: workaround for 14.04 pip/tox
+pyudev              # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)
-pytz
+git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0'
-pyudev  # for ceph-* charm unit tests (not mocked?)
-git+https://github.com/openstack-charmers/zaza.git#egg=zaza
 git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack

tox.ini

@@ -1,8 +1,12 @@
-# Classic charm: ./tox.ini
+# Classic charm (with zaza): ./tox.ini
 # This file is managed centrally by release-tools and should not be modified
 # within individual charm repos.  See the 'global' dir contents for available
 # choices of tox.ini for OpenStack Charms:
 #     https://github.com/openstack-charmers/release-tools
+#
+# TODO: Distill the func test requirements from the lint/unit test
+#       requirements.  They are intertwined.  Also, Zaza itself should specify
+#       all of its own requirements and if it doesn't, fix it there.
 [tox]
 envlist = pep8,py3
 skipsdist = True
@@ -20,11 +24,7 @@ install_command =
 commands = stestr run --slowest {posargs}
 whitelist_externals = juju
 passenv = HOME TERM CS_* OS_* TEST_*
-
+deps = -r{toxinidir}/test-requirements.txt
-[testenv:py3]
-basepython = python3
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 
 [testenv:py35]
 basepython = python3.5
@@ -41,6 +41,16 @@ basepython = python3.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
+[testenv:py38]
+basepython = python3.8
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+
+[testenv:py3]
+basepython = python3
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+
 [testenv:pep8]
 basepython = python3
 deps = -r{toxinidir}/requirements.txt
@@ -80,33 +90,31 @@ omit =
 basepython = python3
 commands = {posargs}
 
-[testenv:func]
+[testenv:func-noop]
-# Charm Functional Test
+basepython = python3
-# Run all bundles (expected to always pass)
+commands =
+    functest-run-suite --help
+
+[testenv:func]
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 commands =
     functest-run-suite --keep-model
 
 [testenv:func-smoke]
-# Charm Functional Test
-# Run a specific bundle as a smoke test (expected to always pass)
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 commands =
     functest-run-suite --keep-model --smoke
 
 [testenv:func-dev]
-# Charm Functional Test
-# Run all development test targets (may not always pass!)
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 commands =
     functest-run-suite --keep-model --dev
 
+[testenv:func-target]
+basepython = python3
+commands =
+    functest-run-suite --keep-model --bundle {posargs}
+
 [flake8]
 ignore = E402,E226
 exclude = */charmhelpers