Add log retrieval to Cassandra

Add system.log retrieval.

The Cassandra's system/general log is located in the cassandra
log directory and is by default called system.log.

Cassandra 2.1 and higher uses 'SLF4J' with 'logback' backend.

Logging properties can be configured at runtime via the nodetool utility
(setlogginglevel) and/or persisted in 'logback.xml' configuration file.

Logging can be enabled on per-module/class basis with configurable
verbosity levels.

The logging in Trove is configured globally (on the root level)
- i.e. for all application modules.

A new configuration property 'system_log_level' was added to control the
log verbosity. It allows the operator define the log level that will be
set when the logging is enabled on an instance.
All logging will be turned off when disabled.

Note: The log level is not changed during prepare. The instances will be
provisioned with the default logging setting (INFO level).

Note: An XML codec was added to allow parsing XML files into/from Python
dicts.

Change-Id: Ia57587bf557771ea6d7f00e2fe9d674789b98559
Depends-On: I4a088b7a541c35a6c32d6985727bad65ff491815
Depends-On: I254b81b45e44aeda3049865bee76cd9075312761
Closes-Bug: 1550557

releasenotes/notes/add-cassandra-log-retrieval-a295f3d0d4c56804.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Enable database log retrieval on Cassandra instances.

requirements.txt

@@ -44,3 +44,4 @@ osprofiler>=1.3.0 # Apache-2.0
 oslo.log>=1.14.0 # Apache-2.0
 oslo.db>=4.10.0 # Apache-2.0
 enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
+xmltodict>=0.10.1  # MIT

trove/common/cfg.py

@@ -860,8 +860,12 @@ cassandra_opts = [
     cfg.ListOpt('ignore_dbs', default=['system', 'system_auth',
                                        'system_traces'],
                 help='Databases to exclude when listing databases.'),
-    cfg.StrOpt('guest_log_exposed_logs', default='',
+    cfg.StrOpt('guest_log_exposed_logs', default='system',
                help='List of Guest Logs to expose for publishing.'),
+    cfg.StrOpt('system_log_level',
+               choices=['ALL', 'TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR'],
+               default='INFO',
+               help='Cassandra log verbosity.'),
     cfg.BoolOpt('cluster_support', default=True,
                 help='Enable clusters to be created and managed.'),
     cfg.StrOpt('api_strategy',

trove/common/stream_codecs.py

@@ -21,6 +21,7 @@ import json
 import re
 import six
 from six.moves.configparser import SafeConfigParser
+import xmltodict
 import yaml
 
 
@@ -427,3 +428,16 @@ class Base64Codec(StreamCodec):
 
         # py27 & py34 seem to understand bytearray the same
         return bytearray([item for item in base64.b64decode(stream)])
+
+
+class XmlCodec(StreamCodec):
+
+    def __init__(self, encoding='utf-8'):
+        self._encoding = encoding
+
+    def serialize(self, dict_data):
+        return xmltodict.unparse(
+            dict_data, output=None, encoding=self._encoding, pretty=True)
+
+    def deserialize(self, stream):
+        return xmltodict.parse(stream, encoding=self._encoding)

trove/guestagent/datastore/experimental/cassandra/manager.py

@@ -25,6 +25,7 @@ from trove.common.notification import EndNotification
 from trove.guestagent import backup
 from trove.guestagent.datastore.experimental.cassandra import service
 from trove.guestagent.datastore import manager
+from trove.guestagent import guest_log
 from trove.guestagent import volume
 
 
@@ -34,6 +35,8 @@ CONF = cfg.CONF
 
 class Manager(manager.Manager):
 
+    GUEST_LOG_DEFS_SYSTEM_LABEL = 'system'
+
     def __init__(self, manager_name='cassandra'):
         super(Manager, self).__init__(manager_name)
         self._app = None
@@ -62,6 +65,29 @@ class Manager(manager.Manager):
     def configuration_manager(self):
         return self.app.configuration_manager
 
+    @property
+    def datastore_log_defs(self):
+        system_log_file = self.validate_log_file(
+            self.app.cassandra_system_log_file, self.app.cassandra_owner)
+        return {
+            self.GUEST_LOG_DEFS_SYSTEM_LABEL: {
+                self.GUEST_LOG_TYPE_LABEL: guest_log.LogType.USER,
+                self.GUEST_LOG_USER_LABEL: self.app.cassandra_owner,
+                self.GUEST_LOG_FILE_LABEL: system_log_file
+            }
+        }
+
+    def guest_log_enable(self, context, log_name, disable):
+        if disable:
+            LOG.debug("Disabling system log.")
+            self.app.set_logging_level('OFF')
+        else:
+            log_level = CONF.get(self.manager_name).get('system_log_level')
+            LOG.debug("Enabling system log with logging level: %s" % log_level)
+            self.app.set_logging_level(log_level)
+
+        return False
+
     def restart(self, context):
         self.app.restart()
 

trove/guestagent/datastore/experimental/cassandra/service.py

@@ -32,6 +32,7 @@ from trove.common import instance as rd_instance
 from trove.common.stream_codecs import IniCodec
 from trove.common.stream_codecs import PropertiesCodec
 from trove.common.stream_codecs import SafeYamlCodec
+from trove.common.stream_codecs import XmlCodec
 from trove.common import utils
 from trove.guestagent.common.configuration import ConfigurationManager
 from trove.guestagent.common.configuration import OneFileOverrideStrategy
@@ -62,6 +63,7 @@ class CassandraApp(object):
 
     CASSANDRA_CONF_FILE = "cassandra.yaml"
     CASSANDRA_TOPOLOGY_FILE = 'cassandra-rackdc.properties'
+    CASSANDRA_LOGBACK_FILE = "logback.xml"
 
     _TOPOLOGY_CODEC = PropertiesCodec(
         delimiter='=', unpack_singletons=True, string_mappings={
@@ -82,6 +84,14 @@ class CassandraApp(object):
             SafeYamlCodec(default_flow_style=False), requires_root=True,
             override_strategy=OneFileOverrideStrategy(revision_dir))
 
+        lb_revision_dir = guestagent_utils.build_file_path(
+            os.path.dirname(self.cassandra_logback), 'logback-overrides')
+        self.logback_conf_manager = ConfigurationManager(
+            self.cassandra_logback,
+            self.cassandra_owner, self.cassandra_owner,
+            XmlCodec(), requires_root=True,
+            override_strategy=OneFileOverrideStrategy(lb_revision_dir))
+
     @property
     def service_candidates(self):
         return ['cassandra']
@@ -117,6 +127,20 @@ class CassandraApp(object):
     def cassandra_working_dir(self):
         return "/var/lib/cassandra"
 
+    @property
+    def cassandra_system_log_file(self):
+        return guestagent_utils.build_file_path(
+            self.cassandra_log_dir, 'system', 'log')
+
+    @property
+    def cassandra_log_dir(self):
+        return "/var/log/cassandra"
+
+    @property
+    def cassandra_logback(self):
+        return guestagent_utils.build_file_path(self.cassandra_conf_dir,
+                                                self.CASSANDRA_LOGBACK_FILE)
+
     @property
     def default_superuser_name(self):
         return "cassandra"
@@ -686,6 +710,16 @@ class CassandraApp(object):
         # <keyspace> ( <table> ... )
         self._run_nodetool_command('flush', keyspace, *tables)
 
+    def set_logging_level(self, log_level):
+        """Set the log Cassandra's system log verbosity level.
+        """
+        # Apply the change at runtime.
+        self._run_nodetool_command('setlogginglevel', 'root', log_level)
+
+        # Persist the change.
+        self.logback_conf_manager.apply_system_override(
+            {'configuration': {'root': {'@level': log_level}}})
+
     def _run_nodetool_command(self, cmd, *args, **kwargs):
         """Execute a nodetool command on this node.
         """

trove/tests/scenario/helpers/cassandra_helper.py

@@ -157,3 +157,6 @@ class CassandraHelper(TestHelper):
     def get_invalid_groups(self):
         return [{'sstable_preemptive_open_interval_in_mb': -1},
                 {'sstable_preemptive_open_interval_in_mb': 'string_value'}]
+
+    def get_exposed_user_log_names(self):
+        return ['system']

trove/tests/scenario/runners/guest_log_runners.py

@@ -674,3 +674,12 @@ class GuestLogRunner(TestRunner):
             expected_type=guest_log.LogType.SYS.name,
             expected_status=guest_log.LogStatus.Ready.name,
             expected_published=0, expected_pending=1)
+
+
+class CassandraGuestLogRunner(GuestLogRunner):
+
+    def run_test_log_show(self):
+        self.assert_log_show(self.auth_client,
+                             self._get_exposed_user_log_name(),
+                             expected_published=0,
+                             expected_pending=None)

trove/tests/unittests/guestagent/test_cassandra_manager.py

@@ -775,3 +775,22 @@ class GuestAgentCassandraDBManagerTest(DatastoreManagerTest):
                           'list_superusers',
                           return_value=[trove_admin, other_admin]):
             self.assertTrue(self.manager.is_root_enabled(self.context))
+
+    def test_guest_log_enable(self):
+        self._assert_guest_log_enable(False, 'INFO')
+        self._assert_guest_log_enable(True, 'OFF')
+
+    def _assert_guest_log_enable(self, disable, expected_level):
+        with patch.multiple(
+                self.manager._app,
+                logback_conf_manager=DEFAULT,
+                _run_nodetool_command=DEFAULT
+        ) as app_mocks:
+            self.assertFalse(self.manager.guest_log_enable(
+                Mock(), Mock(), disable))
+
+            (app_mocks['logback_conf_manager'].apply_system_override.
+             assert_called_once_with(
+                {'configuration': {'root': {'@level': expected_level}}}))
+            app_mocks['_run_nodetool_command'].assert_called_once_with(
+                'setlogginglevel', 'root', expected_level)

trove/tests/unittests/guestagent/test_operating_system.py

@@ -26,7 +26,7 @@ from testtools import ExpectedException
 from trove.common import exception
 from trove.common.stream_codecs import (
     Base64Codec, IdentityCodec, IniCodec, JsonCodec,
-    KeyValueCodec, PropertiesCodec, YamlCodec)
+    KeyValueCodec, PropertiesCodec, XmlCodec, YamlCodec)
 from trove.common import utils
 from trove.guestagent.common import guestagent_utils
 from trove.guestagent.common import operating_system
@@ -132,6 +132,18 @@ class TestOperatingSystem(trove_testtools.TestCase):
 
         self._test_file_codec(data, JsonCodec())
 
+    def test_xml_file_codec(self):
+        data = {'document': {'@name': 'mydocument', '@ttl': '10',
+                             'author': {'@name': 'Jycll ;-)'},
+                             'page': [{'@number': '1', 'paragraph':
+                                       ['lorem ipsum', 'more lorem ipsum']},
+                                      {'@number': '1', 'paragraph':
+                                       ['lorem ipsum', 'more lorem ipsum']}]
+                             }
+                }
+
+        self._test_file_codec(data, XmlCodec())
+
     def _test_file_codec(self, data, read_codec, write_codec=None,
                          expected_data=None,
                          expected_exception=None,