From eb1e16685d7c6df31f2716b189656216570cabe0 Mon Sep 17 00:00:00 2001 From: Cole Walker Date: Mon, 9 Sep 2024 11:38:39 -0400 Subject: [PATCH] Handle incomplete pmc results Improve the logic for handling incomplete results when a pmc query returns empty data. There are intermittent instances where a pmc command returns an empty result. The previous logic would result in ptp-notification considering the ptp state to be in Freerun, but this could lead to undesirable results where incorrect notifications are delivered. This update allows ptp-notification to retry the pmc commands on the next cycle and only updates the ptp status when the results are complete. Additional logging has been added to indicate which pmc results are missing. Test plan: Pass: Verify container image builds Pass: Verify ptp-notification deployment Pass: Verify basic operations (get, subscribe, list, delete) Pass: Verify that ptp-notification does not deliver incorrect freerun notifications when pmc results are incomplete Pass: Verify that ptp-notification delivers a notification when ptp lock is actually lost Story: 2011056 Task: 51049 Signed-off-by: Cole Walker Change-Id: Ic473fcb44e16db629f12208e581a92abc3558480 --- .../common/helpers/ptp_monitor.py | 8 ++++++-- .../trackingfunctionsdk/common/helpers/ptpsync.py | 6 +++--- .../trackingfunctionsdk/common/helpers/ptpsync.py | 12 ++++++++---- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py b/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py index 38d4a64..652c2ad 100644 --- a/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py +++ b/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py @@ -139,8 +139,12 @@ class PtpMonitor: if pmc and ptp4l and phc2sys and ptp4lconf: self.pmc_query_results, total_ptp_keywords, port_count = \ self.ptpsync() - sync_state = utils.check_results(self.pmc_query_results, - total_ptp_keywords, port_count) + try: + sync_state = utils.check_results(self.pmc_query_results, + total_ptp_keywords, port_count) + except RuntimeError as err: + LOG.warning(err) + sync_state = previous_sync_state else: LOG.warning("Missing critical resource: " "PMC %s PTP4L %s PHC2SYS %s PTP4LCONF %s" diff --git a/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py b/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py index 993a0d3..ae040a6 100644 --- a/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py +++ b/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py @@ -74,9 +74,9 @@ def check_results(result, total_ptp_keywords, port_count): # check for a healthy result if len(result) != total_ptp_keywords: - sync_state = constants.FREERUN_PHC_STATE - LOG.warning('results are not complete, returning FREERUN') - return sync_state + LOG.info("Results %s" % result) + LOG.info("Results len %s, total_ptp_keywords %s" % (len(result), total_ptp_keywords)) + raise RuntimeError("PMC results are incomplete, retrying") # determine the current sync state if (result[constants.GM_PRESENT].lower() != constants.GM_IS_PRESENT and result[constants.GRANDMASTER_IDENTITY] != result[constants.CLOCK_IDENTITY]): diff --git a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py index 6c5ec26..867d593 100644 --- a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py +++ b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py @@ -124,9 +124,9 @@ def check_results(result, total_ptp_keywords, port_count): # check for a healthy result if len(result) != total_ptp_keywords: - sync_state = constants.FREERUN_PHC_STATE - LOG.warning('results are not complete, returning FREERUN') - return sync_state + LOG.info("Results %s" % result) + LOG.info("Results len %s, total_ptp_keywords %s" % (len(result), total_ptp_keywords)) + raise RuntimeError("PMC results are incomplete, retrying") # determine the current sync state if (result[constants.GM_PRESENT].lower() != constants.GM_IS_PRESENT and result[constants.GRANDMASTER_IDENTITY] != result[constants.CLOCK_IDENTITY]): @@ -225,7 +225,11 @@ def ptp_status(holdover_time, freq, sync_state, event_time): # run pmc command if preconditions met if pmc and ptp4l and phc2sys and ptp4lconf: result, total_ptp_keywords, port_count = ptpsync() - sync_state = check_results(result, total_ptp_keywords, port_count) + try: + sync_state = check_results(result, total_ptp_keywords, port_count) + except RuntimeError as err: + LOG.warning(err) + sync_state = previous_sync_state else: LOG.warning("Critical resources not available: pmc %s ptp4l %s phc2sys %s ptp4lconf %s" % (pmc, ptp4l, phc2sys, ptp4lconf))