From 87d19911a43818817b90746f121cb3688da8ed76 Mon Sep 17 00:00:00 2001 From: Cole Walker Date: Tue, 20 Sep 2022 17:01:29 -0400 Subject: [PATCH] [PTP] Fix failure after service restart and cleanup state logic Three fixes: 1. Handle a case where the ptp_montor would fail if ptp4l/phc2sys were restarted while ptp-notification is running 2. Correct the logic for determining if the local node is a GM. There was a case where the node was declared a GM if the upstream GM went down, even though local ports were configured not to become master. 3. Correct an issue where the ptp sync state was not set properly on startup and could not transition out of Freerun Test plan: PASS: Build and deploy new container image PASS: PTP state reaches freerun when upstream GM goes down PASS: ptp-notification continues running and resumes monitoring ptp4l state after ptp4l/phc2sys service restart on host Story: 2010056 Task: 46376 Task: 46377 Task: 46378 Signed-off-by: Cole Walker Change-Id: I3ac29fc56f9e72efc5b6ef02fa5472d9d801fbb7 --- .../common/helpers/ptp_monitor.py | 22 ++++++++++++++++--- .../common/helpers/ptpsync.py | 2 +- .../trackingfunctionsdk/services/daemon.py | 12 ++++++++-- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py index e9f4851..e9a7f15 100644 --- a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py +++ b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py @@ -28,11 +28,12 @@ log_helper.config_logger(LOG) class PtpMonitor: _clock_class = None - _ptp_sync_state = PtpState.Freerun + _ptp_sync_state = constants.UNKNOWN_PHC_STATE _new_ptp_sync_event = False _new_clock_class_event = False _ptp_event_time = None _clock_class_event_time = None + _clock_class_retry = 3 # Critical resources ptp4l_service_name = None @@ -73,10 +74,23 @@ class PtpMonitor: self._new_ptp_sync_event = new_ptp_sync_event def get_ptp_sync_state(self): - return self._ptp_sync_state + return self._new_ptp_sync_event, self._ptp_sync_state, self._ptp_event_time def set_ptp_clock_class(self): - clock_class = self.pmc_query_results['clockClass'] + try: + clock_class = self.pmc_query_results['clockClass'] + # Reset retry counter upon getting clock class + self._clock_class_retry = 3 + except KeyError: + LOG.warning("set_ptp_clock_class: Unable to read current clockClass") + if self._clock_class_retry > 0: + self._clock_class_retry -= 1 + LOG.warning("Trying to get clockClass %s more time(s) before " + "setting clockClass 248 (FREERUN)" % self._clock_class_retry) + clock_class = self._clock_class + else: + clock_class = "248" + self._clock_class_retry = 3 if clock_class != self._clock_class: self._clock_class = clock_class self._new_clock_class_event = True @@ -131,6 +145,8 @@ class PtpMonitor: sync_state = PtpState.Freerun # determine if ptp sync state has changed since the last one + LOG.debug("ptp_monitor: sync_state %s, " + "previous_sync_state %s" % (sync_state, previous_sync_state)) if sync_state != previous_sync_state: new_event = True self._ptp_event_time = datetime.datetime.utcnow().timestamp() diff --git a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py index 4a36f0c..2e800ee 100644 --- a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py +++ b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py @@ -74,7 +74,7 @@ def check_results(result, total_ptp_keywords, port_count): if (result[constants.GM_PRESENT].lower() != constants.GM_IS_PRESENT and result[constants.GRANDMASTER_IDENTITY] != result[constants.CLOCK_IDENTITY]): sync_state = constants.FREERUN_PHC_STATE - else: + elif result[constants.GRANDMASTER_IDENTITY] == result[constants.CLOCK_IDENTITY]: local_gm = True LOG.debug("Local node is a GM") for port in range(1, port_count + 1): diff --git a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/services/daemon.py b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/services/daemon.py index cd3144a..2e5104d 100644 --- a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/services/daemon.py +++ b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/services/daemon.py @@ -415,8 +415,15 @@ class PtpWatcherDefault: elif gnss._state == GnssState.Locked and gnss_state != GnssState.Freerun: gnss_state = GnssState.Locked + for ptp4l in self.ptp_monitor_list: + _, read_state, _ = ptp4l.get_ptp_sync_state() + if read_state == PtpState.Holdover or read_state == PtpState.Freerun or \ + read_state == constants.UNKNOWN_PHC_STATE: + ptp_state = PtpState.Freerun + elif read_state == PtpState.Locked and ptp_state != PtpState.Freerun: + ptp_state = PtpState.Locked + os_clock_state = self.os_clock_monitor.get_os_clock_state() - ptp_state = self.ptptracker_context.get('sync_state') if gnss_state is GnssState.Freerun or os_clock_state is OsClockState.Freerun or ptp_state \ is PtpState.Freerun: @@ -443,6 +450,7 @@ class PtpWatcherDefault: def __get_ptp_status(self, holdover_time, freq, sync_state, last_event_time, ptp_monitor): new_event = False new_event_time = last_event_time + ptp_monitor.set_ptp_sync_state() if self.ptp_device_simulated: now = time.time() timediff = now - last_event_time @@ -458,7 +466,7 @@ class PtpWatcherDefault: else: sync_state = PtpState.Freerun else: - new_event, sync_state, new_event_time = ptp_monitor.ptp_status() + new_event, sync_state, new_event_time = ptp_monitor.get_ptp_sync_state() return new_event, sync_state, new_event_time '''announce location'''