GitHub/sdrtrunk
1
0
Fork 0
mirror of https://github.com/DSheirer/sdrtrunk.git synced 2026-11-12 16:47:31 -05:00
Code Issues Packages Projects Releases 2 Wiki Activity

#1830 Health & Diagnostic Monitor feature. (#1831)

Browse Source 
Co-authored-by: Dennis Sheirer <dsheirer@github.com>
This commit is contained in:
Denny Sheirer
2024-02-09 10:05:51 -05:00
committed by GitHub
parent 6b5a1fb827
commit e125d6a452
4 changed files with 226 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/io/github/dsheirer/gui/SDRTrunk.java
View File

@@ -189,10 +189,13 @@ public class SDRTrunk implements Listener<TunerEvent>
EventLogManager eventLogManager = new EventLogManager(aliasModel, mUserPreferences);
mPlaylistManager = new PlaylistManager(mUserPreferences, mTunerManager, aliasModel, eventLogManager, mIconModel);
mDiagnosticMonitor = new DiagnosticMonitor(mUserPreferences, mPlaylistManager.getChannelProcessingManager(),
mTunerManager);
boolean headless = GraphicsEnvironment.isHeadless();
if(!GraphicsEnvironment.isHeadless())
mDiagnosticMonitor = new DiagnosticMonitor(mUserPreferences, mPlaylistManager.getChannelProcessingManager(),
mTunerManager, headless);
mDiagnosticMonitor.start();
if(!headless)
{
mJavaFxWindowManager = new JavaFxWindowManager(mUserPreferences, mTunerManager, mPlaylistManager);
}
@@ -420,7 +423,7 @@ public class SDRTrunk implements Listener<TunerEvent>
processingStatusReportMenuItem.addActionListener(e -> {
try
{
Path path = mDiagnosticMonitor.generateProcessingDiagnosticReport();
Path path = mDiagnosticMonitor.generateProcessingDiagnosticReport("User initiated diagnostic report");
JOptionPane.showMessageDialog(mMainGui, "Report created: " +
path.toString(), "Processing Status Report Created", JOptionPane.INFORMATION_MESSAGE);
@@ -623,6 +626,7 @@ public class SDRTrunk implements Listener<TunerEvent>
private void processShutdown()
{
mLog.info("Application shutdown started ...");
mDiagnosticMonitor.stop();
mUserPreferences.getSwingPreference().setLocation(WINDOW_FRAME_IDENTIFIER, mMainGui.getLocation());
mUserPreferences.getSwingPreference().setDimension(WINDOW_FRAME_IDENTIFIER, mMainGui.getSize());
mUserPreferences.getSwingPreference().setMaximized(WINDOW_FRAME_IDENTIFIER,

62
src/main/java/io/github/dsheirer/gui/preference/application/ApplicationPreferenceEditor.java
View File

@@ -1,6 +1,6 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2023 Dennis Sheirer
* Copyright (C) 2014-2024 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -23,10 +23,16 @@ import io.github.dsheirer.preference.UserPreferences;
import io.github.dsheirer.preference.application.ApplicationPreference;
import javafx.geometry.HPos;
import javafx.geometry.Insets;
import javafx.geometry.Orientation;
import javafx.scene.control.Label;
import javafx.scene.control.Separator;
import javafx.scene.control.Spinner;
import javafx.scene.layout.ColumnConstraints;
import javafx.scene.layout.GridPane;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Priority;
import javafx.scene.layout.VBox;
import org.controlsfx.control.ToggleSwitch;
/**
@@ -38,6 +44,7 @@ public class ApplicationPreferenceEditor extends HBox
private GridPane mEditorPane;
private Label mAutoStartTimeoutLabel;
private Spinner<Integer> mTimeoutSpinner;
private ToggleSwitch mAutomaticDiagnosticMonitoringToggle;
/**
* Constructs an instance
@@ -46,7 +53,14 @@ public class ApplicationPreferenceEditor extends HBox
public ApplicationPreferenceEditor(UserPreferences userPreferences)
{
mApplicationPreference = userPreferences.getApplicationPreference();
getChildren().add(getEditorPane());
setMaxWidth(Double.MAX_VALUE);
VBox vbox = new VBox();
vbox.setMaxHeight(Double.MAX_VALUE);
vbox.setMaxWidth(Double.MAX_VALUE);
vbox.getChildren().add(getEditorPane());
HBox.setHgrow(vbox, Priority.ALWAYS);
getChildren().add(vbox);
}
private GridPane getEditorPane()
@@ -55,13 +69,31 @@ public class ApplicationPreferenceEditor extends HBox
{
int row = 0;
mEditorPane = new GridPane();
mEditorPane.setMaxWidth(Double.MAX_VALUE);
mEditorPane.setVgap(10);
mEditorPane.setHgap(10);
mEditorPane.setHgap(3);
mEditorPane.setPadding(new Insets(10, 10, 10, 10));
GridPane.setHalignment(getAutoStartTimeoutLabel(), HPos.RIGHT);
mEditorPane.add(getAutoStartTimeoutLabel(), 0, row);
mEditorPane.add(getTimeoutSpinner(), 1, row);
mEditorPane.add(new Label("seconds"), 2, row);
Label monitoringLabel = new Label("Application Health and Diagnostic Monitoring.");
mEditorPane.add(monitoringLabel, 0, row, 2, 1);
GridPane.setHalignment(getAutomaticDiagnosticMonitoringToggle(), HPos.RIGHT);
mEditorPane.add(getAutomaticDiagnosticMonitoringToggle(), 0, ++row);
mEditorPane.add(new Label("Enable Diagnostic Monitoring"), 1, row, 2, 1);
Separator separator = new Separator(Orientation.HORIZONTAL);
GridPane.setHgrow(separator, Priority.ALWAYS);
mEditorPane.add(separator, 0, ++row, 3, 1);
mEditorPane.add(getAutoStartTimeoutLabel(), 0, ++row, 2, 1);
GridPane.setHalignment(getTimeoutSpinner(), HPos.RIGHT);
mEditorPane.add(getTimeoutSpinner(), 0, ++row);
mEditorPane.add(new Label("seconds"), 1, row);
ColumnConstraints c1 = new ColumnConstraints();
c1.setPercentWidth(30);
ColumnConstraints c2 = new ColumnConstraints();
c2.setHgrow(Priority.ALWAYS);
mEditorPane.getColumnConstraints().addAll(c1, c2);
}
return mEditorPane;
@@ -91,4 +123,20 @@ public class ApplicationPreferenceEditor extends HBox
return mTimeoutSpinner;
}
/**
* Toggle switch to enable/disable automatic diagnostic monitoring.
*/
private ToggleSwitch getAutomaticDiagnosticMonitoringToggle()
{
if(mAutomaticDiagnosticMonitoringToggle == null)
{
mAutomaticDiagnosticMonitoringToggle = new ToggleSwitch();
mAutomaticDiagnosticMonitoringToggle.setSelected(mApplicationPreference.isAutomaticDiagnosticMonitoring());
mAutomaticDiagnosticMonitoringToggle.selectedProperty().addListener((observable, oldValue, enabled) ->
mApplicationPreference.setAutomaticDiagnosticMonitoring(enabled));
}
return mAutomaticDiagnosticMonitoringToggle;
}
}

138
src/main/java/io/github/dsheirer/monitor/DiagnosticMonitor.java
View File

@@ -20,52 +20,165 @@
package io.github.dsheirer.monitor;
import io.github.dsheirer.controller.channel.ChannelProcessingManager;
import io.github.dsheirer.log.LoggingSuppressor;
import io.github.dsheirer.preference.UserPreferences;
import io.github.dsheirer.source.tuner.manager.TunerManager;
import io.github.dsheirer.util.ThreadPool;
import io.github.dsheirer.util.TimeStamp;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.jar.Attributes;
import java.util.jar.Manifest;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.swing.JOptionPane;
/**
* Utility class for monitoring system components and producing logging reports.
*/
public class DiagnosticMonitor
{
private static final Logger LOGGER = LoggerFactory.getLogger(DiagnosticMonitor.class);
private final LoggingSuppressor LOG_SUPPRESSOR = new LoggingSuppressor(LOGGER);
private static final String DIVIDER = "\n\n=========================================================================\n\n";
private UserPreferences mUserPreferences;
private ChannelProcessingManager mChannelProcessingManager;
private TunerManager mTunerManager;
private ScheduledFuture<?> mBlockedThreadMonitorHandle;
private BlockedThreadMonitor mMonitor = new BlockedThreadMonitor();
private boolean mUserAlertedToBlockedThreadCondition = false;
private Map<Integer,Integer> mBlockedThreadDetectionCountMap = new HashMap<>();
private boolean mHeadless;
/**
* Constructs an instance
* @param userPreferences for application logging directory lookup.
* @param channelProcessingManager for accessing running channel information
* @param tunerManager for accessing allocated tuner channel information
* @param headless to indicate if the thread deadlock monitor should show a user notification.
*/
public DiagnosticMonitor(UserPreferences userPreferences, ChannelProcessingManager channelProcessingManager,
TunerManager tunerManager)
TunerManager tunerManager, boolean headless)
{
mUserPreferences = userPreferences;
mChannelProcessingManager = channelProcessingManager;
mTunerManager = tunerManager;
mHeadless = headless;
}
/**
* Starts monitoring for blocked threads
*/
public void start()
{
if(mBlockedThreadMonitorHandle != null)
{
mBlockedThreadMonitorHandle.cancel(true);
}
if(mUserPreferences.getApplicationPreference().isAutomaticDiagnosticMonitoring())
{
LOGGER.info("Diagnostic monitoring enabled running every 30 seconds");
mBlockedThreadMonitorHandle = ThreadPool.SCHEDULED.scheduleAtFixedRate(mMonitor, 30, 30, TimeUnit.SECONDS);
}
else
{
LOGGER.info("Diagnostic monitoring disabled per user preference (application).");
}
}
/**
* Stops monitoring for blocked threads.
*/
public void stop()
{
if(mBlockedThreadMonitorHandle != null)
{
mBlockedThreadMonitorHandle.cancel(true);
}
mBlockedThreadMonitorHandle = null;
}
/**
* Checks for blocked threads and on discovery, generates a diagnostic report and notifies the user (once).
*/
private void checkForBlockedThreads()
{
if(!mUserAlertedToBlockedThreadCondition)
{
try
{
ThreadMXBean bean = ManagementFactory.getThreadMXBean();
long ids[] = bean.findDeadlockedThreads();
if(ids != null)
{
mUserAlertedToBlockedThreadCondition = true;
ThreadInfo threadInfo[] = bean.getThreadInfo(ids);
StringBuilder sb = new StringBuilder();
sb.append("sdrtrunk detected a critical application error with a threading deadlock, described as follows:\n");
for (ThreadInfo threadInfo1 : threadInfo)
{
sb.append("Thread ID[").append(threadInfo1.getThreadId());
sb.append("] Name [").append(threadInfo1.getThreadName());
sb.append("] Lock [").append(threadInfo1.getLockName());
sb.append("] Owned By [ID:").append(threadInfo1.getLockOwnerId());
sb.append(" | NAME:").append(threadInfo1.getLockName());
sb.append("]\n");
}
LOGGER.error(sb.toString());
Path reportPath = generateProcessingDiagnosticReport(sb + DIVIDER);
LOGGER.error("Thread deadlock report generated: " + reportPath);
if(!mHeadless)
{
String title = "sdrtrunk: Critical Error Detected";
String message = "The sdrtrunk application has detected a thread deadlock situation.\n" +
"The application may degrade over time and eventually run out of memory.\n" +
"A diagnostic report was generated. Please open an issue on the GitHub\n" +
"website and attach this diagnostic report:\n\n" + reportPath.toString();
JOptionPane.showMessageDialog(null, message, title, JOptionPane.ERROR_MESSAGE);
}
}
}
catch(Exception e)
{
LOG_SUPPRESSOR.error("run error", 1, "Error while monitoring for deadlocked " +
"threads: " + e.getLocalizedMessage());
//Set the flag so that we don't try to run again.
mUserAlertedToBlockedThreadCondition = true;
}
}
}
/**
* Creates a diagnostic report containing state information for channels that are in a processing state.
* @param message to prepend to the report
* @return path for the log file that was created.
*/
public Path generateProcessingDiagnosticReport() throws IOException
public Path generateProcessingDiagnosticReport(String message) throws IOException
{
StringBuilder sb = new StringBuilder();
sb.append("sdrtrunk Processing Diagnostic Report\n");
sb.append(message);
sb.append("\n\nsdrtrunk Processing Diagnostic Report\n");
sb.append(DIVIDER);
sb.append(getEnvironmentReport());
sb.append(DIVIDER);
@@ -190,4 +303,23 @@ public class DiagnosticMonitor
return null;
}
/**
* Runnable to periodically check for blocked threads
*/
public class BlockedThreadMonitor implements Runnable
{
@Override
public void run()
{
try
{
checkForBlockedThreads();
}
catch(Throwable t)
{
LOG_SUPPRESSOR.error("Error", 3, "Error while checking for blocked threads", t);
}
}
}
}

29
src/main/java/io/github/dsheirer/preference/application/ApplicationPreference.java
View File

@@ -1,6 +1,6 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2023 Dennis Sheirer
* Copyright (C) 2014-2024 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -31,11 +31,13 @@ import org.slf4j.LoggerFactory;
*/
public class ApplicationPreference extends Preference
{
private static final String PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING = "automatic.diagnostic.monitoring";
private static final String PREFERENCE_KEY_CHANNEL_AUTO_START_TIMEOUT = "channel.auto.start.timeout";
private final static Logger mLog = LoggerFactory.getLogger(ApplicationPreference.class);
private Preferences mPreferences = Preferences.userNodeForPackage(ApplicationPreference.class);
private Integer mChannelAutoStartTimeout;
private Boolean mAutomaticDiagnosticMonitoring;
/**
* Constructs an instance
@@ -77,4 +79,29 @@ public class ApplicationPreference extends Preference
mPreferences.putInt(PREFERENCE_KEY_CHANNEL_AUTO_START_TIMEOUT, timeout);
notifyPreferenceUpdated();
}
/**
* Indicates if automatic diagnostic monitoring is enabled.
* @return enabled.
*/
public boolean isAutomaticDiagnosticMonitoring()
{
if(mAutomaticDiagnosticMonitoring == null)
{
mAutomaticDiagnosticMonitoring = mPreferences.getBoolean(PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING, true);
}
return mAutomaticDiagnosticMonitoring;
}
/**
* Sets the enabled state for automatic diagnostic monitoring.
* @param enabled true to turn on monitoring.
*/
public void setAutomaticDiagnosticMonitoring(boolean enabled)
{
mAutomaticDiagnosticMonitoring = enabled;
mPreferences.putBoolean(PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING, enabled);
notifyPreferenceUpdated();
}
}