#1830 Health & Diagnostic Monitor feature. (#1831)

Co-authored-by: Dennis Sheirer <dsheirer@github.com>

src/main/java/io/github/dsheirer/monitor/DiagnosticMonitor.java

@@ -20,52 +20,165 @@
 package io.github.dsheirer.monitor;
 
 import io.github.dsheirer.controller.channel.ChannelProcessingManager;
+import io.github.dsheirer.log.LoggingSuppressor;
 import io.github.dsheirer.preference.UserPreferences;
 import io.github.dsheirer.source.tuner.manager.TunerManager;
+import io.github.dsheirer.util.ThreadPool;
 import io.github.dsheirer.util.TimeStamp;
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.swing.JOptionPane;
+
 /**
  * Utility class for monitoring system components and producing logging reports.
  */
 public class DiagnosticMonitor
 {
     private static final Logger LOGGER = LoggerFactory.getLogger(DiagnosticMonitor.class);
+    private final LoggingSuppressor LOG_SUPPRESSOR = new LoggingSuppressor(LOGGER);
     private static final String DIVIDER = "\n\n=========================================================================\n\n";
     private UserPreferences mUserPreferences;
     private ChannelProcessingManager mChannelProcessingManager;
     private TunerManager mTunerManager;
+    private ScheduledFuture<?> mBlockedThreadMonitorHandle;
+    private BlockedThreadMonitor mMonitor = new BlockedThreadMonitor();
+    private boolean mUserAlertedToBlockedThreadCondition = false;
+    private Map<Integer,Integer> mBlockedThreadDetectionCountMap = new HashMap<>();
+    private boolean mHeadless;
 
     /**
      * Constructs an instance
      * @param userPreferences for application logging directory lookup.
+     * @param channelProcessingManager for accessing running channel information
+     * @param tunerManager for accessing allocated tuner channel information
+     * @param headless to indicate if the thread deadlock monitor should show a user notification.
      */
     public DiagnosticMonitor(UserPreferences userPreferences, ChannelProcessingManager channelProcessingManager,
-                             TunerManager tunerManager)
+                             TunerManager tunerManager, boolean headless)
     {
         mUserPreferences = userPreferences;
         mChannelProcessingManager = channelProcessingManager;
         mTunerManager = tunerManager;
+        mHeadless = headless;
+    }
+
+    /**
+     * Starts monitoring for blocked threads
+     */
+    public void start()
+    {
+        if(mBlockedThreadMonitorHandle != null)
+        {
+            mBlockedThreadMonitorHandle.cancel(true);
+        }
+
+        if(mUserPreferences.getApplicationPreference().isAutomaticDiagnosticMonitoring())
+        {
+            LOGGER.info("Diagnostic monitoring enabled running every 30 seconds");
+            mBlockedThreadMonitorHandle = ThreadPool.SCHEDULED.scheduleAtFixedRate(mMonitor, 30, 30, TimeUnit.SECONDS);
+        }
+        else
+        {
+            LOGGER.info("Diagnostic monitoring disabled per user preference (application).");
+        }
+    }
+
+    /**
+     * Stops monitoring for blocked threads.
+     */
+    public void stop()
+    {
+        if(mBlockedThreadMonitorHandle != null)
+        {
+            mBlockedThreadMonitorHandle.cancel(true);
+        }
+
+        mBlockedThreadMonitorHandle = null;
+    }
+
+    /**
+     * Checks for blocked threads and on discovery, generates a diagnostic report and notifies the user (once).
+     */
+    private void checkForBlockedThreads()
+    {
+        if(!mUserAlertedToBlockedThreadCondition)
+        {
+            try
+            {
+                ThreadMXBean bean = ManagementFactory.getThreadMXBean();
+
+                long ids[] = bean.findDeadlockedThreads();
+
+                if(ids != null)
+                {
+                    mUserAlertedToBlockedThreadCondition = true;
+
+                    ThreadInfo threadInfo[] = bean.getThreadInfo(ids);
+
+                    StringBuilder sb = new StringBuilder();
+                    sb.append("sdrtrunk detected a critical application error with a threading deadlock, described as follows:\n");
+
+                    for (ThreadInfo threadInfo1 : threadInfo)
+                    {
+                        sb.append("Thread ID[").append(threadInfo1.getThreadId());
+                        sb.append("] Name [").append(threadInfo1.getThreadName());
+                        sb.append("] Lock [").append(threadInfo1.getLockName());
+                        sb.append("] Owned By [ID:").append(threadInfo1.getLockOwnerId());
+                        sb.append(" | NAME:").append(threadInfo1.getLockName());
+                        sb.append("]\n");
+                    }
+
+                    LOGGER.error(sb.toString());
+                    Path reportPath = generateProcessingDiagnosticReport(sb + DIVIDER);
+                    LOGGER.error("Thread deadlock report generated: " + reportPath);
+
+                    if(!mHeadless)
+                    {
+                        String title = "sdrtrunk: Critical Error Detected";
+                        String message = "The sdrtrunk application has detected a thread deadlock situation.\n" +
+                                         "The application may degrade over time and eventually run out of memory.\n" +
+                                         "A diagnostic report was generated.  Please open an issue on the GitHub\n" +
+                                         "website and attach this diagnostic report:\n\n" + reportPath.toString();
+                        JOptionPane.showMessageDialog(null, message, title, JOptionPane.ERROR_MESSAGE);
+                    }
+                }
+            }
+            catch(Exception e)
+            {
+                LOG_SUPPRESSOR.error("run error", 1, "Error while monitoring for deadlocked " +
+                        "threads: " + e.getLocalizedMessage());
+                //Set the flag so that we don't try to run again.
+                mUserAlertedToBlockedThreadCondition = true;
+            }
+        }
     }
 
     /**
      * Creates a diagnostic report containing state information for channels that are in a processing state.
+     * @param message to prepend to the report
      * @return path for the log file that was created.
      */
-    public Path generateProcessingDiagnosticReport() throws IOException
+    public Path generateProcessingDiagnosticReport(String message) throws IOException
     {
         StringBuilder sb = new StringBuilder();
-        sb.append("sdrtrunk Processing Diagnostic Report\n");
+        sb.append(message);
+        sb.append("\n\nsdrtrunk Processing Diagnostic Report\n");
         sb.append(DIVIDER);
         sb.append(getEnvironmentReport());
         sb.append(DIVIDER);
@@ -190,4 +303,23 @@ public class DiagnosticMonitor
 
         return null;
     }
+
+    /**
+     * Runnable to periodically check for blocked threads
+     */
+    public class BlockedThreadMonitor implements Runnable
+    {
+        @Override
+        public void run()
+        {
+            try
+            {
+                checkForBlockedThreads();
+            }
+            catch(Throwable t)
+            {
+                LOG_SUPPRESSOR.error("Error", 3, "Error while checking for blocked threads", t);
+            }
+        }
+    }
 }