#1378 Resolves potential error where tuner shutdown (because of USB I/O error) hangs and causes application to run out of memory. This patch doesn't resolve the original issue with the USB I/O error, but it does ensure that the tuner shutdown which is happening on the USB processing thread, can complete in a timely fashion so that the application doesn't (potentially) crash from out of memory. (#1382)

Co-authored-by: Dennis Sheirer <dsheirer@github.com>
This commit is contained in:
Denny Sheirer
2022-12-24 09:10:17 -05:00
committed by GitHub
parent ddb5b5ef87
commit a87309e792
2 changed files with 79 additions and 28 deletions

View File

@@ -321,7 +321,6 @@ public class IcecastTCPAudioBroadcaster extends IcecastAudioBroadcaster
{
String message = (String) object;
mLog.info("Stream [" + getStreamName() + "] message received: " + message);
if(message != null && !message.trim().isEmpty())
{
if(message.startsWith("HTTP/1.0 200 OK"))

View File

@@ -26,6 +26,7 @@ import io.github.dsheirer.source.tuner.ITunerErrorListener;
import io.github.dsheirer.source.tuner.TunerController;
import io.github.dsheirer.source.tuner.TunerType;
import io.github.dsheirer.source.tuner.manager.TunerManager;
import io.github.dsheirer.util.ThreadPool;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
@@ -267,9 +268,30 @@ public abstract class USBTunerController extends TunerController
public final void stop()
{
mRunning = false;
stopStreaming();
mNativeBufferBroadcaster.clear();
deviceStop();
//Spin the shutdown onto a new thread so that we can set a max wait threshold.
Thread t = new Thread(() -> {
stopStreaming();
mNativeBufferBroadcaster.clear();
deviceStop();
});
t.start();
try
{
//Wait up to 500 milliseconds for the shutdown to complete ... otherwise interrupt it and finish shutdown
t.join(500);
if(t.isAlive())
{
mLog.info("Tuner shutdown exceeded 500ms - forcing shutdown");
t.interrupt();
}
}
catch(InterruptedException ie)
{
}
if(mDeviceHandle != null)
{
@@ -495,6 +517,9 @@ public abstract class USBTunerController extends TunerController
private List<Transfer> mAvailableTransfers;
private LinkedTransferQueue<Transfer> mInProgressTransfers = new LinkedTransferQueue<>();
private boolean mAutoResubmitTransfers = false;
private int mTransferErrorCount = 0;
private List<Transfer> mErrorTransfers = new ArrayList<>();
private int mResubmitFailureLogCount = 0;
/**
* Creates USB Transfers to carry the streaming sample data. Transfer buffers are backed by native memory
@@ -559,34 +584,56 @@ public abstract class USBTunerController extends TunerController
int status = LibUsb.submitTransfer(transfer);
if(status == LibUsb.SUCCESS)
{
mInProgressTransfers.add(transfer);
}
else if(status == LibUsb.ERROR_BUSY)
{
mInProgressTransfers.add(transfer);
//Weird issue/bug with libusb on Windows where libusb tells us the transfer is complete and then tells
//us that the transfer is already submitted when we attempt to re-submit the transfer. We track the
//number of times this occurs and log when the anomaly count exceeds the number of transfer buffers
//in case the application gets to a state where all transfers are no longer usable ... so that we
//know post-mortem what happened to the application.
mAnomalousTransfersDetected++;
if(mAnomalousTransfersDetected < USB_BULK_TRANSFER_BUFFER_POOL_SIZE)
//Attempt to resubmit any previous transfers that failed on submit
if(!mErrorTransfers.isEmpty())
{
mLog.error("USB transfer anomaly detected - continuing - previous status [" + previousStatus +
" ] transferred [" + previousBytesTransferred + "] this has happened [" +
mAnomalousTransfersDetected + "] times");
Transfer toResubmit = mErrorTransfers.get(0);
int resubmitStatus = LibUsb.submitTransfer(toResubmit);
if(resubmitStatus == LibUsb.SUCCESS)
{
mInProgressTransfers.add(transfer);
mTransferErrorCount--;
}
else
{
mErrorTransfers.add(transfer);
if(mResubmitFailureLogCount < mAvailableTransfers.size())
{
mResubmitFailureLogCount++;
mLog.error("Attempt to resubmit previously failed transfer unsuccessful - status: " +
LibUsb.errorName(resubmitStatus));
}
if(mErrorTransfers.size() >= mAvailableTransfers.size())
{
mLog.error("Maximum USB I/O transfer submit errors reached - shutting down USB tuner");
ThreadPool.CACHED.submit(() -> setErrorMessage("USB Error - Transfers"));
}
}
}
}
else if(status == LibUsb.ERROR_BUSY || status == LibUsb.ERROR_IO)
{
mErrorTransfers.add(transfer);
mTransferErrorCount++;
//This should only log once, when the anomaly count matches the transfer count
if(mAnomalousTransfersDetected == USB_BULK_TRANSFER_BUFFER_POOL_SIZE)
//For ERROR_BUSY error, there is a weird issue/bug with libusb on Windows where libusb tells us the
//transfer is complete and then tells us that the transfer is already submitted when we attempt to
//re-submit the transfer. We track the number of times this occurs and log when the anomaly count
//exceeds the number of transfer buffers in case the application gets to a state where all transfers
//are no longer usable ... so that we know post-mortem what happened to the application.
mLog.error("USB error while submitting transfer buffer [" + LibUsb.errorName(status) + "] - this happened [" +
mTransferErrorCount + "] times, but may be temporary. Will attempt to resubmit in future");
if(mErrorTransfers.size() >= mAvailableTransfers.size())
{
mLog.warn("USB transfer anomaly detection count exceeded the total number [8] of available " +
"tranfers - sdrtrunk may no longer be streaming data from this tuner and may appear to " +
"be locked up. If so, restart the application to resolve the issue and please send the " +
"application log to the developer");
mLog.error("Maximum USB I/O transfer submit errors reached - shutting down USB tuner");
ThreadPool.CACHED.submit(() -> setErrorMessage("USB Error - Transfers"));
}
}
else
@@ -610,6 +657,10 @@ public abstract class USBTunerController extends TunerController
{
LibUsb.cancelTransfer(transfer);
}
for(Transfer transfer: mErrorTransfers)
{
LibUsb.cancelTransfer(transfer);
}
}
@Override
@@ -641,14 +692,15 @@ public abstract class USBTunerController extends TunerController
transfer.buffer().rewind();
break;
default:
//Unexpected transfer error - need to reset the bulk transfer interface
//Unexpected transfer error - shutdown the tuner
transfer.buffer().rewind();
//Only set an error if we're not shutting down
if(mAutoResubmitTransfers)
{
setErrorMessage("LibUsb Transfer Error - stopping device - status [" + transfer.status() + "] - " +
LibUsb.errorName(transfer.status()));
//spin this off onto the thread pool, so it doesn't impact the usb processor thread.
ThreadPool.CACHED.submit(() -> setErrorMessage("LibUsb Transfer Error - stopping device - " +
"status [" + transfer.status() + "] - " + LibUsb.errorName(transfer.status())));
}
break;
}