#1378 Resolves potential error where tuner shutdown (because of USB I/O error) hangs and causes application to run out of memory. This patch doesn't resolve the original issue with the USB I/O error, but it does ensure that the tuner shutdown which is happening on the USB processing thread, can complete in a timely fashion so that the application doesn't (potentially) crash from out of memory. (#1382)

Co-authored-by: Dennis Sheirer <dsheirer@github.com>
This commit is contained in:
Denny Sheirer
2022-12-24 09:10:17 -05:00
committed by GitHub
parent ddb5b5ef87
commit a87309e792
2 changed files with 79 additions and 28 deletions

View File

@@ -321,7 +321,6 @@ public class IcecastTCPAudioBroadcaster extends IcecastAudioBroadcaster
{ {
String message = (String) object; String message = (String) object;
mLog.info("Stream [" + getStreamName() + "] message received: " + message);
if(message != null && !message.trim().isEmpty()) if(message != null && !message.trim().isEmpty())
{ {
if(message.startsWith("HTTP/1.0 200 OK")) if(message.startsWith("HTTP/1.0 200 OK"))

View File

@@ -26,6 +26,7 @@ import io.github.dsheirer.source.tuner.ITunerErrorListener;
import io.github.dsheirer.source.tuner.TunerController; import io.github.dsheirer.source.tuner.TunerController;
import io.github.dsheirer.source.tuner.TunerType; import io.github.dsheirer.source.tuner.TunerType;
import io.github.dsheirer.source.tuner.manager.TunerManager; import io.github.dsheirer.source.tuner.manager.TunerManager;
import io.github.dsheirer.util.ThreadPool;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@@ -267,9 +268,30 @@ public abstract class USBTunerController extends TunerController
public final void stop() public final void stop()
{ {
mRunning = false; mRunning = false;
stopStreaming();
mNativeBufferBroadcaster.clear(); //Spin the shutdown onto a new thread so that we can set a max wait threshold.
deviceStop(); Thread t = new Thread(() -> {
stopStreaming();
mNativeBufferBroadcaster.clear();
deviceStop();
});
t.start();
try
{
//Wait up to 500 milliseconds for the shutdown to complete ... otherwise interrupt it and finish shutdown
t.join(500);
if(t.isAlive())
{
mLog.info("Tuner shutdown exceeded 500ms - forcing shutdown");
t.interrupt();
}
}
catch(InterruptedException ie)
{
}
if(mDeviceHandle != null) if(mDeviceHandle != null)
{ {
@@ -495,6 +517,9 @@ public abstract class USBTunerController extends TunerController
private List<Transfer> mAvailableTransfers; private List<Transfer> mAvailableTransfers;
private LinkedTransferQueue<Transfer> mInProgressTransfers = new LinkedTransferQueue<>(); private LinkedTransferQueue<Transfer> mInProgressTransfers = new LinkedTransferQueue<>();
private boolean mAutoResubmitTransfers = false; private boolean mAutoResubmitTransfers = false;
private int mTransferErrorCount = 0;
private List<Transfer> mErrorTransfers = new ArrayList<>();
private int mResubmitFailureLogCount = 0;
/** /**
* Creates USB Transfers to carry the streaming sample data. Transfer buffers are backed by native memory * Creates USB Transfers to carry the streaming sample data. Transfer buffers are backed by native memory
@@ -559,34 +584,56 @@ public abstract class USBTunerController extends TunerController
int status = LibUsb.submitTransfer(transfer); int status = LibUsb.submitTransfer(transfer);
if(status == LibUsb.SUCCESS) if(status == LibUsb.SUCCESS)
{
mInProgressTransfers.add(transfer);
}
else if(status == LibUsb.ERROR_BUSY)
{ {
mInProgressTransfers.add(transfer); mInProgressTransfers.add(transfer);
//Weird issue/bug with libusb on Windows where libusb tells us the transfer is complete and then tells //Attempt to resubmit any previous transfers that failed on submit
//us that the transfer is already submitted when we attempt to re-submit the transfer. We track the if(!mErrorTransfers.isEmpty())
//number of times this occurs and log when the anomaly count exceeds the number of transfer buffers
//in case the application gets to a state where all transfers are no longer usable ... so that we
//know post-mortem what happened to the application.
mAnomalousTransfersDetected++;
if(mAnomalousTransfersDetected < USB_BULK_TRANSFER_BUFFER_POOL_SIZE)
{ {
mLog.error("USB transfer anomaly detected - continuing - previous status [" + previousStatus + Transfer toResubmit = mErrorTransfers.get(0);
" ] transferred [" + previousBytesTransferred + "] this has happened [" + int resubmitStatus = LibUsb.submitTransfer(toResubmit);
mAnomalousTransfersDetected + "] times");
if(resubmitStatus == LibUsb.SUCCESS)
{
mInProgressTransfers.add(transfer);
mTransferErrorCount--;
}
else
{
mErrorTransfers.add(transfer);
if(mResubmitFailureLogCount < mAvailableTransfers.size())
{
mResubmitFailureLogCount++;
mLog.error("Attempt to resubmit previously failed transfer unsuccessful - status: " +
LibUsb.errorName(resubmitStatus));
}
if(mErrorTransfers.size() >= mAvailableTransfers.size())
{
mLog.error("Maximum USB I/O transfer submit errors reached - shutting down USB tuner");
ThreadPool.CACHED.submit(() -> setErrorMessage("USB Error - Transfers"));
}
}
} }
}
else if(status == LibUsb.ERROR_BUSY || status == LibUsb.ERROR_IO)
{
mErrorTransfers.add(transfer);
mTransferErrorCount++;
//This should only log once, when the anomaly count matches the transfer count //For ERROR_BUSY error, there is a weird issue/bug with libusb on Windows where libusb tells us the
if(mAnomalousTransfersDetected == USB_BULK_TRANSFER_BUFFER_POOL_SIZE) //transfer is complete and then tells us that the transfer is already submitted when we attempt to
//re-submit the transfer. We track the number of times this occurs and log when the anomaly count
//exceeds the number of transfer buffers in case the application gets to a state where all transfers
//are no longer usable ... so that we know post-mortem what happened to the application.
mLog.error("USB error while submitting transfer buffer [" + LibUsb.errorName(status) + "] - this happened [" +
mTransferErrorCount + "] times, but may be temporary. Will attempt to resubmit in future");
if(mErrorTransfers.size() >= mAvailableTransfers.size())
{ {
mLog.warn("USB transfer anomaly detection count exceeded the total number [8] of available " + mLog.error("Maximum USB I/O transfer submit errors reached - shutting down USB tuner");
"tranfers - sdrtrunk may no longer be streaming data from this tuner and may appear to " + ThreadPool.CACHED.submit(() -> setErrorMessage("USB Error - Transfers"));
"be locked up. If so, restart the application to resolve the issue and please send the " +
"application log to the developer");
} }
} }
else else
@@ -610,6 +657,10 @@ public abstract class USBTunerController extends TunerController
{ {
LibUsb.cancelTransfer(transfer); LibUsb.cancelTransfer(transfer);
} }
for(Transfer transfer: mErrorTransfers)
{
LibUsb.cancelTransfer(transfer);
}
} }
@Override @Override
@@ -641,14 +692,15 @@ public abstract class USBTunerController extends TunerController
transfer.buffer().rewind(); transfer.buffer().rewind();
break; break;
default: default:
//Unexpected transfer error - need to reset the bulk transfer interface //Unexpected transfer error - shutdown the tuner
transfer.buffer().rewind(); transfer.buffer().rewind();
//Only set an error if we're not shutting down //Only set an error if we're not shutting down
if(mAutoResubmitTransfers) if(mAutoResubmitTransfers)
{ {
setErrorMessage("LibUsb Transfer Error - stopping device - status [" + transfer.status() + "] - " + //spin this off onto the thread pool, so it doesn't impact the usb processor thread.
LibUsb.errorName(transfer.status())); ThreadPool.CACHED.submit(() -> setErrorMessage("LibUsb Transfer Error - stopping device - " +
"status [" + transfer.status() + "] - " + LibUsb.errorName(transfer.status())));
} }
break; break;
} }