Resolves significant memory usage issue - updates Airspy and HackRF with

the recent changes applied to RTL2832 Tuner controller for processing of
libusb transfer buffers.
This commit is contained in:
denny
2016-02-27 06:19:35 -05:00
parent eead22ffad
commit cf2c0ca07d
3 changed files with 252 additions and 214 deletions

View File

@@ -262,6 +262,29 @@ public class AirspyTunerController extends TunerController
} }
} }
public static String getTransferStatus( int status )
{
switch( status )
{
case 0:
return "TRANSFER COMPLETED (0)";
case 1:
return "TRANSFER ERROR (1)";
case 2:
return "TRANSFER TIMED OUT (2)";
case 3:
return "TRANSFER CANCELLED (3)";
case 4:
return "TRANSFER STALL (4)";
case 5:
return "TRANSFER NO DEVICE (5)";
case 6:
return "TRANSFER OVERFLOW (6)";
default:
return "UNKNOWN TRANSFER STATUS (" + status + ")";
}
}
@Override @Override
public void apply( TunerConfiguration config ) throws SourceException public void apply( TunerConfiguration config ) throws SourceException
{ {
@@ -1191,6 +1214,8 @@ public class AirspyTunerController extends TunerController
* not currently running, starts it running in a new thread. * not currently running, starts it running in a new thread.
*/ */
public void addListener( Listener<ComplexBuffer> listener ) public void addListener( Listener<ComplexBuffer> listener )
{
synchronized( mComplexBufferBroadcaster )
{ {
mComplexBufferBroadcaster.addListener( listener ); mComplexBufferBroadcaster.addListener( listener );
@@ -1205,12 +1230,15 @@ public class AirspyTunerController extends TunerController
thread.start(); thread.start();
} }
} }
}
/** /**
* Removes the sample listener. If this is the last registered listener, * Removes the sample listener. If this is the last registered listener,
* shuts down the buffer processing thread. * shuts down the buffer processing thread.
*/ */
public void removeListener( Listener<ComplexBuffer> listener ) public void removeListener( Listener<ComplexBuffer> listener )
{
synchronized( mComplexBufferBroadcaster )
{ {
mComplexBufferBroadcaster.removeListener( listener ); mComplexBufferBroadcaster.removeListener( listener );
@@ -1219,6 +1247,7 @@ public class AirspyTunerController extends TunerController
mBufferProcessor.stop(); mBufferProcessor.stop();
} }
} }
}
/** /**
* Buffer processing thread. Fetches samples from the Airspy Tuner and * Buffer processing thread. Fetches samples from the Airspy Tuner and
@@ -1227,8 +1256,11 @@ public class AirspyTunerController extends TunerController
public class BufferProcessor implements Runnable, TransferCallback public class BufferProcessor implements Runnable, TransferCallback
{ {
private ScheduledFuture<?> mSampleDispatcherTask; private ScheduledFuture<?> mSampleDispatcherTask;
private CopyOnWriteArrayList<Transfer> mTransfers; private LinkedTransferQueue<Transfer> mAvailableTransfers;
private LinkedTransferQueue<Transfer> mTransfersInProgress = new LinkedTransferQueue<>();
private AtomicBoolean mRunning = new AtomicBoolean(); private AtomicBoolean mRunning = new AtomicBoolean();
private ByteBuffer mLibUsbHandlerStatus;
private boolean mCancel = false;
@Override @Override
public void run() public void run()
@@ -1237,33 +1269,64 @@ public class AirspyTunerController extends TunerController
{ {
prepareTransfers(); prepareTransfers();
for( Transfer transfer: mTransfers )
{
int result = LibUsb.submitTransfer( transfer );
if( result != LibUsb.SUCCESS )
{
mLog.error( "error submitting transfer [" +
LibUsb.errorName( result ) + "]" );
break;
}
}
mSampleDispatcherTask = mThreadPoolManager.scheduleFixedRate( mSampleDispatcherTask = mThreadPoolManager.scheduleFixedRate(
ThreadType.SOURCE_SAMPLE_PROCESSING, new BufferDispatcher(), ThreadType.SOURCE_SAMPLE_PROCESSING, new BufferDispatcher(),
20, TimeUnit.MILLISECONDS ); 20, TimeUnit.MILLISECONDS );
mLibUsbHandlerStatus = ByteBuffer.allocateDirect( 4 );
List<Transfer> transfers = new ArrayList<>();
mCancel = false;
while( mRunning.get() ) while( mRunning.get() )
{ {
ByteBuffer completed = ByteBuffer.allocateDirect( 4 ); mAvailableTransfers.drainTo( transfers );
for( Transfer transfer: transfers )
{
int result = LibUsb.submitTransfer( transfer );
if( result == LibUsb.SUCCESS )
{
mTransfersInProgress.add( transfer );
}
else
{
mLog.error( "error submitting transfer [" +
LibUsb.errorName( result ) + "]" );
}
}
int result = LibUsb.handleEventsTimeoutCompleted( int result = LibUsb.handleEventsTimeoutCompleted(
null, USB_TIMEOUT_MS, completed.asIntBuffer() ); null, USB_TIMEOUT_MS, mLibUsbHandlerStatus.asIntBuffer() );
if( result != LibUsb.SUCCESS ) if( result != LibUsb.SUCCESS )
{ {
mLog.error( "error handling events for libusb" ); mLog.error( "error handling events for libusb" );
} }
transfers.clear();
mLibUsbHandlerStatus.rewind();
}
if( mCancel )
{
for( Transfer transfer: mTransfersInProgress )
{
LibUsb.cancelTransfer( transfer );
}
int result = LibUsb.handleEventsTimeoutCompleted( null,
USB_TIMEOUT_MS, mLibUsbHandlerStatus.asIntBuffer() );
if( result != LibUsb.SUCCESS )
{
mLog.error( "error handling events for libusb" );
}
mLibUsbHandlerStatus.rewind();
} }
} }
} }
@@ -1273,10 +1336,10 @@ public class AirspyTunerController extends TunerController
*/ */
public void stop() public void stop()
{ {
mCancel = true;
if( mRunning.compareAndSet( true, false ) ) if( mRunning.compareAndSet( true, false ) )
{ {
cancel();
if( mSampleDispatcherTask != null ) if( mSampleDispatcherTask != null )
{ {
mSampleDispatcherTask.cancel( true ); mSampleDispatcherTask.cancel( true );
@@ -1296,7 +1359,13 @@ public class AirspyTunerController extends TunerController
@Override @Override
public void processTransfer( Transfer transfer ) public void processTransfer( Transfer transfer )
{ {
if( transfer.status() == LibUsb.TRANSFER_COMPLETED ) mTransfersInProgress.remove( transfer );
switch( transfer.status() )
{
case LibUsb.TRANSFER_COMPLETED:
case LibUsb.TRANSFER_STALL:
if( transfer.actualLength() > 0 )
{ {
ByteBuffer buffer = transfer.buffer(); ByteBuffer buffer = transfer.buffer();
@@ -1306,55 +1375,22 @@ public class AirspyTunerController extends TunerController
buffer.rewind(); buffer.rewind();
if( isRunning() )
{
mFilledBuffers.add( data ); mFilledBuffers.add( data );
if( !isRunning() )
{
LibUsb.cancelTransfer( transfer );
} }
} }
switch( transfer.status() )
{
case LibUsb.TRANSFER_COMPLETED:
/* resubmit the transfer */
int result = LibUsb.submitTransfer( transfer );
if( result != LibUsb.SUCCESS )
{
mLog.error( "couldn't resubmit buffer transfer to tuner" );
LibUsb.freeTransfer( transfer );
mTransfers.remove( transfer );
}
break; break;
case LibUsb.TRANSFER_CANCELLED: case LibUsb.TRANSFER_CANCELLED:
/* free the transfer and remove it */
LibUsb.freeTransfer( transfer );
mTransfers.remove( transfer );
break; break;
default: default:
/* unexpected error */ /* unexpected error */
mLog.error( "transfer error" ); mLog.error( "transfer error [" +
} getTransferStatus( transfer.status() ) +
"] transferred actual: " + transfer.actualLength() );
} }
private void cancel() mAvailableTransfers.add( transfer );
{
for( Transfer transfer: mTransfers )
{
LibUsb.cancelTransfer( transfer );
}
ByteBuffer completed = ByteBuffer.allocateDirect( 4 );
int result = LibUsb.handleEventsTimeoutCompleted(
null, USB_TIMEOUT_MS, completed.asIntBuffer() );
if( result != LibUsb.SUCCESS )
{
mLog.error( "error handling usb events during cancel [" +
LibUsb.errorName( result ) + "]" );
}
} }
/** /**
@@ -1363,7 +1399,9 @@ public class AirspyTunerController extends TunerController
*/ */
private void prepareTransfers() throws LibUsbException private void prepareTransfers() throws LibUsbException
{ {
mTransfers = new CopyOnWriteArrayList<Transfer>(); if( mAvailableTransfers == null )
{
mAvailableTransfers = new LinkedTransferQueue<>();
for( int x = 0; x < TRANSFER_BUFFER_POOL_SIZE; x++ ) for( int x = 0; x < TRANSFER_BUFFER_POOL_SIZE; x++ )
{ {
@@ -1378,15 +1416,11 @@ public class AirspyTunerController extends TunerController
final ByteBuffer buffer = final ByteBuffer buffer =
ByteBuffer.allocateDirect( mBufferSize ); ByteBuffer.allocateDirect( mBufferSize );
LibUsb.fillBulkTransfer( transfer, LibUsb.fillBulkTransfer( transfer, mDeviceHandle, USB_ENDPOINT,
mDeviceHandle, buffer, BufferProcessor.this, "Buffer", USB_TIMEOUT_MS );
USB_ENDPOINT,
buffer,
BufferProcessor.this,
"Buffer #" + x,
USB_TIMEOUT_MS );
mTransfers.add( transfer ); mAvailableTransfers.add( transfer );
}
} }
} }
} }

View File

@@ -56,6 +56,7 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder; import java.nio.ByteOrder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedTransferQueue; import java.util.concurrent.LinkedTransferQueue;
import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledFuture;
@@ -91,7 +92,7 @@ public class HackRFTunerController extends TunerController
private final static Logger mLog = private final static Logger mLog =
LoggerFactory.getLogger( HackRFTunerController.class ); LoggerFactory.getLogger( HackRFTunerController.class );
public final static long TIMEOUT_US = 1000000l; //uSeconds public final static long USB_TIMEOUT_MS = 1000000l; //uSeconds
public static final byte USB_ENDPOINT = (byte)0x81; public static final byte USB_ENDPOINT = (byte)0x81;
public static final byte USB_INTERFACE = (byte)0x0; public static final byte USB_INTERFACE = (byte)0x0;
@@ -394,7 +395,7 @@ public class HackRFTunerController extends TunerController
(short)value, (short)value,
(short)index, (short)index,
buffer, buffer,
TIMEOUT_US ); USB_TIMEOUT_MS );
if( transferred < 0 ) if( transferred < 0 )
{ {
@@ -467,7 +468,7 @@ public class HackRFTunerController extends TunerController
(short)value, (short)value,
(short)index, (short)index,
buffer, buffer,
TIMEOUT_US ); USB_TIMEOUT_MS );
if( transferred < 0 ) if( transferred < 0 )
{ {
@@ -885,6 +886,8 @@ public class HackRFTunerController extends TunerController
* not currently running, starts it running in a new thread. * not currently running, starts it running in a new thread.
*/ */
public void addListener( Listener<ComplexBuffer> listener ) public void addListener( Listener<ComplexBuffer> listener )
{
synchronized( mComplexBufferBroadcaster )
{ {
mComplexBufferBroadcaster.addListener( listener ); mComplexBufferBroadcaster.addListener( listener );
@@ -899,12 +902,15 @@ public class HackRFTunerController extends TunerController
thread.start(); thread.start();
} }
} }
}
/** /**
* Removes the sample listener. If this is the last registered listener, * Removes the sample listener. If this is the last registered listener,
* shuts down the buffer processing thread. * shuts down the buffer processing thread.
*/ */
public void removeListener( Listener<ComplexBuffer> listener ) public void removeListener( Listener<ComplexBuffer> listener )
{
synchronized( mComplexBufferBroadcaster )
{ {
mComplexBufferBroadcaster.removeListener( listener ); mComplexBufferBroadcaster.removeListener( listener );
@@ -913,6 +919,7 @@ public class HackRFTunerController extends TunerController
mBufferProcessor.stop(); mBufferProcessor.stop();
} }
} }
}
/** /**
* Buffer processing thread. Fetches samples from the HackRF Tuner and * Buffer processing thread. Fetches samples from the HackRF Tuner and
@@ -921,45 +928,77 @@ public class HackRFTunerController extends TunerController
public class BufferProcessor implements Runnable, TransferCallback public class BufferProcessor implements Runnable, TransferCallback
{ {
private ScheduledFuture<?> mSampleDispatcherTask; private ScheduledFuture<?> mSampleDispatcherTask;
private CopyOnWriteArrayList<Transfer> mTransfers; private LinkedTransferQueue<Transfer> mAvailableTransfers;
private LinkedTransferQueue<Transfer> mTransfersInProgress = new LinkedTransferQueue<>();
private AtomicBoolean mRunning = new AtomicBoolean(); private AtomicBoolean mRunning = new AtomicBoolean();
private ByteBuffer mLibUsbHandlerStatus;
private boolean mCancel = false;
@Override @Override
public void run() public void run()
{ {
if( mRunning.compareAndSet( false, true ) ) if( mRunning.compareAndSet( false, true ) )
{ {
mLog.info( "starting HackRF sample fetch thread" );
prepareTransfers(); prepareTransfers();
for( Transfer transfer: mTransfers )
{
int result = LibUsb.submitTransfer( transfer );
if( result != LibUsb.SUCCESS )
{
mLog.error( "error submitting transfer [" +
LibUsb.errorName( result ) + "]" );
break;
}
}
mSampleDispatcherTask = mThreadPoolManager.scheduleFixedRate( mSampleDispatcherTask = mThreadPoolManager.scheduleFixedRate(
ThreadType.SOURCE_SAMPLE_PROCESSING, new BufferDispatcher(), ThreadType.SOURCE_SAMPLE_PROCESSING, new BufferDispatcher(),
20, TimeUnit.MILLISECONDS ); 20, TimeUnit.MILLISECONDS );
mLibUsbHandlerStatus = ByteBuffer.allocateDirect( 4 );
List<Transfer> transfers = new ArrayList<>();
mCancel = false;
while( mRunning.get() ) while( mRunning.get() )
{ {
ByteBuffer completed = ByteBuffer.allocateDirect( 4 ); mAvailableTransfers.drainTo( transfers );
for( Transfer transfer: transfers )
{
int result = LibUsb.submitTransfer( transfer );
if( result == LibUsb.SUCCESS )
{
mTransfersInProgress.add( transfer );
}
else
{
mLog.error( "error submitting transfer [" +
LibUsb.errorName( result ) + "]" );
}
}
int result = LibUsb.handleEventsTimeoutCompleted( int result = LibUsb.handleEventsTimeoutCompleted(
null, TIMEOUT_US, completed.asIntBuffer() ); null, USB_TIMEOUT_MS, mLibUsbHandlerStatus.asIntBuffer() );
if( result != LibUsb.SUCCESS ) if( result != LibUsb.SUCCESS )
{ {
mLog.error( "error handling events for libusb" ); mLog.error( "error handling events for libusb" );
} }
transfers.clear();
mLibUsbHandlerStatus.rewind();
}
if( mCancel )
{
for( Transfer transfer: mTransfersInProgress )
{
LibUsb.cancelTransfer( transfer );
}
int result = LibUsb.handleEventsTimeoutCompleted(
null, USB_TIMEOUT_MS, mLibUsbHandlerStatus.asIntBuffer() );
if( result != LibUsb.SUCCESS )
{
mLog.error( "error cancelling transfers in progress" );
}
mLibUsbHandlerStatus.rewind();
} }
} }
} }
@@ -969,12 +1008,10 @@ public class HackRFTunerController extends TunerController
*/ */
public void stop() public void stop()
{ {
mCancel = true;
if( mRunning.compareAndSet( true, false ) ) if( mRunning.compareAndSet( true, false ) )
{ {
mLog.info( "stopping HackRF sample fetch thread" );
cancel();
if( mSampleDispatcherTask != null ) if( mSampleDispatcherTask != null )
{ {
mSampleDispatcherTask.cancel( true ); mSampleDispatcherTask.cancel( true );
@@ -994,7 +1031,13 @@ public class HackRFTunerController extends TunerController
@Override @Override
public void processTransfer( Transfer transfer ) public void processTransfer( Transfer transfer )
{ {
if( transfer.status() == LibUsb.TRANSFER_COMPLETED ) mTransfersInProgress.remove( transfer );
switch( transfer.status() )
{
case LibUsb.TRANSFER_COMPLETED:
case LibUsb.TRANSFER_STALL:
if( transfer.actualLength() > 0 )
{ {
ByteBuffer buffer = transfer.buffer(); ByteBuffer buffer = transfer.buffer();
@@ -1004,31 +1047,13 @@ public class HackRFTunerController extends TunerController
buffer.rewind(); buffer.rewind();
if( isRunning() )
{
mFilledBuffers.add( data ); mFilledBuffers.add( data );
if( !isRunning() )
{
LibUsb.cancelTransfer( transfer );
} }
} }
switch( transfer.status() )
{
case LibUsb.TRANSFER_COMPLETED:
/* resubmit the transfer */
int result = LibUsb.submitTransfer( transfer );
if( result != LibUsb.SUCCESS )
{
mLog.error( "couldn't resubmit buffer transfer to tuner" );
LibUsb.freeTransfer( transfer );
mTransfers.remove( transfer );
}
break; break;
case LibUsb.TRANSFER_CANCELLED: case LibUsb.TRANSFER_CANCELLED:
/* free the transfer and remove it */
LibUsb.freeTransfer( transfer );
mTransfers.remove( transfer );
break; break;
default: default:
/* unexpected error */ /* unexpected error */
@@ -1036,25 +1061,8 @@ public class HackRFTunerController extends TunerController
getTransferStatus( transfer.status() ) + getTransferStatus( transfer.status() ) +
"] transferred actual: " + transfer.actualLength() ); "] transferred actual: " + transfer.actualLength() );
} }
}
private void cancel() mAvailableTransfers.add( transfer );
{
for( Transfer transfer: mTransfers )
{
LibUsb.cancelTransfer( transfer );
}
ByteBuffer completed = ByteBuffer.allocateDirect( 4 );
int result = LibUsb.handleEventsTimeoutCompleted(
null, TIMEOUT_US, completed.asIntBuffer() );
if( result != LibUsb.SUCCESS )
{
mLog.error( "error handling usb events during cancel [" +
LibUsb.errorName( result ) + "]" );
}
} }
/** /**
@@ -1063,7 +1071,9 @@ public class HackRFTunerController extends TunerController
*/ */
private void prepareTransfers() throws LibUsbException private void prepareTransfers() throws LibUsbException
{ {
mTransfers = new CopyOnWriteArrayList<Transfer>(); if( mAvailableTransfers == null )
{
mAvailableTransfers = new LinkedTransferQueue<>();
for( int x = 0; x < TRANSFER_BUFFER_POOL_SIZE; x++ ) for( int x = 0; x < TRANSFER_BUFFER_POOL_SIZE; x++ )
{ {
@@ -1078,15 +1088,11 @@ public class HackRFTunerController extends TunerController
final ByteBuffer buffer = final ByteBuffer buffer =
ByteBuffer.allocateDirect( mBufferSize ); ByteBuffer.allocateDirect( mBufferSize );
LibUsb.fillBulkTransfer( transfer, LibUsb.fillBulkTransfer( transfer, mDeviceHandle,USB_ENDPOINT,
mDeviceHandle, buffer, BufferProcessor.this, "Buffer", USB_TIMEOUT_MS );
USB_ENDPOINT,
buffer,
BufferProcessor.this,
"Buffer #" + x,
TIMEOUT_US );
mTransfers.add( transfer ); mAvailableTransfers.add( transfer );
}
} }
} }
} }

View File

@@ -28,7 +28,6 @@ import java.text.DecimalFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedTransferQueue; import java.util.concurrent.LinkedTransferQueue;
import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@@ -72,7 +71,7 @@ public abstract class RTL2832TunerController extends TunerController
public final static int TWO_TO_22_POWER = 4194304; public final static int TWO_TO_22_POWER = 4194304;
public final static byte USB_INTERFACE = (byte)0x0; public final static byte USB_INTERFACE = (byte)0x0;
public final static byte BULK_ENDPOINT_IN = (byte)0x81; public final static byte USB_BULK_ENDPOINT = (byte)0x81;
public final static byte CONTROL_ENDPOINT_IN = public final static byte CONTROL_ENDPOINT_IN =
(byte)( LibUsb.ENDPOINT_IN | LibUsb.REQUEST_TYPE_VENDOR ); (byte)( LibUsb.ENDPOINT_IN | LibUsb.REQUEST_TYPE_VENDOR );
public final static byte CONTROL_ENDPOINT_OUT = public final static byte CONTROL_ENDPOINT_OUT =
@@ -1387,7 +1386,6 @@ public abstract class RTL2832TunerController extends TunerController
private ScheduledFuture<?> mSampleDispatcherTask; private ScheduledFuture<?> mSampleDispatcherTask;
private LinkedTransferQueue<Transfer> mAvailableTransfers; private LinkedTransferQueue<Transfer> mAvailableTransfers;
private LinkedTransferQueue<Transfer> mTransfersInProgress = new LinkedTransferQueue<>(); private LinkedTransferQueue<Transfer> mTransfersInProgress = new LinkedTransferQueue<>();
private CopyOnWriteArrayList<Transfer> mTransfers;
private AtomicBoolean mRunning = new AtomicBoolean(); private AtomicBoolean mRunning = new AtomicBoolean();
private ThreadPoolManager mThreadPoolManager; private ThreadPoolManager mThreadPoolManager;
private ByteBuffer mLibUsbHandlerStatus; private ByteBuffer mLibUsbHandlerStatus;
@@ -1577,7 +1575,7 @@ public abstract class RTL2832TunerController extends TunerController
final ByteBuffer buffer = final ByteBuffer buffer =
ByteBuffer.allocateDirect( mBufferSize ); ByteBuffer.allocateDirect( mBufferSize );
LibUsb.fillBulkTransfer( transfer, mDeviceHandle, BULK_ENDPOINT_IN, LibUsb.fillBulkTransfer( transfer, mDeviceHandle, USB_BULK_ENDPOINT,
buffer, BufferProcessor.this, "Buffer", TIMEOUT_US ); buffer, BufferProcessor.this, "Buffer", TIMEOUT_US );
mAvailableTransfers.add( transfer ); mAvailableTransfers.add( transfer );