From b0689359d5d9cb511f5776969d3868c945dd3b3f Mon Sep 17 00:00:00 2001 From: denny Date: Tue, 17 Nov 2026 02:52:35 -0500 Subject: [PATCH] #2672 Bounds the polyphase buffer queue by sample duration instead of buffer count The polyphase buffer processor queue was capped at 500 native buffers, which is only about 85 ms of samples for an RSP at 6 MSPS (about 1,000 samples per buffer) versus about 6.8 s for an RTL-SDR (32,768 samples per buffer). Short processing stalls overflowed the queue and dropped buffers on RSP tuners. - Dispatcher: adds a weight-bounded queue option (element weigher plus an adjustable maximum queued weight). Elements count toward the weight until the listener has processed them. - PolyphaseChannelManager: bounds the buffer queue at 1 second of samples and updates the bound when the tuner sample rate changes. - RspNativeBuffer: sampleCount() returned twice the complex sample count. This also corrects the tuner sample drop monitor and the spectral display buffer manager for RSP tuners. --- .../channelizer/PolyphaseChannelManager.java | 23 ++++- .../source/tuner/sdrplay/RspNativeBuffer.java | 4 +- .../io/github/dsheirer/util/Dispatcher.java | 87 ++++++++++++++++++- .../github/dsheirer/util/DispatcherTest.java | 56 ++++++++++++ 4 files changed, 163 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelManager.java b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelManager.java index 989fdb76..f7fb91d9 100644 --- a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelManager.java +++ b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelManager.java @@ -68,7 +68,12 @@ public class PolyphaseChannelManager implements ISourceEventProcessor private final static Logger mLog = LoggerFactory.getLogger(PolyphaseChannelManager.class); private static final double CHANNEL_OVERSAMPLING = 2.0; private static final int POLYPHASE_CHANNELIZER_TAPS_PER_CHANNEL = 9; - private static final int MAXIMUM_QUEUED_NATIVE_BUFFERS = 500; + /** + * Maximum duration of queued native buffers awaiting the channelizer. The queue is bounded by sample count instead + * of buffer count because tuner buffer sizes vary widely, for example about 1,000 samples per RSP buffer and 32,768 + * samples per RTL-SDR buffer, so that each tuner can absorb a processing delay of the same duration. + */ + private static final double MAXIMUM_QUEUED_SECONDS = 1.0; private Broadcaster mSourceEventBroadcaster = new Broadcaster<>(); private TunerController mTunerController; @@ -78,7 +83,7 @@ public class PolyphaseChannelManager implements ISourceEventProcessor private ComplexPolyphaseChannelizerM2 mPolyphaseChannelizer; private ChannelSourceEventListener mChannelSourceEventListener = new ChannelSourceEventListener(); private NativeBufferReceiver mNativeBufferReceiver = new NativeBufferReceiver(); - private Dispatcher mBufferDispatcher; + private Dispatcher mBufferDispatcher; private boolean mRunning = true; /** @@ -99,10 +104,21 @@ public class PolyphaseChannelManager implements ISourceEventProcessor mChannelCalculator = new ChannelCalculator(tunerController.getSampleRate(), channelCount, tunerController.getFrequency(), CHANNEL_OVERSAMPLING); - mBufferDispatcher = new Dispatcher("sdrtrunk polyphase buffer processor", 10, MAXIMUM_QUEUED_NATIVE_BUFFERS); + mBufferDispatcher = new Dispatcher<>("sdrtrunk polyphase buffer processor", 10, INativeBuffer::sampleCount, + getMaximumQueuedSamples(tunerController.getSampleRate())); mBufferDispatcher.setListener(mNativeBufferReceiver); } + /** + * Calculates the maximum number of queued samples for the buffer dispatcher. + * @param sampleRate of the tuner + * @return maximum queued samples, at least one + */ + private static long getMaximumQueuedSamples(double sampleRate) + { + return Math.max(1, (long)(sampleRate * MAXIMUM_QUEUED_SECONDS)); + } + /** * Provides a description of the state of this manager. */ @@ -278,6 +294,7 @@ public class PolyphaseChannelManager implements ISourceEventProcessor double sampleRate = sourceEvent.getValue().doubleValue(); int channelCount = ComplexPolyphaseChannelizerM2.getChannelCount(sampleRate); mChannelCalculator.setRates(sampleRate, channelCount); + mBufferDispatcher.setMaximumQueuedWeight(getMaximumQueuedSamples(sampleRate)); break; case NOTIFICATION_FREQUENCY_AND_SAMPLE_RATE_LOCKED: case NOTIFICATION_FREQUENCY_AND_SAMPLE_RATE_UNLOCKED: diff --git a/src/main/java/io/github/dsheirer/source/tuner/sdrplay/RspNativeBuffer.java b/src/main/java/io/github/dsheirer/source/tuner/sdrplay/RspNativeBuffer.java index 749d45b5..237b8c5a 100644 --- a/src/main/java/io/github/dsheirer/source/tuner/sdrplay/RspNativeBuffer.java +++ b/src/main/java/io/github/dsheirer/source/tuner/sdrplay/RspNativeBuffer.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2023 Dennis Sheirer + * Copyright (C) 2014-2026 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -70,7 +70,7 @@ public class RspNativeBuffer extends AbstractNativeBuffer @Override public int sampleCount() { - return mISamples.length * 2; + return mISamples.length; } /** diff --git a/src/main/java/io/github/dsheirer/util/Dispatcher.java b/src/main/java/io/github/dsheirer/util/Dispatcher.java index 4d17acfc..7f450a77 100644 --- a/src/main/java/io/github/dsheirer/util/Dispatcher.java +++ b/src/main/java/io/github/dsheirer/util/Dispatcher.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2025 Dennis Sheirer + * Copyright (C) 2014-2026 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -31,6 +31,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.ToLongFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +56,9 @@ public class Dispatcher implements Listener private HeartbeatManager mHeartbeatManager; private long mDroppedCount = 0; private long mLastOverflowLogTimestamp = 0; + private final ToLongFunction mWeigher; + private volatile long mMaximumQueuedWeight = Long.MAX_VALUE; + private final AtomicLong mQueuedWeight = new AtomicLong(); /** * Constructs an instance of a Dispatcher with integrated heartbeat support. @@ -90,6 +95,7 @@ public class Dispatcher implements Listener mThreadName = threadName; mInterval = interval; mQueue = new LinkedTransferQueue<>(); + mWeigher = null; } /** @@ -103,6 +109,46 @@ public class Dispatcher implements Listener mThreadName = threadName; mInterval = interval; mQueue = new LinkedBlockingQueue<>(maximumQueueSize); + mWeigher = null; + } + + /** + * Constructs an instance that is bounded by the total weight of the queued elements instead of the element count, + * for when element sizes vary widely, for example sample buffers from different tuners. An element is dropped + * when adding it would exceed the maximum weight, except when nothing is queued, so that an element heavier than + * the maximum is still delivered. Elements count toward the weight until the listener has processed them. + * + * @param threadName to name the dispatcher thread + * @param interval for processing each batch in milliseconds. + * @param weigher to calculate the weight of each element, for example a sample count. + * @param maximumQueuedWeight for the queued elements. + */ + public Dispatcher(String threadName, long interval, ToLongFunction weigher, long maximumQueuedWeight) + { + if(weigher == null) + { + throw new IllegalArgumentException("Weigher cannot be null"); + } + + mThreadName = threadName; + mInterval = interval; + mQueue = new LinkedTransferQueue<>(); + mWeigher = weigher; + setMaximumQueuedWeight(maximumQueuedWeight); + } + + /** + * Sets the maximum total weight of the queued elements, for a dispatcher that was constructed with a weigher. + * @param maximumQueuedWeight for the queued elements. + */ + public void setMaximumQueuedWeight(long maximumQueuedWeight) + { + if(maximumQueuedWeight <= 0) + { + throw new IllegalArgumentException("Maximum queued weight must be positive: " + maximumQueuedWeight); + } + + mMaximumQueuedWeight = maximumQueuedWeight; } /** @@ -149,7 +195,27 @@ public class Dispatcher implements Listener return false; } - boolean queued = mQueue.offer(e); + boolean queued; + + if(mWeigher != null) + { + long weight = mWeigher.applyAsLong(e); + long queuedWeight = mQueuedWeight.addAndGet(weight); + + if(queuedWeight > mMaximumQueuedWeight && queuedWeight > weight) + { + mQueuedWeight.addAndGet(-weight); + queued = false; + } + else + { + queued = mQueue.offer(e); + } + } + else + { + queued = mQueue.offer(e); + } if(!queued) { @@ -189,6 +255,7 @@ public class Dispatcher implements Listener } mQueue.clear(); + mQueuedWeight.set(0); mExecutorService = Executors.newSingleThreadScheduledExecutor(new NamingThreadFactory(mThreadName)); Runnable r = (mHeartbeatManager != null ? new ProcessorWithHeartbeat() : new Processor()); @@ -210,6 +277,7 @@ public class Dispatcher implements Listener mScheduledFuture.cancel(false); mScheduledFuture = null; mQueue.clear(); + mQueuedWeight.set(0); } if(mExecutorService != null) @@ -259,6 +327,8 @@ public class Dispatcher implements Listener mListener.getClass() + "]", t); } } + + release(element); } } } @@ -294,6 +364,19 @@ public class Dispatcher implements Listener mListener.getClass() + "]", t); } } + + release(element); + } + } + + /** + * Removes the weight of a dequeued element from the queued weight, once the element is processed. + */ + private void release(E element) + { + if(mWeigher != null) + { + mQueuedWeight.addAndGet(-mWeigher.applyAsLong(element)); } } diff --git a/src/test/java/io/github/dsheirer/util/DispatcherTest.java b/src/test/java/io/github/dsheirer/util/DispatcherTest.java index 50389fb9..fb2873c1 100644 --- a/src/test/java/io/github/dsheirer/util/DispatcherTest.java +++ b/src/test/java/io/github/dsheirer/util/DispatcherTest.java @@ -125,4 +125,60 @@ public class DispatcherTest assertEquals(101, received.size()); } + + /** + * Test: a weight-bounded queue drops elements that would exceed the maximum weight and keeps lighter ones. + */ + @Test + void weightBoundedQueueDropsElementsOverMaximumWeight() throws Exception + { + //Each element weighs its own value: 1 + 2 + 3 = 6 is queued, then 4 would exceed 6 and is dropped, then 0 fits + Dispatcher dispatcher = new Dispatcher<>("test", 1, Integer::longValue, 6); + List received = dispatchWhileBlocked(dispatcher, 4); + dispatcher.receive(0); + awaitSize(received, 5); + Thread.sleep(50); + dispatcher.stop(); + + assertEquals(List.of(0, 1, 2, 3, 0), received); + } + + /** + * Test: an element heavier than the maximum weight is delivered when nothing else is queued. + */ + @Test + void weightBoundedQueueAcceptsHeavyElementWhenEmpty() throws Exception + { + Dispatcher dispatcher = new Dispatcher<>("test", 1, Integer::longValue, 6); + List received = dispatchWhileBlocked(dispatcher, 0); + dispatcher.receive(100); + dispatcher.receive(1); + awaitSize(received, 2); + Thread.sleep(50); + dispatcher.stop(); + + assertEquals(List.of(0, 100), received); + } + + /** + * Test: processed elements release their weight so that the queue accepts new elements after an overflow. + */ + @Test + void weightBoundedQueueRecoversAfterOverflow() throws Exception + { + Dispatcher dispatcher = new Dispatcher<>("test", 1, Integer::longValue, 6); + List received = dispatchWhileBlocked(dispatcher, 10); + awaitSize(received, 4); + Thread.sleep(50); + + for(int x = 0; x < 3; x++) + { + dispatcher.receive(5); + awaitSize(received, 5 + x); + } + + dispatcher.stop(); + + assertEquals(List.of(0, 1, 2, 3, 5, 5, 5), received); + } }