diff --git a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/ComplexPolyphaseChannelizerM2.java b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/ComplexPolyphaseChannelizerM2.java index f82b45dc..243a519e 100644 --- a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/ComplexPolyphaseChannelizerM2.java +++ b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/ComplexPolyphaseChannelizerM2.java @@ -79,9 +79,11 @@ public class ComplexPolyphaseChannelizerM2 extends AbstractComplexPolyphaseChann private static final int MAXIMUM_QUEUED_BATCHES = 50; /** - * Maximum batch sample arrays held for reuse: the queued IFFT batches plus the batches queued for channel output. + * Maximum batch sample arrays held for reuse: the queued IFFT batches plus the batch being filled and the batch + * being transformed. Channel output processors copy their channels from a batch instead of retaining it, so no + * other batches are held. */ - private static final int MAXIMUM_POOLED_BATCHES = 2 * MAXIMUM_QUEUED_BATCHES; + private static final int MAXIMUM_POOLED_BATCHES = MAXIMUM_QUEUED_BATCHES + 2; //Sized to process 40 times per second private IFFTProcessorDispatcher mIFFTProcessorDispatcher = new IFFTProcessorDispatcher(25); diff --git a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelManager.java b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelManager.java index 989fdb76..f7fb91d9 100644 --- a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelManager.java +++ b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelManager.java @@ -68,7 +68,12 @@ public class PolyphaseChannelManager implements ISourceEventProcessor private final static Logger mLog = LoggerFactory.getLogger(PolyphaseChannelManager.class); private static final double CHANNEL_OVERSAMPLING = 2.0; private static final int POLYPHASE_CHANNELIZER_TAPS_PER_CHANNEL = 9; - private static final int MAXIMUM_QUEUED_NATIVE_BUFFERS = 500; + /** + * Maximum duration of queued native buffers awaiting the channelizer. The queue is bounded by sample count instead + * of buffer count because tuner buffer sizes vary widely, for example about 1,000 samples per RSP buffer and 32,768 + * samples per RTL-SDR buffer, so that each tuner can absorb a processing delay of the same duration. + */ + private static final double MAXIMUM_QUEUED_SECONDS = 1.0; private Broadcaster mSourceEventBroadcaster = new Broadcaster<>(); private TunerController mTunerController; @@ -78,7 +83,7 @@ public class PolyphaseChannelManager implements ISourceEventProcessor private ComplexPolyphaseChannelizerM2 mPolyphaseChannelizer; private ChannelSourceEventListener mChannelSourceEventListener = new ChannelSourceEventListener(); private NativeBufferReceiver mNativeBufferReceiver = new NativeBufferReceiver(); - private Dispatcher mBufferDispatcher; + private Dispatcher mBufferDispatcher; private boolean mRunning = true; /** @@ -99,10 +104,21 @@ public class PolyphaseChannelManager implements ISourceEventProcessor mChannelCalculator = new ChannelCalculator(tunerController.getSampleRate(), channelCount, tunerController.getFrequency(), CHANNEL_OVERSAMPLING); - mBufferDispatcher = new Dispatcher("sdrtrunk polyphase buffer processor", 10, MAXIMUM_QUEUED_NATIVE_BUFFERS); + mBufferDispatcher = new Dispatcher<>("sdrtrunk polyphase buffer processor", 10, INativeBuffer::sampleCount, + getMaximumQueuedSamples(tunerController.getSampleRate())); mBufferDispatcher.setListener(mNativeBufferReceiver); } + /** + * Calculates the maximum number of queued samples for the buffer dispatcher. + * @param sampleRate of the tuner + * @return maximum queued samples, at least one + */ + private static long getMaximumQueuedSamples(double sampleRate) + { + return Math.max(1, (long)(sampleRate * MAXIMUM_QUEUED_SECONDS)); + } + /** * Provides a description of the state of this manager. */ @@ -278,6 +294,7 @@ public class PolyphaseChannelManager implements ISourceEventProcessor double sampleRate = sourceEvent.getValue().doubleValue(); int channelCount = ComplexPolyphaseChannelizerM2.getChannelCount(sampleRate); mChannelCalculator.setRates(sampleRate, channelCount); + mBufferDispatcher.setMaximumQueuedWeight(getMaximumQueuedSamples(sampleRate)); break; case NOTIFICATION_FREQUENCY_AND_SAMPLE_RATE_LOCKED: case NOTIFICATION_FREQUENCY_AND_SAMPLE_RATE_UNLOCKED: diff --git a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelSource.java b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelSource.java index 7bc08ce6..f8da87f9 100644 --- a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelSource.java +++ b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/PolyphaseChannelSource.java @@ -180,7 +180,8 @@ public class PolyphaseChannelSource extends TunerChannelSource implements Listen /** * Performs the output processor update. This method must be invoked on the channel results calling thread so that * replacing the output processor doesn't overlap with delivery of channel results to it. The output processor - * applies the configuration changes on its own dispatcher thread. + * applies channel index changes as it copies the next batch, and applies mixer and filter changes on its own + * dispatcher thread at that same batch. * @param channelCalculator for updates * @param filterManager for updates * @throws IllegalArgumentException if there's an issue. diff --git a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/ChannelOutputProcessor.java b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/ChannelOutputProcessor.java index eb1848f3..422ec424 100644 --- a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/ChannelOutputProcessor.java +++ b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/ChannelOutputProcessor.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2024 Dennis Sheirer + * Copyright (C) 2014-2026 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,6 +24,7 @@ import io.github.dsheirer.sample.complex.ComplexSamples; import io.github.dsheirer.source.heartbeat.HeartbeatManager; import io.github.dsheirer.util.Dispatcher; import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; @@ -34,12 +35,20 @@ public abstract class ChannelOutputProcessor implements IPolyphaseChannelOutputP private final static Logger mLog = LoggerFactory.getLogger(ChannelOutputProcessor.class); /** - * Maximum queued channel results batches, about one second of samples. A full batch is dropped when this channel + * Maximum queued channel sample batches, about one second of samples. A full batch is dropped when this channel * falls behind, which preserves the even-sized batch block parity that two-channel synthesis relies on. */ private static final int MAXIMUM_QUEUED_BATCHES = 50; - private Dispatcher mChannelResultsDispatcher; + /** + * Maximum channel sample arrays held for reuse: the queued batches plus the batch being processed and the batch + * being copied. + */ + private static final int MAXIMUM_POOLED_ARRAYS = MAXIMUM_QUEUED_BATCHES + 2; + + private Dispatcher mChannelSamplesDispatcher; + private final Queue mSamplePool = new ArrayBlockingQueue<>(MAXIMUM_POOLED_ARRAYS); + private volatile int[] mChannelOffsets = new int[0]; private final Queue mPendingUpdates = new ConcurrentLinkedQueue<>(); private final AtomicLong mBatchSequence = new AtomicLong(); private HeartbeatManager mHeartbeatManager; @@ -60,25 +69,45 @@ public abstract class ChannelOutputProcessor implements IPolyphaseChannelOutputP mInputChannelCount = inputChannelCount; //Process 1/10th of the sample rate per second at a rate of 20 times a second (200% of anticipated rate) mHeartbeatManager = heartbeatManager; - mChannelResultsDispatcher = new Dispatcher(threadName,50, mHeartbeatManager, MAXIMUM_QUEUED_BATCHES); - mChannelResultsDispatcher.setListener(queued -> { + mChannelSamplesDispatcher = new Dispatcher<>(threadName,50, mHeartbeatManager, MAXIMUM_QUEUED_BATCHES); + mChannelSamplesDispatcher.setListener(queued -> { try { applyPendingUpdates(queued.sequence()); - mCurrentSampleTimestamp = queued.channelResults().getTimestamp(); - process(queued.channelResults()); + mCurrentSampleTimestamp = queued.timestamp(); + process(queued.samples(), queued.blockCount()); } catch(Throwable t) { - mLog.error("Error processing channel results", t); + mLog.error("Error processing channel samples", t); } finally { - queued.channelResults().release(); + mSamplePool.offer(queued.samples()); } }); } + /** + * Sets the I/Q offsets within each channel results block for the polyphase channel(s) that this processor copies. + * The offsets take effect at the first channel results batch received after this method is invoked, the same as a + * change queued with enqueueUpdate(), so a change of offsets and of the mixer frequency apply to the same batch. + * This should only be invoked on the thread that delivers channel results, or before results are delivered. + * @param offsets for each polyphase channel, which is twice the channel index. + */ + protected void setChannelOffsets(int... offsets) + { + mChannelOffsets = offsets.clone(); + } + + /** + * Current I/Q offsets within each channel results block for the polyphase channel(s) that this processor copies. + */ + protected int[] getChannelOffsets() + { + return mChannelOffsets.clone(); + } + /** * Queues a configuration change to apply on the dispatcher thread, so that configuration changes don't overlap with * sample processing. The change takes effect at the first channel results batch received after this method is @@ -119,13 +148,13 @@ public abstract class ChannelOutputProcessor implements IPolyphaseChannelOutputP @Override public void start() { - mChannelResultsDispatcher.start(); + mChannelSamplesDispatcher.start(); } @Override public void stop() { - mChannelResultsDispatcher.stop(); + mChannelSamplesDispatcher.stop(); } /** @@ -147,22 +176,66 @@ public abstract class ChannelOutputProcessor implements IPolyphaseChannelOutputP { } + /** + * Copies this processor's polyphase channel(s) from the channel results batch and queues the copy for processing on + * the dispatcher thread. The copy is small (about 8 kB per polyphase channel for a 1024 block batch), so the batch + * isn't retained beyond this method and its large shared sample array returns to the channelizer's pool as soon as + * the channelizer has dispatched it to every channel. A slow channel then queues only its own samples. + * + * @param channelResults batch to copy from + */ @Override public void receiveChannelResults(ChannelResultsBatch channelResults) { - channelResults.retain(); + long sequence = mBatchSequence.incrementAndGet(); - if(!mChannelResultsDispatcher.offer(new QueuedChannelResults(channelResults, mBatchSequence.incrementAndGet()))) + if(!mChannelSamplesDispatcher.isRunning()) { - channelResults.release(); + return; + } + + int[] offsets = mChannelOffsets; + float[] source = channelResults.getSamples(); + int blockLength = channelResults.getBlockLength(); + int blockCount = channelResults.getBlockCount(); + float[] samples = getPooledArray(blockCount * offsets.length * 2); + int pointer = 0; + + for(int block = 0; block < blockCount; block++) + { + int blockOffset = block * blockLength; + + for(int offset : offsets) + { + samples[pointer++] = source[blockOffset + offset]; + samples[pointer++] = source[blockOffset + offset + 1]; + } + } + + if(!mChannelSamplesDispatcher.offer(new QueuedChannelSamples(samples, blockCount, channelResults.getTimestamp(), + sequence))) + { + mSamplePool.offer(samples); } } /** - * Sub-class implementation to process one polyphase channelizer results batch. - * @param channelResults to process + * Provides a reused sample array of the requested length when one is available, or a new array. */ - public abstract void process(ChannelResultsBatch channelResults); + private float[] getPooledArray(int length) + { + float[] samples = mSamplePool.poll(); + return samples != null && samples.length == length ? samples : new float[length]; + } + + /** + * Sub-class implementation to process one batch of copied channel samples. + * @param samples for each block in order, with the I/Q pair of each polyphase channel in the order of the channel + * offsets, for example I1,Q1,I2,Q2 per block for two channels. + * @param blockCount number of blocks. Batches start on an even block index, so the index of a block within this + * batch has the same parity as its absolute block index. + */ + public abstract void process(float[] samples, int blockCount); @Override public int getInputChannelCount() @@ -171,9 +244,10 @@ public abstract class ChannelOutputProcessor implements IPolyphaseChannelOutputP } /** - * Channel results batch and its position in the sequence of received batches. + * Copied channel samples with the timestamp of the batch they were copied from and the batch's position in the + * sequence of received batches. */ - private record QueuedChannelResults(ChannelResultsBatch channelResults, long sequence) {} + private record QueuedChannelSamples(float[] samples, int blockCount, long timestamp, long sequence) {} /** * Configuration change and the sequence number of the last batch received before the change was requested. diff --git a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/OneChannelOutputProcessor.java b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/OneChannelOutputProcessor.java index 521e03fe..d1372ff9 100644 --- a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/OneChannelOutputProcessor.java +++ b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/OneChannelOutputProcessor.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2024 Dennis Sheirer + * Copyright (C) 2014-2026 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,9 +18,9 @@ */ package io.github.dsheirer.dsp.filter.channelizer.output; -import io.github.dsheirer.dsp.filter.channelizer.ChannelResultsBatch; import io.github.dsheirer.sample.complex.ComplexSamples; import io.github.dsheirer.source.heartbeat.HeartbeatManager; +import java.util.Arrays; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +29,6 @@ public class OneChannelOutputProcessor extends ChannelOutputProcessor { private final static Logger mLog = LoggerFactory.getLogger(OneChannelOutputProcessor.class); private final OneChannelMixerAssembler mMixerAssembler; - private int mChannelOffset; /** * Processor to extract a single channel from a polyphase channelizer and produce an output I/Q complex sample @@ -55,7 +54,7 @@ public class OneChannelOutputProcessor extends ChannelOutputProcessor { StringBuilder sb = new StringBuilder(); sb.append("One Channel Output Processor"); - sb.append("\n\tIndex (doubled) [").append(mChannelOffset).append("]"); + sb.append("\n\tIndex (doubled) ").append(Arrays.toString(getChannelOffsets())); sb.append("\n\tMixer Assembler: none"); return sb.toString(); } @@ -81,8 +80,7 @@ public class OneChannelOutputProcessor extends ChannelOutputProcessor //Set the channelized output results offset to twice the channel index to account for each channel having //an I/Q pair - int channelOffset = indexes.get(0) * 2; - enqueueUpdate(() -> mChannelOffset = channelOffset); + setChannelOffsets(indexes.get(0) * 2); } @Override @@ -92,19 +90,18 @@ public class OneChannelOutputProcessor extends ChannelOutputProcessor } /** - * Extract the channel from the channel results array and pass to the assembler. The assembler will - * apply frequency translation and gain and indicate when a buffer is fully assembled. + * Passes the copied channel samples to the assembler. The assembler will apply frequency translation and gain and + * indicate when a buffer is fully assembled. * - * @param channelResults to process containing blocks of channel I/Q sample pairs (I0,Q0,I1,Q1...In,Qn) + * @param samples containing an I/Q pair for each block (I0,Q0,I1,Q1...In,Qn) + * @param blockCount number of blocks */ @Override - public void process(ChannelResultsBatch channelResults) + public void process(float[] samples, int blockCount) { - float[] samples = channelResults.getSamples(); - int blockLength = channelResults.getBlockLength(); - int channelOffset = mChannelOffset; + int length = blockCount * 2; - for(int index = channelOffset; index < samples.length; index += blockLength) + for(int index = 0; index < length; index += 2) { mMixerAssembler.receive(samples[index], samples[index + 1]); diff --git a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/TwoChannelOutputProcessor.java b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/TwoChannelOutputProcessor.java index 9e3820f0..a35f37ba 100644 --- a/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/TwoChannelOutputProcessor.java +++ b/src/main/java/io/github/dsheirer/dsp/filter/channelizer/output/TwoChannelOutputProcessor.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2024 Dennis Sheirer + * Copyright (C) 2014-2026 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,9 +18,9 @@ */ package io.github.dsheirer.dsp.filter.channelizer.output; -import io.github.dsheirer.dsp.filter.channelizer.ChannelResultsBatch; import io.github.dsheirer.sample.complex.ComplexSamples; import io.github.dsheirer.source.heartbeat.HeartbeatManager; +import java.util.Arrays; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,8 +29,6 @@ public class TwoChannelOutputProcessor extends ChannelOutputProcessor { private final static Logger mLog = LoggerFactory.getLogger(TwoChannelOutputProcessor.class); private TwoChannelMixerAssembler mMixerAssembler; - private int mChannelOffset1; - private int mChannelOffset2; /** * Processor to extract two channels from a polyphase channelizer, synthesize/recombine the channels, apply @@ -60,7 +58,7 @@ public class TwoChannelOutputProcessor extends ChannelOutputProcessor { StringBuilder sb = new StringBuilder(); sb.append("Two Channel Output Processor"); - sb.append("\n\tIndices (doubled) 1 [").append(mChannelOffset1).append("] 2 [").append(mChannelOffset2).append("]"); + sb.append("\n\tIndices (doubled) ").append(Arrays.toString(getChannelOffsets())); sb.append("\n\tMixer Assembler: ").append(mMixerAssembler.getStateDescription()); return sb.toString(); } @@ -98,45 +96,32 @@ public class TwoChannelOutputProcessor extends ChannelOutputProcessor //Set the channelized output results offsets to twice the channel index to account for each channel having //an I/Q pair - int channelOffset1 = indexes.get(0) * 2; - int channelOffset2 = indexes.get(1) * 2; - - enqueueUpdate(() -> { - mChannelOffset1 = channelOffset1; - mChannelOffset2 = channelOffset2; - }); + setChannelOffsets(indexes.get(0) * 2, indexes.get(1) * 2); } /** - * Extract the channel from the channel results array, apply frequency translation, and deliver the - * extracted frequency-corrected channel I/Q sample set to the complex sample listener. + * Synthesizes the two copied channels, applies frequency translation, and delivers the frequency-corrected channel + * I/Q sample set to the complex sample listener. * - * @param channelResults to process containing blocks of channel I/Q sample pairs (I0,Q0,I1,Q1...In,Qn) + * @param samples containing an I/Q pair for each of the two channels per block (I1,Q1,I2,Q2 ...) + * @param blockCount number of blocks */ @Override - public void process(ChannelResultsBatch channelResults) + public void process(float[] samples, int blockCount) { - float[] samples = channelResults.getSamples(); - int blockLength = channelResults.getBlockLength(); - int blockCount = channelResults.getBlockCount(); - int channelOffset1 = mChannelOffset1; - int channelOffset2 = mChannelOffset2; - //Translate channel 2 up by FS/2 (multiply by (-1)^n) so that it sits adjacent to channel 1 for synthesis. The //channelizer dispatches even-sized batches, so the index within the batch carries the absolute block parity. for(int x = 0; x < blockCount; x++) { - int block = x * blockLength; + int block = x * 4; if((x & 1) == 0) { - mMixerAssembler.receive(samples[block + channelOffset1], samples[block + channelOffset1 + 1], - samples[block + channelOffset2], samples[block + channelOffset2 + 1]); + mMixerAssembler.receive(samples[block], samples[block + 1], samples[block + 2], samples[block + 3]); } else { - mMixerAssembler.receive(samples[block + channelOffset1], samples[block + channelOffset1 + 1], - -samples[block + channelOffset2], -samples[block + channelOffset2 + 1]); + mMixerAssembler.receive(samples[block], samples[block + 1], -samples[block + 2], -samples[block + 3]); } if(mMixerAssembler.hasBuffer()) diff --git a/src/main/java/io/github/dsheirer/source/tuner/sdrplay/RspNativeBuffer.java b/src/main/java/io/github/dsheirer/source/tuner/sdrplay/RspNativeBuffer.java index 749d45b5..237b8c5a 100644 --- a/src/main/java/io/github/dsheirer/source/tuner/sdrplay/RspNativeBuffer.java +++ b/src/main/java/io/github/dsheirer/source/tuner/sdrplay/RspNativeBuffer.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2023 Dennis Sheirer + * Copyright (C) 2014-2026 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -70,7 +70,7 @@ public class RspNativeBuffer extends AbstractNativeBuffer @Override public int sampleCount() { - return mISamples.length * 2; + return mISamples.length; } /** diff --git a/src/main/java/io/github/dsheirer/util/Dispatcher.java b/src/main/java/io/github/dsheirer/util/Dispatcher.java index 4d17acfc..7f450a77 100644 --- a/src/main/java/io/github/dsheirer/util/Dispatcher.java +++ b/src/main/java/io/github/dsheirer/util/Dispatcher.java @@ -1,6 +1,6 @@ /* * ***************************************************************************** - * Copyright (C) 2014-2025 Dennis Sheirer + * Copyright (C) 2014-2026 Dennis Sheirer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -31,6 +31,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.ToLongFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +56,9 @@ public class Dispatcher implements Listener private HeartbeatManager mHeartbeatManager; private long mDroppedCount = 0; private long mLastOverflowLogTimestamp = 0; + private final ToLongFunction mWeigher; + private volatile long mMaximumQueuedWeight = Long.MAX_VALUE; + private final AtomicLong mQueuedWeight = new AtomicLong(); /** * Constructs an instance of a Dispatcher with integrated heartbeat support. @@ -90,6 +95,7 @@ public class Dispatcher implements Listener mThreadName = threadName; mInterval = interval; mQueue = new LinkedTransferQueue<>(); + mWeigher = null; } /** @@ -103,6 +109,46 @@ public class Dispatcher implements Listener mThreadName = threadName; mInterval = interval; mQueue = new LinkedBlockingQueue<>(maximumQueueSize); + mWeigher = null; + } + + /** + * Constructs an instance that is bounded by the total weight of the queued elements instead of the element count, + * for when element sizes vary widely, for example sample buffers from different tuners. An element is dropped + * when adding it would exceed the maximum weight, except when nothing is queued, so that an element heavier than + * the maximum is still delivered. Elements count toward the weight until the listener has processed them. + * + * @param threadName to name the dispatcher thread + * @param interval for processing each batch in milliseconds. + * @param weigher to calculate the weight of each element, for example a sample count. + * @param maximumQueuedWeight for the queued elements. + */ + public Dispatcher(String threadName, long interval, ToLongFunction weigher, long maximumQueuedWeight) + { + if(weigher == null) + { + throw new IllegalArgumentException("Weigher cannot be null"); + } + + mThreadName = threadName; + mInterval = interval; + mQueue = new LinkedTransferQueue<>(); + mWeigher = weigher; + setMaximumQueuedWeight(maximumQueuedWeight); + } + + /** + * Sets the maximum total weight of the queued elements, for a dispatcher that was constructed with a weigher. + * @param maximumQueuedWeight for the queued elements. + */ + public void setMaximumQueuedWeight(long maximumQueuedWeight) + { + if(maximumQueuedWeight <= 0) + { + throw new IllegalArgumentException("Maximum queued weight must be positive: " + maximumQueuedWeight); + } + + mMaximumQueuedWeight = maximumQueuedWeight; } /** @@ -149,7 +195,27 @@ public class Dispatcher implements Listener return false; } - boolean queued = mQueue.offer(e); + boolean queued; + + if(mWeigher != null) + { + long weight = mWeigher.applyAsLong(e); + long queuedWeight = mQueuedWeight.addAndGet(weight); + + if(queuedWeight > mMaximumQueuedWeight && queuedWeight > weight) + { + mQueuedWeight.addAndGet(-weight); + queued = false; + } + else + { + queued = mQueue.offer(e); + } + } + else + { + queued = mQueue.offer(e); + } if(!queued) { @@ -189,6 +255,7 @@ public class Dispatcher implements Listener } mQueue.clear(); + mQueuedWeight.set(0); mExecutorService = Executors.newSingleThreadScheduledExecutor(new NamingThreadFactory(mThreadName)); Runnable r = (mHeartbeatManager != null ? new ProcessorWithHeartbeat() : new Processor()); @@ -210,6 +277,7 @@ public class Dispatcher implements Listener mScheduledFuture.cancel(false); mScheduledFuture = null; mQueue.clear(); + mQueuedWeight.set(0); } if(mExecutorService != null) @@ -259,6 +327,8 @@ public class Dispatcher implements Listener mListener.getClass() + "]", t); } } + + release(element); } } } @@ -294,6 +364,19 @@ public class Dispatcher implements Listener mListener.getClass() + "]", t); } } + + release(element); + } + } + + /** + * Removes the weight of a dequeued element from the queued weight, once the element is processed. + */ + private void release(E element) + { + if(mWeigher != null) + { + mQueuedWeight.addAndGet(-mWeigher.applyAsLong(element)); } } diff --git a/src/test/java/io/github/dsheirer/dsp/filter/channelizer/ChannelResultsBatchTest.java b/src/test/java/io/github/dsheirer/dsp/filter/channelizer/ChannelResultsBatchTest.java index b60b1e6b..2478228c 100644 --- a/src/test/java/io/github/dsheirer/dsp/filter/channelizer/ChannelResultsBatchTest.java +++ b/src/test/java/io/github/dsheirer/dsp/filter/channelizer/ChannelResultsBatchTest.java @@ -19,11 +19,16 @@ package io.github.dsheirer.dsp.filter.channelizer; import io.github.dsheirer.dsp.filter.channelizer.output.OneChannelOutputProcessor; +import io.github.dsheirer.dsp.filter.channelizer.output.TwoChannelOutputProcessor; +import io.github.dsheirer.sample.complex.ComplexSamples; import java.util.ArrayDeque; +import java.util.Arrays; import java.util.List; import java.util.Queue; +import java.util.concurrent.CopyOnWriteArrayList; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -77,4 +82,51 @@ public class ChannelResultsBatchTest assertEquals(1, pool.size()); } + + /** + * Test: a running output processor copies its channels instead of retaining the batch, so the batch returns to the + * pool as soon as the channelizer releases it, even when the channel hasn't processed it yet. The copy holds the + * two channels' samples for each block in order. + */ + @Test + void runningOutputProcessorCopiesChannelsWithoutRetainingBatch() throws Exception + { + int blockLength = 8; //4 polyphase channels, I/Q interleaved + int blockCount = 2; + float[] samples = new float[blockLength * blockCount]; + + for(int x = 0; x < samples.length; x++) + { + samples[x] = x; + } + + Queue pool = new ArrayDeque<>(); + ChannelResultsBatch batch = new ChannelResultsBatch(samples, blockLength, 0, pool); + List copied = new CopyOnWriteArrayList<>(); + TwoChannelOutputProcessor processor = new TwoChannelOutputProcessor(50000.0, List.of(1, 3), new float[]{1.0f}, + 1.0f, null, "test") + { + @Override + public void process(float[] channelSamples, int count) + { + copied.add(Arrays.copyOf(channelSamples, count * 4)); + } + }; + processor.setListener((ComplexSamples complexSamples) -> {}); + processor.start(); + processor.receiveChannelResults(batch); + batch.release(); + + assertSame(samples, pool.poll(), "Batch was retained by the output processor"); + + for(int wait = 0; wait < 100 && copied.isEmpty(); wait++) + { + Thread.sleep(20); + } + + processor.stop(); + + //Channel 1 is at offset 2 and channel 3 at offset 6 within each 8 element block + assertArrayEquals(new float[]{2, 3, 6, 7, 10, 11, 14, 15}, copied.get(0)); + } } diff --git a/src/test/java/io/github/dsheirer/util/DispatcherTest.java b/src/test/java/io/github/dsheirer/util/DispatcherTest.java index 50389fb9..fb2873c1 100644 --- a/src/test/java/io/github/dsheirer/util/DispatcherTest.java +++ b/src/test/java/io/github/dsheirer/util/DispatcherTest.java @@ -125,4 +125,60 @@ public class DispatcherTest assertEquals(101, received.size()); } + + /** + * Test: a weight-bounded queue drops elements that would exceed the maximum weight and keeps lighter ones. + */ + @Test + void weightBoundedQueueDropsElementsOverMaximumWeight() throws Exception + { + //Each element weighs its own value: 1 + 2 + 3 = 6 is queued, then 4 would exceed 6 and is dropped, then 0 fits + Dispatcher dispatcher = new Dispatcher<>("test", 1, Integer::longValue, 6); + List received = dispatchWhileBlocked(dispatcher, 4); + dispatcher.receive(0); + awaitSize(received, 5); + Thread.sleep(50); + dispatcher.stop(); + + assertEquals(List.of(0, 1, 2, 3, 0), received); + } + + /** + * Test: an element heavier than the maximum weight is delivered when nothing else is queued. + */ + @Test + void weightBoundedQueueAcceptsHeavyElementWhenEmpty() throws Exception + { + Dispatcher dispatcher = new Dispatcher<>("test", 1, Integer::longValue, 6); + List received = dispatchWhileBlocked(dispatcher, 0); + dispatcher.receive(100); + dispatcher.receive(1); + awaitSize(received, 2); + Thread.sleep(50); + dispatcher.stop(); + + assertEquals(List.of(0, 100), received); + } + + /** + * Test: processed elements release their weight so that the queue accepts new elements after an overflow. + */ + @Test + void weightBoundedQueueRecoversAfterOverflow() throws Exception + { + Dispatcher dispatcher = new Dispatcher<>("test", 1, Integer::longValue, 6); + List received = dispatchWhileBlocked(dispatcher, 10); + awaitSize(received, 4); + Thread.sleep(50); + + for(int x = 0; x < 3; x++) + { + dispatcher.receive(5); + awaitSize(received, 5 + x); + } + + dispatcher.stop(); + + assertEquals(List.of(0, 1, 2, 3, 5, 5, 5), received); + } }