From 48a93990e8b1cbc863621be6f31dd3cf9c527d96 Mon Sep 17 00:00:00 2001 From: Denny Sheirer Date: Sat, 23 Jul 2022 08:14:31 -0400 Subject: [PATCH] #1291 Updates DC removal processing across tuner sample streams to run on a periodic timer to reduce overall processing requirements. Updates build version to 0.5.0 beta 5 (#1292) Co-authored-by: Dennis Sheirer --- build.gradle | 2 +- .../buffer/ByteNativeBufferFactory.java | 15 ++- .../dsheirer/buffer/DcCorrectionManager.java | 114 ++++++++++++++++++ .../buffer/SignedByteNativeBufferFactory.java | 73 ++++++++++- .../airspy/ScalarPackedSampleConverter.java | 60 ++++++--- .../airspy/ScalarUnpackedSampleConverter.java | 49 +++++--- .../airspy/VectorUnpackedSampleConverter.java | 89 +++++++++----- 7 files changed, 330 insertions(+), 72 deletions(-) create mode 100644 src/main/java/io/github/dsheirer/buffer/DcCorrectionManager.java diff --git a/build.gradle b/build.gradle index 52019395..17f1905b 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ repositories { maven { url "https://jitpack.io" } } -version = '0.5.0' +version = '0.5.0-beta5' sourceCompatibility = '17' sourceSets { diff --git a/src/main/java/io/github/dsheirer/buffer/ByteNativeBufferFactory.java b/src/main/java/io/github/dsheirer/buffer/ByteNativeBufferFactory.java index 86225b92..257a5d1e 100644 --- a/src/main/java/io/github/dsheirer/buffer/ByteNativeBufferFactory.java +++ b/src/main/java/io/github/dsheirer/buffer/ByteNativeBufferFactory.java @@ -26,16 +26,20 @@ import java.nio.ByteBuffer; */ public class ByteNativeBufferFactory implements INativeBufferFactory { - private static final float DC_FILTER_GAIN = 0.007f; //Normalizes DC over period of ~142 (1 / .007) buffers - private float mAverageDc = 0.0f; + private DcCorrectionManager mDcCorrectionManager = new DcCorrectionManager(); @Override public INativeBuffer getBuffer(ByteBuffer samples, long timestamp) { byte[] copy = new byte[samples.capacity()]; samples.get(copy); - calculateDc(copy); - return new ByteNativeBuffer(copy, timestamp, mAverageDc); + + if(mDcCorrectionManager.shouldCalculateDc()) + { + calculateDc(copy); + } + + return new ByteNativeBuffer(copy, timestamp, mDcCorrectionManager.getAverageDc()); } /** @@ -55,7 +59,6 @@ public class ByteNativeBufferFactory implements INativeBufferFactory dcAccumulator /= samples.length; dcAccumulator -= 127.5f; dcAccumulator /= 128.0f; - dcAccumulator -= mAverageDc; - mAverageDc += (dcAccumulator * DC_FILTER_GAIN); + mDcCorrectionManager.adjust(dcAccumulator); } } diff --git a/src/main/java/io/github/dsheirer/buffer/DcCorrectionManager.java b/src/main/java/io/github/dsheirer/buffer/DcCorrectionManager.java new file mode 100644 index 00000000..807ed9dc --- /dev/null +++ b/src/main/java/io/github/dsheirer/buffer/DcCorrectionManager.java @@ -0,0 +1,114 @@ +/* + * ***************************************************************************** + * Copyright (C) 2014-2022 Dennis Sheirer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see + * **************************************************************************** + */ + +package io.github.dsheirer.buffer; + +import java.util.concurrent.TimeUnit; + +/** + * Provides DC offset correction value and manages calculation processing interval. + */ +public class DcCorrectionManager +{ +// private static Logger mLog = LoggerFactory.getLogger(DcCorrectionManager.class); +// private DecimalFormat mDecimalFormat = new DecimalFormat("0.00000"); + + /** + * Time (delay) interval for running DC removal calculations + */ + private static final long DC_PROCESSING_INTERVAL = TimeUnit.SECONDS.toMillis(50); + + /** + * Number of buffers to process per DC calculation processing interval. + */ + private static final int DC_CALCULATIONS_PER_INTERVAL = 5; + + /** + * DC offset processing residual threshold for ending DC processing interval + */ + private static final float TARGET_DC_OFFSET_REMAINING = 0.0001f; + + /** + * Number of DC calculations remaining in the current interval + */ + private int mDcCalculationsRemaining = DC_CALCULATIONS_PER_INTERVAL; //Initial value for coarse correction + + /** + * Last time DC offset calculation interval ran + */ + private long mLastDcCalculationTimestamp = 0; + + /** + * DC offset calculation gain. + */ + private static final float DC_FILTER_GAIN = 0.05f; + + /** + * Current DC offset correction value + */ + private float mAverageDc = 0.0f; + + public float getAverageDc() + { + return mAverageDc; + } + + /** + * Adjusts the average DC based on the current calculated DC offset present in a sample buffer. + * @param calculatedDc for the current sample buffer + */ + public void adjust(float calculatedDc) + { + //Calculate the average scaled DC offset so that it can be applied in the native buffer's converted samples + float residualOffsetToRemove = calculatedDc - mAverageDc; + + mAverageDc += (residualOffsetToRemove * DC_FILTER_GAIN); + + if(Math.abs(residualOffsetToRemove) < TARGET_DC_OFFSET_REMAINING) + { + mDcCalculationsRemaining--; + } +// +// mLog.info("DC: " + mDecimalFormat.format(mAverageDc) + +// " RES:" + mDecimalFormat.format(residualOffsetToRemove) + +// " REM:" + mDcCalculationsRemaining); + } + + /** + * Indicates if a DC offset calculation for a buffer should be performed. + */ + public boolean shouldCalculateDc() + { + if(System.currentTimeMillis() > (mLastDcCalculationTimestamp + DC_PROCESSING_INTERVAL)) + { + if(mDcCalculationsRemaining > 0) + { + return true; + } + else + { + //Reset the number of buffers to calculate and the timestamp + mDcCalculationsRemaining = DC_CALCULATIONS_PER_INTERVAL; + mLastDcCalculationTimestamp = System.currentTimeMillis(); + } + } + + return false; + } +} diff --git a/src/main/java/io/github/dsheirer/buffer/SignedByteNativeBufferFactory.java b/src/main/java/io/github/dsheirer/buffer/SignedByteNativeBufferFactory.java index a123d00d..699b946b 100644 --- a/src/main/java/io/github/dsheirer/buffer/SignedByteNativeBufferFactory.java +++ b/src/main/java/io/github/dsheirer/buffer/SignedByteNativeBufferFactory.java @@ -20,14 +20,51 @@ package io.github.dsheirer.buffer; import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; /** * Implements a factory for creating SignedByteNativeBuffer instances */ public class SignedByteNativeBufferFactory implements INativeBufferFactory { - private static final float DC_FILTER_GAIN = 0.02f; //Normalizes DC over period of ~50 (1 / .02) buffers + /** + * DC removal calculations will run once a minute + */ + private static final long DC_PROCESSING_INTERVAL = TimeUnit.SECONDS.toMillis(60); + + /** + * Number of buffers to process per DC calculation processing interval. + */ + private static final int DC_CALCULATIONS_PER_INTERVAL = 5; + + /** + * DC offset processing residual threshold for ending DC processing interval + */ + private static final float TARGET_DC_OFFSET_REMAINING = 0.0002f; + + /** + * Number of DC calculations remaining in the current interval + */ + private int mDcCalculationsRemaining = DC_CALCULATIONS_PER_INTERVAL; //Initial value for coarse correction + + /** + * Last time DC offset calculation interval ran + */ + private long mLastDcCalculationTimestamp = 0; + + /** + * DC offset calculation gain. + */ + private static final float DC_FILTER_GAIN = 0.05f; + + /** + * Current DC offset correction for Inphase samples + */ private float mIAverageDc = 0.0f; + + /** + * Current DC offset correction for Quadrature samples + */ private float mQAverageDc = 0.0f; @Override @@ -35,10 +72,37 @@ public class SignedByteNativeBufferFactory implements INativeBufferFactory { byte[] copy = new byte[samples.capacity()]; samples.get(copy); - calculateDc(copy); + + if(shouldCalculateDc()) + { + calculateDc(copy); + } + return new SignedByteNativeBuffer(copy, timestamp, mIAverageDc, mQAverageDc); } + /** + * Indicates if a DC offset calculation for a buffer should be performed. + */ + private boolean shouldCalculateDc() + { + if(System.currentTimeMillis() > (mLastDcCalculationTimestamp + DC_PROCESSING_INTERVAL)) + { + if(mDcCalculationsRemaining > 0) + { + return true; + } + else + { + //Reset the number of buffers to calculate and the timestamp + mDcCalculationsRemaining = DC_CALCULATIONS_PER_INTERVAL; + mLastDcCalculationTimestamp = System.currentTimeMillis(); + } + } + + return false; + } + /** * Calculates the average DC in the sample stream so that it can be subtracted from the samples when the * native buffer is used. @@ -64,5 +128,10 @@ public class SignedByteNativeBufferFactory implements INativeBufferFactory qDcAccumulator /= 128.0f; qDcAccumulator -= mQAverageDc; mQAverageDc += (qDcAccumulator * DC_FILTER_GAIN); + + if(Math.abs(iDcAccumulator) < TARGET_DC_OFFSET_REMAINING && Math.abs(qDcAccumulator) < TARGET_DC_OFFSET_REMAINING) + { + mDcCalculationsRemaining--; + } } } diff --git a/src/main/java/io/github/dsheirer/buffer/airspy/ScalarPackedSampleConverter.java b/src/main/java/io/github/dsheirer/buffer/airspy/ScalarPackedSampleConverter.java index 0f1e0d60..9648515f 100644 --- a/src/main/java/io/github/dsheirer/buffer/airspy/ScalarPackedSampleConverter.java +++ b/src/main/java/io/github/dsheirer/buffer/airspy/ScalarPackedSampleConverter.java @@ -19,6 +19,7 @@ package io.github.dsheirer.buffer.airspy; +import io.github.dsheirer.buffer.DcCorrectionManager; import java.nio.ByteBuffer; /** @@ -26,40 +27,61 @@ import java.nio.ByteBuffer; */ public class ScalarPackedSampleConverter implements IAirspySampleConverter { - private static final float DC_FILTER_GAIN = 0.007f; //Normalizes DC over period of ~142 (1 / .007) buffers - private float mAverageDc = 0.0f; + /** + * Manages DC calculations and processing interval + */ + private DcCorrectionManager mDcCalculationManager = new DcCorrectionManager(); @Override public short[] convert(ByteBuffer buffer) { + boolean shouldCalculateDc = mDcCalculationManager.shouldCalculateDc(); + int offset = 0; - long dcAccumulator = 0; short sample; short[] samples; byte b1, b2, b3; samples = new short[buffer.capacity() / 3 * 2]; - for(int x = 0; x < samples.length; x += 2) + if(shouldCalculateDc) { - b1 = buffer.get(offset++); - b2 = buffer.get(offset++); - b3 = buffer.get(offset++); + long dcAccumulator = 0; - sample = (short)(((b1 & 0xFF) << 4) | ((b2 & 0xF0) >> 4)); - samples[x] = sample; - dcAccumulator += sample; + for(int x = 0; x < samples.length; x += 2) + { + b1 = buffer.get(offset++); + b2 = buffer.get(offset++); + b3 = buffer.get(offset++); - sample = (short)(((b2 & 0xF) << 8) | (b3 & 0xFF)); - samples[x] = sample; - dcAccumulator += sample; + sample = (short)(((b1 & 0xFF) << 4) | ((b2 & 0xF0) >> 4)); + samples[x] = sample; + dcAccumulator += sample; + + sample = (short)(((b2 & 0xF) << 8) | (b3 & 0xFF)); + samples[x] = sample; + dcAccumulator += sample; + } + + float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f; + averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT; + mDcCalculationManager.adjust(averageDcNow); } + else + { + for(int x = 0; x < samples.length; x += 2) + { + b1 = buffer.get(offset++); + b2 = buffer.get(offset++); + b3 = buffer.get(offset++); - //Calculate the average scaled DC offset so that it can be applied in the native buffer's converted samples - float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f; - averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT; - averageDcNow -= mAverageDc; - mAverageDc += (averageDcNow * DC_FILTER_GAIN); + sample = (short)(((b1 & 0xFF) << 4) | ((b2 & 0xF0) >> 4)); + samples[x] = sample; + + sample = (short)(((b2 & 0xF) << 8) | (b3 & 0xFF)); + samples[x] = sample; + } + } return samples; } @@ -67,7 +89,7 @@ public class ScalarPackedSampleConverter implements IAirspySampleConverter @Override public float getAverageDc() { - return mAverageDc; + return mDcCalculationManager.getAverageDc(); } public static void main(String[] args) diff --git a/src/main/java/io/github/dsheirer/buffer/airspy/ScalarUnpackedSampleConverter.java b/src/main/java/io/github/dsheirer/buffer/airspy/ScalarUnpackedSampleConverter.java index e2a11c4a..6c702c97 100644 --- a/src/main/java/io/github/dsheirer/buffer/airspy/ScalarUnpackedSampleConverter.java +++ b/src/main/java/io/github/dsheirer/buffer/airspy/ScalarUnpackedSampleConverter.java @@ -19,6 +19,7 @@ package io.github.dsheirer.buffer.airspy; +import io.github.dsheirer.buffer.DcCorrectionManager; import java.nio.ByteBuffer; /** @@ -26,34 +27,50 @@ import java.nio.ByteBuffer; */ public class ScalarUnpackedSampleConverter implements IAirspySampleConverter { - private static final float DC_FILTER_GAIN = 0.007f; //Normalizes DC over period of ~142 (1 / .007) buffers - private float mAverageDc = 0.0f; + /** + * Manages DC calculations and processing interval + */ + private DcCorrectionManager mDcCalculationManager = new DcCorrectionManager(); @Override public short[] convert(ByteBuffer buffer) { + boolean shouldCalculateDc = mDcCalculationManager.shouldCalculateDc(); + int offset = 0; - long dcAccumulator = 0; short sample; short[] samples; byte b1, b2; samples = new short[buffer.capacity() / 2]; - for(int x = 0; x < samples.length; x++) + if(shouldCalculateDc) { - b1 = buffer.get(offset++); - b2 = buffer.get(offset++); - sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF)); - samples[x] = sample; - dcAccumulator += sample; - } + long dcAccumulator = 0; - //Calculate the average scaled DC offset so that it can be applied in the native buffer's converted samples - float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f; - averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT; - averageDcNow -= mAverageDc; - mAverageDc += (averageDcNow * DC_FILTER_GAIN); + for(int x = 0; x < samples.length; x++) + { + b1 = buffer.get(offset++); + b2 = buffer.get(offset++); + sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF)); + samples[x] = sample; + dcAccumulator += sample; + } + + float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f; + averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT; + mDcCalculationManager.adjust(averageDcNow); + } + else + { + for(int x = 0; x < samples.length; x++) + { + b1 = buffer.get(offset++); + b2 = buffer.get(offset++); + sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF)); + samples[x] = sample; + } + } return samples; } @@ -61,6 +78,6 @@ public class ScalarUnpackedSampleConverter implements IAirspySampleConverter @Override public float getAverageDc() { - return mAverageDc; + return mDcCalculationManager.getAverageDc(); } } diff --git a/src/main/java/io/github/dsheirer/buffer/airspy/VectorUnpackedSampleConverter.java b/src/main/java/io/github/dsheirer/buffer/airspy/VectorUnpackedSampleConverter.java index f94acf1e..45b889c7 100644 --- a/src/main/java/io/github/dsheirer/buffer/airspy/VectorUnpackedSampleConverter.java +++ b/src/main/java/io/github/dsheirer/buffer/airspy/VectorUnpackedSampleConverter.java @@ -19,25 +19,29 @@ package io.github.dsheirer.buffer.airspy; +import io.github.dsheirer.buffer.DcCorrectionManager; +import java.nio.ByteBuffer; import jdk.incubator.vector.ShortVector; import jdk.incubator.vector.VectorOperators; import jdk.incubator.vector.VectorSpecies; -import java.nio.ByteBuffer; - /** * Scalar implementation of airspy sample converter for un-packed samples. */ public class VectorUnpackedSampleConverter implements IAirspySampleConverter { private static final VectorSpecies VECTOR_SPECIES = ShortVector.SPECIES_PREFERRED; - private static final float DC_FILTER_GAIN = 0.007f; //Normalizes DC over period of ~142 (1 / .007) buffers - private float mAverageDc = 0.0f; + + /** + * Manages DC calculations and processing interval + */ + private DcCorrectionManager mDcCalculationManager = new DcCorrectionManager(); @Override public short[] convert(ByteBuffer buffer) { - long dcAccumulator = 0; + boolean shouldCalculateDc = mDcCalculationManager.shouldCalculateDc(); + short sample; short[] samples; byte b1, b2; @@ -53,44 +57,73 @@ public class VectorUnpackedSampleConverter implements IAirspySampleConverter short[] bytes1 = new short[VECTOR_SPECIES.length()]; short[] bytes2 = new short[VECTOR_SPECIES.length()]; - for(; samplesOffset < VECTOR_SPECIES.loopBound(samples.length); samplesOffset += VECTOR_SPECIES.length()) + if(shouldCalculateDc) { - bytesOffset = 0; + long dcAccumulator = 0; - while(bytesOffset < VECTOR_SPECIES.length()) + for(; samplesOffset < VECTOR_SPECIES.loopBound(samples.length); samplesOffset += VECTOR_SPECIES.length()) { - bytes1[bytesOffset] = (short)(buffer.get(rawPointer++) & 0xFF); - bytes2[bytesOffset++] = (short)(buffer.get(rawPointer++) & 0x0F); + bytesOffset = 0; + + while(bytesOffset < VECTOR_SPECIES.length()) + { + bytes1[bytesOffset] = (short)(buffer.get(rawPointer++) & 0xFF); + bytes2[bytesOffset++] = (short)(buffer.get(rawPointer++) & 0x0F); + } + + vector = ShortVector.fromArray(VECTOR_SPECIES, bytes1, 0) + .or(ShortVector.fromArray(VECTOR_SPECIES, bytes2, 0).lanewise(VectorOperators.LSHL, 8)); + dcAccumulator += vector.reduceLanes(VectorOperators.ADD); + vector.intoArray(samples, samplesOffset); } - vector = ShortVector.fromArray(VECTOR_SPECIES, bytes1, 0) - .or(ShortVector.fromArray(VECTOR_SPECIES, bytes2, 0).lanewise(VectorOperators.LSHL, 8)); - dcAccumulator += vector.reduceLanes(VectorOperators.ADD); - vector.intoArray(samples, samplesOffset); - } + for(; samplesOffset < samples.length; samplesOffset++) + { + b1 = buffer.get(rawPointer++); + b2 = buffer.get(rawPointer++); - for(; samplesOffset < samples.length; samplesOffset++) + sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF)); + samples[samplesOffset] = sample; + dcAccumulator += sample; + } + + float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f; + averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT; + mDcCalculationManager.adjust(averageDcNow); + } + else { - b1 = buffer.get(rawPointer++); - b2 = buffer.get(rawPointer++); + for(; samplesOffset < VECTOR_SPECIES.loopBound(samples.length); samplesOffset += VECTOR_SPECIES.length()) + { + bytesOffset = 0; - sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF)); - samples[samplesOffset] = sample; - dcAccumulator += sample; + while(bytesOffset < VECTOR_SPECIES.length()) + { + bytes1[bytesOffset] = (short)(buffer.get(rawPointer++) & 0xFF); + bytes2[bytesOffset++] = (short)(buffer.get(rawPointer++) & 0x0F); + } + + vector = ShortVector.fromArray(VECTOR_SPECIES, bytes1, 0) + .or(ShortVector.fromArray(VECTOR_SPECIES, bytes2, 0).lanewise(VectorOperators.LSHL, 8)); + vector.intoArray(samples, samplesOffset); + } + + for(; samplesOffset < samples.length; samplesOffset++) + { + b1 = buffer.get(rawPointer++); + b2 = buffer.get(rawPointer++); + + sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF)); + samples[samplesOffset] = sample; + } } - //Calculate the average scaled DC offset so that it can be applied in the native buffer's converted samples - float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f; - averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT; - averageDcNow -= mAverageDc; - mAverageDc += (averageDcNow * DC_FILTER_GAIN); - return samples; } @Override public float getAverageDc() { - return mAverageDc; + return mDcCalculationManager.getAverageDc(); } }