#1291 Updates DC removal processing across tuner sample streams to run on a periodic timer to reduce overall processing requirements. Updates build version to 0.5.0 beta 5 (#1292)

Co-authored-by: Dennis Sheirer <dsheirer@github.com>

build.gradle

@@ -28,7 +28,7 @@ repositories {
     maven { url "https://jitpack.io" }
 }
 
-version = '0.5.0'
+version = '0.5.0-beta5'
 sourceCompatibility = '17'
 
 sourceSets {

src/main/java/io/github/dsheirer/buffer/ByteNativeBuffer.java

@@ -21,7 +21,6 @@ package io.github.dsheirer.buffer;
 
 import io.github.dsheirer.sample.complex.ComplexSamples;
 import io.github.dsheirer.sample.complex.InterleavedComplexSamples;
-
 import java.util.Iterator;
 
 /**
@@ -42,7 +41,7 @@ public class ByteNativeBuffer implements INativeBuffer
 
         for(int x = 0; x < 256; x++)
         {
-            LOOKUP_VALUES[x] = (float)(x - 127) / 128.0f;
+            LOOKUP_VALUES[x] = ((float)x - 127.5f) / 128.0f;
         }
     }
 

src/main/java/io/github/dsheirer/buffer/ByteNativeBufferFactory.java

@@ -26,16 +26,20 @@ import java.nio.ByteBuffer;
  */
 public class ByteNativeBufferFactory implements INativeBufferFactory
 {
-    private static final float DC_FILTER_GAIN = 0.007f; //Normalizes DC over period of ~142 (1 / .007) buffers
+    private DcCorrectionManager mDcCorrectionManager = new DcCorrectionManager();
-    private float mAverageDc = 0.0f;
 
     @Override
     public INativeBuffer getBuffer(ByteBuffer samples, long timestamp)
     {
         byte[] copy = new byte[samples.capacity()];
         samples.get(copy);
-        calculateDc(copy);
+
-        return new ByteNativeBuffer(copy, timestamp, mAverageDc);
+        if(mDcCorrectionManager.shouldCalculateDc())
+        {
+            calculateDc(copy);
+        }
+
+        return new ByteNativeBuffer(copy, timestamp, mDcCorrectionManager.getAverageDc());
     }
 
     /**
@@ -45,7 +49,7 @@ public class ByteNativeBufferFactory implements INativeBufferFactory
      */
     private void calculateDc(byte[] samples)
     {
-        double dcAccumulator = 0;
+        float dcAccumulator = 0;
 
         for(byte sample: samples)
         {
@@ -53,9 +57,8 @@ public class ByteNativeBufferFactory implements INativeBufferFactory
         }
 
         dcAccumulator /= samples.length;
-        dcAccumulator -= 127.0f;
+        dcAccumulator -= 127.5f;
         dcAccumulator /= 128.0f;
-        dcAccumulator -= mAverageDc;
+        mDcCorrectionManager.adjust(dcAccumulator);
-        mAverageDc += (dcAccumulator * DC_FILTER_GAIN);
     }
 }

src/main/java/io/github/dsheirer/buffer/DcCorrectionManager.java

@@ -0,0 +1,114 @@
+/*
+ * *****************************************************************************
+ * Copyright (C) 2014-2022 Dennis Sheirer
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ * ****************************************************************************
+ */
+
+package io.github.dsheirer.buffer;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Provides DC offset correction value and manages calculation processing interval.
+ */
+public class DcCorrectionManager
+{
+//    private static Logger mLog = LoggerFactory.getLogger(DcCorrectionManager.class);
+//    private DecimalFormat mDecimalFormat = new DecimalFormat("0.00000");
+
+    /**
+     * Time (delay) interval for running DC removal calculations
+     */
+    private static final long DC_PROCESSING_INTERVAL = TimeUnit.SECONDS.toMillis(50);
+
+    /**
+     * Number of buffers to process per DC calculation processing interval.
+     */
+    private static final int DC_CALCULATIONS_PER_INTERVAL = 5;
+
+    /**
+     * DC offset processing residual threshold for ending DC processing interval
+     */
+    private static final float TARGET_DC_OFFSET_REMAINING = 0.0001f;
+
+    /**
+     * Number of DC calculations remaining in the current interval
+     */
+    private int mDcCalculationsRemaining = DC_CALCULATIONS_PER_INTERVAL; //Initial value for coarse correction
+
+    /**
+     * Last time DC offset calculation interval ran
+     */
+    private long mLastDcCalculationTimestamp = 0;
+
+    /**
+     * DC offset calculation gain.
+     */
+    private static final float DC_FILTER_GAIN = 0.05f;
+
+    /**
+     * Current DC offset correction value
+     */
+    private float mAverageDc = 0.0f;
+
+    public float getAverageDc()
+    {
+        return mAverageDc;
+    }
+
+    /**
+     * Adjusts the average DC based on the current calculated DC offset present in a sample buffer.
+     * @param calculatedDc for the current sample buffer
+     */
+    public void adjust(float calculatedDc)
+    {
+        //Calculate the average scaled DC offset so that it can be applied in the native buffer's converted samples
+        float residualOffsetToRemove = calculatedDc - mAverageDc;
+
+        mAverageDc += (residualOffsetToRemove * DC_FILTER_GAIN);
+
+        if(Math.abs(residualOffsetToRemove) < TARGET_DC_OFFSET_REMAINING)
+        {
+            mDcCalculationsRemaining--;
+        }
+//
+//        mLog.info("DC: " + mDecimalFormat.format(mAverageDc) +
+//                " RES:" + mDecimalFormat.format(residualOffsetToRemove) +
+//                " REM:" + mDcCalculationsRemaining);
+    }
+
+    /**
+     * Indicates if a DC offset calculation for a buffer should be performed.
+     */
+    public boolean shouldCalculateDc()
+    {
+        if(System.currentTimeMillis() > (mLastDcCalculationTimestamp + DC_PROCESSING_INTERVAL))
+        {
+            if(mDcCalculationsRemaining > 0)
+            {
+                return true;
+            }
+            else
+            {
+                //Reset the number of buffers to calculate and the timestamp
+                mDcCalculationsRemaining = DC_CALCULATIONS_PER_INTERVAL;
+                mLastDcCalculationTimestamp = System.currentTimeMillis();
+            }
+        }
+
+        return false;
+    }
+}

src/main/java/io/github/dsheirer/buffer/SignedByteNativeBufferFactory.java

@@ -20,14 +20,51 @@
 package io.github.dsheirer.buffer;
 
 import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Implements a factory for creating SignedByteNativeBuffer instances
  */
 public class SignedByteNativeBufferFactory implements INativeBufferFactory
 {
-    private static final float DC_FILTER_GAIN = 0.02f; //Normalizes DC over period of ~50 (1 / .02) buffers
+    /**
+     * DC removal calculations will run once a minute
+     */
+    private static final long DC_PROCESSING_INTERVAL = TimeUnit.SECONDS.toMillis(60);
+
+    /**
+     * Number of buffers to process per DC calculation processing interval.
+     */
+    private static final int DC_CALCULATIONS_PER_INTERVAL = 5;
+
+    /**
+     * DC offset processing residual threshold for ending DC processing interval
+      */
+    private static final float TARGET_DC_OFFSET_REMAINING = 0.0002f;
+
+    /**
+     * Number of DC calculations remaining in the current interval
+     */
+    private int mDcCalculationsRemaining = DC_CALCULATIONS_PER_INTERVAL; //Initial value for coarse correction
+
+    /**
+     * Last time DC offset calculation interval ran
+     */
+    private long mLastDcCalculationTimestamp = 0;
+
+    /**
+     * DC offset calculation gain.
+     */
+    private static final float DC_FILTER_GAIN = 0.05f;
+
+    /**
+     * Current DC offset correction for Inphase samples
+     */
     private float mIAverageDc = 0.0f;
+
+    /**
+     * Current DC offset correction for Quadrature samples
+     */
     private float mQAverageDc = 0.0f;
 
     @Override
@@ -35,10 +72,37 @@ public class SignedByteNativeBufferFactory implements INativeBufferFactory
     {
         byte[] copy = new byte[samples.capacity()];
         samples.get(copy);
-        calculateDc(copy);
+
+        if(shouldCalculateDc())
+        {
+            calculateDc(copy);
+        }
+
         return new SignedByteNativeBuffer(copy, timestamp, mIAverageDc, mQAverageDc);
     }
 
+    /**
+     * Indicates if a DC offset calculation for a buffer should be performed.
+     */
+    private boolean shouldCalculateDc()
+    {
+        if(System.currentTimeMillis() > (mLastDcCalculationTimestamp + DC_PROCESSING_INTERVAL))
+        {
+            if(mDcCalculationsRemaining > 0)
+            {
+                return true;
+            }
+            else
+            {
+                //Reset the number of buffers to calculate and the timestamp
+                mDcCalculationsRemaining = DC_CALCULATIONS_PER_INTERVAL;
+                mLastDcCalculationTimestamp = System.currentTimeMillis();
+            }
+        }
+
+        return false;
+    }
+
     /**
      * Calculates the average DC in the sample stream so that it can be subtracted from the samples when the
      * native buffer is used.
@@ -46,8 +110,8 @@ public class SignedByteNativeBufferFactory implements INativeBufferFactory
      */
     private void calculateDc(byte[] samples)
     {
-        double iDcAccumulator = 0;
+        float iDcAccumulator = 0;
-        double qDcAccumulator = 0;
+        float qDcAccumulator = 0;
 
         for(int x = 0; x < samples.length; x += 2)
         {
@@ -64,5 +128,10 @@ public class SignedByteNativeBufferFactory implements INativeBufferFactory
         qDcAccumulator /= 128.0f;
         qDcAccumulator -= mQAverageDc;
         mQAverageDc += (qDcAccumulator * DC_FILTER_GAIN);
+
+        if(Math.abs(iDcAccumulator) < TARGET_DC_OFFSET_REMAINING && Math.abs(qDcAccumulator) < TARGET_DC_OFFSET_REMAINING)
+        {
+            mDcCalculationsRemaining--;
+        }
     }
 }

src/main/java/io/github/dsheirer/buffer/airspy/ScalarPackedSampleConverter.java

@@ -19,6 +19,7 @@
 
 package io.github.dsheirer.buffer.airspy;
 
+import io.github.dsheirer.buffer.DcCorrectionManager;
 import java.nio.ByteBuffer;
 
 /**
@@ -26,40 +27,61 @@ import java.nio.ByteBuffer;
  */
 public class ScalarPackedSampleConverter implements IAirspySampleConverter
 {
-    private static final float DC_FILTER_GAIN = 0.007f; //Normalizes DC over period of ~142 (1 / .007) buffers
+    /**
-    private float mAverageDc = 0.0f;
+     * Manages DC calculations and processing interval
+     */
+    private DcCorrectionManager mDcCalculationManager = new DcCorrectionManager();
 
     @Override
     public short[] convert(ByteBuffer buffer)
     {
+        boolean shouldCalculateDc = mDcCalculationManager.shouldCalculateDc();
+
         int offset = 0;
-        long dcAccumulator = 0;
         short sample;
         short[] samples;
         byte b1, b2, b3;
 
         samples = new short[buffer.capacity() / 3 * 2];
 
-        for(int x = 0; x < samples.length; x += 2)
+        if(shouldCalculateDc)
         {
-            b1 = buffer.get(offset++);
+            long dcAccumulator = 0;
-            b2 = buffer.get(offset++);
-            b3 = buffer.get(offset++);
 
-            sample = (short)(((b1 & 0xFF) << 4) | ((b2 & 0xF0) >> 4));
+            for(int x = 0; x < samples.length; x += 2)
-            samples[x] = sample;
+            {
-            dcAccumulator += sample;
+                b1 = buffer.get(offset++);
+                b2 = buffer.get(offset++);
+                b3 = buffer.get(offset++);
 
-            sample = (short)(((b2 & 0xF) << 8) | (b3 & 0xFF));
+                sample = (short)(((b1 & 0xFF) << 4) | ((b2 & 0xF0) >> 4));
-            samples[x] = sample;
+                samples[x] = sample;
-            dcAccumulator += sample;
+                dcAccumulator += sample;
+
+                sample = (short)(((b2 & 0xF) << 8) | (b3 & 0xFF));
+                samples[x] = sample;
+                dcAccumulator += sample;
+            }
+
+            float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f;
+            averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT;
+            mDcCalculationManager.adjust(averageDcNow);
         }
+        else
+        {
+            for(int x = 0; x < samples.length; x += 2)
+            {
+                b1 = buffer.get(offset++);
+                b2 = buffer.get(offset++);
+                b3 = buffer.get(offset++);
 
-        //Calculate the average scaled DC offset so that it can be applied in the native buffer's converted samples
+                sample = (short)(((b1 & 0xFF) << 4) | ((b2 & 0xF0) >> 4));
-        float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f;
+                samples[x] = sample;
-        averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT;
+
-        averageDcNow -= mAverageDc;
+                sample = (short)(((b2 & 0xF) << 8) | (b3 & 0xFF));
-        mAverageDc += (averageDcNow * DC_FILTER_GAIN);
+                samples[x] = sample;
+            }
+        }
 
         return samples;
     }
@@ -67,7 +89,7 @@ public class ScalarPackedSampleConverter implements IAirspySampleConverter
     @Override
     public float getAverageDc()
     {
-        return mAverageDc;
+        return mDcCalculationManager.getAverageDc();
     }
 
     public static void main(String[] args)

src/main/java/io/github/dsheirer/buffer/airspy/ScalarUnpackedSampleConverter.java

@@ -19,6 +19,7 @@
 
 package io.github.dsheirer.buffer.airspy;
 
+import io.github.dsheirer.buffer.DcCorrectionManager;
 import java.nio.ByteBuffer;
 
 /**
@@ -26,34 +27,50 @@ import java.nio.ByteBuffer;
  */
 public class ScalarUnpackedSampleConverter implements IAirspySampleConverter
 {
-    private static final float DC_FILTER_GAIN = 0.007f; //Normalizes DC over period of ~142 (1 / .007) buffers
+    /**
-    private float mAverageDc = 0.0f;
+     * Manages DC calculations and processing interval
+     */
+    private DcCorrectionManager mDcCalculationManager = new DcCorrectionManager();
 
     @Override
     public short[] convert(ByteBuffer buffer)
     {
+        boolean shouldCalculateDc = mDcCalculationManager.shouldCalculateDc();
+
         int offset = 0;
-        long dcAccumulator = 0;
         short sample;
         short[] samples;
         byte b1, b2;
 
         samples = new short[buffer.capacity() / 2];
 
-        for(int x = 0; x < samples.length; x++)
+        if(shouldCalculateDc)
         {
-            b1 = buffer.get(offset++);
+            long dcAccumulator = 0;
-            b2 = buffer.get(offset++);
-            sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF));
-            samples[x] = sample;
-            dcAccumulator += sample;
-        }
 
-        //Calculate the average scaled DC offset so that it can be applied in the native buffer's converted samples
+            for(int x = 0; x < samples.length; x++)
-        float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f;
+            {
-        averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT;
+                b1 = buffer.get(offset++);
-        averageDcNow -= mAverageDc;
+                b2 = buffer.get(offset++);
-        mAverageDc += (averageDcNow * DC_FILTER_GAIN);
+                sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF));
+                samples[x] = sample;
+                dcAccumulator += sample;
+            }
+
+            float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f;
+            averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT;
+            mDcCalculationManager.adjust(averageDcNow);
+        }
+        else
+        {
+            for(int x = 0; x < samples.length; x++)
+            {
+                b1 = buffer.get(offset++);
+                b2 = buffer.get(offset++);
+                sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF));
+                samples[x] = sample;
+            }
+        }
 
         return samples;
     }
@@ -61,6 +78,6 @@ public class ScalarUnpackedSampleConverter implements IAirspySampleConverter
     @Override
     public float getAverageDc()
     {
-        return mAverageDc;
+        return mDcCalculationManager.getAverageDc();
     }
 }

src/main/java/io/github/dsheirer/buffer/airspy/VectorUnpackedSampleConverter.java

@@ -19,25 +19,29 @@
 
 package io.github.dsheirer.buffer.airspy;
 
+import io.github.dsheirer.buffer.DcCorrectionManager;
+import java.nio.ByteBuffer;
 import jdk.incubator.vector.ShortVector;
 import jdk.incubator.vector.VectorOperators;
 import jdk.incubator.vector.VectorSpecies;
 
-import java.nio.ByteBuffer;
-
 /**
  * Scalar implementation of airspy sample converter for un-packed samples.
  */
 public class VectorUnpackedSampleConverter implements IAirspySampleConverter
 {
     private static final VectorSpecies<Short> VECTOR_SPECIES = ShortVector.SPECIES_PREFERRED;
-    private static final float DC_FILTER_GAIN = 0.007f; //Normalizes DC over period of ~142 (1 / .007) buffers
+
-    private float mAverageDc = 0.0f;
+    /**
+     * Manages DC calculations and processing interval
+     */
+    private DcCorrectionManager mDcCalculationManager = new DcCorrectionManager();
 
     @Override
     public short[] convert(ByteBuffer buffer)
     {
-        long dcAccumulator = 0;
+        boolean shouldCalculateDc = mDcCalculationManager.shouldCalculateDc();
+
         short sample;
         short[] samples;
         byte b1, b2;
@@ -53,44 +57,73 @@ public class VectorUnpackedSampleConverter implements IAirspySampleConverter
         short[] bytes1 = new short[VECTOR_SPECIES.length()];
         short[] bytes2 = new short[VECTOR_SPECIES.length()];
 
-        for(; samplesOffset < VECTOR_SPECIES.loopBound(samples.length); samplesOffset += VECTOR_SPECIES.length())
+        if(shouldCalculateDc)
         {
-            bytesOffset = 0;
+            long dcAccumulator = 0;
 
-            while(bytesOffset < VECTOR_SPECIES.length())
+            for(; samplesOffset < VECTOR_SPECIES.loopBound(samples.length); samplesOffset += VECTOR_SPECIES.length())
             {
-                bytes1[bytesOffset] = (short)(buffer.get(rawPointer++) & 0xFF);
+                bytesOffset = 0;
-                bytes2[bytesOffset++] = (short)(buffer.get(rawPointer++) & 0x0F);
+
+                while(bytesOffset < VECTOR_SPECIES.length())
+                {
+                    bytes1[bytesOffset] = (short)(buffer.get(rawPointer++) & 0xFF);
+                    bytes2[bytesOffset++] = (short)(buffer.get(rawPointer++) & 0x0F);
+                }
+
+                vector = ShortVector.fromArray(VECTOR_SPECIES, bytes1, 0)
+                        .or(ShortVector.fromArray(VECTOR_SPECIES, bytes2, 0).lanewise(VectorOperators.LSHL, 8));
+                dcAccumulator += vector.reduceLanes(VectorOperators.ADD);
+                vector.intoArray(samples, samplesOffset);
             }
 
-            vector = ShortVector.fromArray(VECTOR_SPECIES, bytes1, 0)
+            for(; samplesOffset < samples.length; samplesOffset++)
-                    .or(ShortVector.fromArray(VECTOR_SPECIES, bytes2, 0).lanewise(VectorOperators.LSHL, 8));
+            {
-            dcAccumulator += vector.reduceLanes(VectorOperators.ADD);
+                b1 = buffer.get(rawPointer++);
-            vector.intoArray(samples, samplesOffset);
+                b2 = buffer.get(rawPointer++);
-        }
 
-        for(; samplesOffset < samples.length; samplesOffset++)
+                sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF));
+                samples[samplesOffset] = sample;
+                dcAccumulator += sample;
+            }
+
+            float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f;
+            averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT;
+            mDcCalculationManager.adjust(averageDcNow);
+        }
+        else
         {
-            b1 = buffer.get(rawPointer++);
+            for(; samplesOffset < VECTOR_SPECIES.loopBound(samples.length); samplesOffset += VECTOR_SPECIES.length())
-            b2 = buffer.get(rawPointer++);
+            {
+                bytesOffset = 0;
 
-            sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF));
+                while(bytesOffset < VECTOR_SPECIES.length())
-            samples[samplesOffset] = sample;
+                {
-            dcAccumulator += sample;
+                    bytes1[bytesOffset] = (short)(buffer.get(rawPointer++) & 0xFF);
+                    bytes2[bytesOffset++] = (short)(buffer.get(rawPointer++) & 0x0F);
+                }
+
+                vector = ShortVector.fromArray(VECTOR_SPECIES, bytes1, 0)
+                        .or(ShortVector.fromArray(VECTOR_SPECIES, bytes2, 0).lanewise(VectorOperators.LSHL, 8));
+                vector.intoArray(samples, samplesOffset);
+            }
+
+            for(; samplesOffset < samples.length; samplesOffset++)
+            {
+                b1 = buffer.get(rawPointer++);
+                b2 = buffer.get(rawPointer++);
+
+                sample = (short)(((b2 & 0x0F) << 8) | (b1 & 0xFF));
+                samples[samplesOffset] = sample;
+            }
         }
 
-        //Calculate the average scaled DC offset so that it can be applied in the native buffer's converted samples
-        float averageDcNow = ((float)dcAccumulator / (float)samples.length) - 2048.0f;
-        averageDcNow *= AirspyBufferIterator.SCALE_SIGNED_12_BIT_TO_FLOAT;
-        averageDcNow -= mAverageDc;
-        mAverageDc += (averageDcNow * DC_FILTER_GAIN);
-
         return samples;
     }
 
     @Override
     public float getAverageDc()
     {
-        return mAverageDc;
+        return mDcCalculationManager.getAverageDc();
     }
 }