GitHub/sdrtrunk
1
0
Fork 0
mirror of https://github.com/DSheirer/sdrtrunk.git synced 2026-11-11 08:07:24 -05:00
Code Issues Packages Projects Releases 2 Wiki Activity

Handle streaming audio as frames (#1244)

Browse Source
This commit is contained in:
Doug Hoffman
2022-12-18 04:46:07 -05:00
committed by GitHub
parent 4a68cd9407
commit bbd907514e
8 changed files with 389 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
src/main/java/io/github/dsheirer/audio/broadcast/AudioStreamingBroadcaster.java
View File

@@ -18,8 +18,10 @@
*/
package io.github.dsheirer.audio.broadcast;
import io.github.dsheirer.audio.convert.AudioFrames;
import io.github.dsheirer.audio.convert.ISilenceGenerator;
import io.github.dsheirer.audio.convert.InputAudioFormat;
import io.github.dsheirer.audio.convert.MP3FrameTools;
import io.github.dsheirer.audio.convert.MP3Setting;
import io.github.dsheirer.identifier.IdentifierCollection;
import io.github.dsheirer.util.ThreadPool;
@@ -48,6 +50,7 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
private Queue<AudioRecording> mAudioRecordingQueue = new LinkedTransferQueue<>();
private ISilenceGenerator mSilenceGenerator;
private BroadcastFormat mBroadcastFormat;
private long mDelay;
private long mMaximumRecordingAge;
private AtomicBoolean mStreaming = new AtomicBoolean();
@@ -56,6 +59,8 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
protected int mInlineInterval;
protected int mInlineRemaining = -1;
private int mTimeOverrun = 0;
/**
* AudioBroadcaster for streaming audio recordings to a remote streaming audio server. Audio recordings are
* generated by an internal StreamManager that converts an inbound stream of AudioPackets into a recording of the
@@ -80,10 +85,11 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
public AudioStreamingBroadcaster(T broadcastConfiguration, InputAudioFormat inputAudioFormat, MP3Setting mp3Setting)
{
super(broadcastConfiguration);
mBroadcastFormat = broadcastConfiguration.getBroadcastFormat();
mDelay = getBroadcastConfiguration().getDelay();
mMaximumRecordingAge = getBroadcastConfiguration().getMaximumRecordingAge();
mSilenceGenerator = BroadcastFactory.getSilenceGenerator(broadcastConfiguration.getBroadcastFormat(),
inputAudioFormat, mp3Setting);
inputAudioFormat, mp3Setting);
}
public void dispose()
@@ -270,10 +276,8 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
public class RecordingQueueProcessor implements Runnable
{
private AtomicBoolean mProcessing = new AtomicBoolean();
private ByteArrayInputStream mInputStream;
private AudioFrames mInputFrames;
private IdentifierCollection mInputIdentifierCollection;
private long mFinalSilencePadding = 0;
private int mBytesToStreamPerInterval = 0;
@Override
public void run()
@@ -282,48 +286,35 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
{
try
{
if(mInputStream == null || mInputStream.available() <= 0)
int timeSent = 0;
if(mInputFrames == null || !mInputFrames.hasNextFrame())
{
nextRecording();
}
if(mInputStream != null)
if(mInputFrames != null && mInputFrames.hasNextFrame())
{
int length = FastMath.min(mBytesToStreamPerInterval, mInputStream.available());
byte[] audio = new byte[length];
try
while(mInputFrames.hasNextFrame() && timeSent < PROCESSOR_RUN_INTERVAL_MS)
{
int read = mInputStream.read(audio);
broadcastAudio(audio, mInputIdentifierCollection);
}
catch(IOException ioe)
{
mLog.error("Error reading from in-memory audio recording input stream", ioe);
}
//If this is the final frame fragment then append silence padding to fill the final interval segment
if(length < mBytesToStreamPerInterval && mFinalSilencePadding > 0)
{
List<byte[]> finalSilence = mSilenceGenerator.generate(mFinalSilencePadding);
for(byte[] silence: finalSilence)
{
broadcastAudio(silence, null);
}
mFinalSilencePadding = 0;
mInputFrames.nextFrame();
broadcastAudio(mInputFrames.getCurrentFrame(), mInputIdentifierCollection);
timeSent += mInputFrames.getCurrentFrameDuration();
}
}
else
if((mInputFrames == null || !mInputFrames.hasNextFrame()) && timeSent < PROCESSOR_RUN_INTERVAL_MS)
{
List<byte[]> silenceFrames = mSilenceGenerator.generate(PROCESSOR_RUN_INTERVAL_MS);
for(byte[] silence: silenceFrames)
AudioFrames silenceFrames = mSilenceGenerator.generate(PROCESSOR_RUN_INTERVAL_MS - mTimeOverrun - timeSent);
while(silenceFrames.hasNextFrame())
{
broadcastAudio(silence, null);
silenceFrames.nextFrame();
broadcastAudio(silenceFrames.getCurrentFrame(), null);
timeSent += silenceFrames.getCurrentFrameDuration();
}
}
mTimeOverrun += timeSent - PROCESSOR_RUN_INTERVAL_MS;
}
catch(Throwable t)
{
@@ -341,7 +332,7 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
{
boolean metadataUpdateRequired = false;
if(mInputStream != null)
if(mInputFrames != null)
{
mStreamedAudioCount++;
broadcast(new BroadcastEvent(AudioStreamingBroadcaster.this,
@@ -349,7 +340,7 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
metadataUpdateRequired = true;
}
mInputStream = null;
mInputFrames = null;
mInputIdentifierCollection = null;
//Peek at the next recording but don't remove it from the queue yet, so we can inspect the start time for
@@ -380,20 +371,15 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
if(audio.length > 0)
{
//Calculate how many bytes we'll send with each processor run interval
double intervals = (double)nextRecording.getRecordingLength() / (double)PROCESSOR_RUN_INTERVAL_MS;
mBytesToStreamPerInterval = (int)((double)audio.length / intervals);
mInputStream = new ByteArrayInputStream(audio);
mInputIdentifierCollection = nextRecording.getIdentifierCollection();
mFinalSilencePadding = PROCESSOR_RUN_INTERVAL_MS -
(nextRecording.getRecordingLength() % PROCESSOR_RUN_INTERVAL_MS);
while(mFinalSilencePadding >= PROCESSOR_RUN_INTERVAL_MS)
switch(mBroadcastFormat)
{
mFinalSilencePadding -= PROCESSOR_RUN_INTERVAL_MS;
case MP3:
mInputFrames = MP3FrameTools.split(audio);
break;
default:
throw new IllegalArgumentException("Unsupported broadcast format [" + mBroadcastFormat + "]");
}
mInputIdentifierCollection = nextRecording.getIdentifierCollection();
if(connected())
{
@@ -409,7 +395,7 @@ public abstract class AudioStreamingBroadcaster<T extends BroadcastConfiguration
mLog.error("Stream [" + getBroadcastConfiguration().getName() + "] error reading temporary audio " +
"stream recording [" + nextRecording.getPath().toString() + "] - skipping recording - ", ioe);
mInputStream = null;
mInputFrames = null;
mInputIdentifierCollection = null;
metadataUpdateRequired = false;
}

133
src/main/java/io/github/dsheirer/audio/convert/AudioFrames.java Normal file
View File

@@ -0,0 +1,133 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2022 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.audio.convert;
import java.util.List;
import org.apache.commons.math3.util.FastMath;
public abstract class AudioFrames
{
protected final int audioDuration;
protected final List<byte[]> audioFrames;
protected int mCurrentFrame = -1;
/**
* Create new AudioFrame
* @param audioDuration total duration in milliseconds
* @param audioFrames series of audio frames, split on frame boundaries
*/
public AudioFrames(int audioDuration, List<byte[]> audioFrames)
{
this.audioDuration = audioDuration;
this.audioFrames = audioFrames;
}
/**
* Get total duration of frames
* @return total duration in milliseconds
*/
public int getDuration()
{
return audioDuration;
}
/**
* Get all frames
* @return list of byte arrays, with each element representing a single frame of audio
*/
public List<byte[]> getFrames()
{
return audioFrames;
}
/**
* Get current frame
* @return byte array with the current frame of audio
*/
public byte[] getCurrentFrame()
{
return audioFrames.get(mCurrentFrame);
}
/**
* Get current frame duration
* @return frame duration in milliseconds
*/
public abstract int getCurrentFrameDuration();
/**
* Indicates if there is a frame before the current one
* @return true if the frame exists, otherwise false
*/
public boolean hasPrevFrame()
{
int prevFrame = mCurrentFrame - 1;
return prevFrame >= 0 && prevFrame < audioFrames.size();
}
/**
* Indicates if there is a frame after the current one
* @return true if the frame exists, otherwise false
*/
public boolean hasNextFrame()
{
int nextFrame = mCurrentFrame + 1;
return nextFrame >= 0 && nextFrame < audioFrames.size();
}
/**
* Seek to the previous frame
*/
public void prevFrame()
{
mCurrentFrame -= 1;
}
/**
* Seek to the next frame
*/
public void nextFrame()
{
mCurrentFrame += 1;
}
/**
* Restart from beginning
* Must call nextFrame() to get first frame
*/
public void restart()
{
mCurrentFrame = -1;
}
/**
* Seek by the specified time
* @param duration_ms time in milliseconds, positive or negative
*/
public void seek(int duration_ms)
{
int step = duration_ms < 0 ? -1 : 1;
int actual_ms = 0;
while(FastMath.abs(actual_ms) < FastMath.abs(duration_ms))
{
mCurrentFrame += step;
actual_ms += getCurrentFrameDuration();
}
}
}

2
src/main/java/io/github/dsheirer/audio/convert/ISilenceGenerator.java
View File

@@ -22,5 +22,5 @@ import java.util.List;
public interface ISilenceGenerator
{
List<byte[]> generate(long duration);
AudioFrames generate(long duration);
}

46
src/main/java/io/github/dsheirer/audio/convert/MP3AudioFrames.java Normal file
View File

@@ -0,0 +1,46 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2022 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.audio.convert;
import java.util.List;
public class MP3AudioFrames extends AudioFrames
{
/**
* Create new MP3AudioFrame
* @param audioDuration total duration in milliseconds
* @param audioFrames series of audio frames, split on frame boundaries
*/
public MP3AudioFrames(int audioDuration, List<byte[]> audioFrames)
{
super(audioDuration, audioFrames);
}
/**
* Get current frame duration
* @return frame duration in milliseconds
*/
public int getCurrentFrameDuration()
{
return MP3Header.getFrameDuration(getCurrentFrame(), 0);
}
}

2
src/main/java/io/github/dsheirer/audio/convert/MP3FrameInspector.java
View File

@@ -55,6 +55,7 @@ public class MP3FrameInspector
}
}
/* TODO: Rework for MP3SilenceGenerator returning IAudioFrames
public static void main(String[] args)
{
mLog.info("Starting ...");
@@ -66,4 +67,5 @@ public class MP3FrameInspector
mLog.info("Finished");
}
*/
}

59
src/main/java/io/github/dsheirer/audio/convert/MP3FrameTools.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2022 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.audio.convert;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.math3.util.FastMath;
public class MP3FrameTools {
private MP3FrameTools() {}
/**
* Split MP3 audio at frame boundaries
* @param input byte array of audio
* @return list of byte arrays containing audio frames
*/
public static MP3AudioFrames split(byte[] input)
{
List<byte[]> mp3Frames = new ArrayList<>();
int audioDuration = 0;
int offset = 0;
while(offset < input.length)
{
if(MP3Header.hasSync(input, offset))
{
int framelen = MP3Header.getFrameLength(input, offset);
mp3Frames.add(Arrays.copyOfRange(input, offset, offset + FastMath.min(framelen, input.length - offset)));
audioDuration += MP3Header.getFrameDuration(input, offset);
offset += framelen;
}
else
{
offset += 1;
}
}
return new MP3AudioFrames(audioDuration, mp3Frames);
}
}

69
src/main/java/io/github/dsheirer/audio/convert/MP3Header.java
View File

@@ -19,6 +19,10 @@
package io.github.dsheirer.audio.convert;
import io.github.dsheirer.audio.convert.MPEGLayer;
import io.github.dsheirer.audio.convert.MPEGVersion;
import org.apache.commons.math3.util.FastMath;
/**
* Utility class for inspecting/parsing MP3 frames
*/
@@ -88,6 +92,71 @@ public class MP3Header
return ChannelMode.fromValue((frame[offset + 3] & 0xC0) >> 6);
}
/**
* Indicates if the frame has padding
* @param frame
* @param offset
* @return boolean
*/
public static boolean isPadded(byte[] frame, int offset)
{
return (frame[offset + 2] & 0x01) == 1;
}
/**
* Get the number of bytes in the frame
* @param frame
* @param offset
* @return frame length in bytes
*/
public static int getFrameLength(byte[] frame, int offset)
{
return (getFrameSamples(frame, offset) / 8) * (getBitRate(frame, offset) * 1000) / getSampleRate(frame, offset) + (isPadded(frame, offset) ? 1 : 0);
}
/**
* Get the number of samples in the frame
* @param frame
* @param offset
* @return length of frame in samples
*/
public static int getFrameSamples(byte[] frame, int offset)
{
switch(getMPEGVersion(frame, offset))
{
case V_1:
switch(getMPEGLayer(frame, offset))
{
case LAYER1: return 384;
case LAYER2: return 1152;
case LAYER3: return 1152;
default:
}
case V_2:
case V_2_5:
switch(getMPEGLayer(frame, offset))
{
case LAYER1: return 384;
case LAYER2: return 1152;
case LAYER3: return 576;
default:
}
default:
}
return 0;
}
/**
* Get the frame duration in milliseconds
* @param frame
* @param offset
* @return duration in milliseconds
*/
public static int getFrameDuration(byte[] frame, int offset)
{
return (int)FastMath.round((double)getFrameSamples(frame, offset) / (double)getSampleRate(frame, offset) * 1000);
}
/**
* Inspects the frame bytes and provides details about the header.
*/

54
src/main/java/io/github/dsheirer/audio/convert/MP3SilenceGenerator.java
View File

@@ -19,19 +19,20 @@
package io.github.dsheirer.audio.convert;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.math3.util.FastMath;
public class MP3SilenceGenerator implements ISilenceGenerator
{
private final static Logger mLog = LoggerFactory.getLogger(MP3SilenceGenerator.class);
public static final int MP3_BIT_RATE = 16;
public static final boolean CONSTANT_BIT_RATE = false;
private MP3AudioConverter mMP3AudioConverter;
private InputAudioFormat mInputAudioFormat;
private byte[] mPreviousPartialFrameData;
private byte[] mSilenceFrame;
private int mSilenceFrameDuration;
/**
* Generates MP3 audio silence frames
@@ -40,19 +41,17 @@ public class MP3SilenceGenerator implements ISilenceGenerator
{
mInputAudioFormat = inputAudioFormat;
mMP3AudioConverter = new MP3AudioConverter(inputAudioFormat, setting, false);
generate_frame();
}
/**
* Generates silence frames
* @param duration_ms in milliseconds
* Generates a single silence frame
* @return
*/
public List<byte[]> generate(long duration_ms)
private void generate_frame()
{
double duration_secs = (double)duration_ms / 1000.0;
//We generate silence at 8000 kHz, because it gets resampled by the silence generator to the target rate
int length = (int)(duration_secs * InputAudioFormat.SR_8000.getSampleRate());
int length = (int)(InputAudioFormat.SR_8000.getSampleRate());
List<float[]> silenceBuffers = new ArrayList<>();
@@ -64,7 +63,40 @@ public class MP3SilenceGenerator implements ISilenceGenerator
added += chunk;
}
return mMP3AudioConverter.convert(silenceBuffers);
List<byte[]> mp3Audio = mMP3AudioConverter.convert(silenceBuffers);
byte[] silence = new byte[0];
for(byte[] mp3Chunk: mp3Audio)
{
silence = merge(silence, mp3Chunk);
}
MP3AudioFrames mp3Frames = MP3FrameTools.split(silence);
if(mp3Frames.hasNextFrame())
{
mp3Frames.nextFrame();
mSilenceFrame = mp3Frames.getCurrentFrame();
mSilenceFrameDuration = mp3Frames.getCurrentFrameDuration();
}
}
/**
* Generates silence frames
* @param duration_ms in milliseconds
* @return
*/
public MP3AudioFrames generate(long duration_ms)
{
List<byte[]> silenceFrames = new ArrayList<>();
int silenceDuration = 0;
while(silenceDuration < duration_ms)
{
silenceFrames.add(mSilenceFrame);
silenceDuration += mSilenceFrameDuration;
}
return new MP3AudioFrames(silenceDuration, silenceFrames);
}
private static byte[] merge(byte[] a, byte[] b)
@@ -89,6 +121,7 @@ public class MP3SilenceGenerator implements ISilenceGenerator
return c;
}
/* TODO: Rework for MP3SilenceGenerator returning IAudioFrames
public static void main(String[] args)
{
mLog.debug("Starting ...");
@@ -113,4 +146,5 @@ public class MP3SilenceGenerator implements ISilenceGenerator
mLog.debug("Finished");
}
*/
}