#1761 DMR decoder supports Hytera Tier 3 UTF-16 Talker Alias & new Traffic Channel Talker Status CSBKO 47. Implements a tracking feature to detect when vendors are playing games using non-standard CSBK CRC mask values by tracking when a non-standard mask value is consistently detected and applying that mask when the initial CRC check fails.

This commit is contained in:
Dennis Sheirer
2023-12-14 07:10:30 -05:00
parent b6c6505824
commit ec9ceeefb3
13 changed files with 400 additions and 32 deletions

View File

@@ -20,6 +20,7 @@ package io.github.dsheirer.bits;
import io.github.dsheirer.edac.CRC;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.BitSet;
import org.apache.commons.lang3.Validate;
import org.apache.commons.math3.util.FastMath;
@@ -1278,7 +1279,7 @@ public class BinaryMessage extends BitSet
bytes[x] = getByte(CHARACTER_8_BIT, x * 8 + offset);
}
return new String(bytes);
return new String(bytes, Charset.forName("UTF-16"));
}
/**

View File

@@ -1,18 +1,21 @@
/*******************************************************************************
* sdr-trunk
* Copyright (C) 2014-2018 Dennis Sheirer
/*
* *****************************************************************************
* Copyright (C) 2014-2023 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
* later version.
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program.
* If not, see <http://www.gnu.org/licenses/>
*
******************************************************************************/
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.edac;
import io.github.dsheirer.bits.BinaryMessage;
@@ -177,6 +180,28 @@ public class CRCDMR
return 2;
}
/**
* Calculates the residual CRC value using a mask of zero which should identify the mask value when an alternate
* mask value is employed.
* @param message to be checked
* @param messageStart location
* @param crcStart location
* @return residual CRC check value.
*/
public static int calculateResidual(CorrectedBinaryMessage message, int messageStart, int crcStart)
{
int calculated = 0; //Starting value
/* Iterate the set bits and XOR running checksum with lookup value */
for(int i = message.nextSetBit(messageStart); i >= messageStart && i < crcStart; i = message.nextSetBit(i + 1))
{
calculated ^= CCITT_80_CHECKSUMS[i - messageStart];
}
int checksum = getIntChecksum(message, crcStart, 16);
return calculated ^ checksum;
}
/**
* Error detection for CRC-9 protected Confirmed Packet Data blocks. These
* data blocks have a slightly complicated structure because the checksum

View File

@@ -42,8 +42,11 @@ import io.github.dsheirer.sample.complex.IComplexSamplesListener;
import io.github.dsheirer.source.ISourceEventListener;
import io.github.dsheirer.source.ISourceEventProvider;
import io.github.dsheirer.source.SourceEvent;
import io.github.dsheirer.source.wave.ComplexWaveSource;
import java.io.File;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -332,4 +335,38 @@ public class DMRDecoder extends FeedbackDecoder implements ISourceEventListener,
{
return DMRDecoder.this;
}
public static void main(String[] args)
{
File file = new File("/home/denny/Downloads/TIII,HYT414.585000,.wav");
try
{
ComplexWaveSource source = new ComplexWaveSource(file);
source.open();
System.out.println("Source Sample Rate: " + source.getSampleRate());
DMRDecoder decoder = new DMRDecoder(new DecodeConfigDMR());
decoder.setMessageListener(message -> System.out.println("TS:" + message.getTimeslot() + " " + message));
source.setListener(iNativeBuffer -> {
Iterator<ComplexSamples> it = iNativeBuffer.iterator();
while(it.hasNext())
{
ComplexSamples samples = it.next();
decoder.receive(samples);
}
});
decoder.setSampleRate(source.getSampleRate());
decoder.start();
source.start();
while(true)
{
source.next(65535);
}
}
catch(Exception e)
{
e.printStackTrace();
}
}
}

View File

@@ -42,6 +42,7 @@ import io.github.dsheirer.module.decode.dmr.identifier.DMRTalkgroup;
import io.github.dsheirer.module.decode.dmr.message.DMRMessage;
import io.github.dsheirer.module.decode.dmr.message.data.DataMessage;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.CSBKMessage;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraTrafficChannelTalkerStatus;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.motorola.CapacityMaxAloha;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.motorola.CapacityPlusNeighbors;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.motorola.CapacityPlusSiteStatus;
@@ -708,7 +709,22 @@ public class DMRDecoderState extends TimeslotDecoderState
case HYTERA_08_ANNOUNCEMENT:
case HYTERA_68_ANNOUNCEMENT:
case HYTERA_68_XPT_SITE_STATE:
break;
case HYTERA_08_TRAFFIC_CHANNEL_TALKER_STATUS:
if(csbk instanceof HyteraTrafficChannelTalkerStatus status)
{
if(status.isChannelActive())
{
getIdentifierCollection().update(status.getIdentifiers());
updateCurrentCall(DecodeEventType.CALL_GROUP, "HYTERA TIER 3 CALL", status.getTimestamp());
}
else
{
getIdentifierCollection().remove(Role.FROM);
getIdentifierCollection().update(status.getDestinationRadio());
}
}
break;
case MOTOROLA_CAPPLUS_NEIGHBOR_REPORT:
if(csbk instanceof CapacityPlusNeighbors)
{

View File

@@ -27,6 +27,7 @@ import io.github.dsheirer.module.decode.dmr.message.CACH;
import io.github.dsheirer.module.decode.dmr.message.DMRBurst;
import io.github.dsheirer.module.decode.dmr.message.data.IDLEMessage;
import io.github.dsheirer.module.decode.dmr.message.data.block.DataBlock;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.CSBKMessage;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.standard.Aloha;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.standard.Preamble;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.standard.announcement.Announcement;
@@ -71,6 +72,7 @@ public class DMRMessageProcessor implements Listener<IMessage>
private TalkerAliasAssembler mTalkerAliasAssembler = new TalkerAliasAssembler();
private Listener<IMessage> mMessageListener;
private Map<Integer,TimeslotFrequency> mTimeslotFrequencyMap = new TreeMap<>();
private DmrCrcMaskManager mCrcMaskManager = new DmrCrcMaskManager();
/**
* Constructs an instance
@@ -93,6 +95,17 @@ public class DMRMessageProcessor implements Listener<IMessage>
@Override
public void receive(IMessage message)
{
if(message == null)
{
return;
}
//Detect and correct messages employing an alternate CRC mask pattern.
if(!message.isValid() && message instanceof CSBKMessage csbk)
{
mCrcMaskManager.check(csbk);
}
if(message instanceof FullLCMessage flc)
{
if(flc.getTimeslot() == 1)

View File

@@ -0,0 +1,232 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2023 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.module.decode.dmr;
import io.github.dsheirer.edac.CRCDMR;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.CSBKMessage;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Manages alternate CRC masks that may be employed in a system.
*/
public class DmrCrcMaskManager
{
private Logger LOGGER = LoggerFactory.getLogger(DmrCrcMaskManager.class);
private Map<Integer, ResidualCrcMaskTracker> mCsbkResidualTrackerMap = new TreeMap<>();
private int mDominantMask = 0;
/**
* Constructs an instance
*/
public DmrCrcMaskManager()
{
}
/**
* Creates a deep copy (ie clone) of all tracked values
* @return list of tracker clones.
*/
public List<ResidualCrcMaskTracker> cloneTrackers()
{
List<ResidualCrcMaskTracker>trackers = new ArrayList<>();
for(ResidualCrcMaskTracker tracker: mCsbkResidualTrackerMap.values())
{
trackers.add(tracker.clone());
}
return trackers;
}
/**
* Logs the current tracked CRC values and counts.
*/
public void log()
{
List<ResidualCrcMaskTracker> trackers = new ArrayList<>(mCsbkResidualTrackerMap.values());
Collections.sort(trackers);
StringBuilder sb = new StringBuilder();
sb.append("DMR CRC Mask Manager - Current Tracked Values\n");
for(ResidualCrcMaskTracker tracker: trackers)
{
sb.append("\tTracked Value: ").append(String.format("0x%04X", tracker.getTrackedValue()));
sb.append(" Count:").append(tracker.getCount());
sb.append(" Last Observed: " + new Date(tracker.getLastUpdated())).append("\n");
}
LOGGER.info(sb.toString());
}
/**
* Checks the CSBK message that has a failed CRC checksum to detect if an alternate CRC mask value is employed and
* if so, attempts to correct the message by using the alternate mask value once the alternate mask value is
* detected to be used consistently enough.
* @param csbk to re-check.
*/
public void check(CSBKMessage csbk)
{
if(!csbk.isValid())
{
int residual = CRCDMR.calculateResidual(csbk.getMessage(), 0, 80);
if(mCsbkResidualTrackerMap.containsKey(residual))
{
mCsbkResidualTrackerMap.get(residual).increment();
}
else
{
mCsbkResidualTrackerMap.put(residual, new ResidualCrcMaskTracker(residual));
}
List<ResidualCrcMaskTracker> trackers = new ArrayList<>(mCsbkResidualTrackerMap.values());
Collections.sort(trackers);
if(trackers.size() > 5)
{
mCsbkResidualTrackerMap.remove(trackers.get(0).getTrackedValue());
}
if(trackers.size() > 0)
{
ResidualCrcMaskTracker dominant = trackers.get(trackers.size() - 1);
if(dominant.isValid())
{
mDominantMask = dominant.getTrackedValue();
}
}
if(mDominantMask != 0)
{
csbk.checkCRC(mDominantMask);
//
// if(csbk.isValid())
// {
// LOGGER.info("CSBK fixed using alternate mask: " + Integer.toHexString(mDominantMask).toUpperCase());
// }
}
}
}
/**
* Residual CRC mask value tracker. Tracks the number of observances of a residual CRC calculated value to detect
* when an alternate CRC mask pattern is employed and automatically correct CRC values for messages.
*/
public class ResidualCrcMaskTracker implements Comparable<ResidualCrcMaskTracker>
{
private int mTrackedValue;
private int mCount;
private long mLastUpdated;
/**
* Constructs an instance for the specified residual, sets the count to one and sets the last update to now.
* @param residual value to track.
*/
public ResidualCrcMaskTracker(int residual)
{
mTrackedValue = residual;
mCount = 1;
mLastUpdated = System.currentTimeMillis();
}
/**
* Create a deep copy of this instance.
* @return cloned instance.
*/
public ResidualCrcMaskTracker clone()
{
ResidualCrcMaskTracker clone = new ResidualCrcMaskTracker(getTrackedValue());
clone.mCount = mCount;
clone.mLastUpdated = mLastUpdated;
return clone;
}
/**
* Count of number of times this residual has been observed.
* @return count
*/
public int getCount()
{
return mCount;
}
/**
* Indicates that the residual value has been observed at least 3x times indicating that it is possibly valid.
* @return
*/
public boolean isValid()
{
return mCount >= 3;
}
/**
* Timestamp when this value was last observed.
*/
public long getLastUpdated()
{
return mLastUpdated;
}
/**
* Residual tracked value.
* @return residual
*/
public int getTrackedValue()
{
return mTrackedValue;
}
public void increment()
{
mCount++;
//Prevent rollover by keeping value at max integer value.
if(mCount < 0)
{
mCount = Integer.MAX_VALUE;
}
mLastUpdated = System.currentTimeMillis();
}
/**
* Custom sort order by count and then by last updated timestamp.
* @param o the object to be compared.
* @return
*/
@Override
public int compareTo(ResidualCrcMaskTracker o)
{
int comparison = Integer.compare(getCount(), o.getCount());
if(comparison == 0)
{
comparison = Long.compare(getLastUpdated(), o.getLastUpdated());
}
return comparison;
}
}
}

View File

@@ -1,7 +1,6 @@
/*
* ******************************************************************************
* sdrtrunk
* Copyright (C) 2014-2020 Dennis Sheirer, Zhenyu Mao
* *****************************************************************************
* Copyright (C) 2014-2023 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -15,7 +14,7 @@
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* *****************************************************************************
* ****************************************************************************
*/
package io.github.dsheirer.module.decode.dmr.message.data.csbk;
@@ -72,6 +71,17 @@ public abstract class CSBKMessage extends DataMessage
setValid(correctedBitCount < 2);
}
/**
* Checks CRC using an alternate mask value and sets the message valid flag according to the results.
*/
public void checkCRC(int mask)
{
int correctedBitCount = CRCDMR.correctCCITT80(getMessage(), 0, 80, mask);
//Set message valid flag according to the corrected bit count for the CRC protected message
setValid(correctedBitCount < 2);
}
/**
* Checks crc for multi-block CSBK where the continuation block is an absolute channel parameters block
*

View File

@@ -29,8 +29,8 @@ import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraAdjac
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraAloha;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraAnnouncement;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraCsbko44;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraCsbko47;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraSmsAvailableNotification;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraTrafficChannelTalkerStatus;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraXPTPreamble;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.hytera.HyteraXPTSiteState;
import io.github.dsheirer.module.decode.dmr.message.data.csbk.motorola.CapacityMaxAloha;
@@ -250,8 +250,8 @@ public class CSBKMessageFactory
case HYTERA_08_CSBKO_44:
csbk = new HyteraCsbko44(pattern, message, cach, slotType, timestamp, timeslot);
break;
case HYTERA_08_CSBKO_47:
csbk = new HyteraCsbko47(pattern, message, cach, slotType, timestamp, timeslot);
case HYTERA_08_TRAFFIC_CHANNEL_TALKER_STATUS:
csbk = new HyteraTrafficChannelTalkerStatus(pattern, message, cach, slotType, timestamp, timeslot);
break;
case HYTERA_68_ACKNOWLEDGE:
csbk = new Hytera68Acknowledge(pattern, message, cach, slotType, timestamp, timeslot);

View File

@@ -93,7 +93,7 @@ public enum Opcode
//CSBKO 44 & 47 observed on Tier3 interleaved in voice group call terminator sequence. 44 was continuously transmitted
//and 47 was only transmitted 3x times in succession in the middle of the terminators and CSBKO 44 messages.
HYTERA_08_CSBKO_44(Vendor.HYTERA_8, 44, "HYTERA 08 CSBKO 44"),
HYTERA_08_CSBKO_47(Vendor.HYTERA_8, 47, "HYTERA 08 CSBKO 47"),
HYTERA_08_TRAFFIC_CHANNEL_TALKER_STATUS(Vendor.HYTERA_8, 47, "HYTERA 08 CSBKO 47"),
HYTERA_68_XPT_SITE_STATE(Vendor.HYTERA_68, 10, "HYTERA 68 XPT SITE STATE"),
HYTERA_68_ALOHA(Vendor.HYTERA_68, 25, "HYTERA 68 ALOHA"),
@@ -201,7 +201,7 @@ public enum Opcode
* Hytera opcodes
*/
public static final EnumSet<Opcode> HYTERA = EnumSet.of(HYTERA_08_ACKNOWLEDGE, HYTERA_08_ANNOUNCEMENT,
HYTERA_08_CSBKO_44, HYTERA_08_CSBKO_47, HYTERA_68_XPT_SITE_STATE, HYTERA_68_ALOHA,
HYTERA_08_CSBKO_44, HYTERA_08_TRAFFIC_CHANNEL_TALKER_STATUS, HYTERA_68_XPT_SITE_STATE, HYTERA_68_ALOHA,
HYTERA_68_ACKNOWLEDGE, HYTERA_68_ANNOUNCEMENT, HYTERA_68_XPT_PREAMBLE, HYTERA_68_CSBKO_62);
/**

View File

@@ -34,6 +34,9 @@ import java.util.List;
* Analysis: ... this was transmitted on the Tier III traffic channel during call termination. This CSBK was
* interleaved with every other Terminator message, starting after the first 2x terminator messages. This continued
* through the final teardown which was 3x Clear messages.
*
* On a second example, there were 8 distinct configurations of this message that were repeated in-order, multiple times
* during the traffic channel shutdown ... a total of 51x of these messages were transmitted.
*/
public class HyteraCsbko44 extends CSBKMessage
{

View File

@@ -33,13 +33,14 @@ import java.util.List;
/**
* Hytera DMR Tier III - CSBKO 47 -
*
* Analysis: ... this was transmitted on the traffic channel while the mobile was at the end of a call, in the terminator
* sequence. Since this comes from the SMS gateway 16776906, I wonder if this is a traffic channel message waiting notification?
* Analysis: this message is transmitted three times at the beginning or end of a voice transmission and appears to
* indicate that the traffic channel continues active and signals the start/end of an active speaker.
*/
public class HyteraCsbko47 extends CSBKMessage
public class HyteraTrafficChannelTalkerStatus extends CSBKMessage
{
private static final int[] UNKNOWN_1 = new int[]{16, 17, 18, 19, 20, 21, 22, 23};
private static final int[] UNKNOWN_2 = new int[]{24, 25, 26, 27, 28, 29, 30, 31};
private static final int TALKER_INACTIVE = 30;
private static final int[] DESTINATION_RADIO = new int[]{32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48,
49, 50, 51, 52, 53, 54, 55};
private static final int[] SOURCE_RADIO = new int[]{56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
@@ -59,7 +60,7 @@ public class HyteraCsbko47 extends CSBKMessage
* @param timestamp
* @param timeslot
*/
public HyteraCsbko47(DMRSyncPattern syncPattern, CorrectedBinaryMessage message, CACH cach, SlotType slotType, long timestamp, int timeslot)
public HyteraTrafficChannelTalkerStatus(DMRSyncPattern syncPattern, CorrectedBinaryMessage message, CACH cach, SlotType slotType, long timestamp, int timeslot)
{
super(syncPattern, message, cach, slotType, timestamp, timeslot);
}
@@ -79,17 +80,39 @@ public class HyteraCsbko47 extends CSBKMessage
{
sb.append(" RAS:").append(getBPTCReservedBits());
}
sb.append(" HYTERA CSBKO=47 ######### UNKNOWN");
sb.append(" HYTERA TRAFFIC CHANNEL");
if(isChannelInactive())
{
sb.append(" NOW AVAILABLE - PREVIOUS TALKER:").append(getDestinationRadio());
}
else
{
sb.append(" IN USE - CURRENT TALKER:").append(getDestinationRadio());
}
sb.append(" FM:").append(getSourceRadio());
sb.append(" TO:").append(getDestinationRadio());
sb.append(" UNK1:").append(getUnknown1());
sb.append(" UNK2:").append(getUnknown2());
sb.append(" MSG:").append(getMessage().toHexString());
return sb.toString();
}
/**
* Indicates if the channel is inactive and available for use.
*/
public boolean isChannelInactive()
{
return getMessage().get(TALKER_INACTIVE);
}
/**
* Indicates if the channel is active and in-use by the radio identifiers
*/
public boolean isChannelActive()
{
return !isChannelInactive();
}
/**
* Addressed (ie TO) radio identifier
* @return radio identifier
@@ -108,7 +131,6 @@ public class HyteraCsbko47 extends CSBKMessage
{
return getMessage().getHex(UNKNOWN_1, 2);
}
public String getUnknown2()
{
return getMessage().getHex(UNKNOWN_2, 2);

View File

@@ -118,7 +118,7 @@ public class Clear extends CSBKMessage implements ITimeslotFrequencyReceiver
sb.append(" ENCRYPTED");
}
sb.append(" CLEAR - RETURN TO ").append(getMoveToChannel());
sb.append(" CLEAR - RETURN TO").append(getMoveToChannel());
sb.append(" FM:").append(getSourceRadio());
sb.append(" TO:").append(getDestinationId());
sb.append(" MSG:").append(getMessage().toHexString());

View File

@@ -109,6 +109,15 @@ public class LCMessageFactory
LCOpcode opcode = FullLCMessage.getOpcode(message);
//Some Hytera Tier-3 systems use a zero-valued mask when the FLC is carried in the voice header or terminator
//and in the same transmission it uses the standard mask when carried across the voice frames. It doesn't make
//sense, but maybe it's a bug in their software implementation?
if(!valid && opcode == LCOpcode.FULL_STANDARD_GROUP_VOICE_CHANNEL_USER)
{
//Retry the RS(12,9,4) with a mask value of zero
valid = REED_SOLOMON_12_9_4_DMR.correctFullLinkControl(message, 0);
}
FullLCMessage flc = null;
switch(opcode)