#2742 Database backups with scheduled backups, retention and restore

- Engine-independent backup files (zip with calls, events and metadata as JSON) written from a single read
  transaction on a background thread; restorable into any supported database type.
- Automatic backup interval and number of backups to keep in the database preferences; the oldest backups
  are deleted after each backup.
- Backup list with Back Up Now, Restore and Delete.  A restore runs in one transaction on the database thread
  after making a pre-restore backup; missing recordings are marked deleted and in-progress calls interrupted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
denny
2026-11-22 03:16:23 -05:00
parent d6234600c0
commit bb19547af0
14 changed files with 2119 additions and 2 deletions

View File

@@ -19,6 +19,7 @@
package io.github.dsheirer.database;
import io.github.dsheirer.database.ageoff.AgeOffManager;
import io.github.dsheirer.database.backup.BackupManager;
import io.github.dsheirer.database.call.CallRecord;
import io.github.dsheirer.database.call.ICallRepository;
import io.github.dsheirer.database.call.CallSearchCriteria;
@@ -82,6 +83,7 @@ public class DatabaseManager
private final Map<RowHandle,CallRecord> mPendingCalls = new LinkedHashMap<>();
private final Map<RowHandle,EventRecord> mPendingEvents = new LinkedHashMap<>();
private final AgeOffManager mAgeOffManager;
private final BackupManager mBackupManager;
private volatile CallDatabase mDatabase;
private volatile String mErrorMessage;
private volatile int mGeneration = 1;
@@ -96,6 +98,7 @@ public class DatabaseManager
{
mUserPreferences = userPreferences;
mAgeOffManager = new AgeOffManager(this, userPreferences);
mBackupManager = new BackupManager(this, userPreferences);
}
/**
@@ -116,6 +119,7 @@ public class DatabaseManager
mFlushFuture = mDatabaseExecutor.scheduleWithFixedDelay(this::flush, FLUSH_INTERVAL_MS, FLUSH_INTERVAL_MS,
TimeUnit.MILLISECONDS);
mAgeOffManager.start(mDatabaseExecutor);
mBackupManager.start();
}
/**
@@ -124,6 +128,7 @@ public class DatabaseManager
public void stop()
{
mAgeOffManager.stop();
mBackupManager.stop();
if(mFlushFuture != null)
{
@@ -918,6 +923,53 @@ public class DatabaseManager
return new long[]{transferred, skipped};
}
/**
* Restores the backup into the current database, replacing its calls and events. Runs on the database thread,
* after writing the pending updates. Calls and events in progress continue as new rows. Listeners receive a
* CONNECTED event when the restore is complete.
* @param backup file to restore
* @param progress receives status text
*/
public CompletableFuture<BackupManager.RestoreResult> restore(Path backup, Consumer<String> progress)
{
CompletableFuture<BackupManager.RestoreResult> future = new CompletableFuture<>();
mDatabaseExecutor.execute(() -> {
try
{
CallDatabase database = mDatabase;
if(database == null)
{
throw new IllegalStateException("The call database is not available");
}
flush();
BackupManager.RestoreResult result = BackupManager.restore(database, backup, progress);
//Rows of calls and events in progress were replaced - write them as new rows
mGeneration++;
broadcast(DatabaseEvent.Type.CONNECTED);
future.complete(result);
}
catch(Exception e)
{
LOGGER.error("Error restoring database backup [" + backup + "]", e);
future.completeExceptionally(e);
}
});
return future;
}
/**
* Backup manager
*/
public BackupManager getBackupManager()
{
return mBackupManager;
}
/**
* Age-off rule manager
*/

View File

@@ -0,0 +1,283 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2026 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.database.backup;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import com.google.gson.stream.JsonWriter;
import io.github.dsheirer.database.IdentifierRecord;
import io.github.dsheirer.database.call.CallRecord;
import io.github.dsheirer.database.call.CallState;
import io.github.dsheirer.database.call.PatchedTalkgroup;
import io.github.dsheirer.database.call.RecordingState;
import io.github.dsheirer.database.event.EventRecord;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* JSON form of the call and event records in a backup file. Field names match the database column names, null
* values are omitted and unknown fields are ignored when reading, so that a newer backup format version can add fields.
*/
public final class BackupCodec
{
private BackupCodec()
{
}
public static void write(JsonWriter writer, CallRecord call) throws IOException
{
writer.beginObject();
writeIdentifierFields(writer, call);
writer.name("call_state").value(call.getCallState().name());
string(writer, "talker_alias", call.getTalkerAlias());
writer.name("encrypted").value(call.isEncrypted());
writer.name("duplicate_call").value(call.isDuplicate());
writer.name("priority").value(call.getPriority());
writer.name("alias_record").value(call.isAliasRecord());
string(writer, "record_format", call.getRecordFormat());
string(writer, "file_path", call.getFilePath());
writer.name("file_size").value(call.getFileSize());
writer.name("recording_state").value(call.getRecordingState().name());
writer.endObject();
}
public static void write(JsonWriter writer, EventRecord event) throws IOException
{
writer.beginObject();
writeIdentifierFields(writer, event);
writer.name("duration_ms").value(event.getDuration());
writer.name("event_type").value(event.getEventTypeValue().name());
string(writer, "channel_descriptor", event.getChannelDescriptor());
string(writer, "talker_alias", event.getTalkerAlias());
string(writer, "details", event.getDetails());
writer.endObject();
}
private static void writeIdentifierFields(JsonWriter writer, IdentifierRecord record) throws IOException
{
writer.name("start_time").value(record.getStartTime());
writer.name("end_time").value(record.getEndTime());
string(writer, "protocol", record.getProtocol());
string(writer, "system_name", record.getSystem());
string(writer, "site_name", record.getSite());
string(writer, "channel_name", record.getChannel());
writer.name("frequency").value(record.getFrequency());
writer.name("timeslot").value(record.getTimeslot());
string(writer, "to_id", record.getToId());
number(writer, "to_value", record.getToValue());
string(writer, "to_form", record.getToForm());
string(writer, "to_fq", record.getToFullyQualified());
string(writer, "from_id", record.getFromId());
number(writer, "from_value", record.getFromValue());
string(writer, "from_form", record.getFromForm());
string(writer, "from_fq", record.getFromFullyQualified());
string(writer, "alias_list", record.getAliasList());
if(!record.getPatchedTalkgroups().isEmpty())
{
writer.name("patched_talkgroups").beginArray();
for(PatchedTalkgroup talkgroup: record.getPatchedTalkgroups())
{
writer.beginObject();
writer.name("talkgroup").value(talkgroup.value());
string(writer, "fq", talkgroup.fullyQualified());
writer.endObject();
}
writer.endArray();
}
}
private static void string(JsonWriter writer, String name, String value) throws IOException
{
if(value != null)
{
writer.name(name).value(value);
}
}
private static void number(JsonWriter writer, String name, Long value) throws IOException
{
if(value != null)
{
writer.name(name).value(value);
}
}
/**
* Reads a call record object
*/
public static CallRecord readCall(JsonReader reader) throws IOException
{
CallRecord call = new CallRecord();
reader.beginObject();
while(reader.hasNext())
{
String name = reader.nextName();
if(reader.peek() == JsonToken.NULL)
{
reader.nextNull();
continue;
}
if(readIdentifierField(reader, name, call))
{
continue;
}
switch(name)
{
case "call_state" -> call.setCallState(CallState.fromValue(reader.nextString()));
case "talker_alias" -> call.setTalkerAlias(reader.nextString());
case "encrypted" -> call.setEncrypted(reader.nextBoolean());
case "duplicate_call" -> call.setDuplicate(reader.nextBoolean());
case "priority" -> call.setPriority(reader.nextInt());
case "alias_record" -> call.setAliasRecord(reader.nextBoolean());
case "record_format" -> call.setRecordFormat(reader.nextString());
case "file_path" -> call.setFilePath(reader.nextString());
case "file_size" -> call.setFileSize(reader.nextLong());
case "recording_state" -> call.setRecordingState(RecordingState.fromValue(reader.nextString()));
default -> reader.skipValue();
}
}
reader.endObject();
return call;
}
/**
* Reads an event record object
*/
public static EventRecord readEvent(JsonReader reader) throws IOException
{
EventRecord event = new EventRecord();
reader.beginObject();
while(reader.hasNext())
{
String name = reader.nextName();
if(reader.peek() == JsonToken.NULL)
{
reader.nextNull();
continue;
}
if(readIdentifierField(reader, name, event))
{
continue;
}
switch(name)
{
case "duration_ms" -> event.setDuration(reader.nextLong());
case "event_type" -> event.setEventType(reader.nextString());
case "channel_descriptor" -> event.setChannelDescriptor(reader.nextString());
case "talker_alias" -> event.setTalkerAlias(reader.nextString());
case "details" -> event.setDetails(reader.nextString());
default -> reader.skipValue();
}
}
reader.endObject();
return event;
}
/**
* Reads the value of a common identifier record field.
* @return true if the field is a common field and its value was read
*/
private static boolean readIdentifierField(JsonReader reader, String name, IdentifierRecord record)
throws IOException
{
switch(name)
{
case "start_time" -> record.setStartTime(reader.nextLong());
case "end_time" -> record.setEndTime(reader.nextLong());
case "protocol" -> record.setProtocol(reader.nextString());
case "system_name" -> record.setSystem(reader.nextString());
case "site_name" -> record.setSite(reader.nextString());
case "channel_name" -> record.setChannel(reader.nextString());
case "frequency" -> record.setFrequency(reader.nextLong());
case "timeslot" -> record.setTimeslot(reader.nextInt());
case "to_id" -> record.setToId(reader.nextString());
case "to_value" -> record.setToValue(reader.nextLong());
case "to_form" -> record.setToForm(reader.nextString());
case "to_fq" -> record.setToFullyQualified(reader.nextString());
case "from_id" -> record.setFromId(reader.nextString());
case "from_value" -> record.setFromValue(reader.nextLong());
case "from_form" -> record.setFromForm(reader.nextString());
case "from_fq" -> record.setFromFullyQualified(reader.nextString());
case "alias_list" -> record.setAliasList(reader.nextString());
case "patched_talkgroups" -> record.setPatchedTalkgroups(readPatchedTalkgroups(reader));
default ->
{
return false;
}
}
return true;
}
private static List<PatchedTalkgroup> readPatchedTalkgroups(JsonReader reader) throws IOException
{
List<PatchedTalkgroup> talkgroups = new ArrayList<>();
reader.beginArray();
while(reader.hasNext())
{
long value = 0;
String fq = null;
reader.beginObject();
while(reader.hasNext())
{
String name = reader.nextName();
if(reader.peek() == JsonToken.NULL)
{
reader.nextNull();
}
else if(name.equals("talkgroup"))
{
value = reader.nextLong();
}
else if(name.equals("fq"))
{
fq = reader.nextString();
}
else
{
reader.skipValue();
}
}
reader.endObject();
talkgroups.add(new PatchedTalkgroup(value, fq));
}
reader.endArray();
return talkgroups;
}
}

View File

@@ -0,0 +1,61 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2026 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.database.backup;
import java.nio.file.Path;
/**
* A backup file in the backup directory
*
* @param path of the backup file
* @param created time of the backup, milliseconds since epoch (from the file name)
* @param size of the file in bytes
* @param tag optional tag from the file name (e.g. pre-restore), or null
* @param metadata from the file, or null if the file can't be read
*/
public record BackupInfo(Path path, long created, long size, String tag, Metadata metadata)
{
/**
* Backup file contents summary
*
* @param format version of the backup file format
* @param schemaVersion of the database tables the backup was made from
* @param created time of the backup, milliseconds since epoch
* @param source description of the database that was backed up
* @param calls count of calls in the backup
* @param events count of events in the backup
*/
public record Metadata(int format, int schemaVersion, long created, String source, long calls, long events) {}
/**
* Indicates if the file is a readable backup
*/
public boolean isValid()
{
return metadata != null;
}
/**
* Indicates if this backup was made automatically before restoring another backup
*/
public boolean isPreRestore()
{
return BackupManager.TAG_PRE_RESTORE.equals(tag);
}
}

View File

@@ -0,0 +1,68 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2026 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.database.backup;
import java.util.concurrent.TimeUnit;
/**
* How often the call database is backed up automatically
*/
public enum BackupInterval
{
NEVER("Never (manual only)", 0),
HOURS_6("Every 6 hours", TimeUnit.HOURS.toMillis(6)),
HOURS_12("Every 12 hours", TimeUnit.HOURS.toMillis(12)),
DAY_1("Daily", TimeUnit.DAYS.toMillis(1)),
DAYS_2("Every 2 days", TimeUnit.DAYS.toMillis(2)),
WEEK_1("Weekly", TimeUnit.DAYS.toMillis(7)),
MONTH_1("Monthly (30 days)", TimeUnit.DAYS.toMillis(30));
public static final BackupInterval DEFAULT = DAY_1;
private final String mLabel;
private final long mMilliseconds;
BackupInterval(String label, long milliseconds)
{
mLabel = label;
mMilliseconds = milliseconds;
}
/**
* Time between automatic backups in milliseconds, or 0 when automatic backups are off
*/
public long getMilliseconds()
{
return mMilliseconds;
}
/**
* Indicates if automatic backups are enabled
*/
public boolean isEnabled()
{
return this != NEVER;
}
@Override
public String toString()
{
return mLabel;
}
}

View File

@@ -0,0 +1,792 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2026 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.database.backup;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import io.github.dsheirer.controller.NamingThreadFactory;
import io.github.dsheirer.database.CallDatabase;
import io.github.dsheirer.database.DatabaseManager;
import io.github.dsheirer.database.call.CallRecord;
import io.github.dsheirer.database.call.ICallRepository;
import io.github.dsheirer.database.call.RecordingState;
import io.github.dsheirer.database.event.EventRecord;
import io.github.dsheirer.database.event.IEventRepository;
import io.github.dsheirer.preference.UserPreferences;
import io.github.dsheirer.preference.database.DatabasePreference;
import io.github.dsheirer.sample.Broadcaster;
import io.github.dsheirer.sample.Listener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedOutputStream;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.sql.Connection;
import java.sql.SQLException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
/**
* Database backups: scheduled and on-demand backups of the calls and events, retention of the newest backups, and
* restore of a backup into the current database.
*
* A backup is a zip file with the calls and events as JSON (see {@link BackupCodec}) and a metadata entry. The format
* doesn't depend on the database engine, so a backup can be restored into any of the supported databases. The
* recording files aren't included - restored calls refer to the recording files that still exist.
*
* Backups and restores run one at a time on the backup thread. A backup reads the database in a single read
* transaction while the database thread continues to write. A restore runs on the database thread (writes are held
* until it's done) and first makes a 'pre-restore' backup of the current contents, so it can be undone.
*/
public class BackupManager
{
private static final Logger LOGGER = LoggerFactory.getLogger(BackupManager.class);
public static final int FORMAT_VERSION = 1;
public static final String TAG_PRE_RESTORE = "pre-restore";
static final String FILE_PREFIX = "sdrtrunk-backup-";
static final String FILE_EXTENSION = ".zip";
static final String ENTRY_METADATA = "metadata.json";
static final String ENTRY_CALLS = "calls.json";
static final String ENTRY_EVENTS = "events.json";
private static final Pattern FILE_PATTERN =
Pattern.compile("sdrtrunk-backup-(\\d{8}-\\d{6})(?:-([a-z][a-z0-9-]*))?\\.zip");
private static final DateTimeFormatter FILE_TIME = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");
private static final long INITIAL_DELAY_SECONDS = 120;
private static final long CHECK_INTERVAL_SECONDS = 600;
private static final int PAGE_SIZE = 1000;
private final DatabaseManager mDatabaseManager;
private final DatabasePreference mPreference;
private final Supplier<Path> mDirectory;
private final ScheduledExecutorService mExecutor =
Executors.newSingleThreadScheduledExecutor(new NamingThreadFactory("sdrtrunk database backup"));
private final Broadcaster<BackupInfo> mBackupBroadcaster = new Broadcaster<>();
private ScheduledFuture<?> mScheduledFuture;
/**
* Constructs an instance
* @param databaseManager for the current database
* @param userPreferences for the backup schedule, retention and directory
*/
public BackupManager(DatabaseManager databaseManager, UserPreferences userPreferences)
{
mDatabaseManager = databaseManager;
mPreference = userPreferences.getDatabasePreference();
mDirectory = () -> userPreferences.getDirectoryPreference().getDirectoryDatabaseBackup();
}
/**
* Starts the automatic backup schedule. The schedule is checked two minutes after startup and then every 10
* minutes, and a backup is made when the newest backup is older than the backup interval.
*/
public void start()
{
if(mScheduledFuture == null)
{
mScheduledFuture = mExecutor.scheduleWithFixedDelay(this::checkSchedule, INITIAL_DELAY_SECONDS,
CHECK_INTERVAL_SECONDS, TimeUnit.SECONDS);
}
}
/**
* Stops the schedule. A backup in progress is abandoned and its partial file removed.
*/
public void stop()
{
if(mScheduledFuture != null)
{
mScheduledFuture.cancel(false);
mScheduledFuture = null;
}
mExecutor.shutdownNow();
}
/**
* Directory that holds the backup files
*/
public Path getBackupDirectory()
{
return mDirectory.get();
}
/**
* Registers the listener to be notified of each new backup (scheduled, manual or pre-restore). Notified on the
* backup thread.
*/
public void addBackupListener(Listener<BackupInfo> listener)
{
mBackupBroadcaster.addListener(listener);
}
public void removeBackupListener(Listener<BackupInfo> listener)
{
mBackupBroadcaster.removeListener(listener);
}
private void checkSchedule()
{
try
{
BackupInterval interval = mPreference.getBackupInterval();
List<BackupInfo> backups = list(getBackupDirectory());
long last = backups.isEmpty() ? 0 : backups.getFirst().created();
if(isDue(last, interval, System.currentTimeMillis()) && mDatabaseManager.isConnected())
{
LOGGER.info("Starting scheduled database backup (" + interval + ")");
backup(null);
}
}
catch(Exception e)
{
LOGGER.error("Scheduled database backup failed - " + e.getMessage(), e);
}
}
/**
* Indicates if an automatic backup is due.
* @param lastBackup time of the newest backup, or 0 if there are no backups
* @param interval between automatic backups
* @param now current time
*/
static boolean isDue(long lastBackup, BackupInterval interval, long now)
{
return interval.isEnabled() && (lastBackup <= 0 || now - lastBackup >= interval.getMilliseconds());
}
/**
* Backs up the current database now and then deletes the oldest backups beyond the retention count.
*/
public CompletableFuture<BackupInfo> backupNow()
{
return CompletableFuture.supplyAsync(() -> {
try
{
return backup(null);
}
catch(Exception e)
{
throw new IllegalStateException(e.getMessage(), e);
}
}, mExecutor);
}
/**
* Makes a backup and applies the retention count. Backup thread only.
*/
private BackupInfo backup(String tag) throws IOException, SQLException
{
CallDatabase database = mDatabaseManager.getDatabase();
if(database == null)
{
throw new IllegalStateException("The call database is not available");
}
BackupInfo backup = writeBackup(database, getBackupDirectory(), tag, System.currentTimeMillis());
LOGGER.info("Database backup created [" + backup.path() + "] with " + backup.metadata().calls() +
" calls and " + backup.metadata().events() + " events");
if(tag == null)
{
for(Path deleted: applyRetention(getBackupDirectory(), mPreference.getBackupRetain()))
{
LOGGER.info("Deleted old database backup [" + deleted + "]");
}
}
mBackupBroadcaster.broadcast(backup);
return backup;
}
/**
* Lists the backups, newest first. Runs on the backup thread.
*/
public CompletableFuture<List<BackupInfo>> listBackups()
{
return CompletableFuture.supplyAsync(() -> list(getBackupDirectory()), mExecutor);
}
/**
* Deletes the backup file. Runs on the backup thread.
*/
public CompletableFuture<Void> delete(BackupInfo backup)
{
return CompletableFuture.runAsync(() -> {
try
{
Files.deleteIfExists(backup.path());
LOGGER.info("Deleted database backup [" + backup.path() + "]");
}
catch(IOException ioe)
{
throw new IllegalStateException(ioe.getMessage(), ioe);
}
}, mExecutor);
}
/**
* Replaces the calls and events in the current database with the contents of the backup. A pre-restore backup
* of the current contents is made first (when the database isn't empty).
* @param backup to restore
* @param progress receives status text (on the backup or database thread)
*/
public CompletableFuture<RestoreResult> restore(BackupInfo backup, Consumer<String> progress)
{
return CompletableFuture.supplyAsync(() -> {
try
{
BackupInfo.Metadata metadata = readMetadata(backup.path());
checkCompatible(metadata);
CallDatabase database = mDatabaseManager.getDatabase();
if(database == null)
{
throw new IllegalStateException("The call database is not available");
}
BackupInfo preRestore = null;
if(database.getCallCount() > 0 || database.getEventCount() > 0)
{
progress.accept("Backing up the current database before the restore");
preRestore = backup(TAG_PRE_RESTORE);
}
RestoreResult result = mDatabaseManager.restore(backup.path(), progress).get();
LOGGER.info("Restored database backup [" + backup.path() + "] - " + result.calls() + " calls, " +
result.events() + " events, " + result.missingRecordings() + " missing recordings");
return result.withPreRestore(preRestore);
}
catch(Exception e)
{
Throwable cause = e.getCause() != null && e instanceof java.util.concurrent.ExecutionException ?
e.getCause() : e;
throw new IllegalStateException(cause.getMessage(), cause);
}
}, mExecutor);
}
/**
* Result of a restore
* @param calls restored
* @param events restored
* @param missingRecordings count of restored calls whose recording file no longer exists
* @param preRestore backup of the database contents before the restore, or null if the database was empty
*/
public record RestoreResult(long calls, long events, long missingRecordings, BackupInfo preRestore)
{
RestoreResult withPreRestore(BackupInfo backup)
{
return new RestoreResult(calls, events, missingRecordings, backup);
}
}
/**
* Throws if the backup can't be restored by this version of sdrtrunk.
*/
static void checkCompatible(BackupInfo.Metadata metadata) throws IOException
{
if(metadata == null)
{
throw new IOException("The file is not a readable sdrtrunk database backup");
}
if(metadata.format() > FORMAT_VERSION || metadata.schemaVersion() > CallDatabase.SCHEMA_VERSION)
{
throw new IOException("The backup was made by a newer version of sdrtrunk (backup format " +
metadata.format() + ", schema version " + metadata.schemaVersion() + ")");
}
}
/**
* Writes a backup of the calls and events in the database to a new file in the directory. The file is written
* to a temporary name and renamed when complete, so an interrupted backup never leaves a partial backup file.
* @param database to back up
* @param directory for the backup file
* @param tag for the file name, or null
* @param now backup time
* @return the new backup
*/
public static BackupInfo writeBackup(CallDatabase database, Path directory, String tag, long now)
throws IOException, SQLException
{
Files.createDirectories(directory);
long created = now - (now % 1000);
Path target = directory.resolve(getFileName(created, tag));
//Unique name when two backups are made in the same second
while(Files.exists(target))
{
created += 1000;
target = directory.resolve(getFileName(created, tag));
}
Path temp = directory.resolve(target.getFileName() + ".part");
long calls = 0;
long events = 0;
try(ZipOutputStream zip = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(temp)));
Connection connection = database.getConnection())
{
boolean autoCommit = connection.getAutoCommit();
int isolation = connection.getTransactionIsolation();
try
{
//One read transaction, so that the calls and events are a consistent snapshot
connection.setAutoCommit(false);
setIsolation(connection, Connection.TRANSACTION_REPEATABLE_READ);
ICallRepository callRepository = database.getCallRepository();
JsonWriter writer = startEntry(zip, ENTRY_CALLS);
writer.beginArray();
List<CallRecord> callPage = callRepository.readPage(connection, 0, PAGE_SIZE);
while(!callPage.isEmpty())
{
for(CallRecord call: callPage)
{
BackupCodec.write(writer, call);
}
calls += callPage.size();
callPage = callRepository.readPage(connection, callPage.getLast().getId(), PAGE_SIZE);
}
writer.endArray();
writer.flush();
zip.closeEntry();
IEventRepository eventRepository = database.getEventRepository();
writer = startEntry(zip, ENTRY_EVENTS);
writer.beginArray();
List<EventRecord> eventPage = eventRepository.readPage(connection, 0, PAGE_SIZE);
while(!eventPage.isEmpty())
{
for(EventRecord event: eventPage)
{
BackupCodec.write(writer, event);
}
events += eventPage.size();
eventPage = eventRepository.readPage(connection, eventPage.getLast().getId(), PAGE_SIZE);
}
writer.endArray();
writer.flush();
zip.closeEntry();
connection.rollback();
}
finally
{
setIsolation(connection, isolation);
connection.setAutoCommit(autoCommit);
}
BackupInfo.Metadata metadata = new BackupInfo.Metadata(FORMAT_VERSION, CallDatabase.SCHEMA_VERSION,
created, database.getConfiguration().getDescription(), calls, events);
JsonWriter writer = startEntry(zip, ENTRY_METADATA);
writer.setIndent(" ");
writer.beginObject();
writer.name("format").value(metadata.format());
writer.name("schema_version").value(metadata.schemaVersion());
writer.name("created").value(metadata.created());
writer.name("source").value(metadata.source());
writer.name("calls").value(metadata.calls());
writer.name("events").value(metadata.events());
writer.endObject();
writer.flush();
zip.closeEntry();
}
catch(IOException | SQLException | RuntimeException e)
{
Files.deleteIfExists(temp);
throw e;
}
try
{
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE);
}
catch(AtomicMoveNotSupportedException e)
{
Files.move(temp, target);
}
return read(target);
}
private static void setIsolation(Connection connection, int isolation)
{
try
{
connection.setTransactionIsolation(isolation);
}
catch(SQLException se)
{
LOGGER.debug("Transaction isolation level not supported - " + se.getMessage());
}
}
/**
* Starts a zip entry and returns a JSON writer for it. The writer must be flushed, not closed, since closing it
* would close the zip stream.
*/
private static JsonWriter startEntry(ZipOutputStream zip, String name) throws IOException
{
zip.putNextEntry(new ZipEntry(name));
OutputStream entry = new FilterOutputStream(zip)
{
@Override
public void write(byte[] b, int off, int len) throws IOException
{
out.write(b, off, len);
}
@Override
public void close() throws IOException
{
flush();
}
};
return new JsonWriter(new OutputStreamWriter(entry, StandardCharsets.UTF_8));
}
/**
* Backup file name for the time and tag
*/
static String getFileName(long created, String tag)
{
String time = FILE_TIME.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(created), ZoneId.systemDefault()));
return FILE_PREFIX + time + (tag != null ? "-" + tag : "") + FILE_EXTENSION;
}
/**
* Replaces the calls and events in the database with the contents of the backup, in a single transaction, so
* that a failed restore leaves the database unchanged. Calls that were in progress when the backup was made are
* marked interrupted, and calls whose recording file no longer exists are marked as having a deleted recording.
* Invoke on the database thread only.
* @param database to restore into
* @param backup file
* @param progress receives status text
*/
public static RestoreResult restore(CallDatabase database, Path backup, Consumer<String> progress)
throws IOException, SQLException
{
BackupInfo.Metadata metadata = readMetadata(backup);
checkCompatible(metadata);
long calls = 0;
long events = 0;
long missing = 0;
try(ZipFile zip = new ZipFile(backup.toFile()); Connection connection = database.getConnection())
{
ZipEntry callsEntry = zip.getEntry(ENTRY_CALLS);
ZipEntry eventsEntry = zip.getEntry(ENTRY_EVENTS);
if(callsEntry == null || eventsEntry == null)
{
throw new IOException("The backup file is incomplete");
}
boolean autoCommit = connection.getAutoCommit();
connection.setAutoCommit(false);
try
{
ICallRepository callRepository = database.getCallRepository();
IEventRepository eventRepository = database.getEventRepository();
progress.accept("Removing the current calls and events");
eventRepository.deleteAll(connection);
callRepository.deleteAll(connection);
try(InputStream in = zip.getInputStream(callsEntry);
JsonReader reader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)))
{
List<CallRecord> batch = new ArrayList<>();
reader.beginArray();
while(reader.hasNext())
{
CallRecord call = BackupCodec.readCall(reader);
if(clearMissingRecording(call))
{
missing++;
}
batch.add(call);
if(batch.size() >= PAGE_SIZE)
{
callRepository.insertBatch(connection, batch);
calls += batch.size();
batch.clear();
progress.accept("Restoring calls: " + calls + " of " + metadata.calls());
}
}
reader.endArray();
callRepository.insertBatch(connection, batch);
calls += batch.size();
}
try(InputStream in = zip.getInputStream(eventsEntry);
JsonReader reader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)))
{
List<EventRecord> batch = new ArrayList<>();
reader.beginArray();
while(reader.hasNext())
{
batch.add(BackupCodec.readEvent(reader));
if(batch.size() >= PAGE_SIZE)
{
eventRepository.insertBatch(connection, batch);
events += batch.size();
batch.clear();
progress.accept("Restoring events: " + events + " of " + metadata.events());
}
}
reader.endArray();
eventRepository.insertBatch(connection, batch);
events += batch.size();
}
callRepository.markInterrupted(connection);
progress.accept("Committing the restored calls and events");
connection.commit();
}
catch(IOException | SQLException | RuntimeException e)
{
connection.rollback();
throw e;
}
finally
{
connection.setAutoCommit(autoCommit);
}
}
return new RestoreResult(calls, events, missing, null);
}
/**
* Marks the recording of the call deleted when the recording file no longer exists (e.g. removed by an age-off
* rule after the backup was made).
* @return true if the recording was missing
*/
static boolean clearMissingRecording(CallRecord call)
{
if(call.getFilePath() == null || (call.getRecordingState() != RecordingState.RECORDED &&
call.getRecordingState() != RecordingState.RECORDING))
{
return false;
}
try
{
if(Files.exists(Path.of(call.getFilePath())))
{
return false;
}
}
catch(RuntimeException re)
{
//Invalid path - treat as missing
}
call.setRecordingState(RecordingState.DELETED);
call.setFilePath(null);
call.setFileSize(0);
return true;
}
/**
* Backups in the directory, newest first. Files that match the backup file name but can't be read are included
* with null metadata.
*/
public static List<BackupInfo> list(Path directory)
{
List<BackupInfo> backups = new ArrayList<>();
if(!Files.isDirectory(directory))
{
return backups;
}
try(Stream<Path> files = Files.list(directory))
{
for(Path file: files.toList())
{
BackupInfo backup = read(file);
if(backup != null)
{
backups.add(backup);
}
}
}
catch(IOException ioe)
{
LOGGER.warn("Unable to list database backups in [" + directory + "] - " + ioe.getMessage());
}
backups.sort(Comparator.comparingLong(BackupInfo::created).reversed()
.thenComparing(backup -> backup.path().getFileName().toString(), Comparator.reverseOrder()));
return backups;
}
/**
* Backup details for the file, or null if the file name isn't a backup file name.
*/
static BackupInfo read(Path file)
{
Matcher matcher = FILE_PATTERN.matcher(file.getFileName().toString());
if(!matcher.matches() || !Files.isRegularFile(file))
{
return null;
}
long created;
try
{
created = LocalDateTime.parse(matcher.group(1), FILE_TIME).atZone(ZoneId.systemDefault()).toInstant()
.toEpochMilli();
}
catch(DateTimeParseException e)
{
return null;
}
long size = 0;
try
{
size = Files.size(file);
}
catch(IOException ioe)
{
//Leave as 0
}
BackupInfo.Metadata metadata = null;
try
{
metadata = readMetadata(file);
}
catch(IOException | RuntimeException e)
{
LOGGER.warn("Unable to read database backup [" + file + "] - " + e.getMessage());
}
return new BackupInfo(file, created, size, matcher.group(2), metadata);
}
/**
* Reads the metadata entry of a backup file
*/
static BackupInfo.Metadata readMetadata(Path file) throws IOException
{
try(ZipFile zip = new ZipFile(file.toFile()))
{
ZipEntry entry = zip.getEntry(ENTRY_METADATA);
if(entry == null)
{
return null;
}
try(InputStream in = zip.getInputStream(entry);
InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8))
{
JsonObject json = JsonParser.parseReader(reader).getAsJsonObject();
return new BackupInfo.Metadata(json.get("format").getAsInt(), json.get("schema_version").getAsInt(),
json.get("created").getAsLong(), json.has("source") ? json.get("source").getAsString() : "",
json.get("calls").getAsLong(), json.get("events").getAsLong());
}
}
}
/**
* Deletes the oldest backups so that no more than the retention count remain.
* @return deleted files
*/
public static List<Path> applyRetention(Path directory, int retain)
{
List<Path> deleted = new ArrayList<>();
List<BackupInfo> backups = list(directory);
for(int x = Math.max(1, retain); x < backups.size(); x++)
{
Path path = backups.get(x).path();
try
{
Files.deleteIfExists(path);
deleted.add(path);
}
catch(IOException ioe)
{
LOGGER.warn("Unable to delete old database backup [" + path + "] - " + ioe.getMessage());
}
}
return deleted;
}
}

View File

@@ -420,6 +420,16 @@ public class CallRepository implements ICallRepository
}
}
@Override
public int deleteAll(Connection connection) throws SQLException
{
try(Statement statement = connection.createStatement())
{
statement.executeUpdate("DELETE FROM " + mPatchedTable);
return statement.executeUpdate("DELETE FROM " + mTable);
}
}
@Override
public int deleteActive(Connection connection) throws SQLException
{

View File

@@ -59,6 +59,12 @@ public interface ICallRepository
*/
int deleteActive(Connection connection) throws SQLException;
/**
* Deletes all calls (and their patched talkgroups), for restoring a backup.
* @return count of calls deleted
*/
int deleteAll(Connection connection) throws SQLException;
/**
* Calls matching the criteria, newest first.
*/

View File

@@ -519,6 +519,17 @@ public class EventRepository implements IEventRepository
}
}
@Override
public int deleteAll(Connection connection) throws SQLException
{
try(PreparedStatement patched = connection.prepareStatement("DELETE FROM " + mPatchedTable);
PreparedStatement events = connection.prepareStatement("DELETE FROM " + mTable))
{
patched.executeUpdate();
return events.executeUpdate();
}
}
@Override
public String getSchema()
{

View File

@@ -83,6 +83,12 @@ public interface IEventRepository
*/
int deleteStartedBefore(Connection connection, long startedBefore) throws SQLException;
/**
* Deletes all events (and their patched talkgroups), for restoring a backup.
* @return count of events deleted
*/
int deleteAll(Connection connection) throws SQLException;
/**
* Schema name
*/

View File

@@ -0,0 +1,378 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2026 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.gui.preference.database;
import io.github.dsheirer.database.DatabaseManager;
import io.github.dsheirer.database.backup.BackupInfo;
import io.github.dsheirer.database.backup.BackupInterval;
import io.github.dsheirer.database.backup.BackupManager;
import io.github.dsheirer.preference.database.DatabasePreference;
import io.github.dsheirer.sample.Listener;
import javafx.application.Platform;
import javafx.scene.control.Alert;
import javafx.scene.control.Button;
import javafx.scene.control.ButtonType;
import javafx.scene.control.ComboBox;
import javafx.scene.control.Label;
import javafx.scene.control.ListCell;
import javafx.scene.control.ListView;
import javafx.scene.control.ProgressIndicator;
import javafx.scene.control.Spinner;
import javafx.scene.layout.HBox;
import javafx.scene.layout.VBox;
import javafx.geometry.Pos;
import javafx.stage.Window;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletionException;
/**
* Database backup preferences: automatic backup interval, number of backups to keep, and the list of backups with
* back up now, restore and delete actions.
*/
public class DatabaseBackupPane extends VBox
{
private static final SimpleDateFormat TIME_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static final DecimalFormat SIZE_FORMAT = new DecimalFormat("0.0");
private final DatabasePreference mPreference;
private final DatabaseManager mDatabaseManager;
private final BackupManager mBackupManager;
private final ComboBox<BackupInterval> mIntervalComboBox = new ComboBox<>();
private final Spinner<Integer> mRetainSpinner = new Spinner<>(1, DatabasePreference.MAX_BACKUP_RETAIN,
DatabasePreference.DEFAULT_BACKUP_RETAIN);
private final ListView<BackupInfo> mBackupList = new ListView<>();
private final Button mBackupButton = new Button("Back Up Now");
private final Button mRestoreButton = new Button("Restore Selected Backup...");
private final Button mDeleteButton = new Button("Delete Selected Backup");
private final Button mRefreshButton = new Button("Refresh");
private final ProgressIndicator mProgress = new ProgressIndicator();
private final Label mResultLabel = new Label();
private final Listener<BackupInfo> mBackupListener = backup -> Platform.runLater(this::refresh);
private boolean mBusy;
/**
* Constructs an instance
* @param preference for the backup settings
* @param databaseManager for the backups, or null when the database isn't available
*/
public DatabaseBackupPane(DatabasePreference preference, DatabaseManager databaseManager)
{
super(8);
mPreference = preference;
mDatabaseManager = databaseManager;
mBackupManager = databaseManager != null ? databaseManager.getBackupManager() : null;
Label title = new Label("Backups");
title.setStyle("-fx-font-weight: bold;");
Label description = new Label("A backup holds the calls and events of the database (not the recording " +
"files). Backups can be restored into any database type. Restoring replaces the calls and events in the " +
"database in use, after making a 'pre-restore' backup of them, so a restore can be undone. When there " +
"are more backups than the number to keep, the oldest backups are deleted after each backup.");
description.setWrapText(true);
mIntervalComboBox.getItems().addAll(BackupInterval.values());
mIntervalComboBox.setValue(mPreference.getBackupInterval());
mIntervalComboBox.valueProperty().addListener((obs, old, value) -> {
if(value != null)
{
mPreference.setBackupInterval(value);
}
});
mRetainSpinner.setEditable(true);
mRetainSpinner.setPrefWidth(90);
mRetainSpinner.getValueFactory().setValue(mPreference.getBackupRetain());
mRetainSpinner.valueProperty().addListener((obs, old, value) -> {
if(value != null)
{
mPreference.setBackupRetain(value);
}
});
HBox schedule = new HBox(8, new Label("Back up automatically:"), mIntervalComboBox,
new Label("Keep the newest"), mRetainSpinner, new Label("backups"));
schedule.setAlignment(Pos.CENTER_LEFT);
Label folder = new Label("Backup folder: " + (mBackupManager != null ?
mBackupManager.getBackupDirectory().toString() : "not available"));
folder.setWrapText(true);
mBackupList.setPrefHeight(180);
mBackupList.setPlaceholder(new Label("No backups"));
mBackupList.setCellFactory(list -> new BackupCell());
mBackupList.getSelectionModel().selectedItemProperty().addListener((obs, old, selected) -> updateButtons());
mProgress.setPrefSize(18, 18);
mProgress.setVisible(false);
mResultLabel.setWrapText(true);
mBackupButton.setOnAction(event -> backupNow());
mRestoreButton.setOnAction(event -> restore());
mDeleteButton.setOnAction(event -> delete());
mRefreshButton.setOnAction(event -> refresh());
HBox buttons = new HBox(10, mBackupButton, mRestoreButton, mDeleteButton, mRefreshButton, mProgress);
buttons.setAlignment(Pos.CENTER_LEFT);
getChildren().addAll(title, description, schedule, folder, mBackupList, buttons, mResultLabel);
if(mBackupManager != null)
{
mBackupManager.addBackupListener(mBackupListener);
}
updateButtons();
refresh();
}
/**
* Deregisters from the backup manager
*/
public void dispose()
{
if(mBackupManager != null)
{
mBackupManager.removeBackupListener(mBackupListener);
}
}
/**
* Text for a backup in the list
*/
static String describe(BackupInfo backup)
{
StringBuilder sb = new StringBuilder();
synchronized(TIME_FORMAT)
{
sb.append(TIME_FORMAT.format(new Date(backup.created())));
}
if(backup.isPreRestore())
{
sb.append(" (before restore)");
}
if(backup.isValid())
{
sb.append(" ").append(backup.metadata().calls()).append(" calls, ").append(backup.metadata().events())
.append(" events ");
}
else
{
sb.append(" unreadable backup file ");
}
sb.append(backup.size() < 1024 * 1024 ? Math.max(1, backup.size() / 1024) + " KB" :
SIZE_FORMAT.format(backup.size() / (1024.0 * 1024.0)) + " MB");
if(backup.isValid() && backup.metadata().source() != null && !backup.metadata().source().isEmpty())
{
sb.append(" from ").append(backup.metadata().source());
}
return sb.toString();
}
private static class BackupCell extends ListCell<BackupInfo>
{
@Override
protected void updateItem(BackupInfo item, boolean empty)
{
super.updateItem(item, empty);
setText(empty || item == null ? null : describe(item));
}
}
private void setBusy(boolean busy)
{
mBusy = busy;
mProgress.setVisible(busy);
updateButtons();
}
private void updateButtons()
{
boolean available = mBackupManager != null && !mBusy;
BackupInfo selected = mBackupList.getSelectionModel().getSelectedItem();
mBackupButton.setDisable(!available);
mRefreshButton.setDisable(!available);
mRestoreButton.setDisable(!available || selected == null || !selected.isValid());
mDeleteButton.setDisable(!available || selected == null);
}
/**
* Reloads the list of backups
*/
void refresh()
{
if(mBackupManager == null)
{
return;
}
BackupInfo selected = mBackupList.getSelectionModel().getSelectedItem();
mBackupManager.listBackups().whenComplete((backups, error) -> Platform.runLater(() -> {
if(error != null)
{
mResultLabel.setText("Unable to list backups: " + getMessage(error));
return;
}
mBackupList.getItems().setAll(backups);
if(selected != null)
{
for(BackupInfo backup: backups)
{
if(backup.path().equals(selected.path()))
{
mBackupList.getSelectionModel().select(backup);
}
}
}
updateButtons();
}));
}
private void backupNow()
{
setBusy(true);
mResultLabel.setText("Backing up ...");
mBackupManager.backupNow().whenComplete((backup, error) -> Platform.runLater(() -> {
setBusy(false);
mResultLabel.setText(error != null ? "Backup failed: " + getMessage(error) :
"Backup created: " + backup.path().getFileName() + " (" + backup.metadata().calls() + " calls, " +
backup.metadata().events() + " events)");
refresh();
}));
}
private void restore()
{
BackupInfo backup = mBackupList.getSelectionModel().getSelectedItem();
if(backup == null || !backup.isValid())
{
return;
}
String database = mDatabaseManager.getConfiguration().getDescription();
if(!confirm("Restore Backup", "Restore the backup from " + describeTime(backup) + "?",
"All calls and events in " + database + " are replaced with the " + backup.metadata().calls() +
" calls and " + backup.metadata().events() + " events in the backup.\n\n" +
"The current calls and events are backed up first, so you can undo the restore by restoring that " +
"'before restore' backup.\n\n" +
"Recording files aren't changed. Restored calls whose recording was deleted since the backup can't " +
"be played, and calls recorded after the backup are no longer listed (their files remain on disk)."))
{
return;
}
setBusy(true);
mResultLabel.setText("Restoring ...");
mBackupManager.restore(backup, status -> Platform.runLater(() -> mResultLabel.setText(status)))
.whenComplete((result, error) -> Platform.runLater(() -> {
setBusy(false);
if(error != null)
{
mResultLabel.setText("Restore failed: " + getMessage(error) + "\nThe database wasn't changed.");
}
else
{
mResultLabel.setText("Restored " + result.calls() + " calls and " + result.events() + " events" +
(result.missingRecordings() > 0 ? " (" + result.missingRecordings() + " calls have a " +
"recording that no longer exists)" : "") + "." + (result.preRestore() != null ?
" The previous contents are in backup " + result.preRestore().path().getFileName() + "." :
""));
}
refresh();
}));
}
private void delete()
{
BackupInfo backup = mBackupList.getSelectionModel().getSelectedItem();
if(backup == null || !confirm("Delete Backup", "Delete the backup from " + describeTime(backup) + "?",
backup.path().toString()))
{
return;
}
setBusy(true);
mBackupManager.delete(backup).whenComplete((result, error) -> Platform.runLater(() -> {
setBusy(false);
mResultLabel.setText(error != null ? "Unable to delete the backup: " + getMessage(error) :
"Deleted " + backup.path().getFileName());
refresh();
}));
}
private static String describeTime(BackupInfo backup)
{
synchronized(TIME_FORMAT)
{
return TIME_FORMAT.format(new Date(backup.created()));
}
}
private boolean confirm(String title, String header, String content)
{
Alert alert = new Alert(Alert.AlertType.CONFIRMATION, content, ButtonType.OK, ButtonType.CANCEL);
Window window = getScene() != null ? getScene().getWindow() : null;
if(window != null)
{
alert.initOwner(window);
}
alert.setTitle(title);
alert.setHeaderText(header);
alert.getDialogPane().setMinWidth(560);
Optional<ButtonType> choice = alert.showAndWait();
return choice.isPresent() && choice.get() == ButtonType.OK;
}
private static String getMessage(Throwable error)
{
Throwable cause = error;
while((cause instanceof CompletionException || cause instanceof IllegalStateException) &&
cause.getCause() != null)
{
cause = cause.getCause();
}
return cause.getMessage() != null ? cause.getMessage() : cause.toString();
}
/**
* Backups shown in the list (for testing)
*/
List<BackupInfo> getBackups()
{
return mBackupList.getItems();
}
}

View File

@@ -60,7 +60,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
/**
* Preference editor for the call database selection and the call audio age-off rules.
* Preference editor for the call database selection, the age-off rules and the database backups.
*
* Applying a different database asks whether to transfer the existing calls, and when the target schema already holds
* sdrtrunk calls, whether to append to it, clear it, or create a separately named schema.
@@ -104,6 +104,7 @@ public class DatabasePreferenceEditor extends VBox
private final Spinner<Integer> mEventDaysSpinner = new Spinner<>(1, 36500, 30);
private final Button mRunAgeOffButton = new Button("Run Age-Off Now");
private final Label mAgeOffResultLabel = new Label();
private final DatabaseBackupPane mBackupPane;
/**
* Constructs an instance
@@ -118,7 +119,9 @@ public class DatabasePreferenceEditor extends VBox
setPadding(new Insets(10));
setSpacing(10);
VBox content = new VBox(10, createDatabasePane(), new Separator(), createAgeOffPane());
mBackupPane = new DatabaseBackupPane(mPreference, databaseManager);
VBox content = new VBox(10, createDatabasePane(), new Separator(), createAgeOffPane(), new Separator(),
mBackupPane);
content.setPadding(new Insets(0, 10, 10, 0));
ScrollPane scrollPane = new ScrollPane(content);
scrollPane.setFitToWidth(true);
@@ -139,6 +142,8 @@ public class DatabasePreferenceEditor extends VBox
*/
public void dispose()
{
mBackupPane.dispose();
if(mDatabaseManager != null)
{
mDatabaseManager.removeListener(mDatabaseListener);

View File

@@ -22,6 +22,7 @@ import io.github.dsheirer.database.DatabaseConfiguration;
import io.github.dsheirer.database.DatabaseType;
import io.github.dsheirer.database.ageoff.AgeOffAction;
import io.github.dsheirer.database.ageoff.RetentionPeriod;
import io.github.dsheirer.database.backup.BackupInterval;
import io.github.dsheirer.preference.Preference;
import io.github.dsheirer.preference.PreferenceType;
import io.github.dsheirer.sample.Listener;
@@ -50,11 +51,15 @@ public class DatabasePreference extends Preference
private static final String KEY_SIZE_RULE_ACTION = "ageoff.size.action";
private static final String KEY_DUPLICATE_DNM_RETENTION = "ageoff.duplicate.dnm.retention";
private static final String KEY_EVENT_RULE_ENABLED = "ageoff.events.enabled";
private static final String KEY_BACKUP_INTERVAL = "backup.interval";
private static final String KEY_BACKUP_RETAIN = "backup.retain";
private static final String KEY_EVENT_RULE_DAYS = "ageoff.events.days";
public static final String DEFAULT_POSTGRES_DATABASE = "postgres";
public static final int DEFAULT_AGE_DAYS = 30;
public static final int DEFAULT_EVENT_AGE_DAYS = 30;
public static final int DEFAULT_BACKUP_RETAIN = 7;
public static final int MAX_BACKUP_RETAIN = 365;
public static final int DEFAULT_SIZE_LIMIT_MB = 10 * 1024;
private final Preferences mPreferences = Preferences.userNodeForPackage(DatabasePreference.class);
@@ -134,6 +139,41 @@ public class DatabasePreference extends Preference
mPreferences.get(KEY_USER, ""), mPreferences.get(KEY_PASSWORD, ""));
}
/**
* How often the database is backed up automatically
*/
public BackupInterval getBackupInterval()
{
try
{
return BackupInterval.valueOf(mPreferences.get(KEY_BACKUP_INTERVAL, BackupInterval.DEFAULT.name()));
}
catch(IllegalArgumentException iae)
{
return BackupInterval.DEFAULT;
}
}
public void setBackupInterval(BackupInterval interval)
{
mPreferences.put(KEY_BACKUP_INTERVAL, interval.name());
notifyPreferenceUpdated();
}
/**
* Number of backup files to keep. The oldest backups are deleted after a backup when there are more.
*/
public int getBackupRetain()
{
return Math.clamp(mPreferences.getInt(KEY_BACKUP_RETAIN, DEFAULT_BACKUP_RETAIN), 1, MAX_BACKUP_RETAIN);
}
public void setBackupRetain(int retain)
{
mPreferences.putInt(KEY_BACKUP_RETAIN, Math.clamp(retain, 1, MAX_BACKUP_RETAIN));
notifyPreferenceUpdated();
}
/**
* Indicates if the event age rule (delete decode events older than a number of days) is enabled. Enabled by
* default, because a busy control channel produces many events.

View File

@@ -484,6 +484,11 @@ public class DirectoryPreference extends Preference
/**
* Built-in (H2) call database directory: the 'db' folder in the application root directory.
*/
public Path getDirectoryDatabaseBackup()
{
return getDirectoryDatabase().resolve("backups");
}
public Path getDirectoryDatabase()
{
return getDirectoryApplicationRoot().resolve(DIRECTORY_DATABASE);

View File

@@ -0,0 +1,400 @@
/*
* *****************************************************************************
* Copyright (C) 2014-2026 Dennis Sheirer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>
* ****************************************************************************
*/
package io.github.dsheirer.database.backup;
import io.github.dsheirer.database.CallDatabase;
import io.github.dsheirer.database.DatabaseConfiguration;
import io.github.dsheirer.database.DatabaseType;
import io.github.dsheirer.database.IdentifierRecord;
import io.github.dsheirer.database.call.CallRecord;
import io.github.dsheirer.database.call.CallSearchCriteria;
import io.github.dsheirer.database.call.CallState;
import io.github.dsheirer.database.call.IdentifierFilter;
import io.github.dsheirer.database.call.PatchedTalkgroup;
import io.github.dsheirer.database.call.RecordingState;
import io.github.dsheirer.database.event.EventRecord;
import io.github.dsheirer.database.event.EventSearchCriteria;
import io.github.dsheirer.module.decode.event.DecodeEventType;
import io.github.dsheirer.protocol.Protocol;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.Statement;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Database backup and restore: backup file contents, restore (including across database types), retention and the
* automatic backup schedule.
*
* The restore checks also run against network MariaDB and PostgreSQL servers when the SDRTRUNK_TEST_MARIADB and
* SDRTRUNK_TEST_POSTGRES environment variables are set (see CallDatabaseTest). They use (and drop) the schema
* 'sdrtrunk_backup_test'.
*/
public class BackupManagerTest
{
private static final String TEST_SCHEMA = "sdrtrunk_backup_test";
@TempDir
Path mTempDir;
private static DatabaseConfiguration h2()
{
return new DatabaseConfiguration(DatabaseType.H2, "", 0, "", TEST_SCHEMA, "", "");
}
private static DatabaseConfiguration fromEnvironment(String variable, DatabaseType type)
{
String value = System.getenv(variable);
if(value == null || value.isBlank())
{
return null;
}
String[] parts = value.split(":");
String database = parts.length > 4 ? parts[4] : "postgres";
return new DatabaseConfiguration(type, parts[0], Integer.parseInt(parts[1]), database, TEST_SCHEMA, parts[2],
parts[3]);
}
private CallDatabase open(DatabaseConfiguration configuration) throws Exception
{
CallDatabase database = new CallDatabase(configuration, mTempDir.resolve("db"));
try(Connection connection = database.getConnection(); Statement statement = connection.createStatement())
{
statement.execute(configuration.type().getDialect().getDropSchemaStatement(TEST_SCHEMA));
}
database.install();
return database;
}
private static void drop(CallDatabase database) throws Exception
{
try(Connection connection = database.getConnection(); Statement statement = connection.createStatement())
{
statement.execute(database.getConfiguration().type().getDialect().getDropSchemaStatement(TEST_SCHEMA));
}
}
private static CallRecord call(long start, long talkgroup, String path, RecordingState state, CallState callState)
{
CallRecord call = new CallRecord();
call.setStartTime(start);
call.setEndTime(callState == CallState.ACTIVE ? 0 : start + 2000);
call.setCallState(callState);
call.setProtocol(Protocol.APCO25.name());
call.setSystem("Metro");
call.setChannel("Control");
call.setFrequency(851_012_500);
call.setToId(String.valueOf(talkgroup));
call.setToValue(talkgroup);
call.setToForm(IdentifierRecord.FORM_TALKGROUP);
call.setFromId("5551212");
call.setFromValue(5551212L);
call.setFromForm(IdentifierRecord.FORM_RADIO);
call.setAliasList("Metro");
call.setTalkerAlias("Engine \"4\" ✓");
call.setEncrypted(talkgroup == 3);
call.setPriority(-1);
call.setRecordFormat("MP3");
call.setFilePath(path);
call.setFileSize(path != null ? 1234 : 0);
call.setRecordingState(state);
return call;
}
private static EventRecord event(long start, DecodeEventType type, String details)
{
EventRecord event = new EventRecord();
event.setStartTime(start);
event.setEndTime(start + 500);
event.setDuration(500);
event.setEventType(type.name());
event.setProtocol(Protocol.APCO25.name());
event.setChannelDescriptor("1-0234");
event.setTimeslot(-1);
event.setToId("P:500");
event.setToValue(500L);
event.setToForm(IdentifierRecord.FORM_PATCH_GROUP);
event.setPatchedTalkgroups(List.of(new PatchedTalkgroup(2001, null), new PatchedTalkgroup(2002, "1.2.3")));
event.setAliasList("Metro");
event.setDetails(details);
return event;
}
/**
* Fills the database: a call with an existing recording, a call whose recording was deleted, a call in progress,
* and two events with patched talkgroups.
*/
private void populate(CallDatabase database, Path recording) throws Exception
{
try(Connection connection = database.getConnection())
{
database.getCallRepository().insert(connection, call(1000, 1, recording.toString(),
RecordingState.RECORDED, CallState.COMPLETE));
database.getCallRepository().insert(connection, call(2000, 2, mTempDir.resolve("gone.mp3").toString(),
RecordingState.RECORDED, CallState.COMPLETE));
database.getCallRepository().insert(connection, call(3000, 3, null, RecordingState.NONE,
CallState.ACTIVE));
database.getEventRepository().insert(connection, event(1500, DecodeEventType.CALL_PATCH_GROUP,
"Line 1\nLine \"2\""));
database.getEventRepository().insert(connection, event(2500, DecodeEventType.TEXT_MESSAGE, null));
}
}
@Test
public void h2BackupAndRestore() throws Exception
{
backupAndRestore(h2());
}
@Test
public void mariadbBackupAndRestore() throws Exception
{
DatabaseConfiguration configuration = fromEnvironment("SDRTRUNK_TEST_MARIADB", DatabaseType.MARIADB);
Assumptions.assumeTrue(configuration != null, "SDRTRUNK_TEST_MARIADB not set");
backupAndRestore(configuration);
}
@Test
public void postgresBackupAndRestore() throws Exception
{
DatabaseConfiguration configuration = fromEnvironment("SDRTRUNK_TEST_POSTGRES", DatabaseType.POSTGRESQL);
Assumptions.assumeTrue(configuration != null, "SDRTRUNK_TEST_POSTGRES not set");
backupAndRestore(configuration);
}
private void backupAndRestore(DatabaseConfiguration configuration) throws Exception
{
Path recording = Files.writeString(mTempDir.resolve("call.mp3"), "audio");
Path backups = mTempDir.resolve("backups");
try(CallDatabase database = open(configuration))
{
populate(database, recording);
BackupInfo backup = BackupManager.writeBackup(database, backups, null, System.currentTimeMillis());
assertTrue(backup.isValid());
assertEquals(3, backup.metadata().calls());
assertEquals(2, backup.metadata().events());
assertEquals(BackupManager.FORMAT_VERSION, backup.metadata().format());
assertEquals(CallDatabase.SCHEMA_VERSION, backup.metadata().schemaVersion());
assertNull(backup.tag());
assertFalse(Files.exists(backups.resolve(backup.path().getFileName() + ".part")), "no partial file");
//Change the database after the backup
try(Connection connection = database.getConnection())
{
database.getCallRepository().insert(connection, call(9000, 9, null, RecordingState.NONE,
CallState.COMPLETE));
database.getEventRepository().deleteAll(connection);
}
BackupManager.RestoreResult result = BackupManager.restore(database, backup.path(), status -> {});
assertEquals(3, result.calls());
assertEquals(2, result.events());
assertEquals(1, result.missingRecordings());
verifyRestored(database, recording);
drop(database);
}
//The same backup restores into a different database type
if(configuration.type() != DatabaseType.H2)
{
try(CallDatabase other = new CallDatabase(h2().withSchema("sdrtrunk_backup_other"),
mTempDir.resolve("other")))
{
other.install();
BackupInfo backup = BackupManager.list(backups).getFirst();
BackupManager.restore(other, backup.path(), status -> {});
verifyRestored(other, recording);
}
}
}
private static void verifyRestored(CallDatabase database, Path recording) throws Exception
{
try(Connection connection = database.getConnection())
{
List<CallRecord> calls = database.getCallRepository().search(connection, CallSearchCriteria.all(), 0, 10);
assertEquals(3, calls.size(), "calls added after the backup are removed");
CallRecord active = calls.get(0);
CallRecord missing = calls.get(1);
CallRecord recorded = calls.get(2);
assertEquals(CallState.INTERRUPTED, active.getCallState(), "call in progress at backup is interrupted");
assertTrue(active.isEncrypted());
assertEquals(RecordingState.DELETED, missing.getRecordingState(), "missing recording marked deleted");
assertNull(missing.getFilePath());
assertEquals(RecordingState.RECORDED, recorded.getRecordingState());
assertEquals(recording.toString(), recorded.getFilePath());
assertEquals(1234, recorded.getFileSize());
assertEquals(2000, recorded.getEndTime() - recorded.getStartTime());
assertEquals("Engine \"4\" ✓", recorded.getTalkerAlias());
assertEquals(-1, recorded.getPriority());
assertEquals("Metro", recorded.getAliasList());
assertEquals(5551212L, recorded.getFromValue());
List<EventRecord> events = database.getEventRepository().search(connection, EventSearchCriteria.all(),
0, 10);
assertEquals(2, events.size());
EventRecord patch = events.get(1);
assertEquals("Line 1\nLine \"2\"", patch.getDetails());
assertEquals(500, patch.getDuration());
assertEquals("1-0234", patch.getChannelDescriptor());
assertEquals(-1, patch.getTimeslot());
assertEquals(List.of(new PatchedTalkgroup(2001, null), new PatchedTalkgroup(2002, "1.2.3")),
patch.getPatchedTalkgroups());
assertNull(events.get(0).getDetails());
assertEquals(2, database.getEventRepository().count(connection, new EventSearchCriteria(
IdentifierFilter.text("2002"), null, null, null, null, null, null, 0, 0)), "patched search works");
}
}
@Test
public void failedRestoreLeavesDatabaseUnchanged() throws Exception
{
Path recording = Files.writeString(mTempDir.resolve("call.mp3"), "audio");
Path backups = mTempDir.resolve("backups");
try(CallDatabase database = open(h2()))
{
populate(database, recording);
Files.createDirectories(backups);
//Valid metadata, but the calls entry is truncated JSON
Path broken = backups.resolve(BackupManager.getFileName(System.currentTimeMillis(), null));
writeZip(broken, "{\"format\":1,\"schema_version\":3,\"created\":1,\"calls\":2,\"events\":0}",
"[{\"start_time\":1,\"call_state\":\"COMPLETE\",\"recording_state\":\"NONE\"},{\"start_ti", "[]");
assertThrows(IOException.class, () -> BackupManager.restore(database, broken, status -> {}));
try(Connection connection = database.getConnection())
{
assertEquals(3, database.getCallRepository().count(connection), "rolled back");
assertEquals(2, database.getEventRepository().count(connection));
}
//Newer backup format is rejected before anything is changed
Path newer = backups.resolve(BackupManager.getFileName(System.currentTimeMillis() + 5000, null));
writeZip(newer, "{\"format\":99,\"schema_version\":3,\"created\":1,\"calls\":0,\"events\":0}", "[]",
"[]");
IOException error = assertThrows(IOException.class, () -> BackupManager.restore(database, newer,
status -> {}));
assertTrue(error.getMessage().contains("newer version"));
//Not a zip file: listed as unreadable
Files.writeString(backups.resolve(BackupManager.getFileName(System.currentTimeMillis() + 10_000, null)),
"junk");
List<BackupInfo> listed = BackupManager.list(backups);
assertEquals(3, listed.size());
assertFalse(listed.getFirst().isValid());
}
}
private static void writeZip(Path file, String metadata, String calls, String events) throws IOException
{
try(OutputStream out = Files.newOutputStream(file); ZipOutputStream zip = new ZipOutputStream(out))
{
for(String[] entry: new String[][]{{BackupManager.ENTRY_METADATA, metadata},
{BackupManager.ENTRY_CALLS, calls}, {BackupManager.ENTRY_EVENTS, events}})
{
zip.putNextEntry(new ZipEntry(entry[0]));
zip.write(entry[1].getBytes(StandardCharsets.UTF_8));
zip.closeEntry();
}
}
}
@Test
public void retentionKeepsNewestBackups() throws Exception
{
Path backups = mTempDir.resolve("backups");
try(CallDatabase database = open(h2()))
{
long now = System.currentTimeMillis();
for(int x = 0; x < 5; x++)
{
BackupManager.writeBackup(database, backups, x == 2 ? BackupManager.TAG_PRE_RESTORE : null,
now + x * 60_000L);
}
//Same second as an existing backup gets a unique name
BackupInfo same = BackupManager.writeBackup(database, backups, null, now + 4 * 60_000L);
assertEquals(6, BackupManager.list(backups).size());
assertEquals(now - now % 1000 + 4 * 60_000L + 1000, same.created());
Files.writeString(backups.resolve("notes.txt"), "not a backup");
List<Path> deleted = BackupManager.applyRetention(backups, 3);
assertEquals(3, deleted.size());
List<BackupInfo> remaining = BackupManager.list(backups);
assertEquals(3, remaining.size());
assertEquals(same.path(), remaining.getFirst().path(), "newest first");
assertTrue(remaining.stream().noneMatch(BackupInfo::isPreRestore), "oldest deleted, including tagged");
assertTrue(Files.exists(backups.resolve("notes.txt")), "other files are never deleted");
assertEquals(2, BackupManager.applyRetention(backups, 0).size());
assertEquals(1, BackupManager.list(backups).size(), "retention is at least 1");
}
}
@Test
public void scheduleIsDueAfterInterval()
{
long now = System.currentTimeMillis();
long day = TimeUnit.DAYS.toMillis(1);
assertTrue(BackupManager.isDue(0, BackupInterval.DAY_1, now), "no backups yet");
assertFalse(BackupManager.isDue(now - day + 60_000, BackupInterval.DAY_1, now));
assertTrue(BackupManager.isDue(now - day, BackupInterval.DAY_1, now));
assertFalse(BackupManager.isDue(0, BackupInterval.NEVER, now), "automatic backups off");
assertTrue(BackupManager.isDue(now - TimeUnit.HOURS.toMillis(7), BackupInterval.HOURS_6, now));
}
@Test
public void fileNameRoundTrip() throws Exception
{
long created = 1_700_000_000_000L;
String name = BackupManager.getFileName(created, BackupManager.TAG_PRE_RESTORE);
assertTrue(name.startsWith("sdrtrunk-backup-") && name.endsWith("-pre-restore.zip"), name);
BackupInfo info = BackupManager.read(Files.writeString(mTempDir.resolve(name), "x"));
assertNotNull(info);
assertEquals(created, info.created());
assertTrue(info.isPreRestore());
assertFalse(info.isValid());
assertNull(BackupManager.read(Files.writeString(mTempDir.resolve("sdrtrunk-backup-x.zip"), "x")));
}
}