From bb19547af0b8700b341673db35434b82e436370a Mon Sep 17 00:00:00 2001 From: denny Date: Sun, 22 Nov 2026 03:16:23 -0500 Subject: [PATCH] #2742 Database backups with scheduled backups, retention and restore - Engine-independent backup files (zip with calls, events and metadata as JSON) written from a single read transaction on a background thread; restorable into any supported database type. - Automatic backup interval and number of backups to keep in the database preferences; the oldest backups are deleted after each backup. - Backup list with Back Up Now, Restore and Delete. A restore runs in one transaction on the database thread after making a pre-restore backup; missing recordings are marked deleted and in-progress calls interrupted. Co-Authored-By: Claude Opus 5.5 --- .../dsheirer/database/DatabaseManager.java | 52 ++ .../dsheirer/database/backup/BackupCodec.java | 283 +++++++ .../dsheirer/database/backup/BackupInfo.java | 61 ++ .../database/backup/BackupInterval.java | 68 ++ .../database/backup/BackupManager.java | 792 ++++++++++++++++++ .../database/call/CallRepository.java | 10 + .../database/call/ICallRepository.java | 6 + .../database/event/EventRepository.java | 11 + .../database/event/IEventRepository.java | 6 + .../database/DatabaseBackupPane.java | 378 +++++++++ .../database/DatabasePreferenceEditor.java | 9 +- .../database/DatabasePreference.java | 40 + .../directory/DirectoryPreference.java | 5 + .../database/backup/BackupManagerTest.java | 400 +++++++++ 14 files changed, 2119 insertions(+), 2 deletions(-) create mode 100644 src/main/java/io/github/dsheirer/database/backup/BackupCodec.java create mode 100644 src/main/java/io/github/dsheirer/database/backup/BackupInfo.java create mode 100644 src/main/java/io/github/dsheirer/database/backup/BackupInterval.java create mode 100644 src/main/java/io/github/dsheirer/database/backup/BackupManager.java create mode 100644 src/main/java/io/github/dsheirer/gui/preference/database/DatabaseBackupPane.java create mode 100644 src/test/java/io/github/dsheirer/database/backup/BackupManagerTest.java diff --git a/src/main/java/io/github/dsheirer/database/DatabaseManager.java b/src/main/java/io/github/dsheirer/database/DatabaseManager.java index bb50a6dd..18d40b4d 100644 --- a/src/main/java/io/github/dsheirer/database/DatabaseManager.java +++ b/src/main/java/io/github/dsheirer/database/DatabaseManager.java @@ -19,6 +19,7 @@ package io.github.dsheirer.database; import io.github.dsheirer.database.ageoff.AgeOffManager; +import io.github.dsheirer.database.backup.BackupManager; import io.github.dsheirer.database.call.CallRecord; import io.github.dsheirer.database.call.ICallRepository; import io.github.dsheirer.database.call.CallSearchCriteria; @@ -82,6 +83,7 @@ public class DatabaseManager private final Map mPendingCalls = new LinkedHashMap<>(); private final Map mPendingEvents = new LinkedHashMap<>(); private final AgeOffManager mAgeOffManager; + private final BackupManager mBackupManager; private volatile CallDatabase mDatabase; private volatile String mErrorMessage; private volatile int mGeneration = 1; @@ -96,6 +98,7 @@ public class DatabaseManager { mUserPreferences = userPreferences; mAgeOffManager = new AgeOffManager(this, userPreferences); + mBackupManager = new BackupManager(this, userPreferences); } /** @@ -116,6 +119,7 @@ public class DatabaseManager mFlushFuture = mDatabaseExecutor.scheduleWithFixedDelay(this::flush, FLUSH_INTERVAL_MS, FLUSH_INTERVAL_MS, TimeUnit.MILLISECONDS); mAgeOffManager.start(mDatabaseExecutor); + mBackupManager.start(); } /** @@ -124,6 +128,7 @@ public class DatabaseManager public void stop() { mAgeOffManager.stop(); + mBackupManager.stop(); if(mFlushFuture != null) { @@ -918,6 +923,53 @@ public class DatabaseManager return new long[]{transferred, skipped}; } + /** + * Restores the backup into the current database, replacing its calls and events. Runs on the database thread, + * after writing the pending updates. Calls and events in progress continue as new rows. Listeners receive a + * CONNECTED event when the restore is complete. + * @param backup file to restore + * @param progress receives status text + */ + public CompletableFuture restore(Path backup, Consumer progress) + { + CompletableFuture future = new CompletableFuture<>(); + + mDatabaseExecutor.execute(() -> { + try + { + CallDatabase database = mDatabase; + + if(database == null) + { + throw new IllegalStateException("The call database is not available"); + } + + flush(); + BackupManager.RestoreResult result = BackupManager.restore(database, backup, progress); + + //Rows of calls and events in progress were replaced - write them as new rows + mGeneration++; + broadcast(DatabaseEvent.Type.CONNECTED); + future.complete(result); + } + catch(Exception e) + { + LOGGER.error("Error restoring database backup [" + backup + "]", e); + future.completeExceptionally(e); + } + }); + + return future; + } + + /** + * Backup manager + */ + public BackupManager getBackupManager() + { + return mBackupManager; + } + /** * Age-off rule manager */ diff --git a/src/main/java/io/github/dsheirer/database/backup/BackupCodec.java b/src/main/java/io/github/dsheirer/database/backup/BackupCodec.java new file mode 100644 index 00000000..7b609b1d --- /dev/null +++ b/src/main/java/io/github/dsheirer/database/backup/BackupCodec.java @@ -0,0 +1,283 @@ +/* + * ***************************************************************************** + * Copyright (C) 2014-2026 Dennis Sheirer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see + * **************************************************************************** + */ +package io.github.dsheirer.database.backup; + +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; +import io.github.dsheirer.database.IdentifierRecord; +import io.github.dsheirer.database.call.CallRecord; +import io.github.dsheirer.database.call.CallState; +import io.github.dsheirer.database.call.PatchedTalkgroup; +import io.github.dsheirer.database.call.RecordingState; +import io.github.dsheirer.database.event.EventRecord; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * JSON form of the call and event records in a backup file. Field names match the database column names, null + * values are omitted and unknown fields are ignored when reading, so that a newer backup format version can add fields. + */ +public final class BackupCodec +{ + private BackupCodec() + { + } + + public static void write(JsonWriter writer, CallRecord call) throws IOException + { + writer.beginObject(); + writeIdentifierFields(writer, call); + writer.name("call_state").value(call.getCallState().name()); + string(writer, "talker_alias", call.getTalkerAlias()); + writer.name("encrypted").value(call.isEncrypted()); + writer.name("duplicate_call").value(call.isDuplicate()); + writer.name("priority").value(call.getPriority()); + writer.name("alias_record").value(call.isAliasRecord()); + string(writer, "record_format", call.getRecordFormat()); + string(writer, "file_path", call.getFilePath()); + writer.name("file_size").value(call.getFileSize()); + writer.name("recording_state").value(call.getRecordingState().name()); + writer.endObject(); + } + + public static void write(JsonWriter writer, EventRecord event) throws IOException + { + writer.beginObject(); + writeIdentifierFields(writer, event); + writer.name("duration_ms").value(event.getDuration()); + writer.name("event_type").value(event.getEventTypeValue().name()); + string(writer, "channel_descriptor", event.getChannelDescriptor()); + string(writer, "talker_alias", event.getTalkerAlias()); + string(writer, "details", event.getDetails()); + writer.endObject(); + } + + private static void writeIdentifierFields(JsonWriter writer, IdentifierRecord record) throws IOException + { + writer.name("start_time").value(record.getStartTime()); + writer.name("end_time").value(record.getEndTime()); + string(writer, "protocol", record.getProtocol()); + string(writer, "system_name", record.getSystem()); + string(writer, "site_name", record.getSite()); + string(writer, "channel_name", record.getChannel()); + writer.name("frequency").value(record.getFrequency()); + writer.name("timeslot").value(record.getTimeslot()); + string(writer, "to_id", record.getToId()); + number(writer, "to_value", record.getToValue()); + string(writer, "to_form", record.getToForm()); + string(writer, "to_fq", record.getToFullyQualified()); + string(writer, "from_id", record.getFromId()); + number(writer, "from_value", record.getFromValue()); + string(writer, "from_form", record.getFromForm()); + string(writer, "from_fq", record.getFromFullyQualified()); + string(writer, "alias_list", record.getAliasList()); + + if(!record.getPatchedTalkgroups().isEmpty()) + { + writer.name("patched_talkgroups").beginArray(); + + for(PatchedTalkgroup talkgroup: record.getPatchedTalkgroups()) + { + writer.beginObject(); + writer.name("talkgroup").value(talkgroup.value()); + string(writer, "fq", talkgroup.fullyQualified()); + writer.endObject(); + } + + writer.endArray(); + } + } + + private static void string(JsonWriter writer, String name, String value) throws IOException + { + if(value != null) + { + writer.name(name).value(value); + } + } + + private static void number(JsonWriter writer, String name, Long value) throws IOException + { + if(value != null) + { + writer.name(name).value(value); + } + } + + /** + * Reads a call record object + */ + public static CallRecord readCall(JsonReader reader) throws IOException + { + CallRecord call = new CallRecord(); + reader.beginObject(); + + while(reader.hasNext()) + { + String name = reader.nextName(); + + if(reader.peek() == JsonToken.NULL) + { + reader.nextNull(); + continue; + } + + if(readIdentifierField(reader, name, call)) + { + continue; + } + + switch(name) + { + case "call_state" -> call.setCallState(CallState.fromValue(reader.nextString())); + case "talker_alias" -> call.setTalkerAlias(reader.nextString()); + case "encrypted" -> call.setEncrypted(reader.nextBoolean()); + case "duplicate_call" -> call.setDuplicate(reader.nextBoolean()); + case "priority" -> call.setPriority(reader.nextInt()); + case "alias_record" -> call.setAliasRecord(reader.nextBoolean()); + case "record_format" -> call.setRecordFormat(reader.nextString()); + case "file_path" -> call.setFilePath(reader.nextString()); + case "file_size" -> call.setFileSize(reader.nextLong()); + case "recording_state" -> call.setRecordingState(RecordingState.fromValue(reader.nextString())); + default -> reader.skipValue(); + } + } + + reader.endObject(); + return call; + } + + /** + * Reads an event record object + */ + public static EventRecord readEvent(JsonReader reader) throws IOException + { + EventRecord event = new EventRecord(); + reader.beginObject(); + + while(reader.hasNext()) + { + String name = reader.nextName(); + + if(reader.peek() == JsonToken.NULL) + { + reader.nextNull(); + continue; + } + + if(readIdentifierField(reader, name, event)) + { + continue; + } + + switch(name) + { + case "duration_ms" -> event.setDuration(reader.nextLong()); + case "event_type" -> event.setEventType(reader.nextString()); + case "channel_descriptor" -> event.setChannelDescriptor(reader.nextString()); + case "talker_alias" -> event.setTalkerAlias(reader.nextString()); + case "details" -> event.setDetails(reader.nextString()); + default -> reader.skipValue(); + } + } + + reader.endObject(); + return event; + } + + /** + * Reads the value of a common identifier record field. + * @return true if the field is a common field and its value was read + */ + private static boolean readIdentifierField(JsonReader reader, String name, IdentifierRecord record) + throws IOException + { + switch(name) + { + case "start_time" -> record.setStartTime(reader.nextLong()); + case "end_time" -> record.setEndTime(reader.nextLong()); + case "protocol" -> record.setProtocol(reader.nextString()); + case "system_name" -> record.setSystem(reader.nextString()); + case "site_name" -> record.setSite(reader.nextString()); + case "channel_name" -> record.setChannel(reader.nextString()); + case "frequency" -> record.setFrequency(reader.nextLong()); + case "timeslot" -> record.setTimeslot(reader.nextInt()); + case "to_id" -> record.setToId(reader.nextString()); + case "to_value" -> record.setToValue(reader.nextLong()); + case "to_form" -> record.setToForm(reader.nextString()); + case "to_fq" -> record.setToFullyQualified(reader.nextString()); + case "from_id" -> record.setFromId(reader.nextString()); + case "from_value" -> record.setFromValue(reader.nextLong()); + case "from_form" -> record.setFromForm(reader.nextString()); + case "from_fq" -> record.setFromFullyQualified(reader.nextString()); + case "alias_list" -> record.setAliasList(reader.nextString()); + case "patched_talkgroups" -> record.setPatchedTalkgroups(readPatchedTalkgroups(reader)); + default -> + { + return false; + } + } + + return true; + } + + private static List readPatchedTalkgroups(JsonReader reader) throws IOException + { + List talkgroups = new ArrayList<>(); + reader.beginArray(); + + while(reader.hasNext()) + { + long value = 0; + String fq = null; + reader.beginObject(); + + while(reader.hasNext()) + { + String name = reader.nextName(); + + if(reader.peek() == JsonToken.NULL) + { + reader.nextNull(); + } + else if(name.equals("talkgroup")) + { + value = reader.nextLong(); + } + else if(name.equals("fq")) + { + fq = reader.nextString(); + } + else + { + reader.skipValue(); + } + } + + reader.endObject(); + talkgroups.add(new PatchedTalkgroup(value, fq)); + } + + reader.endArray(); + return talkgroups; + } +} diff --git a/src/main/java/io/github/dsheirer/database/backup/BackupInfo.java b/src/main/java/io/github/dsheirer/database/backup/BackupInfo.java new file mode 100644 index 00000000..06083c96 --- /dev/null +++ b/src/main/java/io/github/dsheirer/database/backup/BackupInfo.java @@ -0,0 +1,61 @@ +/* + * ***************************************************************************** + * Copyright (C) 2014-2026 Dennis Sheirer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see + * **************************************************************************** + */ +package io.github.dsheirer.database.backup; + +import java.nio.file.Path; + +/** + * A backup file in the backup directory + * + * @param path of the backup file + * @param created time of the backup, milliseconds since epoch (from the file name) + * @param size of the file in bytes + * @param tag optional tag from the file name (e.g. pre-restore), or null + * @param metadata from the file, or null if the file can't be read + */ +public record BackupInfo(Path path, long created, long size, String tag, Metadata metadata) +{ + /** + * Backup file contents summary + * + * @param format version of the backup file format + * @param schemaVersion of the database tables the backup was made from + * @param created time of the backup, milliseconds since epoch + * @param source description of the database that was backed up + * @param calls count of calls in the backup + * @param events count of events in the backup + */ + public record Metadata(int format, int schemaVersion, long created, String source, long calls, long events) {} + + /** + * Indicates if the file is a readable backup + */ + public boolean isValid() + { + return metadata != null; + } + + /** + * Indicates if this backup was made automatically before restoring another backup + */ + public boolean isPreRestore() + { + return BackupManager.TAG_PRE_RESTORE.equals(tag); + } +} diff --git a/src/main/java/io/github/dsheirer/database/backup/BackupInterval.java b/src/main/java/io/github/dsheirer/database/backup/BackupInterval.java new file mode 100644 index 00000000..dfce8c2c --- /dev/null +++ b/src/main/java/io/github/dsheirer/database/backup/BackupInterval.java @@ -0,0 +1,68 @@ +/* + * ***************************************************************************** + * Copyright (C) 2014-2026 Dennis Sheirer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see + * **************************************************************************** + */ +package io.github.dsheirer.database.backup; + +import java.util.concurrent.TimeUnit; + +/** + * How often the call database is backed up automatically + */ +public enum BackupInterval +{ + NEVER("Never (manual only)", 0), + HOURS_6("Every 6 hours", TimeUnit.HOURS.toMillis(6)), + HOURS_12("Every 12 hours", TimeUnit.HOURS.toMillis(12)), + DAY_1("Daily", TimeUnit.DAYS.toMillis(1)), + DAYS_2("Every 2 days", TimeUnit.DAYS.toMillis(2)), + WEEK_1("Weekly", TimeUnit.DAYS.toMillis(7)), + MONTH_1("Monthly (30 days)", TimeUnit.DAYS.toMillis(30)); + + public static final BackupInterval DEFAULT = DAY_1; + + private final String mLabel; + private final long mMilliseconds; + + BackupInterval(String label, long milliseconds) + { + mLabel = label; + mMilliseconds = milliseconds; + } + + /** + * Time between automatic backups in milliseconds, or 0 when automatic backups are off + */ + public long getMilliseconds() + { + return mMilliseconds; + } + + /** + * Indicates if automatic backups are enabled + */ + public boolean isEnabled() + { + return this != NEVER; + } + + @Override + public String toString() + { + return mLabel; + } +} diff --git a/src/main/java/io/github/dsheirer/database/backup/BackupManager.java b/src/main/java/io/github/dsheirer/database/backup/BackupManager.java new file mode 100644 index 00000000..370f0210 --- /dev/null +++ b/src/main/java/io/github/dsheirer/database/backup/BackupManager.java @@ -0,0 +1,792 @@ +/* + * ***************************************************************************** + * Copyright (C) 2014-2026 Dennis Sheirer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see + * **************************************************************************** + */ +package io.github.dsheirer.database.backup; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import io.github.dsheirer.controller.NamingThreadFactory; +import io.github.dsheirer.database.CallDatabase; +import io.github.dsheirer.database.DatabaseManager; +import io.github.dsheirer.database.call.CallRecord; +import io.github.dsheirer.database.call.ICallRepository; +import io.github.dsheirer.database.call.RecordingState; +import io.github.dsheirer.database.event.EventRecord; +import io.github.dsheirer.database.event.IEventRepository; +import io.github.dsheirer.preference.UserPreferences; +import io.github.dsheirer.preference.database.DatabasePreference; +import io.github.dsheirer.sample.Broadcaster; +import io.github.dsheirer.sample.Listener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedOutputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +/** + * Database backups: scheduled and on-demand backups of the calls and events, retention of the newest backups, and + * restore of a backup into the current database. + * + * A backup is a zip file with the calls and events as JSON (see {@link BackupCodec}) and a metadata entry. The format + * doesn't depend on the database engine, so a backup can be restored into any of the supported databases. The + * recording files aren't included - restored calls refer to the recording files that still exist. + * + * Backups and restores run one at a time on the backup thread. A backup reads the database in a single read + * transaction while the database thread continues to write. A restore runs on the database thread (writes are held + * until it's done) and first makes a 'pre-restore' backup of the current contents, so it can be undone. + */ +public class BackupManager +{ + private static final Logger LOGGER = LoggerFactory.getLogger(BackupManager.class); + public static final int FORMAT_VERSION = 1; + public static final String TAG_PRE_RESTORE = "pre-restore"; + static final String FILE_PREFIX = "sdrtrunk-backup-"; + static final String FILE_EXTENSION = ".zip"; + static final String ENTRY_METADATA = "metadata.json"; + static final String ENTRY_CALLS = "calls.json"; + static final String ENTRY_EVENTS = "events.json"; + private static final Pattern FILE_PATTERN = + Pattern.compile("sdrtrunk-backup-(\\d{8}-\\d{6})(?:-([a-z][a-z0-9-]*))?\\.zip"); + private static final DateTimeFormatter FILE_TIME = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss"); + private static final long INITIAL_DELAY_SECONDS = 120; + private static final long CHECK_INTERVAL_SECONDS = 600; + private static final int PAGE_SIZE = 1000; + + private final DatabaseManager mDatabaseManager; + private final DatabasePreference mPreference; + private final Supplier mDirectory; + private final ScheduledExecutorService mExecutor = + Executors.newSingleThreadScheduledExecutor(new NamingThreadFactory("sdrtrunk database backup")); + private final Broadcaster mBackupBroadcaster = new Broadcaster<>(); + private ScheduledFuture mScheduledFuture; + + /** + * Constructs an instance + * @param databaseManager for the current database + * @param userPreferences for the backup schedule, retention and directory + */ + public BackupManager(DatabaseManager databaseManager, UserPreferences userPreferences) + { + mDatabaseManager = databaseManager; + mPreference = userPreferences.getDatabasePreference(); + mDirectory = () -> userPreferences.getDirectoryPreference().getDirectoryDatabaseBackup(); + } + + /** + * Starts the automatic backup schedule. The schedule is checked two minutes after startup and then every 10 + * minutes, and a backup is made when the newest backup is older than the backup interval. + */ + public void start() + { + if(mScheduledFuture == null) + { + mScheduledFuture = mExecutor.scheduleWithFixedDelay(this::checkSchedule, INITIAL_DELAY_SECONDS, + CHECK_INTERVAL_SECONDS, TimeUnit.SECONDS); + } + } + + /** + * Stops the schedule. A backup in progress is abandoned and its partial file removed. + */ + public void stop() + { + if(mScheduledFuture != null) + { + mScheduledFuture.cancel(false); + mScheduledFuture = null; + } + + mExecutor.shutdownNow(); + } + + /** + * Directory that holds the backup files + */ + public Path getBackupDirectory() + { + return mDirectory.get(); + } + + /** + * Registers the listener to be notified of each new backup (scheduled, manual or pre-restore). Notified on the + * backup thread. + */ + public void addBackupListener(Listener listener) + { + mBackupBroadcaster.addListener(listener); + } + + public void removeBackupListener(Listener listener) + { + mBackupBroadcaster.removeListener(listener); + } + + private void checkSchedule() + { + try + { + BackupInterval interval = mPreference.getBackupInterval(); + List backups = list(getBackupDirectory()); + long last = backups.isEmpty() ? 0 : backups.getFirst().created(); + + if(isDue(last, interval, System.currentTimeMillis()) && mDatabaseManager.isConnected()) + { + LOGGER.info("Starting scheduled database backup (" + interval + ")"); + backup(null); + } + } + catch(Exception e) + { + LOGGER.error("Scheduled database backup failed - " + e.getMessage(), e); + } + } + + /** + * Indicates if an automatic backup is due. + * @param lastBackup time of the newest backup, or 0 if there are no backups + * @param interval between automatic backups + * @param now current time + */ + static boolean isDue(long lastBackup, BackupInterval interval, long now) + { + return interval.isEnabled() && (lastBackup <= 0 || now - lastBackup >= interval.getMilliseconds()); + } + + /** + * Backs up the current database now and then deletes the oldest backups beyond the retention count. + */ + public CompletableFuture backupNow() + { + return CompletableFuture.supplyAsync(() -> { + try + { + return backup(null); + } + catch(Exception e) + { + throw new IllegalStateException(e.getMessage(), e); + } + }, mExecutor); + } + + /** + * Makes a backup and applies the retention count. Backup thread only. + */ + private BackupInfo backup(String tag) throws IOException, SQLException + { + CallDatabase database = mDatabaseManager.getDatabase(); + + if(database == null) + { + throw new IllegalStateException("The call database is not available"); + } + + BackupInfo backup = writeBackup(database, getBackupDirectory(), tag, System.currentTimeMillis()); + LOGGER.info("Database backup created [" + backup.path() + "] with " + backup.metadata().calls() + + " calls and " + backup.metadata().events() + " events"); + + if(tag == null) + { + for(Path deleted: applyRetention(getBackupDirectory(), mPreference.getBackupRetain())) + { + LOGGER.info("Deleted old database backup [" + deleted + "]"); + } + } + + mBackupBroadcaster.broadcast(backup); + return backup; + } + + /** + * Lists the backups, newest first. Runs on the backup thread. + */ + public CompletableFuture> listBackups() + { + return CompletableFuture.supplyAsync(() -> list(getBackupDirectory()), mExecutor); + } + + /** + * Deletes the backup file. Runs on the backup thread. + */ + public CompletableFuture delete(BackupInfo backup) + { + return CompletableFuture.runAsync(() -> { + try + { + Files.deleteIfExists(backup.path()); + LOGGER.info("Deleted database backup [" + backup.path() + "]"); + } + catch(IOException ioe) + { + throw new IllegalStateException(ioe.getMessage(), ioe); + } + }, mExecutor); + } + + /** + * Replaces the calls and events in the current database with the contents of the backup. A pre-restore backup + * of the current contents is made first (when the database isn't empty). + * @param backup to restore + * @param progress receives status text (on the backup or database thread) + */ + public CompletableFuture restore(BackupInfo backup, Consumer progress) + { + return CompletableFuture.supplyAsync(() -> { + try + { + BackupInfo.Metadata metadata = readMetadata(backup.path()); + checkCompatible(metadata); + CallDatabase database = mDatabaseManager.getDatabase(); + + if(database == null) + { + throw new IllegalStateException("The call database is not available"); + } + + BackupInfo preRestore = null; + + if(database.getCallCount() > 0 || database.getEventCount() > 0) + { + progress.accept("Backing up the current database before the restore"); + preRestore = backup(TAG_PRE_RESTORE); + } + + RestoreResult result = mDatabaseManager.restore(backup.path(), progress).get(); + LOGGER.info("Restored database backup [" + backup.path() + "] - " + result.calls() + " calls, " + + result.events() + " events, " + result.missingRecordings() + " missing recordings"); + return result.withPreRestore(preRestore); + } + catch(Exception e) + { + Throwable cause = e.getCause() != null && e instanceof java.util.concurrent.ExecutionException ? + e.getCause() : e; + throw new IllegalStateException(cause.getMessage(), cause); + } + }, mExecutor); + } + + /** + * Result of a restore + * @param calls restored + * @param events restored + * @param missingRecordings count of restored calls whose recording file no longer exists + * @param preRestore backup of the database contents before the restore, or null if the database was empty + */ + public record RestoreResult(long calls, long events, long missingRecordings, BackupInfo preRestore) + { + RestoreResult withPreRestore(BackupInfo backup) + { + return new RestoreResult(calls, events, missingRecordings, backup); + } + } + + /** + * Throws if the backup can't be restored by this version of sdrtrunk. + */ + static void checkCompatible(BackupInfo.Metadata metadata) throws IOException + { + if(metadata == null) + { + throw new IOException("The file is not a readable sdrtrunk database backup"); + } + + if(metadata.format() > FORMAT_VERSION || metadata.schemaVersion() > CallDatabase.SCHEMA_VERSION) + { + throw new IOException("The backup was made by a newer version of sdrtrunk (backup format " + + metadata.format() + ", schema version " + metadata.schemaVersion() + ")"); + } + } + + /** + * Writes a backup of the calls and events in the database to a new file in the directory. The file is written + * to a temporary name and renamed when complete, so an interrupted backup never leaves a partial backup file. + * @param database to back up + * @param directory for the backup file + * @param tag for the file name, or null + * @param now backup time + * @return the new backup + */ + public static BackupInfo writeBackup(CallDatabase database, Path directory, String tag, long now) + throws IOException, SQLException + { + Files.createDirectories(directory); + long created = now - (now % 1000); + Path target = directory.resolve(getFileName(created, tag)); + + //Unique name when two backups are made in the same second + while(Files.exists(target)) + { + created += 1000; + target = directory.resolve(getFileName(created, tag)); + } + + Path temp = directory.resolve(target.getFileName() + ".part"); + long calls = 0; + long events = 0; + + try(ZipOutputStream zip = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(temp))); + Connection connection = database.getConnection()) + { + boolean autoCommit = connection.getAutoCommit(); + int isolation = connection.getTransactionIsolation(); + + try + { + //One read transaction, so that the calls and events are a consistent snapshot + connection.setAutoCommit(false); + setIsolation(connection, Connection.TRANSACTION_REPEATABLE_READ); + + ICallRepository callRepository = database.getCallRepository(); + JsonWriter writer = startEntry(zip, ENTRY_CALLS); + writer.beginArray(); + List callPage = callRepository.readPage(connection, 0, PAGE_SIZE); + + while(!callPage.isEmpty()) + { + for(CallRecord call: callPage) + { + BackupCodec.write(writer, call); + } + + calls += callPage.size(); + callPage = callRepository.readPage(connection, callPage.getLast().getId(), PAGE_SIZE); + } + + writer.endArray(); + writer.flush(); + zip.closeEntry(); + + IEventRepository eventRepository = database.getEventRepository(); + writer = startEntry(zip, ENTRY_EVENTS); + writer.beginArray(); + List eventPage = eventRepository.readPage(connection, 0, PAGE_SIZE); + + while(!eventPage.isEmpty()) + { + for(EventRecord event: eventPage) + { + BackupCodec.write(writer, event); + } + + events += eventPage.size(); + eventPage = eventRepository.readPage(connection, eventPage.getLast().getId(), PAGE_SIZE); + } + + writer.endArray(); + writer.flush(); + zip.closeEntry(); + connection.rollback(); + } + finally + { + setIsolation(connection, isolation); + connection.setAutoCommit(autoCommit); + } + + BackupInfo.Metadata metadata = new BackupInfo.Metadata(FORMAT_VERSION, CallDatabase.SCHEMA_VERSION, + created, database.getConfiguration().getDescription(), calls, events); + JsonWriter writer = startEntry(zip, ENTRY_METADATA); + writer.setIndent(" "); + writer.beginObject(); + writer.name("format").value(metadata.format()); + writer.name("schema_version").value(metadata.schemaVersion()); + writer.name("created").value(metadata.created()); + writer.name("source").value(metadata.source()); + writer.name("calls").value(metadata.calls()); + writer.name("events").value(metadata.events()); + writer.endObject(); + writer.flush(); + zip.closeEntry(); + } + catch(IOException | SQLException | RuntimeException e) + { + Files.deleteIfExists(temp); + throw e; + } + + try + { + Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE); + } + catch(AtomicMoveNotSupportedException e) + { + Files.move(temp, target); + } + + return read(target); + } + + private static void setIsolation(Connection connection, int isolation) + { + try + { + connection.setTransactionIsolation(isolation); + } + catch(SQLException se) + { + LOGGER.debug("Transaction isolation level not supported - " + se.getMessage()); + } + } + + /** + * Starts a zip entry and returns a JSON writer for it. The writer must be flushed, not closed, since closing it + * would close the zip stream. + */ + private static JsonWriter startEntry(ZipOutputStream zip, String name) throws IOException + { + zip.putNextEntry(new ZipEntry(name)); + OutputStream entry = new FilterOutputStream(zip) + { + @Override + public void write(byte[] b, int off, int len) throws IOException + { + out.write(b, off, len); + } + + @Override + public void close() throws IOException + { + flush(); + } + }; + return new JsonWriter(new OutputStreamWriter(entry, StandardCharsets.UTF_8)); + } + + /** + * Backup file name for the time and tag + */ + static String getFileName(long created, String tag) + { + String time = FILE_TIME.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(created), ZoneId.systemDefault())); + return FILE_PREFIX + time + (tag != null ? "-" + tag : "") + FILE_EXTENSION; + } + + /** + * Replaces the calls and events in the database with the contents of the backup, in a single transaction, so + * that a failed restore leaves the database unchanged. Calls that were in progress when the backup was made are + * marked interrupted, and calls whose recording file no longer exists are marked as having a deleted recording. + * Invoke on the database thread only. + * @param database to restore into + * @param backup file + * @param progress receives status text + */ + public static RestoreResult restore(CallDatabase database, Path backup, Consumer progress) + throws IOException, SQLException + { + BackupInfo.Metadata metadata = readMetadata(backup); + checkCompatible(metadata); + long calls = 0; + long events = 0; + long missing = 0; + + try(ZipFile zip = new ZipFile(backup.toFile()); Connection connection = database.getConnection()) + { + ZipEntry callsEntry = zip.getEntry(ENTRY_CALLS); + ZipEntry eventsEntry = zip.getEntry(ENTRY_EVENTS); + + if(callsEntry == null || eventsEntry == null) + { + throw new IOException("The backup file is incomplete"); + } + + boolean autoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + + try + { + ICallRepository callRepository = database.getCallRepository(); + IEventRepository eventRepository = database.getEventRepository(); + progress.accept("Removing the current calls and events"); + eventRepository.deleteAll(connection); + callRepository.deleteAll(connection); + + try(InputStream in = zip.getInputStream(callsEntry); + JsonReader reader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8))) + { + List batch = new ArrayList<>(); + reader.beginArray(); + + while(reader.hasNext()) + { + CallRecord call = BackupCodec.readCall(reader); + + if(clearMissingRecording(call)) + { + missing++; + } + + batch.add(call); + + if(batch.size() >= PAGE_SIZE) + { + callRepository.insertBatch(connection, batch); + calls += batch.size(); + batch.clear(); + progress.accept("Restoring calls: " + calls + " of " + metadata.calls()); + } + } + + reader.endArray(); + callRepository.insertBatch(connection, batch); + calls += batch.size(); + } + + try(InputStream in = zip.getInputStream(eventsEntry); + JsonReader reader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8))) + { + List batch = new ArrayList<>(); + reader.beginArray(); + + while(reader.hasNext()) + { + batch.add(BackupCodec.readEvent(reader)); + + if(batch.size() >= PAGE_SIZE) + { + eventRepository.insertBatch(connection, batch); + events += batch.size(); + batch.clear(); + progress.accept("Restoring events: " + events + " of " + metadata.events()); + } + } + + reader.endArray(); + eventRepository.insertBatch(connection, batch); + events += batch.size(); + } + + callRepository.markInterrupted(connection); + progress.accept("Committing the restored calls and events"); + connection.commit(); + } + catch(IOException | SQLException | RuntimeException e) + { + connection.rollback(); + throw e; + } + finally + { + connection.setAutoCommit(autoCommit); + } + } + + return new RestoreResult(calls, events, missing, null); + } + + /** + * Marks the recording of the call deleted when the recording file no longer exists (e.g. removed by an age-off + * rule after the backup was made). + * @return true if the recording was missing + */ + static boolean clearMissingRecording(CallRecord call) + { + if(call.getFilePath() == null || (call.getRecordingState() != RecordingState.RECORDED && + call.getRecordingState() != RecordingState.RECORDING)) + { + return false; + } + + try + { + if(Files.exists(Path.of(call.getFilePath()))) + { + return false; + } + } + catch(RuntimeException re) + { + //Invalid path - treat as missing + } + + call.setRecordingState(RecordingState.DELETED); + call.setFilePath(null); + call.setFileSize(0); + return true; + } + + /** + * Backups in the directory, newest first. Files that match the backup file name but can't be read are included + * with null metadata. + */ + public static List list(Path directory) + { + List backups = new ArrayList<>(); + + if(!Files.isDirectory(directory)) + { + return backups; + } + + try(Stream files = Files.list(directory)) + { + for(Path file: files.toList()) + { + BackupInfo backup = read(file); + + if(backup != null) + { + backups.add(backup); + } + } + } + catch(IOException ioe) + { + LOGGER.warn("Unable to list database backups in [" + directory + "] - " + ioe.getMessage()); + } + + backups.sort(Comparator.comparingLong(BackupInfo::created).reversed() + .thenComparing(backup -> backup.path().getFileName().toString(), Comparator.reverseOrder())); + return backups; + } + + /** + * Backup details for the file, or null if the file name isn't a backup file name. + */ + static BackupInfo read(Path file) + { + Matcher matcher = FILE_PATTERN.matcher(file.getFileName().toString()); + + if(!matcher.matches() || !Files.isRegularFile(file)) + { + return null; + } + + long created; + + try + { + created = LocalDateTime.parse(matcher.group(1), FILE_TIME).atZone(ZoneId.systemDefault()).toInstant() + .toEpochMilli(); + } + catch(DateTimeParseException e) + { + return null; + } + + long size = 0; + + try + { + size = Files.size(file); + } + catch(IOException ioe) + { + //Leave as 0 + } + + BackupInfo.Metadata metadata = null; + + try + { + metadata = readMetadata(file); + } + catch(IOException | RuntimeException e) + { + LOGGER.warn("Unable to read database backup [" + file + "] - " + e.getMessage()); + } + + return new BackupInfo(file, created, size, matcher.group(2), metadata); + } + + /** + * Reads the metadata entry of a backup file + */ + static BackupInfo.Metadata readMetadata(Path file) throws IOException + { + try(ZipFile zip = new ZipFile(file.toFile())) + { + ZipEntry entry = zip.getEntry(ENTRY_METADATA); + + if(entry == null) + { + return null; + } + + try(InputStream in = zip.getInputStream(entry); + InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) + { + JsonObject json = JsonParser.parseReader(reader).getAsJsonObject(); + return new BackupInfo.Metadata(json.get("format").getAsInt(), json.get("schema_version").getAsInt(), + json.get("created").getAsLong(), json.has("source") ? json.get("source").getAsString() : "", + json.get("calls").getAsLong(), json.get("events").getAsLong()); + } + } + } + + /** + * Deletes the oldest backups so that no more than the retention count remain. + * @return deleted files + */ + public static List applyRetention(Path directory, int retain) + { + List deleted = new ArrayList<>(); + List backups = list(directory); + + for(int x = Math.max(1, retain); x < backups.size(); x++) + { + Path path = backups.get(x).path(); + + try + { + Files.deleteIfExists(path); + deleted.add(path); + } + catch(IOException ioe) + { + LOGGER.warn("Unable to delete old database backup [" + path + "] - " + ioe.getMessage()); + } + } + + return deleted; + } +} diff --git a/src/main/java/io/github/dsheirer/database/call/CallRepository.java b/src/main/java/io/github/dsheirer/database/call/CallRepository.java index fa95c750..b3334721 100644 --- a/src/main/java/io/github/dsheirer/database/call/CallRepository.java +++ b/src/main/java/io/github/dsheirer/database/call/CallRepository.java @@ -420,6 +420,16 @@ public class CallRepository implements ICallRepository } } + @Override + public int deleteAll(Connection connection) throws SQLException + { + try(Statement statement = connection.createStatement()) + { + statement.executeUpdate("DELETE FROM " + mPatchedTable); + return statement.executeUpdate("DELETE FROM " + mTable); + } + } + @Override public int deleteActive(Connection connection) throws SQLException { diff --git a/src/main/java/io/github/dsheirer/database/call/ICallRepository.java b/src/main/java/io/github/dsheirer/database/call/ICallRepository.java index 0fff8eb6..02c0e447 100644 --- a/src/main/java/io/github/dsheirer/database/call/ICallRepository.java +++ b/src/main/java/io/github/dsheirer/database/call/ICallRepository.java @@ -59,6 +59,12 @@ public interface ICallRepository */ int deleteActive(Connection connection) throws SQLException; + /** + * Deletes all calls (and their patched talkgroups), for restoring a backup. + * @return count of calls deleted + */ + int deleteAll(Connection connection) throws SQLException; + /** * Calls matching the criteria, newest first. */ diff --git a/src/main/java/io/github/dsheirer/database/event/EventRepository.java b/src/main/java/io/github/dsheirer/database/event/EventRepository.java index eed91664..f42f5037 100644 --- a/src/main/java/io/github/dsheirer/database/event/EventRepository.java +++ b/src/main/java/io/github/dsheirer/database/event/EventRepository.java @@ -519,6 +519,17 @@ public class EventRepository implements IEventRepository } } + @Override + public int deleteAll(Connection connection) throws SQLException + { + try(PreparedStatement patched = connection.prepareStatement("DELETE FROM " + mPatchedTable); + PreparedStatement events = connection.prepareStatement("DELETE FROM " + mTable)) + { + patched.executeUpdate(); + return events.executeUpdate(); + } + } + @Override public String getSchema() { diff --git a/src/main/java/io/github/dsheirer/database/event/IEventRepository.java b/src/main/java/io/github/dsheirer/database/event/IEventRepository.java index a875738e..0367a9cd 100644 --- a/src/main/java/io/github/dsheirer/database/event/IEventRepository.java +++ b/src/main/java/io/github/dsheirer/database/event/IEventRepository.java @@ -83,6 +83,12 @@ public interface IEventRepository */ int deleteStartedBefore(Connection connection, long startedBefore) throws SQLException; + /** + * Deletes all events (and their patched talkgroups), for restoring a backup. + * @return count of events deleted + */ + int deleteAll(Connection connection) throws SQLException; + /** * Schema name */ diff --git a/src/main/java/io/github/dsheirer/gui/preference/database/DatabaseBackupPane.java b/src/main/java/io/github/dsheirer/gui/preference/database/DatabaseBackupPane.java new file mode 100644 index 00000000..6be3adb0 --- /dev/null +++ b/src/main/java/io/github/dsheirer/gui/preference/database/DatabaseBackupPane.java @@ -0,0 +1,378 @@ +/* + * ***************************************************************************** + * Copyright (C) 2014-2026 Dennis Sheirer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see + * **************************************************************************** + */ +package io.github.dsheirer.gui.preference.database; + +import io.github.dsheirer.database.DatabaseManager; +import io.github.dsheirer.database.backup.BackupInfo; +import io.github.dsheirer.database.backup.BackupInterval; +import io.github.dsheirer.database.backup.BackupManager; +import io.github.dsheirer.preference.database.DatabasePreference; +import io.github.dsheirer.sample.Listener; +import javafx.application.Platform; +import javafx.scene.control.Alert; +import javafx.scene.control.Button; +import javafx.scene.control.ButtonType; +import javafx.scene.control.ComboBox; +import javafx.scene.control.Label; +import javafx.scene.control.ListCell; +import javafx.scene.control.ListView; +import javafx.scene.control.ProgressIndicator; +import javafx.scene.control.Spinner; +import javafx.scene.layout.HBox; +import javafx.scene.layout.VBox; +import javafx.geometry.Pos; +import javafx.stage.Window; + +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletionException; + +/** + * Database backup preferences: automatic backup interval, number of backups to keep, and the list of backups with + * back up now, restore and delete actions. + */ +public class DatabaseBackupPane extends VBox +{ + private static final SimpleDateFormat TIME_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static final DecimalFormat SIZE_FORMAT = new DecimalFormat("0.0"); + private final DatabasePreference mPreference; + private final DatabaseManager mDatabaseManager; + private final BackupManager mBackupManager; + private final ComboBox mIntervalComboBox = new ComboBox<>(); + private final Spinner mRetainSpinner = new Spinner<>(1, DatabasePreference.MAX_BACKUP_RETAIN, + DatabasePreference.DEFAULT_BACKUP_RETAIN); + private final ListView mBackupList = new ListView<>(); + private final Button mBackupButton = new Button("Back Up Now"); + private final Button mRestoreButton = new Button("Restore Selected Backup..."); + private final Button mDeleteButton = new Button("Delete Selected Backup"); + private final Button mRefreshButton = new Button("Refresh"); + private final ProgressIndicator mProgress = new ProgressIndicator(); + private final Label mResultLabel = new Label(); + private final Listener mBackupListener = backup -> Platform.runLater(this::refresh); + private boolean mBusy; + + /** + * Constructs an instance + * @param preference for the backup settings + * @param databaseManager for the backups, or null when the database isn't available + */ + public DatabaseBackupPane(DatabasePreference preference, DatabaseManager databaseManager) + { + super(8); + mPreference = preference; + mDatabaseManager = databaseManager; + mBackupManager = databaseManager != null ? databaseManager.getBackupManager() : null; + + Label title = new Label("Backups"); + title.setStyle("-fx-font-weight: bold;"); + Label description = new Label("A backup holds the calls and events of the database (not the recording " + + "files). Backups can be restored into any database type. Restoring replaces the calls and events in the " + + "database in use, after making a 'pre-restore' backup of them, so a restore can be undone. When there " + + "are more backups than the number to keep, the oldest backups are deleted after each backup."); + description.setWrapText(true); + + mIntervalComboBox.getItems().addAll(BackupInterval.values()); + mIntervalComboBox.setValue(mPreference.getBackupInterval()); + mIntervalComboBox.valueProperty().addListener((obs, old, value) -> { + if(value != null) + { + mPreference.setBackupInterval(value); + } + }); + mRetainSpinner.setEditable(true); + mRetainSpinner.setPrefWidth(90); + mRetainSpinner.getValueFactory().setValue(mPreference.getBackupRetain()); + mRetainSpinner.valueProperty().addListener((obs, old, value) -> { + if(value != null) + { + mPreference.setBackupRetain(value); + } + }); + + HBox schedule = new HBox(8, new Label("Back up automatically:"), mIntervalComboBox, + new Label("Keep the newest"), mRetainSpinner, new Label("backups")); + schedule.setAlignment(Pos.CENTER_LEFT); + + Label folder = new Label("Backup folder: " + (mBackupManager != null ? + mBackupManager.getBackupDirectory().toString() : "not available")); + folder.setWrapText(true); + + mBackupList.setPrefHeight(180); + mBackupList.setPlaceholder(new Label("No backups")); + mBackupList.setCellFactory(list -> new BackupCell()); + mBackupList.getSelectionModel().selectedItemProperty().addListener((obs, old, selected) -> updateButtons()); + + mProgress.setPrefSize(18, 18); + mProgress.setVisible(false); + mResultLabel.setWrapText(true); + mBackupButton.setOnAction(event -> backupNow()); + mRestoreButton.setOnAction(event -> restore()); + mDeleteButton.setOnAction(event -> delete()); + mRefreshButton.setOnAction(event -> refresh()); + HBox buttons = new HBox(10, mBackupButton, mRestoreButton, mDeleteButton, mRefreshButton, mProgress); + buttons.setAlignment(Pos.CENTER_LEFT); + + getChildren().addAll(title, description, schedule, folder, mBackupList, buttons, mResultLabel); + + if(mBackupManager != null) + { + mBackupManager.addBackupListener(mBackupListener); + } + + updateButtons(); + refresh(); + } + + /** + * Deregisters from the backup manager + */ + public void dispose() + { + if(mBackupManager != null) + { + mBackupManager.removeBackupListener(mBackupListener); + } + } + + /** + * Text for a backup in the list + */ + static String describe(BackupInfo backup) + { + StringBuilder sb = new StringBuilder(); + + synchronized(TIME_FORMAT) + { + sb.append(TIME_FORMAT.format(new Date(backup.created()))); + } + + if(backup.isPreRestore()) + { + sb.append(" (before restore)"); + } + + if(backup.isValid()) + { + sb.append(" ").append(backup.metadata().calls()).append(" calls, ").append(backup.metadata().events()) + .append(" events "); + } + else + { + sb.append(" unreadable backup file "); + } + + sb.append(backup.size() < 1024 * 1024 ? Math.max(1, backup.size() / 1024) + " KB" : + SIZE_FORMAT.format(backup.size() / (1024.0 * 1024.0)) + " MB"); + + if(backup.isValid() && backup.metadata().source() != null && !backup.metadata().source().isEmpty()) + { + sb.append(" from ").append(backup.metadata().source()); + } + + return sb.toString(); + } + + private static class BackupCell extends ListCell + { + @Override + protected void updateItem(BackupInfo item, boolean empty) + { + super.updateItem(item, empty); + setText(empty || item == null ? null : describe(item)); + } + } + + private void setBusy(boolean busy) + { + mBusy = busy; + mProgress.setVisible(busy); + updateButtons(); + } + + private void updateButtons() + { + boolean available = mBackupManager != null && !mBusy; + BackupInfo selected = mBackupList.getSelectionModel().getSelectedItem(); + mBackupButton.setDisable(!available); + mRefreshButton.setDisable(!available); + mRestoreButton.setDisable(!available || selected == null || !selected.isValid()); + mDeleteButton.setDisable(!available || selected == null); + } + + /** + * Reloads the list of backups + */ + void refresh() + { + if(mBackupManager == null) + { + return; + } + + BackupInfo selected = mBackupList.getSelectionModel().getSelectedItem(); + mBackupManager.listBackups().whenComplete((backups, error) -> Platform.runLater(() -> { + if(error != null) + { + mResultLabel.setText("Unable to list backups: " + getMessage(error)); + return; + } + + mBackupList.getItems().setAll(backups); + + if(selected != null) + { + for(BackupInfo backup: backups) + { + if(backup.path().equals(selected.path())) + { + mBackupList.getSelectionModel().select(backup); + } + } + } + + updateButtons(); + })); + } + + private void backupNow() + { + setBusy(true); + mResultLabel.setText("Backing up ..."); + mBackupManager.backupNow().whenComplete((backup, error) -> Platform.runLater(() -> { + setBusy(false); + mResultLabel.setText(error != null ? "Backup failed: " + getMessage(error) : + "Backup created: " + backup.path().getFileName() + " (" + backup.metadata().calls() + " calls, " + + backup.metadata().events() + " events)"); + refresh(); + })); + } + + private void restore() + { + BackupInfo backup = mBackupList.getSelectionModel().getSelectedItem(); + + if(backup == null || !backup.isValid()) + { + return; + } + + String database = mDatabaseManager.getConfiguration().getDescription(); + + if(!confirm("Restore Backup", "Restore the backup from " + describeTime(backup) + "?", + "All calls and events in " + database + " are replaced with the " + backup.metadata().calls() + + " calls and " + backup.metadata().events() + " events in the backup.\n\n" + + "The current calls and events are backed up first, so you can undo the restore by restoring that " + + "'before restore' backup.\n\n" + + "Recording files aren't changed. Restored calls whose recording was deleted since the backup can't " + + "be played, and calls recorded after the backup are no longer listed (their files remain on disk).")) + { + return; + } + + setBusy(true); + mResultLabel.setText("Restoring ..."); + mBackupManager.restore(backup, status -> Platform.runLater(() -> mResultLabel.setText(status))) + .whenComplete((result, error) -> Platform.runLater(() -> { + setBusy(false); + + if(error != null) + { + mResultLabel.setText("Restore failed: " + getMessage(error) + "\nThe database wasn't changed."); + } + else + { + mResultLabel.setText("Restored " + result.calls() + " calls and " + result.events() + " events" + + (result.missingRecordings() > 0 ? " (" + result.missingRecordings() + " calls have a " + + "recording that no longer exists)" : "") + "." + (result.preRestore() != null ? + " The previous contents are in backup " + result.preRestore().path().getFileName() + "." : + "")); + } + + refresh(); + })); + } + + private void delete() + { + BackupInfo backup = mBackupList.getSelectionModel().getSelectedItem(); + + if(backup == null || !confirm("Delete Backup", "Delete the backup from " + describeTime(backup) + "?", + backup.path().toString())) + { + return; + } + + setBusy(true); + mBackupManager.delete(backup).whenComplete((result, error) -> Platform.runLater(() -> { + setBusy(false); + mResultLabel.setText(error != null ? "Unable to delete the backup: " + getMessage(error) : + "Deleted " + backup.path().getFileName()); + refresh(); + })); + } + + private static String describeTime(BackupInfo backup) + { + synchronized(TIME_FORMAT) + { + return TIME_FORMAT.format(new Date(backup.created())); + } + } + + private boolean confirm(String title, String header, String content) + { + Alert alert = new Alert(Alert.AlertType.CONFIRMATION, content, ButtonType.OK, ButtonType.CANCEL); + Window window = getScene() != null ? getScene().getWindow() : null; + + if(window != null) + { + alert.initOwner(window); + } + + alert.setTitle(title); + alert.setHeaderText(header); + alert.getDialogPane().setMinWidth(560); + Optional choice = alert.showAndWait(); + return choice.isPresent() && choice.get() == ButtonType.OK; + } + + private static String getMessage(Throwable error) + { + Throwable cause = error; + + while((cause instanceof CompletionException || cause instanceof IllegalStateException) && + cause.getCause() != null) + { + cause = cause.getCause(); + } + + return cause.getMessage() != null ? cause.getMessage() : cause.toString(); + } + + /** + * Backups shown in the list (for testing) + */ + List getBackups() + { + return mBackupList.getItems(); + } +} diff --git a/src/main/java/io/github/dsheirer/gui/preference/database/DatabasePreferenceEditor.java b/src/main/java/io/github/dsheirer/gui/preference/database/DatabasePreferenceEditor.java index 19475194..c5453e37 100644 --- a/src/main/java/io/github/dsheirer/gui/preference/database/DatabasePreferenceEditor.java +++ b/src/main/java/io/github/dsheirer/gui/preference/database/DatabasePreferenceEditor.java @@ -60,7 +60,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; /** - * Preference editor for the call database selection and the call audio age-off rules. + * Preference editor for the call database selection, the age-off rules and the database backups. * * Applying a different database asks whether to transfer the existing calls, and when the target schema already holds * sdrtrunk calls, whether to append to it, clear it, or create a separately named schema. @@ -104,6 +104,7 @@ public class DatabasePreferenceEditor extends VBox private final Spinner mEventDaysSpinner = new Spinner<>(1, 36500, 30); private final Button mRunAgeOffButton = new Button("Run Age-Off Now"); private final Label mAgeOffResultLabel = new Label(); + private final DatabaseBackupPane mBackupPane; /** * Constructs an instance @@ -118,7 +119,9 @@ public class DatabasePreferenceEditor extends VBox setPadding(new Insets(10)); setSpacing(10); - VBox content = new VBox(10, createDatabasePane(), new Separator(), createAgeOffPane()); + mBackupPane = new DatabaseBackupPane(mPreference, databaseManager); + VBox content = new VBox(10, createDatabasePane(), new Separator(), createAgeOffPane(), new Separator(), + mBackupPane); content.setPadding(new Insets(0, 10, 10, 0)); ScrollPane scrollPane = new ScrollPane(content); scrollPane.setFitToWidth(true); @@ -139,6 +142,8 @@ public class DatabasePreferenceEditor extends VBox */ public void dispose() { + mBackupPane.dispose(); + if(mDatabaseManager != null) { mDatabaseManager.removeListener(mDatabaseListener); diff --git a/src/main/java/io/github/dsheirer/preference/database/DatabasePreference.java b/src/main/java/io/github/dsheirer/preference/database/DatabasePreference.java index 65ce3a0d..be1c4038 100644 --- a/src/main/java/io/github/dsheirer/preference/database/DatabasePreference.java +++ b/src/main/java/io/github/dsheirer/preference/database/DatabasePreference.java @@ -22,6 +22,7 @@ import io.github.dsheirer.database.DatabaseConfiguration; import io.github.dsheirer.database.DatabaseType; import io.github.dsheirer.database.ageoff.AgeOffAction; import io.github.dsheirer.database.ageoff.RetentionPeriod; +import io.github.dsheirer.database.backup.BackupInterval; import io.github.dsheirer.preference.Preference; import io.github.dsheirer.preference.PreferenceType; import io.github.dsheirer.sample.Listener; @@ -50,11 +51,15 @@ public class DatabasePreference extends Preference private static final String KEY_SIZE_RULE_ACTION = "ageoff.size.action"; private static final String KEY_DUPLICATE_DNM_RETENTION = "ageoff.duplicate.dnm.retention"; private static final String KEY_EVENT_RULE_ENABLED = "ageoff.events.enabled"; + private static final String KEY_BACKUP_INTERVAL = "backup.interval"; + private static final String KEY_BACKUP_RETAIN = "backup.retain"; private static final String KEY_EVENT_RULE_DAYS = "ageoff.events.days"; public static final String DEFAULT_POSTGRES_DATABASE = "postgres"; public static final int DEFAULT_AGE_DAYS = 30; public static final int DEFAULT_EVENT_AGE_DAYS = 30; + public static final int DEFAULT_BACKUP_RETAIN = 7; + public static final int MAX_BACKUP_RETAIN = 365; public static final int DEFAULT_SIZE_LIMIT_MB = 10 * 1024; private final Preferences mPreferences = Preferences.userNodeForPackage(DatabasePreference.class); @@ -134,6 +139,41 @@ public class DatabasePreference extends Preference mPreferences.get(KEY_USER, ""), mPreferences.get(KEY_PASSWORD, "")); } + /** + * How often the database is backed up automatically + */ + public BackupInterval getBackupInterval() + { + try + { + return BackupInterval.valueOf(mPreferences.get(KEY_BACKUP_INTERVAL, BackupInterval.DEFAULT.name())); + } + catch(IllegalArgumentException iae) + { + return BackupInterval.DEFAULT; + } + } + + public void setBackupInterval(BackupInterval interval) + { + mPreferences.put(KEY_BACKUP_INTERVAL, interval.name()); + notifyPreferenceUpdated(); + } + + /** + * Number of backup files to keep. The oldest backups are deleted after a backup when there are more. + */ + public int getBackupRetain() + { + return Math.clamp(mPreferences.getInt(KEY_BACKUP_RETAIN, DEFAULT_BACKUP_RETAIN), 1, MAX_BACKUP_RETAIN); + } + + public void setBackupRetain(int retain) + { + mPreferences.putInt(KEY_BACKUP_RETAIN, Math.clamp(retain, 1, MAX_BACKUP_RETAIN)); + notifyPreferenceUpdated(); + } + /** * Indicates if the event age rule (delete decode events older than a number of days) is enabled. Enabled by * default, because a busy control channel produces many events. diff --git a/src/main/java/io/github/dsheirer/preference/directory/DirectoryPreference.java b/src/main/java/io/github/dsheirer/preference/directory/DirectoryPreference.java index 045f0119..c78945f6 100644 --- a/src/main/java/io/github/dsheirer/preference/directory/DirectoryPreference.java +++ b/src/main/java/io/github/dsheirer/preference/directory/DirectoryPreference.java @@ -484,6 +484,11 @@ public class DirectoryPreference extends Preference /** * Built-in (H2) call database directory: the 'db' folder in the application root directory. */ + public Path getDirectoryDatabaseBackup() + { + return getDirectoryDatabase().resolve("backups"); + } + public Path getDirectoryDatabase() { return getDirectoryApplicationRoot().resolve(DIRECTORY_DATABASE); diff --git a/src/test/java/io/github/dsheirer/database/backup/BackupManagerTest.java b/src/test/java/io/github/dsheirer/database/backup/BackupManagerTest.java new file mode 100644 index 00000000..57f79d85 --- /dev/null +++ b/src/test/java/io/github/dsheirer/database/backup/BackupManagerTest.java @@ -0,0 +1,400 @@ +/* + * ***************************************************************************** + * Copyright (C) 2014-2026 Dennis Sheirer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see + * **************************************************************************** + */ +package io.github.dsheirer.database.backup; + +import io.github.dsheirer.database.CallDatabase; +import io.github.dsheirer.database.DatabaseConfiguration; +import io.github.dsheirer.database.DatabaseType; +import io.github.dsheirer.database.IdentifierRecord; +import io.github.dsheirer.database.call.CallRecord; +import io.github.dsheirer.database.call.CallSearchCriteria; +import io.github.dsheirer.database.call.CallState; +import io.github.dsheirer.database.call.IdentifierFilter; +import io.github.dsheirer.database.call.PatchedTalkgroup; +import io.github.dsheirer.database.call.RecordingState; +import io.github.dsheirer.database.event.EventRecord; +import io.github.dsheirer.database.event.EventSearchCriteria; +import io.github.dsheirer.module.decode.event.DecodeEventType; +import io.github.dsheirer.protocol.Protocol; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Statement; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Database backup and restore: backup file contents, restore (including across database types), retention and the + * automatic backup schedule. + * + * The restore checks also run against network MariaDB and PostgreSQL servers when the SDRTRUNK_TEST_MARIADB and + * SDRTRUNK_TEST_POSTGRES environment variables are set (see CallDatabaseTest). They use (and drop) the schema + * 'sdrtrunk_backup_test'. + */ +public class BackupManagerTest +{ + private static final String TEST_SCHEMA = "sdrtrunk_backup_test"; + + @TempDir + Path mTempDir; + + private static DatabaseConfiguration h2() + { + return new DatabaseConfiguration(DatabaseType.H2, "", 0, "", TEST_SCHEMA, "", ""); + } + + private static DatabaseConfiguration fromEnvironment(String variable, DatabaseType type) + { + String value = System.getenv(variable); + + if(value == null || value.isBlank()) + { + return null; + } + + String[] parts = value.split(":"); + String database = parts.length > 4 ? parts[4] : "postgres"; + return new DatabaseConfiguration(type, parts[0], Integer.parseInt(parts[1]), database, TEST_SCHEMA, parts[2], + parts[3]); + } + + private CallDatabase open(DatabaseConfiguration configuration) throws Exception + { + CallDatabase database = new CallDatabase(configuration, mTempDir.resolve("db")); + + try(Connection connection = database.getConnection(); Statement statement = connection.createStatement()) + { + statement.execute(configuration.type().getDialect().getDropSchemaStatement(TEST_SCHEMA)); + } + + database.install(); + return database; + } + + private static void drop(CallDatabase database) throws Exception + { + try(Connection connection = database.getConnection(); Statement statement = connection.createStatement()) + { + statement.execute(database.getConfiguration().type().getDialect().getDropSchemaStatement(TEST_SCHEMA)); + } + } + + private static CallRecord call(long start, long talkgroup, String path, RecordingState state, CallState callState) + { + CallRecord call = new CallRecord(); + call.setStartTime(start); + call.setEndTime(callState == CallState.ACTIVE ? 0 : start + 2000); + call.setCallState(callState); + call.setProtocol(Protocol.APCO25.name()); + call.setSystem("Metro"); + call.setChannel("Control"); + call.setFrequency(851_012_500); + call.setToId(String.valueOf(talkgroup)); + call.setToValue(talkgroup); + call.setToForm(IdentifierRecord.FORM_TALKGROUP); + call.setFromId("5551212"); + call.setFromValue(5551212L); + call.setFromForm(IdentifierRecord.FORM_RADIO); + call.setAliasList("Metro"); + call.setTalkerAlias("Engine \"4\" ✓"); + call.setEncrypted(talkgroup == 3); + call.setPriority(-1); + call.setRecordFormat("MP3"); + call.setFilePath(path); + call.setFileSize(path != null ? 1234 : 0); + call.setRecordingState(state); + return call; + } + + private static EventRecord event(long start, DecodeEventType type, String details) + { + EventRecord event = new EventRecord(); + event.setStartTime(start); + event.setEndTime(start + 500); + event.setDuration(500); + event.setEventType(type.name()); + event.setProtocol(Protocol.APCO25.name()); + event.setChannelDescriptor("1-0234"); + event.setTimeslot(-1); + event.setToId("P:500"); + event.setToValue(500L); + event.setToForm(IdentifierRecord.FORM_PATCH_GROUP); + event.setPatchedTalkgroups(List.of(new PatchedTalkgroup(2001, null), new PatchedTalkgroup(2002, "1.2.3"))); + event.setAliasList("Metro"); + event.setDetails(details); + return event; + } + + /** + * Fills the database: a call with an existing recording, a call whose recording was deleted, a call in progress, + * and two events with patched talkgroups. + */ + private void populate(CallDatabase database, Path recording) throws Exception + { + try(Connection connection = database.getConnection()) + { + database.getCallRepository().insert(connection, call(1000, 1, recording.toString(), + RecordingState.RECORDED, CallState.COMPLETE)); + database.getCallRepository().insert(connection, call(2000, 2, mTempDir.resolve("gone.mp3").toString(), + RecordingState.RECORDED, CallState.COMPLETE)); + database.getCallRepository().insert(connection, call(3000, 3, null, RecordingState.NONE, + CallState.ACTIVE)); + database.getEventRepository().insert(connection, event(1500, DecodeEventType.CALL_PATCH_GROUP, + "Line 1\nLine \"2\"")); + database.getEventRepository().insert(connection, event(2500, DecodeEventType.TEXT_MESSAGE, null)); + } + } + + @Test + public void h2BackupAndRestore() throws Exception + { + backupAndRestore(h2()); + } + + @Test + public void mariadbBackupAndRestore() throws Exception + { + DatabaseConfiguration configuration = fromEnvironment("SDRTRUNK_TEST_MARIADB", DatabaseType.MARIADB); + Assumptions.assumeTrue(configuration != null, "SDRTRUNK_TEST_MARIADB not set"); + backupAndRestore(configuration); + } + + @Test + public void postgresBackupAndRestore() throws Exception + { + DatabaseConfiguration configuration = fromEnvironment("SDRTRUNK_TEST_POSTGRES", DatabaseType.POSTGRESQL); + Assumptions.assumeTrue(configuration != null, "SDRTRUNK_TEST_POSTGRES not set"); + backupAndRestore(configuration); + } + + private void backupAndRestore(DatabaseConfiguration configuration) throws Exception + { + Path recording = Files.writeString(mTempDir.resolve("call.mp3"), "audio"); + Path backups = mTempDir.resolve("backups"); + + try(CallDatabase database = open(configuration)) + { + populate(database, recording); + BackupInfo backup = BackupManager.writeBackup(database, backups, null, System.currentTimeMillis()); + assertTrue(backup.isValid()); + assertEquals(3, backup.metadata().calls()); + assertEquals(2, backup.metadata().events()); + assertEquals(BackupManager.FORMAT_VERSION, backup.metadata().format()); + assertEquals(CallDatabase.SCHEMA_VERSION, backup.metadata().schemaVersion()); + assertNull(backup.tag()); + assertFalse(Files.exists(backups.resolve(backup.path().getFileName() + ".part")), "no partial file"); + + //Change the database after the backup + try(Connection connection = database.getConnection()) + { + database.getCallRepository().insert(connection, call(9000, 9, null, RecordingState.NONE, + CallState.COMPLETE)); + database.getEventRepository().deleteAll(connection); + } + + BackupManager.RestoreResult result = BackupManager.restore(database, backup.path(), status -> {}); + assertEquals(3, result.calls()); + assertEquals(2, result.events()); + assertEquals(1, result.missingRecordings()); + verifyRestored(database, recording); + drop(database); + } + + //The same backup restores into a different database type + if(configuration.type() != DatabaseType.H2) + { + try(CallDatabase other = new CallDatabase(h2().withSchema("sdrtrunk_backup_other"), + mTempDir.resolve("other"))) + { + other.install(); + BackupInfo backup = BackupManager.list(backups).getFirst(); + BackupManager.restore(other, backup.path(), status -> {}); + verifyRestored(other, recording); + } + } + } + + private static void verifyRestored(CallDatabase database, Path recording) throws Exception + { + try(Connection connection = database.getConnection()) + { + List calls = database.getCallRepository().search(connection, CallSearchCriteria.all(), 0, 10); + assertEquals(3, calls.size(), "calls added after the backup are removed"); + CallRecord active = calls.get(0); + CallRecord missing = calls.get(1); + CallRecord recorded = calls.get(2); + assertEquals(CallState.INTERRUPTED, active.getCallState(), "call in progress at backup is interrupted"); + assertTrue(active.isEncrypted()); + assertEquals(RecordingState.DELETED, missing.getRecordingState(), "missing recording marked deleted"); + assertNull(missing.getFilePath()); + assertEquals(RecordingState.RECORDED, recorded.getRecordingState()); + assertEquals(recording.toString(), recorded.getFilePath()); + assertEquals(1234, recorded.getFileSize()); + assertEquals(2000, recorded.getEndTime() - recorded.getStartTime()); + assertEquals("Engine \"4\" ✓", recorded.getTalkerAlias()); + assertEquals(-1, recorded.getPriority()); + assertEquals("Metro", recorded.getAliasList()); + assertEquals(5551212L, recorded.getFromValue()); + + List events = database.getEventRepository().search(connection, EventSearchCriteria.all(), + 0, 10); + assertEquals(2, events.size()); + EventRecord patch = events.get(1); + assertEquals("Line 1\nLine \"2\"", patch.getDetails()); + assertEquals(500, patch.getDuration()); + assertEquals("1-0234", patch.getChannelDescriptor()); + assertEquals(-1, patch.getTimeslot()); + assertEquals(List.of(new PatchedTalkgroup(2001, null), new PatchedTalkgroup(2002, "1.2.3")), + patch.getPatchedTalkgroups()); + assertNull(events.get(0).getDetails()); + assertEquals(2, database.getEventRepository().count(connection, new EventSearchCriteria( + IdentifierFilter.text("2002"), null, null, null, null, null, null, 0, 0)), "patched search works"); + } + } + + @Test + public void failedRestoreLeavesDatabaseUnchanged() throws Exception + { + Path recording = Files.writeString(mTempDir.resolve("call.mp3"), "audio"); + Path backups = mTempDir.resolve("backups"); + + try(CallDatabase database = open(h2())) + { + populate(database, recording); + Files.createDirectories(backups); + + //Valid metadata, but the calls entry is truncated JSON + Path broken = backups.resolve(BackupManager.getFileName(System.currentTimeMillis(), null)); + writeZip(broken, "{\"format\":1,\"schema_version\":3,\"created\":1,\"calls\":2,\"events\":0}", + "[{\"start_time\":1,\"call_state\":\"COMPLETE\",\"recording_state\":\"NONE\"},{\"start_ti", "[]"); + assertThrows(IOException.class, () -> BackupManager.restore(database, broken, status -> {})); + + try(Connection connection = database.getConnection()) + { + assertEquals(3, database.getCallRepository().count(connection), "rolled back"); + assertEquals(2, database.getEventRepository().count(connection)); + } + + //Newer backup format is rejected before anything is changed + Path newer = backups.resolve(BackupManager.getFileName(System.currentTimeMillis() + 5000, null)); + writeZip(newer, "{\"format\":99,\"schema_version\":3,\"created\":1,\"calls\":0,\"events\":0}", "[]", + "[]"); + IOException error = assertThrows(IOException.class, () -> BackupManager.restore(database, newer, + status -> {})); + assertTrue(error.getMessage().contains("newer version")); + + //Not a zip file: listed as unreadable + Files.writeString(backups.resolve(BackupManager.getFileName(System.currentTimeMillis() + 10_000, null)), + "junk"); + List listed = BackupManager.list(backups); + assertEquals(3, listed.size()); + assertFalse(listed.getFirst().isValid()); + } + } + + private static void writeZip(Path file, String metadata, String calls, String events) throws IOException + { + try(OutputStream out = Files.newOutputStream(file); ZipOutputStream zip = new ZipOutputStream(out)) + { + for(String[] entry: new String[][]{{BackupManager.ENTRY_METADATA, metadata}, + {BackupManager.ENTRY_CALLS, calls}, {BackupManager.ENTRY_EVENTS, events}}) + { + zip.putNextEntry(new ZipEntry(entry[0])); + zip.write(entry[1].getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + } + } + + @Test + public void retentionKeepsNewestBackups() throws Exception + { + Path backups = mTempDir.resolve("backups"); + + try(CallDatabase database = open(h2())) + { + long now = System.currentTimeMillis(); + + for(int x = 0; x < 5; x++) + { + BackupManager.writeBackup(database, backups, x == 2 ? BackupManager.TAG_PRE_RESTORE : null, + now + x * 60_000L); + } + + //Same second as an existing backup gets a unique name + BackupInfo same = BackupManager.writeBackup(database, backups, null, now + 4 * 60_000L); + assertEquals(6, BackupManager.list(backups).size()); + assertEquals(now - now % 1000 + 4 * 60_000L + 1000, same.created()); + + Files.writeString(backups.resolve("notes.txt"), "not a backup"); + List deleted = BackupManager.applyRetention(backups, 3); + assertEquals(3, deleted.size()); + List remaining = BackupManager.list(backups); + assertEquals(3, remaining.size()); + assertEquals(same.path(), remaining.getFirst().path(), "newest first"); + assertTrue(remaining.stream().noneMatch(BackupInfo::isPreRestore), "oldest deleted, including tagged"); + assertTrue(Files.exists(backups.resolve("notes.txt")), "other files are never deleted"); + + assertEquals(2, BackupManager.applyRetention(backups, 0).size()); + assertEquals(1, BackupManager.list(backups).size(), "retention is at least 1"); + } + } + + @Test + public void scheduleIsDueAfterInterval() + { + long now = System.currentTimeMillis(); + long day = TimeUnit.DAYS.toMillis(1); + assertTrue(BackupManager.isDue(0, BackupInterval.DAY_1, now), "no backups yet"); + assertFalse(BackupManager.isDue(now - day + 60_000, BackupInterval.DAY_1, now)); + assertTrue(BackupManager.isDue(now - day, BackupInterval.DAY_1, now)); + assertFalse(BackupManager.isDue(0, BackupInterval.NEVER, now), "automatic backups off"); + assertTrue(BackupManager.isDue(now - TimeUnit.HOURS.toMillis(7), BackupInterval.HOURS_6, now)); + } + + @Test + public void fileNameRoundTrip() throws Exception + { + long created = 1_700_000_000_000L; + String name = BackupManager.getFileName(created, BackupManager.TAG_PRE_RESTORE); + assertTrue(name.startsWith("sdrtrunk-backup-") && name.endsWith("-pre-restore.zip"), name); + BackupInfo info = BackupManager.read(Files.writeString(mTempDir.resolve(name), "x")); + assertNotNull(info); + assertEquals(created, info.created()); + assertTrue(info.isPreRestore()); + assertFalse(info.isValid()); + assertNull(BackupManager.read(Files.writeString(mTempDir.resolve("sdrtrunk-backup-x.zip"), "x"))); + } +}