Index socket clients by system; make /systems cheap

A CPU profile during a traffic spike (4,850 sockets, 74 uploads/s) showed
the backend spending ~28% of its busy time in two loops over every
connected socket: notify_clients on each upload, and addSystemClients on
each /systems request.

- Add client_registry.js: sockets in a Map, with active clients indexed
  by system plus a firehose set. notify_clients only visits the
  uploaded call's system, with the same matching rules as before.
- /systems: counts come from the index, the serialized body is reused
  for 5s, and counts now reset to 0 when a system's listeners leave.
  Firehose clients no longer count toward a system.
- nginx: 10s microcache for /systems on api.*.
- Test: node --test test/client_registry.test.js checks recipients and
  counts against the old full-scan implementation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Luke Berndt
2026-11-20 13:22:31 -05:00
parent 5b177eca58
commit a00bc98cc4
5 changed files with 403 additions and 116 deletions

122
backend/client_registry.js Normal file
View File

@@ -0,0 +1,122 @@
// Registry of connected socket.io clients.
//
// Every connected socket has an entry. Active entries are also indexed by
// system (or in the firehose set) so that fanning out an uploaded call only
// visits the listeners of that call's system, instead of every socket on the
// server. At peak that is the difference between ~5,000 iterations per upload
// and a few dozen.
//
// The index has to mirror each entry's active/firehose/shortName fields, so
// call index() after changing them and unindex() before an entry stops
// receiving calls.
const entries = new Map(); // socket id -> entry
const activeBySystem = new Map(); // shortName -> Set<entry>
const firehose = new Set(); // entries that receive every call
function add(id, socket) {
const entry = { socket: socket, active: false, timestamp: new Date() };
entries.set(id, entry);
return entry;
}
function get(id) {
return entries.get(id);
}
function unindex(entry) {
const set = entry.indexedIn;
if (!set) return;
set.delete(entry);
if (set !== firehose && set.size === 0) {
activeBySystem.delete(entry.indexedShortName);
}
entry.indexedIn = null;
entry.indexedShortName = null;
}
// (Re)index an entry from its current fields. Safe to call repeatedly.
function index(entry) {
unindex(entry);
if (!entry.active) return;
let set;
if (entry.firehose === true) {
set = firehose;
} else {
set = activeBySystem.get(entry.shortName);
if (!set) {
set = new Set();
activeBySystem.set(entry.shortName, set);
}
}
set.add(entry);
entry.indexedIn = set;
entry.indexedShortName = entry.shortName;
}
function remove(id) {
const entry = entries.get(id);
if (!entry) return;
unindex(entry);
entries.delete(id);
}
// Same matching rules the old full-scan notify_clients used.
function wantsCall(entry, call) {
if (entry.filterType == "all") return true;
if (entry.filterType == "unit") {
const codeArray = entry.filterCode.split(',');
for (let j = 0; j < codeArray.length; ++j) {
for (let k = 0; k < call.srcList.length; k++) {
if (codeArray[j] == call.srcList[k]) return true;
}
}
return false;
}
const codeArray = entry.talkgroupNums;
for (let j = 0; j < codeArray.length; ++j) {
if (codeArray[j] == call.talkgroupNum) return true;
}
return false;
}
// Send a call to every interested client. Returns how many were sent.
function notify(call, message) {
let sent = 0;
for (const entry of firehose) {
entry.socket.emit("new message", message);
sent++;
}
const listeners = activeBySystem.get(call.shortName.toLowerCase());
if (listeners) {
for (const entry of listeners) {
if (wantsCall(entry, call)) {
entry.socket.emit("new message", message);
sent++;
}
}
}
return sent;
}
// Active listeners per system, e.g. { nhpd: 1428, dcfd: 19 }.
// Firehose clients are not counted against any system.
function systemCounts() {
const counts = {};
for (const [shortName, set] of activeBySystem) {
counts[shortName] = set.size;
}
return counts;
}
module.exports = {
add,
get,
index,
unindex,
remove,
notify,
systemCounts,
size: () => entries.size,
values: () => entries.values(),
};

View File

@@ -15,6 +15,12 @@ const mailjet = new Mailjet({
var systemList = []; var systemList = [];
// Cached /systems response body; rebuilt when stale or when load_systems swaps the list.
const SYSTEMS_BODY_TTL_MS = 5000;
let systemsBody = null;
let systemsBodyList = null;
let systemsBodyTime = 0;
exports.contact_system = async function (req, res) { exports.contact_system = async function (req, res) {
var system = await System.findOne({ var system = await System.findOne({
shortName: req.params.shortName.toLowerCase() shortName: req.params.shortName.toLowerCase()
@@ -181,19 +187,24 @@ exports.get_systems = async function (req, res) {
await load_systems(req.systemClients); await load_systems(req.systemClients);
}*/ }*/
const systemClients = req.systemClients || {}; // The list is a few hundred systems and the homepage requests it constantly,
for (var i = 0; i < systemList.length; i++) { // so serialize it at most every few seconds instead of on every request.
var system = systemList[i]; const now = Date.now();
var shortName = system.shortName; if (!systemsBody || systemsBodyList !== systemList || now - systemsBodyTime > SYSTEMS_BODY_TTL_MS) {
if (systemClients.hasOwnProperty(shortName)) { const systemClients = req.systemClients || {};
system.clientCount = systemClients[shortName]; for (var i = 0; i < systemList.length; i++) {
var system = systemList[i];
system.clientCount = systemClients[system.shortName] || 0;
} }
systemsBody = JSON.stringify({
success: true,
systems: systemList
});
systemsBodyList = systemList;
systemsBodyTime = now;
} }
res.contentType('json'); res.set('Content-Type', 'application/json; charset=utf-8');
res.send(JSON.stringify({ res.send(systemsBody);
success: true,
systems: systemList
}));
} }
exports.authorize_system = async function (req, res) { exports.authorize_system = async function (req, res) {

View File

@@ -90,29 +90,15 @@ var upload = multer({
dest: config.uploadDirectory dest: config.uploadDirectory
}); });
var clients = []; const clients = require("./client_registry");
function addSystemClients(req, res, next) { function addSystemClients(req, res, next) {
var systemClients = {}; req.systemClients = clients.systemCounts();
for (var key in clients) {
if (clients.hasOwnProperty(key)) {
const client = clients[key];
const shortName = client.shortName;
if (client.active) {
if (systemClients.hasOwnProperty(shortName)) {
systemClients[shortName]++;
} else {
systemClients[shortName] = 1;
}
}
}
}
req.systemClients = systemClients;
next(); next();
} }
function addTotalClients(req, res, next) { function addTotalClients(req, res, next) {
req.totalClients = Object.keys(clients).length; req.totalClients = clients.size();
next(); next();
} }
@@ -169,23 +155,21 @@ function get_clients(req, res) {
} }
var total = 0; var total = 0;
var response = []; var response = [];
for (var key in clients) { for (const client of clients.values()) {
if (clients.hasOwnProperty(key)) { total++;
total++; if (!short_name || (client.shortName == short_name)) {
if (!short_name || (clients[key].shortName == short_name)) { var age = (Date.now() - client.timestamp) / 1000;
var age = (Date.now() - clients[key].timestamp) / 1000; var obj = {
var obj = { shortName: client.shortName,
shortName: clients[key].shortName, filterCode: client.filterCode,
filterCode: clients[key].filterCode, filterName: client.filterName,
filterName: clients[key].filterName, filterType: client.filterType,
filterType: clients[key].filterType, filterStarred: client.filterStarred,
filterStarred: clients[key].filterStarred, active: client.active,
active: clients[key].active, talkgroupNums: client.talkgroupNums,
talkgroupNums: clients[key].talkgroupNums, timestamp: age
timestamp: age
}
response.push(obj);
} }
response.push(obj);
} }
} }
response.push({ "total": total }); response.push({ "total": total });
@@ -207,48 +191,7 @@ app.use(function (err, req, res, next) {
function notify_clients(call) { function notify_clients(call) {
call.type = "calls"; call.type = "calls";
const message = JSON.stringify(call); const message = JSON.stringify(call);
var sent = 0; const sent = clients.notify(call, message);
for (var key in clients) {
if (clients.hasOwnProperty(key)) {
var client = clients[key];
if (client.active) {
if ((client.firehose == true) || (client.shortName == call.shortName.toLowerCase())) {
// if client
if ((client.firehose == true) || (client.filterType == "all")) {
sent++;
client.socket.emit("new message", message);
} else if (client.filterType == "unit") {
var codeArray = client.filterCode.split(',');
var success = false;
for (var j = 0; j < codeArray.length; ++j) {
for (var k = 0; k < call.srcList.length; k++) {
if (codeArray[j] == call.srcList[k]) {
sent++;
client.socket.emit("new message", message);
success = true;
break;
}
}
if (success) {
break;
}
}
} else {
var codeArray = client.talkgroupNums;
for (var j = 0; j < codeArray.length; ++j) {
if (codeArray[j] == call.talkgroupNum) {
client.socket.emit("new message", message);
sent++
break;
}
}
}
}
}
}
}
if (sent > 0) { if (sent > 0) {
//console.log("[" + call.shortName.toLowerCase() + "] Sent call to " + sent + " clients"); //console.log("[" + call.shortName.toLowerCase() + "] Sent call to " + sent + " clients");
@@ -269,37 +212,39 @@ io.sockets.on('connection', function (client) {
const cf_ipAddress = client.handshake.headers["cf-connecting-ip"]; const cf_ipAddress = client.handshake.headers["cf-connecting-ip"];
const x_ipAddress = client.handshake.headers["x-forwarded-for"]?.split(",")[0]; const x_ipAddress = client.handshake.headers["x-forwarded-for"]?.split(",")[0];
const ipAddress = cf_ipAddress || x_ipAddress || header_ipAddress; const ipAddress = cf_ipAddress || x_ipAddress || header_ipAddress;
clients[client.id] = { socket: client, active: false }; clients.add(client.id, client);
clients[client.id].timestamp = new Date();
client.on('start', async function (data) { client.on('start', async function (data) {
if (typeof clients[client.id] !== "undefined") { const entry = clients.get(client.id);
if (entry) {
// Stop delivering under any previous filter; re-indexed once this one validates.
clients.unindex(entry);
if ((data.filterType != "firehose") && (!data.shortName || (typeof data.shortName != "string"))) { // If it is not firehose, then it must have a shortName if ((data.filterType != "firehose") && (!data.shortName || (typeof data.shortName != "string"))) { // If it is not firehose, then it must have a shortName
console.error("Error - Socket - Invalid ShortName: " + data.shortName + " ClientID: " + client.id + " IP: " + ipAddress); console.error("Error - Socket - Invalid ShortName: " + data.shortName + " ClientID: " + client.id + " IP: " + ipAddress);
delete clients[client.id]; clients.remove(client.id);
return; return;
} }
clients[client.id].active = true; entry.active = true;
clients[client.id].shortName = data.shortName.toLowerCase(); entry.shortName = data.shortName.toLowerCase();
clients[client.id].filterCode = String(data.filterCode); entry.filterCode = String(data.filterCode);
clients[client.id].filterName = data.filterName; entry.filterName = data.filterName;
clients[client.id].filterStarred = data.filterStarred; entry.filterStarred = data.filterStarred;
clients[client.id].filterType = String(data.filterType); entry.filterType = String(data.filterType);
clients[client.id].talkgroupNums = []; entry.talkgroupNums = [];
clients[client.id].timestamp = new Date(); entry.timestamp = new Date();
clients[client.id].firehose = false; entry.firehose = false;
clients[client.id].ipAddress = ipAddress; entry.ipAddress = ipAddress;
// Validate the filterCode and filterType // Validate the filterCode and filterType
if (typeof clients[client.id].filterCode != "string") { if (typeof entry.filterCode != "string") {
console.error("Error - Socket - Invalid filterCode: " + data.filterCode + " ShortName: " + data.shortName.toLowerCase() + " ClientID: " + client.id + " IP: " + ipAddress); console.error("Error - Socket - Invalid filterCode: " + data.filterCode + " ShortName: " + data.shortName.toLowerCase() + " ClientID: " + client.id + " IP: " + ipAddress);
delete clients[client.id]; clients.remove(client.id);
return; return;
} }
if (typeof clients[client.id].filterType != "string") { if (typeof entry.filterType != "string") {
console.error("Error - Socket - Invalid filterType: " + data.filterType + " ShortName: " + data.shortName.toLowerCase() + " ClientID: " + client.id + " IP: " + ipAddress); console.error("Error - Socket - Invalid filterType: " + data.filterType + " ShortName: " + data.shortName.toLowerCase() + " ClientID: " + client.id + " IP: " + ipAddress);
delete clients[client.id]; clients.remove(client.id);
return; return;
} }
@@ -307,57 +252,61 @@ io.sockets.on('connection', function (client) {
if (data.filterType == "firehose") { if (data.filterType == "firehose") {
if (firehose_key && (typeof data.filterCode == "string") && (data.filterCode == firehose_key)) { if (firehose_key && (typeof data.filterCode == "string") && (data.filterCode == firehose_key)) {
console.log("Enabling Firehose mode for Client: " + client.id + " IP: " + ipAddress); console.log("Enabling Firehose mode for Client: " + client.id + " IP: " + ipAddress);
clients[client.id].firehose = true; entry.firehose = true;
} else { } else {
console.error("Error - Socket - Invalid Firehose Key: " + data.filterCode + " ClientID: " + client.id + " Filter Name: " + client.filterName + " IP: " + ipAddress); console.error("Error - Socket - Invalid Firehose Key: " + data.filterCode + " ClientID: " + client.id + " Filter Name: " + client.filterName + " IP: " + ipAddress);
delete clients[client.id]; clients.remove(client.id);
return; return;
} }
} else if ((data.filterType == "group")) { // Handle filterType == "group" } else if ((data.filterType == "group")) { // Handle filterType == "group"
if ((typeof data.filterCode != "string") || (data.filterCode.indexOf(',') != -1) || !ObjectId.isValid(data.filterCode)) { if ((typeof data.filterCode != "string") || (data.filterCode.indexOf(',') != -1) || !ObjectId.isValid(data.filterCode)) {
console.error("Error - Socket - Invalid Group ID: " + data.filterCode + " ShortName: " + data.shortName.toLowerCase() + " ClientID: " + client.id + " IP: " + ipAddress); console.error("Error - Socket - Invalid Group ID: " + data.filterCode + " ShortName: " + data.shortName.toLowerCase() + " ClientID: " + client.id + " IP: " + ipAddress);
delete clients[client.id]; clients.remove(client.id);
return; return;
} }
const group = await Group.findOne({ 'shortName': data.shortName.toLowerCase(), '_id': ObjectId.createFromHexString(data.filterCode) }); const group = await Group.findOne({ 'shortName': data.shortName.toLowerCase(), '_id': ObjectId.createFromHexString(data.filterCode) });
if (typeof clients[client.id] === "undefined") { if (!clients.get(client.id)) {
console.error("Error - Socket - Client not found ShortName: " + data.shortName + " ClientID: " + client.id + " IP: " + ipAddress); console.error("Error - Socket - Client not found ShortName: " + data.shortName + " ClientID: " + client.id + " IP: " + ipAddress);
return; return;
} }
if (group) { if (group) {
clients[client.id].talkgroupNums = group.talkgroups; entry.talkgroupNums = group.talkgroups;
} else { } else {
console.error("Error - Socket - Invalid group " + data.filterCode + " Shortname: " + data.shortName + " ClientID: " + client.id + " IP: " + ipAddress); console.error("Error - Socket - Invalid group " + data.filterCode + " Shortname: " + data.shortName + " ClientID: " + client.id + " IP: " + ipAddress);
delete clients[client.id]; clients.remove(client.id);
return; return;
} }
// Handle filterType == "talkgroup" // Handle filterType == "talkgroup"
} else if ((data.filterType == "talkgroup") && Array.isArray(data.filterCode)) { } else if ((data.filterType == "talkgroup") && Array.isArray(data.filterCode)) {
clients[client.id].talkgroupNums = data.filterCode; entry.talkgroupNums = data.filterCode;
} else if ((data.filterType == "all") && (data.filterCode == "")) { } else if ((data.filterType == "all") && (data.filterCode == "")) {
// Handle filterType == "all" // Handle filterType == "all"
clients[client.id].talkgroupNums = []; entry.talkgroupNums = [];
} else { } else {
console.error("Error - Socket - Invalid Filter Type: " + data.filterType + " Filter Code: " + data.filterCode + " Filter Name: " + data.filterName + " ShortName: " + data.shortName.toLowerCase() + " ClientID: " + client.id + " IP: " + ipAddress); console.error("Error - Socket - Invalid Filter Type: " + data.filterType + " Filter Code: " + data.filterCode + " Filter Name: " + data.filterName + " ShortName: " + data.shortName.toLowerCase() + " ClientID: " + client.id + " IP: " + ipAddress);
delete clients[client.id]; clients.remove(client.id);
return; return;
} }
// All checks passed: start delivering calls under the new filter.
clients.index(entry);
} else { } else {
console.error("Error - Socket.io [Start] either client not found, shortName:" + data.shortName + " ClientID: " + client.id + " IP: " + ipAddress); console.error("Error - Socket.io [Start] either client not found, shortName:" + data.shortName + " ClientID: " + client.id + " IP: " + ipAddress);
} }
}); });
client.on('stop', function (data) { client.on('stop', function (data) {
if (clients[client.id]) { const entry = clients.get(client.id);
clients[client.id].active = false; if (entry) {
entry.active = false;
clients.unindex(entry);
} else { } else {
console.error("Error - Socket.io [Stop] Client not found "); console.error("Error - Socket.io [Stop] Client not found ");
} }
}); });
client.on('disconnect', function () { client.on('disconnect', function () {
delete clients[client.id]; clients.remove(client.id);
}); });
}); });

View File

@@ -0,0 +1,186 @@
// Run with: node --test test/client_registry.test.js
//
// Drives the registry through random socket lifecycles (connect, start with
// each filter type, re-start on another system, stop, disconnect) and checks
// that every upload reaches exactly the sockets the old full-scan
// notify_clients would have chosen, and that per-system counts match.
const test = require("node:test");
const assert = require("node:assert");
// The old implementation, kept verbatim as the reference.
function oldNotify(clients, call, message) {
var sent = 0;
for (var key in clients) {
if (clients.hasOwnProperty(key)) {
var client = clients[key];
if (client.active) {
if ((client.firehose == true) || (client.shortName == call.shortName.toLowerCase())) {
if ((client.firehose == true) || (client.filterType == "all")) {
sent++;
client.socket.emit("new message", message);
} else if (client.filterType == "unit") {
var codeArray = client.filterCode.split(',');
var success = false;
for (var j = 0; j < codeArray.length; ++j) {
for (var k = 0; k < call.srcList.length; k++) {
if (codeArray[j] == call.srcList[k]) {
sent++;
client.socket.emit("new message", message);
success = true;
break;
}
}
if (success) break;
}
} else {
var codeArray = client.talkgroupNums;
for (var j = 0; j < codeArray.length; ++j) {
if (codeArray[j] == call.talkgroupNum) {
client.socket.emit("new message", message);
sent++;
break;
}
}
}
}
}
}
}
return sent;
}
function oldSystemCounts(clients) {
var systemClients = {};
for (var key in clients) {
const client = clients[key];
// Firehose clients are no longer counted against a system.
if (client.active && !client.firehose) {
systemClients[client.shortName] = (systemClients[client.shortName] || 0) + 1;
}
}
return systemClients;
}
// Small deterministic PRNG so failures are reproducible.
function rng(seed) {
return () => {
seed = (seed * 1664525 + 1013904223) >>> 0;
return seed / 2 ** 32;
};
}
test("registry delivers to the same sockets as the old full scan", () => {
const registry = require("../client_registry");
const rand = rng(42);
const pick = (arr) => arr[Math.floor(rand() * arr.length)];
const systems = ["nhpd", "dcfd", "bcem", "risconn", "Mixed"];
const tgs = [1, 2, 3, 21, 22, "21"];
const units = ["100", "200", "300"];
const old = {}; // the reference model: old-style clients object
const received = new Map(); // socket id -> count, filled by the registry's emits
const ids = [];
let next = 0;
const connect = () => {
const id = "s" + next++;
const socket = { id, emit: () => received.set(id, (received.get(id) || 0) + 1) };
registry.add(id, socket);
old[id] = { socket: { emit() {} }, active: false };
ids.push(id);
};
// Mirrors a successful 'start' as index.js applies it.
const start = (id) => {
const entry = registry.get(id);
registry.unindex(entry);
const filterType = pick(["all", "talkgroup", "group", "unit", "firehose"]);
const fields = {
active: true,
shortName: pick(systems).toLowerCase(),
filterType,
filterCode: filterType === "unit" ? pick(units) + "," + pick(units) : "",
talkgroupNums: filterType === "talkgroup" || filterType === "group" ? [pick(tgs), pick(tgs)] : [],
firehose: filterType === "firehose" && rand() < 0.5,
};
Object.assign(entry, fields);
Object.assign(old[id], fields);
registry.index(entry);
};
const stop = (id) => {
const entry = registry.get(id);
entry.active = false;
registry.unindex(entry);
old[id].active = false;
};
const disconnect = (id) => {
registry.remove(id);
delete old[id];
ids.splice(ids.indexOf(id), 1);
};
for (let i = 0; i < 300; i++) connect();
for (let step = 0; step < 5000; step++) {
const r = rand();
if (r < 0.15 || ids.length === 0) connect();
else if (r < 0.45) start(pick(ids));
else if (r < 0.55) stop(pick(ids));
else if (r < 0.65) disconnect(pick(ids));
else {
const call = {
shortName: pick(systems).toUpperCase(),
talkgroupNum: pick(tgs),
srcList: [pick(units), pick(units)],
};
// Which sockets should get it, per the old code.
const expected = new Set();
for (const id of Object.keys(old)) {
old[id].socket.emit = () => expected.add(id);
}
const oldSent = oldNotify(old, call, "m");
received.clear();
const newSent = registry.notify(call, "m");
assert.strictEqual(newSent, oldSent, `sent count differs at step ${step}`);
assert.deepStrictEqual(new Set(received.keys()), expected, `recipients differ at step ${step}`);
for (const n of received.values()) assert.strictEqual(n, 1, "a socket got the call twice");
}
if (step % 250 === 0) {
assert.deepStrictEqual(registry.systemCounts(), oldSystemCounts(old), `counts differ at step ${step}`);
assert.strictEqual(registry.size(), Object.keys(old).length);
}
}
// Draining every socket leaves no empty per-system sets behind.
for (const id of [...ids]) disconnect(id);
assert.deepStrictEqual(registry.systemCounts(), {});
assert.strictEqual(registry.size(), 0);
});
// Two overlapping 'start' events (one waiting on its Group lookup) can each
// call index() without an unindex() in between. The entry must end up in
// exactly one place, under its latest fields.
test("index() moves an entry instead of duplicating it", () => {
const registry = require("../client_registry");
let got = 0;
const entry = registry.add("race", { emit: () => got++ });
Object.assign(entry, { active: true, shortName: "aaa", filterType: "all", talkgroupNums: [], firehose: false });
registry.index(entry);
entry.shortName = "bbb";
registry.index(entry);
registry.index(entry);
assert.deepStrictEqual(registry.systemCounts(), { bbb: 1 });
assert.strictEqual(registry.notify({ shortName: "AAA", talkgroupNum: 1, srcList: [] }, "m"), 0);
assert.strictEqual(registry.notify({ shortName: "BBB", talkgroupNum: 1, srcList: [] }, "m"), 1);
assert.strictEqual(got, 1);
registry.remove("race");
assert.deepStrictEqual(registry.systemCounts(), {});
});

View File

@@ -142,6 +142,25 @@ server {
access_log off; access_log off;
} }
# System list for the homepage: listener counts can lag a few seconds.
location = /systems {
proxy_pass http://backend_api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
include /etc/nginx/proxy.conf;
proxy_cache api_cache;
proxy_cache_key "$scheme$host$request_uri|$http_origin";
proxy_cache_methods GET HEAD;
proxy_cache_valid 200 10s;
proxy_cache_lock on;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
add_header X-Cache-Status $upstream_cache_status always;
access_log off;
}
location / { location / {
proxy_pass http://backend_api; proxy_pass http://backend_api;
proxy_http_version 1.1; proxy_http_version 1.1;