Reduce backend per-upload CPU to relieve upload timeouts

The Node main thread was saturated at about 39 uploads/s, overflowing the
listen queue so uploads timed out connecting from nginx.

- Enable keepAlive on the S3 agent; every upload did a full TLS handshake
- Remove OpenTelemetry tracing
- Cache the system key lookup for 60s
- Use lean queries when listing calls
- Serialize each call once in notify_clients, not per recipient
- Add an nginx upstream keepalive pool, and hold Node's keepAliveTimeout
  above it so nginx never reuses a connection Node has closed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Luke Berndt
2026-11-09 18:06:48 -05:00
parent c0492ebf53
commit 6825aaf9a4
6 changed files with 220 additions and 290 deletions

View File

@@ -1,4 +1,3 @@
const tracing = require("./agents/otel-tracing");
var express = require("express");
var configureExpress = require("./config/express");
@@ -25,6 +24,12 @@ var app = express();
const server = require('http').createServer(app);
// Must outlast nginx's upstream keepalive_timeout (60s), so nginx is always the
// side that closes an idle connection. Otherwise it can send a request on a
// connection Node has just closed, and nginx won't retry a POST.
server.keepAliveTimeout = 65000;
server.headersTimeout = 70000;
const io = require('socket.io')(server, {
cors: {
@@ -201,6 +206,7 @@ app.use(function (err, req, res, next) {
function notify_clients(call) {
call.type = "calls";
const message = JSON.stringify(call);
var sent = 0;
for (var key in clients) {
@@ -211,7 +217,7 @@ function notify_clients(call) {
// if client
if ((client.firehose == true) || (client.filterType == "all")) {
sent++;
client.socket.emit("new message", JSON.stringify(call));
client.socket.emit("new message", message);
} else if (client.filterType == "unit") {
var codeArray = client.filterCode.split(',');
var success = false;
@@ -219,7 +225,7 @@ function notify_clients(call) {
for (var k = 0; k < call.srcList.length; k++) {
if (codeArray[j] == call.srcList[k]) {
sent++;
client.socket.emit("new message", JSON.stringify(call));
client.socket.emit("new message", message);
success = true;
break;
}
@@ -232,7 +238,7 @@ function notify_clients(call) {
var codeArray = client.talkgroupNums;
for (var j = 0; j < codeArray.length; ++j) {
if (codeArray[j] == call.talkgroupNum) {
client.socket.emit("new message", JSON.stringify(call));
client.socket.emit("new message", message);
sent++
break;
}