From c4a333269d7af92e6219337290dce6fb52596e05 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Sun, 18 Aug 2024 21:25:36 -0400 Subject: [PATCH 1/6] cache system data --- backend/controllers/systems.js | 57 ++++++++++++++++++++++++++++++++-- backend/sys_stats.js | 7 +++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/backend/controllers/systems.js b/backend/controllers/systems.js index 26e9424..f1a645d 100644 --- a/backend/controllers/systems.js +++ b/backend/controllers/systems.js @@ -12,6 +12,9 @@ const mailjet = new Mailjet({ apiSecret: process.env['MAILJET_SECRET'] }); +var systemList = []; +var systemListTime = Date.now(); + exports.contact_system = async function(req, res) { var system = await System.findOne({ shortName: req.params.shortName.toLowerCase() @@ -97,11 +100,59 @@ exports.contact_system = async function(req, res) { } -exports.get_systems = async function (req, res) { +async function load_systems(systemClients) { let fromDate = new Date(Date.now() - 60 * 60 * 24 * 30 * 1000); - //const results = await System.find({lastActive: {$gte: fromDate}}).populate('userId', "screenName").catch( err => { + const results = await System.find({lastActive: {$gte: fromDate}}).populate('userId', "screenName").catch( err => { //const results = await System.find({active: true}).populate('userId', "screenName").catch( err => { - const results = await System.find({active: true}).catch( err => { + //const results = await System.find({active: true}).catch( err => { // super simple query + console.error("Error - get_systems: " + err.message); + }); + systemList = []; + for (var result in results) { + + var clientCount = 0; + if (systemClients.hasOwnProperty(results[result].shortName)) { + clientCount = systemClients[results[result].shortName]; + } + var system = { + name: results[result].name, + shortName: results[result].shortName, + systemType: results[result].systemType, + county: results[result].county, + country: results[result].country, + city: results[result].city, + state: results[result].state, + active: results[result].active, + lastActive: results[result].lastActive, + callAvg: results[result].callAvg, + description: results[result].description, + status: results[result].status, + allowContact: results[result].allowContact, + clientCount: clientCount + } + /* + if (results[result].showScreenName && results[result].userId) { + system.screenName = results[result].userId.screenName + } else { + system.screenName = null; + }*/ + system.screenName = null; + systemList.push(system); + } + +} + + +exports.get_systems = async function (req, res) { + if ((systemList.length == 0) || (systemListTime < (Date.now() - 60 * 1000 * 15))) { + await load_systems(req.systemClients); + systemListTime = Date.now(); + } + + let fromDate = new Date(Date.now() - 60 * 60 * 24 * 30 * 1000); + const results = await System.find({lastActive: {$gte: fromDate}}).populate('userId', "screenName").catch( err => { + //const results = await System.find({active: true}).populate('userId', "screenName").catch( err => { + //const results = await System.find({active: true}).catch( err => { // super simple query console.error("Error - get_systems: " + err.message); res.status(500); res.json({ diff --git a/backend/sys_stats.js b/backend/sys_stats.js index 73c6a20..3e99a41 100644 --- a/backend/sys_stats.js +++ b/backend/sys_stats.js @@ -15,19 +15,20 @@ async function updateActiveSystems() { // go through all the systems // if you have received some calls during that last period, make the system active if ((callTotals[item.shortName] != undefined) && (callTotals[item.shortName][0] > 0)) { - //item.active = true; + item.active = true; item.callAvg = callTotals[item.shortName][0] / timePeriod; item.lastActive = new Date(); } else { - //item.active = false; + item.active = false; item.callAvg = 0; } + /* let fromDate = new Date(Date.now() - 60 * 60 * 24 * 30 * 1000); if ((item.lastActive != null) && (item.lastActive > fromDate)) { item.active = true; } else { item.active = false; - } + }*/ await item.save(); }; } From 290ee216bbf88bba628ac6758c3a43d3749a1584 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Sun, 18 Aug 2024 21:27:34 -0400 Subject: [PATCH 2/6] go time --- backend/controllers/systems.js | 44 ---------------------------------- 1 file changed, 44 deletions(-) diff --git a/backend/controllers/systems.js b/backend/controllers/systems.js index f1a645d..1d83c3b 100644 --- a/backend/controllers/systems.js +++ b/backend/controllers/systems.js @@ -149,50 +149,6 @@ exports.get_systems = async function (req, res) { systemListTime = Date.now(); } - let fromDate = new Date(Date.now() - 60 * 60 * 24 * 30 * 1000); - const results = await System.find({lastActive: {$gte: fromDate}}).populate('userId', "screenName").catch( err => { - //const results = await System.find({active: true}).populate('userId', "screenName").catch( err => { - //const results = await System.find({active: true}).catch( err => { // super simple query - console.error("Error - get_systems: " + err.message); - res.status(500); - res.json({ - success: false, - message: err - }); - return; - }); - var systemList = []; - for (var result in results) { - - var clientCount = 0; - if (req.systemClients.hasOwnProperty(results[result].shortName)) { - clientCount = req.systemClients[results[result].shortName]; - } - var system = { - name: results[result].name, - shortName: results[result].shortName, - systemType: results[result].systemType, - county: results[result].county, - country: results[result].country, - city: results[result].city, - state: results[result].state, - active: results[result].active, - lastActive: results[result].lastActive, - callAvg: results[result].callAvg, - description: results[result].description, - status: results[result].status, - allowContact: results[result].allowContact, - clientCount: clientCount - } - /* - if (results[result].showScreenName && results[result].userId) { - system.screenName = results[result].userId.screenName - } else { - system.screenName = null; - }*/ - system.screenName = null; - systemList.push(system); - } res.contentType('json'); res.send(JSON.stringify({ success: true, From 0cc8caeba040d88210af44e75a95b620d6bf5e07 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Sun, 18 Aug 2024 21:29:02 -0400 Subject: [PATCH 3/6] more sockets --- backend/controllers/uploads.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/controllers/uploads.js b/backend/controllers/uploads.js index cdd5e1a..8babaf4 100644 --- a/backend/controllers/uploads.js +++ b/backend/controllers/uploads.js @@ -13,7 +13,7 @@ const { NodeHttpHandler } = require('@smithy/node-http-handler'); const https = require('https'); const agent = new https.Agent({ - maxSockets: 150 + maxSockets: 250 }); From e3e14a229a25ba115718bef680906a7e556e65b9 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Mon, 19 Aug 2024 17:09:09 -0400 Subject: [PATCH 4/6] ability to use a different public URL for S3 --- backend/controllers/uploads.js | 4 ++-- backend/index.js | 2 +- docker-compose.yml | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/controllers/uploads.js b/backend/controllers/uploads.js index 8babaf4..072078f 100644 --- a/backend/controllers/uploads.js +++ b/backend/controllers/uploads.js @@ -21,7 +21,7 @@ var s3_endpoint = process.env['S3_ENDPOINT'] != null ? process.env['S3_ENDPOINT' var s3_region = process.env['S3_REGION'] != null ? process.env['S3_REGION'] : 'us-west-1'; var s3_bucket = process.env['S3_BUCKET'] != null ? process.env['S3_BUCKET'] : 'openmhz-west'; var s3_profile = process.env['S3_PROFILE'] != null ? process.env['S3_PROFILE'] : 'wasabi-account'; - +var s3_public_url = process.env['S3_PUBLIC_URL'] != null ? process.env['S3_PUBLIC_URL'] : s3_endpoint + "/" + s3_bucket; const client = new S3Client({ requestHandler: new NodeHttpHandler({ @@ -165,7 +165,7 @@ exports.upload = async function (req, res, next) { var endpoint = s3_endpoint; var bucket = s3_bucket; - var url = s3_endpoint + "/" + s3_bucket + "/" + object_key; + var url = s3_public_url + "/" + object_key; var call = new Call({ shortName: shortName, diff --git a/backend/index.js b/backend/index.js index b49bad5..54e481e 100644 --- a/backend/index.js +++ b/backend/index.js @@ -270,7 +270,7 @@ io.sockets.on('connection', function (client) { if (typeof clients[client.id] === "undefined") { console.error("How can it be undefined here!!"); - console.error(clients[client.id]); + console.error(client.id); return; } if (group) { diff --git a/docker-compose.yml b/docker-compose.yml index fdb7e7a..479cc1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,6 +77,7 @@ services: - S3_BUCKET - S3_PROFILE - S3_REGION + - S3_PUBLIC_URL - OTEL_SERVICE_NAME - HONEYCOMB_API_KEY volumes: From cbc324e6267b84d88416384ec8be577458cdad15 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 20 Aug 2024 10:01:38 -0400 Subject: [PATCH 5/6] more prints on stats shifting --- backend/sys_stats.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/sys_stats.js b/backend/sys_stats.js index 3e99a41..05f818e 100644 --- a/backend/sys_stats.js +++ b/backend/sys_stats.js @@ -129,7 +129,7 @@ exports.addCall = function (call) { // This gets called when a Time Period is up exports.shiftStats = async function () { - + console.log("Started Shifting Stats at: " + new Date()); // for all the systems in Error Stats for (var shortName in uploadErrors) { if (uploadErrors.hasOwnProperty(shortName)) { @@ -148,7 +148,7 @@ exports.shiftStats = async function () { uploadErrors[shortName][0] = 0; } } - + console.log("Finished Shifting Upload Errors at: " + new Date()); // for each system in decodeErrorsFreq for (let shortName in decodeErrorsFreq) { @@ -196,7 +196,7 @@ exports.shiftStats = async function () { } } - + console.log("Finished Shifting Decode Errors at: " + new Date()); // for each system in talkgroupStats for (let shortName in talkgroupStats) { @@ -266,7 +266,9 @@ exports.shiftStats = async function () { await SystemStat.updateOne(query, update, options); } } + console.log("Finished Shifting Talkgroup Stats at: " + new Date()); updateActiveSystems(); + console.log("Finished Updating Active Systems at: " + new Date()); } exports.callTotals = function (shortName) { return callTotals[shortName]; From 32af2ca734703920a4c2ae0d85117c09242b7e82 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 20 Aug 2024 10:51:17 -0400 Subject: [PATCH 6/6] Update sys_stats.js --- backend/sys_stats.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/sys_stats.js b/backend/sys_stats.js index 05f818e..4c59347 100644 --- a/backend/sys_stats.js +++ b/backend/sys_stats.js @@ -77,6 +77,7 @@ exports.addError = function (call) { // Keeps track of the number of calls for each talkgroup for a system exports.addCall = function (call) { +/* // if you haven't started keeping track of stats for the System yet if (talkgroupStats[call.shortName] === undefined) { talkgroupStats[call.shortName] = {}; @@ -122,7 +123,7 @@ exports.addCall = function (call) { // add to the call count and total length, Call Average is calc by dividing the two... sysErrors[call.freq].totalLen += call.len; sysErrors[call.freq].errors += call.errorCount; - sysErrors[call.freq].spikes += call.spikeCount; + sysErrors[call.freq].spikes += call.spikeCount;*/ } @@ -149,7 +150,7 @@ exports.shiftStats = async function () { } } console.log("Finished Shifting Upload Errors at: " + new Date()); - +/* // for each system in decodeErrorsFreq for (let shortName in decodeErrorsFreq) { @@ -266,7 +267,7 @@ exports.shiftStats = async function () { await SystemStat.updateOne(query, update, options); } } - console.log("Finished Shifting Talkgroup Stats at: " + new Date()); + console.log("Finished Shifting Talkgroup Stats at: " + new Date());*/ updateActiveSystems(); console.log("Finished Updating Active Systems at: " + new Date()); }