From 31644bcc40c63c52aa94e8085f813c2edb4c71c8 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Sun, 25 Aug 2024 21:30:23 -0400 Subject: [PATCH] more performant stats --- backend/sys_stats.js | 252 ++++++++++++++++++------------------------- 1 file changed, 105 insertions(+), 147 deletions(-) diff --git a/backend/sys_stats.js b/backend/sys_stats.js index 4c59347..7b9b62c 100644 --- a/backend/sys_stats.js +++ b/backend/sys_stats.js @@ -77,53 +77,53 @@ exports.addError = function (call) { // Keeps track of the number of calls for each talkgroup for a system exports.addCall = function (call) { -/* - // if you haven't started keeping track of stats for the System yet - if (talkgroupStats[call.shortName] === undefined) { - talkgroupStats[call.shortName] = {}; - } - var sysTalkgroupStats = talkgroupStats[call.shortName]; - // if you haven't started keeping track of Stats for this TG yet... - if (sysTalkgroupStats[call.talkgroupNum] === undefined) { - sysTalkgroupStats[call.talkgroupNum] = {} - sysTalkgroupStats[call.talkgroupNum].calls = 0; - sysTalkgroupStats[call.talkgroupNum].totalLen = 0; - sysTalkgroupStats[call.talkgroupNum].callCountHistory = new Array(); - sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory = new Array(); - for (var j = 0; j < spots; j++) { - sysTalkgroupStats[call.talkgroupNum].callCountHistory[j] = 0; - sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory[j] = 0; + + // if you haven't started keeping track of stats for the System yet + if (talkgroupStats[call.shortName] === undefined) { + talkgroupStats[call.shortName] = {}; } - } - - // add to the call count and total length, Call Average is calc by dividing the two... - sysTalkgroupStats[call.talkgroupNum].calls++; - sysTalkgroupStats[call.talkgroupNum].totalLen += call.len; - - // if you haven't started keeping track of stats for the System yet - if (decodeErrorsFreq[call.shortName] == undefined) { - decodeErrorsFreq[call.shortName] = {}; - } - - var sysErrors = decodeErrorsFreq[call.shortName]; - - // if you haven't started keeping track of Stats for this TG yet... - if (sysErrors[call.freq] === undefined) { - sysErrors[call.freq] = {} - sysErrors[call.freq].totalLen = 0; - sysErrors[call.freq].errors = 0; - sysErrors[call.freq].spikes = 0; - sysErrors[call.freq].errorHistory = new Array(); - sysErrors[call.freq].spikeHistory = new Array(); - for (var j = 0; j < spots; j++) { - sysErrors[call.freq].errorHistory[j] = 0; - sysErrors[call.freq].spikeHistory[j] = 0; + var sysTalkgroupStats = talkgroupStats[call.shortName]; + // if you haven't started keeping track of Stats for this TG yet... + if (sysTalkgroupStats[call.talkgroupNum] === undefined) { + sysTalkgroupStats[call.talkgroupNum] = {} + sysTalkgroupStats[call.talkgroupNum].calls = 0; + sysTalkgroupStats[call.talkgroupNum].totalLen = 0; + sysTalkgroupStats[call.talkgroupNum].callCountHistory = new Array(); + sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory = new Array(); + for (var j = 0; j < spots; j++) { + sysTalkgroupStats[call.talkgroupNum].callCountHistory[j] = 0; + sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory[j] = 0; + } } - } - // add to the call count and total length, Call Average is calc by dividing the two... - sysErrors[call.freq].totalLen += call.len; - sysErrors[call.freq].errors += call.errorCount; - sysErrors[call.freq].spikes += call.spikeCount;*/ + + // add to the call count and total length, Call Average is calc by dividing the two... + sysTalkgroupStats[call.talkgroupNum].calls++; + sysTalkgroupStats[call.talkgroupNum].totalLen += call.len; + + // if you haven't started keeping track of stats for the System yet + if (decodeErrorsFreq[call.shortName] == undefined) { + decodeErrorsFreq[call.shortName] = {}; + } + + var sysErrors = decodeErrorsFreq[call.shortName]; + + // if you haven't started keeping track of Stats for this TG yet... + if (sysErrors[call.freq] === undefined) { + sysErrors[call.freq] = {} + sysErrors[call.freq].totalLen = 0; + sysErrors[call.freq].errors = 0; + sysErrors[call.freq].spikes = 0; + sysErrors[call.freq].errorHistory = new Array(); + sysErrors[call.freq].spikeHistory = new Array(); + for (var j = 0; j < spots; j++) { + sysErrors[call.freq].errorHistory[j] = 0; + sysErrors[call.freq].spikeHistory[j] = 0; + } + } + // add to the call count and total length, Call Average is calc by dividing the two... + sysErrors[call.freq].totalLen += call.len; + sysErrors[call.freq].errors += call.errorCount; + sysErrors[call.freq].spikes += call.spikeCount; } @@ -149,125 +149,83 @@ exports.shiftStats = async function () { uploadErrors[shortName][0] = 0; } } + const bulkOps = []; console.log("Finished Shifting Upload Errors at: " + new Date()); -/* - // for each system in decodeErrorsFreq - for (let shortName in decodeErrorsFreq) { - - // if the system is in decodeErrorsFreq - if (decodeErrorsFreq.hasOwnProperty(shortName)) { - var sysErrors = decodeErrorsFreq[shortName]; - - - // for each freq in that systems stats - for (var freqNum in sysErrors) { - if (sysErrors.hasOwnProperty(freqNum)) { - var freqErrors = sysErrors[freqNum]; - - if ((freqErrors.errorHistory == undefined) || (freqErrors.spikeHistory == undefined)) { - console.error("[" + shortName + "] Skipping stat for freq: " + freqNum); - continue; - } - // move the history for that freq back - for (let j = spots - 1; j > 0; j--) { - let i = j - 1; - freqErrors.errorHistory[j] = freqErrors.errorHistory[i]; - freqErrors.spikeHistory[j] = freqErrors.spikeHistory[i]; - } - - if (freqErrors.totalLen > 0) { - freqErrors.errorHistory[0] = (freqErrors.errors / freqErrors.totalLen); - freqErrors.spikeHistory[0] = (freqErrors.spikes / freqErrors.totalLen); - } else { - freqErrors.errorHistory[0] = 0; - freqErrors.spikeHistory[0] = 0; - } - freqErrors.totalLen = 0; - freqErrors.errors = 0; - freqErrors.spikes = 0; - } + for (const [shortName, sysErrors] of Object.entries(decodeErrorsFreq)) { + for (const [freqNum, freqErrors] of Object.entries(sysErrors)) { + if (!freqErrors.errorHistory || !freqErrors.spikeHistory) { + console.error(`[${shortName}] Skipping stat for freq: ${freqNum}`); + continue; } - const query = { shortName: shortName }; - const update = { $set: { "decodeErrorsFreq": decodeErrorsFreq[shortName] } }; - const options = { upsert: true }; + freqErrors.errorHistory.unshift(freqErrors.totalLen > 0 ? freqErrors.errors / freqErrors.totalLen : 0); + freqErrors.spikeHistory.unshift(freqErrors.totalLen > 0 ? freqErrors.spikes / freqErrors.totalLen : 0); - await SystemStat.updateOne(query, update, options); + freqErrors.errorHistory.pop(); + freqErrors.spikeHistory.pop(); + freqErrors.totalLen = freqErrors.errors = freqErrors.spikes = 0; } + + bulkOps.push({ + updateOne: { + filter: { shortName }, + update: { $set: { decodeErrorsFreq: sysErrors } }, + upsert: true + } + }); } + + + console.log("Finished Shifting Decode Errors at: " + new Date()); - // for each system in talkgroupStats - for (let shortName in talkgroupStats) { + for (const [shortName, sysTalkgroupStats] of Object.entries(talkgroupStats)) { let callTotal = 0; - // if the system is in stats - if (talkgroupStats.hasOwnProperty(shortName)) { - var sysTalkgroupStats = talkgroupStats[shortName]; - - // for each talkgroup in that systems stats - for (var talkgroupNum in sysTalkgroupStats) { - if (sysTalkgroupStats.hasOwnProperty(talkgroupNum)) { - var tg = sysTalkgroupStats[talkgroupNum]; - var tgHistoryTotal = 0; - if ((tg.callCountHistory == undefined) || (tg.callAvgLenHistory == undefined)) { - console.error("[" + shortName + "] Skipping stat for tg: " + talkgroupNum); - continue; - } - // move the history for that talkgroup back - for (let j = spots - 1; j > 0; j--) { - let i = j - 1; - tgHistoryTotal += tg.callCountHistory[i]; - tg.callCountHistory[j] = tg.callCountHistory[i]; - tg.callAvgLenHistory[j] = tg.callAvgLenHistory[i]; - } - - // add to the total for that group - callTotal += tg.calls; - - - // figure out the history for current period in the history for this talkgroup; - tg.callCountHistory[0] = tg.calls; - tgHistoryTotal += tg.calls; - if (tg.calls > 0) { - tg.callAvgLenHistory[0] = Math.floor(tg.totalLen / tg.calls); - } else { - tg.callAvgLenHistory[0] = 0; - } - tg.calls = 0; - tg.totalLen = 0; - if (tgHistoryTotal == 0) { - // there has been no recent activity on this talkgroup. remove it from the stats. - delete sysTalkgroupStats[talkgroupNum]; - } - } + for (const [talkgroupNum, tg] of Object.entries(sysTalkgroupStats)) { + if (!tg.callCountHistory || !tg.callAvgLenHistory) { + console.error(`[${shortName}] Skipping stat for tg: ${talkgroupNum}`); + continue; } - // figure out call totals for this shortName/Sys - if (callTotals[shortName] == undefined) { - callTotals[shortName] = new Array(); - for (var j = 0; j < spots; j++) { - callTotals[shortName][j] = 0; - } + const tgHistoryTotal = tg.callCountHistory.reduce((sum, count) => sum + count, 0) + tg.calls; + + tg.callCountHistory.unshift(tg.calls); + tg.callAvgLenHistory.unshift(tg.calls > 0 ? Math.floor(tg.totalLen / tg.calls) : 0); + + tg.callCountHistory.pop(); + tg.callAvgLenHistory.pop(); + + callTotal += tg.calls; + tg.calls = tg.totalLen = 0; + + if (tgHistoryTotal === 0) { + delete sysTalkgroupStats[talkgroupNum]; } - - for (var j = spots - 1; j > 0; j--) { - callTotals[shortName][j] = callTotals[shortName][j - 1]; - } - callTotals[shortName][0] = callTotal; - - - - const query = { shortName: shortName }; - const update = { $set: { "talkgroupStats": talkgroupStats[shortName], "callTotals": callTotals[shortName] } }; - const options = { upsert: true }; - - await SystemStat.updateOne(query, update, options); } + + if (!callTotals[shortName]) { + callTotals[shortName] = new Array(spots).fill(0); + } + + callTotals[shortName].unshift(callTotal); + callTotals[shortName].pop(); + + bulkOps.push({ + updateOne: { + filter: { shortName }, + update: { $set: { talkgroupStats: sysTalkgroupStats, callTotals: callTotals[shortName] } }, + upsert: true + } + }); } - console.log("Finished Shifting Talkgroup Stats at: " + new Date());*/ + console.log("Finished Shifting Talkgroup Stats at: " + new Date()); + if (bulkOps.length > 0) { + await SystemStat.bulkWrite(bulkOps); + } + console.log("Finished writing to DB at: " + new Date()); updateActiveSystems(); console.log("Finished Updating Active Systems at: " + new Date()); }