From 49f809a22f5a7d412816893c884297b66bbd83eb Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Sat, 31 Aug 2024 13:57:07 -0400 Subject: [PATCH 1/2] do stats with a file --- backend/sys_stats.js | 311 +++++++++++++++++++++++++++---------------- docker-compose.yml | 1 + 2 files changed, 198 insertions(+), 114 deletions(-) diff --git a/backend/sys_stats.js b/backend/sys_stats.js index 0e66c23..7330bcd 100644 --- a/backend/sys_stats.js +++ b/backend/sys_stats.js @@ -1,6 +1,8 @@ var util = require("util"); const SystemStat = require("./models/system_stat"); var System = require("./models/system"); +const fs = require("fs"); +const talkgroup = require("./models/talkgroup"); var talkgroupStats = {}; var callTotals = {}; var uploadErrors = {}; @@ -13,7 +15,9 @@ async function updateActiveSystems() { // Go through all of the Systems let siteTotal = 0; let activeSystems = 0; + console.log(callTotals); for await (let item of System.find()) { + console.log("Checking: " + item.shortName); // go through all the systems // if you have received some calls during that last period, make the system active if ((callTotals[item.shortName] != undefined) && (callTotals[item.shortName] > 0)) { @@ -27,13 +31,7 @@ async function updateActiveSystems() { item.callAvg = 0; } callTotals[item.shortName] = 0; - /* - let fromDate = new Date(Date.now() - 60 * 60 * 24 * 30 * 1000); - if ((item.lastActive != null) && (item.lastActive > fromDate)) { - item.active = true; - } else { - item.active = false; - }*/ + await item.save(); }; console.log("Site average uploads per minute: " + siteTotal / timePeriod); @@ -42,6 +40,34 @@ async function updateActiveSystems() { exports.initStats = async function () { // get the System Stats collection + + // Check if the talkgroups.json file exists + const filePath = "/data/stats/talkgroups.json"; + if (fs.existsSync(filePath)) { + // Read the contents of the file + const fileContents = fs.readFileSync(filePath, "utf8"); + + // Parse the JSON data + try { + talkgroupStats = JSON.parse(fileContents); + } catch (error) { + console.error("Error parsing JSON data:", error); + } + } + + // Check if the decodeErrorsFreq.json file exists + const decodeErrorsFreqPath = "/data/stats/decodeErrorsFreq.json"; + if (fs.existsSync(decodeErrorsFreqPath)) { + // Read the contents of the file + const fileContents = fs.readFileSync(decodeErrorsFreqPath, "utf8"); + + // Parse the JSON data + try { + decodeErrorsFreq = JSON.parse(fileContents); + } catch (error) { + console.error("Error parsing JSON data:", error); + } + } /* for await (const item of SystemStat.find()) { const obj = item.toObject(); @@ -84,71 +110,85 @@ exports.addError = function (call) { // Keeps track of the number of calls for each talkgroup for a system exports.addCall = function (call) { + // if you haven't started keeping track of stats for the System yet + if (callTotals[call.shortName] === undefined) { + callTotals[call.shortName] = 0; + } callTotals[call.shortName]++; - /* - // if you haven't started keeping track of stats for the System yet - if (talkgroupStats[call.shortName] === undefined) { - talkgroupStats[call.shortName] = {}; + + // if you haven't started keeping track of stats for the System yet + if (talkgroupStats[call.shortName] === undefined) { + talkgroupStats[call.shortName] = {}; + } + var sysTalkgroupStats = talkgroupStats[call.shortName]; + // if you haven't started keeping track of Stats for this TG yet... + if (sysTalkgroupStats[call.talkgroupNum] === undefined) { + sysTalkgroupStats[call.talkgroupNum] = {} + sysTalkgroupStats[call.talkgroupNum].calls = 0; + sysTalkgroupStats[call.talkgroupNum].totalLen = 0; + sysTalkgroupStats[call.talkgroupNum].callCountHistory = new Array(); + sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory = new Array(); + for (var j = 0; j < spots; j++) { + sysTalkgroupStats[call.talkgroupNum].callCountHistory[j] = 0; + sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory[j] = 0; } - var sysTalkgroupStats = talkgroupStats[call.shortName]; - // if you haven't started keeping track of Stats for this TG yet... - if (sysTalkgroupStats[call.talkgroupNum] === undefined) { - sysTalkgroupStats[call.talkgroupNum] = {} - sysTalkgroupStats[call.talkgroupNum].calls = 0; - sysTalkgroupStats[call.talkgroupNum].totalLen = 0; - sysTalkgroupStats[call.talkgroupNum].callCountHistory = new Array(); - sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory = new Array(); - for (var j = 0; j < spots; j++) { - sysTalkgroupStats[call.talkgroupNum].callCountHistory[j] = 0; - sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory[j] = 0; - } + } + + // add to the call count and total length, Call Average is calc by dividing the two... + sysTalkgroupStats[call.talkgroupNum].calls++; + sysTalkgroupStats[call.talkgroupNum].totalLen += call.len; + + // if you haven't started keeping track of stats for the System yet + if (decodeErrorsFreq[call.shortName] == undefined) { + decodeErrorsFreq[call.shortName] = {}; + } + + var sysErrors = decodeErrorsFreq[call.shortName]; + + // if you haven't started keeping track of Stats for this TG yet... + if (sysErrors[call.freq] === undefined) { + sysErrors[call.freq] = {} + sysErrors[call.freq].totalLen = 0; + sysErrors[call.freq].errors = 0; + sysErrors[call.freq].spikes = 0; + sysErrors[call.freq].errorHistory = new Array(); + sysErrors[call.freq].spikeHistory = new Array(); + for (var j = 0; j < spots; j++) { + sysErrors[call.freq].errorHistory[j] = 0; + sysErrors[call.freq].spikeHistory[j] = 0; } - - // add to the call count and total length, Call Average is calc by dividing the two... - sysTalkgroupStats[call.talkgroupNum].calls++; - sysTalkgroupStats[call.talkgroupNum].totalLen += call.len; - - // if you haven't started keeping track of stats for the System yet - if (decodeErrorsFreq[call.shortName] == undefined) { - decodeErrorsFreq[call.shortName] = {}; - } - - var sysErrors = decodeErrorsFreq[call.shortName]; - - // if you haven't started keeping track of Stats for this TG yet... - if (sysErrors[call.freq] === undefined) { - sysErrors[call.freq] = {} - sysErrors[call.freq].totalLen = 0; - sysErrors[call.freq].errors = 0; - sysErrors[call.freq].spikes = 0; - sysErrors[call.freq].errorHistory = new Array(); - sysErrors[call.freq].spikeHistory = new Array(); - for (var j = 0; j < spots; j++) { - sysErrors[call.freq].errorHistory[j] = 0; - sysErrors[call.freq].spikeHistory[j] = 0; - } - } - // add to the call count and total length, Call Average is calc by dividing the two... - sysErrors[call.freq].totalLen += call.len; - sysErrors[call.freq].errors += call.errorCount; - sysErrors[call.freq].spikes += call.spikeCount;*/ + } + // add to the call count and total length, Call Average is calc by dividing the two... + sysErrors[call.freq].totalLen += call.len; + sysErrors[call.freq].errors += call.errorCount; + sysErrors[call.freq].spikes += call.spikeCount; } // This gets called when a Time Period is up exports.shiftStats = async function () { -/* + + // Save decodeErrorsFreq to a JSON file + const decodeErrorsFreqPath = "/data/stats/decodeErrorsFreq.json"; + fs.writeFileSync(decodeErrorsFreqPath, JSON.stringify(decodeErrorsFreq)); + console.log("Saved decodeErrorsFreq to JSON file"); + + // Save talkgroupStats to a JSON file + const filePath = "/data/stats/talkgroups.json"; + fs.writeFileSync(filePath, JSON.stringify(talkgroupStats)); + console.log("Saved talkgroupStats to JSON file"); + console.log("Started Shifting Stats at: " + new Date()); // for all the systems in Error Stats for (var shortName in uploadErrors) { if (uploadErrors.hasOwnProperty(shortName)) { // Update the DB with Error Stats and Error Totals - const query = { shortName: shortName }; + /*const query = { shortName: shortName }; const update = { $set: { "uploadErrors": uploadErrors[shortName] } }; const options = { upsert: true }; - await SystemStat.updateOne(query, update, options); + await SystemStat.updateOne(query, update, options);*/ // move everything back one after updating for (var j = spots - 1; j > 0; j--) { uploadErrors[shortName][j] = uploadErrors[shortName][j - 1]; @@ -157,83 +197,126 @@ exports.shiftStats = async function () { uploadErrors[shortName][0] = 0; } } - const bulkOps = []; console.log("Finished Shifting Upload Errors at: " + new Date()); - for (const [shortName, sysErrors] of Object.entries(decodeErrorsFreq)) { - for (const [freqNum, freqErrors] of Object.entries(sysErrors)) { - if (!freqErrors.errorHistory || !freqErrors.spikeHistory) { - console.error(`[${shortName}] Skipping stat for freq: ${freqNum}`); - continue; + + // for each system in decodeErrorsFreq + for (let shortName in decodeErrorsFreq) { + + // if the system is in decodeErrorsFreq + if (decodeErrorsFreq.hasOwnProperty(shortName)) { + var sysErrors = decodeErrorsFreq[shortName]; + + + // for each freq in that systems stats + for (var freqNum in sysErrors) { + if (sysErrors.hasOwnProperty(freqNum)) { + var freqErrors = sysErrors[freqNum]; + + if ((freqErrors.errorHistory == undefined) || (freqErrors.spikeHistory == undefined)) { + console.error("[" + shortName + "] Skipping stat for freq: " + freqNum); + continue; + } + // move the history for that freq back + for (let j = spots - 1; j > 0; j--) { + let i = j - 1; + freqErrors.errorHistory[j] = freqErrors.errorHistory[i]; + freqErrors.spikeHistory[j] = freqErrors.spikeHistory[i]; + } + + if (freqErrors.totalLen > 0) { + freqErrors.errorHistory[0] = (freqErrors.errors / freqErrors.totalLen); + freqErrors.spikeHistory[0] = (freqErrors.spikes / freqErrors.totalLen); + } else { + freqErrors.errorHistory[0] = 0; + freqErrors.spikeHistory[0] = 0; + } + freqErrors.totalLen = 0; + freqErrors.errors = 0; + freqErrors.spikes = 0; + } } - freqErrors.errorHistory.unshift(freqErrors.totalLen > 0 ? freqErrors.errors / freqErrors.totalLen : 0); - freqErrors.spikeHistory.unshift(freqErrors.totalLen > 0 ? freqErrors.spikes / freqErrors.totalLen : 0); + /* + const query = { shortName: shortName }; + const update = { $set: { "decodeErrorsFreq": decodeErrorsFreq[shortName] } }; + const options = { upsert: true }; - freqErrors.errorHistory.pop(); - freqErrors.spikeHistory.pop(); + await SystemStat.updateOne(query, update, options);*/ - freqErrors.totalLen = freqErrors.errors = freqErrors.spikes = 0; } - - bulkOps.push({ - updateOne: { - filter: { shortName }, - update: { $set: { decodeErrorsFreq: sysErrors } }, - upsert: true - } - }); } - - - console.log("Finished Shifting Decode Errors at: " + new Date()); - for (const [shortName, sysTalkgroupStats] of Object.entries(talkgroupStats)) { + // for each system in talkgroupStats + for (let shortName in talkgroupStats) { let callTotal = 0; - for (const [talkgroupNum, tg] of Object.entries(sysTalkgroupStats)) { - if (!tg.callCountHistory || !tg.callAvgLenHistory) { - console.error(`[${shortName}] Skipping stat for tg: ${talkgroupNum}`); - continue; + // if the system is in stats + if (talkgroupStats.hasOwnProperty(shortName)) { + var sysTalkgroupStats = talkgroupStats[shortName]; + + // for each talkgroup in that systems stats + for (var talkgroupNum in sysTalkgroupStats) { + if (sysTalkgroupStats.hasOwnProperty(talkgroupNum)) { + var tg = sysTalkgroupStats[talkgroupNum]; + var tgHistoryTotal = 0; + if ((tg.callCountHistory == undefined) || (tg.callAvgLenHistory == undefined)) { + console.error("[" + shortName + "] Skipping stat for tg: " + talkgroupNum); + continue; + } + // move the history for that talkgroup back + for (let j = spots - 1; j > 0; j--) { + let i = j - 1; + tgHistoryTotal += tg.callCountHistory[i]; + tg.callCountHistory[j] = tg.callCountHistory[i]; + tg.callAvgLenHistory[j] = tg.callAvgLenHistory[i]; + } + + // add to the total for that group + callTotal += tg.calls; + + + // figure out the history for current period in the history for this talkgroup; + tg.callCountHistory[0] = tg.calls; + tgHistoryTotal += tg.calls; + if (tg.calls > 0) { + tg.callAvgLenHistory[0] = Math.floor(tg.totalLen / tg.calls); + } else { + tg.callAvgLenHistory[0] = 0; + } + tg.calls = 0; + tg.totalLen = 0; + if (tgHistoryTotal == 0) { + // there has been no recent activity on this talkgroup. remove it from the stats. + delete sysTalkgroupStats[talkgroupNum]; + } + } } - const tgHistoryTotal = tg.callCountHistory.reduce((sum, count) => sum + count, 0) + tg.calls; - - tg.callCountHistory.unshift(tg.calls); - tg.callAvgLenHistory.unshift(tg.calls > 0 ? Math.floor(tg.totalLen / tg.calls) : 0); - - tg.callCountHistory.pop(); - tg.callAvgLenHistory.pop(); - - callTotal += tg.calls; - tg.calls = tg.totalLen = 0; - - if (tgHistoryTotal === 0) { - delete sysTalkgroupStats[talkgroupNum]; + // figure out call totals for this shortName/Sys + /* if (callTotals[shortName] == undefined) { + callTotals[shortName] = new Array(); + for (var j = 0; j < spots; j++) { + callTotals[shortName][j] = 0; + } } + + for (var j = spots - 1; j > 0; j--) { + callTotals[shortName][j] = callTotals[shortName][j - 1]; + } + callTotals[shortName][0] = callTotal;*/ + + + /* + const query = { shortName: shortName }; + const update = { $set: { "talkgroupStats": talkgroupStats[shortName], "callTotals": callTotals[shortName] } }; + const options = { upsert: true }; + + await SystemStat.updateOne(query, update, options);*/ } - - if (!callTotals[shortName]) { - callTotals[shortName] = new Array(spots).fill(0); - } - - callTotals[shortName].unshift(callTotal); - callTotals[shortName].pop(); - - bulkOps.push({ - updateOne: { - filter: { shortName }, - update: { $set: { talkgroupStats: sysTalkgroupStats, callTotals: callTotals[shortName] } }, - upsert: true - } - }); } console.log("Finished Shifting Talkgroup Stats at: " + new Date()); - if (bulkOps.length > 0) { - await SystemStat.bulkWrite(bulkOps); - } - console.log("Finished writing to DB at: " + new Date());*/ updateActiveSystems(); console.log("Finished Updating Active Systems at: " + new Date()); } diff --git a/docker-compose.yml b/docker-compose.yml index ca3c72d..7b26ff4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,6 +85,7 @@ services: - HONEYCOMB_API_KEY volumes: - "~/.aws:/home/app/.aws" + - ${PWD}/data/stats:/data/stats depends_on: - mongo ports: From 30164dc55cbbb50641cc2a9c3b8f9b85a7af0e98 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Sat, 31 Aug 2024 14:22:52 -0400 Subject: [PATCH 2/2] less logs --- backend/sys_stats.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/sys_stats.js b/backend/sys_stats.js index 7330bcd..d2347ef 100644 --- a/backend/sys_stats.js +++ b/backend/sys_stats.js @@ -15,9 +15,8 @@ async function updateActiveSystems() { // Go through all of the Systems let siteTotal = 0; let activeSystems = 0; - console.log(callTotals); + for await (let item of System.find()) { - console.log("Checking: " + item.shortName); // go through all the systems // if you have received some calls during that last period, make the system active if ((callTotals[item.shortName] != undefined) && (callTotals[item.shortName] > 0)) {