GitHub/trunk-server
1
0
Fork 0
mirror of https://github.com/openmhz/trunk-server.git synced 2026-11-12 08:47:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

cleaned up stats collection

Browse Source
This commit is contained in:
Luke Berndt
2023-04-07 09:32:58 -04:00
parent 49c794bcaa
commit 70cd2cd14e
5 changed files with 156 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
admin/server/controllers/systems.js
View File

@@ -72,10 +72,9 @@ exports.listSystems = async function (req, res, next) {
sys_stats[stats[i].shortName] = {};
sys_stats[stats[i].shortName].callTotals = stats[i].callTotals;
sys_stats[stats[i].shortName].errorTotals = stats[i].errorTotals;
sys_stats[stats[i].shortName].errorStats = stats[i].errorStats;
sys_stats[stats[i].shortName].usageBytes = stats[i].usageBytes;
sys_stats[stats[i].shortName].usageMb = stats[i].usageMb;
sys_stats[stats[i].shortName].uploadErrors = stats[i].uploadErrors;
sys_stats[stats[i].shortName].talkgroupStats= stats[i].talkgroupStats;
sys_stats[stats[i].shortName].decodeErrorsFreq = stats[i].decodeErrorsFreq;
}
res.json({

80
backend/controllers/stats.js
View File

@@ -1,5 +1,3 @@
var db = require('../db');
var schedule = require('node-schedule');
var sysStats = require("../sys_stats");
const util = require('util');
@@ -8,80 +6,20 @@ exports.init_stats = function() {
sysStats.initStats();
}
exports.get_errors = function(req, res) {
var short_name = req.params.shortName.toLowerCase();
var start = new Date();
start.setHours(start.getHours() - 1);
var collection = db.get().collection("calls");
collection.aggregate([{
$match: {
time: {
$gte: start
},
shortName: short_name
}
},
{
"$unwind": "$freqList"
},
{
"$project": {
"freqList": 1,
"errorRatio": {
$cond: [{
$eq: ["$freqList.len", 0]
}, 0, {
"$multiply": [{
"$divide": ["$freqList.error", "$freqList.len"]
}, 100]
}]
},
"time": "$time"
}
},
{
"$group": {
"_id": "$freqList.freq",
"values": {
"$push": {
"time": "$time",
"errors": "$errorRatio",
"spikes": "$freqList.spikes"
}
}
}
},
{
$sort: {
_id: -1
}
}
], function(err, results) {
if (err) {
console.warn("Error with error stats: " + util.inspect(req.params) + " err: " + err);
res.status(500);
res.send("Error with error stats: " + util.inspect(req.params) + " err: " + err);
} else {
res.contentType('json');
res.send(JSON.stringify(results));
}
});
}
exports.get_stats = function(req, res) {
var short_name = req.params.shortName.toLowerCase();
var callHistory = sysStats.callHistory(short_name);
var tgHistory = sysStats.tgHistory(short_name);
var errorTotals = sysStats.errorTotals(short_name);
if (callHistory && tgHistory) {
var callTotals = sysStats.callTotals(short_name);
var talkgroupStats = sysStats.talkgroupStats(short_name);
var uploadErrors = sysStats.uploadErrors(short_name);
let decodeErrorsFreq = sysStats.decodeErrorsFreq(short_name);
if (callTotals && talkgroupStats) {
res.contentType('json');
res.send(JSON.stringify({
callHistory: callHistory,
tgHistory: tgHistory,
errorTotals: errorTotals
callTotals: callTotals,
talkgroupStats: talkgroupStats,
uploadErrors: uploadErrors,
decodeErrorsFreq: decodeErrorsFreq
}));
} else {
console.log("ShortName didn't exist for stats: " + util.inspect(req.params));

63
backend/controllers/uploads.js
View File

@@ -145,6 +145,8 @@ exports.upload = function (req, res, next) {
return;
}
res.status(200).end();
var base_path = config.mediaDirectory;
var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/";
var target_file = base_path + local_path + path.basename(req.file.originalname);
@@ -183,61 +185,10 @@ exports.upload = function (req, res, next) {
} else {
call.len = (stopTime - time) / 1000; //await callLength(req.file.path);
}
res.status(200).end();
if (!objectStore) {
mkdirp.sync(base_path + local_path, function (err) {
if (err) console.error(err);
});
var src = fs.createReadStream(req.file.path);
var dest = fs.createWriteStream(target_file);
dest.on('unpipe', (src) => {
src.end();
dest.end();
});
src.pipe(dest);
src.unpipe()
src.on('error', function (err) {
console.warn("[" + call.shortName + "] Error /:shortName/upload copying to location - Error: " + err + " Read: " + req.file.path + " Target: " + target_file);
fs.unlink(req.file.path, (err) => {
if (err)
console.log("[" + call.shortName + "] error deleting: " + req.file.path);
});
res.contentType('json');
res.send(JSON.stringify({
success: false,
error: "Upload failed"
}));
});
src.on('end', function () {
fs.unlink(req.file.path);
});
} else {
//var spacesSrc = fs.createReadStream(req.file.path);
//var awsSrc = fs.createReadStream(req.file.path);
var wasabiSrc = fs.createReadStream(req.file.path);
// call S3 to retrieve upload file to specified bucket
/* var spacesParams = {Bucket: 'openmhz', Key: object_key, Body: spacesSrc, ACL: 'public-read'};
// call S3 to retrieve upload file to specified bucket
spacesS3.upload (spacesParams, function (err, data) {
//fs.unlink(req.file.path);
if (err) {
console.log("Spaces Error", err);
}
});*/
/*
var awsParams = {Bucket: 'openmhz', Key: object_key, Body: awsSrc, ACL: 'public-read'};
awsS3.upload (awsParams, function (err, data) {
fs.unlink(req.file.path);
if (err) {
console.log("AWS Error", err);
}
});*/
var wasabiParams = {
Bucket: s3_bucket,
Key: object_key,
@@ -252,7 +203,7 @@ exports.upload = function (req, res, next) {
partSize: 10 * 1024 * 1024, queueSize: 1,
params: wasabiParams
});*/
//wasabiS3.upload(wasabiParams, options, function(err, data) {
wasabiS3.upload(wasabiParams, function (err, data) {
wasabiSrc.destroy();
fs.unlink(req.file.path, (err) => {
@@ -272,24 +223,22 @@ exports.upload = function (req, res, next) {
console.error("[" + call.shortName + "] " + call.name + " - content-length: " + req.headers['content-length'] + " Wasabi Error", err);
}
});
}
} catch (err) {
console.warn("[" + call.shortName + "] Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length']);
res.status(500);
sysStats.addError(call.toObject());
res.contentType('json');
res.status(500);
res.send(JSON.stringify({
success: false,
error: "FFProbe"
error: "File Upload"
}));
fs.unlink(req.file.path, (err) => {
if (err)
console.log("error deleting: " + req.file.path);
});
}
});
});
}

1
backend/index.js
View File

@@ -217,7 +217,6 @@ app.get('/events', events.getEvents);
app.get('/events/:id', events.getEvent);
/*------ STATS ----------*/
app.get('/:shortName/errors', stats.get_errors);
app.get('/:shortName/stats', stats.get_stats);

193
backend/sys_stats.js
View File

@@ -1,8 +1,9 @@
var util = require("util");
var db = require('./db');
var stats = {};
var talkgroupStats = {};
var callTotals = {};
var errorTotals = {};
var uploadErrors = {};
let decodeErrorsFreq = {};
const timePeriod = 15; // in minutes
var spots = (24*60) / timePeriod; // the number of spots needed to keep track of 24 hours of stats
@@ -73,12 +74,17 @@ exports.initStats = function() {
// Talkgroup Stats
if (item.talkgroupStats != undefined) {
stats[item.shortName] = item.talkgroupStats;
talkgroupStats[item.shortName] = item.talkgroupStats;
}
// Error Totals
if (item.errorTotals != undefined) {
errorTotals[item.shortName] = item.errorTotals;
// Decode Errors
if (item.decodeErrorsFreq != undefined) {
decodeErrorsFreq[item.shortName] = item.decodeErrorsFreq;
}
// Upload Error Totals
if (item.uploadErrors != undefined) {
uploadErrors[item.shortName] = item.uploadErrors;
}
if (item.callTotals != undefined) {
@@ -109,30 +115,54 @@ exports.addError = function(call) {
// Keeps track of the number of calls for each talkgroup for a system
exports.addCall = function(call) {
// if you haven't started keeping track of stats for the System yet
if (stats[call.shortName] == undefined) {
stats[call.shortName] = {};
if (talkgroupStats[call.shortName] == undefined) {
talkgroupStats[call.shortName] = {};
}
var stat = stats[call.shortName];
var sysTalkgroupStats = talkgroupStats[call.shortName];
// if you haven't started keeping track of Stats for this TG yet...
if (stat[call.talkgroupNum] == undefined) {
stat[call.talkgroupNum] = {}
stat[call.talkgroupNum].calls = 0;
stat[call.talkgroupNum].totalLen = 0;
stat[call.talkgroupNum].callCount = new Array();
stat[call.talkgroupNum].avgLen = new Array();
if (sysTalkgroupStats[call.talkgroupNum] == undefined) {
sysTalkgroupStats[call.talkgroupNum] = {}
sysTalkgroupStats[call.talkgroupNum].calls = 0;
sysTalkgroupStats[call.talkgroupNum].totalLen = 0;
sysTalkgroupStats[call.talkgroupNum].callCountHistory = new Array();
sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory = new Array();
for (var j = 0; j < spots; j++) {
stat[call.talkgroupNum].callCount[j] = 0;
stat[call.talkgroupNum].avgLen[j] = 0;
sysTalkgroupStats[call.talkgroupNum].callCountHistory[j] = 0;
sysTalkgroupStats[call.talkgroupNum].callAvgLenHistory[j] = 0;
}
}
// add to the call count and total length, Call Average is calc by dividing the two...
stat[call.talkgroupNum].calls++;
stat[call.talkgroupNum].totalLen += call.len;
sysTalkgroupStats[call.talkgroupNum].calls++;
sysTalkgroupStats[call.talkgroupNum].totalLen += call.len;
// if you haven't started keeping track of stats for the System yet
if (decodeErrorsFreq[call.shortName] == undefined) {
decodeErrorsFreq[call.shortName] = {};
}
var sysErrors = decodeErrorsFreq[call.shortName];
// if you haven't started keeping track of Stats for this TG yet...
if (sysErrors[call.freq] == undefined) {
sysErrors[call.freq] = {}
sysErrors[call.freq].totalLen = 0;
sysErrors[call.freq].errors = 0;
sysErrors[call.freq].spikes = 0;
sysErrors[call.freq].errorHistory = new Array();
sysErrors[call.freq].spikeHistory = new Array();
for (var j = 0; j < spots; j++) {
sysErrors[call.freq].errors[j] = 0;
sysErrors[call.freq].spikes [j] = 0;
}
}
// add to the call count and total length, Call Average is calc by dividing the two...
sysErrors[call.freq].totalLen += call.len;
sysErrors[call.freq].errors += call.errorCount;
sysErrors[call.freq].spikes += call.spikeCount;
}
@@ -141,69 +171,121 @@ exports.shiftStats = function() {
db.get().collection('system_stats', function(err, statsCollection) {
// for all the systems in Error Stats
for (var shortName in errorTotals) {
if (stats.hasOwnProperty(shortName)) {
for (var shortName in uploadErrors) {
if (upload.hasOwnProperty(shortName)) {
// Update the DB with Error Stats and Error Totals
statsCollection.update({
shortName: shortName
}, {
$set: {
"errorTotals": errorTotals[shortName]
"uploadErrors": uploadErrors[shortName]
}
}, {
upsert: true
}, function(err, objects) {
if (err) {
console.log("Shortname: " + shortName);
console.log("errorTotals: " + util.inspect(errorTotals[shortName]));
console.log("uploadErrors: " + util.inspect(uploadErrors[shortName]));
console.warn(err.message);
}
});
// move everything back one after updating
for (var j = spots - 1; j > 0; j--) {
errorTotals[shortName][j] = errorTotals[shortName][j - 1];
uploadErrors[shortName][j] = uploadErrors[shortName][j - 1];
}
// reset the first spot back to 0, so you can count the next period
errorTotals[shortName][0] = 0;
uploadErrors[shortName][0] = 0;
}
}
// figure out the stats
// for each system in decodeErrorsFreq
for (let shortName in decodeErrorsFreq) {
// for each system in stats
for (var shortName in stats) {
var callTotal = 0;
// if the system is in decodeErrorsFreq
if (decodeErrorsFreq.hasOwnProperty(shortName)) {
var sysErrors = decodeErrorsFreq[shortName];
// for each freq in that systems stats
for (var freqNum in sysErrors ) {
if (sysErrors.hasOwnProperty(freqNum)) {
var freqErrors = sysErrors[freqNum];
// move the history for that freq back
for (let j = spots - 1; j > 0; j--) {
let i = j - 1;
freqErrors.errorHistory[j] = freqErrors.errorHistory[i];
freqErrors.spikeHistory[j] = freqErrors.spikeHistory[i];
}
if (freqErrors.totalLen > 0) {
freqErrors.errorHistory[0] = (freqErrors.errors / freqErrors.totalLen);
freqErrors.spikeHistory[0] = (freqErrors.spikes / freqErrors.totalLen);
} else {
freqErrors.errorHistory[0] = 0;
freqErrors.spikeHistory[0] = 0;
}
freqErrors.totalLen = 0;
freqErrors.errors = 0;
freqErrors.spikes = 0;
}
}
statsCollection.update({
shortName: shortName
}, {
$set: {
"decodeErrorsFreq": decodeErrorsFreq[shortName],
}
}, {
upsert: true
}, function(err, objects) {
if (err){
console.log("Shortname: " + shortName);
console.log("decodeErrorsFreq: " + util.inspect(decodeErrorsFreq[shortName]));
console.warn(err.message);
}
});
}
}
// for each system in talkgroupStats
for (let shortName in talkgroupStats) {
let callTotal = 0;
// if the system is in stats
if (stats.hasOwnProperty(shortName)) {
var stat = stats[shortName];
if (talkgroupStats.hasOwnProperty(shortName)) {
var sysTalkgroupStats = talkgroupStats[shortName];
// for each talkgroup in that systems stats
for (var talkgroupNum in stat) {
if (stat.hasOwnProperty(talkgroupNum)) {
var tg = stat[talkgroupNum];
for (var talkgroupNum in sysTalkgroupStats ) {
if (sysTalkgroupStats.hasOwnProperty(talkgroupNum)) {
var tg = sysTalkgroupStats[talkgroupNum];
var tgHistoryTotal = 0;
// move the history for that talkgroup back
for (var j = spots - 1; j > 0; j--) {
var i = j - 1;
tgHistoryTotal += tg.callCount[i];
tg.callCount[j] = tg.callCount[i];
tg.avgLen[j] = tg.avgLen[i];
for (let j = spots - 1; j > 0; j--) {
let i = j - 1;
tgHistoryTotal += tg.callCountHistory[i];
tg.callCountHistory[j] = tg.callCountHistory[i];
tg.callAvgLenHistory[j] = tg.callAvgLenHistory[i];
}
// add to the total for that group
callTotal += tg.calls;
// figure out the history for cell 0;
tg.callCount[0] = tg.calls;
// figure out the history for current period in the history for this talkgroup;
tg.callCountHistory[0] = tg.calls;
tgHistoryTotal += tg.calls;
if (tg.calls > 0) {
tg.avgLen[0] = Math.floor(tg.totalLen / tg.calls);
tg.callAvgLenHistory[0] = Math.floor(tg.totalLen / tg.calls);
} else {
tg.avgLen[0] = 0;
tg.callAvgLenHistory[0] = 0;
}
tg.calls = 0;
tg.totalLen = 0;
@@ -222,7 +304,6 @@ exports.shiftStats = function() {
}
}
for (var j = spots - 1; j > 0; j--) {
callTotals[shortName][j] = callTotals[shortName][j - 1];
}
@@ -232,7 +313,7 @@ exports.shiftStats = function() {
shortName: shortName
}, {
$set: {
"talkgroupStats": stats[shortName],
"talkgroupStats": talkgroupStats[shortName],
"callTotals": callTotals[shortName]
}
}, {
@@ -240,7 +321,7 @@ exports.shiftStats = function() {
}, function(err, objects) {
if (err){
console.log("Shortname: " + shortName);
console.log("TgStats: " + util.inspect(stats[shortName]));
console.log("TgStats: " + util.inspect(talkgroupStats[shortName]));
console.log("Call Totals: " + util.inspect(callTotals[shortName]));
console.warn(err.message);
}
@@ -251,13 +332,15 @@ exports.shiftStats = function() {
});
updateActiveSystems();
}
exports.callHistory = function(shortName) {
exports.callTotals = function(shortName) {
return callTotals[shortName];
}
exports.tgHistory = function(shortName) {
return stats[shortName];
exports.talkgroupStats = function(shortName) {
return talkgroupStats[shortName];
}
exports.errorTotals = function(shortName) {
return errorTotals[shortName];
exports.uploadErrors = function(shortName) {
return uploadErrors[shortName];
}
exports.decodeErrorsFreq = function(shortName) {
return decodeErrorsFreq[shortName];
}