From e052aff634e1179560fb74c2f9286a2ba3cf194e Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Tue, 16 May 2023 21:01:55 -0400 Subject: [PATCH] regular cleanups --- backend/controllers/stats.js | 1 + backend/db.js | 69 +++-- backend/index.js | 17 +- backend/support/call_stats.js | 463 --------------------------------- backend/support/error_stats.js | 164 ------------ backend/support/find-remove.js | 202 -------------- backend/support/gen_stats.js | 14 - backend/sys_stats.js | 1 - 8 files changed, 48 insertions(+), 883 deletions(-) delete mode 100644 backend/support/call_stats.js delete mode 100644 backend/support/error_stats.js delete mode 100644 backend/support/find-remove.js delete mode 100644 backend/support/gen_stats.js diff --git a/backend/controllers/stats.js b/backend/controllers/stats.js index 0e68f80..cd0ff45 100644 --- a/backend/controllers/stats.js +++ b/backend/controllers/stats.js @@ -33,3 +33,4 @@ exports.get_stats = function(req, res) { var statSched = schedule.scheduleJob('*/15 * * * *', function() { sysStats.shiftStats(); }); + diff --git a/backend/db.js b/backend/db.js index 3f8bc28..3b0964d 100644 --- a/backend/db.js +++ b/backend/db.js @@ -1,34 +1,49 @@ -var MongoClient = require('mongodb').MongoClient; +const { CloudWatchLogs } = require("aws-sdk"); +var { callModel: Call } = require("./models/call"); +var Event = require("./models/event"); +var Podcast = require("../models/podcast"); - -var host = process.env['MONGO_NODE_DRIVER_HOST'] != null ? process.env['MONGO_NODE_DRIVER_HOST'] : 'localhost'; -var port = process.env['MONGO_NODE_DRIVER_PORT'] != null ? process.env['MONGO_NODE_DRIVER_PORT'] : 27017; - -var state = { - db: null, +exports.cleanOldEvents = async function() { + var date = new Date(); + Event.bulkWrite([ + { + deleteMany: { + filter: { expireTime: {$lt: date} } + } + } + ]).then(res => { + // Prints "1 1 1" + console.log("Removed " + res.deletedCount + " Podcasts"); + }); } -exports.connect = async function (done) { - if (state.db) return done() - - var url = 'mongodb://' + host + ':' + port + '/scanner'; - - const client = new MongoClient(url); - await client.connect(); - state.db = client.db(); - done(); +exports.cleanOldPodcasts = async function() { + var date = new Date(); + Podcast.bulkWrite([ + { + deleteMany: { + filter: { expireTime: {$lt: date} } + } + } + ]).then(res => { + // Prints "1 1 1" + console.log("Removed " + res.deletedCount + " Events"); + }); } -exports.get = function () { - return state.db -} -exports.close = function (done) { - if (state.db) { - state.db.close(function (err, result) { - state.db = null - state.mode = null - done(err) - }) - } + +exports.cleanOldCalls = async function() { + var date = new Date(); + date.setMonth(date.getMonth() - 1); + Call.bulkWrite([ + { + deleteMany: { + filter: { time: {$lt: date} } + } + } + ]).then(res => { + // Prints "1 1 1" + console.log("Removed " + res.deletedCount + " Calls older than " + date); + }); } \ No newline at end of file diff --git a/backend/index.js b/backend/index.js index ee45cbe..94b8f0a 100644 --- a/backend/index.js +++ b/backend/index.js @@ -10,7 +10,8 @@ var groups = require("./controllers/groups"); var stats = require("./controllers/stats"); var events = require("./controllers/events"); var config = require('./config/config.json'); - +let db = require('./db') +var schedule = require('node-schedule'); var mongoose = require("mongoose"); const { ObjectId } = require('mongodb'); const Group = require("./models/group"); @@ -291,6 +292,9 @@ io.sockets.on('connection', function (client) { }); stats.init_stats(); +let cleanupSched = schedule.scheduleJob('3 * * *', function() { + db.cleanOldCalls(); +}); server.listen(app.get("port"), (err) => { if (err) { @@ -300,16 +304,5 @@ server.listen(app.get("port"), (err) => { } }) -/* -var calcRule = new schedule.RecurrenceRule(); -calcRule.minute = 4; - -var calcSched = schedule.scheduleJob(calcRule, function() { - console.log('Time to calulcate stats'); - call_stats.build_call_volume(); - console.log('Time to calulcate usage'); - call_stats.build_usage(); -}); -*/ module.exports = app; diff --git a/backend/support/call_stats.js b/backend/support/call_stats.js deleted file mode 100644 index 239a430..0000000 --- a/backend/support/call_stats.js +++ /dev/null @@ -1,463 +0,0 @@ -var fs = require('fs'); -var path = require('path'); -var config = require('./config/config.json'); -var mongo = require('mongodb'); -var ObjectID = require('mongodb').ObjectID; -mongo.BSONPure = require('bson').BSONPure; -var BSON = mongo.BSONPure; -var MongoClient = require('mongodb').MongoClient, - assert = require('assert'); -var db = require('./db'); -var util = require("util"), - rl = require('readline'), - spawn = require('child_process').spawn; - - - -function build_call_stats(results) { - var historic = new Array(); - - - // go through all of the results - if (results && (results.length > 0)) { - for (var i = 0; i < results.length; i++) { - // if this is the first time we have seen this shortName in the results, create and array for it in the Historic array - if (historic[results[i]._id.shortName] == undefined) { - historic[results[i]._id.shortName] = new Array(); - for (j = 0; j < 121; j++) { - historic[results[i]._id.shortName][j] = 0; - } - } - // add the number of calls for this system at this hour - historic[results[i]._id.shortName][parseInt(results[i]._id.hour)] = results[i].calls; - - } - - db.get().collection('systems', function(err, sysCollection) { - var cursor = sysCollection.find(); - cursor.each(function(err, item) { - - // go through all the systems - if (item) { - // if a system has had no calls, create a blank history array - if (historic[item.shortName] == undefined) { - historic[item.shortName] = new Array(); - for (j = 0; j < 121; j++) { - historic[item.shortName][j] = 0; - } - } - } else { - // when item == null that means that we have gone through all the systems... - // time to save it. - for (var shortName in historic) { - if (historic.hasOwnProperty(shortName)) { - // do stuff - if (historic[shortName][0]>0) { - var active = true; - } else { - var active = false; - } - - sysCollection.update({ - shortName: shortName - }, { - $set: { - "active": active - } - }, { - upsert: true - }, function(err, objects) { - if (err) console.warn(err.message); - - }); - db.get().collection('system_stats', function(err, statsCollection) { - if (err) console.log(err); - //console.log(); - statsCollection.update({ - shortName: shortName - }, { - $set: { - "callHistory": historic[shortName] - } - }, { - upsert: true - }, function(err, objects) { - if (err) console.warn(err.message); - //console.log(util.inspect(objects)); - }); - }); - } - } - } - }); - }); - //db.get().close(); - } -} - -function build_test_stat(results) { - var historic = new Array(); - - - // go through all of the results - if (results && (results.length > 0)) { - for (var i = 0; i < results.length; i++) { - // if this is the first time we have seen this shortName in the results, create and array for it in the Historic array - if (historic[results[i]._id.shortName] == undefined){ - historic[results[i]._id.shortName] = {}; - } - if (historic[results[i]._id.shortName][results[i]._id.talkgroupNum] == undefined) { - historic[results[i]._id.shortName][results[i]._id.talkgroupNum] = new Array(); - for (j = 0; j < 24; j++) { - historic[results[i]._id.shortName][results[i]._id.talkgroupNum][j] = 0; - } - } - // add the number of calls for this system at this hour - if (parseInt(results[i]._id.hour) < 24) - historic[results[i]._id.shortName][results[i]._id.talkgroupNum][parseInt(results[i]._id.hour)] = results[i].value.count; - - } - } - console.log(util.inspect(results)); - /* - var chan_count = 0; - stats = {}; - var db_count = 0; - for (var chan_num in channels) { - var historic = new Array(); - chan_count++; - - for (hour = 0; hour < 24; hour++) { - - historic[hour] = 0; - } - stats[chan_num] = { - name: channels[chan_num].alpha, - desc: channels[chan_num].desc, - num: chan_num, - historic: historic - }; - var query = { - "_id.talkgroup": parseInt(chan_num) - }; - collection.find(query).toArray(function (err, results) { - db_count++; - if (err) console.log(err); - if (results && (results.length > 0)) { - for (var i = 0; i < results.length; i++) { - stats[results[0]._id.talkgroup].historic[results[i]._id.hour] = results[i].value.count; - } - } - if (chan_count == db_count) { - for (var chan_num in stats) { - var chan = stats[chan_num]; - var erase_me = true; - for (var i = 0; i < chan.historic.length; i++) { - if (chan.historic[i] != 0) { - erase_me = false; - break; - } - } - if (erase_me) { - delete stats[chan_num]; - } - } - - } - }); - - }*/ -} -exports.test2 = function() { - - map = function () { - var now = new Date(); - var difference = now.getTime() - this.time.getTime(); - var hour = Math.floor(difference / 1000 / 60 / 60); - emit({ - hour: hour, - talkgroupNum: this.talkgroupNum, - shortName: this.shortName - }, { - count: 1 - }); - } - - reduce = function (key, values) { - var count = 0; - - values.forEach(function (v) { - count += v['count']; - }); - - return { - count: count - }; - } - db.get().collection('calls', function (err, transCollection) { - var yesterday = new Date(); - yesterday.setDate(yesterday.getDate() - 1); - transCollection.mapReduce(map, reduce, { - query: { - time: { - $gte: yesterday - } - }, - out : { inline: 1 }, - }, function (err, results) { - if (err) console.error(err); - build_test_stat(results); - //console.log(util.inspect(results)); - }); - }); -} -exports.test = function() { - var collection = db.get().collection("calls"); - var now = new Date(); - var start = new Date(); - start.setDate(start.getDate() - 50); - collection.aggregate([ - { - $project: { - shortName: 1, - talkgroupNum: 1, - ago: { - $floor: { - $divide: [{ - $subtract: [now, "$time"] - }, 1000 * 60] - } - }, - "interval": { - "$subtract": [ - { "$minute": "$time" }, - { "$mod": [{ "$minute": "$time"}, 15] } - ] - } - } - }, { - "$group": { - "_id": { - shortName: "$shortName", - talkgroupNum: "$talkgroupNum", - ago: "$ago", - "interval": "$interval" - }, - count: {$sum: 1} - } - }, - {$sort:{_id:-1}} -], function(err, results) { - if (err) console.error("ERror" + err); - - console.log(util.inspect(results)); - - -}); -} -exports.build_tg_volume = function() { - var collection = db.get().collection("calls"); - var now = new Date(); - var start = new Date(); - start.setDate(start.getDate() - 50); - collection.aggregate([{ - $match: { - time: { - $gte: start - } - } - }, { - $project: { - shortName: 1, - hour: { - $floor: { - $divide: [{ - $subtract: [now, "$time"] - }, 1000 * 60 * 60] - } - } - } - }, { - $group: { - _id: { - shortName: "$shortName", - hour: "$hour" - }, - calls: { - $sum: 1 - } - } - }], function(err, results) { - if (err) console.error("ERror" + err); - if (results) { - console.log("Building stats..."); - console.log(util.inspect(results)); - build_call_stats(results); - } - - - }); - } -exports.build_call_volume = function() { - var collection = db.get().collection("calls"); - var now = new Date(); - var start = new Date(); - start.setDate(start.getDate() - 5); - collection.aggregate([{ - $match: { - time: { - $gte: start - } - } - }, { - $project: { - shortName: 1, - hour: { - $floor: { - $divide: [{ - $subtract: [now, "$time"] - }, 1000 * 60 * 60] - } - } - } - }, { - $group: { - _id: { - shortName: "$shortName", - hour: "$hour" - }, - calls: { - $sum: 1 - } - } - }], function(err, results) { - if (err) console.error("ERror" + err); - if (results) { - console.log("Building stats..."); - console.log(util.inspect(results)); - build_call_stats(results); - } - - - }); - } - /* - exports.build_call_volume = function() { - - map = function() { - var now = new Date(); - var difference = now.getTime() - this.time.getTime(); - var hour = Math.floor(difference / 1000 / 60 / 60); - emit({ - hour: hour, - shortName: this.shortName - }, { - count: 1 - }); - } - - reduce = function(key, values) { - var count = 0; - - values.forEach(function(v) { - count += v['count']; - }); - printjson(count); - return { - count: count - }; - } - db.get().collection('calls', function(err, callsCollection) { - var start = new Date(); - start.setDate(start.getDate() - 5); - callsCollection.mapReduce(map, reduce, { - query: { - time: { - $gte: start - } - }, - out: { - replace: "call_volume" - } - }, function(err, collection) { - if (err) console.error("ERror" + err); - if (collection) { - console.log("Building stats..."); - build_call_stats(collection); - } - }); - }); - }; - */ - -exports.build_usage = function() { - var base_path = config.mediaDirectory; - var total = 0; - var sysCount = 0; - db.get().collection('systems', function(err, sysCollection) { - var cursor = sysCollection.find(); - cursor.each(function(err, item) { - if (item) { - - var local_path = config.mediaDirectory + "/" + item.shortName + "/"; - console.log("System: " + item.shortName + " [ " + local_path + " ]"); - - var du = spawn('du', ['-s', local_path]); - //var ls = spawn('ls', ['-lh', '/usr']); - - //linereader = rl.createInterface(du.stdout, du.stdin); - - // Read line by line. - //du.stdout.on('data', function (data) { - - du.stderr.on('data', (data) => { - console.log(`stderr: ${data}`); - }); - - du.on('close', (code) => { - //console.log(`child process exited with code ${code}`); - }); - /* - ls.stdout.on('data', (data) => { - console.log(data); - console.log(`stdout: ${data}`); - }); - */ - du.stdout.on('data', function(data) { - var out = String(data); - var arr = out.split('\t'); - var size = parseInt(arr[0]); - console.log(out + " Size: " + size); - - - - - db.get().collection('system_stats', function(err, statsCollection) { - - - statsCollection.update({ - shortName: item.shortName - - }, { - $set: { - "usageBytes": size, - "usageMb": (size / 1024) - } - }, { - upsert: true - }, function(err, objects) { - if (err) console.warn(err.message); - - }); - }); - - }); - } else { - console.log("Processed: " + sysCount); - - } - - }); - }); - -}; diff --git a/backend/support/error_stats.js b/backend/support/error_stats.js deleted file mode 100644 index feab91d..0000000 --- a/backend/support/error_stats.js +++ /dev/null @@ -1,164 +0,0 @@ -var fs = require('fs'); -var path = require('path'); -var config = require('./config/config.json'); -var csvWriter = require('csv-write-stream'); - -var mongo = require('mongodb'); -var ObjectID = require('mongodb').ObjectID; -mongo.BSONPure = require('bson').BSONPure; -var BSON = mongo.BSONPure; -var MongoClient = require('mongodb').MongoClient, - assert = require('assert'); -var db = require('./db'); -var getSize = require('get-folder-size'); -var util = require("util"), - rl = require('readline'), - spawn = require('child_process').spawn; - -exports.test = function() { - - - map = function() { - - for (var idx = 0; idx < this.freqList.length; idx++) { - var key = this.freqList[idx].freq; - if (this.freqList[idx].errors >= 0) { - if (this.freqList[idx].len > 0) { - var errRatio = Math.round((this.freqList[idx].errors / this.freqList[idx].len) * 100 * 1e2) / 1e2 ; - } else { - var errRatio = 0; - } - var value = { - - "spikes": this.freqList[idx].spikes, - "errRatio": errRatio - }; - emit(key, value); - } - } - - } - - - reduce = function(key, values) { - var spikes = new Array(); - var errors = new Array(); - printjson(key); - printjson(values); - values.forEach(function(v) { - spikes.push(v['spikes']); - errors.push(v['errRatio']); - }); - return { - "spikes": spikes, - "errors": errors - }; - } - db.get().collection('calls', function(err, transCollection) { - - transCollection.mapReduce(map, reduce, { - query: { - "shortName": "tester" - }, - out: { - inline: 1 - }, - }, function(err, results) { - if (err) { - console.error(err); - } else { - - for (var freq in results) { - if (results[freq].value.spikes) { - writer.pipe(fs.createWriteStream(results[freq]._id + '.csv')); - - for (var idx = 0; idx < results[freq].value.spikes.length; idx++) { - writer.write({ - "spikes": results[freq].value.spikes[idx], - "errors": results[freq].value.errors[idx] - }); - } - } - /*results[freq].forEach(function(err){ - writer.write(err) - });*/ - - console.log("Freq: " + freq + "\n" + util.inspect(results[freq])); - } - writer.end(); - } - - - - }); - }); -} - -db.connect(function(err) { - if (err) { - console.log('Unable to connect to Mongo.') - process.exit(1) - } else { - //call_stats.build_usage(); - //call_stats.build_call_volume(); - //exports.test(); - var start = new Date(); - start.setHours(start.getHours() - 1); - var collection = db.get().collection("calls"); - collection.aggregate([ - { - $match: { - time: { - $gte: start - }, - shortName: "wmata" - } - }, - - { "$unwind": "$freqList" - }, - { "$project": { - "freqList":1, - "errorRatio": { $cond: [ { $eq: [ "$freqList.len", 0 ] }, 0, {"$multiply": [ {"$divide": ["$freqList.errors", "$freqList.len"]}, 100]} ] }, - "time": { $dateToString: { format: "%H:%M:%S", date: "$time" } } - }}, - { - "$group": { - "_id": { - freq: "$freqList.freq" - }, - - "time": { $push: "$time"}, - "errors": { $push: "$errorRatio"}, - "spikes": { $push: "$freqList.spikes"} - } - }, - {$sort:{_id:-1}} - ], function(err, results) { - if (err) console.error("ERror" + err); - - - for (var freq in results) { - if (results[freq].errors) { - var writer = csvWriter(); - writer.pipe(fs.createWriteStream(results[freq]._id.freq + '.csv')); - - for (var idx = 0; idx < results[freq].spikes.length; idx++) { - writer.write({ - "time": results[freq].time[idx], - "spikes": results[freq].spikes[idx], - "errors": results[freq].errors[idx] - }); - } - writer.end(); - console.log("Freq: " + freq + "\n" + util.inspect(results[freq])); - } - - - - } - - }); - - } -}) diff --git a/backend/support/find-remove.js b/backend/support/find-remove.js deleted file mode 100644 index 5c4c01f..0000000 --- a/backend/support/find-remove.js +++ /dev/null @@ -1,202 +0,0 @@ -var fs = require('fs'), - path = require('path'), - merge = require('fmerge'), - util = require('util'), - rimraf = require('rimraf'), - - now, - testRun - - var totals = {fileSize: 0, deletedSize:0, files: 0, deleted:0} - -function isOlder(path, ageSeconds) { - var stats = fs.statSync(path), - mtime = stats.mtime.getTime(), - expirationTime = (mtime + (ageSeconds * 1000)) - - return now > expirationTime -} - -function hasMaxLevel(options) { - return options && options.hasOwnProperty('maxLevel') -} - -function getMaxLevel(options) { - return hasMaxLevel(options) ? options.maxLevel : -1 -} - -function getAgeSeconds(options) { - return (options && options.age && options.age.seconds) ? options.age.seconds : null -} - -function doDeleteDirectory(currentDir, options, currentLevel) { - - var doDelete = false - var dir = options && options.dir - - if (dir) { - var ageSeconds = getAgeSeconds(options) - var basename = path.basename(currentDir) - - if (util.isArray(dir)) { - doDelete = (dir.indexOf("*") !== 1) || (dir.indexOf(basename) !== -1) - } else if (basename === dir || dir === "*") { - doDelete = true - } - - if (doDelete && hasMaxLevel(options) && currentLevel > 0) { - doDelete = currentLevel <= getMaxLevel(options) - } - - if (ageSeconds && doDelete) { - doDelete = isOlder(currentDir, ageSeconds) - } - } - - return doDelete -} - -function doDeleteFile(currentFile, options) { - // by default it deletes nothing - var doDelete = false - - var extensions = (options && options.extensions) ? options.extensions : null - var files = (options && options.files) ? options.files : null - var dir = (options && options.dir) ? options.dir : null - var ignore = (options && options.ignore) ? options.ignore : null - - // return the last portion of a path, the filename aka basename - var basename = path.basename(currentFile) - - if (files) { - if (util.isArray(files)) - doDelete = (files.indexOf("*.*") !== -1) || (files.indexOf(basename) !== -1) - else { - if (files === '*.*') { - doDelete = true - } else { - doDelete = (basename === files) - } - } - } - - if (!doDelete && extensions) { - var currentExt = path.extname(currentFile) - - if (util.isArray(extensions)) { - doDelete = extensions.indexOf(currentExt) !== -1 - } else { - doDelete = (currentExt === extensions) - } - } - - if (doDelete && ignore) { - if (util.isArray(ignore)) - doDelete = !(ignore.indexOf(basename) !== -1) - else - doDelete = !(basename === ignore) - } - - if (doDelete) { - var ageSeconds = getAgeSeconds(options) - - if (ageSeconds) - doDelete = isOlder(currentFile, ageSeconds) - } - - return doDelete -} - -function isTestRun(options) { - return (options && options.hasOwnProperty('test')) ? options.test : false -} - -/** - * findRemoveSync(currentDir, options) takes any start directory and searches files from there for removal. - * the selection of files for removal depends on the given options. when no options are given, or only the maxLevel - * parameter is given, then everything is removed as if there were no filters. - * - * beware: everything happens synchronously. - * - * - * @param {String} currentDir any directory to operate within. it will seek files and/or directories recursively from there. - * beware that it deletes the given currentDir when no options or only the maxLevel parameter are given. - * @param options json object with optional properties like extensions, files, ignore, maxLevel and age.seconds. - * @return {Object} json object of files and/or directories that were found and successfully removed. - * @api public - */ -var findRemoveSync = module.exports = function(currentDir, options, currentLevel) { - - var removed = {} - - if (fs.existsSync(currentDir)) { - - var maxLevel = getMaxLevel(options), - deleteDirectory = false - - var removeEmpty = (options && options.removeEmpty); - if (currentLevel === undefined) - currentLevel = 0 - else - currentLevel++ - - if (currentLevel < 1) { - now = new Date().getTime() - totals = {fileSize: 0, deletedSize:0, files: 0, deleted:0}; - testRun = isTestRun(options) - } else { - // check directories before deleting files inside. - // this to maintain the original creation time, - // because linux modifies creation date of folders when files within have been deleted. - deleteDirectory = doDeleteDirectory(currentDir, options, currentLevel) - } - - if (maxLevel === -1 || currentLevel < maxLevel) { - var filesInDir = fs.readdirSync(currentDir) - - filesInDir.forEach(function(file) { - - var currentFile = path.join(currentDir, file) - var stat = fs.statSync(currentFile); - if (stat.isDirectory()) { - // the recursive call - var result = findRemoveSync(currentFile, options, currentLevel) - - // merge results - removed = merge(removed, result) - } else { - - if (doDeleteFile(currentFile, options)) { - if (!testRun) - fs.unlinkSync(currentFile) - totals.deletedSize += stat.size - totals.deleted++; - removed[currentFile] = true - } else { - totals.fileSize += stat.size - totals.files++ - } - } - }) - if (removeEmpty && (filesInDir<1)) { - deleteDirectory = true; - } - } - - if (deleteDirectory) { - try { - if (!testRun) - rimraf.sync(currentDir) - - removed[currentDir] = true - } catch (err) { - throw err - } - } - } - if (currentLevel==0) { - return {removed: removed, totals: totals} - } else { - return removed - } -} diff --git a/backend/support/gen_stats.js b/backend/support/gen_stats.js deleted file mode 100644 index b5d013f..0000000 --- a/backend/support/gen_stats.js +++ /dev/null @@ -1,14 +0,0 @@ -var call_stats = require('./call_stats'); -var db = require('./db'); - -db.connect(function(err) { - if (err) { - console.log('Unable to connect to Mongo.') - process.exit(1) - } else { - //call_stats.build_usage(); - //call_stats.build_call_volume(); - call_stats.test2(); - - } -}) diff --git a/backend/sys_stats.js b/backend/sys_stats.js index 3697f49..f4630a4 100644 --- a/backend/sys_stats.js +++ b/backend/sys_stats.js @@ -1,5 +1,4 @@ var util = require("util"); -var db = require('./db'); const SystemStat = require("./models/system_stat"); var System = require("./models/system"); var talkgroupStats = {};