diff --git a/README.md b/README.md index 0445f82..7ce29f7 100644 --- a/README.md +++ b/README.md @@ -242,14 +242,14 @@ It is a huge pain to upgrade MongoDB in place. It turns out to be easier to dump Rough Playbook (use common sense, I may not have this exact): - get into the shell of the mongo container -- `mongodump --db scanner —out /data/db/backup` +- `mongodump --uri="mongodb://127.0.0.1" --db scanner --out /data/db/backup` - exit contianer and go back to host machine - `cd data/db` - `rm *` erase everything... but not the sub-directories because that is where the backup is - upgrade to the latest version of mongo - build and launch the mongo container, which will create an empty DB - get into the shell of the mongo container -- `mongorestore --db scanner --drop /data/db/backup/` +- `mongorestore --db scanner --drop /data/db/backup/scanner` ### Privlege Port Error diff --git a/backend/agents/honeycomb-tracing.js b/backend/agents/honeycomb-tracing.js index a8aa463..ef7a329 100644 --- a/backend/agents/honeycomb-tracing.js +++ b/backend/agents/honeycomb-tracing.js @@ -8,6 +8,7 @@ const { // uses the HONEYCOMB_API_KEY and OTEL_SERVICE_NAME environment variables const sdk = new HoneycombSDK({ + sampleRate: 5, instrumentations: [ getNodeAutoInstrumentations({ // we recommend disabling fs autoinstrumentation since it can be noisy diff --git a/backend/controllers/uploads.js b/backend/controllers/uploads.js index 435b3c4..8e4b9fe 100644 --- a/backend/controllers/uploads.js +++ b/backend/controllers/uploads.js @@ -1,10 +1,12 @@ var path = require('path'); var fs = require('fs'); +const opentelemetry = require('@opentelemetry/api'); var sysStats = require("../sys_stats"); var System = require("../models/system"); var Talkgroup = require("../models/talkgroup"); var { callModel: Call } = require("../models/call"); + const { PutObjectCommand, S3Client } = require("@aws-sdk/client-s3"); const { fromIni } = require("@aws-sdk/credential-providers"); @@ -22,179 +24,266 @@ const client = new S3Client({ }); exports.upload = async function (req, res, next) { - process.nextTick( async () => { - if (!req.file || ((path.extname(req.file.originalname) != '.m4a') && (path.extname(req.file.originalname) != '.mp3'))) { - console.warn("[" + req.params.shortName + "] Error file name is wrong or file does not exist"); - res.status(500); - res.send("Error, invalid filename:\n"); - return; - } - var shortName = req.params.shortName.toLowerCase(); - var apiKey = req.body.api_key; - let item = null; - - try { - item = await System.findOne({ 'shortName': shortName }, ["key", "ignoreUnknownTalkgroup"]); - } catch (err) { - console.warn("[" + req.params.shortName + "] Error /:shortName/upload - Error: " + err); - res.status(500); - res.send("Error, invalid shotName:\n" + err); - return; - } - - if (!item) { - console.info("[" + req.params.shortName + "] Error /:shortName/upload ShortName does not exist"); - res.status(500); - res.send("ShortName does not exist: " + shortName + "\n"); - return; - } - if (apiKey != item.key) { - console.warn("[" + req.params.shortName + "] Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey); - res.status(500); - res.send("API Keys do not match!\n"); - return; - } - - var talkgroupNum = parseInt(req.body.talkgroup_num); - var freq = parseFloat(req.body.freq); - var time = new Date(parseInt(req.body.start_time) * 1000); - var stopTime = new Date(); - if (req.body.stop_time) { - var stopTime = new Date(parseInt(req.body.stop_time) * 1000); - } - var startTime = req.body.start_time; - var emergency = parseInt(req.body.emergency); - - // Blocking sensitive talkgroups -if ((shortName == "hennearmer") && ((talkgroupNum == 3421) || (talkgroupNum == 3423))) { - res.status(200).end(); - return; -} + const tracer = trace.getTracer('upload-tracer'); - if (item.ignoreUnknownTalkgroup == true) { - talkgroupExists = await Talkgroup.exists({ - 'shortName': shortName, - 'num': talkgroupNum + process.nextTick(async () => { + if (!req.file || ((path.extname(req.file.originalname) != '.m4a') && (path.extname(req.file.originalname) != '.mp3'))) { + console.warn("[" + req.params.shortName + "] Error file name is wrong or file does not exist"); + res.status(500); + res.send("Error, invalid filename:\n"); + return; + } + var shortName = req.params.shortName.toLowerCase(); + var apiKey = req.body.api_key; + var talkgroupNum = parseInt(req.body.talkgroup_num); + var freq = parseFloat(req.body.freq); + var time = new Date(parseInt(req.body.start_time) * 1000); + var stopTime = new Date(); + if (req.body.stop_time) { + var stopTime = new Date(parseInt(req.body.stop_time) * 1000); + } + var startTime = req.body.start_time; + var emergency = parseInt(req.body.emergency); + + let errorCount = parseInt(req.body.error_count); + let spikeCount = parseInt(req.body.spike_count); + + if (Number.isNaN(errorCount)) { + errorCount = 0; + } + if (Number.isNaN(spikeCount)) { + spikeCount = 0; + } + + try { + var srcList = JSON.parse(req.body.source_list); + } catch (err) { + var srcList = []; + console.warn("[" + req.params.shortName + "] Error /:shortName/upload Parsing Source/Freq List - Error: " + err); + res.status(500); + res.send("Error parsing sourcelist " + err); + return; + } + + let item = null; + const ctx = propagation.setBaggage( + context.active(), + propagation.createBaggage({ + for_the_children: { shortName: shortName, talkgroupNum: talkgroupNum, startTime: startTime }, + }), + ); + + // Validate that the system exists and the API key is correct + context.with(ctx, () => { + tracer.startActiveSpan('validate-system', async (span) => { + try { + item = await System.findOne({ 'shortName': shortName }, ["key", "ignoreUnknownTalkgroup"]); + } catch (err) { + span.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Error /:shortName/upload - Error: " + err, + }); + console.warn("[" + req.params.shortName + "] Error /:shortName/upload - Error: " + err); + res.status(500); + res.send("Error, invalid shortName:\n" + err); + span.end(); + return; + } + + if (!item) { + span.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey, + }); + console.info("[" + req.params.shortName + "] Error /:shortName/upload ShortName does not exist"); + res.status(500); + res.send("ShortName does not exist: " + shortName + "\n"); + span.end(); + return; + } + if (apiKey != item.key) { + span.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey, + }); + console.warn("[" + req.params.shortName + "] Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey); + res.status(500); + res.send("API Keys do not match!\n"); + span.end(); + return; + } + span.end(); + }); }); - if (!talkgroupExists) { - try { - fs.unlinkSync(req.file.path) - //file removed - } catch (err) { - console.log("[" + call.shortName + "] error deleting: " + req.file.path); - } - - res.status(500); - res.send("Talkgroup does not exist, skipping.\n"); - return; - - } - } - // Add in an API Key check - - let errorCount = parseInt(req.body.error_count); - let spikeCount = parseInt(req.body.spike_count); - - if (Number.isNaN(errorCount)) { - errorCount = 0; - } - if (Number.isNaN(spikeCount)) { - spikeCount = 0; - } - - try { - var srcList = JSON.parse(req.body.source_list); - } catch (err) { - var srcList = []; - console.warn("[" + req.params.shortName + "] Error /:shortName/upload Parsing Source/Freq List - Error: " + err); - res.status(500); - res.send("Error parsing sourcelist " + err); - return; - } - - res.status(200).end(); - - var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/"; - var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + path.extname(req.file.originalname); - - var endpoint = s3_endpoint; - var bucket = s3_bucket; - var url = s3_endpoint + "/" + s3_bucket + "/" + object_key; - - var call = new Call({ - shortName: shortName, - talkgroupNum: talkgroupNum, - objectKey: object_key, - endpoint: endpoint, - bucket: bucket, - time: time, - name: talkgroupNum + "-" + startTime + ".m4a", - freq: freq, - errorCount: errorCount, - spikeCount: spikeCount, - url: url, - emergency: emergency, - path: local_path, - srcList: srcList, - len: -1 - }); - - try { - if (req.body.call_length) { - call.len = parseFloat(req.body.call_length); - } else { - call.len = (stopTime - time) / 1000; - } - } catch (err) { - console.error(err); - } - let fileContent; - try { - fileContent = fs.readFileSync(req.file.path); - //s3Src = fs.createReadStream(req.file.path); - } catch (err) { - console.error("[" + call.shortName + "] Unable to open file: " + req.file.path + " Error: " + err); - return; - } - const command = new PutObjectCommand({ - Bucket: s3_bucket, - Key: object_key, - Body: fileContent, - ACL: 'public-read' - }); - - try { - await client.send(command); - await call.save(); - sysStats.addCall(call.toObject()); - // we only want to notify clients if the clip is longer than 1 second. - if (call.len >= 1) { - req.call = call.toObject(); - next(); - } - } catch (err) { - console.warn("[" + call.shortName + "] Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket); - /*res.status(500); - sysStats.addError(call.toObject()); - res.contentType('json'); - res.status(500); - res.send(JSON.stringify({ - success: false, - error: "File Upload" - }));*/ - } - - try { - fs.unlinkSync(req.file.path); - } catch (err) { - console.warn("There was an Error uploading an deleting: " + req.file.path); - }; + // Parse the request and validate the data + context.with(ctx, () => { + tracer.startActiveSpan('validate-fields', async (span) => { + + // Blocking sensitive talkgroups + if ((shortName == "hennearmer") && ((talkgroupNum == 3421) || (talkgroupNum == 3423))) { + span.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Sensitive Talkgroup", + }); + res.status(200).end(); + span.end(); + return; + } + + if (item.ignoreUnknownTalkgroup == true) { + talkgroupExists = await Talkgroup.exists({ + 'shortName': shortName, + 'num': talkgroupNum + }); + + if (!talkgroupExists) { + try { + fs.unlinkSync(req.file.path) + //file removed + } catch (err) { + console.log("[" + call.shortName + "] error deleting: " + req.file.path); + } + + res.status(500); + res.send("Talkgroup does not exist, skipping.\n"); + span.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Ignore Unknown Talkgroup Set - Talkgroup does not exist: " + talkgroupNum + " ShortName: " + shortName, + }); + span.end(); + return; + } + } + // Add in an API Key check + + + + res.status(200).end(); + span.end(); + }); + }); + + var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/"; + var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + path.extname(req.file.originalname); + + var endpoint = s3_endpoint; + var bucket = s3_bucket; + var url = s3_endpoint + "/" + s3_bucket + "/" + object_key; + + var call = new Call({ + shortName: shortName, + talkgroupNum: talkgroupNum, + objectKey: object_key, + endpoint: endpoint, + bucket: bucket, + time: time, + name: talkgroupNum + "-" + startTime + ".m4a", + freq: freq, + errorCount: errorCount, + spikeCount: spikeCount, + url: url, + emergency: emergency, + path: local_path, + srcList: srcList, + len: -1 + }); + + // Upload the file to S3 + context.with(ctx, () => { + tracer.startActiveSpan('upload-call', async (span) => { + try { + if (req.body.call_length) { + call.len = parseFloat(req.body.call_length); + } else { + call.len = (stopTime - time) / 1000; + } + } catch (err) { + console.error(err); + } + let fileContent; + try { + fileContent = fs.readFileSync(req.file.path); + //s3Src = fs.createReadStream(req.file.path); + } catch (err) { + span.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Unable to open file: " + req.file.path + " Error: " + err, + }); + span.end(); + console.error("[" + call.shortName + "] Unable to open file: " + req.file.path + " Error: " + err); + return; + } + const command = new PutObjectCommand({ + Bucket: s3_bucket, + Key: object_key, + Body: fileContent, + ACL: 'public-read' + }); + + try { + await client.send(command); + + } catch (err) { + span.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket, + }); + console.warn("[" + call.shortName + "] Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket); + /*res.status(500); + sysStats.addError(call.toObject()); + res.contentType('json'); + res.status(500); + res.send(JSON.stringify({ + success: false, + error: "File Upload" + }));*/ + } + span.end(); + }); + }); + + // Save Call to the database + context.with(ctx, () => { + tracer.startActiveSpan('save-call', async (span) => { + try { + await call.save(); + sysStats.addCall(call.toObject()); + // we only want to notify clients if the clip is longer than 1 second. + if (call.len >= 1) { + req.call = call.toObject(); + next(); + } + } catch (err) { + console.warn("[" + call.shortName + "] Error saving call: " + err); + span.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Error saving call: " + err, + }); + /*res.status(500); + sysStats.addError(call.toObject()); + res.contentType('json'); + res.status(500); + res.send(JSON.stringify({ + success: false, + error: "File Upload" + }));*/ + } + + try { + fs.unlinkSync(req.file.path); + } catch (err) { + console.warn("There was an Error uploading an deleting: " + req.file.path); + }; + + span.end(); + }); + }); }); - + } diff --git a/backend/package-lock.json b/backend/package-lock.json index b98a92c..20c0e17 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -13,6 +13,7 @@ "@aws-sdk/credential-provider-process": "^3.306.0", "@aws-sdk/credential-providers": "^3.309.0", "@honeycombio/opentelemetry-node": "^0.6.1", + "@opentelemetry/api": "^1.7.0", "@opentelemetry/auto-instrumentations-node": "^0.41.0", "adm-zip": "^0.5.10", "aws-sdk": "^2.1353", diff --git a/backend/package.json b/backend/package.json index 0ed6456..2cd8e2c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -7,6 +7,7 @@ "@aws-sdk/credential-provider-process": "^3.306.0", "@aws-sdk/credential-providers": "^3.309.0", "@honeycombio/opentelemetry-node": "^0.6.1", + "@opentelemetry/api": "^1.7.0", "@opentelemetry/auto-instrumentations-node": "^0.41.0", "adm-zip": "^0.5.10", "aws-sdk": "^2.1353",