mirror of
https://github.com/openmhz/trunk-server.git
synced 2026-11-12 08:47:40 -05:00
manual tracing for Upload
This commit is contained in:
Luke Berndt
@@ -242,14 +242,14 @@ It is a huge pain to upgrade MongoDB in place. It turns out to be easier to dump
|
||||
|
||||
Rough Playbook (use common sense, I may not have this exact):
|
||||
- get into the shell of the mongo container
|
||||
- `mongodump --db scanner —out /data/db/backup`
|
||||
- `mongodump --uri="mongodb://127.0.0.1" --db scanner --out /data/db/backup`
|
||||
- exit contianer and go back to host machine
|
||||
- `cd data/db`
|
||||
- `rm *` erase everything... but not the sub-directories because that is where the backup is
|
||||
- upgrade to the latest version of mongo
|
||||
- build and launch the mongo container, which will create an empty DB
|
||||
- get into the shell of the mongo container
|
||||
- `mongorestore --db scanner --drop /data/db/backup/`
|
||||
- `mongorestore --db scanner --drop /data/db/backup/scanner`
|
||||
|
||||
### Privlege Port Error
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ const {
|
||||
|
||||
// uses the HONEYCOMB_API_KEY and OTEL_SERVICE_NAME environment variables
|
||||
const sdk = new HoneycombSDK({
|
||||
sampleRate: 5,
|
||||
instrumentations: [
|
||||
getNodeAutoInstrumentations({
|
||||
// we recommend disabling fs autoinstrumentation since it can be noisy
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
var path = require('path');
|
||||
var fs = require('fs');
|
||||
const opentelemetry = require('@opentelemetry/api');
|
||||
var sysStats = require("../sys_stats");
|
||||
var System = require("../models/system");
|
||||
var Talkgroup = require("../models/talkgroup");
|
||||
var { callModel: Call } = require("../models/call");
|
||||
|
||||
|
||||
const { PutObjectCommand, S3Client } = require("@aws-sdk/client-s3");
|
||||
const { fromIni } = require("@aws-sdk/credential-providers");
|
||||
|
||||
@@ -22,179 +24,266 @@ const client = new S3Client({
|
||||
});
|
||||
|
||||
exports.upload = async function (req, res, next) {
|
||||
process.nextTick( async () => {
|
||||
if (!req.file || ((path.extname(req.file.originalname) != '.m4a') && (path.extname(req.file.originalname) != '.mp3'))) {
|
||||
console.warn("[" + req.params.shortName + "] Error file name is wrong or file does not exist");
|
||||
res.status(500);
|
||||
res.send("Error, invalid filename:\n");
|
||||
return;
|
||||
}
|
||||
var shortName = req.params.shortName.toLowerCase();
|
||||
var apiKey = req.body.api_key;
|
||||
let item = null;
|
||||
|
||||
try {
|
||||
item = await System.findOne({ 'shortName': shortName }, ["key", "ignoreUnknownTalkgroup"]);
|
||||
} catch (err) {
|
||||
console.warn("[" + req.params.shortName + "] Error /:shortName/upload - Error: " + err);
|
||||
res.status(500);
|
||||
res.send("Error, invalid shotName:\n" + err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!item) {
|
||||
console.info("[" + req.params.shortName + "] Error /:shortName/upload ShortName does not exist");
|
||||
res.status(500);
|
||||
res.send("ShortName does not exist: " + shortName + "\n");
|
||||
return;
|
||||
}
|
||||
if (apiKey != item.key) {
|
||||
console.warn("[" + req.params.shortName + "] Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey);
|
||||
res.status(500);
|
||||
res.send("API Keys do not match!\n");
|
||||
return;
|
||||
}
|
||||
|
||||
var talkgroupNum = parseInt(req.body.talkgroup_num);
|
||||
var freq = parseFloat(req.body.freq);
|
||||
var time = new Date(parseInt(req.body.start_time) * 1000);
|
||||
var stopTime = new Date();
|
||||
if (req.body.stop_time) {
|
||||
var stopTime = new Date(parseInt(req.body.stop_time) * 1000);
|
||||
}
|
||||
var startTime = req.body.start_time;
|
||||
var emergency = parseInt(req.body.emergency);
|
||||
|
||||
// Blocking sensitive talkgroups
|
||||
if ((shortName == "hennearmer") && ((talkgroupNum == 3421) || (talkgroupNum == 3423))) {
|
||||
res.status(200).end();
|
||||
return;
|
||||
}
|
||||
const tracer = trace.getTracer('upload-tracer');
|
||||
|
||||
|
||||
if (item.ignoreUnknownTalkgroup == true) {
|
||||
talkgroupExists = await Talkgroup.exists({
|
||||
'shortName': shortName,
|
||||
'num': talkgroupNum
|
||||
process.nextTick(async () => {
|
||||
if (!req.file || ((path.extname(req.file.originalname) != '.m4a') && (path.extname(req.file.originalname) != '.mp3'))) {
|
||||
console.warn("[" + req.params.shortName + "] Error file name is wrong or file does not exist");
|
||||
res.status(500);
|
||||
res.send("Error, invalid filename:\n");
|
||||
return;
|
||||
}
|
||||
var shortName = req.params.shortName.toLowerCase();
|
||||
var apiKey = req.body.api_key;
|
||||
var talkgroupNum = parseInt(req.body.talkgroup_num);
|
||||
var freq = parseFloat(req.body.freq);
|
||||
var time = new Date(parseInt(req.body.start_time) * 1000);
|
||||
var stopTime = new Date();
|
||||
if (req.body.stop_time) {
|
||||
var stopTime = new Date(parseInt(req.body.stop_time) * 1000);
|
||||
}
|
||||
var startTime = req.body.start_time;
|
||||
var emergency = parseInt(req.body.emergency);
|
||||
|
||||
let errorCount = parseInt(req.body.error_count);
|
||||
let spikeCount = parseInt(req.body.spike_count);
|
||||
|
||||
if (Number.isNaN(errorCount)) {
|
||||
errorCount = 0;
|
||||
}
|
||||
if (Number.isNaN(spikeCount)) {
|
||||
spikeCount = 0;
|
||||
}
|
||||
|
||||
try {
|
||||
var srcList = JSON.parse(req.body.source_list);
|
||||
} catch (err) {
|
||||
var srcList = [];
|
||||
console.warn("[" + req.params.shortName + "] Error /:shortName/upload Parsing Source/Freq List - Error: " + err);
|
||||
res.status(500);
|
||||
res.send("Error parsing sourcelist " + err);
|
||||
return;
|
||||
}
|
||||
|
||||
let item = null;
|
||||
const ctx = propagation.setBaggage(
|
||||
context.active(),
|
||||
propagation.createBaggage({
|
||||
for_the_children: { shortName: shortName, talkgroupNum: talkgroupNum, startTime: startTime },
|
||||
}),
|
||||
);
|
||||
|
||||
// Validate that the system exists and the API key is correct
|
||||
context.with(ctx, () => {
|
||||
tracer.startActiveSpan('validate-system', async (span) => {
|
||||
try {
|
||||
item = await System.findOne({ 'shortName': shortName }, ["key", "ignoreUnknownTalkgroup"]);
|
||||
} catch (err) {
|
||||
span.setStatus({
|
||||
code: opentelemetry.SpanStatusCode.ERROR,
|
||||
message: "Error /:shortName/upload - Error: " + err,
|
||||
});
|
||||
console.warn("[" + req.params.shortName + "] Error /:shortName/upload - Error: " + err);
|
||||
res.status(500);
|
||||
res.send("Error, invalid shortName:\n" + err);
|
||||
span.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!item) {
|
||||
span.setStatus({
|
||||
code: opentelemetry.SpanStatusCode.ERROR,
|
||||
message: "Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey,
|
||||
});
|
||||
console.info("[" + req.params.shortName + "] Error /:shortName/upload ShortName does not exist");
|
||||
res.status(500);
|
||||
res.send("ShortName does not exist: " + shortName + "\n");
|
||||
span.end();
|
||||
return;
|
||||
}
|
||||
if (apiKey != item.key) {
|
||||
span.setStatus({
|
||||
code: opentelemetry.SpanStatusCode.ERROR,
|
||||
message: "Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey,
|
||||
});
|
||||
console.warn("[" + req.params.shortName + "] Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey);
|
||||
res.status(500);
|
||||
res.send("API Keys do not match!\n");
|
||||
span.end();
|
||||
return;
|
||||
}
|
||||
span.end();
|
||||
});
|
||||
});
|
||||
|
||||
if (!talkgroupExists) {
|
||||
try {
|
||||
fs.unlinkSync(req.file.path)
|
||||
//file removed
|
||||
} catch (err) {
|
||||
console.log("[" + call.shortName + "] error deleting: " + req.file.path);
|
||||
}
|
||||
|
||||
res.status(500);
|
||||
res.send("Talkgroup does not exist, skipping.\n");
|
||||
return;
|
||||
|
||||
}
|
||||
}
|
||||
// Add in an API Key check
|
||||
|
||||
let errorCount = parseInt(req.body.error_count);
|
||||
let spikeCount = parseInt(req.body.spike_count);
|
||||
|
||||
if (Number.isNaN(errorCount)) {
|
||||
errorCount = 0;
|
||||
}
|
||||
if (Number.isNaN(spikeCount)) {
|
||||
spikeCount = 0;
|
||||
}
|
||||
|
||||
try {
|
||||
var srcList = JSON.parse(req.body.source_list);
|
||||
} catch (err) {
|
||||
var srcList = [];
|
||||
console.warn("[" + req.params.shortName + "] Error /:shortName/upload Parsing Source/Freq List - Error: " + err);
|
||||
res.status(500);
|
||||
res.send("Error parsing sourcelist " + err);
|
||||
return;
|
||||
}
|
||||
|
||||
res.status(200).end();
|
||||
|
||||
var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/";
|
||||
var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + path.extname(req.file.originalname);
|
||||
|
||||
var endpoint = s3_endpoint;
|
||||
var bucket = s3_bucket;
|
||||
var url = s3_endpoint + "/" + s3_bucket + "/" + object_key;
|
||||
|
||||
var call = new Call({
|
||||
shortName: shortName,
|
||||
talkgroupNum: talkgroupNum,
|
||||
objectKey: object_key,
|
||||
endpoint: endpoint,
|
||||
bucket: bucket,
|
||||
time: time,
|
||||
name: talkgroupNum + "-" + startTime + ".m4a",
|
||||
freq: freq,
|
||||
errorCount: errorCount,
|
||||
spikeCount: spikeCount,
|
||||
url: url,
|
||||
emergency: emergency,
|
||||
path: local_path,
|
||||
srcList: srcList,
|
||||
len: -1
|
||||
});
|
||||
|
||||
try {
|
||||
if (req.body.call_length) {
|
||||
call.len = parseFloat(req.body.call_length);
|
||||
} else {
|
||||
call.len = (stopTime - time) / 1000;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
let fileContent;
|
||||
try {
|
||||
fileContent = fs.readFileSync(req.file.path);
|
||||
//s3Src = fs.createReadStream(req.file.path);
|
||||
} catch (err) {
|
||||
console.error("[" + call.shortName + "] Unable to open file: " + req.file.path + " Error: " + err);
|
||||
return;
|
||||
}
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: s3_bucket,
|
||||
Key: object_key,
|
||||
Body: fileContent,
|
||||
ACL: 'public-read'
|
||||
});
|
||||
|
||||
try {
|
||||
await client.send(command);
|
||||
await call.save();
|
||||
sysStats.addCall(call.toObject());
|
||||
// we only want to notify clients if the clip is longer than 1 second.
|
||||
if (call.len >= 1) {
|
||||
req.call = call.toObject();
|
||||
next();
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("[" + call.shortName + "] Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket);
|
||||
/*res.status(500);
|
||||
sysStats.addError(call.toObject());
|
||||
res.contentType('json');
|
||||
res.status(500);
|
||||
res.send(JSON.stringify({
|
||||
success: false,
|
||||
error: "File Upload"
|
||||
}));*/
|
||||
}
|
||||
|
||||
try {
|
||||
fs.unlinkSync(req.file.path);
|
||||
} catch (err) {
|
||||
console.warn("There was an Error uploading an deleting: " + req.file.path);
|
||||
};
|
||||
|
||||
|
||||
// Parse the request and validate the data
|
||||
context.with(ctx, () => {
|
||||
tracer.startActiveSpan('validate-fields', async (span) => {
|
||||
|
||||
// Blocking sensitive talkgroups
|
||||
if ((shortName == "hennearmer") && ((talkgroupNum == 3421) || (talkgroupNum == 3423))) {
|
||||
span.setStatus({
|
||||
code: opentelemetry.SpanStatusCode.ERROR,
|
||||
message: "Sensitive Talkgroup",
|
||||
});
|
||||
res.status(200).end();
|
||||
span.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (item.ignoreUnknownTalkgroup == true) {
|
||||
talkgroupExists = await Talkgroup.exists({
|
||||
'shortName': shortName,
|
||||
'num': talkgroupNum
|
||||
});
|
||||
|
||||
if (!talkgroupExists) {
|
||||
try {
|
||||
fs.unlinkSync(req.file.path)
|
||||
//file removed
|
||||
} catch (err) {
|
||||
console.log("[" + call.shortName + "] error deleting: " + req.file.path);
|
||||
}
|
||||
|
||||
res.status(500);
|
||||
res.send("Talkgroup does not exist, skipping.\n");
|
||||
span.setStatus({
|
||||
code: opentelemetry.SpanStatusCode.ERROR,
|
||||
message: "Ignore Unknown Talkgroup Set - Talkgroup does not exist: " + talkgroupNum + " ShortName: " + shortName,
|
||||
});
|
||||
span.end();
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Add in an API Key check
|
||||
|
||||
|
||||
|
||||
res.status(200).end();
|
||||
span.end();
|
||||
});
|
||||
});
|
||||
|
||||
var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/";
|
||||
var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + path.extname(req.file.originalname);
|
||||
|
||||
var endpoint = s3_endpoint;
|
||||
var bucket = s3_bucket;
|
||||
var url = s3_endpoint + "/" + s3_bucket + "/" + object_key;
|
||||
|
||||
var call = new Call({
|
||||
shortName: shortName,
|
||||
talkgroupNum: talkgroupNum,
|
||||
objectKey: object_key,
|
||||
endpoint: endpoint,
|
||||
bucket: bucket,
|
||||
time: time,
|
||||
name: talkgroupNum + "-" + startTime + ".m4a",
|
||||
freq: freq,
|
||||
errorCount: errorCount,
|
||||
spikeCount: spikeCount,
|
||||
url: url,
|
||||
emergency: emergency,
|
||||
path: local_path,
|
||||
srcList: srcList,
|
||||
len: -1
|
||||
});
|
||||
|
||||
// Upload the file to S3
|
||||
context.with(ctx, () => {
|
||||
tracer.startActiveSpan('upload-call', async (span) => {
|
||||
try {
|
||||
if (req.body.call_length) {
|
||||
call.len = parseFloat(req.body.call_length);
|
||||
} else {
|
||||
call.len = (stopTime - time) / 1000;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
let fileContent;
|
||||
try {
|
||||
fileContent = fs.readFileSync(req.file.path);
|
||||
//s3Src = fs.createReadStream(req.file.path);
|
||||
} catch (err) {
|
||||
span.setStatus({
|
||||
code: opentelemetry.SpanStatusCode.ERROR,
|
||||
message: "Unable to open file: " + req.file.path + " Error: " + err,
|
||||
});
|
||||
span.end();
|
||||
console.error("[" + call.shortName + "] Unable to open file: " + req.file.path + " Error: " + err);
|
||||
return;
|
||||
}
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: s3_bucket,
|
||||
Key: object_key,
|
||||
Body: fileContent,
|
||||
ACL: 'public-read'
|
||||
});
|
||||
|
||||
try {
|
||||
await client.send(command);
|
||||
|
||||
} catch (err) {
|
||||
span.setStatus({
|
||||
code: opentelemetry.SpanStatusCode.ERROR,
|
||||
message: "Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket,
|
||||
});
|
||||
console.warn("[" + call.shortName + "] Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket);
|
||||
/*res.status(500);
|
||||
sysStats.addError(call.toObject());
|
||||
res.contentType('json');
|
||||
res.status(500);
|
||||
res.send(JSON.stringify({
|
||||
success: false,
|
||||
error: "File Upload"
|
||||
}));*/
|
||||
}
|
||||
span.end();
|
||||
});
|
||||
});
|
||||
|
||||
// Save Call to the database
|
||||
context.with(ctx, () => {
|
||||
tracer.startActiveSpan('save-call', async (span) => {
|
||||
try {
|
||||
await call.save();
|
||||
sysStats.addCall(call.toObject());
|
||||
// we only want to notify clients if the clip is longer than 1 second.
|
||||
if (call.len >= 1) {
|
||||
req.call = call.toObject();
|
||||
next();
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("[" + call.shortName + "] Error saving call: " + err);
|
||||
span.setStatus({
|
||||
code: opentelemetry.SpanStatusCode.ERROR,
|
||||
message: "Error saving call: " + err,
|
||||
});
|
||||
/*res.status(500);
|
||||
sysStats.addError(call.toObject());
|
||||
res.contentType('json');
|
||||
res.status(500);
|
||||
res.send(JSON.stringify({
|
||||
success: false,
|
||||
error: "File Upload"
|
||||
}));*/
|
||||
}
|
||||
|
||||
try {
|
||||
fs.unlinkSync(req.file.path);
|
||||
} catch (err) {
|
||||
console.warn("There was an Error uploading an deleting: " + req.file.path);
|
||||
};
|
||||
|
||||
span.end();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
1
backend/package-lock.json
generated
1
backend/package-lock.json
generated
@@ -13,6 +13,7 @@
|
||||
"@aws-sdk/credential-provider-process": "^3.306.0",
|
||||
"@aws-sdk/credential-providers": "^3.309.0",
|
||||
"@honeycombio/opentelemetry-node": "^0.6.1",
|
||||
"@opentelemetry/api": "^1.7.0",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.41.0",
|
||||
"adm-zip": "^0.5.10",
|
||||
"aws-sdk": "^2.1353",
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
"@aws-sdk/credential-provider-process": "^3.306.0",
|
||||
"@aws-sdk/credential-providers": "^3.309.0",
|
||||
"@honeycombio/opentelemetry-node": "^0.6.1",
|
||||
"@opentelemetry/api": "^1.7.0",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.41.0",
|
||||
"adm-zip": "^0.5.10",
|
||||
"aws-sdk": "^2.1353",
|
||||
|
||||
Reference in New Issue
Block a user