From 7352c8af82f42aa937a0efc715d6fea14f7655c8 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Wed, 21 Aug 2024 15:02:41 -0400 Subject: [PATCH] Update uploads.js --- backend/controllers/uploads.js | 319 ++++++++++++++++----------------- 1 file changed, 159 insertions(+), 160 deletions(-) diff --git a/backend/controllers/uploads.js b/backend/controllers/uploads.js index 23c7282..854db1c 100644 --- a/backend/controllers/uploads.js +++ b/backend/controllers/uploads.js @@ -17,6 +17,7 @@ const agent = new https.Agent({ }); + var s3_endpoint = process.env['S3_ENDPOINT'] != null ? process.env['S3_ENDPOINT'] : 'https://s3.us-west-1.wasabisys.com'; var s3_region = process.env['S3_REGION'] != null ? process.env['S3_REGION'] : 'us-west-1'; var s3_bucket = process.env['S3_BUCKET'] != null ? process.env['S3_BUCKET'] : 'openmhz-west'; @@ -35,8 +36,9 @@ const client = new S3Client({ }); exports.upload = async function (req, res, next) { + const tracer = trace.getTracer("upload-service") const parentSpan = trace.getActiveSpan(); - await trace.getTracer('upload-service').startActiveSpan('upload_handler', { parent: parentSpan }, async (span) => { + await tracer.startActiveSpan('upload_handler', { parent: parentSpan }, async (span) => { try { span.setAttribute('call.shortName', req.params.shortName.toLowerCase()); span.setAttribute('call.talkgroup_num', req.body.talkgroup_num); @@ -95,208 +97,205 @@ exports.upload = async function (req, res, next) { console.log("Finished parsing request data"); // Validate that the system exists and the API key is correct - await trace.getTracer('upload-service').startActiveSpan('validate_system', { parent: span }, async (validateSpan) => { - - try { - item = await System.findOne({ 'shortName': shortName }, ["key", "ignoreUnknownTalkgroup"]); - validateSpan.setAttribute('system.exists', !!item); + const validateSpan = tracer.startSpan("validate_system"); - if (!item) { - validateSpan.setStatus({ - code: opentelemetry.SpanStatusCode.ERROR, - message: "ShortName does not exist", - }); + try { + item = await System.findOne({ 'shortName': shortName }, ["key", "ignoreUnknownTalkgroup"]); + validateSpan.setAttribute('system.exists', !!item); - console.info("[" + req.params.shortName + "] Error /:shortName/upload ShortName does not exist"); - res.status(500); - res.send("ShortName does not exist: " + shortName + "\n"); - return; - } - if (apiKey != item.key) { - validateSpan.setStatus({ - code: opentelemetry.SpanStatusCode.ERROR, - message: "API Key Mismatch", - }); - console.warn("[" + req.params.shortName + "] Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey); - res.status(500); - res.send("API Keys do not match!\n"); - return; - } - } catch (err) { - validateSpan.recordException(err); + + if (!item) { validateSpan.setStatus({ code: opentelemetry.SpanStatusCode.ERROR, - message: "Error validating system: " + err.message, + message: "ShortName does not exist", }); - res.status(500).send("Error, invalid shortName:\n" + err); + + console.info("[" + req.params.shortName + "] Error /:shortName/upload ShortName does not exist"); + res.status(500); + res.send("ShortName does not exist: " + shortName + "\n"); return; - } finally { - validateSpan.end(); } - }); + if (apiKey != item.key) { + validateSpan.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "API Key Mismatch", + }); + console.warn("[" + req.params.shortName + "] Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey); + res.status(500); + res.send("API Keys do not match!\n"); + return; + } + } catch (err) { + validateSpan.recordException(err); + validateSpan.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Error validating system: " + err.message, + }); + res.status(500).send("Error, invalid shortName:\n" + err); + return; + } finally { + validateSpan.end(); + } + console.log("Finished validating system API key"); // Blocking sensitive talkgroups - await trace.getTracer('upload-service').startActiveSpan('check_sensitive_talkgroups', { parent: span }, async (sensitiveSpan) => { + const sensitiveSpan = tracer.startSpan("check_sensitive_talkgroups"); + if ((shortName == "hennearmer") && ((talkgroupNum == 3421) || (talkgroupNum == 3423))) { + sensitiveSpan.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Sensitive Talkgroup", + }); + //res.status(200).end(); + return; + } + sensitiveSpan.end(); - if ((shortName == "hennearmer") && ((talkgroupNum == 3421) || (talkgroupNum == 3423))) { - sensitiveSpan.setStatus({ - code: opentelemetry.SpanStatusCode.ERROR, - message: "Sensitive Talkgroup", - }); - //res.status(200).end(); - return; - } - sensitiveSpan.end(); - }); console.log("Finished checking sensitive talkgroups"); if (item.ignoreUnknownTalkgroup == true) { - await trace.getTracer('upload-service').startActiveSpan('check_talkgroup_exists', { parent: span }, async (talkgroupSpan) => { - - talkgroupExists = await Talkgroup.exists({ - 'shortName': shortName, - 'num': talkgroupNum - }); - - if (!talkgroupExists) { - try { - fs.unlinkSync(req.file.path) - //file removed - } catch (err) { - console.log("[" + call.shortName + "] error deleting: " + req.file.path); - } - - res.status(500); - res.send("Talkgroup does not exist, skipping.\n"); - - - - return; - } - talkgroupSpan.end(); + const talkgroupSpan = tracer.startSpan("check_talkgroup_exists"); + talkgroupExists = await Talkgroup.exists({ + 'shortName': shortName, + 'num': talkgroupNum }); + + if (!talkgroupExists) { + try { + fs.unlinkSync(req.file.path) + //file removed + } catch (err) { + console.log("[" + call.shortName + "] error deleting: " + req.file.path); + } + + res.status(500); + res.send("Talkgroup does not exist, skipping.\n"); + + + + return; + } + talkgroupSpan.end(); + } console.log("Finished checking talkgroup exists"); - + // Prepare call object let call; - await trace.getTracer('upload-service').startActiveSpan('prepare_call_object', { parent: span }, async (prepareSpan) => { + const prepareSpan = tracer.startSpan('prepare_call_object'); - var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/"; - var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + path.extname(req.file.originalname); + var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/"; + var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + path.extname(req.file.originalname); - var endpoint = s3_endpoint; - var bucket = s3_bucket; - var url = s3_public_url + "/" + object_key; + var endpoint = s3_endpoint; + var bucket = s3_bucket; + var url = s3_public_url + "/" + object_key; - call = new Call({ - shortName: shortName, - talkgroupNum: talkgroupNum, - objectKey: object_key, - endpoint: endpoint, - bucket: bucket, - time: time, - name: talkgroupNum + "-" + startTime + ".m4a", - freq: freq, - errorCount: errorCount, - spikeCount: spikeCount, - url: url, - emergency: emergency, - path: local_path, - srcList: srcList, - len: -1 - }); - - - try { - if (req.body.call_length) { - call.len = parseFloat(req.body.call_length); - } else { - call.len = (stopTime - time) / 1000; - } - } catch (err) { - console.error(err); - } - prepareSpan.setAttribute('call.object_key', object_key); - prepareSpan.setAttribute('call.url', url); - prepareSpan.end(); + call = new Call({ + shortName: shortName, + talkgroupNum: talkgroupNum, + objectKey: object_key, + endpoint: endpoint, + bucket: bucket, + time: time, + name: talkgroupNum + "-" + startTime + ".m4a", + freq: freq, + errorCount: errorCount, + spikeCount: spikeCount, + url: url, + emergency: emergency, + path: local_path, + srcList: srcList, + len: -1 }); + + try { + if (req.body.call_length) { + call.len = parseFloat(req.body.call_length); + } else { + call.len = (stopTime - time) / 1000; + } + } catch (err) { + console.error(err); + } + prepareSpan.setAttribute('call.object_key', object_key); + prepareSpan.setAttribute('call.url', url); + prepareSpan.end(); + + console.log("Finished preparing call object"); let fileContent; // Upload file to S3 - await trace.getTracer('upload-service').startActiveSpan('upload_to_s3',{ parent: span }, async (uploadSpan) => { + const uploadSpan = tracer.startSpan('upload_to_s3'); + try { + fileContent = fs.readFileSync(req.file.path); + //s3Src = fs.createReadStream(req.file.path); - try { - fileContent = fs.readFileSync(req.file.path); - //s3Src = fs.createReadStream(req.file.path); + const command = new PutObjectCommand({ + Bucket: s3_bucket, + Key: object_key, + Body: fileContent, + ACL: 'public-read' + }); + await client.send(command); - const command = new PutObjectCommand({ - Bucket: s3_bucket, - Key: object_key, - Body: fileContent, - ACL: 'public-read' - }); + } catch (err) { + uploadSpan.recordException(err); + uploadSpan.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Upload Error: " + err.message, + }); + console.warn(`[${call.shortName}] Upload Error: ${err}`); + } finally { + uploadSpan.end(); + } - await client.send(command); - - - } catch (err) { - uploadSpan.recordException(err); - uploadSpan.setStatus({ - code: opentelemetry.SpanStatusCode.ERROR, - message: "Upload Error: " + err.message, - }); - console.warn(`[${call.shortName}] Upload Error: ${err}`); - } finally { - uploadSpan.end(); - } - }); - console.log("Finished uploading to S3"); // Save call to database - await trace.getTracer('upload-service').startActiveSpan('save_call',{ parent: span }, async (saveSpan) => { - try { - await call.save(); - sysStats.addCall(call.toObject()); - // we only want to notify clients if the clip is longer than 1 second. - if (call.len >= 1) { - req.call = call.toObject(); - next(); - } - } catch (err) { - saveSpan.recordException(err); - saveSpan.setStatus({ - code: opentelemetry.SpanStatusCode.ERROR, - message: "Error saving call: " + err.message, - }); - console.warn(`[${call.shortName}] Error saving call: ${err}`); - } finally { - saveSpan.end(); + const saveSpan = tracer.startSpan('save_call'); + + try { + await call.save(); + sysStats.addCall(call.toObject()); + // we only want to notify clients if the clip is longer than 1 second. + if (call.len >= 1) { + req.call = call.toObject(); + next(); } - }); + } catch (err) { + saveSpan.recordException(err); + saveSpan.setStatus({ + code: opentelemetry.SpanStatusCode.ERROR, + message: "Error saving call: " + err.message, + }); + console.warn(`[${call.shortName}] Error saving call: ${err}`); + } finally { + saveSpan.end(); + } + console.log("Finished saving call to database"); // Clean up temporary file - await trace.getTracer('upload-service').startActiveSpan('cleanup_temp_file',{ parent: span }, async (cleanupSpan) => { - try { - fs.unlinkSync(req.file.path); - cleanupSpan.setAttribute('file.deleted', true); - } catch (err) { - cleanupSpan.recordException(err); - console.warn("There was an Error deleting: " + req.file.path); - } finally { - cleanupSpan.end(); - } - }); + const cleanupSpan = tracer.startSpan('cleanup_temp_file'); + + try { + fs.unlinkSync(req.file.path); + cleanupSpan.setAttribute('file.deleted', true); + } catch (err) { + cleanupSpan.recordException(err); + console.warn("There was an Error deleting: " + req.file.path); + } finally { + cleanupSpan.end(); + } + } catch (error) { span.recordException(error);