manual tracing for Upload

README.md

@@ -242,14 +242,14 @@ It is a huge pain to upgrade MongoDB in place. It turns out to be easier to dump
 
 Rough Playbook (use common sense, I may not have this exact):
 - get into the shell of the mongo container
-- `mongodump  --db scanner —out /data/db/backup`
+- `mongodump --uri="mongodb://127.0.0.1" --db scanner --out /data/db/backup`
 - exit contianer and go back to host machine
 - `cd data/db`
 - `rm *` erase everything... but not the sub-directories because that is where the backup is
 - upgrade to the latest version of mongo
 - build and launch the mongo container, which will create an empty DB
 - get into the shell of the mongo container
-- `mongorestore --db scanner --drop /data/db/backup/`
+- `mongorestore  --db scanner --drop /data/db/backup/scanner`
 
 ### Privlege Port Error
 

backend/agents/honeycomb-tracing.js

@@ -8,6 +8,7 @@ const {
 
 // uses the HONEYCOMB_API_KEY and OTEL_SERVICE_NAME environment variables
 const sdk = new HoneycombSDK({
+    sampleRate: 5,
     instrumentations: [
         getNodeAutoInstrumentations({
             // we recommend disabling fs autoinstrumentation since it can be noisy

backend/controllers/uploads.js

@@ -1,10 +1,12 @@
 var path = require('path');
 var fs = require('fs');
+const opentelemetry = require('@opentelemetry/api');
 var sysStats = require("../sys_stats");
 var System = require("../models/system");
 var Talkgroup = require("../models/talkgroup");
 var { callModel: Call } = require("../models/call");
 
+
 const { PutObjectCommand, S3Client } = require("@aws-sdk/client-s3");
 const { fromIni } = require("@aws-sdk/credential-providers");
 
@@ -22,177 +24,264 @@ const client = new S3Client({
 });
 
 exports.upload = async function (req, res, next) {
-  process.nextTick( async () => {
+  const tracer = trace.getTracer('upload-tracer');
-  if (!req.file || ((path.extname(req.file.originalname) != '.m4a') && (path.extname(req.file.originalname) != '.mp3'))) {
-    console.warn("[" + req.params.shortName + "] Error file name is wrong or file does not exist");
-    res.status(500);
-    res.send("Error, invalid filename:\n");
-    return;
-  }
-  var shortName = req.params.shortName.toLowerCase();
-  var apiKey = req.body.api_key;
-  let item = null;
-
-  try {
-    item = await System.findOne({ 'shortName': shortName }, ["key", "ignoreUnknownTalkgroup"]);
-  } catch (err) {
-    console.warn("[" + req.params.shortName + "] Error /:shortName/upload - Error: " + err);
-    res.status(500);
-    res.send("Error, invalid shotName:\n" + err);
-    return;
-  }
-
-  if (!item) {
-    console.info("[" + req.params.shortName + "] Error /:shortName/upload ShortName does not exist");
-    res.status(500);
-    res.send("ShortName does not exist: " + shortName + "\n");
-    return;
-  }
-  if (apiKey != item.key) {
-    console.warn("[" + req.params.shortName + "] Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey);
-    res.status(500);
-    res.send("API Keys do not match!\n");
-    return;
-  }
-
-  var talkgroupNum = parseInt(req.body.talkgroup_num);
-  var freq = parseFloat(req.body.freq);
-  var time = new Date(parseInt(req.body.start_time) * 1000);
-  var stopTime = new Date();
-  if (req.body.stop_time) {
-    var stopTime = new Date(parseInt(req.body.stop_time) * 1000);
-  }
-  var startTime = req.body.start_time;
-  var emergency = parseInt(req.body.emergency);
-
-  // Blocking sensitive talkgroups
-if ((shortName == "hennearmer") && ((talkgroupNum == 3421) || (talkgroupNum == 3423))) {
-  res.status(200).end();
-  return;
-}
 
 
-  if (item.ignoreUnknownTalkgroup == true) {
+  process.nextTick(async () => {
-    talkgroupExists = await Talkgroup.exists({
+    if (!req.file || ((path.extname(req.file.originalname) != '.m4a') && (path.extname(req.file.originalname) != '.mp3'))) {
-      'shortName': shortName,
+      console.warn("[" + req.params.shortName + "] Error file name is wrong or file does not exist");
-      'num': talkgroupNum
-    });
-
-    if (!talkgroupExists) {
-      try {
-        fs.unlinkSync(req.file.path)
-        //file removed
-      } catch (err) {
-        console.log("[" + call.shortName + "] error deleting: " + req.file.path);
-      }
-
       res.status(500);
-      res.send("Talkgroup does not exist, skipping.\n");
+      res.send("Error, invalid filename:\n");
       return;
-
     }
-  }
+    var shortName = req.params.shortName.toLowerCase();
-  // Add in an API Key check
+    var apiKey = req.body.api_key;
-
+    var talkgroupNum = parseInt(req.body.talkgroup_num);
-  let errorCount = parseInt(req.body.error_count);
+    var freq = parseFloat(req.body.freq);
-  let spikeCount = parseInt(req.body.spike_count);
+    var time = new Date(parseInt(req.body.start_time) * 1000);
-
+    var stopTime = new Date();
-  if (Number.isNaN(errorCount)) {
+    if (req.body.stop_time) {
-    errorCount = 0;
+      var stopTime = new Date(parseInt(req.body.stop_time) * 1000);
-  }
-  if (Number.isNaN(spikeCount)) {
-    spikeCount = 0;
-  }
-
-  try {
-    var srcList = JSON.parse(req.body.source_list);
-  } catch (err) {
-    var srcList = [];
-    console.warn("[" + req.params.shortName + "] Error /:shortName/upload Parsing Source/Freq List -  Error: " + err);
-    res.status(500);
-    res.send("Error parsing sourcelist " + err);
-    return;
-  }
-
-  res.status(200).end();
-
-  var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/";
-  var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + path.extname(req.file.originalname);
-
-  var endpoint = s3_endpoint;
-  var bucket = s3_bucket;
-  var url = s3_endpoint + "/" + s3_bucket + "/" + object_key;
-
-  var call = new Call({
-    shortName: shortName,
-    talkgroupNum: talkgroupNum,
-    objectKey: object_key,
-    endpoint: endpoint,
-    bucket: bucket,
-    time: time,
-    name: talkgroupNum + "-" + startTime + ".m4a",
-    freq: freq,
-    errorCount: errorCount,
-    spikeCount: spikeCount,
-    url: url,
-    emergency: emergency,
-    path: local_path,
-    srcList: srcList,
-    len: -1
-  });
-
-  try {
-    if (req.body.call_length) {
-      call.len = parseFloat(req.body.call_length);
-    } else {
-      call.len = (stopTime - time) / 1000;
     }
-  } catch (err) {
+    var startTime = req.body.start_time;
-    console.error(err);
+    var emergency = parseInt(req.body.emergency);
-  }
-  let fileContent;
-  try {
-    fileContent = fs.readFileSync(req.file.path);
-    //s3Src = fs.createReadStream(req.file.path);
-  } catch (err) {
-    console.error("[" + call.shortName + "] Unable to open file: " + req.file.path + " Error: " + err);
-    return;
-  }
-  const command = new PutObjectCommand({
-    Bucket: s3_bucket,
-    Key: object_key,
-    Body: fileContent,
-    ACL: 'public-read'
-  });
 
-  try {
+    let errorCount = parseInt(req.body.error_count);
-    await client.send(command);
+    let spikeCount = parseInt(req.body.spike_count);
-    await call.save();
+
-    sysStats.addCall(call.toObject());
+    if (Number.isNaN(errorCount)) {
-    // we only want to notify clients if the clip is longer than 1 second.
+      errorCount = 0;
-    if (call.len >= 1) {
+    }
-      req.call = call.toObject();
+    if (Number.isNaN(spikeCount)) {
-      next();
+      spikeCount = 0;
     }
-  } catch (err) {
-    console.warn("[" + call.shortName + "] Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket);
-    /*res.status(500);
-    sysStats.addError(call.toObject());
-    res.contentType('json');
-    res.status(500);
-    res.send(JSON.stringify({
-      success: false,
-      error: "File Upload"
-    }));*/
-  }
 
     try {
-      fs.unlinkSync(req.file.path);
+      var srcList = JSON.parse(req.body.source_list);
     } catch (err) {
-      console.warn("There was an Error uploading an deleting: " + req.file.path);
+      var srcList = [];
-    };
+      console.warn("[" + req.params.shortName + "] Error /:shortName/upload Parsing Source/Freq List -  Error: " + err);
+      res.status(500);
+      res.send("Error parsing sourcelist " + err);
+      return;
+    }
+
+    let item = null;
+    const ctx = propagation.setBaggage(
+      context.active(),
+      propagation.createBaggage({
+        for_the_children: { shortName: shortName, talkgroupNum: talkgroupNum, startTime: startTime },
+      }),
+    );
+
+    // Validate that the system exists and the API key is correct
+    context.with(ctx, () => {
+      tracer.startActiveSpan('validate-system', async (span) => {
+        try {
+          item = await System.findOne({ 'shortName': shortName }, ["key", "ignoreUnknownTalkgroup"]);
+        } catch (err) {
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: "Error /:shortName/upload - Error: " + err,
+          });
+          console.warn("[" + req.params.shortName + "] Error /:shortName/upload - Error: " + err);
+          res.status(500);
+          res.send("Error, invalid shortName:\n" + err);
+          span.end();
+          return;
+        }
+
+        if (!item) {
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: "Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey,
+          });
+          console.info("[" + req.params.shortName + "] Error /:shortName/upload ShortName does not exist");
+          res.status(500);
+          res.send("ShortName does not exist: " + shortName + "\n");
+          span.end();
+          return;
+        }
+        if (apiKey != item.key) {
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: "Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey,
+          });
+          console.warn("[" + req.params.shortName + "] Error /:shortName/upload API Key Mismatch - Provided key: " + apiKey);
+          res.status(500);
+          res.send("API Keys do not match!\n");
+          span.end();
+          return;
+        }
+        span.end();
+      });
+    });
 
 
+
+    // Parse the request and validate the data
+    context.with(ctx, () => {
+      tracer.startActiveSpan('validate-fields', async (span) => {
+
+        // Blocking sensitive talkgroups
+        if ((shortName == "hennearmer") && ((talkgroupNum == 3421) || (talkgroupNum == 3423))) {
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: "Sensitive Talkgroup",
+          });
+          res.status(200).end();
+          span.end();
+          return;
+        }
+
+        if (item.ignoreUnknownTalkgroup == true) {
+          talkgroupExists = await Talkgroup.exists({
+            'shortName': shortName,
+            'num': talkgroupNum
+          });
+
+          if (!talkgroupExists) {
+            try {
+              fs.unlinkSync(req.file.path)
+              //file removed
+            } catch (err) {
+              console.log("[" + call.shortName + "] error deleting: " + req.file.path);
+            }
+
+            res.status(500);
+            res.send("Talkgroup does not exist, skipping.\n");
+            span.setStatus({
+              code: opentelemetry.SpanStatusCode.ERROR,
+              message: "Ignore Unknown Talkgroup Set - Talkgroup does not exist: " + talkgroupNum + " ShortName: " + shortName,
+            });
+            span.end();
+            return;
+          }
+        }
+        // Add in an API Key check
+
+
+
+        res.status(200).end();
+        span.end();
+      });
+    });
+
+    var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/";
+    var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + path.extname(req.file.originalname);
+
+    var endpoint = s3_endpoint;
+    var bucket = s3_bucket;
+    var url = s3_endpoint + "/" + s3_bucket + "/" + object_key;
+
+    var call = new Call({
+      shortName: shortName,
+      talkgroupNum: talkgroupNum,
+      objectKey: object_key,
+      endpoint: endpoint,
+      bucket: bucket,
+      time: time,
+      name: talkgroupNum + "-" + startTime + ".m4a",
+      freq: freq,
+      errorCount: errorCount,
+      spikeCount: spikeCount,
+      url: url,
+      emergency: emergency,
+      path: local_path,
+      srcList: srcList,
+      len: -1
+    });
+
+    // Upload the file to S3
+    context.with(ctx, () => {
+      tracer.startActiveSpan('upload-call', async (span) => {
+        try {
+          if (req.body.call_length) {
+            call.len = parseFloat(req.body.call_length);
+          } else {
+            call.len = (stopTime - time) / 1000;
+          }
+        } catch (err) {
+          console.error(err);
+        }
+        let fileContent;
+        try {
+          fileContent = fs.readFileSync(req.file.path);
+          //s3Src = fs.createReadStream(req.file.path);
+        } catch (err) {
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: "Unable to open file: " + req.file.path + " Error: " + err,
+          });
+          span.end();
+          console.error("[" + call.shortName + "] Unable to open file: " + req.file.path + " Error: " + err);
+          return;
+        }
+        const command = new PutObjectCommand({
+          Bucket: s3_bucket,
+          Key: object_key,
+          Body: fileContent,
+          ACL: 'public-read'
+        });
+
+        try {
+          await client.send(command);
+
+        } catch (err) {
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: "Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket,
+          });
+          console.warn("[" + call.shortName + "] Upload Error: " + err + " Filename: " + call.name + " content-length: " + req.headers['content-length'] + " Key: " + object_key + " Bucket: " + s3_bucket);
+          /*res.status(500);
+          sysStats.addError(call.toObject());
+          res.contentType('json');
+          res.status(500);
+          res.send(JSON.stringify({
+            success: false,
+            error: "File Upload"
+          }));*/
+        }
+        span.end();
+      });
+    });
+
+    // Save Call to the database
+    context.with(ctx, () => {
+      tracer.startActiveSpan('save-call', async (span) => {
+        try {
+          await call.save();
+          sysStats.addCall(call.toObject());
+          // we only want to notify clients if the clip is longer than 1 second.
+          if (call.len >= 1) {
+            req.call = call.toObject();
+            next();
+          }
+        } catch (err) {
+          console.warn("[" + call.shortName + "] Error saving call: " + err);
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: "Error saving call: " + err,
+          });
+          /*res.status(500);
+          sysStats.addError(call.toObject());
+          res.contentType('json');
+          res.status(500);
+          res.send(JSON.stringify({
+            success: false,
+            error: "File Upload"
+          }));*/
+        }
+
+        try {
+          fs.unlinkSync(req.file.path);
+        } catch (err) {
+          console.warn("There was an Error uploading an deleting: " + req.file.path);
+        };
+
+        span.end();
+      });
+    });
   });
 
 

backend/package-lock.json

@@ -13,6 +13,7 @@
         "@aws-sdk/credential-provider-process": "^3.306.0",
         "@aws-sdk/credential-providers": "^3.309.0",
         "@honeycombio/opentelemetry-node": "^0.6.1",
+        "@opentelemetry/api": "^1.7.0",
         "@opentelemetry/auto-instrumentations-node": "^0.41.0",
         "adm-zip": "^0.5.10",
         "aws-sdk": "^2.1353",

backend/package.json

@@ -7,6 +7,7 @@
     "@aws-sdk/credential-provider-process": "^3.306.0",
     "@aws-sdk/credential-providers": "^3.309.0",
     "@honeycombio/opentelemetry-node": "^0.6.1",
+    "@opentelemetry/api": "^1.7.0",
     "@opentelemetry/auto-instrumentations-node": "^0.41.0",
     "adm-zip": "^0.5.10",
     "aws-sdk": "^2.1353",