Merge pull request #135 from openmhz/event-download

Add in event downloads
2026-11-12 08:47:40 -05:00 · 2023-02-11 10:16:11 -05:00
parent 0c1c00c79a b226b685c7
commit 3a434f4654
6 changed files with 226 additions and 53 deletions
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -1,12 +1,13 @@
 # docker build -t smartnet-player .
 # do a `git pull` in smartnet-player to update

-FROM node:10
+FROM node:19-bullseye
+

 #RUN useradd --user-group --create-home --shell /bin/false app &&\
 #  npm install --global npm@3.7.5
-RUN echo deb http://www.deb-multimedia.org stretch main non-free \
-	>>/etc/apt/sources.list && apt-get update &&\
+RUN echo deb https://www.deb-multimedia.org bullseye main non-free \
+	>>/etc/apt/sources.list && apt-get update -oAcquire::AllowInsecureRepositories=true &&\
 	apt-get install -y --force-yes deb-multimedia-keyring && \
 	apt-get update && \
 	apt-get install -y ffmpeg cron python build-essential g++
--- a/backend/controllers/events.js
+++ b/backend/controllers/events.js
@@ -1,9 +1,32 @@
 var ObjectID = require('mongodb').ObjectID;
+const https = require('https');
 var db = require('../db');
 var mongoose = require("mongoose");
 var Event = require("../models/event");
+const AdmZip = require("adm-zip");
 var { callModel: Call, callSchema } = require("../models/call");
 var frontend_server = process.env['REACT_APP_FRONTEND_SERVER'] != null ? process.env['REACT_APP_FRONTEND_SERVER'] : 'https://openmhz.com';
+const os = require('os');
+var fs = require('fs');
+
+const s3 = require('aws-sdk');
+const { events } = require('../models/event');
+
+var s3_endpoint = process.env['S3_ENDPOINT'] != null ? process.env['S3_ENDPOINT'] : 's3.us-west-1.wasabisys.com';
+var s3_bucket = process.env['S3_BUCKET'] != null ? process.env['S3_BUCKET'] : 'openmhz-west';
+var s3_profile = process.env['S3_PROFILE'] != null ? process.env['S3_PROFILE'] : 'wasabi-account';
+
+const s3Endpoint = new s3.Endpoint(s3_endpoint);
+var s3Credentials = new s3.SharedIniFileCredentials({
+    profile: s3_profile
+});
+
+s3.config.credentials = s3Credentials;
+const s3S3 = new s3.S3({
+    endpoint: s3Endpoint,
+    maxRetries: 2
+});
+


 exports.getEvent = async function (req, res, next) {
@@ -23,7 +46,7 @@ exports.getEvent = async function (req, res, next) {

 exports.getEvents = async function (req, res, next) {
    try {
-        let events = await Event.find({},["title",  "description", "startTime", "endTime", "expireTime", "numCalls", "createdAt", "shortNames"]).exec();
+        let events = await Event.find({}, ["title", "description", "startTime", "endTime", "expireTime", "numCalls", "createdAt", "shortNames"]).exec();
        res.contentType('json');
        res.send(JSON.stringify(events));
    }
@@ -34,6 +57,124 @@ exports.getEvents = async function (req, res, next) {
    }
 }

+
+
+async function downloadFile(folderName, url) {
+    return new Promise((resolve, reject) => {
+        https.get(url, (response) => {
+            const filename = new URL(url).pathname.split('/').pop();
+            const filePath = `${folderName}/${filename}`;
+            console.log("Downloading to filename " + filename + " and filePath " + filePath);
+            let stream = fs.createWriteStream(filePath);
+
+            response.on("error", reject);
+
+            stream.on("finish", () => {
+                stream.close();
+                console.log("Download Completed");
+                resolve();
+            }).on("error", reject);
+
+            response.pipe(stream);
+
+        }).on("error", reject);
+    });
+}
+
+
+const exportEventJson = (eventFolder, event) => {
+    const filename = eventFolder + "/event.json";
+    fs.writeFileSync(filename, JSON.stringify(event));
+}
+
+const createEventZip = async (folder, filename, zipFolder) => {
+    try {
+        const zip = new AdmZip();
+        zip.addLocalFolder(folder, zipFolder);
+        zip.writeZip(filename);
+        console.log(`Created ${filename} successfully for event from ${folder}`);
+    } catch (e) {
+        console.error("Event Error trying to zip: " + filename + " error: " + e);
+    }
+}
+
+const uploadEventZip = (zipFile, s3File) => {
+
+    // If not a pro plan, use S3 storage
+
+    var s3Src = fs.createReadStream(zipFile);
+    return new Promise((resolve, reject) => {
+        const s3Params = {
+            Bucket: s3_bucket,
+            Key: s3File,
+            Body: s3Src,
+            ACL: 'public-read'
+        };
+
+        s3S3.upload(s3Params, function (err, data) {
+            s3Src.destroy();
+
+            if (err) {
+                console.error("Event S3 Upload Error " + err);
+                reject(err);
+            } else {
+                resolve("done");
+            }
+        });
+    });
+}
+
+const cleanupEvent = (folder, filename) => {
+    fs.rmSync(folder, { recursive: true, force: true });
+    fs.rmSync(filename);
+}
+
+const packageEvent = (eventId) => {
+    const event = "tst"
+    const tmpdir = os.tmpdir();
+    return new Promise(async (resolve, reject) => {
+        var objectId = eventId;
+        const event = await Event.findById(objectId).exec();
+        if (!event) {
+            reject("Unable to load Event " + eventId);
+        }
+        let eventFolder = "event_" + Date.now();
+        event.shortNames.forEach((shortName) => {
+            eventFolder = eventFolder + "_" + shortName;
+        });
+        const tmpEventFolder = tmpdir + "/" + eventFolder
+        const zipFile = tmpdir + "/" + eventFolder + ".zip"
+        const s3File = "events/" + eventFolder + ".zip"
+        console.log("Using tmp " + tmpEventFolder);
+
+        try {
+            if (!fs.existsSync(tmpEventFolder)) {
+                fs.mkdirSync(tmpEventFolder);
+            }
+
+            for (const call of event.calls) {
+                await downloadFile(tmpEventFolder, call.url)
+            }
+
+
+            exportEventJson(tmpEventFolder, event);
+            createEventZip(tmpEventFolder, zipFile, eventFolder);
+            await uploadEventZip(zipFile, s3File);
+            var url = 'https://' + s3_endpoint + "/" + s3_bucket + "/" + s3File;
+            event.downloadUrl = url;
+            await event.save();
+
+        } catch (e) {
+            console.error("Failed processing event " + e);
+            reject(e);
+        }
+        cleanupEvent(tmpEventFolder, zipFile);
+        resolve('resolved');
+
+    });
+};
+
+
 const compareCalls = (a, b) => {
    const aTimestamp = new Date(a.time).getTime()
    const bTimestamp = new Date(b.time).getTime()
@@ -59,7 +200,7 @@ exports.addNewEvent = async function (req, res, next) {
        callIds.forEach((id) => {
            objectIds.push(new ObjectID(id));
        })
-        
+
        event.title = title.replace(/[^a-zA-Z0-9 \.\-\_]/g, "");
        event.description = description.replace(/[^a-zA-Z0-9 \.\-\_]/g, "");
        try {
@@ -80,11 +221,12 @@ exports.addNewEvent = async function (req, res, next) {
            });
            event.calls = calls;
            event.numCalls = calls.length;
-            await event.save();
-            
-            const url =  "/events/" + event._id;
+            const savedEvent = await event.save();
+            console.log("Event ID " + savedEvent._id);
+            await packageEvent(savedEvent._id);
+            const url = "/events/" + event._id;
            res.contentType('json');
-            res.send(JSON.stringify({url:url}));
+            res.send(JSON.stringify({ url: url }));
        }
        catch (err) {
            console.error(err);
@@ -94,4 +236,6 @@ exports.addNewEvent = async function (req, res, next) {


    })
-};
+};
+
+
--- a/backend/models/event.js
+++ b/backend/models/event.js
@@ -8,6 +8,7 @@ var eventSchema = mongoose.Schema({
  expireTime: Date,
  startTime: Date,
  endTime: Date,
+  downloadUrl: String,
  shortNames: [String],
  calls: [callSchema],
  numCalls: Number
--- a/backend/package-lock.json
+++ b/backend/package-lock.json
@@ -9,6 +9,7 @@
      "version": "1.0.0",
      "license": "ISC",
      "dependencies": {
+        "adm-zip": "^0.5.10",
        "aws-sdk": "^2.5",
        "axios": "0.18.0",
        "bcrypt-nodejs": "0.0.3",
@@ -28,7 +29,7 @@
        "mongodb": "2.2.36",
        "mongoose": "5.12.3",
        "multer": "1.3.0",
-        "node-ffprobe": "github:ListenerApproved/node-ffprobe",
+        "node-ffprobe": "https://github.com/ListenerApproved/node-ffprobe",
        "node-schedule": "1.2.4",
        "passport": "0.4.0",
        "passport-local": "1.0.0",
@@ -92,6 +93,14 @@
        "node": ">= 0.6"
      }
    },
+    "node_modules/adm-zip": {
+      "version": "0.5.10",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.10.tgz",
+      "integrity": "sha512-x0HvcHqVJNTPk/Bw8JbLWlWoo6Wwnsug0fnYYro1HBrjxZ3G7/AZk7Ahv8JwDe1uIcz8eBqvu86FuF1POiG7vQ==",
+      "engines": {
+        "node": ">=6.0"
+      }
+    },
    "node_modules/append-field": {
      "version": "0.1.0",
      "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz",
@@ -2491,6 +2500,11 @@
        "negotiator": "0.6.3"
      }
    },
+    "adm-zip": {
+      "version": "0.5.10",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.10.tgz",
+      "integrity": "sha512-x0HvcHqVJNTPk/Bw8JbLWlWoo6Wwnsug0fnYYro1HBrjxZ3G7/AZk7Ahv8JwDe1uIcz8eBqvu86FuF1POiG7vQ=="
+    },
    "append-field": {
      "version": "0.1.0",
      "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz",
@@ -3739,7 +3753,7 @@
    },
    "node-ffprobe": {
      "version": "git+ssh://git@github.com/ListenerApproved/node-ffprobe.git#d5e0629ad2545163225f6a1b45b277254ca488a3",
-      "from": "node-ffprobe@github:ListenerApproved/node-ffprobe"
+      "from": "node-ffprobe@https://github.com/ListenerApproved/node-ffprobe"
    },
    "node-gyp-build": {
      "version": "4.6.0",
--- a/backend/package.json
+++ b/backend/package.json
@@ -3,6 +3,7 @@
  "version": "1.0.0",
  "main": "index.js",
  "dependencies": {
+    "adm-zip": "^0.5.10",
    "aws-sdk": "^2.5",
    "axios": "0.18.0",
    "bcrypt-nodejs": "0.0.3",
@@ -31,7 +32,6 @@
    "websocket": "^1.0.24",
    "ws": "^3.1.0"
  },
-  "devDependencies": {},
  "scripts": {
    "test": "mocha --reporter spec"
  },
--- a/frontend/src/Event/ViewEvent.js
+++ b/frontend/src/Event/ViewEvent.js
@@ -65,46 +65,49 @@ function ViewEvent(props) {
      setSelectCallId(_id);
    }
  }
-  
+
  const dateRange = (start, end) => {
    if (start.getYear() != end.getYear()) {
-        return {"part1": start.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + start.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'}), "part2":  end.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'})}; 
+      return { "part1": start.toLocaleDateString('en-us', { weekday: "short", year: "numeric", month: "short", day: "numeric" }) + " " + start.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }), "part2": end.toLocaleDateString('en-us', { weekday: "short", year: "numeric", month: "short", day: "numeric" }) + " " + end.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) };
    } else if (start.getMonth() != end.getMonth()) {
-        return {"part1": start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'}), "part2": end.toLocaleDateString('en-us', { weekday:"short", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'})}; 
-    
+      return { "part1": start.toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" }) + " " + start.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }), "part2": end.toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" }) + " " + end.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) };
+
    } else if (start.getDay() != end.getDay()) {
-        return {"part1": start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'}), "part2": end.toLocaleDateString('en-us', { weekday:"short", day:"numeric"}) + " " + end.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'})}; 
-    
+      return { "part1": start.toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" }) + " " + start.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }), "part2": end.toLocaleDateString('en-us', { weekday: "short", day: "numeric" }) + " " + end.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) };
+
    } else {
-        return {"part1": start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'}), "part2": end.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'})}; 
-    
+      return { "part1": start.toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" }) + " " + start.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }), "part2": end.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) };
+
    }
-}
-  let dates = {part1: "", part2: ""}
+  }
+  let dates = { part1: "", part2: "" }
  let expirationDate = ""
+  let downloadUrl = false
  if (eventData) {
-   dates = dateRange(new Date(eventData.startTime),new Date(eventData.endTime))
-   expirationDate = new Date(eventData.expireTime).toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"})
+    downloadUrl = eventData.downloadUrl;
+    dates = dateRange(new Date(eventData.startTime), new Date(eventData.endTime))
+    expirationDate = new Date(eventData.expireTime).toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" })
  }
-/*
-  const dateRange = (start, end) => {
-    if (start.getYear() != end.getYear()) {
-        return start.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
-    } else if (start.getMonth() != end.getMonth()) {
-        return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
-    
-    } else if (start.getDay() != end.getDay()) {
-        return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
-    
-    } else {
-        return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleTimeString(); 
-    
+
+  /*
+    const dateRange = (start, end) => {
+      if (start.getYear() != end.getYear()) {
+          return start.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
+      } else if (start.getMonth() != end.getMonth()) {
+          return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
+      
+      } else if (start.getDay() != end.getDay()) {
+          return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
+      
+      } else {
+          return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleTimeString(); 
+      
+      }
    }
-  }
-  let dates = "";
-  if (eventData) {
-    dates = dateRange(new Date(eventData.startTime),new Date(eventData.endTime));
-   }*/
+    let dates = "";
+    if (eventData) {
+      dates = dateRange(new Date(eventData.startTime),new Date(eventData.endTime));
+     }*/
  return (
    <div >
      <Sidebar as={Menu} animation='overlay' inverted vertical visible={sidebarOpened}
@@ -127,27 +130,37 @@ function ViewEvent(props) {
      <Grid style={{ paddingTop: '4em' }}>
        <Grid.Column mobile={1} tablet={1} computer={1} ></Grid.Column>
        <Grid.Column mobile={16} tablet={8} computer={4}>
-          
-            {eventData&& 
+
+          {eventData &&
            <>
-            <p>{eventData.description}</p>
-            <Header size='tiny'>Expires {expirationDate}</Header>
+              <p>{eventData.description}</p>
+              <Header size='tiny'>Expires {expirationDate}</Header>
            </>
          }
-          
+
        </Grid.Column>
-        <Grid.Column mobile={8} tablet={8} computer={4}>
-          <Header as="h2">{dates.part1} <br/>→ {dates.part2}</Header>
+        <Grid.Column mobile={8} tablet={8} computer={3}>
+          <Header as="h2">{dates.part1} <br />→ {dates.part2}</Header>
        </Grid.Column>
        <Grid.Column mobile={8} tablet={8} computer={2}>
-        
+
          <Statistic>
            <Statistic.Value>{eventData && eventData.numCalls}</Statistic.Value>
            <Statistic.Label>Calls</Statistic.Label>
          </Statistic>
-          </Grid.Column>
+        </Grid.Column>

-        <Grid.Column mobile={1} tablet={1} computer={5}>
+        <Grid.Column mobile={1} tablet={1} computer={2}>
+          {downloadUrl &&
+            <a href={downloadUrl}>
+              <Statistic small className="desktop-only">
+                <Statistic.Value><Icon name="download" /></Statistic.Value>
+                <Statistic.Label>Download Event</Statistic.Label>
+              </Statistic>
+            </a>
+          }
+        </Grid.Column>
+        <Grid.Column mobile={1} tablet={1} computer={4}>
        </Grid.Column>
      </Grid>
      {eventData && <EventPlayer eventData={eventData} selectCallId={selectCallId} />}