Merge pull request #135 from openmhz/event-download

Add in event downloads

backend/Dockerfile

@@ -1,12 +1,13 @@
 # docker build -t smartnet-player .
 # do a `git pull` in smartnet-player to update
 
-FROM node:10
+FROM node:19-bullseye
+
 
 #RUN useradd --user-group --create-home --shell /bin/false app &&\
 #  npm install --global npm@3.7.5
-RUN echo deb http://www.deb-multimedia.org stretch main non-free \
+RUN echo deb https://www.deb-multimedia.org bullseye main non-free \
-	>>/etc/apt/sources.list && apt-get update &&\
+	>>/etc/apt/sources.list && apt-get update -oAcquire::AllowInsecureRepositories=true &&\
 	apt-get install -y --force-yes deb-multimedia-keyring && \
 	apt-get update && \
 	apt-get install -y ffmpeg cron python build-essential g++

backend/controllers/events.js

@@ -1,9 +1,32 @@
 var ObjectID = require('mongodb').ObjectID;
+const https = require('https');
 var db = require('../db');
 var mongoose = require("mongoose");
 var Event = require("../models/event");
+const AdmZip = require("adm-zip");
 var { callModel: Call, callSchema } = require("../models/call");
 var frontend_server = process.env['REACT_APP_FRONTEND_SERVER'] != null ? process.env['REACT_APP_FRONTEND_SERVER'] : 'https://openmhz.com';
+const os = require('os');
+var fs = require('fs');
+
+const s3 = require('aws-sdk');
+const { events } = require('../models/event');
+
+var s3_endpoint = process.env['S3_ENDPOINT'] != null ? process.env['S3_ENDPOINT'] : 's3.us-west-1.wasabisys.com';
+var s3_bucket = process.env['S3_BUCKET'] != null ? process.env['S3_BUCKET'] : 'openmhz-west';
+var s3_profile = process.env['S3_PROFILE'] != null ? process.env['S3_PROFILE'] : 'wasabi-account';
+
+const s3Endpoint = new s3.Endpoint(s3_endpoint);
+var s3Credentials = new s3.SharedIniFileCredentials({
+    profile: s3_profile
+});
+
+s3.config.credentials = s3Credentials;
+const s3S3 = new s3.S3({
+    endpoint: s3Endpoint,
+    maxRetries: 2
+});
+
 
 
 exports.getEvent = async function (req, res, next) {
@@ -23,7 +46,7 @@ exports.getEvent = async function (req, res, next) {
 
 exports.getEvents = async function (req, res, next) {
     try {
-        let events = await Event.find({},["title",  "description", "startTime", "endTime", "expireTime", "numCalls", "createdAt", "shortNames"]).exec();
+        let events = await Event.find({}, ["title", "description", "startTime", "endTime", "expireTime", "numCalls", "createdAt", "shortNames"]).exec();
         res.contentType('json');
         res.send(JSON.stringify(events));
     }
@@ -34,6 +57,124 @@ exports.getEvents = async function (req, res, next) {
     }
 }
 
+
+
+async function downloadFile(folderName, url) {
+    return new Promise((resolve, reject) => {
+        https.get(url, (response) => {
+            const filename = new URL(url).pathname.split('/').pop();
+            const filePath = `${folderName}/${filename}`;
+            console.log("Downloading to filename " + filename + " and filePath " + filePath);
+            let stream = fs.createWriteStream(filePath);
+
+            response.on("error", reject);
+
+            stream.on("finish", () => {
+                stream.close();
+                console.log("Download Completed");
+                resolve();
+            }).on("error", reject);
+
+            response.pipe(stream);
+
+        }).on("error", reject);
+    });
+}
+
+
+const exportEventJson = (eventFolder, event) => {
+    const filename = eventFolder + "/event.json";
+    fs.writeFileSync(filename, JSON.stringify(event));
+}
+
+const createEventZip = async (folder, filename, zipFolder) => {
+    try {
+        const zip = new AdmZip();
+        zip.addLocalFolder(folder, zipFolder);
+        zip.writeZip(filename);
+        console.log(`Created ${filename} successfully for event from ${folder}`);
+    } catch (e) {
+        console.error("Event Error trying to zip: " + filename + " error: " + e);
+    }
+}
+
+const uploadEventZip = (zipFile, s3File) => {
+
+    // If not a pro plan, use S3 storage
+
+    var s3Src = fs.createReadStream(zipFile);
+    return new Promise((resolve, reject) => {
+        const s3Params = {
+            Bucket: s3_bucket,
+            Key: s3File,
+            Body: s3Src,
+            ACL: 'public-read'
+        };
+
+        s3S3.upload(s3Params, function (err, data) {
+            s3Src.destroy();
+
+            if (err) {
+                console.error("Event S3 Upload Error " + err);
+                reject(err);
+            } else {
+                resolve("done");
+            }
+        });
+    });
+}
+
+const cleanupEvent = (folder, filename) => {
+    fs.rmSync(folder, { recursive: true, force: true });
+    fs.rmSync(filename);
+}
+
+const packageEvent = (eventId) => {
+    const event = "tst"
+    const tmpdir = os.tmpdir();
+    return new Promise(async (resolve, reject) => {
+        var objectId = eventId;
+        const event = await Event.findById(objectId).exec();
+        if (!event) {
+            reject("Unable to load Event " + eventId);
+        }
+        let eventFolder = "event_" + Date.now();
+        event.shortNames.forEach((shortName) => {
+            eventFolder = eventFolder + "_" + shortName;
+        });
+        const tmpEventFolder = tmpdir + "/" + eventFolder
+        const zipFile = tmpdir + "/" + eventFolder + ".zip"
+        const s3File = "events/" + eventFolder + ".zip"
+        console.log("Using tmp " + tmpEventFolder);
+
+        try {
+            if (!fs.existsSync(tmpEventFolder)) {
+                fs.mkdirSync(tmpEventFolder);
+            }
+
+            for (const call of event.calls) {
+                await downloadFile(tmpEventFolder, call.url)
+            }
+
+
+            exportEventJson(tmpEventFolder, event);
+            createEventZip(tmpEventFolder, zipFile, eventFolder);
+            await uploadEventZip(zipFile, s3File);
+            var url = 'https://' + s3_endpoint + "/" + s3_bucket + "/" + s3File;
+            event.downloadUrl = url;
+            await event.save();
+
+        } catch (e) {
+            console.error("Failed processing event " + e);
+            reject(e);
+        }
+        cleanupEvent(tmpEventFolder, zipFile);
+        resolve('resolved');
+
+    });
+};
+
+
 const compareCalls = (a, b) => {
     const aTimestamp = new Date(a.time).getTime()
     const bTimestamp = new Date(b.time).getTime()
@@ -80,11 +221,12 @@ exports.addNewEvent = async function (req, res, next) {
             });
             event.calls = calls;
             event.numCalls = calls.length;
-            await event.save();
+            const savedEvent = await event.save();
-            
+            console.log("Event ID " + savedEvent._id);
-            const url =  "/events/" + event._id;
+            await packageEvent(savedEvent._id);
+            const url = "/events/" + event._id;
             res.contentType('json');
-            res.send(JSON.stringify({url:url}));
+            res.send(JSON.stringify({ url: url }));
         }
         catch (err) {
             console.error(err);
@@ -95,3 +237,5 @@ exports.addNewEvent = async function (req, res, next) {
 
     })
 };
+
+

backend/models/event.js

@@ -8,6 +8,7 @@ var eventSchema = mongoose.Schema({
   expireTime: Date,
   startTime: Date,
   endTime: Date,
+  downloadUrl: String,
   shortNames: [String],
   calls: [callSchema],
   numCalls: Number

backend/package-lock.json

@@ -9,6 +9,7 @@
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
+        "adm-zip": "^0.5.10",
         "aws-sdk": "^2.5",
         "axios": "0.18.0",
         "bcrypt-nodejs": "0.0.3",
@@ -28,7 +29,7 @@
         "mongodb": "2.2.36",
         "mongoose": "5.12.3",
         "multer": "1.3.0",
-        "node-ffprobe": "github:ListenerApproved/node-ffprobe",
+        "node-ffprobe": "https://github.com/ListenerApproved/node-ffprobe",
         "node-schedule": "1.2.4",
         "passport": "0.4.0",
         "passport-local": "1.0.0",
@@ -92,6 +93,14 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/adm-zip": {
+      "version": "0.5.10",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.10.tgz",
+      "integrity": "sha512-x0HvcHqVJNTPk/Bw8JbLWlWoo6Wwnsug0fnYYro1HBrjxZ3G7/AZk7Ahv8JwDe1uIcz8eBqvu86FuF1POiG7vQ==",
+      "engines": {
+        "node": ">=6.0"
+      }
+    },
     "node_modules/append-field": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz",
@@ -2491,6 +2500,11 @@
         "negotiator": "0.6.3"
       }
     },
+    "adm-zip": {
+      "version": "0.5.10",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.10.tgz",
+      "integrity": "sha512-x0HvcHqVJNTPk/Bw8JbLWlWoo6Wwnsug0fnYYro1HBrjxZ3G7/AZk7Ahv8JwDe1uIcz8eBqvu86FuF1POiG7vQ=="
+    },
     "append-field": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz",
@@ -3739,7 +3753,7 @@
     },
     "node-ffprobe": {
       "version": "git+ssh://git@github.com/ListenerApproved/node-ffprobe.git#d5e0629ad2545163225f6a1b45b277254ca488a3",
-      "from": "node-ffprobe@github:ListenerApproved/node-ffprobe"
+      "from": "node-ffprobe@https://github.com/ListenerApproved/node-ffprobe"
     },
     "node-gyp-build": {
       "version": "4.6.0",

backend/package.json

@@ -3,6 +3,7 @@
   "version": "1.0.0",
   "main": "index.js",
   "dependencies": {
+    "adm-zip": "^0.5.10",
     "aws-sdk": "^2.5",
     "axios": "0.18.0",
     "bcrypt-nodejs": "0.0.3",
@@ -31,7 +32,6 @@
     "websocket": "^1.0.24",
     "ws": "^3.1.0"
   },
-  "devDependencies": {},
   "scripts": {
     "test": "mocha --reporter spec"
   },

frontend/src/Event/ViewEvent.js

@@ -68,43 +68,46 @@ function ViewEvent(props) {
 
   const dateRange = (start, end) => {
     if (start.getYear() != end.getYear()) {
-        return {"part1": start.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + start.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'}), "part2":  end.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'})}; 
+      return { "part1": start.toLocaleDateString('en-us', { weekday: "short", year: "numeric", month: "short", day: "numeric" }) + " " + start.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }), "part2": end.toLocaleDateString('en-us', { weekday: "short", year: "numeric", month: "short", day: "numeric" }) + " " + end.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) };
     } else if (start.getMonth() != end.getMonth()) {
-        return {"part1": start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'}), "part2": end.toLocaleDateString('en-us', { weekday:"short", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'})}; 
+      return { "part1": start.toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" }) + " " + start.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }), "part2": end.toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" }) + " " + end.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) };
 
     } else if (start.getDay() != end.getDay()) {
-        return {"part1": start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'}), "part2": end.toLocaleDateString('en-us', { weekday:"short", day:"numeric"}) + " " + end.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'})}; 
+      return { "part1": start.toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" }) + " " + start.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }), "part2": end.toLocaleDateString('en-us', { weekday: "short", day: "numeric" }) + " " + end.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) };
 
     } else {
-        return {"part1": start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'}), "part2": end.toLocaleTimeString([], {hour: 'numeric', minute:'2-digit'})}; 
+      return { "part1": start.toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" }) + " " + start.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }), "part2": end.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) };
 
     }
-}
+  }
-  let dates = {part1: "", part2: ""}
+  let dates = { part1: "", part2: "" }
   let expirationDate = ""
+  let downloadUrl = false
   if (eventData) {
-   dates = dateRange(new Date(eventData.startTime),new Date(eventData.endTime))
+    downloadUrl = eventData.downloadUrl;
-   expirationDate = new Date(eventData.expireTime).toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"})
+    dates = dateRange(new Date(eventData.startTime), new Date(eventData.endTime))
+    expirationDate = new Date(eventData.expireTime).toLocaleDateString('en-us', { weekday: "short", month: "short", day: "numeric" })
   }
-/*
-  const dateRange = (start, end) => {
-    if (start.getYear() != end.getYear()) {
-        return start.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
-    } else if (start.getMonth() != end.getMonth()) {
-        return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
 
-    } else if (start.getDay() != end.getDay()) {
+  /*
-        return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
+    const dateRange = (start, end) => {
+      if (start.getYear() != end.getYear()) {
+          return start.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", year:"numeric", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
+      } else if (start.getMonth() != end.getMonth()) {
+          return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", month:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
       
-    } else {
+      } else if (start.getDay() != end.getDay()) {
-        return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleTimeString(); 
+          return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleDateString('en-us', { weekday:"short", day:"numeric"}) + " " + end.toLocaleTimeString(); 
       
+      } else {
+          return start.toLocaleDateString('en-us', { weekday:"short",  month:"short", day:"numeric"}) + " " + start.toLocaleTimeString() + " - " + end.toLocaleTimeString(); 
+      
+      }
     }
-  }
+    let dates = "";
-  let dates = "";
+    if (eventData) {
-  if (eventData) {
+      dates = dateRange(new Date(eventData.startTime),new Date(eventData.endTime));
-    dates = dateRange(new Date(eventData.startTime),new Date(eventData.endTime));
+     }*/
-   }*/
   return (
     <div >
       <Sidebar as={Menu} animation='overlay' inverted vertical visible={sidebarOpened}
@@ -128,16 +131,16 @@ function ViewEvent(props) {
         <Grid.Column mobile={1} tablet={1} computer={1} ></Grid.Column>
         <Grid.Column mobile={16} tablet={8} computer={4}>
 
-            {eventData&& 
+          {eventData &&
             <>
-            <p>{eventData.description}</p>
+              <p>{eventData.description}</p>
-            <Header size='tiny'>Expires {expirationDate}</Header>
+              <Header size='tiny'>Expires {expirationDate}</Header>
             </>
           }
 
         </Grid.Column>
-        <Grid.Column mobile={8} tablet={8} computer={4}>
+        <Grid.Column mobile={8} tablet={8} computer={3}>
-          <Header as="h2">{dates.part1} <br/>→ {dates.part2}</Header>
+          <Header as="h2">{dates.part1} <br />→ {dates.part2}</Header>
         </Grid.Column>
         <Grid.Column mobile={8} tablet={8} computer={2}>
 
@@ -145,9 +148,19 @@ function ViewEvent(props) {
             <Statistic.Value>{eventData && eventData.numCalls}</Statistic.Value>
             <Statistic.Label>Calls</Statistic.Label>
           </Statistic>
-          </Grid.Column>
+        </Grid.Column>
 
-        <Grid.Column mobile={1} tablet={1} computer={5}>
+        <Grid.Column mobile={1} tablet={1} computer={2}>
+          {downloadUrl &&
+            <a href={downloadUrl}>
+              <Statistic small className="desktop-only">
+                <Statistic.Value><Icon name="download" /></Statistic.Value>
+                <Statistic.Label>Download Event</Statistic.Label>
+              </Statistic>
+            </a>
+          }
+        </Grid.Column>
+        <Grid.Column mobile={1} tablet={1} computer={4}>
         </Grid.Column>
       </Grid>
       {eventData && <EventPlayer eventData={eventData} selectCallId={selectCallId} />}