From e86bfb1418577a755b094c592d04278e09314503 Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Thu, 26 Nov 2020 19:00:17 -0600 Subject: [PATCH 1/9] Adding htpasswd, fails without it --- nginx-proxy/htpasswd | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 nginx-proxy/htpasswd diff --git a/nginx-proxy/htpasswd b/nginx-proxy/htpasswd new file mode 100644 index 0000000..e69de29 From 744300b9d9bbe92ae9aa5fad1a8de5639b0965cb Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Thu, 26 Nov 2020 19:09:45 -0600 Subject: [PATCH 2/9] Adding S3_BUCKET to define the bucket name --- docker-compose.yml | 1 + docker-prod.sh | 3 ++- docker-test.sh | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 18b9267..1a7898c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,6 +67,7 @@ services: - FREE_PLAN_ARCHIVE - PRO_PLAN_ARCHIVE - S3_ENDPOINT + - S3_BUCKET - S3_PROFILE volumes: - "${MEDIA}:/home/app/trunk-server-backend/media" diff --git a/docker-prod.sh b/docker-prod.sh index a26ec0d..db4397f 100755 --- a/docker-prod.sh +++ b/docker-prod.sh @@ -15,7 +15,8 @@ export SOCKET_SERVER="wss://socket.${DOMAIN_NAME}" export ACCOUNT_SERVER="${PROTOCOL}account.${DOMAIN_NAME}" export COOKIE_DOMAIN=".${DOMAIN_NAME}" export S3_PROFILE='wasabi-account' -export S3_ENPOINT='s3.us-west-1.wasabisys.com' +export S3_ENDPOINT='s3.us-west-1.wasabisys.com' +export S3_BUCKET='openmhz-west' export SITE_NAME="OpenMHz" export ADMIN_EMAIL="luke@openmhz.com" export FREE_PLAN=0 diff --git a/docker-test.sh b/docker-test.sh index 7407925..9d57cc5 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -19,7 +19,8 @@ export COOKIE_DOMAIN=".${DOMAIN_NAME}" export SITE_NAME="OpenMHz" export ADMIN_EMAIL="admin@email.com" export S3_PROFILE='default' -export S3_ENPOINT='s3.us-west-1.wasabisys.com' +export S3_ENDPOINT='s3.us-west-1.wasabisys.com' +export S3_BUCKET='openmhz-test' export FREE_PLAN=0 export PRO_PLAN=10 export FREE_PLAN_PRICE=0 From 2c991d261fab2f436996e18cc371a38678365bb1 Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Thu, 26 Nov 2020 19:15:42 -0600 Subject: [PATCH 3/9] Modify upload script to use S3_BUCKET variable --- backend/controllers/uploads.js | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/backend/controllers/uploads.js b/backend/controllers/uploads.js index ffe4915..e4d3118 100644 --- a/backend/controllers/uploads.js +++ b/backend/controllers/uploads.js @@ -15,6 +15,7 @@ var config = require('../config/config.json'); var media_server = process.env['MEDIA_SERVER'] != null ? process.env['MEDIA_SERVER'] : 'https://media.openmhz.com'; var s3_endpoint = process.env['S3_ENDPOINT'] != null ? process.env['S3_ENDPOINT'] : 's3.us-west-1.wasabisys.com'; +var s3_bucket = process.env['S3_BUCKET'] != null ? process.env['S3_BUCKET'] : 'openmhz-west'; var s3_profile = process.env['S3_PROFILE'] != null ? process.env['S3_PROFILE'] : 'wasabi-account'; const wasabi = require('aws-sdk'); @@ -126,23 +127,21 @@ process.nextTick(function() { var local_path = "/" + shortName + "/" + time.getFullYear() + "/" + (time.getMonth() + 1) + "/" + time.getDate() + "/"; var target_file = base_path + local_path + path.basename(req.file.originalname); var object_key = "media/" + shortName + "-" + talkgroupNum + "-" + startTime + ".m4a"; - //var endpoint = 'https://s3.amazonaws.com/openmhz/'; - //var endpoint = 'https://s3.wasabisys.com/openmhz/'; + var endpoint = s3_endpoint; - var bucket = 'openmhz' - //var url = 'https://openmhz.nyc3.cdn.digitaloceanspaces.com/' + object_key; - - var url = endpoint + object_key; - //var url = 'https://s3.wasabisys.com/openmhz/' + object_key; - //var url = 'https://s3.amazonaws.com/openmhz/' + object_key; - var objectStore = true; + var bucket = s3_bucket; + if (item.plan && (item.plan.type == "pro")) { + // If a pro plan, use local storage + // TODO: Why? url = media_server + local_path + talkgroupNum + "-" + startTime + ".m4a"; objectStore = false; + } else { + // If not a pro plan, use S3 storage + var url = 'https://' + s3_endpoint + "/" + s3_bucket + "/" + object_key; + var objectStore = true; } - - var call = new Call({ shortName: shortName, talkgroupNum: talkgroupNum, @@ -222,7 +221,7 @@ process.nextTick(function() { });*/ var wasabiParams = { - Bucket: 'openmhz-west', + Bucket: s3_bucket, Key: object_key, Body: wasabiSrc, ACL: 'public-read' From 4d05397cce787512e93eebd754bc788850d9f567 Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Thu, 26 Nov 2020 19:19:13 -0600 Subject: [PATCH 4/9] Updating README to detail S3 config. --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d256574..79cd80c 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,18 @@ There are a lot of different components that make up the system. All of the serv You can run things in 2 different modes, `test` and `prod`. The big difference is that `prod` expects all of the calls to be https and gets cranky when they are not. I have gotten `test` to run fine on my laptop, so that is probably a good starting point. +Currently, both `test` and `prod` expect to use S3-based storage instead of local storage. Switching to use local storage would be relatively easy - but for the sake of testing, let's just say use something S3-compatible. Make sure that ~/.aws/credentials has the credentials you'd like to use with your S3-compatible storage provider.. IE: +``` +[default] +aws_access_key_id = [..] +aws_secret_access_key = [..] +``` `./docker-test.sh` - sets up docker-compose to run with the correct environment variables for testing `./docker-prod.sh` - sets up docker-compose to run with the correct environment variables for production -Fill out the Environment variables in both these files. You don't need to worry about the Stripe stuff, since that is dormant. +Fill out the Environment variables in both these files. S3_PROFILE should match the profile that is defined in the aws/credentials file. You don't need to worry about the Stripe stuff, since that is dormant. If you are doing frontend work, you can turn off the de-obfuscation. To do this, edit the Dockerfile for the server you are working with. Change the CMD to: `CMD ["npm", "run", "start:dev"]` Just remember to turn it off when you make the code public. @@ -91,4 +97,4 @@ After doing this, you should have the following domains: api.domain.com, account ### Mailjet Sign up for MailJet! -https://www.mailjet.com \ No newline at end of file +https://www.mailjet.com From feb9d0eb3747c9ff57926e183586866f32575b6a Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Fri, 27 Nov 2020 10:20:33 -0600 Subject: [PATCH 5/9] Remove htpasswd again, and remove reference from Dockerfile instead - I see that htpasswd was intentionally removed. --- nginx-proxy/Dockerfile | 1 - nginx-proxy/htpasswd | 0 2 files changed, 1 deletion(-) delete mode 100644 nginx-proxy/htpasswd diff --git a/nginx-proxy/Dockerfile b/nginx-proxy/Dockerfile index 07af6fe..6dbeb24 100644 --- a/nginx-proxy/Dockerfile +++ b/nginx-proxy/Dockerfile @@ -21,7 +21,6 @@ RUN apt-get update \ # Add main NGINX config COPY *.conf /etc/nginx/ -COPY htpasswd /etc/nginx/ COPY ${STAGE}/certs/ /etc/nginx/certs/ # Add virtual hosts diff --git a/nginx-proxy/htpasswd b/nginx-proxy/htpasswd deleted file mode 100644 index e69de29..0000000 From 4d4f2a13d497d8e3a02c9cc3fb6e0165fd4b9094 Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Thu, 26 Nov 2020 20:05:59 -0600 Subject: [PATCH 6/9] Rename docker-prod/docker-test to example, and gitignore the actual files. --- .gitignore | 2 ++ README.md | 2 ++ docker-prod.sh => docker-prod.sh.example | 0 docker-test.sh => docker-test.sh.example | 0 4 files changed, 4 insertions(+) create mode 100644 .gitignore rename docker-prod.sh => docker-prod.sh.example (100%) rename docker-test.sh => docker-test.sh.example (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cd96ea1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +docker-prod.sh +docker-test.sh diff --git a/README.md b/README.md index 79cd80c..0c5b16f 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ aws_access_key_id = [..] aws_secret_access_key = [..] ``` +Copy docker-test.sh.example to docker-test.sh, and/or docker-prod.sh.example to docker-prod.sh. + `./docker-test.sh` - sets up docker-compose to run with the correct environment variables for testing `./docker-prod.sh` - sets up docker-compose to run with the correct environment variables for production diff --git a/docker-prod.sh b/docker-prod.sh.example similarity index 100% rename from docker-prod.sh rename to docker-prod.sh.example diff --git a/docker-test.sh b/docker-test.sh.example similarity index 100% rename from docker-test.sh rename to docker-test.sh.example From da90ad5c7e7558b839632ddfbdfd47d6bf90a52e Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Thu, 26 Nov 2020 20:29:05 -0600 Subject: [PATCH 7/9] Increase memory to avoid running out while building --- frontend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 4fafcc0..014b4b3 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -16,6 +16,6 @@ ENV PATH /usr/app/node_modules/.bin:$PATH WORKDIR $HOME COPY . $HOME/ -RUN npm run build +RUN node --max_old_space_size=8000 $(which npm) run build CMD ["npm", "run", "start"] From a26606faf2526e1f09e237901ae1a92f7ba8d903 Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Thu, 26 Nov 2020 21:43:00 -0600 Subject: [PATCH 8/9] Increase the memory limit for real --- frontend/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 014b4b3..d69058e 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -5,6 +5,7 @@ FROM node:8.11 ENV HOME=/home/app +ENV NODE_OPTIONS="--max_old_space_size=8000" RUN mkdir -p /home/app COPY ./package.json /tmp @@ -16,6 +17,6 @@ ENV PATH /usr/app/node_modules/.bin:$PATH WORKDIR $HOME COPY . $HOME/ -RUN node --max_old_space_size=8000 $(which npm) run build +RUN npm run build -CMD ["npm", "run", "start"] +CMD ["npm", "run", "start:dev"] From e014fe0c97f9ba662e5b2f08150340e4f67389ba Mon Sep 17 00:00:00 2001 From: Nate Carlson Date: Fri, 27 Nov 2020 10:30:58 -0600 Subject: [PATCH 9/9] Oops, remove change that starts in dev mode. --- frontend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index d69058e..797cc95 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -19,4 +19,4 @@ WORKDIR $HOME COPY . $HOME/ RUN npm run build -CMD ["npm", "run", "start:dev"] +CMD ["npm", "run", "start"]