Files
trunk-server/nginx-proxy/prod/vhosts/site.template
Luke Berndt 5b177eca58 Raise nginx connection limits and microcache /calls polling
A traffic surge exhausted nginx's 3 x 4096 worker_connections (each
proxied websocket uses two), so Cloudflare got empty responses (520)
and the backend's listen queue filled.

- worker_processes auto (8 cores), worker_connections 16384,
  worker_rlimit_nofile 65535.
- Cache public /calls endpoints on api.*: 5s for live lists, 60s for
  historical ones, with cache lock and stale-while-updating so bursts
  collapse to one backend query. Origin is in the cache key because
  the backend's CORS header varies by Origin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 22:42:11 -05:00

218 lines
9.0 KiB
Plaintext

limit_req_zone $binary_remote_addr zone=perip:10m rate=1r/s;
# Only send "Connection: upgrade" for real WebSocket requests, so ordinary
# requests can reuse the upstream keepalive pool.
map $http_upgrade $backend_connection {
default upgrade;
'' '';
}
# Microcache for the public /calls polling endpoints. Responses carry no
# per-user data; the CORS header varies by Origin, so Origin is in the key.
proxy_cache_path /var/cache/nginx/api levels=1:2 keys_zone=api_cache:20m
max_size=500m inactive=10m use_temp_path=off;
upstream backend_api {
server backend:3005;
keepalive 64;
keepalive_timeout 60s;
}
server {
listen 80;
server_name account.${DOMAIN_NAME} api.${DOMAIN_NAME} admin.${DOMAIN_NAME} www.${DOMAIN_NAME} ${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
return 301 https://$host$request_uri;
}
server {
listen 443 ssl;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
#ssl_stapling on;
#ssl_stapling_verify on;
#add_header Strict-Transport-Security max-age=15768000;
server_name ${DOMAIN_NAME} www.${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location /static {
proxy_pass http://frontend:3000;
access_log off;
}
location /favicon.ico {
proxy_pass http://frontend:3000;
expires 1M;
add_header Surrogate-Control "public, max-age=2629746";
add_header Cache-Control "max-age=2629746, public";
access_log off;
}
location /*.png {
proxy_pass http://frontend:3000;
access_log off;
expires 1M;
add_header Surrogate-Control "public, max-age=2629746";
add_header Cache-Control "max-age=2629746, public";
}
location /*.m4a {
proxy_pass http://frontend:3000;
access_log off;
expires 1M;
add_header Surrogate-Control "public, max-age=2629746";
add_header Cache-Control "max-age=2629746, public";
}
location / {
proxy_pass http://frontend:3000;
access_log /dev/stdout main;
}
}
server {
listen 443 ssl;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
server_name api.${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
# Live call lists: short TTL; the socket pushes anything newer.
location ~ ^/[^/]+/calls(/(latest|newer|next))?$ {
proxy_pass http://backend_api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
include /etc/nginx/proxy.conf;
proxy_cache api_cache;
proxy_cache_key "$scheme$host$request_uri|$http_origin";
proxy_cache_methods GET HEAD;
proxy_cache_valid 200 5s;
proxy_cache_lock on;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
add_header X-Cache-Status $upstream_cache_status always;
access_log off;
}
# Historical call lists: calls before a fixed time rarely change.
location ~ ^/[^/]+/calls/(older|date|[0-9]+/older)$ {
proxy_pass http://backend_api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
include /etc/nginx/proxy.conf;
proxy_cache api_cache;
proxy_cache_key "$scheme$host$request_uri|$http_origin";
proxy_cache_methods GET HEAD;
proxy_cache_valid 200 60s;
proxy_cache_lock on;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
add_header X-Cache-Status $upstream_cache_status always;
access_log off;
}
location / {
proxy_pass http://backend_api;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $backend_connection;
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
include /etc/nginx/proxy.conf;
access_log off;
}
}
server {
listen 443 ssl;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
server_name account.${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location / {
proxy_pass http://account:3009;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
include /etc/nginx/proxy.conf;
access_log off;
}
}
server {
listen 443 ssl;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
server_name admin.${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location / {
proxy_pass http://admin:3008;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
include /etc/nginx/proxy.conf;
access_log off;
}
}