Files
trunk-server/nginx-proxy/prod/vhosts/site.template
Luke Berndt a00bc98cc4 Index socket clients by system; make /systems cheap
A CPU profile during a traffic spike (4,850 sockets, 74 uploads/s) showed
the backend spending ~28% of its busy time in two loops over every
connected socket: notify_clients on each upload, and addSystemClients on
each /systems request.

- Add client_registry.js: sockets in a Map, with active clients indexed
  by system plus a firehose set. notify_clients only visits the
  uploaded call's system, with the same matching rules as before.
- /systems: counts come from the index, the serialized body is reused
  for 5s, and counts now reset to 0 when a system's listeners leave.
  Firehose clients no longer count toward a system.
- nginx: 10s microcache for /systems on api.*.
- Test: node --test test/client_registry.test.js checks recipients and
  counts against the old full-scan implementation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 13:22:31 -05:00

237 lines
9.7 KiB
Plaintext

limit_req_zone $binary_remote_addr zone=perip:10m rate=1r/s;
# Only send "Connection: upgrade" for real WebSocket requests, so ordinary
# requests can reuse the upstream keepalive pool.
map $http_upgrade $backend_connection {
default upgrade;
'' '';
}
# Microcache for the public /calls polling endpoints. Responses carry no
# per-user data; the CORS header varies by Origin, so Origin is in the key.
proxy_cache_path /var/cache/nginx/api levels=1:2 keys_zone=api_cache:20m
max_size=500m inactive=10m use_temp_path=off;
upstream backend_api {
server backend:3005;
keepalive 64;
keepalive_timeout 60s;
}
server {
listen 80;
server_name account.${DOMAIN_NAME} api.${DOMAIN_NAME} admin.${DOMAIN_NAME} www.${DOMAIN_NAME} ${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
return 301 https://$host$request_uri;
}
server {
listen 443 ssl;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
#ssl_stapling on;
#ssl_stapling_verify on;
#add_header Strict-Transport-Security max-age=15768000;
server_name ${DOMAIN_NAME} www.${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location /static {
proxy_pass http://frontend:3000;
access_log off;
}
location /favicon.ico {
proxy_pass http://frontend:3000;
expires 1M;
add_header Surrogate-Control "public, max-age=2629746";
add_header Cache-Control "max-age=2629746, public";
access_log off;
}
location /*.png {
proxy_pass http://frontend:3000;
access_log off;
expires 1M;
add_header Surrogate-Control "public, max-age=2629746";
add_header Cache-Control "max-age=2629746, public";
}
location /*.m4a {
proxy_pass http://frontend:3000;
access_log off;
expires 1M;
add_header Surrogate-Control "public, max-age=2629746";
add_header Cache-Control "max-age=2629746, public";
}
location / {
proxy_pass http://frontend:3000;
access_log /dev/stdout main;
}
}
server {
listen 443 ssl;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
server_name api.${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
# Live call lists: short TTL; the socket pushes anything newer.
location ~ ^/[^/]+/calls(/(latest|newer|next))?$ {
proxy_pass http://backend_api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
include /etc/nginx/proxy.conf;
proxy_cache api_cache;
proxy_cache_key "$scheme$host$request_uri|$http_origin";
proxy_cache_methods GET HEAD;
proxy_cache_valid 200 5s;
proxy_cache_lock on;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
add_header X-Cache-Status $upstream_cache_status always;
access_log off;
}
# Historical call lists: calls before a fixed time rarely change.
location ~ ^/[^/]+/calls/(older|date|[0-9]+/older)$ {
proxy_pass http://backend_api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
include /etc/nginx/proxy.conf;
proxy_cache api_cache;
proxy_cache_key "$scheme$host$request_uri|$http_origin";
proxy_cache_methods GET HEAD;
proxy_cache_valid 200 60s;
proxy_cache_lock on;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
add_header X-Cache-Status $upstream_cache_status always;
access_log off;
}
# System list for the homepage: listener counts can lag a few seconds.
location = /systems {
proxy_pass http://backend_api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
include /etc/nginx/proxy.conf;
proxy_cache api_cache;
proxy_cache_key "$scheme$host$request_uri|$http_origin";
proxy_cache_methods GET HEAD;
proxy_cache_valid 200 10s;
proxy_cache_lock on;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
add_header X-Cache-Status $upstream_cache_status always;
access_log off;
}
location / {
proxy_pass http://backend_api;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $backend_connection;
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
include /etc/nginx/proxy.conf;
access_log off;
}
}
server {
listen 443 ssl;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
server_name account.${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location / {
proxy_pass http://account:3009;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
include /etc/nginx/proxy.conf;
access_log off;
}
}
server {
listen 443 ssl;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
server_name admin.${DOMAIN_NAME};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location / {
proxy_pass http://admin:3008;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
include /etc/nginx/proxy.conf;
access_log off;
}
}