From 5b177eca58f23797986fc0967bbde770fc325594 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Thu, 19 Nov 2026 22:42:11 -0500 Subject: [PATCH] Raise nginx connection limits and microcache /calls polling A traffic surge exhausted nginx's 3 x 4096 worker_connections (each proxied websocket uses two), so Cloudflare got empty responses (520) and the backend's listen queue filled. - worker_processes auto (8 cores), worker_connections 16384, worker_rlimit_nofile 65535. - Cache public /calls endpoints on api.*: 5s for live lists, 60s for historical ones, with cache lock and stale-while-updating so bursts collapse to one backend query. Origin is in the cache key because the backend's CORS header varies by Origin. Co-Authored-By: Claude Opus 5.5 --- nginx-proxy/nginx.conf | 5 ++-- nginx-proxy/prod/vhosts/site.template | 43 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/nginx-proxy/nginx.conf b/nginx-proxy/nginx.conf index aa88782..17a2fb6 100644 --- a/nginx-proxy/nginx.conf +++ b/nginx-proxy/nginx.conf @@ -1,10 +1,11 @@ user www-data; -worker_processes 3; ## default is 1 +worker_processes auto; +worker_rlimit_nofile 65535; pid /var/run/nginx.pid; events { - worker_connections 4096; ## Default: 1024 + worker_connections 16384; } diff --git a/nginx-proxy/prod/vhosts/site.template b/nginx-proxy/prod/vhosts/site.template index b72c9c4..5cdcb1a 100644 --- a/nginx-proxy/prod/vhosts/site.template +++ b/nginx-proxy/prod/vhosts/site.template @@ -7,6 +7,11 @@ map $http_upgrade $backend_connection { '' ''; } +# Microcache for the public /calls polling endpoints. Responses carry no +# per-user data; the CORS header varies by Origin, so Origin is in the key. +proxy_cache_path /var/cache/nginx/api levels=1:2 keys_zone=api_cache:20m + max_size=500m inactive=10m use_temp_path=off; + upstream backend_api { server backend:3005; keepalive 64; @@ -99,6 +104,44 @@ server { } + # Live call lists: short TTL; the socket pushes anything newer. + location ~ ^/[^/]+/calls(/(latest|newer|next))?$ { + proxy_pass http://backend_api; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + include /etc/nginx/proxy.conf; + proxy_cache api_cache; + proxy_cache_key "$scheme$host$request_uri|$http_origin"; + proxy_cache_methods GET HEAD; + proxy_cache_valid 200 5s; + proxy_cache_lock on; + proxy_cache_lock_timeout 5s; + proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504; + proxy_cache_background_update on; + add_header X-Cache-Status $upstream_cache_status always; + access_log off; + } + + # Historical call lists: calls before a fixed time rarely change. + location ~ ^/[^/]+/calls/(older|date|[0-9]+/older)$ { + proxy_pass http://backend_api; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + include /etc/nginx/proxy.conf; + proxy_cache api_cache; + proxy_cache_key "$scheme$host$request_uri|$http_origin"; + proxy_cache_methods GET HEAD; + proxy_cache_valid 200 60s; + proxy_cache_lock on; + proxy_cache_lock_timeout 5s; + proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504; + proxy_cache_background_update on; + add_header X-Cache-Status $upstream_cache_status always; + access_log off; + } + location / { proxy_pass http://backend_api; proxy_http_version 1.1;