From 4d29c862b1a94c9925ec3dae17e0207ae000d46a Mon Sep 17 00:00:00 2001 From: DecFox <33030671+DecFox@users.noreply.github.com> Date: Thu, 20 Feb 2025 00:03:56 +0530 Subject: [PATCH 1/6] refactor: disable orchestrate and point to new ooniprobe ecs service --- .../ooni-backend/templates/nginx-api-fsn.conf | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf index 9d6e1451..3341ecb3 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -82,7 +82,7 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; - proxy_pass https://registry.ooni.io:443; + proxy_pass https://ooniprobe.prod.ooni.io:443; } # Selectively route test-list/urls to the API @@ -111,20 +111,20 @@ server { # Orchestrate # Should match: # - /api/v1/test-list - location ~^/api/v1/(test-list|urls) { - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - - proxy_pass https://orchestrate.ooni.io:443; - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - add_header X-Frame-Options DENY always; - add_header X-Content-Type-Options nosniff always; - - } + # location ~^/api/v1/(test-list|urls) { + # proxy_http_version 1.1; + # proxy_set_header Host $http_host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # proxy_read_timeout 900; + + # proxy_pass https://orchestrate.ooni.io:443; + # add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + # add_header X-Frame-Options DENY always; + # add_header X-Content-Type-Options nosniff always; + + # } # Web Connectivity Test Helper # Should match: From 35436f98fc95ab3c45813f6b6fe58f790a4e27d0 Mon Sep 17 00:00:00 2001 From: DecFox <33030671+DecFox@users.noreply.github.com> Date: Thu, 20 Feb 2025 17:59:25 +0530 Subject: [PATCH 2/6] refactor: sync nginx conf with current backend-fsn --- .../ooni-backend/templates/nginx-api-fsn.conf | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf index 3341ecb3..445372bd 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -69,6 +69,20 @@ server { resolver 127.0.0.1; + set $external_remote_addr $remote_addr; + if ($remote_addr = "188.166.93.143") { + # If remote_addr is ams-pg-test trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + if ($remote_addr = "142.93.237.101") { + # If remote_addr is ams-pg trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + if ($remote_addr ~ "^3.") { + # If remote_addr is AWS trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + # Registry # Should match: # - /api/v1/login @@ -86,7 +100,8 @@ server { } # Selectively route test-list/urls to the API - location ~^/api/v1/test-list/urls { + # location ~^/api/v1/test-list/urls { + location ~^/api/v1/(test-list|urls) { proxy_pass http://127.0.0.1:8000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -109,6 +124,7 @@ server { } # Orchestrate + # NOTE: we are now serving this from the ooniapi monolith # Should match: # - /api/v1/test-list # location ~^/api/v1/(test-list|urls) { @@ -209,15 +225,6 @@ server { proxy_pass https://0.th.ooni.org; } - set $external_remote_addr $remote_addr; - if ($remote_addr = "188.166.93.143") { - # If remote_addr is ams-pg-test trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } - if ($remote_addr = "142.93.237.101") { - # If remote_addr is ams-pg trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } proxy_set_header X-Real-IP $external_remote_addr; proxy_cache apicache; From 6915fcaff395fef93d8987167a0d3170f0121de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Mar 2025 13:56:03 +0100 Subject: [PATCH 3/6] Update nginx-api-fsn.conf Add rule for additional AWS prefix --- ansible/roles/ooni-backend/templates/nginx-api-fsn.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf index 445372bd..806ed825 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -82,6 +82,10 @@ server { # If remote_addr is AWS trust the X-Real-IP header set $external_remote_addr $http_x_real_ip; } + if ($remote_addr ~ "^18.") { + # If remote_addr is AWS trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } # Registry # Should match: From 3e5284e3d9ccab41e3b0650ef25f9fce03d9c8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Mar 2025 14:28:30 +0100 Subject: [PATCH 4/6] Update nginx-api-fsn.conf Fix x-real-ip extraction --- .../ooni-backend/templates/nginx-api-fsn.conf | 72 +++++++++++-------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf index 806ed825..b70053b3 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -5,6 +5,14 @@ proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M max_size=5g inactive=24h use_temp_path=off; +# Trust getting real-ip from X-Forwarded-For when the addresses are AWS IPs +# see: https://serverfault.com/a/414166 +set_real_ip_from 10.0.0.0/16; +set_real_ip_from 18.0.0.0/8; +set_real_ip_from 3.0.0.0/8; +real_ip_header X-Forwarded-For; +real_ip_recursive on; + # anonymize ipaddr map $remote_addr $remote_addr_anon { ~(?P\d+\.\d+\.\d+)\. $ip.0; @@ -19,11 +27,15 @@ map $http_x_forwarded_for $remote_fwd_anon { default 0.0.0.0; } - # log anonymized ipaddr and caching status -log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $upstream_cache_status [$time_local] ' +log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $remote_addr $upstream_cache_status [$time_local] ' '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; +log_format netdata '$remote_addr_anon - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '$request_length $request_time $upstream_response_time ' + '"$http_referer" "$http_user_agent"'; + server { # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests # over Tor correctly land to the proper backend. We are listening on this custom port @@ -34,6 +46,8 @@ server { listen [::]:443 ssl http2 default_server; server_name _; access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; + access_log /var/log/nginx/access.log netdata; + error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; gzip on; gzip_types text/plain application/xml application/json; @@ -42,9 +56,9 @@ server { # and everyhing else client_max_body_size 200M; # for measurement POST - ssl_certificate {{ certpath }}{{ inventory_hostname }}/fullchain.pem; - ssl_certificate_key {{ certpath }}{{ inventory_hostname }}/privkey.pem; - ssl_trusted_certificate {{ certpath }}{{ inventory_hostname }}/chain.pem; # for ssl_stapling_verify + ssl_certificate /var/lib/dehydrated/certs/backend-fsn.ooni.org/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/backend-fsn.ooni.org/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/backend-fsn.ooni.org/chain.pem; # for ssl_stapling_verify # Use the intermediate configuration to support legacy probes # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 @@ -70,22 +84,22 @@ server { resolver 127.0.0.1; set $external_remote_addr $remote_addr; - if ($remote_addr = "188.166.93.143") { - # If remote_addr is ams-pg-test trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } - if ($remote_addr = "142.93.237.101") { - # If remote_addr is ams-pg trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } - if ($remote_addr ~ "^3.") { - # If remote_addr is AWS trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } - if ($remote_addr ~ "^18.") { - # If remote_addr is AWS trust the X-Real-IP header - set $external_remote_addr $http_x_real_ip; - } + #if ($remote_addr = "188.166.93.143") { + # # If remote_addr is ams-pg-test trust the X-Real-IP header + # set $external_remote_addr $http_x_real_ip; + #} + #if ($remote_addr = "142.93.237.101") { + # # If remote_addr is ams-pg trust the X-Real-IP header + # set $external_remote_addr $http_x_real_ip; + #} + #if ($remote_addr ~ "^3.") { + # # If remote_addr is AWS trust the X-Real-IP header + # set $external_remote_addr $http_x_real_ip; + #} + #if ($remote_addr ~ "^18.") { + # # If remote_addr is AWS trust the X-Real-IP header + # set $external_remote_addr $http_x_real_ip; + #} # Registry # Should match: @@ -95,7 +109,7 @@ server { location ~^/api/v1/(login|register|update) { proxy_http_version 1.1; proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Real-IP $external_remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; @@ -108,7 +122,8 @@ server { location ~^/api/v1/(test-list|urls) { proxy_pass http://127.0.0.1:8000; proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Real-IP $external_remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_cache apicache; proxy_cache_min_uses 1; proxy_cache_lock on; @@ -134,7 +149,7 @@ server { # location ~^/api/v1/(test-list|urls) { # proxy_http_version 1.1; # proxy_set_header Host $http_host; - # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Real-IP $external_remote_addr; # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # proxy_set_header X-Forwarded-Proto $scheme; # proxy_read_timeout 900; @@ -159,19 +174,20 @@ server { location ~^/web-connectivity/(status) { proxy_http_version 1.1; proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Real-IP $external_remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; - proxy_pass https://wcth.ooni.io; + proxy_pass https://0.th.ooni.org; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; add_header X-Frame-Options DENY always; add_header X-Content-Type-Options nosniff always; + add_header Host "0.th.ooni.org"; } location /whoami { - return 200 "{{ inventory_hostname }}"; + return 200 "backend-fsn.ooni.org"; } location /metrics { @@ -189,7 +205,7 @@ server { rewrite ^/netdata/badge /api/v1/badge.svg break; proxy_pass http://127.0.0.1:19999; proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Real-IP $external_remote_addr; } # Expose package version badges From c221ca3ef973131aaa46c47a8cbd232f5d647f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Mar 2025 14:29:39 +0100 Subject: [PATCH 5/6] Update nginx-api-fsn.conf Drop remote_addr debug from logging --- ansible/roles/ooni-backend/templates/nginx-api-fsn.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf index b70053b3..7804ca75 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -28,7 +28,7 @@ map $http_x_forwarded_for $remote_fwd_anon { } # log anonymized ipaddr and caching status -log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $remote_addr $upstream_cache_status [$time_local] ' +log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $upstream_cache_status [$time_local] ' '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; log_format netdata '$remote_addr_anon - $remote_user [$time_local] ' From 58e6a4df78a60f523c24f87aa65cae65caa81b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 20 May 2025 16:28:26 +0200 Subject: [PATCH 6/6] Add incidents reverse proxy --- .../roles/ooni-backend/templates/nginx-api-fsn.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf index 7804ca75..e288edef 100644 --- a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -142,6 +142,18 @@ server { add_header X-Content-Type-Options nosniff always; } + # reverse proxy incidents + location ~^/api/v1/incidents { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $external_remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://oonifindings.prod.ooni.io:443; + } + # Orchestrate # NOTE: we are now serving this from the ooniapi monolith # Should match: