From efe37c39dcb3446f91d69fce33e3c62ba78cdfc3 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Mon, 25 May 2026 20:38:25 +0300 Subject: [PATCH] fix(nginx): add X-Robots-Tag noindex on preview hostnames Preview hostnames proxy SDK/Client/CLI paths straight to apify.github.io, so they serve production content even after the apify-docs preview S3 bucket is cleaned up. A `site:preview.docs.apify.com` search currently returns dozens of these URLs. Add a server-level `X-Robots-Tag` header so every response under a preview hostname -- including upstream proxies, error pages, and future routes -- tells search engines not to index, follow, or archive it. --- nginx.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nginx.conf b/nginx.conf index 1b8bb1b244..4d40e8cad5 100644 --- a/nginx.conf +++ b/nginx.conf @@ -810,6 +810,12 @@ server { resolver 172.20.0.10; server_name ~^(?[^.]+)\.preview\.docs\.apify\.com$; + # Block search indexing on every response from preview hostnames, including the + # apify.github.io proxy_passes (SDK/Client/CLI) that have no preview bucket and + # would otherwise serve production content under the preview hostname. + # `always` keeps the header on 4xx/5xx so error pages aren't indexed either. + add_header X-Robots-Tag "noindex, nofollow, noarchive" always; + # add trailing slashes to the root of GH pages docs rewrite ^/api/client/js$ /api/client/js/ redirect; rewrite ^/api/client/python$ /api/client/python/ redirect;