From eaa3ff03427f7a488687e8f7bc99b54b953698b0 Mon Sep 17 00:00:00 2001 From: Julien Wajsberg Date: Tue, 19 Sep 2023 15:29:38 +0200 Subject: [PATCH] Add a robots.txt file to disallow indexing our profile links --- res/robots.txt | 4 ++++ webpack.config.js | 1 + 2 files changed, 5 insertions(+) create mode 100644 res/robots.txt diff --git a/res/robots.txt b/res/robots.txt new file mode 100644 index 0000000000..9016964190 --- /dev/null +++ b/res/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Disallow: /public/ +Disallow: /from-url/ +Disallow: /compare/ diff --git a/webpack.config.js b/webpack.config.js index f2c03dde94..da6c8c0037 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -99,6 +99,7 @@ const config = { 'res/_redirects', 'res/zee-worker.js', 'res/contribute.json', + 'res/robots.txt', 'res/service-worker-compat.js', { from: 'docs-user', to: 'docs' }, { from: 'locales', to: 'locales' },