diff --git a/res/robots.txt b/res/robots.txt new file mode 100644 index 0000000000..9016964190 --- /dev/null +++ b/res/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Disallow: /public/ +Disallow: /from-url/ +Disallow: /compare/ diff --git a/webpack.config.js b/webpack.config.js index f2c03dde94..da6c8c0037 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -99,6 +99,7 @@ const config = { 'res/_redirects', 'res/zee-worker.js', 'res/contribute.json', + 'res/robots.txt', 'res/service-worker-compat.js', { from: 'docs-user', to: 'docs' }, { from: 'locales', to: 'locales' },