diff options
Diffstat (limited to 'public/robots.txt')
| -rw-r--r-- | public/robots.txt | 19 |
1 files changed, 19 insertions, 0 deletions
diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 00000000..a4ee103d --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,19 @@ +# This says to apply these settings to ALL search engine spiders/crawlers +# User-agent: * + +User-agent: AhrefsBot +User-agent: Adsbot-Google +User-agent: Yandex +User-agent: Googlebot +Disallow: /shop/product/pdf/* +Disallow: /shop/product/https://* +Disallow: /v1/* + +User-agent: * +Disallow: sentral.indoteknik.com/* +Disallow: erp.indoteknik.com/* + +Sitemap: https://indoteknik.com/sitemap/products.xml +Sitemap: https://indoteknik.com/sitemap/brands.xml +Sitemap: https://indoteknik.com/sitemap/categories.xml +Sitemap: https://indoteknik.com/sitemap/videos.xml
\ No newline at end of file |