diff options
| -rw-r--r-- | public/robots.txt | 11 |
1 files changed, 11 insertions, 0 deletions
diff --git a/public/robots.txt b/public/robots.txt index a4ee103d..12aef5a6 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,13 +1,24 @@ # This says to apply these settings to ALL search engine spiders/crawlers # User-agent: * +# Version 1.0 User-agent: AhrefsBot User-agent: Adsbot-Google User-agent: Yandex User-agent: Googlebot +User-agent: Googlebot-mobile Disallow: /shop/product/pdf/* +Disallow: /shop/product/wishlist/* Disallow: /shop/product/https://* Disallow: /v1/* +Disallow: /shop/search?product_name=* +Disallow: /shop/search?q=* +Disallow: /shop/cart/ +Disallow: /shop/checkout/ +Disallow: /my/* +Disallow: /shop/variant/* +Disallow: /shop/search/* +Disallow: /promo/* User-agent: * Disallow: sentral.indoteknik.com/* |