From c89f620717483331534c4c8e7b1c9208799c2c11 Mon Sep 17 00:00:00 2001 From: Rafi Zadanly Date: Tue, 30 May 2023 16:10:58 +0700 Subject: Update robots.txt --- public/robots.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/public/robots.txt b/public/robots.txt index a4ee103d..12aef5a6 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,13 +1,24 @@ # This says to apply these settings to ALL search engine spiders/crawlers # User-agent: * +# Version 1.0 User-agent: AhrefsBot User-agent: Adsbot-Google User-agent: Yandex User-agent: Googlebot +User-agent: Googlebot-mobile Disallow: /shop/product/pdf/* +Disallow: /shop/product/wishlist/* Disallow: /shop/product/https://* Disallow: /v1/* +Disallow: /shop/search?product_name=* +Disallow: /shop/search?q=* +Disallow: /shop/cart/ +Disallow: /shop/checkout/ +Disallow: /my/* +Disallow: /shop/variant/* +Disallow: /shop/search/* +Disallow: /promo/* User-agent: * Disallow: sentral.indoteknik.com/* -- cgit v1.2.3