From c89f620717483331534c4c8e7b1c9208799c2c11 Mon Sep 17 00:00:00 2001 From: Rafi Zadanly Date: Tue, 30 May 2023 16:10:58 +0700 Subject: Update robots.txt --- public/robots.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'public/robots.txt') diff --git a/public/robots.txt b/public/robots.txt index a4ee103d..12aef5a6 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,13 +1,24 @@ # This says to apply these settings to ALL search engine spiders/crawlers # User-agent: * +# Version 1.0 User-agent: AhrefsBot User-agent: Adsbot-Google User-agent: Yandex User-agent: Googlebot +User-agent: Googlebot-mobile Disallow: /shop/product/pdf/* +Disallow: /shop/product/wishlist/* Disallow: /shop/product/https://* Disallow: /v1/* +Disallow: /shop/search?product_name=* +Disallow: /shop/search?q=* +Disallow: /shop/cart/ +Disallow: /shop/checkout/ +Disallow: /my/* +Disallow: /shop/variant/* +Disallow: /shop/search/* +Disallow: /promo/* User-agent: * Disallow: sentral.indoteknik.com/* -- cgit v1.2.3 From 81d3bd413bdf56963d032afbcb04bfd3ce3b9d76 Mon Sep 17 00:00:00 2001 From: stephanchrst Date: Mon, 19 Jun 2023 15:42:51 +0700 Subject: revise robots.txt requested by renca --- public/robots.txt | 1 - 1 file changed, 1 deletion(-) (limited to 'public/robots.txt') diff --git a/public/robots.txt b/public/robots.txt index 12aef5a6..418e72f1 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -3,7 +3,6 @@ # Version 1.0 User-agent: AhrefsBot -User-agent: Adsbot-Google User-agent: Yandex User-agent: Googlebot User-agent: Googlebot-mobile -- cgit v1.2.3