From 344fe1857b8307933082110c15b737af9118ac89 Mon Sep 17 00:00:00 2001 From: Rafi Zadanly Date: Fri, 12 May 2023 17:01:37 +0700 Subject: add robots.txt --- public/robots.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 public/robots.txt (limited to 'public/robots.txt') diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 00000000..62114b1f --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,12 @@ +# This says to apply these settings to ALL search engine spiders/crawlers +# User-agent: * +User-agent: Googlebot +Disalow: /shop/product/pdf/* +Disalow: /shop/product/https://* +Disallow: /v1/* +Disallow: /sentral.indoteknik.com* + +Sitemap: https://indoteknik.com/sitemap/products.xml +Sitemap: https://indoteknik.com/sitemap/brands.xml +Sitemap: https://indoteknik.com/sitemap/categories.xml +Sitemap: https://indoteknik.com/sitemap/videos.xml \ No newline at end of file -- cgit v1.2.3 From 711993092eb75aacc9766f0da1c264e5dde55218 Mon Sep 17 00:00:00 2001 From: Rafi Zadanly Date: Tue, 16 May 2023 11:22:00 +0700 Subject: add robots.txt --- public/robots.txt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'public/robots.txt') diff --git a/public/robots.txt b/public/robots.txt index 62114b1f..a4ee103d 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,10 +1,17 @@ # This says to apply these settings to ALL search engine spiders/crawlers # User-agent: * + +User-agent: AhrefsBot +User-agent: Adsbot-Google +User-agent: Yandex User-agent: Googlebot -Disalow: /shop/product/pdf/* -Disalow: /shop/product/https://* +Disallow: /shop/product/pdf/* +Disallow: /shop/product/https://* Disallow: /v1/* -Disallow: /sentral.indoteknik.com* + +User-agent: * +Disallow: sentral.indoteknik.com/* +Disallow: erp.indoteknik.com/* Sitemap: https://indoteknik.com/sitemap/products.xml Sitemap: https://indoteknik.com/sitemap/brands.xml -- cgit v1.2.3