# Crawlers Setup
User-agent: *

# Allow specific paths so Googlebot can crawl them and read the
# X-Robots-Tag: noindex header set in .htaccess. Required to remove
# these URLs from the "Indexed, though blocked by robots.txt" report
# in Google Search Console. Allow rules MUST come before the broader
# Disallow rules below for both Googlebot and Bing to honour them.
Allow: /directory/currency/switch/
Allow: /catalog/seo_sitemap/
Allow: /customer/account/

# Directories
Disallow: /checkout/
Disallow: /catalogsearch/
Disallow: /app/
Disallow: /downloader/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /catalog/product_compare/
Disallow: /catalog/
Disallow: /catalog/product/view/id/
Disallow: /customer/
Disallow: /wishlist/
Disallow: /404/
Disallow: /admin/
Disallow: /adminlogin/
Disallow: /tigerdragon/
Disallow: /api/
Disallow: /install/
Disallow: /oldsite/
Disallow: /directory/
Disallow: /captcha/
Disallow: /tag/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /poll/
Disallow: /newsletter/

# URL parameters (layered nav, sort, limit, pagination).
# NOTE: dir/order/price/cat/manufacturer/limit/mode/p are intentionally
# NOT disallowed. Faceted-nav / sort / limit / paginated views must be
# crawlable so Googlebot can read the X-Robots-Tag: noindex, follow header
# served for these param URLs (see .htaccess). Blocking them in robots.txt
# is exactly what put them in the "Indexed, though blocked by robots.txt"
# GSC report — a disallowed URL can never be fetched to see the noindex,
# so it never leaves the index. Unblock + noindex,follow is the only way
# to deindex them while keeping product links crawlable.
#
# Session / store-switch params (SID, ___store, ___from_store) are also
# NOT disallowed, for the same reason: Google had already indexed some
# ?SID=/?___store= URLs, and blocking them kept those stuck in the GSC
# "Indexed, though blocked by robots.txt" report. They now serve
# X-Robots-Tag: noindex, follow via .htaccess (or resolve as a
# redirect/404), which deindexes them once crawled.

# Files
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /meta-certified-media-buying-professional-400-101-exam-prep-640.html
Disallow: /meta-certified*

# Sitemap
Sitemap: https://www.tertiarycourses.com.my/sitemap.xml