# Sitemap
Sitemap: http://www.thesupermarketonline.com.au/sitemap

# No need to index this stuff
User-agent: *
Disallow: /connect/
Disallow: /html/
Disallow: /posting
Disallow: /scripts/
Disallow: /remote
Disallow: /site-remote
Disallow: /*?fb_xd_fragment
Disallow: /*?qty=*
Disallow: /*?mobile=*
Disallow: /orders
Disallow: /invoice
# Avoid duplicated content
Disallow: /*?f=*
Disallow: /*?m=*

# Disallow known useless bots and crawlers
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: rogerbot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /