# Disallow aggressive crawling of mj12bot
User-agent: MJ12bot
Disallow: /

# Disallow search engines from crawling solr search result pages with lots of different paramters (for some reason googlebot crawled a lot of those and thus hugely inflating the tx_realurl_urlencodecache)
User-agent: *
Disallow: /*?q=*

User-agent: *
Disallow: /*&q=*

User-agent: *
Disallow: /*tx_solr*

# This pagetype is used to load the sidebar content in the forum
# Google for some reason started to index these XHR requests, so exclude them here
User-agent: *
Disallow: /*type=1478165154*

# #24870
# This path is used to load forum comments via ajax and should not be indexed
User-agent: *
Disallow: */commentsonly/*

# Default: Allow all robots.
User-agent: *
Allow: /