User-Agent: Googlebot
Allow: /

User-Agent: Googlebot-Mobile
Allow: /

User-Agent: msnbot
Allow: /

User-Agent: bingbot
Allow: /

# Adsense
User-Agent: Mediapartners-Google
Allow: / 

# Blekko
User-Agent: ScoutJet
Allow: / 

User-Agent: Yandex
Allow: / 

User-agent: baiduspider
Allow: / 

User-agent: DuckDuckBot
Allow: / 

# CommonCrawl
User-agent: ccbot
Allow: / 

User-agent: NerdyBot
Disallow: /

User-agent: ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com
Disallow: /

User-agent: A6-Indexer/1.0
Disallow: /

User-agent: WeSEE:Ads/PageBot
Disallow: /

user-agent: AhrefsBot
disallow: / 

user-agent: A6-Indexer/1.0 (http://www.a6corp.com/a6-web-scraping-policy/)
disallow: /