User-agent: Adsbot-Google 
Disallow:                           # allow anything

User-Agent: Mediapartners-Google
Disallow:                           # allow anything

User-agent: Googlebot-Mobile
Disallow: /languages.xhtml          # block language switcher 
Disallow:                           # allow anything

User-Agent: taptubot                # touch phone mobi directory    
Disallow: /languages.xhtml
Disallow:                           # allow anything

User-agent: Exabot
Disallow: /languages.xhtml
Disallow:                           # allow anything

User-Agent: Yandex                  # Russian Search Engine
Disallow: /languages.xhtml
Disallow:                           # allow anything

User-agent: YahooSeeker/M1A1-R2D2   # Yahoo Mobile
Disallow: /languages.xhtml
Disallow: /                         # complete block
Crawl-delay: 10 

User-agent: MSNBOT-MOBILE           # Bing Mobile
Disallow: /languages.xhtml
Disallow:                           # allow anything
Crawl-delay: 10 

User-agent: Baiduspider             # chinese search engine
Disallow: /languages.xhtml
Disallow:                           # allow everything else

User-agent: Googlebot
Disallow: /languages.xhtml          # block language switcher 
Disallow:                           # allow anything

User-agent: msnbot                  # Bing
Disallow: /languages.xhtml          # block language switcher 
Disallow:                           # allow anything

User-agent: ia_archiver
Disallow: /                         # complete block
User-agent: BlitzBOT
Disallow: /                         # complete block
User-agent: psbot
Disallow: /                         # complete block
User-agent: TurnitinBot             # www.turnitin.com/robot/crawlerinfo.html
Disallow: /                         # complete block
User-agent: Twiceler                # www.twiceler.com
Disallow: /                         # complete block
User-agent: Charlotte               # http://www.searchme.com/support/pages/spider.php
Disallow: /                         # complete block
User-agent: heritrix                # http://www.metacarta.com
Disallow: /                         # complete block
User-agent: dotbot
Disallow: / 

User-agent: *
Disallow: /