#Most entries taken from here: http://twitter.com/robots.txt

#Google Search Engine Robot
User-agent: Googlebot
Allow: /?_escaped_fragment_
Disallow: /images/*
Disallow: /simple/*
Disallow: /auto/*
Disallow: /flv/*
Disallow: /codes/*

User-agent: Applebot
Allow: /?_escaped_fragment_
Disallow: /images/*
Disallow: /simple/*
Disallow: /auto/*
Disallow: /flv/*
Disallow: /codes/*

#Yahoo! Search Engine Robot
User-Agent: Slurp
Allow: /?_escaped_fragment_
Disallow: /images/*
Disallow: /simple/*
Disallow: /auto/*
Disallow: /flv/*
Disallow: /codes/*

#Yandex Search Engine Robot
User-agent: Yandex
Allow: /?_escaped_fragment_
Disallow: /images/*
Disallow: /simple/*
Disallow: /auto/*
Disallow: /flv/*
Disallow: /codes/*

#Microsoft Search Engine Robot
User-Agent: msnbot
Allow: /?_escaped_fragment_
Disallow: /images/*
Disallow: /simple/*
Disallow: /auto/*
Disallow: /flv/*
Disallow: /codes/*

User-agent: Cliqzbot
Disallow: /  

# Every bot that might possibly read and respect this file.
User-agent: *
Disallow: /images/*
Disallow: /simple/*
Disallow: /auto/*
Disallow: /flv/*
Disallow: /codes/*
Disallow: /search

# Wait 1 second between successive requests. See ONBOARD-2698 for details.
Crawl-delay: 1

# Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
# the protocol of the sitemap.
Sitemap: http://slack-time.com/sitemap.xml

Sitemap: http://slack-time.com/rss/newvideos.xml