User-agent: AppleNewsBot
Crawl-delay: 100
Allow: *

User-agent: Adsbot-Google
Crawl-delay: 100
Allow: *

User-agent: Googlebot
Crawl-delay: 100
Allow: *

User-agent: Twitterbot
Crawl-delay: 100
Disallow: 

User-agent: Mediapartners-Google
Crawl-delay: 100
Allow: *

User-agent: Googlebot-Image
Disallow: /

User-agent: Googlebot-Mobile
Crawl-delay: 100
Allow: *

User-agent: Googlebot-News
Crawl-delay: 100
Allow: *

User-agent: Googlebot-Video
Disallow: /

User-agent: bingbot
Crawl-delay: 100
Allow: *

User-agent: bing
Crawl-delay: 100
Allow: *

User-agent: msnbot
Disallow: /

User-agent: facebot
Disallow: /

User-agent: Yahoo-slurp
Disallow: /

User-agent: slurp
Disallow: /

User-agent: ia_archiver
Crawl-delay: 100
Allow: *

User-agent: yandex
Disallow: /

User-agent: *
Disallow: /