#stop spamming us please
User-agent: msnbot
Disallow: /

#stop spamming us please
User-agent: Slurp
Disallow: /

#stop spamming us please
User-agent: TurnitinBot
Disallow: /

#stop spamming us please
User-Agent: panscient.com
Disallow: /

#Robots are welcome to crawl the html parts of the site, but not the various ebook formats
User-agent: *
Disallow: /pdf/
Disallow: /facsimile/
Disallow: /msreader/
Disallow: /oeb/
Disallow: /shop/
Disallow: /solr/
Disallow: /lucene/
Disallow: /daisy/
Disallow: /html-package/
Disallow: /tm/basic/
Disallow: /tm/scholarly/search/
Crawl-delay: 60
#smart robots only crawl the content
Sitemap: http://www.nzetc.org/sitemapindex.xml