## robots.txt -- for more info see: ## http://info.webcrawler.com/mak/projects/robots/norobots.html # # # This is for some buggy spiders that don't seem to recognize case ... though, # if they aren't smart enough to recognize case, will they be smart enough to # parse a robots.txt file ... User-agent: * Disallow: /servlet/articlenews/ Disallow: /servlet/page/ Disallow: /servlet/htmltemplate/ Sitemap: http://www.theglobeandmail.com/sitemap.xml