# # Agent-specific settings for indexing the site. # # Allow Google for full text crawling User-agent: GoogleBot # Google ignores Crawl-delay # Crawl-delay: 2 # URL prefixes # URL wildcards Disallow: /admin/ Disallow: /alerts/ Disallow: /cache/ Disallow: /carnet/ Disallow: /cart Disallow: /cart/ Disallow: /cartregistration/ Disallow: /cc/ Disallow: /commenting/ Disallow: /counterstats/ Disallow: /deliver/ Disallow: /editcontent/ Disallow: /email Disallow: /email/ Disallow: /error/ Disallow: /export/ Disallow: /librarians/uploadbranding Disallow: /licence-admin/ Disallow: /managesavedsearches Disallow: /markedlist/ Disallow: /mostviewed/ Disallow: /offer/ Disallow: /openurl/profile Disallow: /orders Disallow: /orders/ Disallow: /pricing/ Disallow: /profile Disallow: /profile/ Disallow: /pubstats/ Disallow: /recommendto Disallow: /reference/ Disallow: /regactivation Disallow: /regactivation/ Disallow: /registration Disallow: /registration/ Disallow: /resourcelookup Disallow: /runsavedsearch Disallow: /savedsearches Disallow: /search Disallow: /search/ Disallow: /selfservice/ Disallow: /session Disallow: /session/ Disallow: /statslogredirect Disallow: /statslogwrapper Disallow: /subscribedtitles Disallow: /uploadfile Disallow: /user/ Disallow: /workflow/ Disallow: /content/contributor Disallow: /content/contributor/ Disallow: /content/institution Disallow: /content/institution/ # # Yandex have two bots so they can work twice as slowly # User-agent: YandexBot Crawl-delay: 4 Disallow: /admin/ Disallow: /alerts/ Disallow: /cache/ Disallow: /carnet/ Disallow: /cart Disallow: /cart/ Disallow: /cartregistration/ Disallow: /cc/ Disallow: /commenting/ Disallow: /counterstats/ Disallow: /deliver/ Disallow: /editcontent/ Disallow: /email Disallow: /email/ Disallow: /error/ Disallow: /export/ Disallow: /librarians/uploadbranding Disallow: /licence-admin/ Disallow: /managesavedsearches Disallow: /markedlist/ Disallow: /mostviewed/ Disallow: /offer/ Disallow: /openurl/profile Disallow: /orders Disallow: /orders/ Disallow: /pricing/ Disallow: /profile Disallow: /profile/ Disallow: /pubstats/ Disallow: /recommendto Disallow: /reference/ Disallow: /regactivation Disallow: /regactivation/ Disallow: /registration Disallow: /registration/ Disallow: /resourcelookup Disallow: /runsavedsearch Disallow: /savedsearches Disallow: /search Disallow: /search/ Disallow: /selfservice/ Disallow: /session Disallow: /session/ Disallow: /statslogredirect Disallow: /statslogwrapper Disallow: /subscribedtitles Disallow: /uploadfile Disallow: /user/ Disallow: /workflow/ Disallow: /content/contributor Disallow: /content/contributor/ Disallow: /content/institution Disallow: /content/institution/ # # Yandex have two bots so they can work twice as slowly # User-agent: YandexImages Crawl-delay: 4 Disallow: /admin/ Disallow: /alerts/ Disallow: /cache/ Disallow: /carnet/ Disallow: /cart Disallow: /cart/ Disallow: /cartregistration/ Disallow: /cc/ Disallow: /commenting/ Disallow: /counterstats/ Disallow: /deliver/ Disallow: /editcontent/ Disallow: /email Disallow: /email/ Disallow: /error/ Disallow: /export/ Disallow: /librarians/uploadbranding Disallow: /licence-admin/ Disallow: /managesavedsearches Disallow: /markedlist/ Disallow: /mostviewed/ Disallow: /offer/ Disallow: /openurl/profile Disallow: /orders Disallow: /orders/ Disallow: /pricing/ Disallow: /profile Disallow: /profile/ Disallow: /pubstats/ Disallow: /recommendto Disallow: /reference/ Disallow: /regactivation Disallow: /regactivation/ Disallow: /registration Disallow: /registration/ Disallow: /resourcelookup Disallow: /runsavedsearch Disallow: /savedsearches Disallow: /search Disallow: /search/ Disallow: /selfservice/ Disallow: /session Disallow: /session/ Disallow: /statslogredirect Disallow: /statslogwrapper Disallow: /subscribedtitles Disallow: /uploadfile Disallow: /user/ Disallow: /workflow/ Disallow: /content/contributor Disallow: /content/contributor/ Disallow: /content/institution Disallow: /content/institution/ # # Bing specific configuration # User-agent: bingbot Crawl-delay: 2 Disallow: /admin/ Disallow: /alerts/ Disallow: /cache/ Disallow: /carnet/ Disallow: /cart Disallow: /cart/ Disallow: /cartregistration/ Disallow: /cc/ Disallow: /commenting/ Disallow: /counterstats/ Disallow: /deliver/ Disallow: /editcontent/ Disallow: /email Disallow: /email/ Disallow: /error/ Disallow: /export/ Disallow: /librarians/uploadbranding Disallow: /licence-admin/ Disallow: /managesavedsearches Disallow: /markedlist/ Disallow: /mostviewed/ Disallow: /offer/ Disallow: /openurl/profile Disallow: /orders Disallow: /orders/ Disallow: /pricing/ Disallow: /profile Disallow: /profile/ Disallow: /pubstats/ Disallow: /recommendto Disallow: /reference/ Disallow: /regactivation Disallow: /regactivation/ Disallow: /registration Disallow: /registration/ Disallow: /resourcelookup Disallow: /runsavedsearch Disallow: /savedsearches Disallow: /search Disallow: /search/ Disallow: /selfservice/ Disallow: /session Disallow: /session/ Disallow: /statslogredirect Disallow: /statslogwrapper Disallow: /subscribedtitles Disallow: /uploadfile Disallow: /user/ Disallow: /workflow/ Disallow: /content/contributor Disallow: /content/contributor/ Disallow: /content/institution Disallow: /content/institution/ # # Baidu specific configuration # User-agent: Baiduspider Crawl-delay: 2 Disallow: /admin/ Disallow: /alerts/ Disallow: /cache/ Disallow: /carnet/ Disallow: /cart Disallow: /cart/ Disallow: /cartregistration/ Disallow: /cc/ Disallow: /commenting/ Disallow: /counterstats/ Disallow: /deliver/ Disallow: /editcontent/ Disallow: /email Disallow: /email/ Disallow: /error/ Disallow: /export/ Disallow: /librarians/uploadbranding Disallow: /licence-admin/ Disallow: /managesavedsearches Disallow: /markedlist/ Disallow: /mostviewed/ Disallow: /offer/ Disallow: /openurl/profile Disallow: /orders Disallow: /orders/ Disallow: /pricing/ Disallow: /profile Disallow: /profile/ Disallow: /pubstats/ Disallow: /recommendto Disallow: /reference/ Disallow: /regactivation Disallow: /regactivation/ Disallow: /registration Disallow: /registration/ Disallow: /resourcelookup Disallow: /runsavedsearch Disallow: /savedsearches Disallow: /search Disallow: /search/ Disallow: /selfservice/ Disallow: /session Disallow: /session/ Disallow: /statslogredirect Disallow: /statslogwrapper Disallow: /subscribedtitles Disallow: /uploadfile Disallow: /user/ Disallow: /workflow/ Disallow: /content/contributor Disallow: /content/contributor/ Disallow: /content/institution Disallow: /content/institution/ User-agent: facebookexternalhit User-agent: LinkedInBot User-agent: Twitterbot Allow: / User-agent: GPTBot Disallow: / # # Generic settings for indexing the site. # User-agent: * Crawl-delay: 2 Disallow: /admin/ Disallow: /alerts/ Disallow: /cache/ Disallow: /carnet/ Disallow: /cart Disallow: /cart/ Disallow: /cartregistration/ Disallow: /cc/ Disallow: /commenting/ Disallow: /counterstats/ Disallow: /deliver/ Disallow: /editcontent/ Disallow: /email Disallow: /email/ Disallow: /error/ Disallow: /export/ Disallow: /librarians/uploadbranding Disallow: /licence-admin/ Disallow: /managesavedsearches Disallow: /markedlist/ Disallow: /mostviewed/ Disallow: /offer/ Disallow: /openurl/profile Disallow: /orders Disallow: /orders/ Disallow: /pricing/ Disallow: /profile Disallow: /profile/ Disallow: /pubstats/ Disallow: /recommendto Disallow: /reference/ Disallow: /regactivation Disallow: /regactivation/ Disallow: /registration Disallow: /registration/ Disallow: /resourcelookup Disallow: /runsavedsearch Disallow: /savedsearches Disallow: /search Disallow: /search/ Disallow: /selfservice/ Disallow: /session Disallow: /session/ Disallow: /statslogredirect Disallow: /statslogwrapper Disallow: /subscribedtitles Disallow: /uploadfile Disallow: /user/ Disallow: /workflow/ Disallow: /content/contributor Disallow: /content/contributor/ Disallow: /content/institution Disallow: /content/institution/