# This content comes from the Main/RobotsTxt topic

# This file needs to conform to a normal robots.txt file, dont use any other formatting
# comments need to be done like this, ie with a # (hash). A CGI then cats this our as /robots.txt

User-agent: *
Crawl-delay: 5
Disallow: /SIE
Disallow: /view/SIE
Disallow: /rdiff
Disallow: /oops
Disallow: /edit
Disallow: /search
Disallow: /view/TWiki/TWikiRegistration