# robots.txt # Exclusionary file to prevent Web spiders and robots from # scanning selected directories of domain # Place in www directory; do not remove User-agent line User-agent: * Disallow: /cgi-bin/ Disallow: /formproc/ Disallow: /forums/ Disallow: /graphics/ Disallow: /itsmblog/ Disallow: /private/ Disallow: /test/ Disallow: /tmp/ Disallow: /uploads/ User-agent: URL_Spider_Pro Disallow: / User-agent: CherryPicker Disallow: /