User-agent: * # directed to all spiders Allow: /search.shtml Allow: /crosslinks.shtml Disallow: /cgi-bin/ Disallow: /images/