User-agent: * # Prevent indexing of raw search refinements, price ranges, sorting rules # Disallow crawling of 3+ Refinements Disallow: /*prefn3=* Disallow: /*prefv3=* Disallow: /*prefn4=* Disallow: /*prefv4=* Disallow: /*prefn5=* Disallow: /*prefv5=* Disallow: /*prefn6=* Disallow: /*prefv6=* Disallow: /*prefn7=* Disallow: /*prefv7=* Disallow: /*pmin=* Disallow: /*pmax=* Disallow: *|* Disallow: *%7C* # Prevent indexing of checkout and cart sessions Disallow: /cart* Disallow: /checkout* # Prevent indexing of un-aliased storefront URLs Disallow: */demandware.store* #Copy and uncomment to allow access to certain resources #Allow: */demandware.store/*/Pipeline-Name Allow: */demandware.store/*/Search-UpdateGrid Allow: / # Taxonomy Sitemaps Sitemap: https://www.ross-simons.com/sitemap_index.xml #Crawl delays for common bots User-agent: * Crawl-delay: 2 User-agent: Pinterestbot Crawl-delay: 0 User-agent: FacebookBot Crawl-delay: 0 User-agent: facebookexternalhit/1.1 Crawl-delay: 0 User-agent: facebookexternalhit/1.0 Crawl-delay: 0