# see http://www.robotstxt.org/orig.html for documentation User-agent: * # disable filters in categories and subcategories Disallow: /c/*%2B* Disallow: /c/*/asizes-* Disallow: /c/*/as-* Disallow: /c/*/asize-* Disallow: /c/*/g_price- Disallow: /c/*/weight-* Disallow: /c/*/sustainability-* Disallow: /c/*/rada-* Disallow: /c/*/sport-* Disallow: /c/*/warehouse-* Disallow: /page/event/* Disallow: /category-v2/* # disable dynamic content and filters Disallow: /c/*? Disallow: /s/*? Disallow: /pg/*? Disallow: /*?*lb.f* # special content Disallow: /basket/* Disallow: /order/* Disallow: /user/* Disallow: /category/* # ajax loading Disallow: /product/gear-up* Disallow: /product/similar-products* Disallow: /*?q=* Disallow: /load-reviews* #User-agent: Googlebot #Disallow: User-agent: Googlebot-image Disallow: # we don't use and causing non-trivial traffic User-agent: Amazonbot Disallow: / User-agent: BLEXBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: trendkite-akashic-crawler Disallow: / User-agent: MJ12bot Disallow: / User-agent: AwarioRssBot User-agent: Timpibot User-agent: RainBot User-agent: AwarioSmartBot Disallow: / User-agent: SERankingBacklinksBot Disallow: / Sitemap: https://top4running.es/export/sitemap_index.xml