# For all robots
User-agent: *

# Block access to specific groups of pages
Disallow: /customers/basket
Disallow: /customers/checkout
Disallow: /trade/login
Disallow: /customers/login
Disallow: /*.php$
Disallow: *?keywords=*
Disallow: *&title=desc
Disallow: *&title=asc
Disallow: *&price=asc
Disallow: *&price=desc
Disallow: /category/
Disallow: /category/view/
Disallow: /product/view/
Disallow: /customers/basket
Disallow: /customers/checkout
Disallow: /customers/login
Disallow: /trade/login
Disallow: /checkout
Disallow: /cart1
Disallow: /user-forgot-password
Disallow: /customer-my-account
Disallow: /error
Disallow: /trade/refresh-mini-cart
Disallow: /trade/remove-from-cart
Disallow: /repusers/
Disallow: /ajax_
Disallow: /cms-custome-module/
Disallow: /our-producers/country/region/producer/product


# ignore search queries 

Disallow: /wines?keywords=

# Allow search crawlers to discover the sitemap
Sitemap: https://www.eurowines.co.uk/sitemap.xml

# Disallowed bots (aggressive or irrelevant crawlers)
User-agent: AhrefsBot
Disallow: /

# Semrush
User-agent: SiteAuditBot
User-agent: SplitSignalBot
Disallow: /

User-agent: CazoodleBot
User-agent: dotbot/1.0
User-agent: Gigabot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: AwarioRssBot
User-agent: AwarioSmartBot
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: YandexBot
User-agent: YandexImages
User-agent: YandexUserproxy
User-agent: YandexRenderResourcesBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Qwantbot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: MojeekBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ArchiveBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: coccocbot-image
Disallow: /

User-agent: MJ12bot
Disallow: /

# Limit rate and time for visit
# maximum rate is one page every 5 seconds
Request-rate: 1/10

# 5 seconds between page requests
Crawl-delay: 10

# only visit between 02:00 and 07:00 UTC
Visit-time: 0200-0700