# Matching is done after the domain "/" to the last character of the string
# Partial matches within a path may require a "*" to achieve the match
# SEs need access to CSS and JS in order to render content correctly
# Sitemap
Sitemap: https://www.giftsandmerchandise.co.uk/sitemap_index.xml

# Disallow directories and sensitive paths

User-agent: *

Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /cgi-sys/
Disallow: /wp-content/uploads/vj-wp-import-export/
Disallow: /comments/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /*/feed
Disallow: /wp-json/
Disallow: /uncategorized-
Disallow: /search
Disallow: /wp-content/uploads/fibofilters/
Disallow: /sitemap.xml
Disallow: /wp-*.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-comments-post.php
Disallow: /*/xmlrpc.php

# Disallow specific query strings

Disallow: /*/trackback
Disallow: /*?blackhole
Disallow: /*blackhole
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*?remove_item=
Disallow: /*remove_item=
Disallow: /*?removed_item=
Disallow: /*removed_item=
Disallow: /*?undo_item=
Disallow: /*undo_item=
Disallow: /*?attribute*=
Disallow: /*attribute*=
Disallow: /*?attribute_pa_=
Disallow: /*attribute_pa_=
Disallow: /*?taxonomy=
Disallow: /*taxonomy=
Disallow: /*?currency=
Disallow: /*currency=
Disallow: /*?orderby=
Disallow: /*orderby=
Disallow: /*?s=
Disallow: /*s=
Disallow: /*?filter_
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?rating=
Disallow: /*?post_type=
Disallow: /*?action=

# Allow access to resources required for rendering

Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/plugins/
Allow: /wp-content/themes/
Allow: /wp-includes/js/
Allow: /wp-includes/css/

# Allow access to root

Allow: /

# ALLOWS HERE ARE JUST FOR REFERENCE AS WILCARD ABOVE

# Main Search Engines
User-agent: Googlebot
User-agent: Googlebot-Mobile
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: DuckDuckBot
Allow: /

# Social Media Crawlers
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterestbot
User-agent: Discordbot
User-agent: WhatsApp
User-agent: SkypeUriPreview
User-agent: TelegramBot
User-agent: BingPreview
Allow: /

# SEO and backlink crawlers (optional, beneficial for exposure)
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
Allow: /

# Allow AI agents for search/discovery (not training)
User-agent: OAI-SearchBot
User-agent: ChatGPT-User  
User-agent: PerplexityBot
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
User-agent: Applebot
User-agent: PetalBot
Allow: /