User-agent: *
# Disallow 1.0 because we want the equivalent /latest indexed instead
Disallow: /1.0
# Disallow known dev sites
Disallow: /1.0-dev
# Disallow translations that exist, but aren't actually translated yet
# (this ensures that search engines won't just see them as duplicate content for now)
# (Note that not all spiders support wildcards here).
Disallow: /*/es

# Different config for Googlebot, as that supports Allows (so we can restrict more)
User-agent: Googlebot
Allow: /latest$
Allow: /latest/$
Allow: /latest/en
Allow: /validator
Allow: /$
Disallow: /

User-agent: *
# Disallow 1.0 because we want the equivalent /latest indexed instead
Disallow: /1.0
# Disallow known dev sites
Disallow: /1.0-dev
# Disallow translations that exist, but aren't actually translated yet
# (this ensures that search engines won't just see them as duplicate content for now)
# (Note that not all spiders support wildcards here).
Disallow: /*/es

# Different config for Googlebot, as that supports Allows (so we can restrict more)
User-agent: Googlebot
Allow: /latest$
Allow: /latest/$
Allow: /latest/en
Allow: /validator
Allow: /$
Disallow: /

User-Agent: LinkChecker
Allow: /

Allow linkchecker to crawl all of standard.open-contracting.org
Allow linkchecker to crawl all of standard.open-contracting.org
