user-agent: *
{% if disallow_all -%}
disallow: /
{% else -%}
disallow: /*/download/
disallow: /*/firstrun/
disallow: /*/newsletter/existing/
disallow: /*/whatsnew/
disallow: /*/firefox/os/1.1/
disallow: /*/firefox/os/1.3/
disallow: /*/firefox/os/1.3t/
disallow: /*/firefox/os/1.4/
disallow: /b/
{% endif %}

user-agent: LinkChecker
disallow:
crawl-delay: 1
user-agent: *
{% if disallow_all -%}
disallow: /
{% else -%}
disallow: /*/download/
disallow: /*/firstrun/
disallow: /*/newsletter/existing/
disallow: /*/whatsnew/
disallow: /*/firefox/os/1.1/
disallow: /*/firefox/os/1.3/
disallow: /*/firefox/os/1.3t/
disallow: /*/firefox/os/1.4/
disallow: /b/
{% endif %}

Allow LinkChecker to crawl the site for broken links
Allow LinkChecker to crawl the site for broken links
