<?php
//
// Web Robots https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
//

include 'content/config.php';
include 'content/cache.php'; cache::me();
header('Content-Type: text/plain');

// Global rules
echo "User-agent: *\nDisallow:";
// Removing Documents From the Wayback Machine archive.org
echo "\n\nUser-agent: ia_archiver\nDisallow: /";
// Disable *.domaintools.com crawler
echo "\n\nUser-agent: SurveyBot\nDisallow: /";
// Be sure your sitemap.xml path is correct
echo "\n\nSitemap: http://$host{$path}sitemap.xml";
<?php
//
// Web Robots https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
//

include 'content/config.php';
include 'content/cache.php'; cache::me();

// Allow crawl all content
echo "User-agent: *\nDisallow:";

// Removing Documents From the Wayback Machine archive.org
echo "\n\nUser-agent: ia_archiver\nDisallow: /";

// Reduce bandwidth on blocking unwanted crawlers
echo "\n\nUser-agent: AhrefsBot\nDisallow: /
\nUser-agent: Exabot\nDisallow: /
\nUser-agent: meanpathbot\nDisallow: /
\nUser-agent: MJ12bot\nDisallow: /
\nUser-agent: nibbler\nDisallow: /
\nUser-agent: SurveyBot\nDisallow: /
\nUser-agent: XoviBot\nDisallow: /";

// Add sitemap.xml if exists
if (file_exists('sitemap.xml')) {
    echo "\n\nSitemap: $scheme://$host$path/sitemap.xml";
}

Reduce bandwidth, add sitemap.xml if exists
Reduce bandwidth, add sitemap.xml if exists

Reduce bandwidth on blocking unwanted crawlers
Add sitemap.xml if exists
MIME Type defined in Apache configuration
