async fn crawl_sitemaps(url: Url, tx: mpsc::Sender<Link>, limit: usize, client: reqwest::Client) {
    let mut robottxt_url = url.clone();
    robottxt_url.set_path("robots.txt");
    let robottxt = match get_page(robottxt_url.as_str(), &client).await {
        Ok(x) => match x.text().await {
            Ok(x) => x,
            Err(_) => return,
        },
        Err(_) => return,
    };
    let url_str = url.to_string();
    robottxt
        .lines()
        .filter(|x| x.contains("Sitemap"))
        .filter_map(|x| x[9..].split_whitespace().next())
        .map(|x| x.trim())
        .filter_map(|x| links::normalize_url(x, &url_str))
        .for_each(|x| {
            let tx_clone = tx.clone();
            let client_clone = client.clone();
            let limit_clone = limit.clone();
            tokio::spawn(async move {
                crawl_sitemap(x.url, tx_clone, limit_clone, client_clone).await;
            });
        });
}
/// Function to find and crawl sitemaps from robottxt.
Function to find and crawl sitemaps from robottxt.
Function to find and crawl sitemaps from robottxt .
async fn crawl_sitemaps(url: Url, tx: mpsc::Sender<Link>, limit: usize, client: reqwest::Client) {
    let mut robottxt_url = url.clone();
    robottxt_url.set_path("robots.txt");
    let robottxt = match get_page(robottxt_url.as_str(), &client).await {
        Ok(x) => match x.text().await {
            Ok(x) => x,
            Err(_) => return,
        },
        Err(_) => return,
    };
    let url_str = url.to_string();
    robottxt
        .lines()
        .filter(|x| x.contains("Sitemap"))
        .filter_map(|x| x[9..].split_whitespace().next())
        .map(|x| x.trim())
        .filter_map(|x| links::normalize_url(x, &url_str))
        .for_each(|x| {
            let tx_clone = tx.clone();
            let client_clone = client.clone();
            let limit_clone = limit.clone();
            tokio::spawn(async move {
                crawl_sitemap(x.url, tx_clone, limit_clone, client_clone).await;
            });
        });
}
async fn crawl_sitemaps ( url : Url , tx : mpsc :: Sender < Link > , limit : usize , client : reqwest :: Client ) { let mut robottxt_url = url . clone ( ) ; robottxt_url . set_path ( " " ) ; let robottxt = match get_page ( robottxt_url . as_str ( ) , & client ) . await { Ok ( x ) => match x . text ( ) . await { Ok ( x ) => x , Err ( _ ) => return , } , Err ( _ ) => return , } ; let url_str = url . to_string ( ) ; robottxt . lines ( ) . filter ( | x | x . contains ( " " ) ) . filter_map ( | x | x [ 9 .. ] . split_whitespace ( ) . next ( ) ) . map ( | x | x . trim ( ) ) . filter_map ( | x | links :: normalize_url ( x , & url_str ) ) . for_each ( | x | { let tx_clone = tx . clone ( ) ; let client_clone = client . clone ( ) ; let limit_clone = limit . clone ( ) ; tokio :: spawn ( async move { crawl_sitemap ( x . url , tx_clone , limit_clone , client_clone ) . await ; } ) ; } ) ; }
Function to find and crawl sitemaps from robottxt.
Function to find and crawl sitemaps from robottxt .
url
Url
tx
mpsc::Sender<Link>
limit
usize
client
reqwest::Client
url
Url
tx
mpsc::Sender<Link>
limit
usize
client
reqwest::Client