pub async fn crawl_with_depth(
    origin_url: Link,
    crawl_depth: usize,
    whitelist: Option<HashSet<url::Host>>,
    blacklist: Option<HashSet<url::Host>>,
    word_list: HashSet<String>,
    tx_output: mpsc::Sender<Link>,
    tx_selenium: mpsc::Sender<String>,
    task_limit: usize,
    timeout: u64,
) -> Result<(), String> {
    let mut to_crawl: HashSet<Url> = HashSet::new();
    let mut crawled: HashSet<Url> = HashSet::new();
    let mut dont_crawl: HashSet<Url> = HashSet::new();
    let word_list = Arc::new(word_list);

    let client = init_reqwest_client(timeout)?;
    let resolver = init_dns_resolver()?;

    to_crawl.insert(origin_url.url);

    for _ in 0..crawl_depth {
        println!("Crawling {} URls", to_crawl.len());

        let (tx_cralwer, mut rx_crawler) = mpsc::channel::<Link>(task_limit);

        to_crawl.iter().cloned().for_each(|x| {
            let tx_clone = tx_cralwer.clone();
            let tx_selenium_clone = tx_selenium.clone();
            let client_clone = client.clone();
            let resolver_clone = resolver.clone();
            let word_list_clone = word_list.clone();
            tokio::spawn(async move {
                crawl_page(
                    x,
                    client_clone,
                    tx_clone,
                    tx_selenium_clone,
                    task_limit,
                    resolver_clone,
                    word_list_clone,
                )
                .await
            });
        });

        to_crawl.clear();

        drop(tx_cralwer);

        while let Some(link) = rx_crawler.recv().await {
            if link.crawled {
                crawled.insert(link.url.clone());
                if let Err(_) = tx_output.send(link).await {
                    return Err("Output Connection Closed".to_string());
                }
            } else {
                let should_crawl = link.should_crawl(&whitelist, &blacklist);
                if should_crawl && !crawled.contains(&link.url) {
                    to_crawl.insert(link.url);
                } else if !should_crawl && !dont_crawl.contains(&link.url) {
                    dont_crawl.insert(link.url.clone());
                    if let Err(_) = tx_output.send(link).await {
                        return Err("Output Connection Closed".to_string());
                    }
                }
            }
        }
    }

    stream::iter(to_crawl)
        .map(|x| links::Link::new_from_url(&x))
        .for_each_concurrent(task_limit, |x| async {
            let _ = tx_output.send(x).await;
        })
        .await;
    Ok(())
}
/// Funtion to start crawling when depth is specified.
/// Does not use Sitemaps.
Funtion to start crawling when depth is specified.
Does not use Sitemaps.
Funtion to start crawling when depth is specified . Does not use Sitemaps .
pub async fn crawl_with_depth(
    origin_url: Link,
    crawl_depth: usize,
    whitelist: Option<HashSet<url::Host>>,
    blacklist: Option<HashSet<url::Host>>,
    word_list: HashSet<String>,
    tx_output: mpsc::Sender<Link>,
    tx_selenium: mpsc::Sender<String>,
    task_limit: usize,
    timeout: u64,
) -> Result<(), String> {
    let mut to_crawl: HashSet<Url> = HashSet::new();
    let mut crawled: HashSet<Url> = HashSet::new();
    let mut dont_crawl: HashSet<Url> = HashSet::new();
    let word_list = Arc::new(word_list);
    let client = init_reqwest_client(timeout)?;
    let resolver = init_dns_resolver()?;
    to_crawl.insert(origin_url.url);
    for _ in 0..crawl_depth {
        println!("Crawling {} URls", to_crawl.len());
        let (tx_cralwer, mut rx_crawler) = mpsc::channel::<Link>(task_limit);
        to_crawl.iter().cloned().for_each(|x| {
            let tx_clone = tx_cralwer.clone();
            let tx_selenium_clone = tx_selenium.clone();
            let client_clone = client.clone();
            let resolver_clone = resolver.clone();
            let word_list_clone = word_list.clone();
            tokio::spawn(async move {
                crawl_page(
                    x,
                    client_clone,
                    tx_clone,
                    tx_selenium_clone,
                    task_limit,
                    resolver_clone,
                    word_list_clone,
                )
                .await
            });
        });
        to_crawl.clear();
        drop(tx_cralwer);
        while let Some(link) = rx_crawler.recv().await {
            if link.crawled {
                crawled.insert(link.url.clone());
                if let Err(_) = tx_output.send(link).await {
                    return Err("Output Connection Closed".to_string());
                }
            } else {
                let should_crawl = link.should_crawl(&whitelist, &blacklist);
                if should_crawl && !crawled.contains(&link.url) {
                    to_crawl.insert(link.url);
                } else if !should_crawl && !dont_crawl.contains(&link.url) {
                    dont_crawl.insert(link.url.clone());
                    if let Err(_) = tx_output.send(link).await {
                        return Err("Output Connection Closed".to_string());
                    }
                }
            }
        }
    }
    stream::iter(to_crawl)
        .map(|x| links::Link::new_from_url(&x))
        .for_each_concurrent(task_limit, |x| async {
            let _ = tx_output.send(x).await;
        })
        .await;
    Ok(())
}
pub async fn crawl_with_depth ( origin_url : Link , crawl_depth : usize , whitelist : Option < HashSet < url :: Host > > , blacklist : Option < HashSet < url :: Host > > , word_list : HashSet < String > , tx_output : mpsc :: Sender < Link > , tx_selenium : mpsc :: Sender < String > , task_limit : usize , timeout : u64 , ) -> Result < ( ) , String > { let mut to_crawl : HashSet < Url > = HashSet :: new ( ) ; let mut crawled : HashSet < Url > = HashSet :: new ( ) ; let mut dont_crawl : HashSet < Url > = HashSet :: new ( ) ; let word_list = Arc :: new ( word_list ) ; let client = init_reqwest_client ( timeout ) ? ; let resolver = init_dns_resolver ( ) ? ; to_crawl . insert ( origin_url . url ) ; for _ in 0 .. crawl_depth { println ! ( " " to_crawl len ( ) ) ; let ( tx_cralwer , mut rx_crawler ) = mpsc :: channel :: < Link > ( task_limit ) ; to_crawl . iter ( ) . cloned ( ) . for_each ( | x | { let tx_clone = tx_cralwer . clone ( ) ; let tx_selenium_clone = tx_selenium . clone ( ) ; let client_clone = client . clone ( ) ; let resolver_clone = resolver . clone ( ) ; let word_list_clone = word_list . clone ( ) ; tokio :: spawn ( async move { crawl_page ( x , client_clone , tx_clone , tx_selenium_clone , task_limit , resolver_clone , word_list_clone , ) . await } ) ; } ) ; to_crawl . clear ( ) ; drop ( tx_cralwer ) ; while let Some ( link ) = rx_crawler . recv ( ) . await { if link . crawled { crawled . insert ( link . url . clone ( ) ) ; if let Err ( _ ) = tx_output . send ( link ) . await { return Err ( " " . to_string ( ) ) ; } } else { let should_crawl = link . should_crawl ( & whitelist , & blacklist ) ; if should_crawl && ! crawled . contains ( & link . url ) { to_crawl . insert ( link . url ) ; } else if ! should_crawl && ! dont_crawl . contains ( & link . url ) { dont_crawl . insert ( link . url . clone ( ) ) ; if let Err ( _ ) = tx_output . send ( link ) . await { return Err ( " " . to_string ( ) ) ; } } } } } stream :: iter ( to_crawl ) . map ( | x | links :: Link :: new_from_url ( & x ) ) . for_each_concurrent ( task_limit , | x | async { let _ = tx_output . send ( x ) . await ; } ) . await ; Ok ( ( ) ) }
Funtion to start crawling when depth is specified.
Funtion to start crawling when depth is specified .
origin_url
Link
crawl_depth
usize
whitelist
Option<HashSet<url::Host>>
blacklist
Option<HashSet<url::Host>>
word_list
HashSet<String>
tx_output
mpsc::Sender<Link>
tx_selenium
mpsc::Sender<String>
task_limit
usize
timeout
u64
origin_url
Link
crawl_depth
usize
whitelist
Option<HashSet<url::Host>>
blacklist
Option<HashSet<url::Host>>
word_list
HashSet<String>
tx_output
mpsc::Sender<Link>
tx_selenium
mpsc::Sender<String>
task_limit
usize
timeout
u64