pub async fn collect_links(self, inputs: &[Input]) -> Result<HashSet<Request>> {
        let (contents_tx, mut contents_rx) = tokio::sync::mpsc::channel(self.max_concurrency);

        // extract input contents
        for input in inputs.iter().cloned() {
            let sender = contents_tx.clone();

            let skip_missing_inputs = self.skip_missing_inputs;
            tokio::spawn(async move {
                let contents = input.get_contents(None, skip_missing_inputs).await;
                sender.send(contents).await
            });
        }

        // receiver will get None once all tasks are done
        drop(contents_tx);

        // extract links from input contents
        let mut extract_links_handles = vec![];

        while let Some(result) = contents_rx.recv().await {
            for input_content in result? {
                let base = self.base.clone();
                let handle = tokio::task::spawn_blocking(move || {
                    let mut extractor = Extractor::new(base);
                    extractor.extract(&input_content)
                });
                extract_links_handles.push(handle);
            }
        }

        // Note: we could dispatch links to be checked as soon as we get them,
        //       instead of building a HashSet with all links.
        //       This optimization would speed up cases where there's
        //       a lot of inputs and/or the inputs are large (e.g. big files).
        let mut links: HashSet<Request> = HashSet::new();

        for handle in extract_links_handles {
            let new_links = handle.await?;
            links.extend(new_links?);
        }

        Ok(links)
    }
/// Fetch all unique links from a slice of inputs
/// All relative URLs get prefixed with `base` if given.
/// (This can be a directory or a base URL)
///
/// # Errors
///
/// Will return `Err` if links cannot be extracted from an input
Fetch all unique links from a slice of inputs
All relative URLs get prefixed with `base` if given.
(This can be a directory or a base URL)
Fetch all unique links from a slice of inputs All relative URLs get prefixed with ` base ` if given . ( This can be a directory or a base URL )
pub async fn collect_links(self, inputs: &[Input]) -> Result<HashSet<Request>> {
        let (contents_tx, mut contents_rx) = tokio::sync::mpsc::channel(self.max_concurrency);
        // extract input contents
        for input in inputs.iter().cloned() {
            let sender = contents_tx.clone();
            let skip_missing_inputs = self.skip_missing_inputs;
            tokio::spawn(async move {
                let contents = input.get_contents(None, skip_missing_inputs).await;
                sender.send(contents).await
            });
        }
        // receiver will get None once all tasks are done
        drop(contents_tx);
        // extract links from input contents
        let mut extract_links_handles = vec![];
        while let Some(result) = contents_rx.recv().await {
            for input_content in result? {
                let base = self.base.clone();
                let handle = tokio::task::spawn_blocking(move || {
                    let mut extractor = Extractor::new(base);
                    extractor.extract(&input_content)
                });
                extract_links_handles.push(handle);
            }
        }
        // Note: we could dispatch links to be checked as soon as we get them,
        //       instead of building a HashSet with all links.
        //       This optimization would speed up cases where there's
        //       a lot of inputs and/or the inputs are large (e.g. big files).
        let mut links: HashSet<Request> = HashSet::new();
        for handle in extract_links_handles {
            let new_links = handle.await?;
            links.extend(new_links?);
        }
        Ok(links)
    }
pub async fn collect_links ( self , inputs : & [ Input ] ) -> Result < HashSet < Request > > { let ( contents_tx , mut contents_rx ) = tokio :: sync :: mpsc :: channel ( self . max_concurrency ) ; // extract input contents for input in inputs . iter ( ) . cloned ( ) { let sender = contents_tx . clone ( ) ; let skip_missing_inputs = self . skip_missing_inputs ; tokio :: spawn ( async move { let contents = input . get_contents ( None , skip_missing_inputs ) . await ; sender . send ( contents ) . await } ) ; } // receiver will get None once all tasks are done drop ( contents_tx ) ; // extract links from input contents let mut extract_links_handles = vec ! [ ] ; while let Some ( result ) = contents_rx . recv ( ) . await { for input_content in result ? { let base = self . base . clone ( ) ; let handle = tokio :: task :: spawn_blocking ( move | | { let mut extractor = Extractor :: new ( base ) ; extractor . extract ( & input_content ) } ) ; extract_links_handles . push ( handle ) ; } } // Note: we could dispatch links to be checked as soon as we get them, //       instead of building a HashSet with all links. //       This optimization would speed up cases where there's //       a lot of inputs and/or the inputs are large (e.g. big files). let mut links : HashSet < Request > = HashSet :: new ( ) ; for handle in extract_links_handles { let new_links = handle . await ? ; links . extend ( new_links ? ) ; } Ok ( links ) }
Fetch all unique links from a slice of inputs
All relative URLs get prefixed with `base` if given.
Fetch all unique links from a slice of inputs All relative URLs get prefixed with ` base ` if given .
self
inputs
&[Input]
errors
Will return `Err` if links cannot be extracted from an input
Will return ` Err ` if links cannot be extracted from an input