pub fn run(&mut self) {
        /* Push the origin URL and depth (0) through the channel */
        self.map_url_path(&self.args.origin, url_helper::to_path(&self.args.origin));
        Scraper::push(&self.transmitter, self.args.origin.clone(), 0, 0);

        thread::scope(|thread_scope| {
            for _ in 0..self.args.jobs {
                let tx = self.transmitter.clone();
                let rx = self.receiver.clone();
                let self_clone = &self;

                thread_scope.spawn(move |_| {
                    let mut counter = 0;
                    // For a random delay
                    let mut rng = rand::thread_rng();

                    while counter < MAX_EMPTY_RECEIVES {
                        match rx.try_recv() {
                            Err(e) => match e {
                                TryRecvError::Empty => {
                                    counter += 1;
                                    std::thread::sleep(SLEEP_DURATION);
                                }
                                TryRecvError::Disconnected => panic!("{}", e),
                            },
                            Ok((url, depth, ext_depth)) => {
                                counter = 0;
                                Scraper::handle_url(&self_clone, &tx, url, depth, ext_depth);
                                self_clone.sleep(&mut rng);
                            }
                        }
                    }
                });
            }
        })
        .unwrap();
    }
/// Run through the channel and complete it
Run through the channel and complete it
Run through the channel and complete it
pub fn run(&mut self) {
        /* Push the origin URL and depth (0) through the channel */
        self.map_url_path(&self.args.origin, url_helper::to_path(&self.args.origin));
        Scraper::push(&self.transmitter, self.args.origin.clone(), 0, 0);
        thread::scope(|thread_scope| {
            for _ in 0..self.args.jobs {
                let tx = self.transmitter.clone();
                let rx = self.receiver.clone();
                let self_clone = &self;
                thread_scope.spawn(move |_| {
                    let mut counter = 0;
                    // For a random delay
                    let mut rng = rand::thread_rng();
                    while counter < MAX_EMPTY_RECEIVES {
                        match rx.try_recv() {
                            Err(e) => match e {
                                TryRecvError::Empty => {
                                    counter += 1;
                                    std::thread::sleep(SLEEP_DURATION);
                                }
                                TryRecvError::Disconnected => panic!("{}", e),
                            },
                            Ok((url, depth, ext_depth)) => {
                                counter = 0;
                                Scraper::handle_url(&self_clone, &tx, url, depth, ext_depth);
                                self_clone.sleep(&mut rng);
                            }
                        }
                    }
                });
            }
        })
        .unwrap();
    }
pub fn run ( & mut self ) { /* Push the origin URL and depth (0) through the channel */ self . map_url_path ( & self . args . origin , url_helper :: to_path ( & self . args . origin ) ) ; Scraper :: push ( & self . transmitter , self . args . origin . clone ( ) , 0 , 0 ) ; thread :: scope ( | thread_scope | { for _ in 0 .. self . args . jobs { let tx = self . transmitter . clone ( ) ; let rx = self . receiver . clone ( ) ; let self_clone = & self ; thread_scope . spawn ( move | _ | { let mut counter = 0 ; // For a random delay let mut rng = rand :: thread_rng ( ) ; while counter < MAX_EMPTY_RECEIVES { match rx . try_recv ( ) { Err ( e ) => match e { TryRecvError :: Empty => { counter += 1 ; std :: thread :: sleep ( SLEEP_DURATION ) ; } TryRecvError :: Disconnected => panic ! ( " " e ) , } , Ok ( ( url , depth , ext_depth ) ) => { counter = 0 ; Scraper :: handle_url ( & self_clone , & tx , url , depth , ext_depth ) ; self_clone . sleep ( & mut rng ) ; } } } } ) ; } } ) . unwrap ( ) ; }
Run through the channel and complete it
Run through the channel and complete it
&mut self
&mut self