fn make_request(&self, url: &Url) -> Result<Response, reqwest::Error> {
        let req = self.client.get(url.clone());
        let req = match self.get_auth(url) {
            Some((username, password)) => req.basic_auth(username, password.clone()),
            None => req,
        };
        match req.send() {
            Ok(mut data) => {
                lazy_static! {
                    static ref DATA_TYPE_REGEX: Regex =
                        Regex::new(r#"^.*(\b[a-z]+/[a-z-+\.]+).*$"#).unwrap();
                    static ref CHARSET_REGEX: Regex =
                        Regex::new(r#"^.*charset\s*=\s*["']?([^"'\s;]+).*$"#).unwrap();
                }

                let (data_type, charset): (String, Option<String>) =
                    match data.headers().get("content-type") {
                        Some(content_type_header) => {
                            let content_type = content_type_header.to_str().unwrap();
                            let data_type_captures =
                                DATA_TYPE_REGEX.captures_iter(&content_type).nth(0);
                            let data_type = data_type_captures
                                .map_or(String::from("text/html"), |first| {
                                    String::from(first.get(1).unwrap().as_str().to_lowercase())
                                });
                            let charset_captures =
                                CHARSET_REGEX.captures_iter(&content_type).nth(0);
                            let charset = charset_captures.map(|first| {
                                String::from(first.get(1).unwrap().as_str().to_lowercase())
                            });
                            (data_type, charset)
                        }
                        None => (String::from("text/html"), None),
                    };

                let filename = if !Downloader::is_html(&data_type) {
                    Downloader::get_filename(data.headers())
                } else {
                    None
                };

                let mut raw_data: Vec<u8> = Vec::new();
                data.copy_to(&mut raw_data).unwrap();
                let response_data = if Downloader::is_html(&data_type) {
                    ResponseData::Html(raw_data)
                } else {
                    ResponseData::Other(raw_data)
                };

                Ok(Response::new(response_data, filename, charset))
            }

            Err(e) => {
                warn!("Downloader.get() has encountered an error: {}", e);
                Err(e)
            }
        }
    }
///Download the content at this url
Download the content at this url
Download the content at this url
fn make_request(&self, url: &Url) -> Result<Response, reqwest::Error> {
        let req = self.client.get(url.clone());
        let req = match self.get_auth(url) {
            Some((username, password)) => req.basic_auth(username, password.clone()),
            None => req,
        };
        match req.send() {
            Ok(mut data) => {
                lazy_static! {
                    static ref DATA_TYPE_REGEX: Regex =
                        Regex::new(r#"^.*(\b[a-z]+/[a-z-+\.]+).*$"#).unwrap();
                    static ref CHARSET_REGEX: Regex =
                        Regex::new(r#"^.*charset\s*=\s*["']?([^"'\s;]+).*$"#).unwrap();
                }
                let (data_type, charset): (String, Option<String>) =
                    match data.headers().get("content-type") {
                        Some(content_type_header) => {
                            let content_type = content_type_header.to_str().unwrap();
                            let data_type_captures =
                                DATA_TYPE_REGEX.captures_iter(&content_type).nth(0);
                            let data_type = data_type_captures
                                .map_or(String::from("text/html"), |first| {
                                    String::from(first.get(1).unwrap().as_str().to_lowercase())
                                });
                            let charset_captures =
                                CHARSET_REGEX.captures_iter(&content_type).nth(0);
                            let charset = charset_captures.map(|first| {
                                String::from(first.get(1).unwrap().as_str().to_lowercase())
                            });
                            (data_type, charset)
                        }
                        None => (String::from("text/html"), None),
                    };
                let filename = if !Downloader::is_html(&data_type) {
                    Downloader::get_filename(data.headers())
                } else {
                    None
                };
                let mut raw_data: Vec<u8> = Vec::new();
                data.copy_to(&mut raw_data).unwrap();
                let response_data = if Downloader::is_html(&data_type) {
                    ResponseData::Html(raw_data)
                } else {
                    ResponseData::Other(raw_data)
                };
                Ok(Response::new(response_data, filename, charset))
            }
            Err(e) => {
                warn!("Downloader.get() has encountered an error: {}", e);
                Err(e)
            }
        }
    }
fn make_request ( & self , url : & Url ) -> Result < Response , reqwest :: Error > { let req = self . client . get ( url . clone ( ) ) ; let req = match self . get_auth ( url ) { Some ( ( username , password ) ) => req . basic_auth ( username , password . clone ( ) ) , None => req , } ; match req . send ( ) { Ok ( mut data ) => { lazy_static ! { static ref DATA_TYPE_REGEX Regex Regex new ( r#"^.*(\b[a-z]+/[a-z-+\.]+).*$"# ) unwrap ( ) static ref CHARSET_REGEX Regex Regex new ( r#"^.*charset\s*=\s*["']?([^"'\s;]+).*$"# ) unwrap ( ) } let ( data_type , charset ) : ( String , Option < String > ) = match data . headers ( ) . get ( " " ) { Some ( content_type_header ) => { let content_type = content_type_header . to_str ( ) . unwrap ( ) ; let data_type_captures = DATA_TYPE_REGEX . captures_iter ( & content_type ) . nth ( 0 ) ; let data_type = data_type_captures . map_or ( String :: from ( " " ) , | first | { String :: from ( first . get ( 1 ) . unwrap ( ) . as_str ( ) . to_lowercase ( ) ) } ) ; let charset_captures = CHARSET_REGEX . captures_iter ( & content_type ) . nth ( 0 ) ; let charset = charset_captures . map ( | first | { String :: from ( first . get ( 1 ) . unwrap ( ) . as_str ( ) . to_lowercase ( ) ) } ) ; ( data_type , charset ) } None => ( String :: from ( " " ) , None ) , } ; let filename = if ! Downloader :: is_html ( & data_type ) { Downloader :: get_filename ( data . headers ( ) ) } else { None } ; let mut raw_data : Vec < u8 > = Vec :: new ( ) ; data . copy_to ( & mut raw_data ) . unwrap ( ) ; let response_data = if Downloader :: is_html ( & data_type ) { ResponseData :: Html ( raw_data ) } else { ResponseData :: Other ( raw_data ) } ; Ok ( Response :: new ( response_data , filename , charset ) ) } Err ( e ) => { warn ! ( " " e ) ; Err ( e ) } } }
Download the content at this url
Download the content at this url
&self
url
&Url
&self
url
&Url