pub async fn crawl_url(
    client: &Client,
    url: &str,
    user_agent: Option<String>,
) -> Result<Article, Error> {
    info!("Start crawling article with URL {}", url);
    if !is_supported_url(url) {
        error!("not supported URL {}", url);
        return Err(Error::InvalidUrl);
    }

    let document = transform_to_document(client, url, user_agent).await?;
    let result = parser::parse(&document).map_err(|_| Error::InvalidResponse);
    info!("Finish crawling article with URL {}", url);
    result
}
/// Given a URL, crawls the page and parses it into an Article.
Given a URL, crawls the page and parses it into an Article.
Given a URL crawls the page and parses it into an Article .
pub async fn crawl_url(
    client: &Client,
    url: &str,
    user_agent: Option<String>,
) -> Result<Article, Error> {
    info!("Start crawling article with URL {}", url);
    if !is_supported_url(url) {
        error!("not supported URL {}", url);
        return Err(Error::InvalidUrl);
    }
    let document = transform_to_document(client, url, user_agent).await?;
    let result = parser::parse(&document).map_err(|_| Error::InvalidResponse);
    info!("Finish crawling article with URL {}", url);
    result
}
pub async fn crawl_url ( client : & Client , url : & str , user_agent : Option < String > , ) -> Result < Article , Error > { info ! ( " " url ) ; if ! is_supported_url ( url ) { error ! ( " " url ) ; return Err ( Error :: InvalidUrl ) ; } let document = transform_to_document ( client , url , user_agent ) . await ? ; let result = parser :: parse ( & document ) . map_err ( | _ | Error :: InvalidResponse ) ; info ! ( " " url ) ; result }
Given a URL, crawls the page and parses it into an Article.
Given a URL crawls the page and parses it into an Article .
client
&Client
url
&str
user_agent
Option<String>