async fn request_robots_txt_without_proxy() -> Result<()> {
    let handles = Arc::new(Handles::for_testing(None, None).0);

    let srv = MockServer::start();

    let mock = srv.mock(|when, then| {
        when.method(GET).path("/robots.txt");
        then.status(200).body("this is a test");
    });

    let extractor = Extractor {
        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
        response: None,
        url: srv.url("/api/users/stuff/things"),
        target: ExtractionTarget::RobotsTxt,
        handles,
    };

    let resp = extractor.request_robots_txt().await?;

    assert!(matches!(resp.status(), &StatusCode::OK));
    println!("{}", resp);
    assert_eq!(resp.content_length(), 14);
    assert_eq!(mock.hits(), 1);
    Ok(())
}
/// test that /robots.txt is correctly requested given a base url (happy path)
test that /robots.txt is correctly requested given a base url (happy path)
test that / robots . txt is correctly requested given a base url ( happy path )
async fn request_robots_txt_without_proxy() -> Result<()> {
    let handles = Arc::new(Handles::for_testing(None, None).0);
    let srv = MockServer::start();
    let mock = srv.mock(|when, then| {
        when.method(GET).path("/robots.txt");
        then.status(200).body("this is a test");
    });
    let extractor = Extractor {
        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
        response: None,
        url: srv.url("/api/users/stuff/things"),
        target: ExtractionTarget::RobotsTxt,
        handles,
    };
    let resp = extractor.request_robots_txt().await?;
    assert!(matches!(resp.status(), &StatusCode::OK));
    println!("{}", resp);
    assert_eq!(resp.content_length(), 14);
    assert_eq!(mock.hits(), 1);
    Ok(())
}
async fn request_robots_txt_without_proxy ( ) -> Result < ( ) > { let handles = Arc :: new ( Handles :: for_testing ( None , None ) . 0 ) ; let srv = MockServer :: start ( ) ; let mock = srv . mock ( | when , then | { when . method ( GET ) . path ( " " ) ; then . status ( 200 ) . body ( " " ) ; } ) ; let extractor = Extractor { links_regex : Regex :: new ( LINKFINDER_REGEX ) . unwrap ( ) , robots_regex : Regex :: new ( ROBOTS_TXT_REGEX ) . unwrap ( ) , response : None , url : srv . url ( " " ) , target : ExtractionTarget :: RobotsTxt , handles , } ; let resp = extractor . request_robots_txt ( ) . await ? ; assert ! ( matches ( resp status ( ) StatusCode OK ) ) ; println ! ( " " resp ) ; assert_eq ! ( resp content_length ( ) 14 ) ; assert_eq ! ( mock hits ( ) 1 ) ; Ok ( ( ) ) }
test that /robots.txt is correctly requested given a base url (happy path)
test that / robots . txt is correctly requested given a base url ( happy path )