async fn request_robots_txt_with_proxy() -> Result<()> {
    let handles = Arc::new(Handles::for_testing(None, None).0);
    let mut config = Configuration::new()?;

    let srv = MockServer::start();

    let mock = srv.mock(|when, then| {
        when.method(GET).path("/robots.txt");
        then.status(200).body("this is also a test");
    });

    // note: the proxy doesn't actually do anything other than hit a different code branch
    // in this unit test; it would however have an effect on an integration test
    config.proxy = srv.url("/ima-proxy");
    config.no_recursion = true;

    let extractor = ExtractorBuilder::default()
        .url(&srv.url("/api/different/path"))
        .target(ExtractionTarget::RobotsTxt)
        .handles(handles)
        .build()?;

    let resp = extractor.request_robots_txt().await?;

    assert!(matches!(resp.status(), &StatusCode::OK));
    assert_eq!(resp.content_length(), 19);
    assert_eq!(mock.hits(), 1);
    Ok(())
}
/// test that /robots.txt is correctly requested given a base url (happy path) when a proxy is used
test that /robots.txt is correctly requested given a base url (happy path) when a proxy is used
test that / robots . txt is correctly requested given a base url ( happy path ) when a proxy is used
async fn request_robots_txt_with_proxy() -> Result<()> {
    let handles = Arc::new(Handles::for_testing(None, None).0);
    let mut config = Configuration::new()?;
    let srv = MockServer::start();
    let mock = srv.mock(|when, then| {
        when.method(GET).path("/robots.txt");
        then.status(200).body("this is also a test");
    });
    // note: the proxy doesn't actually do anything other than hit a different code branch
    // in this unit test; it would however have an effect on an integration test
    config.proxy = srv.url("/ima-proxy");
    config.no_recursion = true;
    let extractor = ExtractorBuilder::default()
        .url(&srv.url("/api/different/path"))
        .target(ExtractionTarget::RobotsTxt)
        .handles(handles)
        .build()?;
    let resp = extractor.request_robots_txt().await?;
    assert!(matches!(resp.status(), &StatusCode::OK));
    assert_eq!(resp.content_length(), 19);
    assert_eq!(mock.hits(), 1);
    Ok(())
}
async fn request_robots_txt_with_proxy ( ) -> Result < ( ) > { let handles = Arc :: new ( Handles :: for_testing ( None , None ) . 0 ) ; let mut config = Configuration :: new ( ) ? ; let srv = MockServer :: start ( ) ; let mock = srv . mock ( | when , then | { when . method ( GET ) . path ( " " ) ; then . status ( 200 ) . body ( " " ) ; } ) ; // note: the proxy doesn't actually do anything other than hit a different code branch // in this unit test; it would however have an effect on an integration test config . proxy = srv . url ( " " ) ; config . no_recursion = true ; let extractor = ExtractorBuilder :: default ( ) . url ( & srv . url ( " " ) ) . target ( ExtractionTarget :: RobotsTxt ) . handles ( handles ) . build ( ) ? ; let resp = extractor . request_robots_txt ( ) . await ? ; assert ! ( matches ( resp status ( ) StatusCode OK ) ) ; assert_eq ! ( resp content_length ( ) 19 ) ; assert_eq ! ( mock hits ( ) 1 ) ; Ok ( ( ) ) }
test that /robots.txt is correctly requested given a base url (happy path) when a proxy is used
test that / robots . txt is correctly requested given a base url ( happy path ) when a proxy is used