@Test public void findLinksInRobotsTxt() { String path = "/robots.txt"; getMockServer() .when(HttpRequest.request() .withMethod("GET") .withPath(path), Times.exactly(1)) .respond(HttpResponse.response() .withStatusCode(200) .withHeader("Content-Type", "text/plain; utf-8") .withBody(createRobotsTxt()) ); LinkExplorer explorer = new LinkExplorer(getHttpclient(), getRsContext(), LinkExplorer.robotsReader); ResultIndex index = new ResultIndex(); Result<LinkList> result = explorer.explore(composeUri(path), index, null); result.getErrors().forEach(Throwable::printStackTrace); assertThat(result.getErrors().isEmpty(), is(true)); assertThat(result.getContent().isPresent(), is(true)); LinkList linkList = result.getContent().orElse(new LinkList()); Set<URI> validUris = linkList.getValidUris(); assertThat(validUris, containsInAnyOrder( composeUri("dataset1/resourcelist1.xml"), composeUri("/dataset2/resourcelist.xml"), composeUri("/some/other/just_a_sitemap.xml"))); assertThat(linkList.getInvalidUris().size(), equalTo(0)); assertThat(index.contains(composeUri(path)), is(true)); }
@Override public Result<LinkList> explore(URI uri, ResultIndex index, String authString) { Result<List<String>> stringResult = execute(uri, responseReader, authString); Result<LinkList> result = stringResult.map(stringListToLinkListConverter); result.getInvalidUris().addAll(result.getContent().map(LinkList::getInvalidUris).orElse(Collections.emptySet())); index.add(result); RsExplorer rsExplorer = new RsExplorer(getHttpClient(), rsContext); for (URI rsUri : result.getContent().orElse(new LinkList()).getValidUris()) { if (!index.contains(rsUri)) { Result<RsRoot> child = rsExplorer.explore(rsUri, index, null); result.addChild(child); } } return result; }
LinkExplorer extends AbstractUriExplorer { @Override public Result<LinkList> explore(URI uri, ResultIndex index, String authString) { Result<List<String>> stringResult = execute(uri, responseReader, authString); Result<LinkList> result = stringResult.map(stringListToLinkListConverter); result.getInvalidUris().addAll(result.getContent().map(LinkList::getInvalidUris).orElse(Collections.emptySet())); index.add(result); RsExplorer rsExplorer = new RsExplorer(getHttpClient(), rsContext); for (URI rsUri : result.getContent().orElse(new LinkList()).getValidUris()) { if (!index.contains(rsUri)) { Result<RsRoot> child = rsExplorer.explore(rsUri, index, null); result.addChild(child); } } return result; } }
LinkExplorer extends AbstractUriExplorer { @Override public Result<LinkList> explore(URI uri, ResultIndex index, String authString) { Result<List<String>> stringResult = execute(uri, responseReader, authString); Result<LinkList> result = stringResult.map(stringListToLinkListConverter); result.getInvalidUris().addAll(result.getContent().map(LinkList::getInvalidUris).orElse(Collections.emptySet())); index.add(result); RsExplorer rsExplorer = new RsExplorer(getHttpClient(), rsContext); for (URI rsUri : result.getContent().orElse(new LinkList()).getValidUris()) { if (!index.contains(rsUri)) { Result<RsRoot> child = rsExplorer.explore(rsUri, index, null); result.addChild(child); } } return result; } LinkExplorer(CloseableHttpClient httpClient,
                      ResourceSyncContext rsContext,
                      Function_WithExceptions<HttpResponse, List<String>, ?> responseReader); }
LinkExplorer extends AbstractUriExplorer { @Override public Result<LinkList> explore(URI uri, ResultIndex index, String authString) { Result<List<String>> stringResult = execute(uri, responseReader, authString); Result<LinkList> result = stringResult.map(stringListToLinkListConverter); result.getInvalidUris().addAll(result.getContent().map(LinkList::getInvalidUris).orElse(Collections.emptySet())); index.add(result); RsExplorer rsExplorer = new RsExplorer(getHttpClient(), rsContext); for (URI rsUri : result.getContent().orElse(new LinkList()).getValidUris()) { if (!index.contains(rsUri)) { Result<RsRoot> child = rsExplorer.explore(rsUri, index, null); result.addChild(child); } } return result; } LinkExplorer(CloseableHttpClient httpClient,
                      ResourceSyncContext rsContext,
                      Function_WithExceptions<HttpResponse, List<String>, ?> responseReader); @Override Result<LinkList> explore(URI uri, ResultIndex index, String authString); }
LinkExplorer extends AbstractUriExplorer { @Override public Result<LinkList> explore(URI uri, ResultIndex index, String authString) { Result<List<String>> stringResult = execute(uri, responseReader, authString); Result<LinkList> result = stringResult.map(stringListToLinkListConverter); result.getInvalidUris().addAll(result.getContent().map(LinkList::getInvalidUris).orElse(Collections.emptySet())); index.add(result); RsExplorer rsExplorer = new RsExplorer(getHttpClient(), rsContext); for (URI rsUri : result.getContent().orElse(new LinkList()).getValidUris()) { if (!index.contains(rsUri)) { Result<RsRoot> child = rsExplorer.explore(rsUri, index, null); result.addChild(child); } } return result; } LinkExplorer(CloseableHttpClient httpClient,
                      ResourceSyncContext rsContext,
                      Function_WithExceptions<HttpResponse, List<String>, ?> responseReader); @Override Result<LinkList> explore(URI uri, ResultIndex index, String authString);  }