HttpContentRetriever implements ContentRetriever { @Override public ContentRetrieverResponse retrieveUrlContent(CharSequence url) { long startTime = System.currentTimeMillis(); log.info("starting content retrieval for url: {}", url); try { return retrieveUrlContent(url.toString(), 0); } catch (Exception e) { log.error("content retrieval failed for url: " + url, e); return new ContentRetrieverResponse(e); } finally { log.info("finished content retrieval for url: {} in {} ms", url, (System.currentTimeMillis() - startTime)); } } HttpContentRetriever(int connectionTimeout, int readTimeout, int maxPageContentLength,
            long throttleSleepTime, int maxRetriesCount); @Override ContentRetrieverResponse retrieveUrlContent(CharSequence url);  }
@Test public void testGetContentForHttp200() throws Exception { String expectedResult = "this is expected result"; HttpContentRetriever service = prepareValidService(); CloseableHttpResponse getContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getContentStatusLine = mock(StatusLine.class); HttpEntity getContentHttpEntity = mock(HttpEntity.class); when(getContentHttpResponse.getStatusLine()).thenReturn(getContentStatusLine); when(getContentStatusLine.getStatusCode()).thenReturn(200); when(getContentHttpResponse.getEntity()).thenReturn(getContentHttpEntity); when(getContentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(expectedResult.getBytes())); when(httpClient.execute(any(HttpGet.class))).thenReturn(getContentHttpResponse); ContentRetrieverResponse response = service.retrieveUrlContent("someUrl"); assertNotNull(response); assertNull(response.getException()); assertEquals(expectedResult, response.getContent()); }
@Test public void testGetNoContentForHttp404() throws Exception { HttpContentRetriever service = prepareValidService(); CloseableHttpResponse getContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getContentStatusLine = mock(StatusLine.class); when(getContentHttpResponse.getStatusLine()).thenReturn(getContentStatusLine); when(getContentStatusLine.getStatusCode()).thenReturn(404); when(httpClient.execute(any(HttpGet.class))).thenReturn(getContentHttpResponse); ContentRetrieverResponse response = service.retrieveUrlContent("someUrl"); assertNotNull(response); assertEquals("", response.getContent()); assertTrue(response.getException() instanceof NoSuchElementException); }
@Test public void testGetContentResultsInExceptionForHttp500() throws Exception { HttpContentRetriever service = prepareValidService(); CloseableHttpResponse getContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getContentStatusLine = mock(StatusLine.class); when(getContentHttpResponse.getStatusLine()).thenReturn(getContentStatusLine); when(getContentStatusLine.getStatusCode()).thenReturn(500); when(httpClient.execute(any(HttpGet.class))).thenReturn(getContentHttpResponse); ContentRetrieverResponse response = service.retrieveUrlContent("someUrl"); assertNotNull(response); assertEquals("", response.getContent()); assertTrue(response.getException() instanceof RuntimeException); }
@Test public void testGetMovedContentForHttp301NoLocationHeader() throws Exception { String originalResult = "this is original result"; String movedResult = "this is moved result"; HttpContentRetriever service = prepareValidService(); CloseableHttpResponse getContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getContentStatusLine = mock(StatusLine.class); HttpEntity getContentHttpEntity = mock(HttpEntity.class); when(getContentHttpResponse.getStatusLine()).thenReturn(getContentStatusLine); when(getContentStatusLine.getStatusCode()).thenReturn(301); when(getContentHttpResponse.getEntity()).thenReturn(getContentHttpEntity); when(getContentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(originalResult.getBytes())); CloseableHttpResponse getMovedContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getMovedContentStatusLine = mock(StatusLine.class); HttpEntity getMovedContentHttpEntity = mock(HttpEntity.class); when(getMovedContentHttpResponse.getStatusLine()).thenReturn(getMovedContentStatusLine); when(getMovedContentStatusLine.getStatusCode()).thenReturn(200); when(getMovedContentHttpResponse.getEntity()).thenReturn(getMovedContentHttpEntity); when(getMovedContentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(movedResult.getBytes())); when(httpClient.execute(any(HttpGet.class))).thenReturn(getContentHttpResponse, getMovedContentHttpResponse); ContentRetrieverResponse response = service.retrieveUrlContent("someUrl"); assertNotNull(response); assertTrue(response.getException() instanceof RuntimeException); assertEquals("", response.getContent()); }
@Test public void testGetMovedContentForHttp301And200() throws Exception { String originalResult = "this is original result"; String movedResult = "this is moved result"; HttpContentRetriever service = prepareValidService(); Header mockedHeader = mock(Header.class); when(mockedHeader.getName()).thenReturn("Location"); when(mockedHeader.getValue()).thenReturn("newUrl"); Header[] headers = new Header[] {mockedHeader}; CloseableHttpResponse getContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getContentStatusLine = mock(StatusLine.class); HttpEntity getContentHttpEntity = mock(HttpEntity.class); when(getContentHttpResponse.getStatusLine()).thenReturn(getContentStatusLine); when(getContentStatusLine.getStatusCode()).thenReturn(301); when(getContentHttpResponse.getAllHeaders()).thenReturn(headers); when(getContentHttpResponse.getEntity()).thenReturn(getContentHttpEntity); when(getContentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(originalResult.getBytes())); CloseableHttpResponse getMovedContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getMovedContentStatusLine = mock(StatusLine.class); HttpEntity getMovedContentHttpEntity = mock(HttpEntity.class); when(getMovedContentHttpResponse.getStatusLine()).thenReturn(getMovedContentStatusLine); when(getMovedContentStatusLine.getStatusCode()).thenReturn(200); when(getMovedContentHttpResponse.getEntity()).thenReturn(getMovedContentHttpEntity); when(getMovedContentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(movedResult.getBytes())); when(httpClient.execute(any(HttpGet.class))).thenReturn(getContentHttpResponse, getMovedContentHttpResponse); ContentRetrieverResponse response = service.retrieveUrlContent("someUrl"); assertNotNull(response); assertNull(response.getException()); assertEquals(movedResult, response.getContent()); }
@Test public void testGetContentForHttp429RetryLimitExceeded() throws Exception { HttpContentRetriever service = prepareValidService(); CloseableHttpResponse getContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getContentStatusLine = mock(StatusLine.class); when(getContentHttpResponse.getStatusLine()).thenReturn(getContentStatusLine); when(getContentStatusLine.getStatusCode()).thenReturn(429); HttpEntity getContentHttpEntity = mock(HttpEntity.class); when(getContentHttpResponse.getEntity()).thenReturn(getContentHttpEntity); when(getContentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream("".getBytes())); when(httpClient.execute(any(HttpGet.class))).thenReturn(getContentHttpResponse); ContentRetrieverResponse response = service.retrieveUrlContent("someUrl"); assertNotNull(response); assertTrue(response.getException() instanceof RetryLimitExceededException); assertEquals("", response.getContent()); }
@Test public void testGetContentForHttp429And200() throws Exception { String validResult = "this is valid result"; HttpContentRetriever service = prepareValidService(); CloseableHttpResponse getRateLimitedContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getContentStatusLine = mock(StatusLine.class); when(getRateLimitedContentHttpResponse.getStatusLine()).thenReturn(getContentStatusLine); when(getContentStatusLine.getStatusCode()).thenReturn(429); HttpEntity getContentHttpEntity = mock(HttpEntity.class); when(getRateLimitedContentHttpResponse.getEntity()).thenReturn(getContentHttpEntity); when(getContentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream("".getBytes())); CloseableHttpResponse getValidContentHttpResponse = mock(CloseableHttpResponse.class); StatusLine getMovedContentStatusLine = mock(StatusLine.class); HttpEntity getMovedContentHttpEntity = mock(HttpEntity.class); when(getValidContentHttpResponse.getStatusLine()).thenReturn(getMovedContentStatusLine); when(getMovedContentStatusLine.getStatusCode()).thenReturn(200); when(getValidContentHttpResponse.getEntity()).thenReturn(getMovedContentHttpEntity); when(getMovedContentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(validResult.getBytes())); when(httpClient.execute(any(HttpGet.class))).thenReturn(getRateLimitedContentHttpResponse, getValidContentHttpResponse); ContentRetrieverResponse response = service.retrieveUrlContent("someUrl"); assertNotNull(response); assertNull(response.getException()); assertEquals(validResult, response.getContent()); }