NormalizerImpl implements UriNormalizer { @Override public CrawleableUri normalize(CrawleableUri uri) { URI uriObject = uri.getUri(); boolean changed = false; String path = uriObject.getRawPath(); if (path != null) { String temp = normalizePath(path); if (temp != path) { path = temp; changed = true; } } String query = uriObject.getQuery(); if (query != null) { if (query.length() > 0) { String[] queryList = query.split("&"); Arrays.sort(queryList); List<String> queries = new ArrayList<>(Arrays.asList(queryList)); List<String> toRemove = new ArrayList<>(); for (String queryParameter : queries) { if (sessionIDs.contains(queryParameter.split("=")[0].toLowerCase())) { toRemove.add(queryParameter); } } queries.removeAll(toRemove); String newQuery = String.join("&", queries); if (!query.equals(newQuery)) { query = newQuery; changed = true; } } else { query = null; changed = true; } } int port = uriObject.getPort(); String scheme = uriObject.getScheme() != null ? uriObject.getScheme() : ""; if (port != -1 && defaultPortMap.containsKey(scheme) && port == defaultPortMap.get(scheme)) { port = -1; changed = true; } String fragment = uriObject.getFragment(); if ((fragment != null) && (fragment.length() > 0)) { changed = true; } String host = uriObject.getHost() != null ? uriObject.getHost() : ""; String lowerCaseHost = host != null ? host.toLowerCase() : ""; String lowerCaseScheme = scheme != null ? scheme.toLowerCase() : ""; if (!scheme.equals(lowerCaseScheme) || !host.equals(lowerCaseHost)) { scheme = lowerCaseScheme; host = lowerCaseHost; changed = true; } if (changed) { URIBuilder builder = new URIBuilder(uriObject); builder.setFragment(null); builder.setPath(path); builder.setCustomQuery(query); builder.setPort(port); builder.setHost(host); builder.setScheme(scheme); CrawleableUri normalizedUri = null; try { normalizedUri = new CrawleableUri(builder.build()); normalizedUri.setData(uri.getData()); } catch (URISyntaxException e) { LOGGER.error("Exception while normalizing URI. Returning original URI.", e); return uri; } return normalizedUri; } else return uri; } NormalizerImpl(List<String> sessionIDs, Map<String, Integer> defaultPortMap); @Override CrawleableUri normalize(CrawleableUri uri); String normalizePath(String path);  }
@Test public void test() { List<String> sessionIDs = new ArrayList<String>(); sessionIDs.add("sessionid"); sessionIDs.add("jsessionids"); sessionIDs.add("phpsessid"); sessionIDs.add("sid"); Map<String, Integer> mapDefaultPort = new HashedMap<String, Integer>(); mapDefaultPort.put("http", 80); mapDefaultPort.put("https", 443); mapDefaultPort.put("ftp", 21); mapDefaultPort.put("ftps", 90); mapDefaultPort.put("sftp", 22); NormalizerImpl normalizer = new NormalizerImpl(sessionIDs, mapDefaultPort); CrawleableUri normUri = normalizer.normalize(originalUri); Assert.assertEquals(expectedUri, normUri); Map<String, Object> expectedData = expectedUri.getData(); Map<String, Object> normData = normUri.getData(); for (Entry<String, Object> expected : expectedData.entrySet()) { Assert.assertTrue( "The normalized URI does not contain data with the expected key \"" + expected.getKey() + "\".", normData.containsKey(expected.getKey())); Assert.assertEquals(expected.getValue(), normData.get(expected.getKey())); } Assert.assertEquals("The normalized URI seems to have a different amount of appended data elemenets.", expectedData.size(), normData.size()); }