@Ignore @Test public void testFetch() throws Exception { LuceneIndexer indexer = new LuceneIndexer(new AppContext()); try { catalogue.fetch(indexer, "Hindi"); } finally { System.out.println("Closing the Index"); indexer.close(); } }
public void fetch(LuceneIndexer indexer, String language) throws IOException { boolean continueLoop = true; int perPage = 500; int start = 0; int end = 200000; int current = start; while (continueLoop && (current <= end)) { continueLoop = false; System.out.println("start = " + start + " , perPage = " + perPage); Document doc = Jsoup.connect("http: try{ end=Integer.parseInt(doc.select("table tbody tr td b").get(1).text()); }catch (Exception e){System.out.println("Error finding the total number of pages, defaulting to 2 lac.");} Elements links = doc.select("table tbody tr td a"); for (Element link : links) { String result = link.toString(); try { Map<String, String> urlParameters = Utils.getUrlParameters(result.replaceAll("&amp;", "&").replaceAll("[\\n\\t]", "")); if (urlParameters.containsKey(AppConstants.BARCODE)) { ++current; continueLoop = true; indexer.index(urlParameters); } } catch (Exception e) { System.out.println(result); System.err.println(e.getMessage()); } } indexer.commit(); start += perPage; } }
DownloadCatalogue { public void fetch(LuceneIndexer indexer, String language) throws IOException { boolean continueLoop = true; int perPage = 500; int start = 0; int end = 200000; int current = start; while (continueLoop && (current <= end)) { continueLoop = false; System.out.println("start = " + start + " , perPage = " + perPage); Document doc = Jsoup.connect("http: try{ end=Integer.parseInt(doc.select("table tbody tr td b").get(1).text()); }catch (Exception e){System.out.println("Error finding the total number of pages, defaulting to 2 lac.");} Elements links = doc.select("table tbody tr td a"); for (Element link : links) { String result = link.toString(); try { Map<String, String> urlParameters = Utils.getUrlParameters(result.replaceAll("&amp;", "&").replaceAll("[\\n\\t]", "")); if (urlParameters.containsKey(AppConstants.BARCODE)) { ++current; continueLoop = true; indexer.index(urlParameters); } } catch (Exception e) { System.out.println(result); System.err.println(e.getMessage()); } } indexer.commit(); start += perPage; } } }
DownloadCatalogue { public void fetch(LuceneIndexer indexer, String language) throws IOException { boolean continueLoop = true; int perPage = 500; int start = 0; int end = 200000; int current = start; while (continueLoop && (current <= end)) { continueLoop = false; System.out.println("start = " + start + " , perPage = " + perPage); Document doc = Jsoup.connect("http: try{ end=Integer.parseInt(doc.select("table tbody tr td b").get(1).text()); }catch (Exception e){System.out.println("Error finding the total number of pages, defaulting to 2 lac.");} Elements links = doc.select("table tbody tr td a"); for (Element link : links) { String result = link.toString(); try { Map<String, String> urlParameters = Utils.getUrlParameters(result.replaceAll("&amp;", "&").replaceAll("[\\n\\t]", "")); if (urlParameters.containsKey(AppConstants.BARCODE)) { ++current; continueLoop = true; indexer.index(urlParameters); } } catch (Exception e) { System.out.println(result); System.err.println(e.getMessage()); } } indexer.commit(); start += perPage; } }  }
DownloadCatalogue { public void fetch(LuceneIndexer indexer, String language) throws IOException { boolean continueLoop = true; int perPage = 500; int start = 0; int end = 200000; int current = start; while (continueLoop && (current <= end)) { continueLoop = false; System.out.println("start = " + start + " , perPage = " + perPage); Document doc = Jsoup.connect("http: try{ end=Integer.parseInt(doc.select("table tbody tr td b").get(1).text()); }catch (Exception e){System.out.println("Error finding the total number of pages, defaulting to 2 lac.");} Elements links = doc.select("table tbody tr td a"); for (Element link : links) { String result = link.toString(); try { Map<String, String> urlParameters = Utils.getUrlParameters(result.replaceAll("&amp;", "&").replaceAll("[\\n\\t]", "")); if (urlParameters.containsKey(AppConstants.BARCODE)) { ++current; continueLoop = true; indexer.index(urlParameters); } } catch (Exception e) { System.out.println(result); System.err.println(e.getMessage()); } } indexer.commit(); start += perPage; } }  void fetch(LuceneIndexer indexer, String language); }
DownloadCatalogue { public void fetch(LuceneIndexer indexer, String language) throws IOException { boolean continueLoop = true; int perPage = 500; int start = 0; int end = 200000; int current = start; while (continueLoop && (current <= end)) { continueLoop = false; System.out.println("start = " + start + " , perPage = " + perPage); Document doc = Jsoup.connect("http: try{ end=Integer.parseInt(doc.select("table tbody tr td b").get(1).text()); }catch (Exception e){System.out.println("Error finding the total number of pages, defaulting to 2 lac.");} Elements links = doc.select("table tbody tr td a"); for (Element link : links) { String result = link.toString(); try { Map<String, String> urlParameters = Utils.getUrlParameters(result.replaceAll("&amp;", "&").replaceAll("[\\n\\t]", "")); if (urlParameters.containsKey(AppConstants.BARCODE)) { ++current; continueLoop = true; indexer.index(urlParameters); } } catch (Exception e) { System.out.println(result); System.err.println(e.getMessage()); } } indexer.commit(); start += perPage; } }  void fetch(LuceneIndexer indexer, String language);  }