@Ignore @Test public void testParsingLargeFile() throws ParseException, IOException { InputStream is = GDELTParserTest.class.getResourceAsStream("20140701.export.txt"); BufferedReader reader = new BufferedReader(new InputStreamReader(is)); String line; try { while ((line = reader.readLine()) != null) { parser.parseLine(line); } } finally { try { reader.close(); } catch (IOException e) { System.err.println("Failed to close reader: " + e.toString()); } } }
public GDELTEvent parseLine(String gdeltTabDelimitedLine) throws ParseException { String[] columns = gdeltTabDelimitedLine.split("\\t"); GDELTEvent event = new GDELTEvent(); List<Method> methods = getGDELTMethods(GDELTEvent.class); for (Method method : methods) { method.setAccessible(true); GDELTField annotation = method.getAnnotation(GDELTField.class); int index = headerMapping.get(annotation.name()); if (index > columns.length - 1) { throw new ParseException("Annotated field index is beyond source column array index", index); } String column = columns[index]; if (column == null || column.trim().equals("")) { if (annotation.required()) { throw new ParseException(annotation.name() + " field is required", index); } continue; } column = column.trim(); Class type = method.getParameterTypes()[0]; try { if (type.equals(String.class)) { method.invoke(event, column); } else if (type.equals(Date.class)) { String formatString = annotation.dateFormat(); if (formatString == null) { throw new ParseException("Date type requires dataFormat annotation parameter", index); } Date date = new SimpleDateFormat(formatString).parse(column); method.invoke(event, date); } else if (type.equals(Integer.class) || type.equals(Integer.TYPE)) { method.invoke(event, Integer.parseInt(column)); } else if (type.equals(Double.class) || type.equals(Double.TYPE)) { method.invoke(event, Double.parseDouble(column)); } else if (type.equals(Boolean.class) || type.equals(Boolean.TYPE)) { method.invoke(event, column.equals("1")); } else { throw new ParseException(type + " is not supported", index); } } catch (InvocationTargetException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } return event; }
GDELTParser { public GDELTEvent parseLine(String gdeltTabDelimitedLine) throws ParseException { String[] columns = gdeltTabDelimitedLine.split("\\t"); GDELTEvent event = new GDELTEvent(); List<Method> methods = getGDELTMethods(GDELTEvent.class); for (Method method : methods) { method.setAccessible(true); GDELTField annotation = method.getAnnotation(GDELTField.class); int index = headerMapping.get(annotation.name()); if (index > columns.length - 1) { throw new ParseException("Annotated field index is beyond source column array index", index); } String column = columns[index]; if (column == null || column.trim().equals("")) { if (annotation.required()) { throw new ParseException(annotation.name() + " field is required", index); } continue; } column = column.trim(); Class type = method.getParameterTypes()[0]; try { if (type.equals(String.class)) { method.invoke(event, column); } else if (type.equals(Date.class)) { String formatString = annotation.dateFormat(); if (formatString == null) { throw new ParseException("Date type requires dataFormat annotation parameter", index); } Date date = new SimpleDateFormat(formatString).parse(column); method.invoke(event, date); } else if (type.equals(Integer.class) || type.equals(Integer.TYPE)) { method.invoke(event, Integer.parseInt(column)); } else if (type.equals(Double.class) || type.equals(Double.TYPE)) { method.invoke(event, Double.parseDouble(column)); } else if (type.equals(Boolean.class) || type.equals(Boolean.TYPE)) { method.invoke(event, column.equals("1")); } else { throw new ParseException(type + " is not supported", index); } } catch (InvocationTargetException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } return event; } }
GDELTParser { public GDELTEvent parseLine(String gdeltTabDelimitedLine) throws ParseException { String[] columns = gdeltTabDelimitedLine.split("\\t"); GDELTEvent event = new GDELTEvent(); List<Method> methods = getGDELTMethods(GDELTEvent.class); for (Method method : methods) { method.setAccessible(true); GDELTField annotation = method.getAnnotation(GDELTField.class); int index = headerMapping.get(annotation.name()); if (index > columns.length - 1) { throw new ParseException("Annotated field index is beyond source column array index", index); } String column = columns[index]; if (column == null || column.trim().equals("")) { if (annotation.required()) { throw new ParseException(annotation.name() + " field is required", index); } continue; } column = column.trim(); Class type = method.getParameterTypes()[0]; try { if (type.equals(String.class)) { method.invoke(event, column); } else if (type.equals(Date.class)) { String formatString = annotation.dateFormat(); if (formatString == null) { throw new ParseException("Date type requires dataFormat annotation parameter", index); } Date date = new SimpleDateFormat(formatString).parse(column); method.invoke(event, date); } else if (type.equals(Integer.class) || type.equals(Integer.TYPE)) { method.invoke(event, Integer.parseInt(column)); } else if (type.equals(Double.class) || type.equals(Double.TYPE)) { method.invoke(event, Double.parseDouble(column)); } else if (type.equals(Boolean.class) || type.equals(Boolean.TYPE)) { method.invoke(event, column.equals("1")); } else { throw new ParseException(type + " is not supported", index); } } catch (InvocationTargetException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } return event; } GDELTParser(); }
GDELTParser { public GDELTEvent parseLine(String gdeltTabDelimitedLine) throws ParseException { String[] columns = gdeltTabDelimitedLine.split("\\t"); GDELTEvent event = new GDELTEvent(); List<Method> methods = getGDELTMethods(GDELTEvent.class); for (Method method : methods) { method.setAccessible(true); GDELTField annotation = method.getAnnotation(GDELTField.class); int index = headerMapping.get(annotation.name()); if (index > columns.length - 1) { throw new ParseException("Annotated field index is beyond source column array index", index); } String column = columns[index]; if (column == null || column.trim().equals("")) { if (annotation.required()) { throw new ParseException(annotation.name() + " field is required", index); } continue; } column = column.trim(); Class type = method.getParameterTypes()[0]; try { if (type.equals(String.class)) { method.invoke(event, column); } else if (type.equals(Date.class)) { String formatString = annotation.dateFormat(); if (formatString == null) { throw new ParseException("Date type requires dataFormat annotation parameter", index); } Date date = new SimpleDateFormat(formatString).parse(column); method.invoke(event, date); } else if (type.equals(Integer.class) || type.equals(Integer.TYPE)) { method.invoke(event, Integer.parseInt(column)); } else if (type.equals(Double.class) || type.equals(Double.TYPE)) { method.invoke(event, Double.parseDouble(column)); } else if (type.equals(Boolean.class) || type.equals(Boolean.TYPE)) { method.invoke(event, column.equals("1")); } else { throw new ParseException(type + " is not supported", index); } } catch (InvocationTargetException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } return event; } GDELTParser(); GDELTEvent parseLine(String gdeltTabDelimitedLine); }
GDELTParser { public GDELTEvent parseLine(String gdeltTabDelimitedLine) throws ParseException { String[] columns = gdeltTabDelimitedLine.split("\\t"); GDELTEvent event = new GDELTEvent(); List<Method> methods = getGDELTMethods(GDELTEvent.class); for (Method method : methods) { method.setAccessible(true); GDELTField annotation = method.getAnnotation(GDELTField.class); int index = headerMapping.get(annotation.name()); if (index > columns.length - 1) { throw new ParseException("Annotated field index is beyond source column array index", index); } String column = columns[index]; if (column == null || column.trim().equals("")) { if (annotation.required()) { throw new ParseException(annotation.name() + " field is required", index); } continue; } column = column.trim(); Class type = method.getParameterTypes()[0]; try { if (type.equals(String.class)) { method.invoke(event, column); } else if (type.equals(Date.class)) { String formatString = annotation.dateFormat(); if (formatString == null) { throw new ParseException("Date type requires dataFormat annotation parameter", index); } Date date = new SimpleDateFormat(formatString).parse(column); method.invoke(event, date); } else if (type.equals(Integer.class) || type.equals(Integer.TYPE)) { method.invoke(event, Integer.parseInt(column)); } else if (type.equals(Double.class) || type.equals(Double.TYPE)) { method.invoke(event, Double.parseDouble(column)); } else if (type.equals(Boolean.class) || type.equals(Boolean.TYPE)) { method.invoke(event, column.equals("1")); } else { throw new ParseException(type + " is not supported", index); } } catch (InvocationTargetException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } return event; } GDELTParser(); GDELTEvent parseLine(String gdeltTabDelimitedLine);  }