DateParser extends AbstractParser { public List<Date> processing(String input) { if (input == null) return null; List<Date> dates = null; List<String> dateBlocks = new ArrayList<String>(); try { List<String> tokenizations = analyzer.tokenize(input, new Language("en", 1.0)); if (tokenizations.size() == 0) return null; for(String tok : tokenizations) { if (!tok.equals(" ") && !tok.equals("\n")) { tok = tok.replaceAll("[ \n]", ""); dateBlocks.add(tok + " <date>"); } } String headerDate = FeaturesVectorDate.addFeaturesDate(dateBlocks); String res = label(headerDate); StringTokenizer st2 = new StringTokenizer(res, "\n"); String lastTag = null; org.grobid.core.data.Date date = new Date(); int lineCount = 0; String currentMarker = null; while (st2.hasMoreTokens()) { String line = st2.nextToken(); if ((line.trim().length() == 0)) { if (date.isNotNull()) { if (dates == null) dates = new ArrayList<Date>(); normalize(date); dates.add(date); } date = new Date(); continue; } StringTokenizer st3 = new StringTokenizer(line, "\t "); int ll = st3.countTokens(); int i = 0; String s1 = null; String s2 = null; while (st3.hasMoreTokens()) { String s = st3.nextToken().trim(); if (i == 0) { s2 = s; } else if (i == ll - 1) { s1 = s; } i++; } if (s1.equals("<year>") || s1.equals("I-<year>")) { if (date.getYearString() != null) { if ((s1.equals("I-<year>")) || (!s1.equals(lastTag) && !lastTag.equals("I-<year>")) ) { if (date.isNotNull()) { if (dates == null) dates = new ArrayList<Date>(); normalize(date); dates.add(date); } date = new Date(); date.setYearString(s2); } else { if (date.getYearString().length() == 0) date.setYearString(s2); else if ((date.getYearString().charAt(date.getYearString().length() - 1) == '-') | (date.getYearString().charAt(date.getYearString().length() - 1) == '\'')) date.setYearString(date.getYearString() + s2); else date.setYearString(date.getYearString() + " " + s2); } } else { date.setYearString(s2); } } else if (s1.equals("<month>") || s1.equals("I-<month>")) { if (date.getMonthString() != null) { if ((s1.equals("I-<month>")) || (!s1.equals(lastTag) && !lastTag.equals("I-<month>")) ) { if (date.isNotNull()) { if (dates == null) dates = new ArrayList<Date>(); normalize(date); dates.add(date); } date = new Date(); date.setMonthString(s2); } else { if (date.getMonthString().length() == 0) date.setMonthString(s2); else if ((date.getMonthString().charAt(date.getMonthString().length() - 1) == '-') | (date.getMonthString().charAt(date.getMonthString().length() - 1) == '\'')) date.setMonthString(date.getMonthString() + s2); else date.setMonthString(date.getMonthString() + " " + s2); } } else { date.setMonthString(s2); } } else if (s1.equals("<day>") || s1.equals("I-<day>")) { if (date.getDayString() != null) { if ((s1.equals("I-<day>")) || (!s1.equals(lastTag) && !lastTag.equals("I-<day>")) ) { if (date.isNotNull()) { if (dates == null) dates = new ArrayList<Date>(); normalize(date); dates.add(date); } date = new Date(); date.setDayString(s2); } else { if (date.getDayString().length() == 0) date.setDayString(s2); else if ((date.getDayString().charAt(date.getDayString().length() - 1) == '-') | (date.getDayString().charAt(date.getDayString().length() - 1) == '\'')) date.setDayString(date.getDayString() + s2); else date.setDayString(date.getDayString() + " " + s2); } } else { date.setDayString(s2); } } lastTag = s1; lineCount++; } if (date.isNotNull()) { if (dates == null) dates = new ArrayList<Date>(); normalize(date); dates.add(date); } } catch (Exception e) { throw new GrobidException("An exception occured while running Grobid.", e); } return dates; } DateParser(); List<Date> processing(String input); void normalize(Date date); StringBuilder trainingExtraction(List<String> inputs); static final Pattern jan; static final Pattern feb; static final Pattern mar; static final Pattern apr; static final Pattern may; static final Pattern jun; static final Pattern jul; static final Pattern aug; static final Pattern sep; static final Pattern oct; static final Pattern nov; static final Pattern dec; static final Pattern[] months; }
@Test public void processing_englishStandardDate_shouldWork() throws Exception { List<Date> output = target.processing("19 January 1983"); assertThat(output, hasSize(1)); final Date date = output.get(0); assertThat(date.getDay(), is(19)); assertThat(date.getMonth(), is(1)); assertThat(date.getYear(), is(1983)); assertThat(date.getDayString(), is("19")); assertThat(date.getMonthString(), is("January")); assertThat(date.getYearString(), is("1983")); }
@Test public void processing_englishStandardDate1_shouldWork() throws Exception { List<Date> output = target.processing("19. January 19 83"); assertThat(output, hasSize(1)); final Date date = output.get(0); assertThat(date.getDay(), is(19)); assertThat(date.getMonth(), is(1)); assertThat(date.getYear(), is(1983)); assertThat(date.getDayString(), is("19")); assertThat(date.getMonthString(), is("January")); assertThat(date.getYearString(), is("19 83")); }
@Test public void processing_englishStandardDate2_shouldWork() throws Exception { List<Date> output = target.processing("1918-1939"); assertThat(output, hasSize(2)); final Date date1 = output.get(0); assertThat(date1.getDay(), is(-1)); assertThat(date1.getMonth(), is(-1)); assertThat(date1.getYear(), is(1918)); assertThat(date1.getDayString(), nullValue()); assertThat(date1.getMonthString(), nullValue()); assertThat(date1.getYearString(), is("1918")); final Date date2 = output.get(1); assertThat(date2.getDay(), is(-1)); assertThat(date2.getMonth(), is(-1)); assertThat(date2.getYear(), is(1939)); assertThat(date2.getDayString(), nullValue()); assertThat(date2.getMonthString(), nullValue()); assertThat(date2.getYearString(), is("1939")); }
@Test public void processing_englishStandardDate3_shouldWork() throws Exception { List<Date> output = target.processing("16.06.1942-28.04.1943"); assertThat(output, hasSize(2)); final Date date1 = output.get(0); assertThat(date1.getDay(), is(16)); assertThat(date1.getMonth(), is(6)); assertThat(date1.getYear(), is(1942)); assertThat(date1.getDayString(), is("16")); assertThat(date1.getMonthString(), is("06")); assertThat(date1.getYearString(), is("1942")); final Date date2 = output.get(1); assertThat(date2.getDay(), is(28)); assertThat(date2.getMonth(), is(4)); assertThat(date2.getYear(), is(1943)); assertThat(date2.getDayString(), is("28")); assertThat(date2.getMonthString(), is("04")); assertThat(date2.getYearString(), is("1943")); }
@Ignore("Need more training data, perhaps") @Test public void processing_englishStandardDate4_shouldWork() throws Exception { List<Date> output = target.processing("4.01.1943-21.10.1943"); assertThat(output, hasSize(2)); final Date date1 = output.get(0); assertThat(date1.getDay(), is(4)); assertThat(date1.getMonth(), is(1)); assertThat(date1.getYear(), is(1943)); assertThat(date1.getDayString(), is("4")); assertThat(date1.getMonthString(), is("1")); assertThat(date1.getYearString(), is("1943")); final Date date2 = output.get(1); assertThat(date2.getDay(), is(21)); assertThat(date2.getMonth(), is(10)); assertThat(date2.getYear(), is(1943)); assertThat(date2.getDayString(), is("21")); assertThat(date2.getMonthString(), is("04")); assertThat(date2.getYearString(), is("1943")); }
@Test public void processing_englishStandardDate5_shouldWork() throws Exception { List<Date> output = target.processing("12.03.1942-10.1943"); assertThat(output, hasSize(2)); final Date date1 = output.get(0); assertThat(date1.getDay(), is(12)); assertThat(date1.getMonth(), is(3)); assertThat(date1.getYear(), is(1942)); assertThat(date1.getDayString(), is("12")); assertThat(date1.getMonthString(), is("03")); assertThat(date1.getYearString(), is("1942")); final Date date2 = output.get(1); assertThat(date2.getDay(), is(-1)); assertThat(date2.getMonth(), is(10)); assertThat(date2.getYear(), is(1943)); assertThat(date2.getDayString(), nullValue()); assertThat(date2.getMonthString(), is("10")); assertThat(date2.getYearString(), is("1943")); }
@Ignore("Need more training data, perhaps") @Test public void processing_englishStandardDate6_shouldWork() throws Exception { List<Date> output = target.processing("1941-45"); assertThat(output, hasSize(2)); final Date date1 = output.get(0); assertThat(date1.getDay(), is(-1)); assertThat(date1.getMonth(), is(-1)); assertThat(date1.getYear(), is(1941)); assertThat(date1.getDayString(), nullValue()); assertThat(date1.getMonthString(), nullValue()); assertThat(date1.getYearString(), is("1941")); final Date date2 = output.get(1); assertThat(date2.getDay(), is(-1)); assertThat(date2.getMonth(), is(-1)); assertThat(date2.getYear(), is(45)); assertThat(date2.getDayString(), nullValue()); assertThat(date2.getMonthString(), nullValue()); assertThat(date2.getYearString(), is("45")); }
@Test public void processing_englishStandardDate7_shouldWork() throws Exception { List<Date> output = target.processing("2015-10-21"); assertThat(output, hasSize(1)); final Date date1 = output.get(0); assertThat(date1.getDay(), is(21)); assertThat(date1.getMonth(), is(10)); assertThat(date1.getYear(), is(2015)); assertThat(date1.getDayString(), is("" + date1.getDay())); assertThat(date1.getMonthString(), is("" + date1.getMonth())); assertThat(date1.getYearString(), is("" + date1.getYear())); }
@Test public void processing_englishStandardDate9_shouldWork() throws Exception { List<Date> output = target.processing("2015-10-21 10-12-2016"); assertThat(output, hasSize(2)); final Date date1 = output.get(0); assertThat(date1.getDay(), is(21)); assertThat(date1.getMonth(), is(10)); assertThat(date1.getYear(), is(2015)); assertThat(date1.getDayString(), is("" + date1.getDay())); assertThat(date1.getMonthString(), is("" + date1.getMonth())); assertThat(date1.getYearString(), is("" + date1.getYear())); final Date date2 = output.get(1); assertThat(date2.getDay(), is(10)); assertThat(date2.getMonth(), is(12)); assertThat(date2.getYear(), is(2016)); assertThat(date2.getDayString(), is("" + date2.getDay())); assertThat(date2.getMonthString(), is("" + date2.getMonth())); assertThat(date2.getYearString(), is("" + date2.getYear())); }
@Test public void testMayAndMarchOverlap() throws Exception { DateParser dateParser = new DateParser(); { List<Date> dates = dateParser.processing("May 2003"); assertEquals(1, dates.size()); Date date = dates.get(0); assertEquals(2003, date.getYear()); assertEquals(5, date.getMonth()); } { List<Date> dates = dateParser.processing("Mar 2003"); assertEquals(1, dates.size()); Date date = dates.get(0); assertEquals(2003, date.getYear()); assertEquals(3, date.getMonth()); } }