@Test public void testCoveringAnnotation() throws Exception { TypeSystemDescription tsDesc; TypePriorities typePrios; AnalysisEngine ae; JCas jCas; ParagraphTestType p; SentenceTestType s1; SentenceTestType s2; FeaturePathSets featurePathSets; List<RawToken> tokens; tsDesc = TypeSystemDescriptionFactory .createTypeSystemDescription("de.julielab.jcore.consumer.es.testTypes"); typePrios = TypePrioritiesFactory.createTypePriorities(ParagraphTestType.class, SentenceTestType.class); ae = AnalysisEngineFactory.createEngine(NoOpAnnotator.class, tsDesc, typePrios); jCas = ae.newJCas(); jCas.setDocumentText("A first sentence. And a second sentence."); p = new ParagraphTestType(jCas, 0, 40); s1 = new SentenceTestType(jCas, 0, 17); s2 = new SentenceTestType(jCas, 18, 40); p.addToIndexes(); s1.addToIndexes(); s2.addToIndexes(); featurePathSets = new FeaturePathSets(new FeaturePathSet(SentenceTestType.type, null, null, null)); tokens = getTokensForAnnotationIndexes(featurePathSets, null, false, RawToken.class, p, null, jCas); assertEquals(2, tokens.size()); assertEquals("A first sentence.", tokens.get(0).token); assertEquals("And a second sentence.", tokens.get(1).token); }
@SuppressWarnings("unchecked") public <T extends IToken> List<T> getTokensForAnnotationIndexes(FeaturePathSets featurePathSets, Filter filterForField, boolean sort, Class<T> tokenClass, AnnotationFS coveringAnnotation, DistributedField<? extends IToken> distributedFields, JCas aJCas) throws CASException { List<T> l = null; if (null == distributedFields) l = new ArrayList<>(); if (null != filterForField) filterForField.reset(); LinkedHashSet<Integer> annotationIndexes = new LinkedHashSet<>(featurePathSets.size()); for (Integer type : featurePathSets.getUimaTypes()) annotationIndexes.add(type); AnnotationIndexMerger indexMerger; try { indexMerger = new AnnotationIndexMerger(annotationIndexes, sort, coveringAnnotation, aJCas); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Type could be found in the type system.", e); } int positionIncrement = 1; boolean valueForCurrentOffsetFound = false; while (indexMerger.incrementAnnotation()) { TOP a = indexMerger.getAnnotation(); int start = indexMerger.getCurrentBegin(); int end = indexMerger.getCurrentEnd(); if (null != coveringAnnotation) { start -= coveringAnnotation.getBegin(); end -= coveringAnnotation.getBegin(); } if (sort) { if (indexMerger.hasBeginOffsetChanged()) { if (!valueForCurrentOffsetFound && !indexMerger.firstToken()) positionIncrement++; else positionIncrement = 1; valueForCurrentOffsetFound = false; } else if (!indexMerger.firstToken) { positionIncrement = 0; } } int type = a.getTypeIndexID(); Collection<FeaturePathSet> featurePaths = featurePathSets.getFeaturePathSetsForType(type); if (featurePaths.isEmpty()) featurePaths = featurePathSets.findFeaturePathSetsForSupertypes(a.getType(), aJCas); for (FeaturePathSet featureSet : featurePaths) { Filter filter = featureSet.getFilter(); if (null != filter) filter.reset(); String featureConcatenationString = featureSet.getConcatenationString(); List<List<String>> featureValues = new ArrayList<>(featurePaths.size()); long time = System.currentTimeMillis(); if (null != featureSet.getFeaturePaths() && featureSet.getFeaturePaths().size() > 0) { for (String featurePath : featureSet.getFeaturePaths()) { JCoReFeaturePath fp = getCachedFeaturePath(featurePath); List<String> valueList = fp.getValueAsStringList(a); valueList = applyFilter(valueList, filter); if (!valueList.isEmpty()) featureValues.add(valueList); } } else if (a instanceof Annotation) { try { List<String> value = new ArrayList<>(); value.add(((Annotation) a).getCoveredText()); value = applyFilter(value, filter); featureValues.add(value); } catch (StringIndexOutOfBoundsException e) { log.error( "An annotation occurred with offsets that exceed the document text. The annotation: \n{}\nthe document text length: {}", a, a.getCAS().getDocumentText().length()); throw e; } } else { throw new IllegalArgumentException("No feature path has been given for feature structure " + a + " but since it does not cover any text no field value can be retrieved."); } time = System.currentTimeMillis() - time; valueDeterminationTime += time; time = System.currentTimeMillis(); if (!featureValues.isEmpty() && !StringUtils.isEmpty(featureConcatenationString)) { for (int i = 0; i < featureValues.get(0).size(); ++i) { List<String> parallelFeatureValues = new ArrayList<>(featureValues.size()); for (int j = 0; j < featureValues.size(); ++j) { String v = featureValues.get(j).get(i); if (null != v) parallelFeatureValues.add(v); } String singleConcatValue = StringUtils.join(parallelFeatureValues, featureConcatenationString); List<String> filteredValues = applyFilter(singleConcatValue, filterForField); for (String concatValue : filteredValues) { if (concatValue != null && concatValue.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, concatValue, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(concatValue, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } else if (!featureValues.isEmpty()) { for (int i = 0; i < featureValues.size(); i++) { List<String> valueArray = featureValues.get(i); List<String> filteredList = applyFilter(valueArray, filterForField); for (int j = 0; j < filteredList.size(); j++) { String value = filteredList.get(j); if (value != null && value.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, value, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(value, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } time = System.currentTimeMillis() - time; valueToTokenConversionTime += time; } } return l; }
AbstractFieldGenerator { @SuppressWarnings("unchecked") public <T extends IToken> List<T> getTokensForAnnotationIndexes(FeaturePathSets featurePathSets, Filter filterForField, boolean sort, Class<T> tokenClass, AnnotationFS coveringAnnotation, DistributedField<? extends IToken> distributedFields, JCas aJCas) throws CASException { List<T> l = null; if (null == distributedFields) l = new ArrayList<>(); if (null != filterForField) filterForField.reset(); LinkedHashSet<Integer> annotationIndexes = new LinkedHashSet<>(featurePathSets.size()); for (Integer type : featurePathSets.getUimaTypes()) annotationIndexes.add(type); AnnotationIndexMerger indexMerger; try { indexMerger = new AnnotationIndexMerger(annotationIndexes, sort, coveringAnnotation, aJCas); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Type could be found in the type system.", e); } int positionIncrement = 1; boolean valueForCurrentOffsetFound = false; while (indexMerger.incrementAnnotation()) { TOP a = indexMerger.getAnnotation(); int start = indexMerger.getCurrentBegin(); int end = indexMerger.getCurrentEnd(); if (null != coveringAnnotation) { start -= coveringAnnotation.getBegin(); end -= coveringAnnotation.getBegin(); } if (sort) { if (indexMerger.hasBeginOffsetChanged()) { if (!valueForCurrentOffsetFound && !indexMerger.firstToken()) positionIncrement++; else positionIncrement = 1; valueForCurrentOffsetFound = false; } else if (!indexMerger.firstToken) { positionIncrement = 0; } } int type = a.getTypeIndexID(); Collection<FeaturePathSet> featurePaths = featurePathSets.getFeaturePathSetsForType(type); if (featurePaths.isEmpty()) featurePaths = featurePathSets.findFeaturePathSetsForSupertypes(a.getType(), aJCas); for (FeaturePathSet featureSet : featurePaths) { Filter filter = featureSet.getFilter(); if (null != filter) filter.reset(); String featureConcatenationString = featureSet.getConcatenationString(); List<List<String>> featureValues = new ArrayList<>(featurePaths.size()); long time = System.currentTimeMillis(); if (null != featureSet.getFeaturePaths() && featureSet.getFeaturePaths().size() > 0) { for (String featurePath : featureSet.getFeaturePaths()) { JCoReFeaturePath fp = getCachedFeaturePath(featurePath); List<String> valueList = fp.getValueAsStringList(a); valueList = applyFilter(valueList, filter); if (!valueList.isEmpty()) featureValues.add(valueList); } } else if (a instanceof Annotation) { try { List<String> value = new ArrayList<>(); value.add(((Annotation) a).getCoveredText()); value = applyFilter(value, filter); featureValues.add(value); } catch (StringIndexOutOfBoundsException e) { log.error( "An annotation occurred with offsets that exceed the document text. The annotation: \n{}\nthe document text length: {}", a, a.getCAS().getDocumentText().length()); throw e; } } else { throw new IllegalArgumentException("No feature path has been given for feature structure " + a + " but since it does not cover any text no field value can be retrieved."); } time = System.currentTimeMillis() - time; valueDeterminationTime += time; time = System.currentTimeMillis(); if (!featureValues.isEmpty() && !StringUtils.isEmpty(featureConcatenationString)) { for (int i = 0; i < featureValues.get(0).size(); ++i) { List<String> parallelFeatureValues = new ArrayList<>(featureValues.size()); for (int j = 0; j < featureValues.size(); ++j) { String v = featureValues.get(j).get(i); if (null != v) parallelFeatureValues.add(v); } String singleConcatValue = StringUtils.join(parallelFeatureValues, featureConcatenationString); List<String> filteredValues = applyFilter(singleConcatValue, filterForField); for (String concatValue : filteredValues) { if (concatValue != null && concatValue.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, concatValue, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(concatValue, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } else if (!featureValues.isEmpty()) { for (int i = 0; i < featureValues.size(); i++) { List<String> valueArray = featureValues.get(i); List<String> filteredList = applyFilter(valueArray, filterForField); for (int j = 0; j < filteredList.size(); j++) { String value = filteredList.get(j); if (value != null && value.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, value, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(value, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } time = System.currentTimeMillis() - time; valueToTokenConversionTime += time; } } return l; } }
AbstractFieldGenerator { @SuppressWarnings("unchecked") public <T extends IToken> List<T> getTokensForAnnotationIndexes(FeaturePathSets featurePathSets, Filter filterForField, boolean sort, Class<T> tokenClass, AnnotationFS coveringAnnotation, DistributedField<? extends IToken> distributedFields, JCas aJCas) throws CASException { List<T> l = null; if (null == distributedFields) l = new ArrayList<>(); if (null != filterForField) filterForField.reset(); LinkedHashSet<Integer> annotationIndexes = new LinkedHashSet<>(featurePathSets.size()); for (Integer type : featurePathSets.getUimaTypes()) annotationIndexes.add(type); AnnotationIndexMerger indexMerger; try { indexMerger = new AnnotationIndexMerger(annotationIndexes, sort, coveringAnnotation, aJCas); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Type could be found in the type system.", e); } int positionIncrement = 1; boolean valueForCurrentOffsetFound = false; while (indexMerger.incrementAnnotation()) { TOP a = indexMerger.getAnnotation(); int start = indexMerger.getCurrentBegin(); int end = indexMerger.getCurrentEnd(); if (null != coveringAnnotation) { start -= coveringAnnotation.getBegin(); end -= coveringAnnotation.getBegin(); } if (sort) { if (indexMerger.hasBeginOffsetChanged()) { if (!valueForCurrentOffsetFound && !indexMerger.firstToken()) positionIncrement++; else positionIncrement = 1; valueForCurrentOffsetFound = false; } else if (!indexMerger.firstToken) { positionIncrement = 0; } } int type = a.getTypeIndexID(); Collection<FeaturePathSet> featurePaths = featurePathSets.getFeaturePathSetsForType(type); if (featurePaths.isEmpty()) featurePaths = featurePathSets.findFeaturePathSetsForSupertypes(a.getType(), aJCas); for (FeaturePathSet featureSet : featurePaths) { Filter filter = featureSet.getFilter(); if (null != filter) filter.reset(); String featureConcatenationString = featureSet.getConcatenationString(); List<List<String>> featureValues = new ArrayList<>(featurePaths.size()); long time = System.currentTimeMillis(); if (null != featureSet.getFeaturePaths() && featureSet.getFeaturePaths().size() > 0) { for (String featurePath : featureSet.getFeaturePaths()) { JCoReFeaturePath fp = getCachedFeaturePath(featurePath); List<String> valueList = fp.getValueAsStringList(a); valueList = applyFilter(valueList, filter); if (!valueList.isEmpty()) featureValues.add(valueList); } } else if (a instanceof Annotation) { try { List<String> value = new ArrayList<>(); value.add(((Annotation) a).getCoveredText()); value = applyFilter(value, filter); featureValues.add(value); } catch (StringIndexOutOfBoundsException e) { log.error( "An annotation occurred with offsets that exceed the document text. The annotation: \n{}\nthe document text length: {}", a, a.getCAS().getDocumentText().length()); throw e; } } else { throw new IllegalArgumentException("No feature path has been given for feature structure " + a + " but since it does not cover any text no field value can be retrieved."); } time = System.currentTimeMillis() - time; valueDeterminationTime += time; time = System.currentTimeMillis(); if (!featureValues.isEmpty() && !StringUtils.isEmpty(featureConcatenationString)) { for (int i = 0; i < featureValues.get(0).size(); ++i) { List<String> parallelFeatureValues = new ArrayList<>(featureValues.size()); for (int j = 0; j < featureValues.size(); ++j) { String v = featureValues.get(j).get(i); if (null != v) parallelFeatureValues.add(v); } String singleConcatValue = StringUtils.join(parallelFeatureValues, featureConcatenationString); List<String> filteredValues = applyFilter(singleConcatValue, filterForField); for (String concatValue : filteredValues) { if (concatValue != null && concatValue.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, concatValue, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(concatValue, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } else if (!featureValues.isEmpty()) { for (int i = 0; i < featureValues.size(); i++) { List<String> valueArray = featureValues.get(i); List<String> filteredList = applyFilter(valueArray, filterForField); for (int j = 0; j < filteredList.size(); j++) { String value = filteredList.get(j); if (value != null && value.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, value, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(value, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } time = System.currentTimeMillis() - time; valueToTokenConversionTime += time; } } return l; } @Deprecated  AbstractFieldGenerator(String fieldname); @Deprecated  AbstractFieldGenerator(String fieldname, FilterRegistry filterRegistry);  AbstractFieldGenerator();  AbstractFieldGenerator(FilterRegistry filterRegistry); }
AbstractFieldGenerator { @SuppressWarnings("unchecked") public <T extends IToken> List<T> getTokensForAnnotationIndexes(FeaturePathSets featurePathSets, Filter filterForField, boolean sort, Class<T> tokenClass, AnnotationFS coveringAnnotation, DistributedField<? extends IToken> distributedFields, JCas aJCas) throws CASException { List<T> l = null; if (null == distributedFields) l = new ArrayList<>(); if (null != filterForField) filterForField.reset(); LinkedHashSet<Integer> annotationIndexes = new LinkedHashSet<>(featurePathSets.size()); for (Integer type : featurePathSets.getUimaTypes()) annotationIndexes.add(type); AnnotationIndexMerger indexMerger; try { indexMerger = new AnnotationIndexMerger(annotationIndexes, sort, coveringAnnotation, aJCas); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Type could be found in the type system.", e); } int positionIncrement = 1; boolean valueForCurrentOffsetFound = false; while (indexMerger.incrementAnnotation()) { TOP a = indexMerger.getAnnotation(); int start = indexMerger.getCurrentBegin(); int end = indexMerger.getCurrentEnd(); if (null != coveringAnnotation) { start -= coveringAnnotation.getBegin(); end -= coveringAnnotation.getBegin(); } if (sort) { if (indexMerger.hasBeginOffsetChanged()) { if (!valueForCurrentOffsetFound && !indexMerger.firstToken()) positionIncrement++; else positionIncrement = 1; valueForCurrentOffsetFound = false; } else if (!indexMerger.firstToken) { positionIncrement = 0; } } int type = a.getTypeIndexID(); Collection<FeaturePathSet> featurePaths = featurePathSets.getFeaturePathSetsForType(type); if (featurePaths.isEmpty()) featurePaths = featurePathSets.findFeaturePathSetsForSupertypes(a.getType(), aJCas); for (FeaturePathSet featureSet : featurePaths) { Filter filter = featureSet.getFilter(); if (null != filter) filter.reset(); String featureConcatenationString = featureSet.getConcatenationString(); List<List<String>> featureValues = new ArrayList<>(featurePaths.size()); long time = System.currentTimeMillis(); if (null != featureSet.getFeaturePaths() && featureSet.getFeaturePaths().size() > 0) { for (String featurePath : featureSet.getFeaturePaths()) { JCoReFeaturePath fp = getCachedFeaturePath(featurePath); List<String> valueList = fp.getValueAsStringList(a); valueList = applyFilter(valueList, filter); if (!valueList.isEmpty()) featureValues.add(valueList); } } else if (a instanceof Annotation) { try { List<String> value = new ArrayList<>(); value.add(((Annotation) a).getCoveredText()); value = applyFilter(value, filter); featureValues.add(value); } catch (StringIndexOutOfBoundsException e) { log.error( "An annotation occurred with offsets that exceed the document text. The annotation: \n{}\nthe document text length: {}", a, a.getCAS().getDocumentText().length()); throw e; } } else { throw new IllegalArgumentException("No feature path has been given for feature structure " + a + " but since it does not cover any text no field value can be retrieved."); } time = System.currentTimeMillis() - time; valueDeterminationTime += time; time = System.currentTimeMillis(); if (!featureValues.isEmpty() && !StringUtils.isEmpty(featureConcatenationString)) { for (int i = 0; i < featureValues.get(0).size(); ++i) { List<String> parallelFeatureValues = new ArrayList<>(featureValues.size()); for (int j = 0; j < featureValues.size(); ++j) { String v = featureValues.get(j).get(i); if (null != v) parallelFeatureValues.add(v); } String singleConcatValue = StringUtils.join(parallelFeatureValues, featureConcatenationString); List<String> filteredValues = applyFilter(singleConcatValue, filterForField); for (String concatValue : filteredValues) { if (concatValue != null && concatValue.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, concatValue, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(concatValue, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } else if (!featureValues.isEmpty()) { for (int i = 0; i < featureValues.size(); i++) { List<String> valueArray = featureValues.get(i); List<String> filteredList = applyFilter(valueArray, filterForField); for (int j = 0; j < filteredList.size(); j++) { String value = filteredList.get(j); if (value != null && value.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, value, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(value, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } time = System.currentTimeMillis() - time; valueToTokenConversionTime += time; } } return l; } @Deprecated  AbstractFieldGenerator(String fieldname); @Deprecated  AbstractFieldGenerator(String fieldname, FilterRegistry filterRegistry);  AbstractFieldGenerator();  AbstractFieldGenerator(FilterRegistry filterRegistry); void setFilterRegistry(FilterRegistry filterRegistry); @SuppressWarnings("unchecked") List<T> getTokensForAnnotationIndexes(FeaturePathSets featurePathSets,
			Filter filterForField, boolean sort, Class<T> tokenClass, AnnotationFS coveringAnnotation,
			DistributedField<? extends IToken> distributedFields, JCas aJCas); @SuppressWarnings("unchecked") T createRawFieldValueForString(String input, Filter f); ArrayFieldValue createRawArrayFieldValue(Collection<T> fieldValues); @SuppressWarnings("unchecked") ArrayFieldValue createRawArrayFieldValue(Collection<T> fieldValues, Filter filter); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] fs, String featurePath, Filter filter,
			Filter overallFilter); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] fs, String featurePath, Filter filter); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] fs, String featurePath); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] a, String[] featurePaths); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] a, String[] featurePaths,
			Filter[] filters); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] fss, String[] featurePaths,
			Filter[] filters, Filter overallFilter); ArrayFieldValue createRawFieldValueForAnnotation(FeatureStructure annotation, String[] featurePaths,
			Filter[] filters); IFieldValue createRawFieldValueForAnnotation(FeatureStructure fs, String featurePath, Filter f); PreanalyzedFieldValue createPreanalyzedFieldValue(String stringValue, List<PreanalyzedToken> tokens); PreanalyzedToken createPreanalyzedTokenInTokenSequence(List<?> fieldValueList, String term, int start,
			int end, int posIncr, byte[] payload, String type, int flags); String getFieldname(); void setFieldname(String fieldname); void addFieldSubgenerator(FieldGenerator subgenerator); FieldGenerator getFieldSubgenerator(Class<? extends FieldGenerator> generatorClass); void addInnerDocumentGenerator(FieldValueGenerator subgenerator); FieldValueGenerator getInnerDocumentGenerator(Class<? extends FieldValueGenerator> generatorClass); @Deprecated void addSubgeneratorFeatureStructure(Object key, TOP featureStructure); Map<Class<?>, List<TOP>> getSubgeneratorFS(); }
AbstractFieldGenerator { @SuppressWarnings("unchecked") public <T extends IToken> List<T> getTokensForAnnotationIndexes(FeaturePathSets featurePathSets, Filter filterForField, boolean sort, Class<T> tokenClass, AnnotationFS coveringAnnotation, DistributedField<? extends IToken> distributedFields, JCas aJCas) throws CASException { List<T> l = null; if (null == distributedFields) l = new ArrayList<>(); if (null != filterForField) filterForField.reset(); LinkedHashSet<Integer> annotationIndexes = new LinkedHashSet<>(featurePathSets.size()); for (Integer type : featurePathSets.getUimaTypes()) annotationIndexes.add(type); AnnotationIndexMerger indexMerger; try { indexMerger = new AnnotationIndexMerger(annotationIndexes, sort, coveringAnnotation, aJCas); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Type could be found in the type system.", e); } int positionIncrement = 1; boolean valueForCurrentOffsetFound = false; while (indexMerger.incrementAnnotation()) { TOP a = indexMerger.getAnnotation(); int start = indexMerger.getCurrentBegin(); int end = indexMerger.getCurrentEnd(); if (null != coveringAnnotation) { start -= coveringAnnotation.getBegin(); end -= coveringAnnotation.getBegin(); } if (sort) { if (indexMerger.hasBeginOffsetChanged()) { if (!valueForCurrentOffsetFound && !indexMerger.firstToken()) positionIncrement++; else positionIncrement = 1; valueForCurrentOffsetFound = false; } else if (!indexMerger.firstToken) { positionIncrement = 0; } } int type = a.getTypeIndexID(); Collection<FeaturePathSet> featurePaths = featurePathSets.getFeaturePathSetsForType(type); if (featurePaths.isEmpty()) featurePaths = featurePathSets.findFeaturePathSetsForSupertypes(a.getType(), aJCas); for (FeaturePathSet featureSet : featurePaths) { Filter filter = featureSet.getFilter(); if (null != filter) filter.reset(); String featureConcatenationString = featureSet.getConcatenationString(); List<List<String>> featureValues = new ArrayList<>(featurePaths.size()); long time = System.currentTimeMillis(); if (null != featureSet.getFeaturePaths() && featureSet.getFeaturePaths().size() > 0) { for (String featurePath : featureSet.getFeaturePaths()) { JCoReFeaturePath fp = getCachedFeaturePath(featurePath); List<String> valueList = fp.getValueAsStringList(a); valueList = applyFilter(valueList, filter); if (!valueList.isEmpty()) featureValues.add(valueList); } } else if (a instanceof Annotation) { try { List<String> value = new ArrayList<>(); value.add(((Annotation) a).getCoveredText()); value = applyFilter(value, filter); featureValues.add(value); } catch (StringIndexOutOfBoundsException e) { log.error( "An annotation occurred with offsets that exceed the document text. The annotation: \n{}\nthe document text length: {}", a, a.getCAS().getDocumentText().length()); throw e; } } else { throw new IllegalArgumentException("No feature path has been given for feature structure " + a + " but since it does not cover any text no field value can be retrieved."); } time = System.currentTimeMillis() - time; valueDeterminationTime += time; time = System.currentTimeMillis(); if (!featureValues.isEmpty() && !StringUtils.isEmpty(featureConcatenationString)) { for (int i = 0; i < featureValues.get(0).size(); ++i) { List<String> parallelFeatureValues = new ArrayList<>(featureValues.size()); for (int j = 0; j < featureValues.size(); ++j) { String v = featureValues.get(j).get(i); if (null != v) parallelFeatureValues.add(v); } String singleConcatValue = StringUtils.join(parallelFeatureValues, featureConcatenationString); List<String> filteredValues = applyFilter(singleConcatValue, filterForField); for (String concatValue : filteredValues) { if (concatValue != null && concatValue.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, concatValue, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(concatValue)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(concatValue, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } else if (!featureValues.isEmpty()) { for (int i = 0; i < featureValues.size(); i++) { List<String> valueArray = featureValues.get(i); List<String> filteredList = applyFilter(valueArray, filterForField); for (int j = 0; j < filteredList.size(); j++) { String value = filteredList.get(j); if (value != null && value.length() > 0) { if (null == distributedFields) { if (tokenClass.equals(RawToken.class)) l.add((T) new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) l.add((T) createPreanalyzedTokenInTokenSequence(l, value, start, end, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } else { if (tokenClass.equals(RawToken.class)) distributedFields.addTokenForTerm(new RawToken(value)); else if (tokenClass.equals(PreanalyzedToken.class)) distributedFields.addTokenForTerm(createPreanalyzedToken(value, -1, -1, positionIncrement, null, null, 0)); else throw new IllegalArgumentException( "Token class " + tokenClass + " is not supported."); } valueForCurrentOffsetFound = true; positionIncrement = 0; } } } } time = System.currentTimeMillis() - time; valueToTokenConversionTime += time; } } return l; } @Deprecated  AbstractFieldGenerator(String fieldname); @Deprecated  AbstractFieldGenerator(String fieldname, FilterRegistry filterRegistry);  AbstractFieldGenerator();  AbstractFieldGenerator(FilterRegistry filterRegistry); void setFilterRegistry(FilterRegistry filterRegistry); @SuppressWarnings("unchecked") List<T> getTokensForAnnotationIndexes(FeaturePathSets featurePathSets,
			Filter filterForField, boolean sort, Class<T> tokenClass, AnnotationFS coveringAnnotation,
			DistributedField<? extends IToken> distributedFields, JCas aJCas); @SuppressWarnings("unchecked") T createRawFieldValueForString(String input, Filter f); ArrayFieldValue createRawArrayFieldValue(Collection<T> fieldValues); @SuppressWarnings("unchecked") ArrayFieldValue createRawArrayFieldValue(Collection<T> fieldValues, Filter filter); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] fs, String featurePath, Filter filter,
			Filter overallFilter); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] fs, String featurePath, Filter filter); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] fs, String featurePath); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] a, String[] featurePaths); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] a, String[] featurePaths,
			Filter[] filters); ArrayFieldValue createRawFieldValueForAnnotations(FeatureStructure[] fss, String[] featurePaths,
			Filter[] filters, Filter overallFilter); ArrayFieldValue createRawFieldValueForAnnotation(FeatureStructure annotation, String[] featurePaths,
			Filter[] filters); IFieldValue createRawFieldValueForAnnotation(FeatureStructure fs, String featurePath, Filter f); PreanalyzedFieldValue createPreanalyzedFieldValue(String stringValue, List<PreanalyzedToken> tokens); PreanalyzedToken createPreanalyzedTokenInTokenSequence(List<?> fieldValueList, String term, int start,
			int end, int posIncr, byte[] payload, String type, int flags); String getFieldname(); void setFieldname(String fieldname); void addFieldSubgenerator(FieldGenerator subgenerator); FieldGenerator getFieldSubgenerator(Class<? extends FieldGenerator> generatorClass); void addInnerDocumentGenerator(FieldValueGenerator subgenerator); FieldValueGenerator getInnerDocumentGenerator(Class<? extends FieldValueGenerator> generatorClass); @Deprecated void addSubgeneratorFeatureStructure(Object key, TOP featureStructure); Map<Class<?>, List<TOP>> getSubgeneratorFS();  }