@Test public void testWhitespaceSplitterGetInputFieldValue() throws Exception { Schema input = Schema.recordOf("input", Schema.Field.of("offset", Schema.of(Schema.Type.INT)), Schema.Field.of("body", Schema.of(Schema.Type.STRING))); StructuredRecord record = StructuredRecord.builder(input).set("offset", 1).set("body", "Hi heard about Spark") .build(); Splitter splitter = Splitter.on(Pattern.compile("\\s+")); List<String> expected = new ArrayList<>(); expected.add("Hi"); expected.add("heard"); expected.add("about"); expected.add("Spark"); Assert.assertEquals(expected, SparkUtils.getInputFieldValue(record, "body", splitter)); record = StructuredRecord.builder(input).set("offset", 2).set("body", "Classes in Java").build(); expected = new ArrayList<>(); expected.add("Classes"); expected.add("in"); expected.add("Java"); Assert.assertEquals(expected, SparkUtils.getInputFieldValue(record, "body", splitter)); }
public static List<String> getInputFieldValue(StructuredRecord input, String inputField, Splitter splitter) { List<String> text = new ArrayList<>(); Schema schema = input.getSchema().getField(inputField).getSchema(); Schema.Type type = schema.isNullable() ? schema.getNonNullable().getType() : schema.getType(); try { if (type == Schema.Type.ARRAY) { Object value = input.get(inputField); if (value instanceof List) { text = input.get(inputField); } else { text = Lists.newArrayList((String[]) value); } } else { String value = input.get(inputField); if (value != null) { text = Lists.newArrayList(splitter.split(value)); } } } catch (ClassCastException e) { throw new IllegalArgumentException(String.format("Schema type mismatch for field %s. Please make sure the " + "value to be used for feature generation is an array of " + "string or a string.", inputField), e); } return text; }
SparkUtils { public static List<String> getInputFieldValue(StructuredRecord input, String inputField, Splitter splitter) { List<String> text = new ArrayList<>(); Schema schema = input.getSchema().getField(inputField).getSchema(); Schema.Type type = schema.isNullable() ? schema.getNonNullable().getType() : schema.getType(); try { if (type == Schema.Type.ARRAY) { Object value = input.get(inputField); if (value instanceof List) { text = input.get(inputField); } else { text = Lists.newArrayList((String[]) value); } } else { String value = input.get(inputField); if (value != null) { text = Lists.newArrayList(splitter.split(value)); } } } catch (ClassCastException e) { throw new IllegalArgumentException(String.format("Schema type mismatch for field %s. Please make sure the " + "value to be used for feature generation is an array of " + "string or a string.", inputField), e); } return text; } }
SparkUtils { public static List<String> getInputFieldValue(StructuredRecord input, String inputField, Splitter splitter) { List<String> text = new ArrayList<>(); Schema schema = input.getSchema().getField(inputField).getSchema(); Schema.Type type = schema.isNullable() ? schema.getNonNullable().getType() : schema.getType(); try { if (type == Schema.Type.ARRAY) { Object value = input.get(inputField); if (value instanceof List) { text = input.get(inputField); } else { text = Lists.newArrayList((String[]) value); } } else { String value = input.get(inputField); if (value != null) { text = Lists.newArrayList(splitter.split(value)); } } } catch (ClassCastException e) { throw new IllegalArgumentException(String.format("Schema type mismatch for field %s. Please make sure the " + "value to be used for feature generation is an array of " + "string or a string.", inputField), e); } return text; } private  SparkUtils(); }
SparkUtils { public static List<String> getInputFieldValue(StructuredRecord input, String inputField, Splitter splitter) { List<String> text = new ArrayList<>(); Schema schema = input.getSchema().getField(inputField).getSchema(); Schema.Type type = schema.isNullable() ? schema.getNonNullable().getType() : schema.getType(); try { if (type == Schema.Type.ARRAY) { Object value = input.get(inputField); if (value instanceof List) { text = input.get(inputField); } else { text = Lists.newArrayList((String[]) value); } } else { String value = input.get(inputField); if (value != null) { text = Lists.newArrayList(splitter.split(value)); } } } catch (ClassCastException e) { throw new IllegalArgumentException(String.format("Schema type mismatch for field %s. Please make sure the " + "value to be used for feature generation is an array of " + "string or a string.", inputField), e); } return text; } private  SparkUtils(); static void validateConfigParameters(Schema inputSchema, @Nullable String featuresToInclude,
                                       @Nullable String featuresToExclude, String predictionField,
                                       @Nullable String cardinalityMapping); static Map<String, Integer> getFeatureList(Schema inputSchema, @Nullable String featuresToInclude,
                                             @Nullable String featuresToExclude, String predictionField); static Map<Integer, Integer> getCategoricalFeatureInfo(Schema inputSchema,
                                                         @Nullable String featuresToInclude,
                                                         @Nullable String featuresToExclude,
                                                         String labelField,
                                                         @Nullable String cardinalityMapping); static void validateLabelFieldForTrainer(Schema inputSchema, String labelField); static StructuredRecord.Builder cloneRecord(StructuredRecord record, Schema outputSchema,
                                              String predictionField); static Schema getOutputSchema(Schema inputSchema, String predictionField); static void validateFeatureGeneratorConfig(Schema inputSchema, Map<String, String> map, String pattern); static void validateTextField(Schema inputSchema, String key); static List<String> getInputFieldValue(StructuredRecord input, String inputField, Splitter splitter); }
SparkUtils { public static List<String> getInputFieldValue(StructuredRecord input, String inputField, Splitter splitter) { List<String> text = new ArrayList<>(); Schema schema = input.getSchema().getField(inputField).getSchema(); Schema.Type type = schema.isNullable() ? schema.getNonNullable().getType() : schema.getType(); try { if (type == Schema.Type.ARRAY) { Object value = input.get(inputField); if (value instanceof List) { text = input.get(inputField); } else { text = Lists.newArrayList((String[]) value); } } else { String value = input.get(inputField); if (value != null) { text = Lists.newArrayList(splitter.split(value)); } } } catch (ClassCastException e) { throw new IllegalArgumentException(String.format("Schema type mismatch for field %s. Please make sure the " + "value to be used for feature generation is an array of " + "string or a string.", inputField), e); } return text; } private  SparkUtils(); static void validateConfigParameters(Schema inputSchema, @Nullable String featuresToInclude,
                                       @Nullable String featuresToExclude, String predictionField,
                                       @Nullable String cardinalityMapping); static Map<String, Integer> getFeatureList(Schema inputSchema, @Nullable String featuresToInclude,
                                             @Nullable String featuresToExclude, String predictionField); static Map<Integer, Integer> getCategoricalFeatureInfo(Schema inputSchema,
                                                         @Nullable String featuresToInclude,
                                                         @Nullable String featuresToExclude,
                                                         String labelField,
                                                         @Nullable String cardinalityMapping); static void validateLabelFieldForTrainer(Schema inputSchema, String labelField); static StructuredRecord.Builder cloneRecord(StructuredRecord record, Schema outputSchema,
                                              String predictionField); static Schema getOutputSchema(Schema inputSchema, String predictionField); static void validateFeatureGeneratorConfig(Schema inputSchema, Map<String, String> map, String pattern); static void validateTextField(Schema inputSchema, String key); static List<String> getInputFieldValue(StructuredRecord input, String inputField, Splitter splitter);  }