@Test void tokenize() { String text = ",aa,,fo\"oooo\",\"bar\",co\"o'l,e\"c,df'hk,lm',"; List<String> tokens = Utils.tokenize(',', null, false, text); assertThat(tokens, contains("aa", "fo\"oooo\"", "\"bar\"", "co\"o'l", "e\"c", "df'hk", "lm'")); tokens = Utils.tokenize(',', null, true, text); assertThat(tokens, contains("", "aa", "", "fo\"oooo\"", "\"bar\"", "co\"o'l", "e\"c", "df'hk", "lm'", "")); tokens = Utils.tokenize(',', "\"", false, text); assertThat(tokens, contains("aa", "fo\"oooo\"", "\"bar\"", "co\"o'l,e\"c", "df'hk", "lm'")); tokens = Utils.tokenize(',', "'", false, text); assertThat(tokens, contains("aa", "fo\"oooo\"", "\"bar\"", "co\"o'l,e\"c,df'hk", "lm',")); tokens = Utils.tokenize(',', "\"'", false, text); assertThat(tokens, contains("aa", "fo\"oooo\"", "\"bar\"", "co\"o'l,e\"c", "df'hk,lm'")); tokens = Utils.tokenize(',', "\"'", true, text); assertThat(tokens, contains("", "aa", "", "fo\"oooo\"", "\"bar\"", "co\"o'l,e\"c", "df'hk,lm'", "")); tokens = Utils.tokenize(';', "\"'", true, text); assertThat(tokens, iterableWithSize(1)); assertThat(tokens.get(0), is(text)); }
public static List<String> tokenize(char separator, String quoteChars, boolean includeEmptyTokens, String text) { char[] quotes = quoteChars == null ? new char[0] : quoteChars.toCharArray(); StringBuilder token = new StringBuilder(); List<String> result = new ArrayList<>(); boolean quoted = false; char lastQuoteCharacter = ' '; for (int i = 0; i < text.length(); i++) { char ch = text.charAt(i); if (quoted) { if (ch == lastQuoteCharacter) { quoted = false; } token.append(ch); } else { if (ch == separator) { if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } token.setLength(0); } else { for (char quote : quotes) { if (ch == quote) { quoted = true; lastQuoteCharacter = ch; break; } } token.append(ch); } } } if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } return result; }
Utils { public static List<String> tokenize(char separator, String quoteChars, boolean includeEmptyTokens, String text) { char[] quotes = quoteChars == null ? new char[0] : quoteChars.toCharArray(); StringBuilder token = new StringBuilder(); List<String> result = new ArrayList<>(); boolean quoted = false; char lastQuoteCharacter = ' '; for (int i = 0; i < text.length(); i++) { char ch = text.charAt(i); if (quoted) { if (ch == lastQuoteCharacter) { quoted = false; } token.append(ch); } else { if (ch == separator) { if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } token.setLength(0); } else { for (char quote : quotes) { if (ch == quote) { quoted = true; lastQuoteCharacter = ch; break; } } token.append(ch); } } } if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } return result; } }
Utils { public static List<String> tokenize(char separator, String quoteChars, boolean includeEmptyTokens, String text) { char[] quotes = quoteChars == null ? new char[0] : quoteChars.toCharArray(); StringBuilder token = new StringBuilder(); List<String> result = new ArrayList<>(); boolean quoted = false; char lastQuoteCharacter = ' '; for (int i = 0; i < text.length(); i++) { char ch = text.charAt(i); if (quoted) { if (ch == lastQuoteCharacter) { quoted = false; } token.append(ch); } else { if (ch == separator) { if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } token.setLength(0); } else { for (char quote : quotes) { if (ch == quote) { quoted = true; lastQuoteCharacter = ch; break; } } token.append(ch); } } } if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } return result; } private  Utils(); }
Utils { public static List<String> tokenize(char separator, String quoteChars, boolean includeEmptyTokens, String text) { char[] quotes = quoteChars == null ? new char[0] : quoteChars.toCharArray(); StringBuilder token = new StringBuilder(); List<String> result = new ArrayList<>(); boolean quoted = false; char lastQuoteCharacter = ' '; for (int i = 0; i < text.length(); i++) { char ch = text.charAt(i); if (quoted) { if (ch == lastQuoteCharacter) { quoted = false; } token.append(ch); } else { if (ch == separator) { if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } token.setLength(0); } else { for (char quote : quotes) { if (ch == quote) { quoted = true; lastQuoteCharacter = ch; break; } } token.append(ch); } } } if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } return result; } private  Utils(); static List<String> tokenize(char separator, String quoteChars, boolean includeEmptyTokens, String text); static String unwrap(String str); static void write(ByteBuffer byteBuffer, OutputStream out); static byte[] toByteArray(ByteBuffer byteBuffer); }
Utils { public static List<String> tokenize(char separator, String quoteChars, boolean includeEmptyTokens, String text) { char[] quotes = quoteChars == null ? new char[0] : quoteChars.toCharArray(); StringBuilder token = new StringBuilder(); List<String> result = new ArrayList<>(); boolean quoted = false; char lastQuoteCharacter = ' '; for (int i = 0; i < text.length(); i++) { char ch = text.charAt(i); if (quoted) { if (ch == lastQuoteCharacter) { quoted = false; } token.append(ch); } else { if (ch == separator) { if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } token.setLength(0); } else { for (char quote : quotes) { if (ch == quote) { quoted = true; lastQuoteCharacter = ch; break; } } token.append(ch); } } } if (includeEmptyTokens || token.length() > 0) { result.add(token.toString()); } return result; } private  Utils(); static List<String> tokenize(char separator, String quoteChars, boolean includeEmptyTokens, String text); static String unwrap(String str); static void write(ByteBuffer byteBuffer, OutputStream out); static byte[] toByteArray(ByteBuffer byteBuffer);  }