@Test public void testOldDirectivesWithNewSyntax() throws Exception { String recipe = "parse-as-csv :body '\t' true; drop :body;"; GrammarMigrator migrator = new MigrateToV2(recipe); migrator.migrate(); }
@Override public String migrate() throws DirectiveParseException { List<String> transformed = new ArrayList<>(); int lineno = 1; for (String directive : recipe) { directive = directive.trim(); if (directive.isEmpty() || directive.startsWith(" || (directive.startsWith("#") && !directive.startsWith("#pragma"))) { continue; } if (directive.contains("exp:") || directive.contains("prop:")) { if (directive.endsWith(";")) { transformed.add(directive); } else { transformed.add(directive + ";"); } continue; } if (directive.startsWith("#pragma")) { transformed.add(directive); continue; } if (directive.endsWith(";")) { directive = directive.substring(0, directive.length() - 1); } StringTokenizer tokenizer = new StringTokenizer(directive, " "); String command = tokenizer.nextToken(); switch (command) { case "set": { switch (tokenizer.nextToken()) { case "column": { String column = getNextToken(tokenizer, "set column", "column-name", lineno); String expr = getNextToken(tokenizer, "\n", "set column", "jexl-expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "columns": { String columns = getNextToken(tokenizer, "\n", "set columns", "name1, name2, ...", lineno); String cols[] = columns.split(","); transformed.add(String.format("set-headers %s;", toColumArray(cols))); } break; } } break; case "rename": { String oldcol = getNextToken(tokenizer, command, "old", lineno); String newcol = getNextToken(tokenizer, command, "new", lineno); transformed.add(String.format("rename %s %s;", col(oldcol), col(newcol))); } break; case "set-type": { String col = getNextToken(tokenizer, command, "col", lineno); String type = getNextToken(tokenizer, command, "type", lineno); transformed.add(String.format("set-type %s %s;", col(col), type)); } break; case "drop": { String columns = getNextToken(tokenizer, command, "column", lineno); String cols[] = columns.split(","); transformed.add(String.format("drop %s;", toColumArray(cols))); } break; case "merge": { String col1 = getNextToken(tokenizer, command, "first", lineno); String col2 = getNextToken(tokenizer, command, "second", lineno); String dest = getNextToken(tokenizer, command, "new-column", lineno); String delimiter = getNextToken(tokenizer, "\n", command, "delimiter", lineno); transformed.add(String.format("merge %s %s %s %s;", col(col1), col(col2), col(dest), quote(delimiter))); } break; case "uppercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("uppercase %s;", col(col))); } break; case "lowercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("lowercase %s;", col(col))); } break; case "titlecase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("titlecase %s;", col(col))); } break; case "indexsplit": { String source = getNextToken(tokenizer, command, "source", lineno); String startStr = getNextToken(tokenizer, command, "start", lineno); String endStr = getNextToken(tokenizer, command, "end", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("indexsplit %s %s %s %s;", col(source), startStr, endStr, col(destination))); } break; case "split": { String source = getNextToken(tokenizer, command, "source-column-name", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String firstCol = getNextToken(tokenizer, command, "new-column-1", lineno); String secondCol = getNextToken(tokenizer, command, "new-column-2", lineno); transformed.add(String.format("split %s %s %s %s;", col(source), quote(delimiter), col(firstCol), col(secondCol))); } break; case "filter-row-if-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-not-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-true": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } break; case "filter-row-if-false": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } break; case "filter-rows-on" : { String cmd = getNextToken(tokenizer, command, "command", lineno); if (cmd.equalsIgnoreCase("condition-false")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } else if (cmd.equalsIgnoreCase("condition-true")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } else if (cmd.equalsIgnoreCase("empty-or-null-columns")) { String columns = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("filter-empty-or-null %s;", toColumArray(columns.split(",")))); } else if (cmd.equalsIgnoreCase("regex-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } else if (cmd.equalsIgnoreCase("regex-not-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } else { throw new DirectiveParseException( "filter-rows-on", String.format("Unknown option '%s' specified at line no %s", cmd, lineno) ); } } break; case "set-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-variable %s exp:{%s};", column, expression)); } break; case "increment-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, command, "value", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("increment-variable %s %s exp:{%s};", column, value, expression)); } break; case "mask-number": { String column = getNextToken(tokenizer, command, "column", lineno); String mask = getNextToken(tokenizer, command, "pattern", lineno); transformed.add(String.format("mask-number %s %s;", col(column), quote(mask))); } break; case "mask-shuffle": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("mask-shuffle %s;", col(column))); } break; case "format-date": { String column = getNextToken(tokenizer, command, "column", 1); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("format-date %s %s;", col(column), quote(format))); } break; case "format-unix-timestamp": { String column = getNextToken(tokenizer, command, "column", lineno); String dstDatePattern = getNextToken(tokenizer, "\n", command, "destination-format", lineno); transformed.add(String.format("format-unix-timestamp %s %s;", col(column), quote(dstDatePattern))); } break; case "quantize": { String column1 = getNextToken(tokenizer, command, "source-column", lineno); String column2 = getNextToken(tokenizer, command, "destination-column", lineno); String ranges = getNextToken(tokenizer, "\n", command, "destination-column", lineno); transformed.add(String.format("quantize %s %s %s;", col(column1), col(column2), ranges)); } break; case "find-and-replace" : { String columns = getNextToken(tokenizer, command, "columns", lineno); String expression = getNextToken(tokenizer, "\n", command, "sed-script", lineno); transformed.add(String.format("find-and-replace %s %s;", toColumArray(columns.split(",")), quote(expression))); } break; case "parse-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimStr = getNextToken(tokenizer, command, "delimiter", lineno); if (delimStr.endsWith(";")) { delimStr = delimStr.substring(0, delimStr.length() - 1); } String hasHeaderLinesOpt = getNextToken(tokenizer, "\n", command, "true|false", lineno, true); transformed.add(String.format("parse-as-csv %s %s %s;", col(column), quote(delimStr), hasHeaderLinesOpt == null ? "" : hasHeaderLinesOpt)); } break; case "parse-as-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-json %s %s;", col(column), depthOpt)); } break; case "parse-as-avro" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String type = getNextToken(tokenizer, command, "type", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-avro %s %s %s %s;", col(column), schemaId, type, versionOpt)); } break; case "parse-as-protobuf" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String recordName = getNextToken(tokenizer, command, "record-name", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-protobuf %s %s %s %s;", col(column), schemaId, quote(recordName), versionOpt)); } break; case "json-path" : { String src = getNextToken(tokenizer, command, "source", lineno); String dest = getNextToken(tokenizer, command, "dest", lineno); String path = getNextToken(tokenizer, "\n", command, "json-path", lineno); transformed.add(String.format("json-path %s %s %s;", col(src), col(dest), quote(path))); } break; case "set-charset" : { String column = getNextToken(tokenizer, command, "column", lineno); String charset = getNextToken(tokenizer, "\n", command, "charset", lineno, true); transformed.add(String.format("set-charset %s %s;", col(column), charset)); } break; case "invoke-http" : { String url = getNextToken(tokenizer, command, "url", lineno); String columnsOpt = getNextToken(tokenizer, command, "columns", lineno); String headers = getNextToken(tokenizer, "\n", command, "headers", lineno, true); transformed.add(String.format("invoke-http %s %s %s;", quote(url), toColumArray(columnsOpt.split(",")), quote(headers))); } break; case "set-record-delim" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String limitStr = getNextToken(tokenizer, "\n", column, "limit", lineno, true); transformed.add(String.format("set-record-delim %s %s %s;", col(column), quote(delimiter), limitStr)); } break; case "parse-as-fixed-length" : { String column = getNextToken(tokenizer, command, "column", lineno); String widthStr = getNextToken(tokenizer, command, "widths", lineno); String padding = getNextToken(tokenizer, "\n", column, "padding", lineno, true); transformed.add(String.format("parse-as-fixed-length %s %s %s;", col(column), widthStr, quote(padding))); } break; case "split-to-rows" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "separator", lineno); transformed.add(String.format("split-to-rows %s %s;", col(column), quote(regex))); } break; case "split-to-columns" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "regex", lineno); transformed.add(String.format("split-to-columns %s %s;", col(column), quote(regex))); } break; case "parse-xml-to-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-xml-to-json %s %s;", col(column), depthOpt)); } break; case "parse-as-xml" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-xml %s;", col(column))); } break; case "xpath" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "xpath-array" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath-array %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "flatten" : { String cols = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("flatten %s;", toColumArray(cols.split(",")))); } break; case "copy" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String forceOpt = getNextToken(tokenizer, "\n", command, "force", lineno, true); if (forceOpt == null || forceOpt.isEmpty()) { transformed.add(String.format("copy %s %s;", col(source), col(destination))); } else { transformed.add(String.format("copy %s %s %s;", col(source), col(destination), forceOpt)); } } break; case "fill-null-or-empty" : { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, "\n", command, "fixed-value", lineno, false); transformed.add(String.format("fill-null-or-empty %s %s;", col(column), quote(value))); } break; case "cut-character" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String range = getNextToken(tokenizer, command, "range", lineno); transformed.add(String.format("cut-character %s %s %s;", col(source), col(destination), quote(range))); } break; case "generate-uuid" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("generate-uuid %s;", col(column))); } break; case "url-encode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-encode %s;", col(column))); } break; case "url-decode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-decode %s;", col(column))); } break; case "parse-as-log" : { String column = getNextToken(tokenizer, command, "column", lineno); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-log %s %s;", col(column), quote(format))); } break; case "parse-as-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String timezone = getNextToken(tokenizer, "\n", command, "timezone", lineno, true); transformed.add(String.format("parse-as-date %s %s;", col(column), quote(timezone))); } break; case "parse-as-simple-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-simple-date %s %s;", col(column), quote(pattern))); } break; case "diff-date" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destColumn = getNextToken(tokenizer, "\n", command, "destColumn", lineno); transformed.add(String.format("diff-date %s %s %s;", col(column1), col(column2), col(destColumn))); } break; case "keep" : { String columns = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("keep %s;", toColumArray(columns.split(",")))); } break; case "parse-as-hl7" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-hl7 %s %s;", col(column), depthOpt)); } break; case "split-email" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-email %s;", col(column))); } break; case "swap" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); transformed.add(String.format("swap %s %s;", col(column1), col(column2))); } break; case "hash" : { String column = getNextToken(tokenizer, command, "column", lineno); String algorithm = getNextToken(tokenizer, command, "algorithm", lineno); String encodeOpt = getNextToken(tokenizer, "\n", command, "encode", lineno, true); transformed.add(String.format("hash %s %s %s;", col(column), quote(algorithm), encodeOpt)); } break; case "write-as-json-map" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-json-map %s;", col(column))); } break; case "write-as-json-object" : { String column = getNextToken(tokenizer, command, "column", lineno); String columnsStr = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("write-as-json-object %s %s;", col(column), toColumArray(columnsStr.split(",")))); } break; case "write-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-csv %s;", col(column))); } break; case "parse-as-avro-file": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-avro-file %s;", col(column))); } break; case "send-to-error": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("send-to-error exp:{%s};", condition)); } break; case "fail": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("fail exp:{%s};", condition)); } break; case "text-distance" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-distance %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "text-metric" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-metric %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "catalog-lookup" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("catalog-lookup %s %s;", quote(type), col(column))); } break; case "table-lookup" : { String column = getNextToken(tokenizer, command, "column", lineno); String table = getNextToken(tokenizer, command, "table", lineno); transformed.add(String.format("table-lookup %s %s;", col(column), quote(table))); } break; case "stemming" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("stemming %s;", col(column))); } break; case "columns-replace" : { String sed = getNextToken(tokenizer, command, "sed-expression", lineno); transformed.add(String.format("columns-replace %s;", quote(sed))); } break; case "extract-regex-groups" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, command, "regex", lineno); transformed.add(String.format("extract-regex-groups %s %s;", col(column), quote(regex))); } break; case "split-url" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-url %s;", col(column))); } break; case "cleanse-column-names" : { transformed.add("cleanse-column-names;"); } break; case "change-column-case" : { String casing = getNextToken(tokenizer, command, "case", lineno); transformed.add(String.format("change-column-case %s;", casing)); } break; case "set-column" : { String column = getNextToken(tokenizer, command, "column", lineno); String expr = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "encode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("encode %s %s;", quote(type), col(column))); } break; case "decode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("decode %s %s;", quote(type), col(column))); } break; case "trim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("trim %s;", col(col))); } break; case "ltrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("ltrim %s;", col(col))); } break; case "rtrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("rtrim %s;", col(col))); } break; default: if (!directive.endsWith(";") && !directive.startsWith("$")) { transformed.add(directive + ";"); } else { transformed.add(directive); } break; } lineno = lineno + 1; } return Joiner.on('\n').join(transformed); }
MigrateToV2 implements GrammarMigrator { @Override public String migrate() throws DirectiveParseException { List<String> transformed = new ArrayList<>(); int lineno = 1; for (String directive : recipe) { directive = directive.trim(); if (directive.isEmpty() || directive.startsWith(" || (directive.startsWith("#") && !directive.startsWith("#pragma"))) { continue; } if (directive.contains("exp:") || directive.contains("prop:")) { if (directive.endsWith(";")) { transformed.add(directive); } else { transformed.add(directive + ";"); } continue; } if (directive.startsWith("#pragma")) { transformed.add(directive); continue; } if (directive.endsWith(";")) { directive = directive.substring(0, directive.length() - 1); } StringTokenizer tokenizer = new StringTokenizer(directive, " "); String command = tokenizer.nextToken(); switch (command) { case "set": { switch (tokenizer.nextToken()) { case "column": { String column = getNextToken(tokenizer, "set column", "column-name", lineno); String expr = getNextToken(tokenizer, "\n", "set column", "jexl-expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "columns": { String columns = getNextToken(tokenizer, "\n", "set columns", "name1, name2, ...", lineno); String cols[] = columns.split(","); transformed.add(String.format("set-headers %s;", toColumArray(cols))); } break; } } break; case "rename": { String oldcol = getNextToken(tokenizer, command, "old", lineno); String newcol = getNextToken(tokenizer, command, "new", lineno); transformed.add(String.format("rename %s %s;", col(oldcol), col(newcol))); } break; case "set-type": { String col = getNextToken(tokenizer, command, "col", lineno); String type = getNextToken(tokenizer, command, "type", lineno); transformed.add(String.format("set-type %s %s;", col(col), type)); } break; case "drop": { String columns = getNextToken(tokenizer, command, "column", lineno); String cols[] = columns.split(","); transformed.add(String.format("drop %s;", toColumArray(cols))); } break; case "merge": { String col1 = getNextToken(tokenizer, command, "first", lineno); String col2 = getNextToken(tokenizer, command, "second", lineno); String dest = getNextToken(tokenizer, command, "new-column", lineno); String delimiter = getNextToken(tokenizer, "\n", command, "delimiter", lineno); transformed.add(String.format("merge %s %s %s %s;", col(col1), col(col2), col(dest), quote(delimiter))); } break; case "uppercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("uppercase %s;", col(col))); } break; case "lowercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("lowercase %s;", col(col))); } break; case "titlecase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("titlecase %s;", col(col))); } break; case "indexsplit": { String source = getNextToken(tokenizer, command, "source", lineno); String startStr = getNextToken(tokenizer, command, "start", lineno); String endStr = getNextToken(tokenizer, command, "end", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("indexsplit %s %s %s %s;", col(source), startStr, endStr, col(destination))); } break; case "split": { String source = getNextToken(tokenizer, command, "source-column-name", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String firstCol = getNextToken(tokenizer, command, "new-column-1", lineno); String secondCol = getNextToken(tokenizer, command, "new-column-2", lineno); transformed.add(String.format("split %s %s %s %s;", col(source), quote(delimiter), col(firstCol), col(secondCol))); } break; case "filter-row-if-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-not-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-true": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } break; case "filter-row-if-false": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } break; case "filter-rows-on" : { String cmd = getNextToken(tokenizer, command, "command", lineno); if (cmd.equalsIgnoreCase("condition-false")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } else if (cmd.equalsIgnoreCase("condition-true")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } else if (cmd.equalsIgnoreCase("empty-or-null-columns")) { String columns = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("filter-empty-or-null %s;", toColumArray(columns.split(",")))); } else if (cmd.equalsIgnoreCase("regex-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } else if (cmd.equalsIgnoreCase("regex-not-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } else { throw new DirectiveParseException( "filter-rows-on", String.format("Unknown option '%s' specified at line no %s", cmd, lineno) ); } } break; case "set-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-variable %s exp:{%s};", column, expression)); } break; case "increment-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, command, "value", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("increment-variable %s %s exp:{%s};", column, value, expression)); } break; case "mask-number": { String column = getNextToken(tokenizer, command, "column", lineno); String mask = getNextToken(tokenizer, command, "pattern", lineno); transformed.add(String.format("mask-number %s %s;", col(column), quote(mask))); } break; case "mask-shuffle": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("mask-shuffle %s;", col(column))); } break; case "format-date": { String column = getNextToken(tokenizer, command, "column", 1); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("format-date %s %s;", col(column), quote(format))); } break; case "format-unix-timestamp": { String column = getNextToken(tokenizer, command, "column", lineno); String dstDatePattern = getNextToken(tokenizer, "\n", command, "destination-format", lineno); transformed.add(String.format("format-unix-timestamp %s %s;", col(column), quote(dstDatePattern))); } break; case "quantize": { String column1 = getNextToken(tokenizer, command, "source-column", lineno); String column2 = getNextToken(tokenizer, command, "destination-column", lineno); String ranges = getNextToken(tokenizer, "\n", command, "destination-column", lineno); transformed.add(String.format("quantize %s %s %s;", col(column1), col(column2), ranges)); } break; case "find-and-replace" : { String columns = getNextToken(tokenizer, command, "columns", lineno); String expression = getNextToken(tokenizer, "\n", command, "sed-script", lineno); transformed.add(String.format("find-and-replace %s %s;", toColumArray(columns.split(",")), quote(expression))); } break; case "parse-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimStr = getNextToken(tokenizer, command, "delimiter", lineno); if (delimStr.endsWith(";")) { delimStr = delimStr.substring(0, delimStr.length() - 1); } String hasHeaderLinesOpt = getNextToken(tokenizer, "\n", command, "true|false", lineno, true); transformed.add(String.format("parse-as-csv %s %s %s;", col(column), quote(delimStr), hasHeaderLinesOpt == null ? "" : hasHeaderLinesOpt)); } break; case "parse-as-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-json %s %s;", col(column), depthOpt)); } break; case "parse-as-avro" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String type = getNextToken(tokenizer, command, "type", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-avro %s %s %s %s;", col(column), schemaId, type, versionOpt)); } break; case "parse-as-protobuf" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String recordName = getNextToken(tokenizer, command, "record-name", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-protobuf %s %s %s %s;", col(column), schemaId, quote(recordName), versionOpt)); } break; case "json-path" : { String src = getNextToken(tokenizer, command, "source", lineno); String dest = getNextToken(tokenizer, command, "dest", lineno); String path = getNextToken(tokenizer, "\n", command, "json-path", lineno); transformed.add(String.format("json-path %s %s %s;", col(src), col(dest), quote(path))); } break; case "set-charset" : { String column = getNextToken(tokenizer, command, "column", lineno); String charset = getNextToken(tokenizer, "\n", command, "charset", lineno, true); transformed.add(String.format("set-charset %s %s;", col(column), charset)); } break; case "invoke-http" : { String url = getNextToken(tokenizer, command, "url", lineno); String columnsOpt = getNextToken(tokenizer, command, "columns", lineno); String headers = getNextToken(tokenizer, "\n", command, "headers", lineno, true); transformed.add(String.format("invoke-http %s %s %s;", quote(url), toColumArray(columnsOpt.split(",")), quote(headers))); } break; case "set-record-delim" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String limitStr = getNextToken(tokenizer, "\n", column, "limit", lineno, true); transformed.add(String.format("set-record-delim %s %s %s;", col(column), quote(delimiter), limitStr)); } break; case "parse-as-fixed-length" : { String column = getNextToken(tokenizer, command, "column", lineno); String widthStr = getNextToken(tokenizer, command, "widths", lineno); String padding = getNextToken(tokenizer, "\n", column, "padding", lineno, true); transformed.add(String.format("parse-as-fixed-length %s %s %s;", col(column), widthStr, quote(padding))); } break; case "split-to-rows" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "separator", lineno); transformed.add(String.format("split-to-rows %s %s;", col(column), quote(regex))); } break; case "split-to-columns" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "regex", lineno); transformed.add(String.format("split-to-columns %s %s;", col(column), quote(regex))); } break; case "parse-xml-to-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-xml-to-json %s %s;", col(column), depthOpt)); } break; case "parse-as-xml" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-xml %s;", col(column))); } break; case "xpath" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "xpath-array" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath-array %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "flatten" : { String cols = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("flatten %s;", toColumArray(cols.split(",")))); } break; case "copy" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String forceOpt = getNextToken(tokenizer, "\n", command, "force", lineno, true); if (forceOpt == null || forceOpt.isEmpty()) { transformed.add(String.format("copy %s %s;", col(source), col(destination))); } else { transformed.add(String.format("copy %s %s %s;", col(source), col(destination), forceOpt)); } } break; case "fill-null-or-empty" : { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, "\n", command, "fixed-value", lineno, false); transformed.add(String.format("fill-null-or-empty %s %s;", col(column), quote(value))); } break; case "cut-character" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String range = getNextToken(tokenizer, command, "range", lineno); transformed.add(String.format("cut-character %s %s %s;", col(source), col(destination), quote(range))); } break; case "generate-uuid" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("generate-uuid %s;", col(column))); } break; case "url-encode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-encode %s;", col(column))); } break; case "url-decode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-decode %s;", col(column))); } break; case "parse-as-log" : { String column = getNextToken(tokenizer, command, "column", lineno); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-log %s %s;", col(column), quote(format))); } break; case "parse-as-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String timezone = getNextToken(tokenizer, "\n", command, "timezone", lineno, true); transformed.add(String.format("parse-as-date %s %s;", col(column), quote(timezone))); } break; case "parse-as-simple-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-simple-date %s %s;", col(column), quote(pattern))); } break; case "diff-date" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destColumn = getNextToken(tokenizer, "\n", command, "destColumn", lineno); transformed.add(String.format("diff-date %s %s %s;", col(column1), col(column2), col(destColumn))); } break; case "keep" : { String columns = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("keep %s;", toColumArray(columns.split(",")))); } break; case "parse-as-hl7" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-hl7 %s %s;", col(column), depthOpt)); } break; case "split-email" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-email %s;", col(column))); } break; case "swap" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); transformed.add(String.format("swap %s %s;", col(column1), col(column2))); } break; case "hash" : { String column = getNextToken(tokenizer, command, "column", lineno); String algorithm = getNextToken(tokenizer, command, "algorithm", lineno); String encodeOpt = getNextToken(tokenizer, "\n", command, "encode", lineno, true); transformed.add(String.format("hash %s %s %s;", col(column), quote(algorithm), encodeOpt)); } break; case "write-as-json-map" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-json-map %s;", col(column))); } break; case "write-as-json-object" : { String column = getNextToken(tokenizer, command, "column", lineno); String columnsStr = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("write-as-json-object %s %s;", col(column), toColumArray(columnsStr.split(",")))); } break; case "write-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-csv %s;", col(column))); } break; case "parse-as-avro-file": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-avro-file %s;", col(column))); } break; case "send-to-error": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("send-to-error exp:{%s};", condition)); } break; case "fail": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("fail exp:{%s};", condition)); } break; case "text-distance" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-distance %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "text-metric" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-metric %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "catalog-lookup" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("catalog-lookup %s %s;", quote(type), col(column))); } break; case "table-lookup" : { String column = getNextToken(tokenizer, command, "column", lineno); String table = getNextToken(tokenizer, command, "table", lineno); transformed.add(String.format("table-lookup %s %s;", col(column), quote(table))); } break; case "stemming" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("stemming %s;", col(column))); } break; case "columns-replace" : { String sed = getNextToken(tokenizer, command, "sed-expression", lineno); transformed.add(String.format("columns-replace %s;", quote(sed))); } break; case "extract-regex-groups" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, command, "regex", lineno); transformed.add(String.format("extract-regex-groups %s %s;", col(column), quote(regex))); } break; case "split-url" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-url %s;", col(column))); } break; case "cleanse-column-names" : { transformed.add("cleanse-column-names;"); } break; case "change-column-case" : { String casing = getNextToken(tokenizer, command, "case", lineno); transformed.add(String.format("change-column-case %s;", casing)); } break; case "set-column" : { String column = getNextToken(tokenizer, command, "column", lineno); String expr = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "encode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("encode %s %s;", quote(type), col(column))); } break; case "decode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("decode %s %s;", quote(type), col(column))); } break; case "trim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("trim %s;", col(col))); } break; case "ltrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("ltrim %s;", col(col))); } break; case "rtrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("rtrim %s;", col(col))); } break; default: if (!directive.endsWith(";") && !directive.startsWith("$")) { transformed.add(directive + ";"); } else { transformed.add(directive); } break; } lineno = lineno + 1; } return Joiner.on('\n').join(transformed); } }
MigrateToV2 implements GrammarMigrator { @Override public String migrate() throws DirectiveParseException { List<String> transformed = new ArrayList<>(); int lineno = 1; for (String directive : recipe) { directive = directive.trim(); if (directive.isEmpty() || directive.startsWith(" || (directive.startsWith("#") && !directive.startsWith("#pragma"))) { continue; } if (directive.contains("exp:") || directive.contains("prop:")) { if (directive.endsWith(";")) { transformed.add(directive); } else { transformed.add(directive + ";"); } continue; } if (directive.startsWith("#pragma")) { transformed.add(directive); continue; } if (directive.endsWith(";")) { directive = directive.substring(0, directive.length() - 1); } StringTokenizer tokenizer = new StringTokenizer(directive, " "); String command = tokenizer.nextToken(); switch (command) { case "set": { switch (tokenizer.nextToken()) { case "column": { String column = getNextToken(tokenizer, "set column", "column-name", lineno); String expr = getNextToken(tokenizer, "\n", "set column", "jexl-expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "columns": { String columns = getNextToken(tokenizer, "\n", "set columns", "name1, name2, ...", lineno); String cols[] = columns.split(","); transformed.add(String.format("set-headers %s;", toColumArray(cols))); } break; } } break; case "rename": { String oldcol = getNextToken(tokenizer, command, "old", lineno); String newcol = getNextToken(tokenizer, command, "new", lineno); transformed.add(String.format("rename %s %s;", col(oldcol), col(newcol))); } break; case "set-type": { String col = getNextToken(tokenizer, command, "col", lineno); String type = getNextToken(tokenizer, command, "type", lineno); transformed.add(String.format("set-type %s %s;", col(col), type)); } break; case "drop": { String columns = getNextToken(tokenizer, command, "column", lineno); String cols[] = columns.split(","); transformed.add(String.format("drop %s;", toColumArray(cols))); } break; case "merge": { String col1 = getNextToken(tokenizer, command, "first", lineno); String col2 = getNextToken(tokenizer, command, "second", lineno); String dest = getNextToken(tokenizer, command, "new-column", lineno); String delimiter = getNextToken(tokenizer, "\n", command, "delimiter", lineno); transformed.add(String.format("merge %s %s %s %s;", col(col1), col(col2), col(dest), quote(delimiter))); } break; case "uppercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("uppercase %s;", col(col))); } break; case "lowercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("lowercase %s;", col(col))); } break; case "titlecase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("titlecase %s;", col(col))); } break; case "indexsplit": { String source = getNextToken(tokenizer, command, "source", lineno); String startStr = getNextToken(tokenizer, command, "start", lineno); String endStr = getNextToken(tokenizer, command, "end", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("indexsplit %s %s %s %s;", col(source), startStr, endStr, col(destination))); } break; case "split": { String source = getNextToken(tokenizer, command, "source-column-name", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String firstCol = getNextToken(tokenizer, command, "new-column-1", lineno); String secondCol = getNextToken(tokenizer, command, "new-column-2", lineno); transformed.add(String.format("split %s %s %s %s;", col(source), quote(delimiter), col(firstCol), col(secondCol))); } break; case "filter-row-if-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-not-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-true": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } break; case "filter-row-if-false": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } break; case "filter-rows-on" : { String cmd = getNextToken(tokenizer, command, "command", lineno); if (cmd.equalsIgnoreCase("condition-false")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } else if (cmd.equalsIgnoreCase("condition-true")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } else if (cmd.equalsIgnoreCase("empty-or-null-columns")) { String columns = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("filter-empty-or-null %s;", toColumArray(columns.split(",")))); } else if (cmd.equalsIgnoreCase("regex-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } else if (cmd.equalsIgnoreCase("regex-not-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } else { throw new DirectiveParseException( "filter-rows-on", String.format("Unknown option '%s' specified at line no %s", cmd, lineno) ); } } break; case "set-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-variable %s exp:{%s};", column, expression)); } break; case "increment-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, command, "value", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("increment-variable %s %s exp:{%s};", column, value, expression)); } break; case "mask-number": { String column = getNextToken(tokenizer, command, "column", lineno); String mask = getNextToken(tokenizer, command, "pattern", lineno); transformed.add(String.format("mask-number %s %s;", col(column), quote(mask))); } break; case "mask-shuffle": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("mask-shuffle %s;", col(column))); } break; case "format-date": { String column = getNextToken(tokenizer, command, "column", 1); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("format-date %s %s;", col(column), quote(format))); } break; case "format-unix-timestamp": { String column = getNextToken(tokenizer, command, "column", lineno); String dstDatePattern = getNextToken(tokenizer, "\n", command, "destination-format", lineno); transformed.add(String.format("format-unix-timestamp %s %s;", col(column), quote(dstDatePattern))); } break; case "quantize": { String column1 = getNextToken(tokenizer, command, "source-column", lineno); String column2 = getNextToken(tokenizer, command, "destination-column", lineno); String ranges = getNextToken(tokenizer, "\n", command, "destination-column", lineno); transformed.add(String.format("quantize %s %s %s;", col(column1), col(column2), ranges)); } break; case "find-and-replace" : { String columns = getNextToken(tokenizer, command, "columns", lineno); String expression = getNextToken(tokenizer, "\n", command, "sed-script", lineno); transformed.add(String.format("find-and-replace %s %s;", toColumArray(columns.split(",")), quote(expression))); } break; case "parse-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimStr = getNextToken(tokenizer, command, "delimiter", lineno); if (delimStr.endsWith(";")) { delimStr = delimStr.substring(0, delimStr.length() - 1); } String hasHeaderLinesOpt = getNextToken(tokenizer, "\n", command, "true|false", lineno, true); transformed.add(String.format("parse-as-csv %s %s %s;", col(column), quote(delimStr), hasHeaderLinesOpt == null ? "" : hasHeaderLinesOpt)); } break; case "parse-as-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-json %s %s;", col(column), depthOpt)); } break; case "parse-as-avro" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String type = getNextToken(tokenizer, command, "type", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-avro %s %s %s %s;", col(column), schemaId, type, versionOpt)); } break; case "parse-as-protobuf" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String recordName = getNextToken(tokenizer, command, "record-name", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-protobuf %s %s %s %s;", col(column), schemaId, quote(recordName), versionOpt)); } break; case "json-path" : { String src = getNextToken(tokenizer, command, "source", lineno); String dest = getNextToken(tokenizer, command, "dest", lineno); String path = getNextToken(tokenizer, "\n", command, "json-path", lineno); transformed.add(String.format("json-path %s %s %s;", col(src), col(dest), quote(path))); } break; case "set-charset" : { String column = getNextToken(tokenizer, command, "column", lineno); String charset = getNextToken(tokenizer, "\n", command, "charset", lineno, true); transformed.add(String.format("set-charset %s %s;", col(column), charset)); } break; case "invoke-http" : { String url = getNextToken(tokenizer, command, "url", lineno); String columnsOpt = getNextToken(tokenizer, command, "columns", lineno); String headers = getNextToken(tokenizer, "\n", command, "headers", lineno, true); transformed.add(String.format("invoke-http %s %s %s;", quote(url), toColumArray(columnsOpt.split(",")), quote(headers))); } break; case "set-record-delim" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String limitStr = getNextToken(tokenizer, "\n", column, "limit", lineno, true); transformed.add(String.format("set-record-delim %s %s %s;", col(column), quote(delimiter), limitStr)); } break; case "parse-as-fixed-length" : { String column = getNextToken(tokenizer, command, "column", lineno); String widthStr = getNextToken(tokenizer, command, "widths", lineno); String padding = getNextToken(tokenizer, "\n", column, "padding", lineno, true); transformed.add(String.format("parse-as-fixed-length %s %s %s;", col(column), widthStr, quote(padding))); } break; case "split-to-rows" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "separator", lineno); transformed.add(String.format("split-to-rows %s %s;", col(column), quote(regex))); } break; case "split-to-columns" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "regex", lineno); transformed.add(String.format("split-to-columns %s %s;", col(column), quote(regex))); } break; case "parse-xml-to-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-xml-to-json %s %s;", col(column), depthOpt)); } break; case "parse-as-xml" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-xml %s;", col(column))); } break; case "xpath" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "xpath-array" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath-array %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "flatten" : { String cols = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("flatten %s;", toColumArray(cols.split(",")))); } break; case "copy" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String forceOpt = getNextToken(tokenizer, "\n", command, "force", lineno, true); if (forceOpt == null || forceOpt.isEmpty()) { transformed.add(String.format("copy %s %s;", col(source), col(destination))); } else { transformed.add(String.format("copy %s %s %s;", col(source), col(destination), forceOpt)); } } break; case "fill-null-or-empty" : { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, "\n", command, "fixed-value", lineno, false); transformed.add(String.format("fill-null-or-empty %s %s;", col(column), quote(value))); } break; case "cut-character" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String range = getNextToken(tokenizer, command, "range", lineno); transformed.add(String.format("cut-character %s %s %s;", col(source), col(destination), quote(range))); } break; case "generate-uuid" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("generate-uuid %s;", col(column))); } break; case "url-encode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-encode %s;", col(column))); } break; case "url-decode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-decode %s;", col(column))); } break; case "parse-as-log" : { String column = getNextToken(tokenizer, command, "column", lineno); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-log %s %s;", col(column), quote(format))); } break; case "parse-as-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String timezone = getNextToken(tokenizer, "\n", command, "timezone", lineno, true); transformed.add(String.format("parse-as-date %s %s;", col(column), quote(timezone))); } break; case "parse-as-simple-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-simple-date %s %s;", col(column), quote(pattern))); } break; case "diff-date" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destColumn = getNextToken(tokenizer, "\n", command, "destColumn", lineno); transformed.add(String.format("diff-date %s %s %s;", col(column1), col(column2), col(destColumn))); } break; case "keep" : { String columns = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("keep %s;", toColumArray(columns.split(",")))); } break; case "parse-as-hl7" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-hl7 %s %s;", col(column), depthOpt)); } break; case "split-email" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-email %s;", col(column))); } break; case "swap" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); transformed.add(String.format("swap %s %s;", col(column1), col(column2))); } break; case "hash" : { String column = getNextToken(tokenizer, command, "column", lineno); String algorithm = getNextToken(tokenizer, command, "algorithm", lineno); String encodeOpt = getNextToken(tokenizer, "\n", command, "encode", lineno, true); transformed.add(String.format("hash %s %s %s;", col(column), quote(algorithm), encodeOpt)); } break; case "write-as-json-map" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-json-map %s;", col(column))); } break; case "write-as-json-object" : { String column = getNextToken(tokenizer, command, "column", lineno); String columnsStr = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("write-as-json-object %s %s;", col(column), toColumArray(columnsStr.split(",")))); } break; case "write-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-csv %s;", col(column))); } break; case "parse-as-avro-file": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-avro-file %s;", col(column))); } break; case "send-to-error": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("send-to-error exp:{%s};", condition)); } break; case "fail": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("fail exp:{%s};", condition)); } break; case "text-distance" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-distance %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "text-metric" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-metric %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "catalog-lookup" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("catalog-lookup %s %s;", quote(type), col(column))); } break; case "table-lookup" : { String column = getNextToken(tokenizer, command, "column", lineno); String table = getNextToken(tokenizer, command, "table", lineno); transformed.add(String.format("table-lookup %s %s;", col(column), quote(table))); } break; case "stemming" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("stemming %s;", col(column))); } break; case "columns-replace" : { String sed = getNextToken(tokenizer, command, "sed-expression", lineno); transformed.add(String.format("columns-replace %s;", quote(sed))); } break; case "extract-regex-groups" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, command, "regex", lineno); transformed.add(String.format("extract-regex-groups %s %s;", col(column), quote(regex))); } break; case "split-url" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-url %s;", col(column))); } break; case "cleanse-column-names" : { transformed.add("cleanse-column-names;"); } break; case "change-column-case" : { String casing = getNextToken(tokenizer, command, "case", lineno); transformed.add(String.format("change-column-case %s;", casing)); } break; case "set-column" : { String column = getNextToken(tokenizer, command, "column", lineno); String expr = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "encode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("encode %s %s;", quote(type), col(column))); } break; case "decode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("decode %s %s;", quote(type), col(column))); } break; case "trim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("trim %s;", col(col))); } break; case "ltrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("ltrim %s;", col(col))); } break; case "rtrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("rtrim %s;", col(col))); } break; default: if (!directive.endsWith(";") && !directive.startsWith("$")) { transformed.add(directive + ";"); } else { transformed.add(directive); } break; } lineno = lineno + 1; } return Joiner.on('\n').join(transformed); } MigrateToV2(List<String> recipe);  MigrateToV2(String[] recipe);  MigrateToV2(@Nullable String recipe, String delimiter);  MigrateToV2(String recipe); }
MigrateToV2 implements GrammarMigrator { @Override public String migrate() throws DirectiveParseException { List<String> transformed = new ArrayList<>(); int lineno = 1; for (String directive : recipe) { directive = directive.trim(); if (directive.isEmpty() || directive.startsWith(" || (directive.startsWith("#") && !directive.startsWith("#pragma"))) { continue; } if (directive.contains("exp:") || directive.contains("prop:")) { if (directive.endsWith(";")) { transformed.add(directive); } else { transformed.add(directive + ";"); } continue; } if (directive.startsWith("#pragma")) { transformed.add(directive); continue; } if (directive.endsWith(";")) { directive = directive.substring(0, directive.length() - 1); } StringTokenizer tokenizer = new StringTokenizer(directive, " "); String command = tokenizer.nextToken(); switch (command) { case "set": { switch (tokenizer.nextToken()) { case "column": { String column = getNextToken(tokenizer, "set column", "column-name", lineno); String expr = getNextToken(tokenizer, "\n", "set column", "jexl-expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "columns": { String columns = getNextToken(tokenizer, "\n", "set columns", "name1, name2, ...", lineno); String cols[] = columns.split(","); transformed.add(String.format("set-headers %s;", toColumArray(cols))); } break; } } break; case "rename": { String oldcol = getNextToken(tokenizer, command, "old", lineno); String newcol = getNextToken(tokenizer, command, "new", lineno); transformed.add(String.format("rename %s %s;", col(oldcol), col(newcol))); } break; case "set-type": { String col = getNextToken(tokenizer, command, "col", lineno); String type = getNextToken(tokenizer, command, "type", lineno); transformed.add(String.format("set-type %s %s;", col(col), type)); } break; case "drop": { String columns = getNextToken(tokenizer, command, "column", lineno); String cols[] = columns.split(","); transformed.add(String.format("drop %s;", toColumArray(cols))); } break; case "merge": { String col1 = getNextToken(tokenizer, command, "first", lineno); String col2 = getNextToken(tokenizer, command, "second", lineno); String dest = getNextToken(tokenizer, command, "new-column", lineno); String delimiter = getNextToken(tokenizer, "\n", command, "delimiter", lineno); transformed.add(String.format("merge %s %s %s %s;", col(col1), col(col2), col(dest), quote(delimiter))); } break; case "uppercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("uppercase %s;", col(col))); } break; case "lowercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("lowercase %s;", col(col))); } break; case "titlecase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("titlecase %s;", col(col))); } break; case "indexsplit": { String source = getNextToken(tokenizer, command, "source", lineno); String startStr = getNextToken(tokenizer, command, "start", lineno); String endStr = getNextToken(tokenizer, command, "end", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("indexsplit %s %s %s %s;", col(source), startStr, endStr, col(destination))); } break; case "split": { String source = getNextToken(tokenizer, command, "source-column-name", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String firstCol = getNextToken(tokenizer, command, "new-column-1", lineno); String secondCol = getNextToken(tokenizer, command, "new-column-2", lineno); transformed.add(String.format("split %s %s %s %s;", col(source), quote(delimiter), col(firstCol), col(secondCol))); } break; case "filter-row-if-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-not-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-true": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } break; case "filter-row-if-false": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } break; case "filter-rows-on" : { String cmd = getNextToken(tokenizer, command, "command", lineno); if (cmd.equalsIgnoreCase("condition-false")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } else if (cmd.equalsIgnoreCase("condition-true")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } else if (cmd.equalsIgnoreCase("empty-or-null-columns")) { String columns = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("filter-empty-or-null %s;", toColumArray(columns.split(",")))); } else if (cmd.equalsIgnoreCase("regex-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } else if (cmd.equalsIgnoreCase("regex-not-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } else { throw new DirectiveParseException( "filter-rows-on", String.format("Unknown option '%s' specified at line no %s", cmd, lineno) ); } } break; case "set-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-variable %s exp:{%s};", column, expression)); } break; case "increment-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, command, "value", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("increment-variable %s %s exp:{%s};", column, value, expression)); } break; case "mask-number": { String column = getNextToken(tokenizer, command, "column", lineno); String mask = getNextToken(tokenizer, command, "pattern", lineno); transformed.add(String.format("mask-number %s %s;", col(column), quote(mask))); } break; case "mask-shuffle": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("mask-shuffle %s;", col(column))); } break; case "format-date": { String column = getNextToken(tokenizer, command, "column", 1); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("format-date %s %s;", col(column), quote(format))); } break; case "format-unix-timestamp": { String column = getNextToken(tokenizer, command, "column", lineno); String dstDatePattern = getNextToken(tokenizer, "\n", command, "destination-format", lineno); transformed.add(String.format("format-unix-timestamp %s %s;", col(column), quote(dstDatePattern))); } break; case "quantize": { String column1 = getNextToken(tokenizer, command, "source-column", lineno); String column2 = getNextToken(tokenizer, command, "destination-column", lineno); String ranges = getNextToken(tokenizer, "\n", command, "destination-column", lineno); transformed.add(String.format("quantize %s %s %s;", col(column1), col(column2), ranges)); } break; case "find-and-replace" : { String columns = getNextToken(tokenizer, command, "columns", lineno); String expression = getNextToken(tokenizer, "\n", command, "sed-script", lineno); transformed.add(String.format("find-and-replace %s %s;", toColumArray(columns.split(",")), quote(expression))); } break; case "parse-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimStr = getNextToken(tokenizer, command, "delimiter", lineno); if (delimStr.endsWith(";")) { delimStr = delimStr.substring(0, delimStr.length() - 1); } String hasHeaderLinesOpt = getNextToken(tokenizer, "\n", command, "true|false", lineno, true); transformed.add(String.format("parse-as-csv %s %s %s;", col(column), quote(delimStr), hasHeaderLinesOpt == null ? "" : hasHeaderLinesOpt)); } break; case "parse-as-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-json %s %s;", col(column), depthOpt)); } break; case "parse-as-avro" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String type = getNextToken(tokenizer, command, "type", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-avro %s %s %s %s;", col(column), schemaId, type, versionOpt)); } break; case "parse-as-protobuf" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String recordName = getNextToken(tokenizer, command, "record-name", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-protobuf %s %s %s %s;", col(column), schemaId, quote(recordName), versionOpt)); } break; case "json-path" : { String src = getNextToken(tokenizer, command, "source", lineno); String dest = getNextToken(tokenizer, command, "dest", lineno); String path = getNextToken(tokenizer, "\n", command, "json-path", lineno); transformed.add(String.format("json-path %s %s %s;", col(src), col(dest), quote(path))); } break; case "set-charset" : { String column = getNextToken(tokenizer, command, "column", lineno); String charset = getNextToken(tokenizer, "\n", command, "charset", lineno, true); transformed.add(String.format("set-charset %s %s;", col(column), charset)); } break; case "invoke-http" : { String url = getNextToken(tokenizer, command, "url", lineno); String columnsOpt = getNextToken(tokenizer, command, "columns", lineno); String headers = getNextToken(tokenizer, "\n", command, "headers", lineno, true); transformed.add(String.format("invoke-http %s %s %s;", quote(url), toColumArray(columnsOpt.split(",")), quote(headers))); } break; case "set-record-delim" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String limitStr = getNextToken(tokenizer, "\n", column, "limit", lineno, true); transformed.add(String.format("set-record-delim %s %s %s;", col(column), quote(delimiter), limitStr)); } break; case "parse-as-fixed-length" : { String column = getNextToken(tokenizer, command, "column", lineno); String widthStr = getNextToken(tokenizer, command, "widths", lineno); String padding = getNextToken(tokenizer, "\n", column, "padding", lineno, true); transformed.add(String.format("parse-as-fixed-length %s %s %s;", col(column), widthStr, quote(padding))); } break; case "split-to-rows" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "separator", lineno); transformed.add(String.format("split-to-rows %s %s;", col(column), quote(regex))); } break; case "split-to-columns" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "regex", lineno); transformed.add(String.format("split-to-columns %s %s;", col(column), quote(regex))); } break; case "parse-xml-to-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-xml-to-json %s %s;", col(column), depthOpt)); } break; case "parse-as-xml" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-xml %s;", col(column))); } break; case "xpath" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "xpath-array" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath-array %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "flatten" : { String cols = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("flatten %s;", toColumArray(cols.split(",")))); } break; case "copy" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String forceOpt = getNextToken(tokenizer, "\n", command, "force", lineno, true); if (forceOpt == null || forceOpt.isEmpty()) { transformed.add(String.format("copy %s %s;", col(source), col(destination))); } else { transformed.add(String.format("copy %s %s %s;", col(source), col(destination), forceOpt)); } } break; case "fill-null-or-empty" : { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, "\n", command, "fixed-value", lineno, false); transformed.add(String.format("fill-null-or-empty %s %s;", col(column), quote(value))); } break; case "cut-character" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String range = getNextToken(tokenizer, command, "range", lineno); transformed.add(String.format("cut-character %s %s %s;", col(source), col(destination), quote(range))); } break; case "generate-uuid" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("generate-uuid %s;", col(column))); } break; case "url-encode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-encode %s;", col(column))); } break; case "url-decode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-decode %s;", col(column))); } break; case "parse-as-log" : { String column = getNextToken(tokenizer, command, "column", lineno); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-log %s %s;", col(column), quote(format))); } break; case "parse-as-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String timezone = getNextToken(tokenizer, "\n", command, "timezone", lineno, true); transformed.add(String.format("parse-as-date %s %s;", col(column), quote(timezone))); } break; case "parse-as-simple-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-simple-date %s %s;", col(column), quote(pattern))); } break; case "diff-date" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destColumn = getNextToken(tokenizer, "\n", command, "destColumn", lineno); transformed.add(String.format("diff-date %s %s %s;", col(column1), col(column2), col(destColumn))); } break; case "keep" : { String columns = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("keep %s;", toColumArray(columns.split(",")))); } break; case "parse-as-hl7" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-hl7 %s %s;", col(column), depthOpt)); } break; case "split-email" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-email %s;", col(column))); } break; case "swap" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); transformed.add(String.format("swap %s %s;", col(column1), col(column2))); } break; case "hash" : { String column = getNextToken(tokenizer, command, "column", lineno); String algorithm = getNextToken(tokenizer, command, "algorithm", lineno); String encodeOpt = getNextToken(tokenizer, "\n", command, "encode", lineno, true); transformed.add(String.format("hash %s %s %s;", col(column), quote(algorithm), encodeOpt)); } break; case "write-as-json-map" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-json-map %s;", col(column))); } break; case "write-as-json-object" : { String column = getNextToken(tokenizer, command, "column", lineno); String columnsStr = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("write-as-json-object %s %s;", col(column), toColumArray(columnsStr.split(",")))); } break; case "write-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-csv %s;", col(column))); } break; case "parse-as-avro-file": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-avro-file %s;", col(column))); } break; case "send-to-error": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("send-to-error exp:{%s};", condition)); } break; case "fail": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("fail exp:{%s};", condition)); } break; case "text-distance" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-distance %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "text-metric" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-metric %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "catalog-lookup" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("catalog-lookup %s %s;", quote(type), col(column))); } break; case "table-lookup" : { String column = getNextToken(tokenizer, command, "column", lineno); String table = getNextToken(tokenizer, command, "table", lineno); transformed.add(String.format("table-lookup %s %s;", col(column), quote(table))); } break; case "stemming" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("stemming %s;", col(column))); } break; case "columns-replace" : { String sed = getNextToken(tokenizer, command, "sed-expression", lineno); transformed.add(String.format("columns-replace %s;", quote(sed))); } break; case "extract-regex-groups" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, command, "regex", lineno); transformed.add(String.format("extract-regex-groups %s %s;", col(column), quote(regex))); } break; case "split-url" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-url %s;", col(column))); } break; case "cleanse-column-names" : { transformed.add("cleanse-column-names;"); } break; case "change-column-case" : { String casing = getNextToken(tokenizer, command, "case", lineno); transformed.add(String.format("change-column-case %s;", casing)); } break; case "set-column" : { String column = getNextToken(tokenizer, command, "column", lineno); String expr = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "encode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("encode %s %s;", quote(type), col(column))); } break; case "decode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("decode %s %s;", quote(type), col(column))); } break; case "trim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("trim %s;", col(col))); } break; case "ltrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("ltrim %s;", col(col))); } break; case "rtrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("rtrim %s;", col(col))); } break; default: if (!directive.endsWith(";") && !directive.startsWith("$")) { transformed.add(directive + ";"); } else { transformed.add(directive); } break; } lineno = lineno + 1; } return Joiner.on('\n').join(transformed); } MigrateToV2(List<String> recipe);  MigrateToV2(String[] recipe);  MigrateToV2(@Nullable String recipe, String delimiter);  MigrateToV2(String recipe); @Override String migrate(); static String getNextToken(StringTokenizer tokenizer, String directive,
                                    String field, int lineno); static String getNextToken(StringTokenizer tokenizer, String delimiter,
                                    String directive, String field, int lineno); static String getNextToken(StringTokenizer tokenizer, String delimiter,
                                    String directive, String field, int lineno, boolean optional); }
MigrateToV2 implements GrammarMigrator { @Override public String migrate() throws DirectiveParseException { List<String> transformed = new ArrayList<>(); int lineno = 1; for (String directive : recipe) { directive = directive.trim(); if (directive.isEmpty() || directive.startsWith(" || (directive.startsWith("#") && !directive.startsWith("#pragma"))) { continue; } if (directive.contains("exp:") || directive.contains("prop:")) { if (directive.endsWith(";")) { transformed.add(directive); } else { transformed.add(directive + ";"); } continue; } if (directive.startsWith("#pragma")) { transformed.add(directive); continue; } if (directive.endsWith(";")) { directive = directive.substring(0, directive.length() - 1); } StringTokenizer tokenizer = new StringTokenizer(directive, " "); String command = tokenizer.nextToken(); switch (command) { case "set": { switch (tokenizer.nextToken()) { case "column": { String column = getNextToken(tokenizer, "set column", "column-name", lineno); String expr = getNextToken(tokenizer, "\n", "set column", "jexl-expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "columns": { String columns = getNextToken(tokenizer, "\n", "set columns", "name1, name2, ...", lineno); String cols[] = columns.split(","); transformed.add(String.format("set-headers %s;", toColumArray(cols))); } break; } } break; case "rename": { String oldcol = getNextToken(tokenizer, command, "old", lineno); String newcol = getNextToken(tokenizer, command, "new", lineno); transformed.add(String.format("rename %s %s;", col(oldcol), col(newcol))); } break; case "set-type": { String col = getNextToken(tokenizer, command, "col", lineno); String type = getNextToken(tokenizer, command, "type", lineno); transformed.add(String.format("set-type %s %s;", col(col), type)); } break; case "drop": { String columns = getNextToken(tokenizer, command, "column", lineno); String cols[] = columns.split(","); transformed.add(String.format("drop %s;", toColumArray(cols))); } break; case "merge": { String col1 = getNextToken(tokenizer, command, "first", lineno); String col2 = getNextToken(tokenizer, command, "second", lineno); String dest = getNextToken(tokenizer, command, "new-column", lineno); String delimiter = getNextToken(tokenizer, "\n", command, "delimiter", lineno); transformed.add(String.format("merge %s %s %s %s;", col(col1), col(col2), col(dest), quote(delimiter))); } break; case "uppercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("uppercase %s;", col(col))); } break; case "lowercase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("lowercase %s;", col(col))); } break; case "titlecase": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("titlecase %s;", col(col))); } break; case "indexsplit": { String source = getNextToken(tokenizer, command, "source", lineno); String startStr = getNextToken(tokenizer, command, "start", lineno); String endStr = getNextToken(tokenizer, command, "end", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("indexsplit %s %s %s %s;", col(source), startStr, endStr, col(destination))); } break; case "split": { String source = getNextToken(tokenizer, command, "source-column-name", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String firstCol = getNextToken(tokenizer, command, "new-column-1", lineno); String secondCol = getNextToken(tokenizer, command, "new-column-2", lineno); transformed.add(String.format("split %s %s %s %s;", col(source), quote(delimiter), col(firstCol), col(secondCol))); } break; case "filter-row-if-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-not-matched": { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } break; case "filter-row-if-true": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } break; case "filter-row-if-false": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } break; case "filter-rows-on" : { String cmd = getNextToken(tokenizer, command, "command", lineno); if (cmd.equalsIgnoreCase("condition-false")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} false;", condition)); } else if (cmd.equalsIgnoreCase("condition-true")) { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("filter-row exp:{%s} true;", condition)); } else if (cmd.equalsIgnoreCase("empty-or-null-columns")) { String columns = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("filter-empty-or-null %s;", toColumArray(columns.split(",")))); } else if (cmd.equalsIgnoreCase("regex-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-matched %s %s;", col(column), quote(pattern))); } else if (cmd.equalsIgnoreCase("regex-not-match")) { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "regex", lineno); transformed.add(String.format("filter-by-regex if-not-matched %s %s;", col(column), quote(pattern))); } else { throw new DirectiveParseException( "filter-rows-on", String.format("Unknown option '%s' specified at line no %s", cmd, lineno) ); } } break; case "set-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-variable %s exp:{%s};", column, expression)); } break; case "increment-variable": { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, command, "value", lineno); String expression = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("increment-variable %s %s exp:{%s};", column, value, expression)); } break; case "mask-number": { String column = getNextToken(tokenizer, command, "column", lineno); String mask = getNextToken(tokenizer, command, "pattern", lineno); transformed.add(String.format("mask-number %s %s;", col(column), quote(mask))); } break; case "mask-shuffle": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("mask-shuffle %s;", col(column))); } break; case "format-date": { String column = getNextToken(tokenizer, command, "column", 1); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("format-date %s %s;", col(column), quote(format))); } break; case "format-unix-timestamp": { String column = getNextToken(tokenizer, command, "column", lineno); String dstDatePattern = getNextToken(tokenizer, "\n", command, "destination-format", lineno); transformed.add(String.format("format-unix-timestamp %s %s;", col(column), quote(dstDatePattern))); } break; case "quantize": { String column1 = getNextToken(tokenizer, command, "source-column", lineno); String column2 = getNextToken(tokenizer, command, "destination-column", lineno); String ranges = getNextToken(tokenizer, "\n", command, "destination-column", lineno); transformed.add(String.format("quantize %s %s %s;", col(column1), col(column2), ranges)); } break; case "find-and-replace" : { String columns = getNextToken(tokenizer, command, "columns", lineno); String expression = getNextToken(tokenizer, "\n", command, "sed-script", lineno); transformed.add(String.format("find-and-replace %s %s;", toColumArray(columns.split(",")), quote(expression))); } break; case "parse-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimStr = getNextToken(tokenizer, command, "delimiter", lineno); if (delimStr.endsWith(";")) { delimStr = delimStr.substring(0, delimStr.length() - 1); } String hasHeaderLinesOpt = getNextToken(tokenizer, "\n", command, "true|false", lineno, true); transformed.add(String.format("parse-as-csv %s %s %s;", col(column), quote(delimStr), hasHeaderLinesOpt == null ? "" : hasHeaderLinesOpt)); } break; case "parse-as-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-json %s %s;", col(column), depthOpt)); } break; case "parse-as-avro" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String type = getNextToken(tokenizer, command, "type", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-avro %s %s %s %s;", col(column), schemaId, type, versionOpt)); } break; case "parse-as-protobuf" : { String column = getNextToken(tokenizer, command, "column", lineno); String schemaId = getNextToken(tokenizer, command, "schema-id", lineno); String recordName = getNextToken(tokenizer, command, "record-name", lineno); String versionOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-protobuf %s %s %s %s;", col(column), schemaId, quote(recordName), versionOpt)); } break; case "json-path" : { String src = getNextToken(tokenizer, command, "source", lineno); String dest = getNextToken(tokenizer, command, "dest", lineno); String path = getNextToken(tokenizer, "\n", command, "json-path", lineno); transformed.add(String.format("json-path %s %s %s;", col(src), col(dest), quote(path))); } break; case "set-charset" : { String column = getNextToken(tokenizer, command, "column", lineno); String charset = getNextToken(tokenizer, "\n", command, "charset", lineno, true); transformed.add(String.format("set-charset %s %s;", col(column), charset)); } break; case "invoke-http" : { String url = getNextToken(tokenizer, command, "url", lineno); String columnsOpt = getNextToken(tokenizer, command, "columns", lineno); String headers = getNextToken(tokenizer, "\n", command, "headers", lineno, true); transformed.add(String.format("invoke-http %s %s %s;", quote(url), toColumArray(columnsOpt.split(",")), quote(headers))); } break; case "set-record-delim" : { String column = getNextToken(tokenizer, command, "column", lineno); String delimiter = getNextToken(tokenizer, command, "delimiter", lineno); String limitStr = getNextToken(tokenizer, "\n", column, "limit", lineno, true); transformed.add(String.format("set-record-delim %s %s %s;", col(column), quote(delimiter), limitStr)); } break; case "parse-as-fixed-length" : { String column = getNextToken(tokenizer, command, "column", lineno); String widthStr = getNextToken(tokenizer, command, "widths", lineno); String padding = getNextToken(tokenizer, "\n", column, "padding", lineno, true); transformed.add(String.format("parse-as-fixed-length %s %s %s;", col(column), widthStr, quote(padding))); } break; case "split-to-rows" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "separator", lineno); transformed.add(String.format("split-to-rows %s %s;", col(column), quote(regex))); } break; case "split-to-columns" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, "\n", "regex", lineno); transformed.add(String.format("split-to-columns %s %s;", col(column), quote(regex))); } break; case "parse-xml-to-json" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-xml-to-json %s %s;", col(column), depthOpt)); } break; case "parse-as-xml" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-xml %s;", col(column))); } break; case "xpath" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "xpath-array" : { String column = getNextToken(tokenizer, command, "column", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String xpath = getNextToken(tokenizer, "\n", command, "xpath", lineno); transformed.add(String.format("xpath-array %s %s %s;", col(column), col(destination), quote(xpath))); } break; case "flatten" : { String cols = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("flatten %s;", toColumArray(cols.split(",")))); } break; case "copy" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String forceOpt = getNextToken(tokenizer, "\n", command, "force", lineno, true); if (forceOpt == null || forceOpt.isEmpty()) { transformed.add(String.format("copy %s %s;", col(source), col(destination))); } else { transformed.add(String.format("copy %s %s %s;", col(source), col(destination), forceOpt)); } } break; case "fill-null-or-empty" : { String column = getNextToken(tokenizer, command, "column", lineno); String value = getNextToken(tokenizer, "\n", command, "fixed-value", lineno, false); transformed.add(String.format("fill-null-or-empty %s %s;", col(column), quote(value))); } break; case "cut-character" : { String source = getNextToken(tokenizer, command, "source", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); String range = getNextToken(tokenizer, command, "range", lineno); transformed.add(String.format("cut-character %s %s %s;", col(source), col(destination), quote(range))); } break; case "generate-uuid" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("generate-uuid %s;", col(column))); } break; case "url-encode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-encode %s;", col(column))); } break; case "url-decode" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("url-decode %s;", col(column))); } break; case "parse-as-log" : { String column = getNextToken(tokenizer, command, "column", lineno); String format = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-log %s %s;", col(column), quote(format))); } break; case "parse-as-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String timezone = getNextToken(tokenizer, "\n", command, "timezone", lineno, true); transformed.add(String.format("parse-as-date %s %s;", col(column), quote(timezone))); } break; case "parse-as-simple-date" : { String column = getNextToken(tokenizer, command, "column", lineno); String pattern = getNextToken(tokenizer, "\n", command, "format", lineno); transformed.add(String.format("parse-as-simple-date %s %s;", col(column), quote(pattern))); } break; case "diff-date" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destColumn = getNextToken(tokenizer, "\n", command, "destColumn", lineno); transformed.add(String.format("diff-date %s %s %s;", col(column1), col(column2), col(destColumn))); } break; case "keep" : { String columns = getNextToken(tokenizer, command, "columns", lineno); transformed.add(String.format("keep %s;", toColumArray(columns.split(",")))); } break; case "parse-as-hl7" : { String column = getNextToken(tokenizer, command, "column", lineno); String depthOpt = getNextToken(tokenizer, "\n", command, "depth", lineno, true); transformed.add(String.format("parse-as-hl7 %s %s;", col(column), depthOpt)); } break; case "split-email" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-email %s;", col(column))); } break; case "swap" : { String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); transformed.add(String.format("swap %s %s;", col(column1), col(column2))); } break; case "hash" : { String column = getNextToken(tokenizer, command, "column", lineno); String algorithm = getNextToken(tokenizer, command, "algorithm", lineno); String encodeOpt = getNextToken(tokenizer, "\n", command, "encode", lineno, true); transformed.add(String.format("hash %s %s %s;", col(column), quote(algorithm), encodeOpt)); } break; case "write-as-json-map" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-json-map %s;", col(column))); } break; case "write-as-json-object" : { String column = getNextToken(tokenizer, command, "column", lineno); String columnsStr = getNextToken(tokenizer, "\n", command, "columns", lineno); transformed.add(String.format("write-as-json-object %s %s;", col(column), toColumArray(columnsStr.split(",")))); } break; case "write-as-csv" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("write-as-csv %s;", col(column))); } break; case "parse-as-avro-file": { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("parse-as-avro-file %s;", col(column))); } break; case "send-to-error": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("send-to-error exp:{%s};", condition)); } break; case "fail": { String condition = getNextToken(tokenizer, "\n", command, "condition", lineno); transformed.add(String.format("fail exp:{%s};", condition)); } break; case "text-distance" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-distance %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "text-metric" : { String method = getNextToken(tokenizer, command, "method", lineno); String column1 = getNextToken(tokenizer, command, "column1", lineno); String column2 = getNextToken(tokenizer, command, "column2", lineno); String destination = getNextToken(tokenizer, command, "destination", lineno); transformed.add(String.format("text-metric %s %s %s %s;", quote(method), col(column1), col(column2), col(destination))); } break; case "catalog-lookup" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("catalog-lookup %s %s;", quote(type), col(column))); } break; case "table-lookup" : { String column = getNextToken(tokenizer, command, "column", lineno); String table = getNextToken(tokenizer, command, "table", lineno); transformed.add(String.format("table-lookup %s %s;", col(column), quote(table))); } break; case "stemming" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("stemming %s;", col(column))); } break; case "columns-replace" : { String sed = getNextToken(tokenizer, command, "sed-expression", lineno); transformed.add(String.format("columns-replace %s;", quote(sed))); } break; case "extract-regex-groups" : { String column = getNextToken(tokenizer, command, "column", lineno); String regex = getNextToken(tokenizer, command, "regex", lineno); transformed.add(String.format("extract-regex-groups %s %s;", col(column), quote(regex))); } break; case "split-url" : { String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("split-url %s;", col(column))); } break; case "cleanse-column-names" : { transformed.add("cleanse-column-names;"); } break; case "change-column-case" : { String casing = getNextToken(tokenizer, command, "case", lineno); transformed.add(String.format("change-column-case %s;", casing)); } break; case "set-column" : { String column = getNextToken(tokenizer, command, "column", lineno); String expr = getNextToken(tokenizer, "\n", command, "expression", lineno); transformed.add(String.format("set-column %s exp:{%s};", col(column), expr)); } break; case "encode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("encode %s %s;", quote(type), col(column))); } break; case "decode" : { String type = getNextToken(tokenizer, command, "type", lineno); String column = getNextToken(tokenizer, command, "column", lineno); transformed.add(String.format("decode %s %s;", quote(type), col(column))); } break; case "trim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("trim %s;", col(col))); } break; case "ltrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("ltrim %s;", col(col))); } break; case "rtrim": { String col = getNextToken(tokenizer, command, "col", lineno); transformed.add(String.format("rtrim %s;", col(col))); } break; default: if (!directive.endsWith(";") && !directive.startsWith("$")) { transformed.add(directive + ";"); } else { transformed.add(directive); } break; } lineno = lineno + 1; } return Joiner.on('\n').join(transformed); } MigrateToV2(List<String> recipe);  MigrateToV2(String[] recipe);  MigrateToV2(@Nullable String recipe, String delimiter);  MigrateToV2(String recipe); @Override String migrate(); static String getNextToken(StringTokenizer tokenizer, String directive,
                                    String field, int lineno); static String getNextToken(StringTokenizer tokenizer, String delimiter,
                                    String directive, String field, int lineno); static String getNextToken(StringTokenizer tokenizer, String delimiter,
                                    String directive, String field, int lineno, boolean optional);  }