@Test public void testNoRewrite() throws LensException { String query = "select count(distinct id) from location_dim"; String actual = qtest.rewrite(query, conf, hconf); String expected = "select count( distinct id ) from location_dim "; compareQueries(actual, expected); String query2 = "select count(distinct id) from location_dim location_dim"; String actual2 = qtest.rewrite(query2, conf, hconf); String expected2 = "select count( distinct id ) from location_dim location_dim___location_dim"; compareQueries(expected2, actual2); String query3 = "select count(distinct location_dim.id) from db.location_dim location_dim"; String actual3 = qtest.rewrite(query3, conf, hconf); String expected3 = "select count( distinct ( location_dim__db_location_dim_location_dim . id )) " + "from db.location_dim location_dim__db_location_dim_location_dim"; compareQueries(expected3, actual3); String query4 = "select count(distinct location_dim.id) from db.location_dim location_dim " + "left outer join db.item_dim item_dim on location_dim.id = item_dim.id " + "right outer join time_dim time_dim on location_dim.id = time_dim.id "; String actual4 = qtest.rewrite(query4, conf, hconf); String expected4 = "select count( distinct ( location_dim__db_location_dim_location_dim . id )) " + "from db.location_dim location_dim__db_location_dim_location_dim left outer join (select id from db.item_dim) " + "item_dim__db_item_dim_item_dim on (( location_dim__db_location_dim_location_dim . id ) = " + "( item_dim__db_item_dim_item_dim . id )) right outer join (select id from time_dim) time_dim___time_dim on " + "(( location_dim__db_location_dim_location_dim . id ) = ( time_dim___time_dim . id ))"; compareQueries(expected4, actual4); }
@Override public String rewrite(String query, Configuration conf, HiveConf metastoreConf) throws LensException { this.query = query; StringBuilder mergedQuery; rewrittenQuery.setLength(0); String queryReplacedUdf; reset(); try { String finalRewrittenQuery; if (query.toLowerCase().matches("(.*)union all(.*)")) { finalRewrittenQuery = ""; String[] queries = query.toLowerCase().split("union all"); for (int i = 0; i < queries.length; i++) { log.info("Union Query Part {} : {}", i, queries[i]); ast = HQLParser.parseHQL(queries[i], metastoreConf); buildQuery(conf, metastoreConf); mergedQuery = rewrittenQuery.append(" union all "); finalRewrittenQuery = mergedQuery.toString().substring(0, mergedQuery.lastIndexOf("union all")); reset(); } } else { ast = HQLParser.parseHQL(query, metastoreConf); buildQuery(conf, metastoreConf); finalRewrittenQuery = rewrittenQuery.toString(); } queryReplacedUdf = replaceUDFForDB(finalRewrittenQuery); log.info("Input Query : {}", query); log.info("Rewritten Query : {}", queryReplacedUdf); } catch (SemanticException e) { throw new LensException(e); } return queryReplacedUdf; }
ColumnarSQLRewriter implements QueryRewriter { @Override public String rewrite(String query, Configuration conf, HiveConf metastoreConf) throws LensException { this.query = query; StringBuilder mergedQuery; rewrittenQuery.setLength(0); String queryReplacedUdf; reset(); try { String finalRewrittenQuery; if (query.toLowerCase().matches("(.*)union all(.*)")) { finalRewrittenQuery = ""; String[] queries = query.toLowerCase().split("union all"); for (int i = 0; i < queries.length; i++) { log.info("Union Query Part {} : {}", i, queries[i]); ast = HQLParser.parseHQL(queries[i], metastoreConf); buildQuery(conf, metastoreConf); mergedQuery = rewrittenQuery.append(" union all "); finalRewrittenQuery = mergedQuery.toString().substring(0, mergedQuery.lastIndexOf("union all")); reset(); } } else { ast = HQLParser.parseHQL(query, metastoreConf); buildQuery(conf, metastoreConf); finalRewrittenQuery = rewrittenQuery.toString(); } queryReplacedUdf = replaceUDFForDB(finalRewrittenQuery); log.info("Input Query : {}", query); log.info("Rewritten Query : {}", queryReplacedUdf); } catch (SemanticException e) { throw new LensException(e); } return queryReplacedUdf; } }
ColumnarSQLRewriter implements QueryRewriter { @Override public String rewrite(String query, Configuration conf, HiveConf metastoreConf) throws LensException { this.query = query; StringBuilder mergedQuery; rewrittenQuery.setLength(0); String queryReplacedUdf; reset(); try { String finalRewrittenQuery; if (query.toLowerCase().matches("(.*)union all(.*)")) { finalRewrittenQuery = ""; String[] queries = query.toLowerCase().split("union all"); for (int i = 0; i < queries.length; i++) { log.info("Union Query Part {} : {}", i, queries[i]); ast = HQLParser.parseHQL(queries[i], metastoreConf); buildQuery(conf, metastoreConf); mergedQuery = rewrittenQuery.append(" union all "); finalRewrittenQuery = mergedQuery.toString().substring(0, mergedQuery.lastIndexOf("union all")); reset(); } } else { ast = HQLParser.parseHQL(query, metastoreConf); buildQuery(conf, metastoreConf); finalRewrittenQuery = rewrittenQuery.toString(); } queryReplacedUdf = replaceUDFForDB(finalRewrittenQuery); log.info("Input Query : {}", query); log.info("Rewritten Query : {}", queryReplacedUdf); } catch (SemanticException e) { throw new LensException(e); } return queryReplacedUdf; } ColumnarSQLRewriter(); }
ColumnarSQLRewriter implements QueryRewriter { @Override public String rewrite(String query, Configuration conf, HiveConf metastoreConf) throws LensException { this.query = query; StringBuilder mergedQuery; rewrittenQuery.setLength(0); String queryReplacedUdf; reset(); try { String finalRewrittenQuery; if (query.toLowerCase().matches("(.*)union all(.*)")) { finalRewrittenQuery = ""; String[] queries = query.toLowerCase().split("union all"); for (int i = 0; i < queries.length; i++) { log.info("Union Query Part {} : {}", i, queries[i]); ast = HQLParser.parseHQL(queries[i], metastoreConf); buildQuery(conf, metastoreConf); mergedQuery = rewrittenQuery.append(" union all "); finalRewrittenQuery = mergedQuery.toString().substring(0, mergedQuery.lastIndexOf("union all")); reset(); } } else { ast = HQLParser.parseHQL(query, metastoreConf); buildQuery(conf, metastoreConf); finalRewrittenQuery = rewrittenQuery.toString(); } queryReplacedUdf = replaceUDFForDB(finalRewrittenQuery); log.info("Input Query : {}", query); log.info("Rewritten Query : {}", queryReplacedUdf); } catch (SemanticException e) { throw new LensException(e); } return queryReplacedUdf; } ColumnarSQLRewriter(); @Override void init(Configuration conf); String getClause(); void analyzeInternal(Configuration conf, HiveConf hconf); String getTableFromTabRefNode(ASTNode tree); void getJoinCond(ASTNode node); StringBuilder constructJoinChain(); int getColumnCount(ASTNode node); boolean isExpressionsAnswerableFromFact(ASTNode node); void getFilterInJoinCond(ASTNode node); String getFactAlias(); void factFilterPushDown(ASTNode node); void getFactKeysFromNode(ASTNode node); void getAllFactKeys(); void getAllDimColumns(ASTNode node); void buildDimSubqueries(); void buildSubqueries(ASTNode node); String getTableOrAlias(String keyString, String type); ArrayList<String> getAggregateColumns(ASTNode node, MutableInt count); ArrayList<String> getTablesAndColumns(ASTNode node); String getLimitClause(ASTNode node); void getAllFilters(ASTNode node); String getFactNameAlias(ASTNode fromAST); void reset(); String replaceUDFForDB(String query); void replaceAliasInAST(); void buildQuery(Configuration conf, HiveConf hconf); @Override String rewrite(String query, Configuration conf, HiveConf metastoreConf); }
ColumnarSQLRewriter implements QueryRewriter { @Override public String rewrite(String query, Configuration conf, HiveConf metastoreConf) throws LensException { this.query = query; StringBuilder mergedQuery; rewrittenQuery.setLength(0); String queryReplacedUdf; reset(); try { String finalRewrittenQuery; if (query.toLowerCase().matches("(.*)union all(.*)")) { finalRewrittenQuery = ""; String[] queries = query.toLowerCase().split("union all"); for (int i = 0; i < queries.length; i++) { log.info("Union Query Part {} : {}", i, queries[i]); ast = HQLParser.parseHQL(queries[i], metastoreConf); buildQuery(conf, metastoreConf); mergedQuery = rewrittenQuery.append(" union all "); finalRewrittenQuery = mergedQuery.toString().substring(0, mergedQuery.lastIndexOf("union all")); reset(); } } else { ast = HQLParser.parseHQL(query, metastoreConf); buildQuery(conf, metastoreConf); finalRewrittenQuery = rewrittenQuery.toString(); } queryReplacedUdf = replaceUDFForDB(finalRewrittenQuery); log.info("Input Query : {}", query); log.info("Rewritten Query : {}", queryReplacedUdf); } catch (SemanticException e) { throw new LensException(e); } return queryReplacedUdf; } ColumnarSQLRewriter(); @Override void init(Configuration conf); String getClause(); void analyzeInternal(Configuration conf, HiveConf hconf); String getTableFromTabRefNode(ASTNode tree); void getJoinCond(ASTNode node); StringBuilder constructJoinChain(); int getColumnCount(ASTNode node); boolean isExpressionsAnswerableFromFact(ASTNode node); void getFilterInJoinCond(ASTNode node); String getFactAlias(); void factFilterPushDown(ASTNode node); void getFactKeysFromNode(ASTNode node); void getAllFactKeys(); void getAllDimColumns(ASTNode node); void buildDimSubqueries(); void buildSubqueries(ASTNode node); String getTableOrAlias(String keyString, String type); ArrayList<String> getAggregateColumns(ASTNode node, MutableInt count); ArrayList<String> getTablesAndColumns(ASTNode node); String getLimitClause(ASTNode node); void getAllFilters(ASTNode node); String getFactNameAlias(ASTNode fromAST); void reset(); String replaceUDFForDB(String query); void replaceAliasInAST(); void buildQuery(Configuration conf, HiveConf hconf); @Override String rewrite(String query, Configuration conf, HiveConf metastoreConf);  }