@Test public void testFactFilterPushDown() throws LensException { String query = "SELECT (dim1 . date) date , sum((f . msr1)) msr1 , (dim2 . name) dim2_name " + "FROM fact f INNER JOIN dim1 dim1 ON f.dim1_id = dim1.id and f.m2 = '1234' " + "INNER JOIN dim2 dim2 ON f.dim2_id = dim2.id and f.m3 > 3000 " + "WHERE ((dim1 . date) = '2014-11-25 00:00:00') and f.m4 is not null " + "GROUP BY (dim1 . date), (dim2 . name)"; String actual = qtest.rewrite(query, conf, hconf); String expected = "select ( dim1___dim1 . date ) as `date` , sum(alias1) as `msr1` , ( dim2___dim2 . name ) " + "as `dim2_name` from (select fact___f.dim1_id, fact___f.m2, fact___f.dim2_id, fact___f.m3, fact___f.m4, " + "sum(( fact___f . msr1 )) as alias1 from fact fact___f where ( fact___f . m4 ) " + "is not null and (( fact___f . m2 ) = '1234' ) and (( fact___f . m3 ) > 3000 ) and " + "fact___f.dim1_id in ( select dim1 .id from dim1 where (( dim1. date ) = '2014-11-25 00:00:00' ) ) " + "group by fact___f.dim1_id, fact___f.m2, fact___f.dim2_id, fact___f.m3, fact___f.m4) fact___f " + "inner join (select id, date from dim1) dim1___dim1 on ((( fact___f . dim1_id ) = ( dim1___dim1 . id )) and " + "(( fact___f . m2 ) = '1234' )) inner join (select id, name from dim2) " + "dim2___dim2 on ((( fact___f . dim2_id ) = " + "( dim2___dim2 . id )) and (( fact___f . m3 ) > 3000 )) where ((( dim1___dim1 . date )" + " = '2014-11-25 00:00:00' ) and ( fact___f . m4 ) is not null ) " + "group by ( dim1___dim1 . date ), ( dim2___dim2 . name )"; compareQueries(actual, expected); }
public void factFilterPushDown(ASTNode node) { if (node == null) { log.debug("Join AST is null "); return; } String filterCond = ""; if (node.getToken().getType() == HiveParser.KW_AND) { ASTNode parentNode = (ASTNode) node.getChild(0).getParent(); if (parentNode.getChild(0).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(0).getChild(1).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(1).getType() == HiveParser.DOT) { return; } ASTNode right = (ASTNode) node.getChild(1); filterCond = HQLParser.getString(right); } String factAlias = getFactAlias(); if (filterCond.matches("(.*)" + factAlias + "(.*)")) { factFilterPush.append(filterCond).append(" and "); } for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); factFilterPushDown(child); } }
ColumnarSQLRewriter implements QueryRewriter { public void factFilterPushDown(ASTNode node) { if (node == null) { log.debug("Join AST is null "); return; } String filterCond = ""; if (node.getToken().getType() == HiveParser.KW_AND) { ASTNode parentNode = (ASTNode) node.getChild(0).getParent(); if (parentNode.getChild(0).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(0).getChild(1).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(1).getType() == HiveParser.DOT) { return; } ASTNode right = (ASTNode) node.getChild(1); filterCond = HQLParser.getString(right); } String factAlias = getFactAlias(); if (filterCond.matches("(.*)" + factAlias + "(.*)")) { factFilterPush.append(filterCond).append(" and "); } for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); factFilterPushDown(child); } } }
ColumnarSQLRewriter implements QueryRewriter { public void factFilterPushDown(ASTNode node) { if (node == null) { log.debug("Join AST is null "); return; } String filterCond = ""; if (node.getToken().getType() == HiveParser.KW_AND) { ASTNode parentNode = (ASTNode) node.getChild(0).getParent(); if (parentNode.getChild(0).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(0).getChild(1).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(1).getType() == HiveParser.DOT) { return; } ASTNode right = (ASTNode) node.getChild(1); filterCond = HQLParser.getString(right); } String factAlias = getFactAlias(); if (filterCond.matches("(.*)" + factAlias + "(.*)")) { factFilterPush.append(filterCond).append(" and "); } for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); factFilterPushDown(child); } } ColumnarSQLRewriter(); }
ColumnarSQLRewriter implements QueryRewriter { public void factFilterPushDown(ASTNode node) { if (node == null) { log.debug("Join AST is null "); return; } String filterCond = ""; if (node.getToken().getType() == HiveParser.KW_AND) { ASTNode parentNode = (ASTNode) node.getChild(0).getParent(); if (parentNode.getChild(0).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(0).getChild(1).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(1).getType() == HiveParser.DOT) { return; } ASTNode right = (ASTNode) node.getChild(1); filterCond = HQLParser.getString(right); } String factAlias = getFactAlias(); if (filterCond.matches("(.*)" + factAlias + "(.*)")) { factFilterPush.append(filterCond).append(" and "); } for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); factFilterPushDown(child); } } ColumnarSQLRewriter(); @Override void init(Configuration conf); String getClause(); void analyzeInternal(Configuration conf, HiveConf hconf); String getTableFromTabRefNode(ASTNode tree); void getJoinCond(ASTNode node); StringBuilder constructJoinChain(); int getColumnCount(ASTNode node); boolean isExpressionsAnswerableFromFact(ASTNode node); void getFilterInJoinCond(ASTNode node); String getFactAlias(); void factFilterPushDown(ASTNode node); void getFactKeysFromNode(ASTNode node); void getAllFactKeys(); void getAllDimColumns(ASTNode node); void buildDimSubqueries(); void buildSubqueries(ASTNode node); String getTableOrAlias(String keyString, String type); ArrayList<String> getAggregateColumns(ASTNode node, MutableInt count); ArrayList<String> getTablesAndColumns(ASTNode node); String getLimitClause(ASTNode node); void getAllFilters(ASTNode node); String getFactNameAlias(ASTNode fromAST); void reset(); String replaceUDFForDB(String query); void replaceAliasInAST(); void buildQuery(Configuration conf, HiveConf hconf); @Override String rewrite(String query, Configuration conf, HiveConf metastoreConf); }
ColumnarSQLRewriter implements QueryRewriter { public void factFilterPushDown(ASTNode node) { if (node == null) { log.debug("Join AST is null "); return; } String filterCond = ""; if (node.getToken().getType() == HiveParser.KW_AND) { ASTNode parentNode = (ASTNode) node.getChild(0).getParent(); if (parentNode.getChild(0).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(0).getChild(1).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(0).getType() == HiveParser.DOT && parentNode.getChild(1).getChild(1).getType() == HiveParser.DOT) { return; } ASTNode right = (ASTNode) node.getChild(1); filterCond = HQLParser.getString(right); } String factAlias = getFactAlias(); if (filterCond.matches("(.*)" + factAlias + "(.*)")) { factFilterPush.append(filterCond).append(" and "); } for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); factFilterPushDown(child); } } ColumnarSQLRewriter(); @Override void init(Configuration conf); String getClause(); void analyzeInternal(Configuration conf, HiveConf hconf); String getTableFromTabRefNode(ASTNode tree); void getJoinCond(ASTNode node); StringBuilder constructJoinChain(); int getColumnCount(ASTNode node); boolean isExpressionsAnswerableFromFact(ASTNode node); void getFilterInJoinCond(ASTNode node); String getFactAlias(); void factFilterPushDown(ASTNode node); void getFactKeysFromNode(ASTNode node); void getAllFactKeys(); void getAllDimColumns(ASTNode node); void buildDimSubqueries(); void buildSubqueries(ASTNode node); String getTableOrAlias(String keyString, String type); ArrayList<String> getAggregateColumns(ASTNode node, MutableInt count); ArrayList<String> getTablesAndColumns(ASTNode node); String getLimitClause(ASTNode node); void getAllFilters(ASTNode node); String getFactNameAlias(ASTNode fromAST); void reset(); String replaceUDFForDB(String query); void replaceAliasInAST(); void buildQuery(Configuration conf, HiveConf hconf); @Override String rewrite(String query, Configuration conf, HiveConf metastoreConf);  }