message: adding chunk merge to sort join
diff: @@ -54,9 +54,26 @@ arrow::Status do_sorted_join(const std::shared_ptr<arrow::Table> &left_tab,
  twisterx::join::config::JoinType join_type,
  std::shared_ptr<arrow::Table> *joined_table,
  arrow::MemoryPool *memory_pool) {
  + // combine chunks if multiple chunks are available
  + std::shared_ptr<arrow::Table> left_tab_comb, right_tab_comb;
  + arrow::Status lstatus, rstatus;
  + auto t11 = std::chrono::high_resolution_clock::now();
  +
  + lstatus = twisterx::join::util::CombineChunks(left_tab, left_join_column_idx, left_tab_comb, memory_pool);
  + rstatus = twisterx::join::util::CombineChunks(right_tab, right_join_column_idx, right_tab_comb, memory_pool);
  +
  + auto t22 = std::chrono::high_resolution_clock::now();
  +
  + if (!lstatus.ok() || !rstatus.ok()) {
  + LOG(ERROR) << "Combining chunks failed!";
  + return arrow::Status::Invalid("Sort join failed!");
  + }
  +
  + LOG(INFO) << "Combine chunks time : " << std::chrono::duration_cast<std::chrono::milliseconds>(t22 - t11).count();
  +
  //sort columns
  - auto left_join_column = left_tab->column(left_join_column_idx)->chunk(0);
  - auto right_join_column = right_tab->column(right_join_column_idx)->chunk(0);
  + auto left_join_column = left_tab_comb->column(left_join_column_idx)->chunk(0);
  + auto right_join_column = right_tab_comb->column(right_join_column_idx)->chunk(0);
  auto t1 = std::chrono::high_resolution_clock::now();
  std::shared_ptr<arrow::Array> left_index_sorted_column;
  @@ -197,8 +214,8 @@ arrow::Status do_sorted_join(const std::shared_ptr<arrow::Table> &left_tab,
  // build final table
  status = twisterx::join::util::build_final_table(
  left_indices, right_indices,
  - left_tab,
  - right_tab,
  + left_tab_comb,
  + right_tab_comb,
  joined_table,
  memory_pool
  );
  @@ -244,7 +261,7 @@ arrow::Status do_hash_join(const std::shared_ptr<arrow::Table> &left_tab,
  return arrow::Status::Invalid("Hash join failed!");
  }
  - LOG(INFO) << "Combine chunk time : " << std::chrono::duration_cast<std::chrono::milliseconds>(t22 - t11).count();
  + LOG(INFO) << "Combine chunks time : " << std::chrono::duration_cast<std::chrono::milliseconds>(t22 - t11).count();
  //sort columns
  std::shared_ptr<arrow::Array> left_idx_column = left_tab_comb->column(left_join_column_idx)->chunk(0);
  