message: Tune a constant that results in a 20-50% improvement on some benchmarks, modest improvement on others and perf neutral for the rest.
diff: @@ -217,7 +217,9 @@ struct LaunchBatchMatMul<CPUDevice, Scalar> {
  in_x.dim_size(1) * in_x.dim_size(2) * out->dim_size(2);
  const int64 small_dim = std::min(
  std::min(in_x.dim_size(1), in_x.dim_size(2)), out->dim_size(2));
  - const int64 kMaxCostOuterParallelism = 128 * 128 * 256; // heuristic.
  + // NOTE(nikhilsarda): This heuristic is optimal in benchmarks as of
  + // Jan 21, 2020.
  + const int64 kMaxCostOuterParallelism = 128 * 128; // heuristic.
  auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
  if (small_dim > 1 &&
  (batch_size == 1 || cost_per_unit > kMaxCostOuterParallelism)) {
  