orttraining/orttraining/python/orttraining_pybind_state.cc
TrainingConfigurationResult ConfigureSessionForTraining(
    training::TrainingSession* sess, TrainingParameters& parameters) {
  //TODO tix, refactor the mpi related code to populate all fields correctly by default.
  ORT_ENFORCE(parameters.horizontal_parallel_size <= parameters.world_size);
  ORT_ENFORCE(parameters.data_parallel_size <= parameters.world_size);

  if (parameters.world_size % parameters.data_parallel_size != 0) {
    throw std::runtime_error("Cannot split data parallel group because world_size is not divisible");
  }

  if (parameters.world_size % parameters.horizontal_parallel_size != 0) {
    throw std::runtime_error("Cannot split horizontal parallel group because world_size is not divisible");
  }

  auto data_group_size = parameters.world_size / (parameters.horizontal_parallel_size * parameters.pipeline_parallel_size);
  if (data_group_size != parameters.data_parallel_size) {
    LOGS(*(sess->GetLogger()), WARNING) << "data_parallel_size is not correct, tuned automatically to "
                                        << data_group_size;
    parameters.data_parallel_size = data_group_size;
  }

  training::TrainingSession::TrainingConfiguration config{};
  config.weight_names_to_train = parameters.weights_to_train;
  config.weight_names_to_not_train = parameters.weights_not_to_train;
  config.immutable_weights = parameters.immutable_weights;

  config.gradient_accumulation_steps = parameters.gradient_accumulation_steps;

  config.distributed_config.world_rank = parameters.world_rank;
  config.distributed_config.world_size = parameters.world_size;
  config.distributed_config.local_rank = parameters.local_rank;
  config.distributed_config.local_size = parameters.local_size;
  config.distributed_config.data_parallel_size = parameters.data_parallel_size;
  config.distributed_config.horizontal_parallel_size = parameters.horizontal_parallel_size;

  if (parameters.use_mixed_precision) {
    training::TrainingSession::TrainingConfiguration::MixedPrecisionConfiguration mp{};
    mp.use_mixed_precision_initializers = true;

    config.mixed_precision_config = mp;
  }

  config.loss_name = parameters.loss_output_name;

  if (!parameters.training_optimizer_name.empty()) {
    training::TrainingSession::TrainingConfiguration::OptimizerConfiguration opt{};
    opt.name = parameters.training_optimizer_name;
    opt.learning_rate_input_name = parameters.lr_params_feed_name;
    opt.weight_attributes_generator = [&parameters](const std::string& weight_name) {
      const auto it = parameters.optimizer_attributes_map.find(weight_name);
      ORT_ENFORCE(
          it != parameters.optimizer_attributes_map.end(),
          "Failed to find attribute map for weight ", weight_name);
      return it->second;
    };
    opt.weight_int_attributes_generator = [&parameters](const std::string& weight_name) {
      const auto it = parameters.optimizer_int_attributes_map.find(weight_name);
      ORT_ENFORCE(
          it != parameters.optimizer_int_attributes_map.end(),
          "Failed to find int attribute map for weight ", weight_name);
      return it->second;
    };
    opt.use_mixed_precision_moments = parameters.use_fp16_moments;
    opt.do_all_reduce_in_mixed_precision_type = true;
    // TODO: this mapping is temporary.
    // For now, nccl allreduce kernel only implements for allreduce_post_accumulation
    // hovorod allreduce kernel only implements for not allreduce_post_accumulation.
    // eventually we will have one all reduce kernel and let opt to have
    // an allreduce_post_accumulation option and remove the use_nccl option.
    opt.use_nccl = parameters.allreduce_post_accumulation;
    opt.deepspeed_zero = onnxruntime::training::ZeROConfig(parameters.deepspeed_zero_stage);
    // TODO: The norm clipping value is 1.0f which is the default used in most frameworks.
    // Need to have another option to support more values in the future.
    opt.enable_grad_norm_clip = parameters.enable_grad_norm_clip;

    // TODO reduction types
    if (parameters.enable_adasum) {
#ifdef USE_CUDA
      opt.adasum_reduction_type = training::AdasumReductionType::GpuHierarchicalReduction;
#else
      opt.adasum_reduction_type = training::AdasumReductionType::CpuReduction;
#endif
    }

    config.optimizer_config = opt;
  }

  if (!parameters.optimizer_initial_state.empty()) {
    config.init_optimizer_states = parameters.optimizer_initial_state;
  }

  config.gradient_graph_config.use_invertible_layernorm_grad = parameters.use_invertible_layernorm_grad;
  config.gradient_graph_config.set_gradients_as_graph_outputs = parameters.set_gradients_as_graph_outputs;

  config.graph_transformer_config.attn_dropout_recompute = parameters.attn_dropout_recompute;
  config.graph_transformer_config.gelu_recompute = parameters.gelu_recompute;
  config.graph_transformer_config.transformer_layer_recompute = parameters.transformer_layer_recompute;
  config.graph_transformer_config.number_recompute_layers = parameters.number_recompute_layers;

  training::TrainingSession::TrainingConfigurationResult config_result{};

  OrtPybindThrowIfError(sess->ConfigureForTraining(config, config_result));

  TrainingConfigurationResult python_config_result{};
  if (config_result.mixed_precision_config_result.has_value()) {
    const auto& mp_config_result = config_result.mixed_precision_config_result.value();
    python_config_result.loss_scale_input_name = mp_config_result.loss_scale_input_name;
  }

  return python_config_result;
}
// TODO: this method does not handle parallel optimization.
this method does not handle parallel optimization.
this method does not handle parallel optimization .
TrainingConfigurationResult ConfigureSessionForTraining(
    training::TrainingSession* sess, TrainingParameters& parameters) {
  ORT_ENFORCE(parameters.horizontal_parallel_size <= parameters.world_size);
  ORT_ENFORCE(parameters.data_parallel_size <= parameters.world_size);
  if (parameters.world_size % parameters.data_parallel_size != 0) {
    throw std::runtime_error("Cannot split data parallel group because world_size is not divisible");
  }
  if (parameters.world_size % parameters.horizontal_parallel_size != 0) {
    throw std::runtime_error("Cannot split horizontal parallel group because world_size is not divisible");
  }
  auto data_group_size = parameters.world_size / (parameters.horizontal_parallel_size * parameters.pipeline_parallel_size);
  if (data_group_size != parameters.data_parallel_size) {
    LOGS(*(sess->GetLogger()), WARNING) << "data_parallel_size is not correct, tuned automatically to "
                                        << data_group_size;
    parameters.data_parallel_size = data_group_size;
  }
  training::TrainingSession::TrainingConfiguration config{};
  config.weight_names_to_train = parameters.weights_to_train;
  config.weight_names_to_not_train = parameters.weights_not_to_train;
  config.immutable_weights = parameters.immutable_weights;
  config.gradient_accumulation_steps = parameters.gradient_accumulation_steps;
  config.distributed_config.world_rank = parameters.world_rank;
  config.distributed_config.world_size = parameters.world_size;
  config.distributed_config.local_rank = parameters.local_rank;
  config.distributed_config.local_size = parameters.local_size;
  config.distributed_config.data_parallel_size = parameters.data_parallel_size;
  config.distributed_config.horizontal_parallel_size = parameters.horizontal_parallel_size;
  if (parameters.use_mixed_precision) {
    training::TrainingSession::TrainingConfiguration::MixedPrecisionConfiguration mp{};
    mp.use_mixed_precision_initializers = true;
    config.mixed_precision_config = mp;
  }
  config.loss_name = parameters.loss_output_name;
  if (!parameters.training_optimizer_name.empty()) {
    training::TrainingSession::TrainingConfiguration::OptimizerConfiguration opt{};
    opt.name = parameters.training_optimizer_name;
    opt.learning_rate_input_name = parameters.lr_params_feed_name;
    opt.weight_attributes_generator = [&parameters](const std::string& weight_name) {
      const auto it = parameters.optimizer_attributes_map.find(weight_name);
      ORT_ENFORCE(
          it != parameters.optimizer_attributes_map.end(),
          "Failed to find attribute map for weight ", weight_name);
      return it->second;
    };
    opt.weight_int_attributes_generator = [&parameters](const std::string& weight_name) {
      const auto it = parameters.optimizer_int_attributes_map.find(weight_name);
      ORT_ENFORCE(
          it != parameters.optimizer_int_attributes_map.end(),
          "Failed to find int attribute map for weight ", weight_name);
      return it->second;
    };
    opt.use_mixed_precision_moments = parameters.use_fp16_moments;
    opt.do_all_reduce_in_mixed_precision_type = true;
    opt.use_nccl = parameters.allreduce_post_accumulation;
    opt.deepspeed_zero = onnxruntime::training::ZeROConfig(parameters.deepspeed_zero_stage);
    opt.enable_grad_norm_clip = parameters.enable_grad_norm_clip;
    if (parameters.enable_adasum) {
#ifdef USE_CUDA
      opt.adasum_reduction_type = training::AdasumReductionType::GpuHierarchicalReduction;
#else
      opt.adasum_reduction_type = training::AdasumReductionType::CpuReduction;
#endif
    }
    config.optimizer_config = opt;
  }
  if (!parameters.optimizer_initial_state.empty()) {
    config.init_optimizer_states = parameters.optimizer_initial_state;
  }
  config.gradient_graph_config.use_invertible_layernorm_grad = parameters.use_invertible_layernorm_grad;
  config.gradient_graph_config.set_gradients_as_graph_outputs = parameters.set_gradients_as_graph_outputs;
  config.graph_transformer_config.attn_dropout_recompute = parameters.attn_dropout_recompute;
  config.graph_transformer_config.gelu_recompute = parameters.gelu_recompute;
  config.graph_transformer_config.transformer_layer_recompute = parameters.transformer_layer_recompute;
  config.graph_transformer_config.number_recompute_layers = parameters.number_recompute_layers;
  training::TrainingSession::TrainingConfigurationResult config_result{};
  OrtPybindThrowIfError(sess->ConfigureForTraining(config, config_result));
  TrainingConfigurationResult python_config_result{};
  if (config_result.mixed_precision_config_result.has_value()) {
    const auto& mp_config_result = config_result.mixed_precision_config_result.value();
    python_config_result.loss_scale_input_name = mp_config_result.loss_scale_input_name;
  }
  return python_config_result;
}
TrainingConfigurationResult ConfigureSessionForTraining ( training :: TrainingSession * sess , TrainingParameters & parameters ) { ORT_ENFORCE ( parameters . horizontal_parallel_size <= parameters . world_size ) ; ORT_ENFORCE ( parameters . data_parallel_size <= parameters . world_size ) ; if ( parameters . world_size % parameters . data_parallel_size != 0 ) { throw std :: runtime_error ( " " ) ; } if ( parameters . world_size % parameters . horizontal_parallel_size != 0 ) { throw std :: runtime_error ( " " ) ; } auto data_group_size = parameters . world_size / ( parameters . horizontal_parallel_size * parameters . pipeline_parallel_size ) ; if ( data_group_size != parameters . data_parallel_size ) { LOGS ( * ( sess -> GetLogger ( ) ) , WARNING ) << " " << data_group_size ; parameters . data_parallel_size = data_group_size ; } training :: TrainingSession :: TrainingConfiguration config { } ; config . weight_names_to_train = parameters . weights_to_train ; config . weight_names_to_not_train = parameters . weights_not_to_train ; config . immutable_weights = parameters . immutable_weights ; config . gradient_accumulation_steps = parameters . gradient_accumulation_steps ; config . distributed_config . world_rank = parameters . world_rank ; config . distributed_config . world_size = parameters . world_size ; config . distributed_config . local_rank = parameters . local_rank ; config . distributed_config . local_size = parameters . local_size ; config . distributed_config . data_parallel_size = parameters . data_parallel_size ; config . distributed_config . horizontal_parallel_size = parameters . horizontal_parallel_size ; if ( parameters . use_mixed_precision ) { training :: TrainingSession :: TrainingConfiguration :: MixedPrecisionConfiguration mp { } ; mp . use_mixed_precision_initializers = true ; config . mixed_precision_config = mp ; } config . loss_name = parameters . loss_output_name ; if ( ! parameters . training_optimizer_name . empty ( ) ) { training :: TrainingSession :: TrainingConfiguration :: OptimizerConfiguration opt { } ; opt . name = parameters . training_optimizer_name ; opt . learning_rate_input_name = parameters . lr_params_feed_name ; opt . weight_attributes_generator = [ & parameters ] ( const std :: string & weight_name ) { const auto it = parameters . optimizer_attributes_map . find ( weight_name ) ; ORT_ENFORCE ( it != parameters . optimizer_attributes_map . end ( ) , " " , weight_name ) ; return it -> second ; } ; opt . weight_int_attributes_generator = [ & parameters ] ( const std :: string & weight_name ) { const auto it = parameters . optimizer_int_attributes_map . find ( weight_name ) ; ORT_ENFORCE ( it != parameters . optimizer_int_attributes_map . end ( ) , " " , weight_name ) ; return it -> second ; } ; opt . use_mixed_precision_moments = parameters . use_fp16_moments ; opt . do_all_reduce_in_mixed_precision_type = true ; opt . use_nccl = parameters . allreduce_post_accumulation ; opt . deepspeed_zero = onnxruntime :: training :: ZeROConfig ( parameters . deepspeed_zero_stage ) ; opt . enable_grad_norm_clip = parameters . enable_grad_norm_clip ; if ( parameters . enable_adasum ) { #ifdef USE_CUDA opt . adasum_reduction_type = training :: AdasumReductionType :: GpuHierarchicalReduction ; #else opt . adasum_reduction_type = training :: AdasumReductionType :: CpuReduction ; #endif } config . optimizer_config = opt ; } if ( ! parameters . optimizer_initial_state . empty ( ) ) { config . init_optimizer_states = parameters . optimizer_initial_state ; } config . gradient_graph_config . use_invertible_layernorm_grad = parameters . use_invertible_layernorm_grad ; config . gradient_graph_config . set_gradients_as_graph_outputs = parameters . set_gradients_as_graph_outputs ; config . graph_transformer_config . attn_dropout_recompute = parameters . attn_dropout_recompute ; config . graph_transformer_config . gelu_recompute = parameters . gelu_recompute ; config . graph_transformer_config . transformer_layer_recompute = parameters . transformer_layer_recompute ; config . graph_transformer_config . number_recompute_layers = parameters . number_recompute_layers ; training :: TrainingSession :: TrainingConfigurationResult config_result { } ; OrtPybindThrowIfError ( sess -> ConfigureForTraining ( config , config_result ) ) ; TrainingConfigurationResult python_config_result { } ; if ( config_result . mixed_precision_config_result . has_value ( ) ) { const auto & mp_config_result = config_result . mixed_precision_config_result . value ( ) ; python_config_result . loss_scale_input_name = mp_config_result . loss_scale_input_name ; } return python_config_result ; }
TODO: this method does not handle parallel optimization.
TODO : this method does not handle parallel optimization .
//TODO tix, refactor the mpi related code to populate all fields correctly by default. // TODO: this mapping is temporary. // For now, nccl allreduce kernel only implements for allreduce_post_accumulation // hovorod allreduce kernel only implements for not allreduce_post_accumulation. // eventually we will have one all reduce kernel and let opt to have // an allreduce_post_accumulation option and remove the use_nccl option. // TODO: The norm clipping value is 1.0f which is the default used in most frameworks. // Need to have another option to support more values in the future. // TODO reduction types