paleo/simulation.py
def simulate_model_parallel(nested_list, layer_list, batch_size, device,
                            network, use_pipeline, use_only_gemm, ppp_comp,
                            ppp_comm):
    """Run simulation for model parallel."""
    # Get times for different batch_sizes:
    forward_times, params_in_bytes = _profile_for_batch_size(
        layer_list, 'forward', device, batch_size, use_only_gemm, ppp_comp,
        ppp_comm)
    backward_times, _ = _profile_for_batch_size(layer_list, 'backward', device,
                                                batch_size, use_only_gemm,
                                                ppp_comp, ppp_comm)

    result_times = []
    for direction, times in [('Fwd', forward_times), ('Bwd', backward_times)]:
        lower, upper = _sum_with_parallel(nested_list, layer_list, times)
        result_times.append([direction, lower, upper])
    headers = ['Pass', 'Lowerbound, Upperbound']
    return headers, result_times
Run simulation for model parallel.
Run simulation for model parallel.
Run simulation for model parallel .
def simulate_model_parallel(nested_list, layer_list, batch_size, device,
                            network, use_pipeline, use_only_gemm, ppp_comp,
                            ppp_comm):
    forward_times, params_in_bytes = _profile_for_batch_size(
        layer_list, 'forward', device, batch_size, use_only_gemm, ppp_comp,
        ppp_comm)
    backward_times, _ = _profile_for_batch_size(layer_list, 'backward', device,
                                                batch_size, use_only_gemm,
                                                ppp_comp, ppp_comm)
    result_times = []
    for direction, times in [('Fwd', forward_times), ('Bwd', backward_times)]:
        lower, upper = _sum_with_parallel(nested_list, layer_list, times)
        result_times.append([direction, lower, upper])
    headers = ['Pass', 'Lowerbound, Upperbound']
    return headers, result_times
def simulate_model_parallel ( nested_list , layer_list , batch_size , device , network , use_pipeline , use_only_gemm , ppp_comp , ppp_comm ) : forward_times , params_in_bytes = _profile_for_batch_size ( layer_list , 'forward' , device , batch_size , use_only_gemm , ppp_comp , ppp_comm ) backward_times , _ = _profile_for_batch_size ( layer_list , 'backward' , device , batch_size , use_only_gemm , ppp_comp , ppp_comm ) result_times = [ ] for direction , times in [ ( 'Fwd' , forward_times ) , ( 'Bwd' , backward_times ) ] : lower , upper = _sum_with_parallel ( nested_list , layer_list , times ) result_times . append ( [ direction , lower , upper ] ) headers = [ 'Pass' , 'Lowerbound, Upperbound' ] return headers , result_times
Run simulation for model parallel.
Run simulation for model parallel .
"""Run simulation for model parallel.""" # Get times for different batch_sizes:
nested_list
layer_list
batch_size
device
network
use_pipeline
use_only_gemm
ppp_comp
ppp_comm