deprecated_code/workflows/mpi/imaging/imaging_mpi.py
def invert_list_mpi_workflow(vis_list, template_model_imagelist, context, dopsf=False, normalize=True,
                                    facets=1, vis_slices=1, gcfcf=None, 
                                comm=MPI.COMM_WORLD, **kwargs):
    """ Sum results from invert, iterating over the scattered image and vis_list

    :param vis_list: Only full for rank==0
    :param template_model_imagelist: Model used to determine image parameters
    (in rank=0)
    :param dopsf: Make the PSF instead of the dirty image
    :param facets: Number of facets
    :param normalize: Normalize by sumwt
    :param vis_slices: Number of slices
    :param context: Imaging context
    :param gcfcg: tuple containing grid correction and convolution function (in
    rank=0)
    :param comm:MPI Communicator
    :param kwargs: Parameters for functions in components
    :return: List of (image, sumwt) tuple
   """
    
   # NOTE: Be careful with normalization as normalizing parts is not the 
   # same as normalizing the whole, normalization happens for each image in a
   # frequency window (in this versio we only parallelize at freqwindows
    def concat_tuples(list_of_tuples):
        if len(list_of_tuples)<2:
            result_list=list_of_tuples
        else:
            result_list=list_of_tuples[0]
            for l in list_of_tuples[1:]:
                result_list+=l
        return result_list


    rank = comm.Get_rank()
    size = comm.Get_size()
    log.info('%d: In invert_list_mpi_workflow: %d elements in vis_list %d in model' %
          (rank,len(vis_list),len(template_model_imagelist)))

    assert get_parameter(kwargs, "use_serial_invert", True),"Only freq paralellization implemented"
    #if get_parameter(kwargs, "use_serial_invert", False):
    if get_parameter(kwargs, "use_serial_invert", True):
        from workflows.serial.imaging.imaging_serial import invert_list_serial_workflow

        results_vislist = list()
        # Distribute visibilities and model by freq
        sub_vis_list= numpy.array_split(vis_list, size)
        sub_vis_list=comm.scatter(sub_vis_list,root=0)
        sub_template_model_imagelist= numpy.array_split(template_model_imagelist, size)
        sub_template_model_imagelist=comm.scatter(sub_template_model_imagelist,root=0)
        if gcfcf is not None:
            sub_gcfcf = numpy.array_split(gcfcf,size)
            sub_gcfcf=comm.scatter(sub_gcfcf,root=0)
        isinstance(sub_vis_list[0], Visibility)
        sub_results_vislist = [invert_list_serial_workflow(vis_list=[sub_vis_list[i]],
                     template_model_imagelist=[sub_template_model_imagelist[i]],
                     context=context, dopsf=dopsf, normalize=normalize, 
                                                       vis_slices=vis_slices,
                     facets=facets, gcfcf=gcfcf, **kwargs)[0]
                for i, _ in enumerate(sub_vis_list)]
        #print("%d sub_results_vislist" %rank,sub_results_vislist)
        results_vislist=comm.gather(sub_results_vislist,root=0)
        #print("%d results_vislist before concatenate"%rank,results_vislist)
        if rank == 0:
            #image_results_list_list=[x for x in image_results_list_list if x]
            #results_vislist=numpy.concatenate(results_vislist)
            # TODO: concatenate dos not concatenate well a list of tuples 
            # it returns a 2d array instead of a concatenated list of tuples

            results_vislist =concat_tuples(results_vislist)
        else:
            results_vislist=list()
    #print("%d results_vislist"%rank,results_vislist)
    return results_vislist
 Sum results from invert, iterating over the scattered image and vis_list

    :param vis_list: Only full for rank==0
    :param template_model_imagelist: Model used to determine image parameters
    (in rank=0)
    :param dopsf: Make the PSF instead of the dirty image
    :param facets: Number of facets
    :param normalize: Normalize by sumwt
    :param vis_slices: Number of slices
    :param context: Imaging context
    :param gcfcg: tuple containing grid correction and convolution function (in
    rank=0)
    :param comm:MPI Communicator
    :param kwargs: Parameters for functions in components
    :return: List of (image, sumwt) tuple
   
Sum results from invert, iterating over the scattered image and vis_list
Sum results from invert iterating over the scattered image and vis_list
def invert_list_mpi_workflow(vis_list, template_model_imagelist, context, dopsf=False, normalize=True,
                                    facets=1, vis_slices=1, gcfcf=None, 
                                comm=MPI.COMM_WORLD, **kwargs):
    def concat_tuples(list_of_tuples):
        if len(list_of_tuples)<2:
            result_list=list_of_tuples
        else:
            result_list=list_of_tuples[0]
            for l in list_of_tuples[1:]:
                result_list+=l
        return result_list
    rank = comm.Get_rank()
    size = comm.Get_size()
    log.info('%d: In invert_list_mpi_workflow: %d elements in vis_list %d in model' %
          (rank,len(vis_list),len(template_model_imagelist)))
    assert get_parameter(kwargs, "use_serial_invert", True),"Only freq paralellization implemented"
    if get_parameter(kwargs, "use_serial_invert", True):
        from workflows.serial.imaging.imaging_serial import invert_list_serial_workflow
        results_vislist = list()
        sub_vis_list= numpy.array_split(vis_list, size)
        sub_vis_list=comm.scatter(sub_vis_list,root=0)
        sub_template_model_imagelist= numpy.array_split(template_model_imagelist, size)
        sub_template_model_imagelist=comm.scatter(sub_template_model_imagelist,root=0)
        if gcfcf is not None:
            sub_gcfcf = numpy.array_split(gcfcf,size)
            sub_gcfcf=comm.scatter(sub_gcfcf,root=0)
        isinstance(sub_vis_list[0], Visibility)
        sub_results_vislist = [invert_list_serial_workflow(vis_list=[sub_vis_list[i]],
                     template_model_imagelist=[sub_template_model_imagelist[i]],
                     context=context, dopsf=dopsf, normalize=normalize, 
                                                       vis_slices=vis_slices,
                     facets=facets, gcfcf=gcfcf, **kwargs)[0]
                for i, _ in enumerate(sub_vis_list)]
        results_vislist=comm.gather(sub_results_vislist,root=0)
        if rank == 0:
            results_vislist =concat_tuples(results_vislist)
        else:
            results_vislist=list()
    return results_vislist
def invert_list_mpi_workflow ( vis_list , template_model_imagelist , context , dopsf = False , normalize = True , facets = 1 , vis_slices = 1 , gcfcf = None , comm = MPI . COMM_WORLD , ** kwargs ) : def concat_tuples ( list_of_tuples ) : if len ( list_of_tuples ) < 2 : result_list = list_of_tuples else : result_list = list_of_tuples [ 0 ] for l in list_of_tuples [ 1 : ] : result_list += l return result_list rank = comm . Get_rank ( ) size = comm . Get_size ( ) log . info ( '%d: In invert_list_mpi_workflow: %d elements in vis_list %d in model' % ( rank , len ( vis_list ) , len ( template_model_imagelist ) ) ) assert get_parameter ( kwargs , "use_serial_invert" , True ) , "Only freq paralellization implemented" if get_parameter ( kwargs , "use_serial_invert" , True ) : from workflows . serial . imaging . imaging_serial import invert_list_serial_workflow results_vislist = list ( ) sub_vis_list = numpy . array_split ( vis_list , size ) sub_vis_list = comm . scatter ( sub_vis_list , root = 0 ) sub_template_model_imagelist = numpy . array_split ( template_model_imagelist , size ) sub_template_model_imagelist = comm . scatter ( sub_template_model_imagelist , root = 0 ) if gcfcf is not None : sub_gcfcf = numpy . array_split ( gcfcf , size ) sub_gcfcf = comm . scatter ( sub_gcfcf , root = 0 ) isinstance ( sub_vis_list [ 0 ] , Visibility ) sub_results_vislist = [ invert_list_serial_workflow ( vis_list = [ sub_vis_list [ i ] ] , template_model_imagelist = [ sub_template_model_imagelist [ i ] ] , context = context , dopsf = dopsf , normalize = normalize , vis_slices = vis_slices , facets = facets , gcfcf = gcfcf , ** kwargs ) [ 0 ] for i , _ in enumerate ( sub_vis_list ) ] results_vislist = comm . gather ( sub_results_vislist , root = 0 ) if rank == 0 : results_vislist = concat_tuples ( results_vislist ) else : results_vislist = list ( ) return results_vislist
Sum results from invert, iterating over the scattered image and vis_list
Sum results from invert iterating over the scattered image and vis_list
""" Sum results from invert, iterating over the scattered image and vis_list

    :param vis_list: Only full for rank==0
    :param template_model_imagelist: Model used to determine image parameters
    (in rank=0)
    :param dopsf: Make the PSF instead of the dirty image
    :param facets: Number of facets
    :param normalize: Normalize by sumwt
    :param vis_slices: Number of slices
    :param context: Imaging context
    :param gcfcg: tuple containing grid correction and convolution function (in
    rank=0)
    :param comm:MPI Communicator
    :param kwargs: Parameters for functions in components
    :return: List of (image, sumwt) tuple
   """ # NOTE: Be careful with normalization as normalizing parts is not the  # same as normalizing the whole, normalization happens for each image in a # frequency window (in this versio we only parallelize at freqwindows #if get_parameter(kwargs, "use_serial_invert", False): # Distribute visibilities and model by freq #print("%d sub_results_vislist" %rank,sub_results_vislist) #print("%d results_vislist before concatenate"%rank,results_vislist) #image_results_list_list=[x for x in image_results_list_list if x] #results_vislist=numpy.concatenate(results_vislist) # TODO: concatenate dos not concatenate well a list of tuples  # it returns a 2d array instead of a concatenated list of tuples #print("%d results_vislist"%rank,results_vislist)
List of (image, sumwt) tuple
List of ( image sumwt ) tuple
vis_list
Only full for rank==0
Only full for rank == 0
template_model_imagelist
Model used to determine image parameters
(in rank=0)
Model used to determine image parameters ( in rank = 0 )
context
None
dopsf
Make the PSF instead of the dirty image
Make the PSF instead of the dirty image
normalize
Normalize by sumwt
Normalize by sumwt
facets
Number of facets
Number of facets
vis_slices
Number of slices
Number of slices
gcfcf
comm
None
gcfcg
tuple containing grid correction and convolution function (in
rank=0)
tuple containing grid correction and convolution function ( in rank = 0 )
kwargs
Parameters for functions in components
Parameters for functions in components