pypet/trajectory.py
def _merge_parameters(self, other_trajectory, remove_duplicates=False,
                          trial_parameter_name=None,
                          ignore_data=()):
        """Merges parameters from the other trajectory into the current one.

        The explored parameters in the current trajectory are directly enlarged (in RAM),
        no storage service is needed here. Later on in `f_merge` the storage service
        will be requested to store the enlarge parameters to disk.

        Note explored parameters are always enlarged. Unexplored parameters might become
        new explored parameters if they differ in their default values
        in the current and the other trajectory, respectively.

        :return: A tuple with two elements:

            1.

                Dictionary of run index mappings from old trajectroy to the new one.

            2.

                List of names of parameters that were altered.

        """

        if trial_parameter_name:
            if remove_duplicates:
                self._logger.warning('You have given a trial parameter and you want to '
                                     'remove_items duplicates. There cannot be any duplicates '
                                     'when adding trials, I will not look for duplicates.')
                remove_duplicates = False

        # Dictionary containing full parameter names as keys
        # and pairs of parameters from both trajectories as values.
        # Parameters kept in this dictionary are marked for merging and will be enlarged
        # with ranges and values of corresponding parameters in the other trajectory
        params_to_change = {}

        if trial_parameter_name:
            # We want to merge a trial parameter
            # First make some sanity checks
            my_trial_parameter = self.f_get(trial_parameter_name)
            other_trial_parameter = other_trajectory.f_get(trial_parameter_name)
            if not isinstance(my_trial_parameter, BaseParameter):
                raise TypeError('Your trial_parameter `%s` does not evaluate to a real parameter'
                                ' in the trajectory' % trial_parameter_name)

            # Extract the ranges of both trial parameters
            if my_trial_parameter.f_has_range():
                my_trial_list = my_trial_parameter.f_get_range(copy=False)
            else:
                # If we only have a single trial, we need to make a range of length 1
                # This is probably a very exceptional case
                my_trial_list = [my_trial_parameter.f_get()]

            if other_trial_parameter.f_has_range():
                other_trial_list = other_trial_parameter.f_get_range(copy=False)
            else:
                other_trial_list = [other_trial_parameter.f_get()]

            # Make sanity checks if both ranges contain all numbers from 0 to T1
            # for the current trajectory and 0 to T2 for the other trajectory
            mytrialset = set(my_trial_list)
            mymaxtrial_T1 = max(mytrialset)  # maximum trial index in current trajectory aka T1

            if mytrialset != set(range(mymaxtrial_T1 + 1)):
                raise TypeError('In order to specify a trial parameter, this parameter must '
                                'contain integers from 0 to %d, but it in fact it '
                                'contains `%s`.' % (mymaxtrial_T1, str(mytrialset)))

            othertrialset = set(other_trial_list)
            othermaxtrial_T2 = max(othertrialset)  # maximum trial index in other trajectory aka T2
            if othertrialset != set(range(othermaxtrial_T2 + 1)):
                raise TypeError('In order to specify a trial parameter, this parameter must '
                                'contain integers from 0 to %d, but it infact it contains `%s` '
                                'in the other trajectory.' %
                                (othermaxtrial_T2, str(othertrialset)))

            # If the trial parameter's name was just given in parts we update it here
            # to the full name
            trial_parameter_name = my_trial_parameter.v_full_name

            # If we had the very exceptional case, that our trial parameter was not explored,
            # aka we only had 1 trial, we have to add it to the explored parameters
            if not trial_parameter_name in self._explored_parameters:
                self._explored_parameters[trial_parameter_name] = my_trial_parameter

            # We need to mark the trial parameter for merging
            params_to_change[trial_parameter_name] = (my_trial_parameter, other_trial_parameter)

        # Dictionary containing all parameters of the other trajectory, we will iterate through it
        # to spot parameters that need to be enlarge or become new explored parameters
        params_to_merge = other_trajectory._parameters.copy()

        params_to_merge.update(other_trajectory._derived_parameters)
        for ignore in ignore_data:
            if ignore in params_to_merge:
                del params_to_merge[ignore]

        run_name_dummys = set([f(-1) for f in other_trajectory._wildcard_functions.values()])
        # Iterate through all parameters of the other trajectory
        # and check which differ from the parameters of the current trajectory
        for key in params_to_merge:
            other_param = params_to_merge[key]

            # We don't need to merge anything based on wildcards
            split_key = key.split('.')
            if any(x in other_trajectory._reversed_wildcards for x in split_key):
                continue

            my_param = self.f_get(key)
            if not my_param._values_of_same_type(my_param.f_get(), other_param.f_get()):
                raise TypeError('The parameters with name `%s` are not of the same type, cannot '
                                'merge trajectory.' % key)

            # We have taken care about the trial parameter before, it is already
            # marked for merging
            if my_param.v_full_name == trial_parameter_name:
                continue

            # If a parameter was explored in one of the trajectories or two unexplored
            # parameters differ, we need to mark them for merge
            if (my_param.f_has_range() or
                    other_param.f_has_range() or
                    not my_param._equal_values(my_param.f_get(), other_param.f_get())):

                # If two unexplored parameters differ, that means they differ in every run,
                # accordingly we do not need to check for duplicate runs anymore
                params_to_change[key] = (my_param, other_param)
                if not my_param.f_has_range() and not other_param.f_has_range():
                    remove_duplicates = False

        # Check if we use all runs or remove duplicates:
        used_runs = {}
        for idx in range(len(other_trajectory)):
            used_runs[idx] = idx
        if remove_duplicates:

            # We need to compare all parameter combinations in the current trajectory
            # to all parameter combinations in the other trajectory to spot duplicate points.
            # Quadratic Complexity!
            for irun in range(len(other_trajectory)):
                for jrun in range(len(self)):
                    change = True

                    # Check all marked parameters
                    for my_param, other_param in params_to_change.values():
                        if other_param.f_has_range():
                            other_param._set_parameter_access(irun)

                        if my_param.f_has_range():
                            my_param._set_parameter_access(jrun)

                        val1 = my_param.f_get()
                        val2 = other_param.f_get()

                        # If only one parameter differs, the parameter space point differs
                        # and we can skip the rest of the parameters
                        if not my_param._equal_values(val1, val2):
                            change = False
                            break

                    # If we found one parameter space point in the current trajectory
                    # that matches the ith point in the other, we do not need the ith
                    # point. We can also skip comparing to the rest of the points in the
                    # current trajectory
                    if change:
                        del used_runs[irun]
                        break

            # Restore changed default values
            for my_param, other_param in params_to_change.values():
                other_param._restore_default()
                my_param._restore_default()


        # Merge parameters into the current trajectory
        adding_length = len(used_runs)
        starting_length = len(self)
        if adding_length == 0:
            return used_runs, []
        count = 0
        for key in sorted(used_runs.keys()):
            used_runs[key] = starting_length + count
            count += 1

        for my_param, other_param in params_to_change.values():
            fullname = my_param.v_full_name

            # We need new ranges to enlarge all parameters marked for merging
            if fullname == trial_parameter_name:
                # The trial parameter now has to cover the range 0 to T1+T2+1
                other_range = [x + mymaxtrial_T1 + 1 for x in other_trial_list]
            else:
                # In case we do not use all runs we need to filter the ranges of the
                # parameters of the other trajectory
                if other_param.f_has_range():
                    other_range = (x for jdx, x in enumerate(other_param.f_get_range(copy=False))
                                                             if jdx in used_runs)
                else:
                    other_range = (other_param.f_get() for _ in range(adding_length))

            # If a parameter in the current trajectory was marked for merging but was not
            # explored before, we need to explore it first, simply by creating the range of
            # the current trajectory's length containing only it's default value
            if not my_param.f_has_range():
                my_param.f_unlock()
                my_param._explore((my_param.f_get() for _ in  range(len(self))))

            # After determining the new range extension `other_range`,
            # expand the parameters
            my_param.f_unlock()
            my_param._expand(other_range)

            if not fullname in self._explored_parameters:
                self._explored_parameters[fullname] = my_param

        return used_runs, list(params_to_change.keys())
def _merge_parameters(self, other_trajectory, remove_duplicates=False,
                          trial_parameter_name=None,
                          ignore_data=()):
        """Merges parameters from the other trajectory into the current one.

        The explored parameters in the current trajectory are directly enlarged (in RAM),
        no storage service is needed here. Later on in `f_merge` the storage service
        will be requested to store the enlarge parameters to disk.

        Note explored parameters are always enlarged. Unexplored parameters might become
        new explored parameters if they differ in their default values
        in the current and the other trajectory, respectively.

        :return: A tuple with two elements:

            1.

                Dictionary of run index mappings from old trajectroy to the new one.

            2.

                List of names of parameters that were altered.

        """

        if trial_parameter_name:
            if remove_duplicates:
                self._logger.warning('You have given a trial parameter and you want to '
                                     'remove_items duplicates. There cannot be any duplicates '
                                     'when adding trials, I will not look for duplicates.')
                remove_duplicates = False

        # Dictionary containing full parameter names as keys
        # and pairs of parameters from both trajectories as values.
        # Parameters kept in this dictionary are marked for merging and will be enlarged
        # with ranges and values of corresponding parameters in the other trajectory
        params_to_change = {}

        if trial_parameter_name:
            # We want to merge a trial parameter
            # First make some sanity checks
            my_trial_parameter = self.f_get(trial_parameter_name)
            other_trial_parameter = other_trajectory.f_get(trial_parameter_name)
            if not isinstance(my_trial_parameter, BaseParameter):
                raise TypeError('Your trial_parameter `%s` does not evaluate to a real parameter'
                                ' in the trajectory' % trial_parameter_name)

            # Extract the ranges of both trial parameters
            if my_trial_parameter.f_has_range():
                my_trial_list = my_trial_parameter.f_get_range(copy=False)
            else:
                # If we only have a single trial, we need to make a range of length 1
                # This is probably a very exceptional case
                my_trial_list = [my_trial_parameter.f_get()]

            if other_trial_parameter.f_has_range():
                other_trial_list = other_trial_parameter.f_get_range(copy=False)
            else:
                other_trial_list = [other_trial_parameter.f_get()]

            # Make sanity checks if both ranges contain all numbers from 0 to T1
            # for the current trajectory and 0 to T2 for the other trajectory
            mytrialset = set(my_trial_list)
            mymaxtrial_T1 = max(mytrialset)  # maximum trial index in current trajectory aka T1

            if mytrialset != set(range(mymaxtrial_T1 + 1)):
                raise TypeError('In order to specify a trial parameter, this parameter must '
                                'contain integers from 0 to %d, but it in fact it '
                                'contains `%s`.' % (mymaxtrial_T1, str(mytrialset)))

            othertrialset = set(other_trial_list)
            othermaxtrial_T2 = max(othertrialset)  # maximum trial index in other trajectory aka T2
            if othertrialset != set(range(othermaxtrial_T2 + 1)):
                raise TypeError('In order to specify a trial parameter, this parameter must '
                                'contain integers from 0 to %d, but it infact it contains `%s` '
                                'in the other trajectory.' %
                                (othermaxtrial_T2, str(othertrialset)))

            # If the trial parameter's name was just given in parts we update it here
            # to the full name
            trial_parameter_name = my_trial_parameter.v_full_name

            # If we had the very exceptional case, that our trial parameter was not explored,
            # aka we only had 1 trial, we have to add it to the explored parameters
            if not trial_parameter_name in self._explored_parameters:
                self._explored_parameters[trial_parameter_name] = my_trial_parameter

            # We need to mark the trial parameter for merging
            params_to_change[trial_parameter_name] = (my_trial_parameter, other_trial_parameter)

        # Dictionary containing all parameters of the other trajectory, we will iterate through it
        # to spot parameters that need to be enlarge or become new explored parameters
        params_to_merge = other_trajectory._parameters.copy()

        params_to_merge.update(other_trajectory._derived_parameters)
        for ignore in ignore_data:
            if ignore in params_to_merge:
                del params_to_merge[ignore]

        run_name_dummys = set([f(-1) for f in other_trajectory._wildcard_functions.values()])
        # Iterate through all parameters of the other trajectory
        # and check which differ from the parameters of the current trajectory
        for key in params_to_merge:
            other_param = params_to_merge[key]

            # We don't need to merge anything based on wildcards
            split_key = key.split('.')
            if any(x in other_trajectory._reversed_wildcards for x in split_key):
                continue

            my_param = self.f_get(key)
            if not my_param._values_of_same_type(my_param.f_get(), other_param.f_get()):
                raise TypeError('The parameters with name `%s` are not of the same type, cannot '
                                'merge trajectory.' % key)

            # We have taken care about the trial parameter before, it is already
            # marked for merging
            if my_param.v_full_name == trial_parameter_name:
                continue

            # If a parameter was explored in one of the trajectories or two unexplored
            # parameters differ, we need to mark them for merge
            if (my_param.f_has_range() or
                    other_param.f_has_range() or
                    not my_param._equal_values(my_param.f_get(), other_param.f_get())):

                # If two unexplored parameters differ, that means they differ in every run,
                # accordingly we do not need to check for duplicate runs anymore
                params_to_change[key] = (my_param, other_param)
                if not my_param.f_has_range() and not other_param.f_has_range():
                    remove_duplicates = False

        # Check if we use all runs or remove duplicates:
        used_runs = {}
        for idx in range(len(other_trajectory)):
            used_runs[idx] = idx
        if remove_duplicates:

            # We need to compare all parameter combinations in the current trajectory
            # to all parameter combinations in the other trajectory to spot duplicate points.
            # Quadratic Complexity!
            for irun in range(len(other_trajectory)):
                for jrun in range(len(self)):
                    change = True

                    # Check all marked parameters
                    for my_param, other_param in params_to_change.values():
                        if other_param.f_has_range():
                            other_param._set_parameter_access(irun)

                        if my_param.f_has_range():
                            my_param._set_parameter_access(jrun)

                        val1 = my_param.f_get()
                        val2 = other_param.f_get()

                        # If only one parameter differs, the parameter space point differs
                        # and we can skip the rest of the parameters
                        if not my_param._equal_values(val1, val2):
                            change = False
                            break

                    # If we found one parameter space point in the current trajectory
                    # that matches the ith point in the other, we do not need the ith
                    # point. We can also skip comparing to the rest of the points in the
                    # current trajectory
                    if change:
                        del used_runs[irun]
                        break

            # Restore changed default values
            for my_param, other_param in params_to_change.values():
                other_param._restore_default()
                my_param._restore_default()


        # Merge parameters into the current trajectory
        adding_length = len(used_runs)
        starting_length = len(self)
        if adding_length == 0:
            return used_runs, []
        count = 0
        for key in sorted(used_runs.keys()):
            used_runs[key] = starting_length + count
            count += 1

        for my_param, other_param in params_to_change.values():
            fullname = my_param.v_full_name

            # We need new ranges to enlarge all parameters marked for merging
            if fullname == trial_parameter_name:
                # The trial parameter now has to cover the range 0 to T1+T2+1
                other_range = [x + mymaxtrial_T1 + 1 for x in other_trial_list]
            else:
                # In case we do not use all runs we need to filter the ranges of the
                # parameters of the other trajectory
                if other_param.f_has_range():
                    other_range = (x for jdx, x in enumerate(other_param.f_get_range(copy=False))
                                                             if jdx in used_runs)
                else:
                    other_range = (other_param.f_get() for _ in range(adding_length))

            # If a parameter in the current trajectory was marked for merging but was not
            # explored before, we need to explore it first, simply by creating the range of
            # the current trajectory's length containing only it's default value
            if not my_param.f_has_range():
                my_param.f_unlock()
                my_param._explore((my_param.f_get() for _ in  range(len(self))))

            # After determining the new range extension `other_range`,
            # expand the parameters
            my_param.f_unlock()
            my_param._expand(other_range)

            if not fullname in self._explored_parameters:
                self._explored_parameters[fullname] = my_param

        return used_runs, list(params_to_change.keys())
def _merge_parameters ( self , other_trajectory , remove_duplicates = False , trial_parameter_name = None , ignore_data = ( ) ) : if trial_parameter_name : if remove_duplicates : self . _logger . warning ( 'You have given a trial parameter and you want to ' 'remove_items duplicates. There cannot be any duplicates ' 'when adding trials, I will not look for duplicates.' ) remove_duplicates = False # Dictionary containing full parameter names as keys # and pairs of parameters from both trajectories as values. # Parameters kept in this dictionary are marked for merging and will be enlarged # with ranges and values of corresponding parameters in the other trajectory params_to_change = { } if trial_parameter_name : # We want to merge a trial parameter # First make some sanity checks my_trial_parameter = self . f_get ( trial_parameter_name ) other_trial_parameter = other_trajectory . f_get ( trial_parameter_name ) if not isinstance ( my_trial_parameter , BaseParameter ) : raise TypeError ( 'Your trial_parameter `%s` does not evaluate to a real parameter' ' in the trajectory' % trial_parameter_name ) # Extract the ranges of both trial parameters if my_trial_parameter . f_has_range ( ) : my_trial_list = my_trial_parameter . f_get_range ( copy = False ) else : # If we only have a single trial, we need to make a range of length 1 # This is probably a very exceptional case my_trial_list = [ my_trial_parameter . f_get ( ) ] if other_trial_parameter . f_has_range ( ) : other_trial_list = other_trial_parameter . f_get_range ( copy = False ) else : other_trial_list = [ other_trial_parameter . f_get ( ) ] # Make sanity checks if both ranges contain all numbers from 0 to T1 # for the current trajectory and 0 to T2 for the other trajectory mytrialset = set ( my_trial_list ) mymaxtrial_T1 = max ( mytrialset ) # maximum trial index in current trajectory aka T1 if mytrialset != set ( range ( mymaxtrial_T1 + 1 ) ) : raise TypeError ( 'In order to specify a trial parameter, this parameter must ' 'contain integers from 0 to %d, but it in fact it ' 'contains `%s`.' % ( mymaxtrial_T1 , str ( mytrialset ) ) ) othertrialset = set ( other_trial_list ) othermaxtrial_T2 = max ( othertrialset ) # maximum trial index in other trajectory aka T2 if othertrialset != set ( range ( othermaxtrial_T2 + 1 ) ) : raise TypeError ( 'In order to specify a trial parameter, this parameter must ' 'contain integers from 0 to %d, but it infact it contains `%s` ' 'in the other trajectory.' % ( othermaxtrial_T2 , str ( othertrialset ) ) ) # If the trial parameter's name was just given in parts we update it here # to the full name trial_parameter_name = my_trial_parameter . v_full_name # If we had the very exceptional case, that our trial parameter was not explored, # aka we only had 1 trial, we have to add it to the explored parameters if not trial_parameter_name in self . _explored_parameters : self . _explored_parameters [ trial_parameter_name ] = my_trial_parameter # We need to mark the trial parameter for merging params_to_change [ trial_parameter_name ] = ( my_trial_parameter , other_trial_parameter ) # Dictionary containing all parameters of the other trajectory, we will iterate through it # to spot parameters that need to be enlarge or become new explored parameters params_to_merge = other_trajectory . _parameters . copy ( ) params_to_merge . update ( other_trajectory . _derived_parameters ) for ignore in ignore_data : if ignore in params_to_merge : del params_to_merge [ ignore ] run_name_dummys = set ( [ f ( - 1 ) for f in other_trajectory . _wildcard_functions . values ( ) ] ) # Iterate through all parameters of the other trajectory # and check which differ from the parameters of the current trajectory for key in params_to_merge : other_param = params_to_merge [ key ] # We don't need to merge anything based on wildcards split_key = key . split ( '.' ) if any ( x in other_trajectory . _reversed_wildcards for x in split_key ) : continue my_param = self . f_get ( key ) if not my_param . _values_of_same_type ( my_param . f_get ( ) , other_param . f_get ( ) ) : raise TypeError ( 'The parameters with name `%s` are not of the same type, cannot ' 'merge trajectory.' % key ) # We have taken care about the trial parameter before, it is already # marked for merging if my_param . v_full_name == trial_parameter_name : continue # If a parameter was explored in one of the trajectories or two unexplored # parameters differ, we need to mark them for merge if ( my_param . f_has_range ( ) or other_param . f_has_range ( ) or not my_param . _equal_values ( my_param . f_get ( ) , other_param . f_get ( ) ) ) : # If two unexplored parameters differ, that means they differ in every run, # accordingly we do not need to check for duplicate runs anymore params_to_change [ key ] = ( my_param , other_param ) if not my_param . f_has_range ( ) and not other_param . f_has_range ( ) : remove_duplicates = False # Check if we use all runs or remove duplicates: used_runs = { } for idx in range ( len ( other_trajectory ) ) : used_runs [ idx ] = idx if remove_duplicates : # We need to compare all parameter combinations in the current trajectory # to all parameter combinations in the other trajectory to spot duplicate points. # Quadratic Complexity! for irun in range ( len ( other_trajectory ) ) : for jrun in range ( len ( self ) ) : change = True # Check all marked parameters for my_param , other_param in params_to_change . values ( ) : if other_param . f_has_range ( ) : other_param . _set_parameter_access ( irun ) if my_param . f_has_range ( ) : my_param . _set_parameter_access ( jrun ) val1 = my_param . f_get ( ) val2 = other_param . f_get ( ) # If only one parameter differs, the parameter space point differs # and we can skip the rest of the parameters if not my_param . _equal_values ( val1 , val2 ) : change = False break # If we found one parameter space point in the current trajectory # that matches the ith point in the other, we do not need the ith # point. We can also skip comparing to the rest of the points in the # current trajectory if change : del used_runs [ irun ] break # Restore changed default values for my_param , other_param in params_to_change . values ( ) : other_param . _restore_default ( ) my_param . _restore_default ( ) # Merge parameters into the current trajectory adding_length = len ( used_runs ) starting_length = len ( self ) if adding_length == 0 : return used_runs , [ ] count = 0 for key in sorted ( used_runs . keys ( ) ) : used_runs [ key ] = starting_length + count count += 1 for my_param , other_param in params_to_change . values ( ) : fullname = my_param . v_full_name # We need new ranges to enlarge all parameters marked for merging if fullname == trial_parameter_name : # The trial parameter now has to cover the range 0 to T1+T2+1 other_range = [ x + mymaxtrial_T1 + 1 for x in other_trial_list ] else : # In case we do not use all runs we need to filter the ranges of the # parameters of the other trajectory if other_param . f_has_range ( ) : other_range = ( x for jdx , x in enumerate ( other_param . f_get_range ( copy = False ) ) if jdx in used_runs ) else : other_range = ( other_param . f_get ( ) for _ in range ( adding_length ) ) # If a parameter in the current trajectory was marked for merging but was not # explored before, we need to explore it first, simply by creating the range of # the current trajectory's length containing only it's default value if not my_param . f_has_range ( ) : my_param . f_unlock ( ) my_param . _explore ( ( my_param . f_get ( ) for _ in range ( len ( self ) ) ) ) # After determining the new range extension `other_range`, # expand the parameters my_param . f_unlock ( ) my_param . _expand ( other_range ) if not fullname in self . _explored_parameters : self . _explored_parameters [ fullname ] = my_param return used_runs , list ( params_to_change . keys ( ) )
Merges parameters from the other trajectory into the current one.

        The explored parameters in the current trajectory are directly enlarged (in RAM),
        no storage service is needed here. Later on in `f_merge` the storage service
        will be requested to store the enlarge parameters to disk.

        Note explored parameters are always enlarged. Unexplored parameters might become
        new explored parameters if they differ in their default values
        in the current and the other trajectory, respectively.

        :return: A tuple with two elements:

            1.

                Dictionary of run index mappings from old trajectroy to the new one.

            2.

                List of names of parameters that were altered.
Merges parameters from the other trajectory into the current one .