def parallelNelderMead(objFunc,guess,perturb=None,P=1,ftol=0.000001,xtol=0.00000001,maxiter=np.inf,maxeval=np.inf,r_param=1.0,e_param=1.0,c_param=0.5,s_param=0.5,maxcores=None,name=None,resume=False,savefreq=None,verbose=1):
    '''
    A parallel implementation of the Nelder-Mead minimization algorithm, as
    described in Lee and Wiswall.  For long optimization procedures, it can
    save progress between iterations and resume later.

    Parameters
    ----------
    objFunc : function
        The objective function to be minimized. Takes a single 1D array as input.
    guess : np.array
        Initial starting point for the simplex, representing an input for objFunc.
    perturb : np.array
        Perturbation vector for the simplex, of the same length as an input to
        objFunc.  If perturb[j] is non-zero, a simplex point will be created
        that perturbs the j-th element of guess by perturb[j]; if it is zero,
        then the j-th parameter of objFunc will not be optimized over.  By
        default, guess=None, indicating that all parameters should be optimized,
        with an initial perturbation of 0.1*guess.
    P : int
        Degree of parallelization: the number of vertices of the simplex to try
        to update on each iteration of the process.
    ftol : float
        Absolute tolerance of the objective function for convergence.  If suc-
        cessive iterations return minimum function values that differ by less
        than ftol, the process terminates successfully.
    xtol : float
        Absolute tolerance of the input values for convergence.  If the maximum
        distance between the current minimum point and the worst point in the
        simplex is less than xtol, then the process terminates successfully.
    maxiter : int
        Maximum number of Nelder-Mead iterations; reaching iters=maxiter is
        reported as an "unsuccessful" minimization.
    maxeval : int
        Maximum number of evaluations of objFunc (across all processes); reaching
        evals=maxeval is reported as an "unsuccessful" minimization.
    r_param: float
        Parameter indicating magnitude of the reflection point calculation.
    e_param: float
        Parameter indicating magnitude of the expansion point calculation.
    c_param: float
        Parameter indicating magnitude of the contraction point calculation.
    s_param: float
        Parameter indicating magnitude of the shrink calculation.
    maxcores : int
        The maximum number of CPU cores that the optimization should use,
        regardless of the size of the problem.
    name : string
        A filename for (optionally) saving the progress of the Nelder-Mead search,
        and for resuming a previous search (when resume=True).  Useful for long
        searches that could potentially be interrupted by computer down time.
    resume : boolean
        An indicator for whether the search should resume from earlier progress.
        When True, the process will load a progress file named in input name.
    savefreq : int
        When not None, search progress will be saved to name.txt every savefreq
        iterations, to be loaded later with resume=True).
    verbose : int
        Indicator for the verbosity of the optimization routine.  Higher values
        generate more text output; verbose=0 produces no text output.

    Returns
    -------
    min_point : np.array
        The input that minimizes objFunc, as found by the minimization.
    fmin : float
        The minimum of objFunc; fmin = objFunc(min_point).
    '''
    # If this is a resumed search, load the data
    if resume:
        simplex, fvals, iters, evals = loadNelderMeadData(name)
        dim_count = fvals.size - 1
        N = dim_count+1 # Number of points in simplex
        K = simplex.shape[1] # Total number of parameters
    # Otherwise, construct the initial simplex and array of function values
    else:
        if perturb is None: # Default: perturb each parameter by 10%
            perturb = 0.1*guess
            guess[guess == 0] = 0.1
        params_to_opt = np.where(perturb != 0)[0] # Indices of which parameters to optimize
        dim_count = params_to_opt.size # Number of parameters to search over
        N = dim_count+1 # Number of points in simplex
        K = guess.size # Total number of parameters
        simplex = np.tile(guess,(N,1))
        for j in range(dim_count): # Perturb each parameter to optimize by the specified distance
            simplex[j+1,params_to_opt[j]] = simplex[j+1,params_to_opt[j]] + perturb[params_to_opt[j]]
        # Initialize a few
        fvals = np.zeros(dim_count+1) + np.nan
        iters = 0
        evals = 0

    # Create the pool of worker processes
    cpu_cores = multiprocessing.cpu_count() # Total number of available CPU cores
    cores_to_use = min(cpu_cores,dim_count)
    if maxcores is not None: # Cap the number of cores if desired
        cores_to_use = min(cores_to_use,maxcores)
    parallel = Parallel(n_jobs=cores_to_use)

    # Begin a new Nelder-Mead search
    if not resume:
        temp_simplex = list(simplex) # Evaluate the initial simplex
        fvals = np.array(parallel(delayed(objFunc)(params) for params in temp_simplex))
        evals += N
        # Reorder the initial simplex
        order = np.argsort(fvals)
        fvals = fvals[order]
        simplex = simplex[order,:]
        fmin = fvals[0]
        f_dist = np.abs(fmin - fvals[-1])
        x_dist = np.max(np.sqrt(np.sum(simplex**2.0 - np.tile(simplex[0,:],(N,1))**2.0,axis=1)))
        if verbose > 0:
            print('Evaluated the initial simplex: fmin=' + str(fmin) + ', f_dist=' + str(f_dist) + ', x_dist=' + str(x_dist))
    else: # Resume an existing search that was cut short
        if verbose > 0:
            print('Resuming search after ' + str(iters) + ' iterations and ' + str(evals) + ' function evaluations.')

    # Initialize some inputs for the multithreader
    j_list = list(range(N-P,N))
    opt_params= [r_param,c_param,e_param]

    # Run the Nelder-Mead algorithm until a terminal condition is met
    go = True
    while go:
        t_start = clock()
        iters += 1
        if verbose > 0:
            print('Beginning iteration #' + str(iters) + ' now.')

        # Update the P worst points of the simplex
        output = parallel(delayed(parallelNelderMeadWorker)(objFunc,simplex,fvals,j,P,opt_params) for j in j_list)
        new_subsimplex = np.zeros((P,K)) + np.nan
        new_vals = np.zeros(P) + np.nan
        new_evals = 0
        for i in range(P):
            new_subsimplex[i,:] = output[i][0]
            new_vals[i] = output[i][1]
            new_evals += output[i][2]
        evals += new_evals

        # Check whether any updates actually happened
        old_subsimplex = simplex[(N-P):N,:]
        if np.max(np.abs(new_subsimplex - old_subsimplex)) == 0:
            if verbose > 0:
                print('Updated the simplex, but must perform a shrink step.')
            # If every attempted update was unsuccessful, must shrink the simplex
            simplex = s_param*np.tile(simplex[0,:],(N,1)) + (1.0-s_param)*simplex
            temp_simplex = list(simplex[1:N,:])
            fvals = np.array([fvals[0]] + parallel(delayed(objFunc)(params) for params in temp_simplex))
            new_evals += N-1
            evals += N-1
        else:
            if verbose > 0:
                print('Updated the simplex successfully.')
            # Otherwise, update the simplex with the new results
            simplex[(N-P):N,:] = new_subsimplex
            fvals[(N-P):N] = new_vals

        # Reorder the simplex from best to worst
        order = np.argsort(fvals)
        fvals = fvals[order]
        simplex = simplex[order,:]
        fmin = fvals[0]
        f_dist = np.abs(fmin - fvals[-1])
        x_dist = np.max(np.sqrt(np.sum(simplex**2.0 - np.tile(simplex[0,:],(N,1))**2.0,axis=1)))
        t_end = clock()
        if verbose > 0:
            t_iter = t_end - t_start
            print('Finished iteration #' + str(iters) +' with ' + str(new_evals) + ' evaluations (' + str(evals) + ' cumulative) in ' + str(t_iter) + ' seconds.')
            print('Simplex status: fmin=' + str(fmin) + ', f_dist=' + str(f_dist) + ', x_dist=' + str(x_dist))

        # Check for terminal conditions
        if iters >= maxiter:
            go = False
            print('Maximum iterations reached, terminating unsuccessfully.')
        if evals >= maxeval:
            go = False
            print('Maximum evaluations reached, terminating unsuccessfully.')
        if f_dist < ftol:
            go = False
            print('Function tolerance reached, terminating successfully.')
        if x_dist < xtol:
            go = False
            print('Parameter tolerance reached, terminating successfully.')

        # Save the progress of the estimation if desired
        if savefreq is not None:
            if (iters % savefreq) == 0:
                 saveNelderMeadData(name, simplex, fvals, iters, evals)
                 if verbose > 0:
                     print('Saved search progress in ' + name + '.txt')

    # Return the results
    xopt = simplex[0,:]
    return xopt, fmin
A parallel implementation of the Nelder-Mead minimization algorithm, as
    described in Lee and Wiswall.  For long optimization procedures, it can
    save progress between iterations and resume later.

    Parameters
    ----------
    objFunc : function
        The objective function to be minimized. Takes a single 1D array as input.
    guess : np.array
        Initial starting point for the simplex, representing an input for objFunc.
    perturb : np.array
        Perturbation vector for the simplex, of the same length as an input to
        objFunc.  If perturb[j] is non-zero, a simplex point will be created
        that perturbs the j-th element of guess by perturb[j]; if it is zero,
        then the j-th parameter of objFunc will not be optimized over.  By
        default, guess=None, indicating that all parameters should be optimized,
        with an initial perturbation of 0.1*guess.
    P : int
        Degree of parallelization: the number of vertices of the simplex to try
        to update on each iteration of the process.
    ftol : float
        Absolute tolerance of the objective function for convergence.  If suc-
        cessive iterations return minimum function values that differ by less
        than ftol, the process terminates successfully.
    xtol : float
        Absolute tolerance of the input values for convergence.  If the maximum
        distance between the current minimum point and the worst point in the
        simplex is less than xtol, then the process terminates successfully.
    maxiter : int
        Maximum number of Nelder-Mead iterations; reaching iters=maxiter is
        reported as an "unsuccessful" minimization.
    maxeval : int
        Maximum number of evaluations of objFunc (across all processes); reaching
        evals=maxeval is reported as an "unsuccessful" minimization.
    r_param: float
        Parameter indicating magnitude of the reflection point calculation.
    e_param: float
        Parameter indicating magnitude of the expansion point calculation.
    c_param: float
        Parameter indicating magnitude of the contraction point calculation.
    s_param: float
        Parameter indicating magnitude of the shrink calculation.
    maxcores : int
        The maximum number of CPU cores that the optimization should use,
        regardless of the size of the problem.
    name : string
        A filename for (optionally) saving the progress of the Nelder-Mead search,
        and for resuming a previous search (when resume=True).  Useful for long
        searches that could potentially be interrupted by computer down time.
    resume : boolean
        An indicator for whether the search should resume from earlier progress.
        When True, the process will load a progress file named in input name.
    savefreq : int
        When not None, search progress will be saved to name.txt every savefreq
        iterations, to be loaded later with resume=True).
    verbose : int
        Indicator for the verbosity of the optimization routine.  Higher values
        generate more text output; verbose=0 produces no text output.

    Returns
    -------
    min_point : np.array
        The input that minimizes objFunc, as found by the minimization.
    fmin : float
        The minimum of objFunc; fmin = objFunc(min_point).