title: minimizing non-convex function with Nelder-Mead
body: <p>I am using <code>scipy.optimize.minimize</code>, with the default method ('Neldear-Mead').
  The function I try to minimize is not strictly convex. It stays at the same value on some significant areas.</p>
  
  <p>The issue that I have is that the steps taken by the algorithm are too small. For example my starting point has a first coordinate <code>x0 = 0.2</code> . I know that the function will result in a different value only for a significant step, for example moving by 0.05. Unfortunately, I can see that the algorithm makes very small step (moving by around 0.000001). As a result, my function returns the same value, and the algorithm does not converge. Can I change that behaviour?</p>
  
  <p>For convenience, here's the scipy code:</p>
  
  <pre><code>def _minimize_neldermead(func, x0, args=(), callback=None,
                           xtol=1e-4, ftol=1e-4, maxiter=None, maxfev=None,
                           disp=False, return_all=False,
                           **unknown_options):
      """
      Minimization of scalar function of one or more variables using the
      Nelder-Mead algorithm.
  
      Options for the Nelder-Mead algorithm are:
          disp : bool
              Set to True to print convergence messages.
          xtol : float
              Relative error in solution `xopt` acceptable for convergence.
          ftol : float
              Relative error in ``fun(xopt)`` acceptable for convergence.
          maxiter : int
              Maximum number of iterations to perform.
          maxfev : int
              Maximum number of function evaluations to make.
  
      This function is called by the `minimize` function with
      `method=Nelder-Mead`. It is not supposed to be called directly.
      """
      _check_unknown_options(unknown_options)
      maxfun = maxfev
      retall = return_all
  
      fcalls, func = wrap_function(func, args)
      x0 = asfarray(x0).flatten()
      N = len(x0)
      rank = len(x0.shape)
      if not -1 &lt; rank &lt; 2:
          raise ValueError("Initial guess must be a scalar or rank-1 sequence.")
      if maxiter is None:
          maxiter = N * 200
      if maxfun is None:
          maxfun = N * 200
  
      rho = 1
      chi = 2
      psi = 0.5
      sigma = 0.5
      one2np1 = list(range(1, N + 1))
  
      if rank == 0:
          sim = numpy.zeros((N + 1,), dtype=x0.dtype)
      else:
          sim = numpy.zeros((N + 1, N), dtype=x0.dtype)
      fsim = numpy.zeros((N + 1,), float)
      sim[0] = x0
      if retall:
          allvecs = [sim[0]]
      fsim[0] = func(x0)
      nonzdelt = 0.05
      zdelt = 0.00025
      for k in range(0, N):
          y = numpy.array(x0, copy=True)
          if y[k] != 0:
              y[k] = (1 + nonzdelt)*y[k]
          else:
              y[k] = zdelt
  
          sim[k + 1] = y
          f = func(y)
          fsim[k + 1] = f
  
      ind = numpy.argsort(fsim)
      fsim = numpy.take(fsim, ind, 0)
      # sort so sim[0,:] has the lowest function value
      sim = numpy.take(sim, ind, 0)
  
      iterations = 1
  
      while (fcalls[0] &lt; maxfun and iterations &lt; maxiter):
          if (numpy.max(numpy.ravel(numpy.abs(sim[1:] - sim[0]))) &lt;= xtol and
                  numpy.max(numpy.abs(fsim[0] - fsim[1:])) &lt;= ftol):
              break
  
          xbar = numpy.add.reduce(sim[:-1], 0) / N
          xr = (1 + rho) * xbar - rho * sim[-1]
          fxr = func(xr)
          doshrink = 0
  
          if fxr &lt; fsim[0]:
              xe = (1 + rho * chi) * xbar - rho * chi * sim[-1]
              fxe = func(xe)
  
              if fxe &lt; fxr:
                  sim[-1] = xe
                  fsim[-1] = fxe
              else:
                  sim[-1] = xr
                  fsim[-1] = fxr
          else:  # fsim[0] &lt;= fxr
              if fxr &lt; fsim[-2]:
                  sim[-1] = xr
                  fsim[-1] = fxr
              else:  # fxr &gt;= fsim[-2]
                  # Perform contraction
                  if fxr &lt; fsim[-1]:
                      xc = (1 + psi * rho) * xbar - psi * rho * sim[-1]
                      fxc = func(xc)
  
                      if fxc &lt;= fxr:
                          sim[-1] = xc
                          fsim[-1] = fxc
                      else:
                          doshrink = 1
                  else:
                      # Perform an inside contraction
                      xcc = (1 - psi) * xbar + psi * sim[-1]
                      fxcc = func(xcc)
  
                      if fxcc &lt; fsim[-1]:
                          sim[-1] = xcc
                          fsim[-1] = fxcc
                      else:
                          doshrink = 1
  
                  if doshrink:
                      for j in one2np1:
                          sim[j] = sim[0] + sigma * (sim[j] - sim[0])
                          fsim[j] = func(sim[j])
  
          ind = numpy.argsort(fsim)
          sim = numpy.take(sim, ind, 0)
          fsim = numpy.take(fsim, ind, 0)
          if callback is not None:
              callback(sim[0])
          iterations += 1
          if retall:
              allvecs.append(sim[0])
  
      x = sim[0]
      fval = numpy.min(fsim)
      warnflag = 0
  
      if fcalls[0] &gt;= maxfun:
          warnflag = 1
          msg = _status_message['maxfev']
          if disp:
              print('Warning: ' + msg)
      elif iterations &gt;= maxiter:
          warnflag = 2
          msg = _status_message['maxiter']
          if disp:
              print('Warning: ' + msg)
      else:
          msg = _status_message['success']
          if disp:
              print(msg)
              print("         Current function value: %f" % fval)
              print("         Iterations: %d" % iterations)
              print("         Function evaluations: %d" % fcalls[0])
  
      result = OptimizeResult(fun=fval, nit=iterations, nfev=fcalls[0],
                              status=warnflag, success=(warnflag == 0),
                              message=msg, x=x)
      if retall:
          result['allvecs'] = allvecs
      return result
  </code></pre>