'Measure runtime for computing fprop and bprop separately, as well as
full minibatch run times. For inference case, only the fprop is measured.
Arguments:
dataset (NervanaDataIterator) Dataset iterator to perform fit on
cost (Cost): Defines the function which the model is minimizing based
on the output of the last layer and the input labels
niterations (optional, int): Number of minibatches to average over
nskip (optional, int): Number of iterations at the beginning to skip
when calculating the runtime statistics
inference (bool, optional): Is inference use case
optimizer (Optimizer): Defines the learning rule for updating the model parameters.
Returns:
dictionary with fprop, bprop run times'

def benchmark(self, dataset, inference=False, cost=None, optimizer=None, niterations=20, nskip=2):

    if ((inference is False) and ((cost is None) or (optimizer is None))):
        raise RuntimeError('Need    cost    and    optimizer    to    benchmark    bprop')
    self.cost = cost
    self.initialize(dataset, cost)
    self.optimizer = optimizer
    self.total_cost = np.empty((1, 1))
    self.total_cost[:] = 0
    times = OrderedDict()
    time_keys = (['fprop'] if inference else ['fprop', 'bprop', 'iteration'])
    for ky in time_keys:
        times[ky] = np.full((niterations + nskip), (-1.0))
    count = 0
    fprop_start = self.be.init_mark()
    fprop_end = self.be.init_mark()
    bprop_end = self.be.init_mark()
    while (count < (niterations + nskip)):
        dataset.reset()
        for (mb_idx, (x, t)) in enumerate(dataset):
            self.be.record_mark(fprop_start)
            x = self.fprop(x, inference)
            if (inference is False):
                self.total_cost[:] = (self.total_cost + self.cost.get_cost(x, t))
            self.be.record_mark(fprop_end)
            if (inference is False):
                delta = self.cost.get_errors(x, t)
                self.bprop(delta)
                self.optimizer.optimize(self.layers_to_optimize, epoch=0)
                self.be.record_mark(bprop_end)
                self.be.synchronize_mark(bprop_end)
            else:
                self.be.synchronize_mark(fprop_end)
            times['fprop'][count] = self.be.get_time(fprop_start, fprop_end)
            if (inference is False):
                times['bprop'][count] = self.be.get_time(fprop_end, bprop_end)
                times['iteration'][count] = (times['fprop'][count] + times['bprop'][count])
            count += 1
            if (count >= (niterations + nskip)):
                break
    header = ('Func', 'Mean', 'Median', 'Min', 'Max', 'Units')
    stats = tuple((stat.lower() for stat in header[1:(-1)]))
    fmt_titles = (('|    {:^11}    ' * len(header)) + '|')
    fmt_nums = (('|    {func:<11}    ' + (('|        {%s:<10.5g}    ' * len(stats)) % stats)) + '|    {units:^11}    |')
    head_str = fmt_titles.format(*header)
    sep = ('-' * len(head_str))
    neon_logger.display(sep)
    neon_logger.display(head_str)
    neon_logger.display(sep)
    out_stats = {}
    for step in times:
        timesu = np.array(times[step][nskip:])
        out_stats[step] = {}
        for stat in stats:
            out_stats[step][stat] = getattr(np, stat)(timesu)
        neon_logger.display(fmt_nums.format(units='msec', func=step, **out_stats[step]))
    neon_logger.display(sep)
    return out_stats
