def _populate_grad_dict(var_to_app_to_idx,
                        grad_dict, wrt, cost_name=None):
    """
        Helper function for grad function.

        var_to_app_to_idx: a dictionary mapping a variable to
                a second dictionary.
                the second dictionary maps apply nodes acting on
                this variable to the variable's index in the apply
                node's input list

        grad_dict: A dictionary mapping variables to their gradients.
                   Should be populated by grad function, which should:
                       -Set the gradient with respect to the cost to 1
                       -Load all gradients from known_grads, possibly
                        overriding the cost
                       -Set the gradient for disconnected
                        inputs to a variable with type DisconnectedType()

        wrt: the minimal set of variables that must be included in grad_dict

        cost_name: The name of the cost being differentiated, optional.
                    used to name the grad with respect to x as
                    (d<cost_name>/dx)

        returns: a list of gradients corresponding to wrt

    """
    # build a dict mapping node to the terms node contributes to each of
    # its inputs' gradients
    term_dict = OrderedDict()

    def access_term_cache(node):
        """ Populates term_dict[node] and returns it """

        if node not in term_dict:

            inputs = node.inputs

            output_grads = [access_grad_cache(var) for var in node.outputs]

            # list of bools indicating if each output is connected to the cost
            outputs_connected = [not isinstance(g.type, DisconnectedType)
                                 for g in output_grads]

            connection_pattern = _node_to_pattern(node)

            # list of bools indicating if each input is connected to the cost
            inputs_connected = [
                (True in [input_to_output and output_to_cost for
                          input_to_output, output_to_cost in
                          zip(input_to_outputs, outputs_connected)]) for
                input_to_outputs in connection_pattern
            ]

            # List of bools indicating if each output is an integer dtype
            output_is_int = [hasattr(output.type, 'dtype') and
                             output.type.dtype in theano.tensor.discrete_dtypes
                             for output in node.outputs]

            # List of bools indicating if each output is NullType
            ograd_is_nan = [isinstance(output.type, NullType)
                            for output in output_grads]

            # List of bools indicating if each input only has NullType outputs
            only_connected_to_nan = [
                (True not in
                 [in_to_out and out_to_cost and not out_nan
                  for in_to_out, out_to_cost, out_nan in
                  zip(in_to_outs, outputs_connected, ograd_is_nan)])
                for in_to_outs in connection_pattern]

            if True not in inputs_connected:
                # All outputs of this op are disconnected so we can skip
                # Calling the op's grad method and report that the inputs
                # are disconnected
                # (The op's grad method could do this too, but this saves the
                # implementer the trouble of worrying about this case)
                input_grads = [disconnected_type() for ipt in inputs]
            elif False not in only_connected_to_nan:
                # All inputs are only connected to nan gradients, so we don't
                # need to bother calling the grad method. We know the gradient
                # with respect to all connected inputs is nan.
                input_grads = []
                for connected in inputs_connected:
                    if connected:
                        input_grads.append(null_type())
                    else:
                        input_grads.append(disconnected_type())
            else:
                # At least one input of this op is connected to the cost so and
                # not all output gradients are undefined so we must
                # call the op's grad method

                # Each Op's grad function requires inputs and output_grads
                # If the Op destroys any input, but the grad expression uses
                # it, then chances are the resulting graph will have a
                # dependency cycle. We avoid this cycle by passing (symbolic)
                # copies of each destroyed input.
                try:
                    dinputs = [node.inputs[x[0]] for x in
                               itervalues(node.op.destroy_map)]
                except __HOLE__:
                    dinputs = []

                def try_to_copy_if_needed(var):
                    if var in dinputs and hasattr(var, 'copy'):
                        return var.copy()
                    return var

                inputs = [try_to_copy_if_needed(ipt) for ipt in inputs]

                # Build a list of output gradients with the same dtype as
                # the corresponding output variable.
                # If an output is of a float dtype, we want to cast the
                # output gradient into the same dtype, to avoid having a
                # gradient graph with double precision (taking more memory,
                # and more computation).
                # If an output is of an integer dtype, then we just leave it
                # alone.
                # DO NOT force integer variables to have zero grad. This causes
                # bugs where we fail to detect disconnected or undefined
                # gradients.
                # DO NOT force integer variables to have integer dtype.
                # This is a violation of the op contract.
                new_output_grads = []
                for o, og in zip(node.outputs, output_grads):
                    o_dt = getattr(o.type, 'dtype', None)
                    og_dt = getattr(og.type, 'dtype', None)
                    if (o_dt not in theano.tensor.discrete_dtypes and
                            og_dt and o_dt != og_dt):
                        new_output_grads.append(og.astype(o_dt))
                    else:
                        new_output_grads.append(og)

                # Make sure that, if new_output_grads[i] has a floating point
                # dtype, it is the same dtype as outputs[i]
                for o, ng in zip(node.outputs, new_output_grads):
                    o_dt = getattr(o.type, 'dtype', None)
                    ng_dt = getattr(ng.type, 'dtype', None)
                    if (ng_dt is not None and
                            o_dt not in theano.tensor.discrete_dtypes):
                        assert ng_dt == o_dt

                # Someone who had obviously not read the Op contract tried
                # to modify this part of the function.
                # If you ever think it is a good idea to make an integer
                # valued gradient, please
                # 1) Read the Op contract again
                # 2) Talk to Ian Goodfellow
                # (Both of these sources will tell you not to do it)
                for ng in new_output_grads:
                    assert (getattr(ng.type, 'dtype', None)
                            not in theano.tensor.discrete_dtypes)

                # If config.compute_test_value is turned on, check that the
                # gradients on the outputs of this node have the right shape.
                # We also check the gradient on the inputs later--both checks
                # are needed, because some gradients are only ever specified
                # by the user, not computed by Op.grad, and some gradients are
                # only computed and returned, but never passed as another
                # node's output grads.
                for idx, packed in enumerate(izip(node.outputs,
                                             new_output_grads)):
                    orig_output, new_output_grad = packed
                    if not hasattr(orig_output, 'shape'):
                        continue
                    if isinstance(new_output_grad.type, DisconnectedType):
                        continue
                    for orig_output_v, new_output_grad_v in get_debug_values(
                            *packed):
                        o_shape = orig_output_v.shape
                        g_shape = new_output_grad_v.shape
                        if o_shape != g_shape:
                            raise ValueError(
                                "Got a gradient of shape " +
                                str(o_shape) + " on an output of shape " +
                                str(g_shape))

                input_grads = node.op.grad(inputs, new_output_grads)

                if input_grads is None:
                    raise TypeError("%s.grad returned NoneType, "
                                    "expected iterable." % str(node.op))

                if len(input_grads) != len(inputs):
                    raise ValueError(("%s returned the wrong number of" +
                                      " gradient terms.") % str(node.op))
# We can not enforce this, as AdvancedSubtensor1 has an option to
# return the sparse grad for optimization reason.

                    #            for ig, i in zip(input_grads, inputs):
#                if (not isinstance(ig.type, (DisconnectedType, NullType)) and
#                    type(ig.type) != type(i.type)):
#                    raise ValueError(
#                        "%s returned the wrong type for gradient terms."
#                        " Sparse inputs must have sparse grads and dense"
#                        " inputs must have dense grad. Got %s, expected %s" %(
#                            str(node.op), ig.type, i.type))

            # must convert to list in case the op returns a tuple
            # we won't be able to post-process out the Nones if it does that
            input_grads = list(input_grads)

            # Need to propagate the NullType gradients; if an input grad is
            # not disconnected and the corresponding input is connected
            # to at least one output whose gradient is NullType then the input
            # grad should be NullType.
            for inp_idx in range(len(input_grads)):
                for out_idx in range(len(ograd_is_nan)):
                    if (ograd_is_nan[out_idx] and
                            connection_pattern[inp_idx][out_idx] and
                            not isinstance(input_grads[inp_idx].type,
                                           DisconnectedType)):
                        input_grads[inp_idx] = output_grads[out_idx]

            # Do type checking on the result

            # List of bools indicating if each input only has integer outputs
            only_connected_to_int = [
                (True not in
                 [in_to_out and out_to_cost and not out_int
                  for in_to_out, out_to_cost, out_int in
                  zip(in_to_outs, outputs_connected, output_is_int)])
                for in_to_outs in connection_pattern]

            for i, term in enumerate(input_grads):

                # Disallow Nones
                if term is None:
                    # We don't know what None means. in the past it has been
                    # used to mean undefined, zero, or disconnected.
                    # We therefore don't allow it because its usage has become
                    # so muddied.
                    raise TypeError(
                        ('%s.grad returned None for' +
                         ' a gradient term, '
                         'this is prohibited. Instead of None,'
                         'return zeros_like(input), disconnected_type(),'
                         ' or a NullType variable such as those made with '
                         'the grad_undefined or grad_unimplemented helper '
                         'functions.') % node.op)

                # Check that the gradient term for this input
                # has the right shape
                if hasattr(term, 'shape'):
                    orig_ipt = inputs[i]
                    for orig_ipt_v, term_v in get_debug_values(orig_ipt, term):
                        i_shape = orig_ipt_v.shape
                        t_shape = term_v.shape
                        if i_shape != t_shape:
                            raise ValueError(
                                "%s.grad returned object of "
                                "shape %s as gradient term on input %d "
                                "of shape %s" % (node.op, t_shape, i, i_shape))

                if not isinstance(term.type,
                                  (NullType, DisconnectedType)):
                    if term.type.dtype not in theano.tensor.float_dtypes:
                        raise TypeError(str(node.op) + '.grad illegally '
                                        ' returned an integer-valued variable.'
                                        ' (Input index %d, dtype %s)' % (
                                            i, term.type.dtype))

                    if only_connected_to_nan[i]:
                        assert isinstance(term.type, NullType)

                    if only_connected_to_int[i]:
                        # This term has only integer outputs and we know
                        # it's not undefined or disconnected
                        # The only other valid thing it can be is 0

                        is_zero = _is_zero(term)
                        assert is_zero in ['yes', 'no', 'maybe']
                        if is_zero == 'maybe':
                            msg = "%s.grad returned %s of type %s for input"
                            msg += " %d. This input's only connections to "
                            msg += "the cost through this op are via "
                            msg += "integer-valued outputs so it should be "
                            msg += "NullType, DisconnectedType, or some form "
                            msg += "of zeros. It is not NullType or "
                            msg += "DisconnectedType and theano can't "
                            msg += "simplify it to a constant, so it's not "
                            msg += "verifiably zeros."

                            msg = msg % (str(node.op), str(term),
                                         str(type(term)), i)

                        if is_zero == 'no':
                            msg = "%s.grad returned %s of type %s for input"
                            msg += " %d. Since this input is only connected "
                            msg += "to integer-valued outputs, it should "
                            msg += "evaluate to zeros, but it evaluates to"
                            msg += "%s."

                            msg % (node.op, term, type(term), i,
                                   theano.get_scalar_constant_value(term))

                            raise ValueError(msg)

            # Check that op.connection_pattern matches the connectivity
            # logic driving the op.grad method
            for i, packed in enumerate(zip(inputs, input_grads,
                                           inputs_connected)):
                ipt, ig, connected = packed
                actually_connected = \
                    not isinstance(ig.type, DisconnectedType)

                if actually_connected and not connected:
                    msg = "%s.grad returned %s of type %s for input %d."
                    msg += " Expected DisconnectedType instance based on "
                    msg += " the output of the op's connection_pattern "
                    msg += "method."
                    msg = msg % (str(node.op), str(ig), str(ig.type), i)
                    raise TypeError(msg)

                if connected and not actually_connected:
                    msg = "%s.grad returned DisconnectedType for input"
                    msg += " %d."
                    msg = msg % (str(node.op), i)
                    if hasattr(node.op, 'connection_pattern'):
                        msg += ' Its connection_pattern method does not'
                        msg += ' allow this.'
                        raise TypeError(msg)
                    else:
                        msg += ' You may want to implement a '
                        msg += 'connection_pattern method for it.'
                        warnings.warn(msg)

            # cache the result
            term_dict[node] = input_grads

        return term_dict[node]

    # populate grad_dict[var] and return it
    def access_grad_cache(var):
        if var not in grad_dict:
            # If var is not in grad_dict already, we must compute it
            if var in var_to_app_to_idx:
                null_terms = []
                terms = []
                node_to_idx = var_to_app_to_idx[var]
                for node in node_to_idx:
                    for idx in node_to_idx[node]:

                        term = access_term_cache(node)[idx]

                        if not isinstance(term, gof.Variable):
                            raise TypeError(
                                "%s.grad returned %s, expected"
                                " Variable instance." % (str(node.op),
                                                         type(term)))

                        if isinstance(term.type, NullType):
                            null_terms.append(term)
                            continue

                        # Don't try to sum up DisconnectedType placeholders
                        if isinstance(term.type, DisconnectedType):
                            continue

                        if hasattr(var, 'ndim') and term.ndim != var.ndim:
                            raise ValueError(
                                ("%s.grad returned a term with"
                                 " %d dimensions, but %d are required.") % (
                                     str(node.op), term.ndim, var.ndim))

                        terms.append(term)

                # Add up the terms to get the total gradient on this variable
                if len(null_terms) > 0:
                    # At least one term is a NullType : the total gradient
                    # will also be a NullType
                    grad_dict[var] = null_terms[0]
                elif len(terms) > 0:
                    # the next line is like sum(terms) but doesn't add an
                    # extraneous TensorConstant(0)
                    grad_dict[var] = reduce(lambda x, y: x + y, terms)
                else:
                    grad_dict[var] = disconnected_type()

                if cost_name is not None and var.name is not None:
                    grad_dict[var].name = '(d%s/d%s)' % (cost_name, var.name)
            else:
                # this variable isn't connected to the cost in the
                # computational graph
                grad_dict[var] = disconnected_type()
        # end if cache miss
        return grad_dict[var]

    rval = [access_grad_cache(elem) for elem in wrt]

    return rval
dataset/ETHPy150Open rizar/attention-lvcsr/libs/Theano/theano/gradient.py/_populate_grad_dict