def test_careduce():
    """
    test sum pattern 1, 11, 10, 01, 001, 010, 100, 110, 011, 111,
    0011, 0101, 0111, 1011, 1111

    test sum pattern implemented with reshape:
    1000, 0100, 0010, 0001, 11111

    others implemented by reshape that are not tested
    0011,0101,0110,1001,1010,1100
    1110,1101,1011

    TODO: test with broadcast

    We test with the pre_scalar_op sqr in all cases. This cover all
    code, with and without it the pre_scalar_op.

    """
    for scalar_op, careduce_op in [
            (theano.scalar.mul, tensor.elemwise.CAReduceDtype),
            (theano.scalar.add, tensor.elemwise.CAReduceDtype),
            (theano.scalar.maximum, tensor.CAReduce),
            (theano.scalar.minimum, tensor.CAReduce)
            # The following 2 cases could work if the scalar_op.c_code work with float* dtype.
            # Currently we have this error:
            # error: invalid operands of types 'npy_float32' and 'npy_float32' to binary 'operator&'
            #(theano.scalar.and_, tensor.elemwise.CAReduce),
            #(theano.scalar.or_, tensor.elemwise.CAReduce),
    ]:
        for shape, pattern in [((1, 1), (1,)),
                               ((1, 0), (1,)),
                               ((0, 1), (1,)),
                               ((0, 0), (1,)),
                               ((0, 0, 0), (1, 2)),
                               ((0, 0, 0, 0), (1, 2, 3)),
                               ((2, 1), (1,)),
                               ((1, 2), (1,)),
                               ((100, 3, 1300), [1]),
                               ((0,), [0]), ((5,), [0]),
                               ((0, 0), [0, 1]), ((1, 0), [0, 1]), ((5, 4), [0, 1]), ((33, 31), [0, 1]), ((5, 4), [1]), ((5, 4), [0]),  # need something bigger then 32 for some opt test.
                               ((5, 4, 3), [0]), ((5, 4, 3), [1]), ((5, 4, 3), [0, 1]), ((5, 4, 3), [2]), ((5, 4, 3), [1, 2]), ((5, 4, 3), [0, 1, 2]),
                               ((0, 0, 0, 0), [0, 1, 2, 3]),
                               ((5, 4, 3, 20), [2, 3]), ((5, 4, 3, 2), [0, 1, 2, 3]), ((5, 4, 3, 2), [0, 2, 3]), ((5, 4, 3, 2), [1, 2, 3]),
                               ((5, 4, 3, 10, 11), [1, 2]),
                               ((5, 4, 3, 20), [2, 3]), ((5, 4, 3, 2), [0, 1, 2, 3]), ((5, 4, 3, 2), [0, 2, 3]), ((5, 4, 3, 2), [1, 2, 3]),

                               # test shape bigger then 4096 on each dimension to make sure that we work correctly when we don't have enough thread/block in each dimensions
                               ((4100, 3), [0]), ((3, 4101), [0]),  # 10
                               ((1024, 33), [0]), ((33, 1024), [0]),  # 10
                               ((1025, 33), [0]), ((33, 1025), [0]),  # 10

                               ((4100, 3), [1]), ((3, 4101), [1]),  # 01
                               ((1024, 33), [1]), ((33, 1024), [1]),  # 01
                               ((1025, 33), [1]), ((33, 1025), [1]),  # 01

                               ((4100, 3), [0, 1]), ((3, 4101), [0, 1]),  # 11
                               ((1024, 33), [0, 1]), ((33, 1024), [0, 1]),  # 01
                               ((1025, 33), [0, 1]), ((33, 1025), [0, 1]),  # 01

                               ((4100, 4, 3), [0]), ((5, 4100, 3), [0]), ((5, 4, 4100), [0]), ((3, 65536, 1), [0]),  # 100
                               ((4100, 4, 3), [1]), ((5, 4100, 3), [1]), ((5, 4, 4100), [1]),  # 010
                               ((4100, 4, 3), [2]), ((5, 4100, 3), [2]), ((5, 4, 4100), [2]),  # 001
                               ((4100, 4, 3), [0, 1]), ((5, 4100, 3), [0, 1]), ((5, 4, 4100), [0, 1]),  # 110
                               ((4100, 4, 3), [1, 2]), ((5, 4100, 3), [1, 2]), ((5, 4, 4100), [1, 2]),  # 011
                               ((4100,4,3),[0,2]),((5,4100,3),[0,2]),((5,4,4100),[0,2]),
                               ((4100, 4, 3), [0, 1, 2]), ((5, 4100, 3), [0, 1, 2]), ((5, 4, 4100), [0, 1, 2]),  # 111
                               ((65, 4, 3), [0, 1, 2]), ((5, 65, 3), [0, 1, 2]), ((5, 4, 65), [0, 1, 2]),  # 111

                               # test pattern implemented by reshape
                               ((4100, 4, 3, 2), [0]), ((4, 4100, 3, 2), [0]), ((4, 3, 4100, 2), [0]), ((4, 3, 2, 4100), [0]),  # 1000
                               ((4100, 4, 3, 2), [1]), ((4, 4100, 3, 2), [1]), ((4, 3, 4100, 2), [1]), ((4, 3, 2, 4100), [1]),  # 0100
                               ((4100, 4, 3, 2), [2]), ((4, 4100, 3, 2), [2]), ((4, 3, 4100, 2), [2]), ((4, 3, 2, 4100), [2]),  # 0010
                               ((4100, 4, 3, 2), [3]), ((4, 4100, 3, 2), [3]), ((4, 3, 4100, 2), [3]), ((4, 3, 2, 4100), [3]),  # 0001

                               # reduce over 2d
                               ((4100, 4, 3, 2), [2, 3]), ((4, 4100, 3, 2), [2, 3]), ((4, 3, 4100, 2), [2, 3]), ((4, 3, 2, 4100), [2, 3]),  # 0011
                               ((4100, 4, 3, 2), [1, 3]), ((4, 4100, 3, 2), [1, 3]), ((4, 3, 4100, 2), [1, 3]), ((4, 3, 2, 4100), [1, 3]),  # 0101
                               ((4100, 4, 3, 2), [1, 2]), ((4, 4100, 3, 2), [1, 2]), ((4, 3, 4100, 2), [1, 2]), ((4, 3, 2, 4100), [1, 2]),  # 0110
                               ((4100,4,3,2),[0,3]),((4,4100,3,2),[0,3]),((4,3,4100,2),[0,3]),((4,3,2,4100),[0,3]),#1001
#                               ((4100,4,3,2),[0,2]),((4,4100,3,2),[0,2]),((4,3,4100,2),[0,2]),((4,3,2,4100),[0,2]),#1010 not implemented
                               ((4100, 4, 3, 2), [0, 1]), ((4, 4100, 3, 2), [0, 1]), ((4, 3, 4100, 2), [0, 1]), ((4, 3, 2, 4100), [0, 1]),  # 1100

                               # reduce over 3d
                               # 3d not tested: 1101, 1110, 1111
                               ((4100,4,3,2),[0,1,3]),((4,4100,3,2),[0,1,3]),((4,3,4100,2),[0,1,3]),((4,3,2,4100),[0,1,3]),#1101
                               ((4100, 4, 3, 2), [0, 1, 2]), ((4, 4100, 3, 2), [0, 1, 2]), ((4, 3, 4100, 2), [0, 1, 2]), ((4, 3, 2, 4100), [0, 1, 2]),  # 1110
                               ((4100, 4, 3, 2), [0, 2, 3]), ((4, 4100, 3, 2), [0, 2, 3]), ((4, 3, 4100, 2), [0, 2, 3]),  # ((4,3,2,4100),[0,2,3]),#1011
                               ((4100, 4, 3, 2), [1, 2, 3]), ((4, 4100, 3, 2), [1, 2, 3]), ((4, 3, 4100, 2), [1, 2, 3]), ((4, 3, 2, 4100), [1, 2, 3]),  # 0111
                               ((65, 4, 3, 2), [1, 2, 3]), ((4, 65, 3, 2), [1, 2, 3]), ((4, 3, 65, 2), [1, 2, 3]), ((4, 3, 2, 65), [1, 2, 3]),  # 0111

                               # reduce over 4d
                               ((4100, 2, 3, 4), [0, 1, 2, 3]), ((2, 4100, 3, 4), [0, 1, 2, 3]), ((2, 3, 4100, 4), [0, 1, 2, 3]), ((2, 3, 4, 4100), [0, 1, 2, 3]), ((128, 1, 3, 3), [0, 1, 2, 3]),  # 1111

                               # reduce over 5d
                               ((1100, 2, 3, 4, 5), [0, 1, 2, 3, 4]), ((2, 1100, 3, 4, 5), [0, 1, 2, 3, 4]), ((2, 3, 1100, 4, 5), [0, 1, 2, 3, 4]), ((2, 3, 4, 1100, 5), [0, 1, 2, 3, 4]), ((2, 3, 4, 5, 1100), [0, 1, 2, 3, 4]),  # 11111

                               ]:

            op = careduce_op(scalar_op, axis=pattern)
            pat = tensor_pattern_to_gpu_pattern(shape, pattern)

            a = tensor.TensorType('float32', (False,) * len(shape))()
            b = op(a*a)
            val = numpy.random.rand(numpy.prod(shape)).reshape(shape)
    #        val = numpy.ones(shape)
    #        val = numpy.arange(numpy.prod(shape)).reshape(shape)
            val = theano._asarray(val, dtype='float32')
            f = theano.function([a], b, mode=mode_with_gpu)
            f2 = theano.function([a], b, mode=mode_without_gpu)
            assert tcn.GpuCAReduce in [x.op.__class__
                                       for x in f.maker.fgraph.toposort()], (
                                           scalar_op, shape, pattern)
            if tcn.GpuElemwise in [x.op.__class__
                                   for x in f.maker.fgraph.toposort()]:
                assert tcn.GpuReshape in [x.op.__class__
                                          for x in f.maker.fgraph.toposort()]
            assert op.__class__ in [x.op.__class__
                                    for x in f2.maker.fgraph.toposort()], (
                                           scalar_op, shape, pattern)
            f_caused_value_error = False
            try:
                f_out = f(val)
            except ValueError as e:
                exc = e
                f_caused_value_error = True
            except __HOLE__:
                if (numpy.prod(shape) == 0 and
                    getattr(scalar_op, 'identity', None) != 0):
                    continue
                raise

            f2_caused_value_error = False
            try:
                f2_out = f2(val)
            except ValueError as e:
                exc2 = e
                f2_caused_value_error = True

            if f_caused_value_error != f2_caused_value_error:
                if f_caused_value_error:
                    print('f caused this value error:')
                    print(exc)
                else:
                    print('f did not raise a value error, but should have')
                if f2_caused_value_error:
                    print('f2 caused this value error:')
                    print(exc2)
                else:
                    print('f should not have raised a value error')
                print('shape was: ', shape)
                print('pattern was: ', pattern)
                assert False

            try:
                # We raise the error threashold as we sum big matrix
                # and this cause small rounding difference with some seed
                # example in debug mode with unittests.rseed=9275
                orig_rtol = theano.tensor.basic.float32_rtol
                theano.tensor.basic.float32_rtol = 2e-5
                assert _allclose(f_out, f2_out), ('shape', shape,
                                                    'pattern', pattern,
                                                    scalar_op,
                                                    sum([shape[i] for i in pattern]),
                                                    f2(val), f(val), val)
            finally:
                theano.tensor.basic.float32_rtol = orig_rtol

            # test with dimshuffle
            # we shuffle the 2 outer dims.
        for shape, pattern in [  # ((5,),[0]),
                               ((5, 4), [0, 1]), ((5, 4), [0]),
                               ((5, 4, 3), [0]), ((5, 4, 3), [0, 1]), ((5, 4, 3), [2]), ((5, 4, 3), [0, 1, 2]),
                               ((5, 4, 3, 2), [0, 1, 2, 3]), ((5, 4, 3, 2), [0, 2, 3]),
                               ((128, 1, 3, 3), [0, 1, 2, 3]),
        ]:
            op = careduce_op(scalar_op, axis=pattern)
            pat = tensor_pattern_to_gpu_pattern(shape, pattern)

            a = tensor.TensorType('float32', (False,) * len(shape))()
            dim_pattern = list(range(len(shape)))
            dim_pattern[0] = 1
            dim_pattern[1] = 0
            a = a.dimshuffle(dim_pattern)
            b = op(a*a)
            val = numpy.random.rand(numpy.prod(shape)).reshape(shape)
    #        val = numpy.ones(shape)
    #        val = numpy.arange(numpy.prod(shape)).reshape(shape)
            val = theano._asarray(val, dtype='float32')
            f = theano.function([a], b, mode=mode_with_gpu)
            f2 = theano.function([a], b, mode=mode_without_gpu)
            assert tcn.GpuCAReduce in [x.op.__class__
                                       for x in f.maker.fgraph.toposort()], (
                                           scalar_op, shape, pattern)
            assert tcn.GpuElemwise not in [x.op.__class__
                                           for x in f.maker.fgraph.toposort()]
            assert op.__class__ in [x.op.__class__
                                    for x in f2.maker.fgraph.toposort()], (
                                           scalar_op, shape, pattern)
            assert _allclose(f2(val), f(val)), ('shape', shape,
                                                'pattern', pattern,
                                                scalar_op,
                                                sum([shape[i] for i in pattern]))

            # test with broadcast
        for shape, pattern in [((5,), [0]),
                               ((5, 4), [0, 1]), ((5, 4), [0]),
                               ((5, 4, 3), [0]), ((5, 4, 3), [0, 1]),
                               ((5, 4, 3), [2]), ((5, 4, 3), [0, 1, 2]),
                               ((5, 4, 3, 2), [0, 1, 2, 3]), ((5, 4, 3, 2), [0, 2, 3]),
                               ((128, 1, 3, 3), [0, 1, 2, 3]),
        ]:
            op = careduce_op(scalar_op, axis=pattern)
            pat = tensor_pattern_to_gpu_pattern(shape, pattern)

            shape = numpy.asarray(shape) * 2
            a = tensor.TensorType('float32', (False,) * len(shape))()
            a2 = tcn.CudaNdarrayType((False,) * len(shape))()
            b = op(a*a)
            b2 = op(a2*a2)
            val = numpy.random.rand(numpy.prod(shape)).reshape(shape)
    #        val = numpy.ones(shape)
    #        val = numpy.arange(numpy.prod(shape)).reshape(shape)
            val = theano._asarray(val, dtype='float32')
            val2 = cuda.CudaNdarray(val)
            if len(shape) == 1:
                val = val[::2]
                val2 = val2[::2]
            elif len(shape) == 2:
                val = val[::2, ::2]
                val2 = val2[::2, ::2]
            elif len(shape) == 3:
                val = val[::2, ::2, ::2]
                val2 = val2[::2, ::2, ::2]
            elif len(shape) == 4:
                val = val[::2, ::2, ::2, ::2]
                val2 = val2[::2, ::2, ::2, ::2]
            f = theano.function([a], b, mode=mode_without_gpu)
            f2 = theano.function([a2], b2, mode=mode_with_gpu)
            assert tcn.GpuCAReduce in [x.op.__class__
                                       for x in f2.maker.fgraph.toposort()], (
                                           scalar_op, shape, pattern)
            assert tcn.GpuElemwise not in [x.op.__class__
                                           for x in f.maker.fgraph.toposort()]
            assert op.__class__ in [x.op.__class__
                                    for x in f.maker.fgraph.toposort()], (
                                           scalar_op, shape, pattern)
            assert _allclose(f2(val2), f(val)), ('shape', shape,
                                                 'pattern', pattern,
                                                 sum([shape[i] for i in pattern]))
dataset/ETHPy150Open rizar/attention-lvcsr/libs/Theano/theano/sandbox/cuda/tests/test_basic_ops.py/test_careduce