Theano -- Unresolved symbol when multiplying two matrices. All works for vectors and tensor3
Question: When I try to run the following code:

    
    
    a = T.matrix('a')
    b = T.matrix('b')
    f = theano.function([a, b], T.batched_dot(a,b))
    f([[1, 2], [5, 6]],[[3,4],[7,8]])
    

I get the following error and stack trace:

    
    
    ---------------------------------------------------------------------------
    ImportError                               Traceback (most recent call last)
    <ipython-input-13-42cb5508791b> in <module>()
          1 a = T.matrix('a')
          2 b = T.matrix('b')
    ----> 3 f = theano.function([a, b], T.batched_dot(a,b))
          4 f([[1, 2], [5, 6]],[[3,4],[7,8]])
    
    /usr/local/lib/python2.7/dist-packages/theano/compile/function.pyc in function(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)
        264                 allow_input_downcast=allow_input_downcast,
        265                 on_unused_input=on_unused_input,
    --> 266                 profile=profile)
        267     # We need to add the flag check_aliased inputs if we have any mutable or
        268     # borrowed used defined inputs
    
    /usr/local/lib/python2.7/dist-packages/theano/compile/pfunc.pyc in pfunc(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)
        509     return orig_function(inputs, cloned_outputs, mode,
        510             accept_inplace=accept_inplace, name=name, profile=profile,
    --> 511             on_unused_input=on_unused_input)
        512 
        513 
    
    /usr/local/lib/python2.7/dist-packages/theano/compile/function_module.pyc in orig_function(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input)
       1464                    profile=profile,
       1465                    on_unused_input=on_unused_input).create(
    -> 1466                        defaults)
       1467 
       1468     t2 = time.time()
    
    /usr/local/lib/python2.7/dist-packages/theano/compile/function_module.pyc in create(self, input_storage, trustme)
       1322             theano.config.traceback.limit = 0
       1323             _fn, _i, _o = self.linker.make_thunk(
    -> 1324                 input_storage=input_storage_lists)
       1325         finally:
       1326             theano.config.traceback.limit = limit_orig
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/link.pyc in make_thunk(self, input_storage, output_storage)
        517     def make_thunk(self, input_storage=None, output_storage=None):
        518         return self.make_all(input_storage=input_storage,
    --> 519                              output_storage=output_storage)[:3]
        520 
        521     def make_all(self, input_storage, output_storage):
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/vm.pyc in make_all(self, profiler, input_storage, output_storage)
        895                                                  storage_map,
        896                                                  compute_map,
    --> 897                                                  no_recycling))
        898                 if not hasattr(thunks[-1], 'lazy'):
        899                     # We don't want all ops maker to think about lazy Ops.
    
    /usr/local/lib/python2.7/dist-packages/theano/scan_module/scan_op.pyc in make_thunk(self, node, storage_map, compute_map, no_recycling)
        592                                name=self.name,
        593                                profile=profile,
    --> 594                                on_unused_input='ignore')
        595 
        596         try:
    
    /usr/local/lib/python2.7/dist-packages/theano/compile/function.pyc in function(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)
        264                 allow_input_downcast=allow_input_downcast,
        265                 on_unused_input=on_unused_input,
    --> 266                 profile=profile)
        267     # We need to add the flag check_aliased inputs if we have any mutable or
        268     # borrowed used defined inputs
    
    /usr/local/lib/python2.7/dist-packages/theano/compile/pfunc.pyc in pfunc(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)
        509     return orig_function(inputs, cloned_outputs, mode,
        510             accept_inplace=accept_inplace, name=name, profile=profile,
    --> 511             on_unused_input=on_unused_input)
        512 
        513 
    
    /usr/local/lib/python2.7/dist-packages/theano/compile/function_module.pyc in orig_function(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input)
       1464                    profile=profile,
       1465                    on_unused_input=on_unused_input).create(
    -> 1466                        defaults)
       1467 
       1468     t2 = time.time()
    
    /usr/local/lib/python2.7/dist-packages/theano/compile/function_module.pyc in create(self, input_storage, trustme)
       1322             theano.config.traceback.limit = 0
       1323             _fn, _i, _o = self.linker.make_thunk(
    -> 1324                 input_storage=input_storage_lists)
       1325         finally:
       1326             theano.config.traceback.limit = limit_orig
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/link.pyc in make_thunk(self, input_storage, output_storage)
        517     def make_thunk(self, input_storage=None, output_storage=None):
        518         return self.make_all(input_storage=input_storage,
    --> 519                              output_storage=output_storage)[:3]
        520 
        521     def make_all(self, input_storage, output_storage):
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/vm.pyc in make_all(self, profiler, input_storage, output_storage)
        895                                                  storage_map,
        896                                                  compute_map,
    --> 897                                                  no_recycling))
        898                 if not hasattr(thunks[-1], 'lazy'):
        899                     # We don't want all ops maker to think about lazy Ops.
    
    /usr/local/lib/python2.7/dist-packages/theano/sandbox/cuda/__init__.pyc in make_thunk(self, node, storage_map, compute_map, no_recycling)
        257                                     enable_cuda=False)
        258         return super(GpuOp, self).make_thunk(node, storage_map,
    --> 259                                              compute_map, no_recycling)
        260 
        261 theano.compile.debugmode.default_make_thunk.append(
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/op.pyc in make_thunk(self, node, storage_map, compute_map, no_recycling)
        737                 logger.debug('Trying CLinker.make_thunk')
        738                 outputs = cl.make_thunk(input_storage=node_input_storage,
    --> 739                                         output_storage=node_output_storage)
        740                 fill_storage, node_input_filters, node_output_filters = outputs
        741 
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/cc.pyc in make_thunk(self, input_storage, output_storage, keep_lock)
       1071         cthunk, in_storage, out_storage, error_storage = self.__compile__(
       1072             input_storage, output_storage,
    -> 1073             keep_lock=keep_lock)
       1074 
       1075         res = _CThunk(cthunk, init_tasks, tasks, error_storage)
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/cc.pyc in __compile__(self, input_storage, output_storage, keep_lock)
       1013                                     input_storage,
       1014                                     output_storage,
    -> 1015                                     keep_lock=keep_lock)
       1016         return (thunk,
       1017                 [link.Container(input, storage) for input, storage in
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/cc.pyc in cthunk_factory(self, error_storage, in_storage, out_storage, keep_lock)
       1440         else:
       1441             module = get_module_cache().module_from_key(
    -> 1442                 key=key, lnk=self, keep_lock=keep_lock)
       1443 
       1444         vars = self.inputs + self.outputs + self.orphans
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/cmodule.pyc in module_from_key(self, key, lnk, keep_lock)
       1074             try:
       1075                 location = dlimport_workdir(self.dirname)
    -> 1076                 module = lnk.compile_cmodule(location)
       1077                 name = module.__file__
       1078                 assert name.startswith(location)
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/cc.pyc in compile_cmodule(self, location)
       1352                 lib_dirs=self.lib_dirs(),
       1353                 libs=libs,
    -> 1354                 preargs=preargs)
       1355         except Exception, e:
       1356             e.args += (str(self.fgraph),)
    
    /usr/local/lib/python2.7/dist-packages/theano/sandbox/cuda/nvcc_compiler.pyc in compile_str(module_name, src_code, location, include_dirs, lib_dirs, libs, preargs, rpaths, py_module)
        432             #touch the __init__ file
        433             open(os.path.join(location, "__init__.py"), 'w').close()
    --> 434             return dlimport(lib_filename)
    
    /usr/local/lib/python2.7/dist-packages/theano/gof/cmodule.pyc in dlimport(fullpath, suffix)
        291                 importlib.invalidate_caches()
        292         t0 = time.time()
    --> 293         rval = __import__(module_name, {}, {}, [module_name])
        294         t1 = time.time()
        295         import_time += t1 - t0
    
    ImportError: ('The following error happened while compiling the node', for{gpu,scan_fn}(Elemwise{minimum,no_inplace}.0, GpuSubtensor{int64:int64:int8}.0, GpuSubtensor{int64:int64:int8}.0, Elemwise{minimum,no_inplace}.0), '\n', 'The following error happened while compiling the node', GpuAlloc{memset_0=True}(CudaNdarrayConstant{0.0}, TensorConstant{1}), '\n', '/home/alex/.theano/compiledir_Linux-3.13--generic-x86_64-with-Ubuntu-14.04-trusty-x86_64-2.7.6-64/tmppLzatX/e0c22e1de788a177e13f39a93c579f18.so: undefined symbol: _Z17CudaNdarray_SIZEtPK11CudaNdarray', '[GpuAlloc{memset_0=True}(CudaNdarrayConstant{0.0}, TensorConstant{1})]')
    

If I replace `matrix` with `vector` or `tensor3` (and supply properly shaped
tensors to the function call), it compiles and runs. For `matrix` none of
`dot`, `tensor_dot`, `batched_dot` work, but addition and subtraction work.

I ran `pip install --upgrade theano`, and the errors stayed. `pip` reports my
current version to be `0.7`. Since it only fails for matrices, could it be
because of some library `theano` depends on that I somehow happen to have
misconfigured?


Answer: What I did wrong was `pip install --upgrade theano`. Turned out I already had
theano 0.7, which is the latest, so `pip install --upgrade` did nothing (but
it was not apparent because it did upgrade the dependencies).

`pip uninstall theano && pip install theano` fixed the problem

