text: # -*- coding: utf-8 -*-
  ''' Nose test generators
  
  Need function load / save / roundtrip tests
  
  '''
  import os
  from collections import OrderedDict
  from os.path import join as pjoin, dirname
  from glob import glob
  from io import BytesIO
  import re
  from tempfile import mkdtemp
  
  import warnings
  import shutil
  import gzip
  
  from numpy.testing import (assert_array_equal, assert_array_almost_equal,
                             assert_equal, assert_, assert_warns)
  import pytest
  from pytest import raises as assert_raises
  
  import numpy as np
  from numpy import array
  import scipy.sparse as SP
  
  import scipy.io
  from scipy.io.matlab import MatlabOpaque, MatlabFunction, MatlabObject
  import scipy.io.matlab._byteordercodes as boc
  from scipy.io.matlab._miobase import (
      matdims, MatWriteError, MatReadError, matfile_version)
  from scipy.io.matlab._mio import mat_reader_factory, loadmat, savemat, whosmat
  from scipy.io.matlab._mio5 import (
      MatFile5Writer, MatFile5Reader, varmats_from_mat, to_writeable,
      EmptyStructMarker)
  import scipy.io.matlab._mio5_params as mio5p
  
  test_data_path = pjoin(dirname(__file__), 'data')
  
  
  def mlarr(*args, **kwargs):
      """Convenience function to return matlab-compatible 2-D array."""
      arr = np.array(*args, **kwargs)
      arr.shape = matdims(arr)
      return arr
  
  
  # Define cases to test
  theta = np.pi/4*np.arange(9,dtype=float).reshape(1,9)
  case_table4 = [
      {'name': 'double',
       'classes': {'testdouble': 'double'},
       'expected': {'testdouble': theta}
       }]
  case_table4.append(
      {'name': 'string',
       'classes': {'teststring': 'char'},
       'expected': {'teststring':
                    array(['"Do nine men interpret?" "Nine men," I nod.'])}
       })
  case_table4.append(
      {'name': 'complex',
       'classes': {'testcomplex': 'double'},
       'expected': {'testcomplex': np.cos(theta) + 1j*np.sin(theta)}
       })
  A = np.zeros((3,5))
  A[0] = list(range(1,6))
  A[:,0] = list(range(1,4))
  case_table4.append(
      {'name': 'matrix',
       'classes': {'testmatrix': 'double'},
       'expected': {'testmatrix': A},
       })
  case_table4.append(
      {'name': 'sparse',
       'classes': {'testsparse': 'sparse'},
       'expected': {'testsparse': SP.coo_matrix(A)},
       })
  B = A.astype(complex)
  B[0,0] += 1j
  case_table4.append(
      {'name': 'sparsecomplex',
       'classes': {'testsparsecomplex': 'sparse'},
       'expected': {'testsparsecomplex': SP.coo_matrix(B)},
       })
  case_table4.append(
      {'name': 'multi',
       'classes': {'theta': 'double', 'a': 'double'},
       'expected': {'theta': theta, 'a': A},
       })
  case_table4.append(
      {'name': 'minus',
       'classes': {'testminus': 'double'},
       'expected': {'testminus': mlarr(-1)},
       })
  case_table4.append(
      {'name': 'onechar',
       'classes': {'testonechar': 'char'},
       'expected': {'testonechar': array(['r'])},
       })
  # Cell arrays stored as object arrays
  CA = mlarr((  # tuple for object array creation
          [],
          mlarr([1]),
          mlarr([[1,2]]),
          mlarr([[1,2,3]])), dtype=object).reshape(1,-1)
  CA[0,0] = array(
      ['This cell contains this string and 3 arrays of increasing length'])
  case_table5 = [
      {'name': 'cell',
       'classes': {'testcell': 'cell'},
       'expected': {'testcell': CA}}]
  CAE = mlarr((  # tuple for object array creation
      mlarr(1),
      mlarr(2),
      mlarr([]),
      mlarr([]),
      mlarr(3)), dtype=object).reshape(1,-1)
  objarr = np.empty((1,1),dtype=object)
  objarr[0,0] = mlarr(1)
  case_table5.append(
      {'name': 'scalarcell',
       'classes': {'testscalarcell': 'cell'},
       'expected': {'testscalarcell': objarr}
       })
  case_table5.append(
      {'name': 'emptycell',
       'classes': {'testemptycell': 'cell'},
       'expected': {'testemptycell': CAE}})
  case_table5.append(
      {'name': 'stringarray',
       'classes': {'teststringarray': 'char'},
       'expected': {'teststringarray': array(
      ['one  ', 'two  ', 'three'])},
       })
  case_table5.append(
      {'name': '3dmatrix',
       'classes': {'test3dmatrix': 'double'},
       'expected': {
      'test3dmatrix': np.transpose(np.reshape(list(range(1,25)), (4,3,2)))}
       })
  st_sub_arr = array([np.sqrt(2),np.exp(1),np.pi]).reshape(1,3)
  dtype = [(n, object) for n in ['stringfield', 'doublefield', 'complexfield']]
  st1 = np.zeros((1,1), dtype)
  st1['stringfield'][0,0] = array(['Rats live on no evil star.'])
  st1['doublefield'][0,0] = st_sub_arr
  st1['complexfield'][0,0] = st_sub_arr * (1 + 1j)
  case_table5.append(
      {'name': 'struct',
       'classes': {'teststruct': 'struct'},
       'expected': {'teststruct': st1}
       })
  CN = np.zeros((1,2), dtype=object)
  CN[0,0] = mlarr(1)
  CN[0,1] = np.zeros((1,3), dtype=object)
  CN[0,1][0,0] = mlarr(2, dtype=np.uint8)
  CN[0,1][0,1] = mlarr([[3]], dtype=np.uint8)
  CN[0,1][0,2] = np.zeros((1,2), dtype=object)
  CN[0,1][0,2][0,0] = mlarr(4, dtype=np.uint8)
  CN[0,1][0,2][0,1] = mlarr(5, dtype=np.uint8)
  case_table5.append(
      {'name': 'cellnest',
       'classes': {'testcellnest': 'cell'},
       'expected': {'testcellnest': CN},
       })
  st2 = np.empty((1,1), dtype=[(n, object) for n in ['one', 'two']])
  st2[0,0]['one'] = mlarr(1)
  st2[0,0]['two'] = np.empty((1,1), dtype=[('three', object)])
  st2[0,0]['two'][0,0]['three'] = array(['number 3'])
  case_table5.append(
      {'name': 'structnest',
       'classes': {'teststructnest': 'struct'},
       'expected': {'teststructnest': st2}
       })
  a = np.empty((1,2), dtype=[(n, object) for n in ['one', 'two']])
  a[0,0]['one'] = mlarr(1)
  a[0,0]['two'] = mlarr(2)
  a[0,1]['one'] = array(['number 1'])
  a[0,1]['two'] = array(['number 2'])
  case_table5.append(
      {'name': 'structarr',
       'classes': {'teststructarr': 'struct'},
       'expected': {'teststructarr': a}
       })
  ODT = np.dtype([(n, object) for n in
                   ['expr', 'inputExpr', 'args',
                    'isEmpty', 'numArgs', 'version']])
  MO = MatlabObject(np.zeros((1,1), dtype=ODT), 'inline')
  m0 = MO[0,0]
  m0['expr'] = array(['x'])
  m0['inputExpr'] = array([' x = INLINE_INPUTS_{1};'])
  m0['args'] = array(['x'])
  m0['isEmpty'] = mlarr(0)
  m0['numArgs'] = mlarr(1)
  m0['version'] = mlarr(1)
  case_table5.append(
      {'name': 'object',
       'classes': {'testobject': 'object'},
       'expected': {'testobject': MO}
       })
  fp_u_str = open(pjoin(test_data_path, 'japanese_utf8.txt'), 'rb')
  u_str = fp_u_str.read().decode('utf-8')
  fp_u_str.close()
  case_table5.append(
      {'name': 'unicode',
       'classes': {'testunicode': 'char'},
      'expected': {'testunicode': array([u_str])}
       })
  case_table5.append(
      {'name': 'sparse',
       'classes': {'testsparse': 'sparse'},
       'expected': {'testsparse': SP.coo_matrix(A)},
       })
  case_table5.append(
      {'name': 'sparsecomplex',
       'classes': {'testsparsecomplex': 'sparse'},
       'expected': {'testsparsecomplex': SP.coo_matrix(B)},
       })
  case_table5.append(
      {'name': 'bool',
       'classes': {'testbools': 'logical'},
       'expected': {'testbools':
                    array([[True], [False]])},
       })
  
  case_table5_rt = case_table5[:]
  # Inline functions can't be concatenated in matlab, so RT only
  case_table5_rt.append(
      {'name': 'objectarray',
       'classes': {'testobjectarray': 'object'},
       'expected': {'testobjectarray': np.repeat(MO, 2).reshape(1,2)}})
  
  
  def types_compatible(var1, var2):
      """Check if types are same or compatible.
  
      0-D numpy scalars are compatible with bare python scalars.
      """
      type1 = type(var1)
      type2 = type(var2)
      if type1 is type2:
          return True
      if type1 is np.ndarray and var1.shape == ():
          return type(var1.item()) is type2
      if type2 is np.ndarray and var2.shape == ():
          return type(var2.item()) is type1
      return False
  
  
  def _check_level(label, expected, actual):
      """ Check one level of a potentially nested array """
      if SP.issparse(expected):  # allow different types of sparse matrices
          assert_(SP.issparse(actual))
          assert_array_almost_equal(actual.toarray(),
                                    expected.toarray(),
                                    err_msg=label,
                                    decimal=5)
          return
      # Check types are as expected
      assert_(types_compatible(expected, actual),
              "Expected type %s, got %s at %s" %
              (type(expected), type(actual), label))
      # A field in a record array may not be an ndarray
      # A scalar from a record array will be type np.void
      if not isinstance(expected,
                        (np.void, np.ndarray, MatlabObject)):
          assert_equal(expected, actual)
          return
      # This is an ndarray-like thing
      assert_(expected.shape == actual.shape,
              msg='Expected shape %s, got %s at %s' % (expected.shape,
                                                       actual.shape,
                                                       label))
      ex_dtype = expected.dtype
      if ex_dtype.hasobject:  # array of objects
          if isinstance(expected, MatlabObject):
              assert_equal(expected.classname, actual.classname)
          for i, ev in enumerate(expected):
              level_label = "%s, [%d], " % (label, i)
              _check_level(level_label, ev, actual[i])
          return
      if ex_dtype.fields:  # probably recarray
          for fn in ex_dtype.fields:
              level_label = "%s, field %s, " % (label, fn)
              _check_level(level_label,
                           expected[fn], actual[fn])
          return
      if ex_dtype.type in (str,  # string or bool
                           np.unicode_,
                           np.bool_):
          assert_equal(actual, expected, err_msg=label)
          return
      # Something numeric
      assert_array_almost_equal(actual, expected, err_msg=label, decimal=5)
  
  
  def _load_check_case(name, files, case):
      for file_name in files:
          matdict = loadmat(file_name, struct_as_record=True)
          label = "test %s; file %s" % (name, file_name)
          for k, expected in case.items():
              k_label = "%s, variable %s" % (label, k)
              assert_(k in matdict, "Missing key at %s" % k_label)
              _check_level(k_label, expected, matdict[k])
  
  
  def _whos_check_case(name, files, case, classes):
      for file_name in files:
          label = "test %s; file %s" % (name, file_name)
  
          whos = whosmat(file_name)
  
          expected_whos = [
              (k, expected.shape, classes[k]) for k, expected in case.items()]
  
          whos.sort()
          expected_whos.sort()
          assert_equal(whos, expected_whos,
                       "%s: %r != %r" % (label, whos, expected_whos)
                       )
  
  
  # Round trip tests
  def _rt_check_case(name, expected, format):
      mat_stream = BytesIO()
      savemat(mat_stream, expected, format=format)
      mat_stream.seek(0)
      _load_check_case(name, [mat_stream], expected)
  
  
  # generator for tests
  def _cases(version, filt='test%(name)s_*.mat'):
      if version == '4':
          cases = case_table4
      elif version == '5':
          cases = case_table5
      else:
          assert version == '5_rt'
          cases = case_table5_rt
      for case in cases:
          name = case['name']
          expected = case['expected']
          if filt is None:
              files = None
          else:
              use_filt = pjoin(test_data_path, filt % dict(name=name))
              files = glob(use_filt)
              assert len(files) > 0, \
                  "No files for test %s using filter %s" % (name, filt)
          classes = case['classes']
          yield name, files, expected, classes
  
  
  @pytest.mark.parametrize('version', ('4', '5'))
  def test_load(version):
      for case in _cases(version):
          _load_check_case(*case[:3])
  
  
  @pytest.mark.parametrize('version', ('4', '5'))
  def test_whos(version):
      for case in _cases(version):
          _whos_check_case(*case)
  
  
  # generator for round trip tests
  @pytest.mark.parametrize('version, fmts', [
      ('4', ['4', '5']),
      ('5_rt', ['5']),
  ])
  def test_round_trip(version, fmts):
      for case in _cases(version, filt=None):
          for fmt in fmts:
              _rt_check_case(case[0], case[2], fmt)
  
  
  def test_gzip_simple():
      xdense = np.zeros((20,20))
      xdense[2,3] = 2.3
      xdense[4,5] = 4.5
      x = SP.csc_matrix(xdense)
  
      name = 'gzip_test'
      expected = {'x':x}
      format = '4'
  
      tmpdir = mkdtemp()
      try:
          fname = pjoin(tmpdir,name)
          mat_stream = gzip.open(fname, mode='wb')
          savemat(mat_stream, expected, format=format)
          mat_stream.close()
  
          mat_stream = gzip.open(fname, mode='rb')
          actual = loadmat(mat_stream, struct_as_record=True)
          mat_stream.close()
      finally:
          shutil.rmtree(tmpdir)
  
      assert_array_almost_equal(actual['x'].toarray(),
                                expected['x'].toarray(),
                                err_msg=repr(actual))
  
  
  def test_multiple_open():
      # Ticket #1039, on Windows: check that files are not left open
      tmpdir = mkdtemp()
      try:
          x = dict(x=np.zeros((2, 2)))
  
          fname = pjoin(tmpdir, "a.mat")
  
          # Check that file is not left open
          savemat(fname, x)
          os.unlink(fname)
          savemat(fname, x)
          loadmat(fname)
          os.unlink(fname)
  
          # Check that stream is left open
          f = open(fname, 'wb')
          savemat(f, x)
          f.seek(0)
          f.close()
  
          f = open(fname, 'rb')
          loadmat(f)
          f.seek(0)
          f.close()
      finally:
          shutil.rmtree(tmpdir)
  
  
  def test_mat73():
      # Check any hdf5 files raise an error
      filenames = glob(
          pjoin(test_data_path, 'testhdf5*.mat'))
      assert_(len(filenames) > 0)
      for filename in filenames:
          fp = open(filename, 'rb')
          assert_raises(NotImplementedError,
                        loadmat,
                        fp,
                        struct_as_record=True)
          fp.close()
  
  
  def test_warnings():
      # This test is an echo of the previous behavior, which was to raise a
      # warning if the user triggered a search for mat files on the Python system
      # path. We can remove the test in the next version after upcoming (0.13).
      fname = pjoin(test_data_path, 'testdouble_7.1_GLNX86.mat')
      with warnings.catch_warnings():
          warnings.simplefilter('error')
          # This should not generate a warning
          loadmat(fname, struct_as_record=True)
          # This neither
          loadmat(fname, struct_as_record=False)
  
  
  def test_regression_653():
      # Saving a dictionary with only invalid keys used to raise an error. Now we
      # save this as an empty struct in matlab space.
      sio = BytesIO()
      savemat(sio, {'d':{1:2}}, format='5')
      back = loadmat(sio)['d']
      # Check we got an empty struct equivalent
      assert_equal(back.shape, (1,1))
      assert_equal(back.dtype, np.dtype(object))
      assert_(back[0,0] is None)
  
  
  def test_structname_len():
      # Test limit for length of field names in structs
      lim = 31
      fldname = 'a' * lim
      st1 = np.zeros((1,1), dtype=[(fldname, object)])
      savemat(BytesIO(), {'longstruct': st1}, format='5')
      fldname = 'a' * (lim+1)
      st1 = np.zeros((1,1), dtype=[(fldname, object)])
      assert_raises(ValueError, savemat, BytesIO(),
                    {'longstruct': st1}, format='5')
  
  
  def test_4_and_long_field_names_incompatible():
      # Long field names option not supported in 4
      my_struct = np.zeros((1,1),dtype=[('my_fieldname',object)])
      assert_raises(ValueError, savemat, BytesIO(),
                    {'my_struct':my_struct}, format='4', long_field_names=True)
  
  
  def test_long_field_names():
      # Test limit for length of field names in structs
      lim = 63
      fldname = 'a' * lim
      st1 = np.zeros((1,1), dtype=[(fldname, object)])
      savemat(BytesIO(), {'longstruct': st1}, format='5',long_field_names=True)
      fldname = 'a' * (lim+1)
      st1 = np.zeros((1,1), dtype=[(fldname, object)])
      assert_raises(ValueError, savemat, BytesIO(),
                    {'longstruct': st1}, format='5',long_field_names=True)
  
  
  def test_long_field_names_in_struct():
      # Regression test - long_field_names was erased if you passed a struct
      # within a struct
      lim = 63
      fldname = 'a' * lim
      cell = np.ndarray((1,2),dtype=object)
      st1 = np.zeros((1,1), dtype=[(fldname, object)])
      cell[0,0] = st1
      cell[0,1] = st1
      savemat(BytesIO(), {'longstruct': cell}, format='5',long_field_names=True)
      #
      # Check to make sure it fails with long field names off
      #
      assert_raises(ValueError, savemat, BytesIO(),
                    {'longstruct': cell}, format='5', long_field_names=False)
  
  
  def test_cell_with_one_thing_in_it():
      # Regression test - make a cell array that's 1 x 2 and put two
      # strings in it. It works. Make a cell array that's 1 x 1 and put
      # a string in it. It should work but, in the old days, it didn't.
      cells = np.ndarray((1,2),dtype=object)
      cells[0,0] = 'Hello'
      cells[0,1] = 'World'
      savemat(BytesIO(), {'x': cells}, format='5')
  
      cells = np.ndarray((1,1),dtype=object)
      cells[0,0] = 'Hello, world'
      savemat(BytesIO(), {'x': cells}, format='5')
  
  
  def test_writer_properties():
      # Tests getting, setting of properties of matrix writer
      mfw = MatFile5Writer(BytesIO())
      assert_equal(mfw.global_vars, [])
      mfw.global_vars = ['avar']
      assert_equal(mfw.global_vars, ['avar'])
      assert_equal(mfw.unicode_strings, False)
      mfw.unicode_strings = True
      assert_equal(mfw.unicode_strings, True)
      assert_equal(mfw.long_field_names, False)
      mfw.long_field_names = True
      assert_equal(mfw.long_field_names, True)
  
  
  def test_use_small_element():
      # Test whether we're using small data element or not
      sio = BytesIO()
      wtr = MatFile5Writer(sio)
      # First check size for no sde for name
      arr = np.zeros(10)
      wtr.put_variables({'aaaaa': arr})
      w_sz = len(sio.getvalue())
      # Check small name results in largish difference in size
      sio.truncate(0)
      sio.seek(0)
      wtr.put_variables({'aaaa': arr})
      assert_(w_sz - len(sio.getvalue()) > 4)
      # Whereas increasing name size makes less difference
      sio.truncate(0)
      sio.seek(0)
      wtr.put_variables({'aaaaaa': arr})
      assert_(len(sio.getvalue()) - w_sz < 4)
  
  
  def test_save_dict():
      # Test that both dict and OrderedDict can be saved (as recarray),
      # loaded as matstruct, and preserve order
      ab_exp = np.array([[(1, 2)]], dtype=[('a', object), ('b', object)])
      for dict_type in (dict, OrderedDict):
          # Initialize with tuples to keep order
          d = dict_type([('a', 1), ('b', 2)])
          stream = BytesIO()
          savemat(stream, {'dict': d})
          stream.seek(0)
          vals = loadmat(stream)['dict']
          assert_equal(vals.dtype.names, ('a', 'b'))
          assert_array_equal(vals, ab_exp)
  
  
  def test_1d_shape():
      # New 5 behavior is 1D -> row vector
      arr = np.arange(5)
      for format in ('4', '5'):
          # Column is the default
          stream = BytesIO()
          savemat(stream, {'oned': arr}, format=format)
          vals = loadmat(stream)
          assert_equal(vals['oned'].shape, (1, 5))
          # can be explicitly 'column' for oned_as
          stream = BytesIO()
          savemat(stream, {'oned':arr},
                  format=format,
                  oned_as='column')
          vals = loadmat(stream)
          assert_equal(vals['oned'].shape, (5,1))
          # but different from 'row'
          stream = BytesIO()
          savemat(stream, {'oned':arr},
                  format=format,
                  oned_as='row')
          vals = loadmat(stream)
          assert_equal(vals['oned'].shape, (1,5))
  
  
  def test_compression():
      arr = np.zeros(100).reshape((5,20))
      arr[2,10] = 1
      stream = BytesIO()
      savemat(stream, {'arr':arr})
      raw_len = len(stream.getvalue())
      vals = loadmat(stream)
      assert_array_equal(vals['arr'], arr)
      stream = BytesIO()
      savemat(stream, {'arr':arr}, do_compression=True)
      compressed_len = len(stream.getvalue())
      vals = loadmat(stream)
      assert_array_equal(vals['arr'], arr)
      assert_(raw_len > compressed_len)
      # Concatenate, test later
      arr2 = arr.copy()
      arr2[0,0] = 1
      stream = BytesIO()
      savemat(stream, {'arr':arr, 'arr2':arr2}, do_compression=False)
      vals = loadmat(stream)
      assert_array_equal(vals['arr2'], arr2)
      stream = BytesIO()
      savemat(stream, {'arr':arr, 'arr2':arr2}, do_compression=True)
      vals = loadmat(stream)
      assert_array_equal(vals['arr2'], arr2)
  
  
  def test_single_object():
      stream = BytesIO()
      savemat(stream, {'A':np.array(1, dtype=object)})
  
  
  def test_skip_variable():
      # Test skipping over the first of two variables in a MAT file
      # using mat_reader_factory and put_variables to read them in.
      #
      # This is a regression test of a problem that's caused by
      # using the compressed file reader seek instead of the raw file
      # I/O seek when skipping over a compressed chunk.
      #
      # The problem arises when the chunk is large: this file has
      # a 256x256 array of random (uncompressible) doubles.
      #
      filename = pjoin(test_data_path,'test_skip_variable.mat')
      #
      # Prove that it loads with loadmat
      #
      d = loadmat(filename, struct_as_record=True)
      assert_('first' in d)
      assert_('second' in d)
      #
      # Make the factory
      #
      factory, file_opened = mat_reader_factory(filename, struct_as_record=True)
      #
      # This is where the factory breaks with an error in MatMatrixGetter.to_next
      #
      d = factory.get_variables('second')
      assert_('second' in d)
      factory.mat_stream.close()
  
  
  def test_empty_struct():
      # ticket 885
      filename = pjoin(test_data_path,'test_empty_struct.mat')
      # before ticket fix, this would crash with ValueError, empty data
      # type
      d = loadmat(filename, struct_as_record=True)
      a = d['a']
      assert_equal(a.shape, (1,1))
      assert_equal(a.dtype, np.dtype(object))
      assert_(a[0,0] is None)
      stream = BytesIO()
      arr = np.array((), dtype='U')
      # before ticket fix, this used to give data type not understood
      savemat(stream, {'arr':arr})
      d = loadmat(stream)
      a2 = d['arr']
      assert_array_equal(a2, arr)
  
  
  def test_save_empty_dict():
      # saving empty dict also gives empty struct
      stream = BytesIO()
      savemat(stream, {'arr': {}})
      d = loadmat(stream)
      a = d['arr']
      assert_equal(a.shape, (1,1))
      assert_equal(a.dtype, np.dtype(object))
      assert_(a[0,0] is None)
  
  
  def assert_any_equal(output, alternatives):
      """ Assert `output` is equal to at least one element in `alternatives`
      """
      one_equal = False
      for expected in alternatives:
          if np.all(output == expected):
              one_equal = True
              break
      assert_(one_equal)
  
  
  def test_to_writeable():
      # Test to_writeable function
      res = to_writeable(np.array([1]))  # pass through ndarrays
      assert_equal(res.shape, (1,))
      assert_array_equal(res, 1)
      # Dict fields can be written in any order
      expected1 = np.array([(1, 2)], dtype=[('a', '|O8'), ('b', '|O8')])
      expected2 = np.array([(2, 1)], dtype=[('b', '|O8'), ('a', '|O8')])
      alternatives = (expected1, expected2)
      assert_any_equal(to_writeable({'a':1,'b':2}), alternatives)
      # Fields with underscores discarded
      assert_any_equal(to_writeable({'a':1,'b':2, '_c':3}), alternatives)
      # Not-string fields discarded
      assert_any_equal(to_writeable({'a':1,'b':2, 100:3}), alternatives)
      # String fields that are valid Python identifiers discarded
      assert_any_equal(to_writeable({'a':1,'b':2, '99':3}), alternatives)
      # Object with field names is equivalent
  
      class klass:
          pass
  
      c = klass
      c.a = 1
      c.b = 2
      assert_any_equal(to_writeable(c), alternatives)
      # empty list and tuple go to empty array
      res = to_writeable([])
      assert_equal(res.shape, (0,))
      assert_equal(res.dtype.type, np.float64)
      res = to_writeable(())
      assert_equal(res.shape, (0,))
      assert_equal(res.dtype.type, np.float64)
      # None -> None
      assert_(to_writeable(None) is None)
      # String to strings
      assert_equal(to_writeable('a string').dtype.type, np.str_)
      # Scalars to numpy to NumPy scalars
      res = to_writeable(1)
      assert_equal(res.shape, ())
      assert_equal(res.dtype.type, np.array(1).dtype.type)
      assert_array_equal(res, 1)
      # Empty dict returns EmptyStructMarker
      assert_(to_writeable({}) is EmptyStructMarker)
      # Object does not have (even empty) __dict__
      assert_(to_writeable(object()) is None)
      # Custom object does have empty __dict__, returns EmptyStructMarker
  
      class C:
          pass
  
      assert_(to_writeable(c()) is EmptyStructMarker)
      # dict keys with legal characters are convertible
      res = to_writeable({'a': 1})['a']
      assert_equal(res.shape, (1,))
      assert_equal(res.dtype.type, np.object_)
      # Only fields with illegal characters, falls back to EmptyStruct
      assert_(to_writeable({'1':1}) is EmptyStructMarker)
      assert_(to_writeable({'_a':1}) is EmptyStructMarker)
      # Unless there are valid fields, in which case structured array
      assert_equal(to_writeable({'1':1, 'f': 2}),
                   np.array([(2,)], dtype=[('f', '|O8')]))
  
  
  def test_recarray():
      # check roundtrip of structured array
      dt = [('f1', 'f8'),
            ('f2', 'S10')]
      arr = np.zeros((2,), dtype=dt)
      arr[0]['f1'] = 0.5
      arr[0]['f2'] = 'python'
      arr[1]['f1'] = 99
      arr[1]['f2'] = 'not perl'
      stream = BytesIO()
      savemat(stream, {'arr': arr})
      d = loadmat(stream, struct_as_record=False)
      a20 = d['arr'][0,0]
      assert_equal(a20.f1, 0.5)
      assert_equal(a20.f2, 'python')
      d = loadmat(stream, struct_as_record=True)
      a20 = d['arr'][0,0]
      assert_equal(a20['f1'], 0.5)
      assert_equal(a20['f2'], 'python')
      # structs always come back as object types
      assert_equal(a20.dtype, np.dtype([('f1', 'O'),
                                        ('f2', 'O')]))
      a21 = d['arr'].flat[1]
      assert_equal(a21['f1'], 99)
      assert_equal(a21['f2'], 'not perl')
  
  
  def test_save_object():
      class C:
          pass
      c = C()
      c.field1 = 1
      c.field2 = 'a string'
      stream = BytesIO()
      savemat(stream, {'c': c})
      d = loadmat(stream, struct_as_record=False)
      c2 = d['c'][0,0]
      assert_equal(c2.field1, 1)
      assert_equal(c2.field2, 'a string')
      d = loadmat(stream, struct_as_record=True)
      c2 = d['c'][0,0]
      assert_equal(c2['field1'], 1)
      assert_equal(c2['field2'], 'a string')
  
  
  def test_read_opts():
      # tests if read is seeing option sets, at initialization and after
      # initialization
      arr = np.arange(6).reshape(1,6)
      stream = BytesIO()
      savemat(stream, {'a': arr})
      rdr = MatFile5Reader(stream)
      back_dict = rdr.get_variables()
      rarr = back_dict['a']
      assert_array_equal(rarr, arr)
      rdr = MatFile5Reader(stream, squeeze_me=True)
      assert_array_equal(rdr.get_variables()['a'], arr.reshape((6,)))
      rdr.squeeze_me = False
      assert_array_equal(rarr, arr)
      rdr = MatFile5Reader(stream, byte_order=boc.native_code)
      assert_array_equal(rdr.get_variables()['a'], arr)
      # inverted byte code leads to error on read because of swapped
      # header etc.
      rdr = MatFile5Reader(stream, byte_order=boc.swapped_code)
      assert_raises(Exception, rdr.get_variables)
      rdr.byte_order = boc.native_code
      assert_array_equal(rdr.get_variables()['a'], arr)
      arr = np.array(['a string'])
      stream.truncate(0)
      stream.seek(0)
      savemat(stream, {'a': arr})
      rdr = MatFile5Reader(stream)
      assert_array_equal(rdr.get_variables()['a'], arr)
      rdr = MatFile5Reader(stream, chars_as_strings=False)
      carr = np.atleast_2d(np.array(list(arr.item()), dtype='U1'))
      assert_array_equal(rdr.get_variables()['a'], carr)
      rdr.chars_as_strings = True
      assert_array_equal(rdr.get_variables()['a'], arr)
  
  
  def test_empty_string():
      # make sure reading empty string does not raise error
      estring_fname = pjoin(test_data_path, 'single_empty_string.mat')
      fp = open(estring_fname, 'rb')
      rdr = MatFile5Reader(fp)
      d = rdr.get_variables()
      fp.close()
      assert_array_equal(d['a'], np.array([], dtype='U1'))
      # Empty string round trip. Matlab cannot distinguish
      # between a string array that is empty, and a string array
      # containing a single empty string, because it stores strings as
      # arrays of char. There is no way of having an array of char that
      # is not empty, but contains an empty string.
      stream = BytesIO()
      savemat(stream, {'a': np.array([''])})
      rdr = MatFile5Reader(stream)
      d = rdr.get_variables()
      assert_array_equal(d['a'], np.array([], dtype='U1'))
      stream.truncate(0)
      stream.seek(0)
      savemat(stream, {'a': np.array([], dtype='U1')})
      rdr = MatFile5Reader(stream)
      d = rdr.get_variables()
      assert_array_equal(d['a'], np.array([], dtype='U1'))
      stream.close()
  
  
  def test_corrupted_data():
      import zlib
      for exc, fname in [(ValueError, 'corrupted_zlib_data.mat'),
                         (zlib.error, 'corrupted_zlib_checksum.mat')]:
          with open(pjoin(test_data_path, fname), 'rb') as fp:
              rdr = MatFile5Reader(fp)
              assert_raises(exc, rdr.get_variables)
  
  
  def test_corrupted_data_check_can_be_disabled():
      with open(pjoin(test_data_path, 'corrupted_zlib_data.mat'), 'rb') as fp:
          rdr = MatFile5Reader(fp, verify_compressed_data_integrity=False)
          rdr.get_variables()
  
  
  def test_read_both_endian():
      # make sure big- and little- endian data is read correctly
      for fname in ('big_endian.mat', 'little_endian.mat'):
          fp = open(pjoin(test_data_path, fname), 'rb')
          rdr = MatFile5Reader(fp)
          d = rdr.get_variables()
          fp.close()
          assert_array_equal(d['strings'],
                             np.array([['hello'],
                                       ['world']], dtype=object))
          assert_array_equal(d['floats'],
                             np.array([[2., 3.],
                                       [3., 4.]], dtype=np.float32))
  
  
  def test_write_opposite_endian():
      # We don't support writing opposite endian .mat files, but we need to behave
      # correctly if the user supplies an other-endian NumPy array to write out.
      float_arr = np.array([[2., 3.],
                            [3., 4.]])
      int_arr = np.arange(6).reshape((2, 3))
      uni_arr = np.array(['hello', 'world'], dtype='U')
      stream = BytesIO()
      savemat(stream, {'floats': float_arr.byteswap().newbyteorder(),
                              'ints': int_arr.byteswap().newbyteorder(),
                              'uni_arr': uni_arr.byteswap().newbyteorder()})
      rdr = MatFile5Reader(stream)
      d = rdr.get_variables()
      assert_array_equal(d['floats'], float_arr)
      assert_array_equal(d['ints'], int_arr)
      assert_array_equal(d['uni_arr'], uni_arr)
      stream.close()
  
  
  def test_logical_array():
      # The roundtrip test doesn't verify that we load the data up with the
      # correct (bool) dtype
      with open(pjoin(test_data_path, 'testbool_8_WIN64.mat'), 'rb') as fobj:
          rdr = MatFile5Reader(fobj, mat_dtype=True)
          d = rdr.get_variables()
      x = np.array([[True], [False]], dtype=np.bool_)
      assert_array_equal(d['testbools'], x)
      assert_equal(d['testbools'].dtype, x.dtype)
  
  
  def test_logical_out_type():
      # Confirm that bool type written as uint8, uint8 class
      # See gh-4022
      stream = BytesIO()
      barr = np.array([False, True, False])
      savemat(stream, {'barray': barr})
      stream.seek(0)
      reader = MatFile5Reader(stream)
      reader.initialize_read()
      reader.read_file_header()
      hdr, _ = reader.read_var_header()
      assert_equal(hdr.mclass, mio5p.mxUINT8_CLASS)
      assert_equal(hdr.is_logical, True)
      var = reader.read_var_array(hdr, False)
      assert_equal(var.dtype.type, np.uint8)
  
  
  def test_roundtrip_zero_dimensions():
      stream = BytesIO()
      savemat(stream, {'d':np.empty((10, 0))})
      d = loadmat(stream)
      assert d['d'].shape == (10, 0)
  
  
  def test_mat4_3d():
      # test behavior when writing 3-D arrays to matlab 4 files
      stream = BytesIO()
      arr = np.arange(24).reshape((2,3,4))
      assert_raises(ValueError, savemat, stream, {'a': arr}, True, '4')
  
  
  def test_func_read():
      func_eg = pjoin(test_data_path, 'testfunc_7.4_GLNX86.mat')
      fp = open(func_eg, 'rb')
      rdr = MatFile5Reader(fp)
      d = rdr.get_variables()
      fp.close()
      assert isinstance(d['testfunc'], MatlabFunction)
      stream = BytesIO()
      wtr = MatFile5Writer(stream)
      assert_raises(MatWriteError, wtr.put_variables, d)
  
  
  def test_mat_dtype():
      double_eg = pjoin(test_data_path, 'testmatrix_6.1_SOL2.mat')
      fp = open(double_eg, 'rb')
      rdr = MatFile5Reader(fp, mat_dtype=False)
      d = rdr.get_variables()
      fp.close()
      assert_equal(d['testmatrix'].dtype.kind, 'u')
  
      fp = open(double_eg, 'rb')
      rdr = MatFile5Reader(fp, mat_dtype=True)
      d = rdr.get_variables()
      fp.close()
      assert_equal(d['testmatrix'].dtype.kind, 'f')
  
  
  def test_sparse_in_struct():
      # reproduces bug found by DC where Cython code was insisting on
      # ndarray return type, but getting sparse matrix
      st = {'sparsefield': SP.coo_matrix(np.eye(4))}
      stream = BytesIO()
      savemat(stream, {'a':st})
      d = loadmat(stream, struct_as_record=True)
      assert_array_equal(d['a'][0, 0]['sparsefield'].toarray(), np.eye(4))
  
  
  def test_mat_struct_squeeze():
      stream = BytesIO()
      in_d = {'st':{'one':1, 'two':2}}
      savemat(stream, in_d)
      # no error without squeeze
      loadmat(stream, struct_as_record=False)
      # previous error was with squeeze, with mat_struct
      loadmat(stream, struct_as_record=False, squeeze_me=True)
  
  
  def test_scalar_squeeze():
      stream = BytesIO()
      in_d = {'scalar': [[0.1]], 'string': 'my name', 'st':{'one':1, 'two':2}}
      savemat(stream, in_d)
      out_d = loadmat(stream, squeeze_me=True)
      assert_(isinstance(out_d['scalar'], float))
      assert_(isinstance(out_d['string'], str))
      assert_(isinstance(out_d['st'], np.ndarray))
  
  
  def test_str_round():
      # from report by Angus McMorland on mailing list 3 May 2010
      stream = BytesIO()
      in_arr = np.array(['Hello', 'Foob'])
      out_arr = np.array(['Hello', 'Foob '])
      savemat(stream, dict(a=in_arr))
      res = loadmat(stream)
      # resulted in ['HloolFoa', 'elWrdobr']
      assert_array_equal(res['a'], out_arr)
      stream.truncate(0)
      stream.seek(0)
      # Make Fortran ordered version of string
      in_str = in_arr.tobytes(order='F')
      in_from_str = np.ndarray(shape=a.shape,
                               dtype=in_arr.dtype,
                               order='F',
                               buffer=in_str)
      savemat(stream, dict(a=in_from_str))
      assert_array_equal(res['a'], out_arr)
      # unicode save did lead to buffer too small error
      stream.truncate(0)
      stream.seek(0)
      in_arr_u = in_arr.astype('U')
      out_arr_u = out_arr.astype('U')
      savemat(stream, {'a': in_arr_u})
      res = loadmat(stream)
      assert_array_equal(res['a'], out_arr_u)
  
  
  def test_fieldnames():
      # Check that field names are as expected
      stream = BytesIO()
      savemat(stream, {'a': {'a':1, 'b':2}})
      res = loadmat(stream)
      field_names = res['a'].dtype.names
      assert_equal(set(field_names), set(('a', 'b')))
  
  
  def test_loadmat_varnames():
      # Test that we can get just one variable from a mat file using loadmat
      mat5_sys_names = ['__globals__',
                        '__header__',
                        '__version__']
      for eg_file, sys_v_names in (
          (pjoin(test_data_path, 'testmulti_4.2c_SOL2.mat'), []), (pjoin(
              test_data_path, 'testmulti_7.4_GLNX86.mat'), mat5_sys_names)):
          vars = loadmat(eg_file)
          assert_equal(set(vars.keys()), set(['a', 'theta'] + sys_v_names))
          vars = loadmat(eg_file, variable_names='a')
          assert_equal(set(vars.keys()), set(['a'] + sys_v_names))
          vars = loadmat(eg_file, variable_names=['a'])
          assert_equal(set(vars.keys()), set(['a'] + sys_v_names))
          vars = loadmat(eg_file, variable_names=['theta'])
          assert_equal(set(vars.keys()), set(['theta'] + sys_v_names))
          vars = loadmat(eg_file, variable_names=('theta',))
          assert_equal(set(vars.keys()), set(['theta'] + sys_v_names))
          vars = loadmat(eg_file, variable_names=[])
          assert_equal(set(vars.keys()), set(sys_v_names))
          vnames = ['theta']
          vars = loadmat(eg_file, variable_names=vnames)
          assert_equal(vnames, ['theta'])
  
  
  def test_round_types():
      # Check that saving, loading preserves dtype in most cases
      arr = np.arange(10)
      stream = BytesIO()
      for dts in ('f8','f4','i8','i4','i2','i1',
                  'u8','u4','u2','u1','c16','c8'):
          stream.truncate(0)
          stream.seek(0)  # needed for BytesIO in Python 3
          savemat(stream, {'arr': arr.astype(dts)})
          vars = loadmat(stream)
          assert_equal(np.dtype(dts), vars['arr'].dtype)
  
  
  def test_varmats_from_mat():
      # Make a mat file with several variables, write it, read it back
      names_vars = (('arr', mlarr(np.arange(10))),
                    ('mystr', mlarr('a string')),
                    ('mynum', mlarr(10)))
  
      # Dict like thing to give variables in defined order
      class C:
          def items(self):
              return names_vars
      stream = BytesIO()
      savemat(stream, C())
      varmats = varmats_from_mat(stream)
      assert_equal(len(varmats), 3)
      for i in range(3):
          name, var_stream = varmats[i]
          exp_name, exp_res = names_vars[i]
          assert_equal(name, exp_name)
          res = loadmat(var_stream)
          assert_array_equal(res[name], exp_res)
  
  
  def test_one_by_zero():
      # Test 1x0 chars get read correctly
      func_eg = pjoin(test_data_path, 'one_by_zero_char.mat')
      fp = open(func_eg, 'rb')
      rdr = MatFile5Reader(fp)
      d = rdr.get_variables()
      fp.close()
      assert_equal(d['var'].shape, (0,))
  
  
  def test_load_mat4_le():
      # We were getting byte order wrong when reading little-endian floa64 dense
      # matrices on big-endian platforms
      mat4_fname = pjoin(test_data_path, 'test_mat4_le_floats.mat')
      vars = loadmat(mat4_fname)
      assert_array_equal(vars['a'], [[0.1, 1.2]])
  
  
  def test_unicode_mat4():
      # Mat4 should save unicode as latin1
      bio = BytesIO()
      var = {'second_cat': 'Schrödinger'}
      savemat(bio, var, format='4')
      var_back = loadmat(bio)
      assert_equal(var_back['second_cat'], var['second_cat'])
  
  
  def test_logical_sparse():
      # Test we can read logical sparse stored in mat file as bytes.
      # See https://github.com/scipy/scipy/issues/3539.
      # In some files saved by MATLAB, the sparse data elements (Real Part
      # Subelement in MATLAB speak) are stored with apparent type double
      # (miDOUBLE) but are in fact single bytes.
      filename = pjoin(test_data_path,'logical_sparse.mat')
      # Before fix, this would crash with:
      # ValueError: indices and data should have the same size
      d = loadmat(filename, struct_as_record=True)
      log_sp = d['sp_log_5_4']
      assert_(isinstance(log_sp, SP.csc_matrix))
      assert_equal(log_sp.dtype.type, np.bool_)
      assert_array_equal(log_sp.toarray(),
                         [[True, True, True, False],
                          [False, False, True, False],
                          [False, False, True, False],
                          [False, False, False, False],
                          [False, False, False, False]])
  
  
  def test_empty_sparse():
      # Can we read empty sparse matrices?
      sio = BytesIO()
      import scipy.sparse
      empty_sparse = scipy.sparse.csr_matrix([[0,0],[0,0]])
      savemat(sio, dict(x=empty_sparse))
      sio.seek(0)
      res = loadmat(sio)
      assert_array_equal(res['x'].shape, empty_sparse.shape)
      assert_array_equal(res['x'].toarray(), 0)
      # Do empty sparse matrices get written with max nnz 1?
      # See https://github.com/scipy/scipy/issues/4208
      sio.seek(0)
      reader = MatFile5Reader(sio)
      reader.initialize_read()
      reader.read_file_header()
      hdr, _ = reader.read_var_header()
      assert_equal(hdr.nzmax, 1)
  
  
  def test_empty_mat_error():
      # Test we get a specific warning for an empty mat file
      sio = BytesIO()
      assert_raises(MatReadError, loadmat, sio)
  
  
  def test_miuint32_compromise():
      # Reader should accept miUINT32 for miINT32, but check signs
      # mat file with miUINT32 for miINT32, but OK values
      filename = pjoin(test_data_path, 'miuint32_for_miint32.mat')
      res = loadmat(filename)
      assert_equal(res['an_array'], np.arange(10)[None, :])
      # mat file with miUINT32 for miINT32, with negative value
      filename = pjoin(test_data_path, 'bad_miuint32.mat')
      with assert_raises(ValueError):
          loadmat(filename)
  
  
  def test_miutf8_for_miint8_compromise():
      # Check reader accepts ascii as miUTF8 for array names
      filename = pjoin(test_data_path, 'miutf8_array_name.mat')
      res = loadmat(filename)
      assert_equal(res['array_name'], [[1]])
      # mat file with non-ascii utf8 name raises error
      filename = pjoin(test_data_path, 'bad_miutf8_array_name.mat')
      with assert_raises(ValueError):
          loadmat(filename)
  
  
  def test_bad_utf8():
      # Check that reader reads bad UTF with 'replace' option
      filename = pjoin(test_data_path,'broken_utf8.mat')
      res = loadmat(filename)
      assert_equal(res['bad_string'],
                   b'\x80 am broken'.decode('utf8', 'replace'))
  
  
  def test_save_unicode_field(tmpdir):
      filename = os.path.join(str(tmpdir), 'test.mat')
      test_dict = {u'a':{u'b':1,u'c':'test_str'}}
      savemat(filename, test_dict)
  
  
  def test_filenotfound():
      # Check the correct error is thrown
      assert_raises(OSError, loadmat, "NotExistentFile00.mat")
      assert_raises(OSError, loadmat, "NotExistentFile00")
  
  
  def test_simplify_cells():
      # Test output when simplify_cells=True
      filename = pjoin(test_data_path, 'testsimplecell.mat')
      res1 = loadmat(filename, simplify_cells=True)
      res2 = loadmat(filename, simplify_cells=False)
      assert_(isinstance(res1["s"], dict))
      assert_(isinstance(res2["s"], np.ndarray))
      assert_array_equal(res1["s"]["mycell"], np.array(["a", "b", "c"]))
  
  
  @pytest.mark.parametrize('version, filt, regex', [
      (0, '_4*_*', None),
      (1, '_5*_*', None),
      (1, '_6*_*', None),
      (1, '_7*_*', '^((?!hdf5).)*$'),  # not containing hdf5
      (2, '_7*_*', '.*hdf5.*'),
      (1, '8*_*', None),
  ])
  def test_matfile_version(version, filt, regex):
      use_filt = pjoin(test_data_path, 'test*%s.mat' % filt)
      files = glob(use_filt)
      if regex is not None:
          files = [file for file in files if re.match(regex, file) is not None]
      assert len(files) > 0, \
          "No files for version %s using filter %s" % (version, filt)
      for file in files:
          got_version = matfile_version(file)
          assert got_version[0] == version
  
  
  def test_opaque():
      """Test that we can read a MatlabOpaque object."""
      data = loadmat(pjoin(test_data_path, 'parabola.mat'))
      assert isinstance(data['parabola'], MatlabFunction)
      assert isinstance(data['parabola'].item()[3].item()[3], MatlabOpaque)
  
  
  def test_deprecation():
      """Test that access to previous attributes still works."""
      # This should be accessible immediately from scipy.io import
      with assert_warns(DeprecationWarning):
          scipy.io.matlab.mio5_params.MatlabOpaque  # noqa
  
      # These should be importable but warn as well
      with assert_warns(DeprecationWarning):
          from scipy.io.matlab.miobase import MatReadError  # noqa
  