code: """
  High level interface to PyTables for reading and writing pandas data structures
  to disk
  """
  
  # pylint: disable-msg=E1101,W0613,W0603
  from datetime import datetime, date
  import time
  import re
  import copy
  import itertools
  import warnings
  import os
  
  from pandas.types.common import (is_list_like,
                                   is_categorical_dtype,
                                   is_timedelta64_dtype,
                                   is_datetime64tz_dtype,
                                   is_datetime64_dtype,
                                   _ensure_object,
                                   _ensure_int64,
                                   _ensure_platform_int)
  from pandas.types.missing import array_equivalent
  
  import numpy as np
  
  import pandas as pd
  from pandas import (Series, DataFrame, Panel, Panel4D, Index,
                      MultiIndex, Int64Index, isnull)
  from pandas.core import config
  from pandas.io.common import _stringify_path
  from pandas.sparse.api import SparseSeries, SparseDataFrame
  from pandas.sparse.array import BlockIndex, IntIndex
  from pandas.tseries.api import PeriodIndex, DatetimeIndex
  from pandas.tseries.tdi import TimedeltaIndex
  from pandas.core.base import StringMixin
  from pandas.formats.printing import adjoin, pprint_thing
  from pandas.core.common import _asarray_tuplesafe, PerformanceWarning
  from pandas.core.algorithms import match, unique
  from pandas.core.categorical import Categorical
  from pandas.core.internals import (BlockManager, make_block,
                                     _block2d_to_blocknd,
                                     _factor_indexer, _block_shape)
  from pandas.core.index import _ensure_index
  from pandas.tools.merge import concat
  from pandas import compat
  from pandas.compat import u_safe as u, PY3, range, lrange, string_types, filter
  from pandas.core.config import get_option
  from pandas.computation.pytables import Expr, maybe_expression
  
  import pandas.lib as lib
  import pandas.algos as algos
  import pandas.tslib as tslib
  
  from distutils.version import LooseVersion
  
  # versioning attribute
  _version = '0.15.2'
  
  # encoding
  # PY3 encoding if we don't specify
  _default_encoding = 'UTF-8'
  
  
  def _ensure_decoded(s):
      """ if we have bytes, decode them to unicode """
      if isinstance(s, np.bytes_):
          s = s.decode('UTF-8')
      return s
  
  
  def _ensure_encoding(encoding):
      # set the encoding if we need
      if encoding is None:
          if PY3:
              encoding = _default_encoding
      return encoding
  
  Term = Expr
  
  
  def _ensure_term(where, scope_level):
      """
      ensure that the where is a Term or a list of Term
      this makes sure that we are capturing the scope of variables
      that are passed
      create the terms here with a frame_level=2 (we are 2 levels down)
      """
  
      # only consider list/tuple here as an ndarray is automaticaly a coordinate
      # list
      level = scope_level + 1
      if isinstance(where, (list, tuple)):
          wlist = []
          for w in filter(lambda x: x is not None, where):
              if not maybe_expression(w):
                  wlist.append(w)
              else:
                  wlist.append(Term(w, scope_level=level))
          where = wlist
      elif maybe_expression(where):
          where = Term(where, scope_level=level)
      return where
  
  
  class PossibleDataLossError(Exception):
      pass
  
  
  class ClosedFileError(Exception):
      pass
  
  
  class IncompatibilityWarning(Warning):
      pass
  
  incompatibility_doc = """
  where criteria is being ignored as this version [%s] is too old (or
  not-defined), read the file in and write it out to a new file to upgrade (with
  the copy_to method)
  """
  
  
  class AttributeConflictWarning(Warning):
      pass
  
  attribute_conflict_doc = """
  the [%s] attribute of the existing index is [%s] which conflicts with the new
  [%s], resetting the attribute to None
  """
  
  
  class DuplicateWarning(Warning):
      pass
  
  duplicate_doc = """
  duplicate entries in table, taking most recently appended
  """
  
  performance_doc = """
  your performance may suffer as PyTables will pickle object types that it cannot
  map directly to c-types [inferred_type->%s,key->%s] [items->%s]
  """
  
  # formats
  _FORMAT_MAP = {
      u('f'): 'fixed',
      u('fixed'): 'fixed',
      u('t'): 'table',
      u('table'): 'table',
  }
  
  format_deprecate_doc = """
  the table keyword has been deprecated
  use the format='fixed(f)|table(t)' keyword instead
    fixed(f) : specifies the Fixed format
               and is the default for put operations
    table(t) : specifies the Table format
               and is the default for append operations
  """
  
  # map object types
  _TYPE_MAP = {
  
      Series: u('series'),
      SparseSeries: u('sparse_series'),
      pd.TimeSeries: u('series'),
      DataFrame: u('frame'),
      SparseDataFrame: u('sparse_frame'),
      Panel: u('wide'),
      Panel4D: u('ndim'),
  }
  
  # storer class map
  _STORER_MAP = {
      u('TimeSeries'): 'LegacySeriesFixed',
      u('Series'): 'LegacySeriesFixed',
      u('DataFrame'): 'LegacyFrameFixed',
      u('DataMatrix'): 'LegacyFrameFixed',
      u('series'): 'SeriesFixed',
      u('sparse_series'): 'SparseSeriesFixed',
      u('frame'): 'FrameFixed',
      u('sparse_frame'): 'SparseFrameFixed',
      u('wide'): 'PanelFixed',
  }
  
  # table class map
  _TABLE_MAP = {
      u('generic_table'): 'GenericTable',
      u('appendable_series'): 'AppendableSeriesTable',
      u('appendable_multiseries'): 'AppendableMultiSeriesTable',
      u('appendable_frame'): 'AppendableFrameTable',
      u('appendable_multiframe'): 'AppendableMultiFrameTable',
      u('appendable_panel'): 'AppendablePanelTable',
      u('appendable_ndim'): 'AppendableNDimTable',
      u('worm'): 'WORMTable',
      u('legacy_frame'): 'LegacyFrameTable',
      u('legacy_panel'): 'LegacyPanelTable',
  }
  
  # axes map
  _AXES_MAP = {
      DataFrame: [0],
      Panel: [1, 2],
      Panel4D: [1, 2, 3],
  }
  
  # register our configuration options
  dropna_doc = """
  : boolean
      drop ALL nan rows when appending to a table
  """
  format_doc = """
  : format
      default format writing format, if None, then
      put will default to 'fixed' and append will default to 'table'
  """
  
  with config.config_prefix('io.hdf'):
      config.register_option('dropna_table', False, dropna_doc,
                             validator=config.is_bool)
      config.register_option(
          'default_format', None, format_doc,
          validator=config.is_one_of_factory(['fixed', 'table', None])
      )
  
  # oh the troubles to reduce import time
  _table_mod = None
  _table_file_open_policy_is_strict = False
  
  
  def _tables():
      global _table_mod
      global _table_file_open_policy_is_strict
      if _table_mod is None:
          import tables
          _table_mod = tables
  
          # version requirements
          if LooseVersion(tables.__version__) < '3.0.0':
              raise ImportError("PyTables version >= 3.0.0 is required")
  
          # set the file open policy
          # return the file open policy; this changes as of pytables 3.1
          # depending on the HDF5 version
          try:
              _table_file_open_policy_is_strict = (
                  tables.file._FILE_OPEN_POLICY == 'strict')
          except:
              pass
  
      return _table_mod
  
  # interface to/from ###
  
  
  def to_hdf(path_or_buf, key, value, mode=None, complevel=None, complib=None,
             append=None, **kwargs):
      """ store this object, close it if we opened it """
  
      if append:
          f = lambda store: store.append(key, value, **kwargs)
      else:
          f = lambda store: store.put(key, value, **kwargs)
  
      path_or_buf = _stringify_path(path_or_buf)
      if isinstance(path_or_buf, string_types):
          with HDFStore(path_or_buf, mode=mode, complevel=complevel,
                        complib=complib) as store:
              f(store)
      else:
          f(path_or_buf)
  
  
  def read_hdf(path_or_buf, key=None, **kwargs):
      """ read from the store, close it if we opened it
  
          Retrieve pandas object stored in file, optionally based on where
          criteria
  
          Parameters
          ----------
          path_or_buf : path (string), buffer, or path object (pathlib.Path or
              py._path.local.LocalPath) to read from
  
              .. versionadded:: 0.19.0 support for pathlib, py.path.
  
          key : group identifier in the store. Can be omitted if the HDF file
              contains a single pandas object.
          where : list of Term (or convertable) objects, optional
          start : optional, integer (defaults to None), row number to start
              selection
          stop  : optional, integer (defaults to None), row number to stop
              selection
          columns : optional, a list of columns that if not None, will limit the
              return columns
          iterator : optional, boolean, return an iterator, default False
          chunksize : optional, nrows to include in iteration, return an iterator
  
          Returns
          -------
          The selected object
  
          """
  
      if kwargs.get('mode', 'a') not in ['r', 'r+', 'a']:
          raise ValueError('mode {0} is not allowed while performing a read. '
                           'Allowed modes are r, r+ and a.'
                           .format(kwargs.get('mode')))
      # grab the scope
      if 'where' in kwargs:
          kwargs['where'] = _ensure_term(kwargs['where'], scope_level=1)
  
      path_or_buf = _stringify_path(path_or_buf)
      if isinstance(path_or_buf, string_types):
  
          try:
              exists = os.path.exists(path_or_buf)
  
          # if filepath is too long
          except (TypeError, ValueError):
              exists = False
  
          if not exists:
              raise IOError('File %s does not exist' % path_or_buf)
  
          # can't auto open/close if we are using an iterator
          # so delegate to the iterator
          store = HDFStore(path_or_buf, **kwargs)
          auto_close = True
  
      elif isinstance(path_or_buf, HDFStore):
          if not path_or_buf.is_open:
              raise IOError('The HDFStore must be open for reading.')
  
          store = path_or_buf
          auto_close = False
  
      else:
          raise NotImplementedError('Support for generic buffers has not been '
                                    'implemented.')
  
      try:
          if key is None:
              groups = store.groups()
              if len(groups) == 0:
                  raise ValueError('No dataset in HDF5 file.')
              candidate_only_group = groups[0]
  
              # For the HDF file to have only one dataset, all other groups
              # should then be metadata groups for that candidate group. (This
              # assumes that the groups() method enumerates parent groups
              # before their children.)
              for group_to_check in groups[1:]:
                  if not _is_metadata_of(group_to_check, candidate_only_group):
                      raise ValueError('key must be provided when HDF5 file '
                                       'contains multiple datasets.')
              key = candidate_only_group._v_pathname
          return store.select(key, auto_close=auto_close, **kwargs)
      except:
          # if there is an error, close the store
          try:
              store.close()
          except:
              pass
  
          raise
  
  
  def _is_metadata_of(group, parent_group):
      """Check if a given group is a metadata group for a given parent_group."""
      if group._v_depth <= parent_group._v_depth:
          return False
  
      current = group
      while current._v_depth > 1:
          parent = current._v_parent
          if parent == parent_group and current._v_name == 'meta':
              return True
          current = current._v_parent
      return False
  
  
  class HDFStore(StringMixin):
  
      """
      dict-like IO interface for storing pandas objects in PyTables
      either Fixed or Table format.
  
      Parameters
      ----------
      path : string
          File path to HDF5 file
      mode : {'a', 'w', 'r', 'r+'}, default 'a'
  
          ``'r'``
              Read-only; no data can be modified.
          ``'w'``
              Write; a new file is created (an existing file with the same
              name would be deleted).
          ``'a'``
              Append; an existing file is opened for reading and writing,
              and if the file does not exist it is created.
          ``'r+'``
              It is similar to ``'a'``, but the file must already exist.
      complevel : int, 1-9, default 0
              If a complib is specified compression will be applied
              where possible
      complib : {'zlib', 'bzip2', 'lzo', 'blosc', None}, default None
              If complevel is > 0 apply compression to objects written
              in the store wherever possible
      fletcher32 : bool, default False
              If applying compression use the fletcher32 checksum
  
      Examples
      --------
      >>> from pandas import DataFrame
      >>> from numpy.random import randn
      >>> bar = DataFrame(randn(10, 4))
      >>> store = HDFStore('test.h5')
      >>> store['foo'] = bar   # write to HDF5
      >>> bar = store['foo']   # retrieve
      >>> store.close()
      """
  
      def __init__(self, path, mode=None, complevel=None, complib=None,
                   fletcher32=False, **kwargs):
          try:
              import tables  # noqa
          except ImportError as ex:  # pragma: no cover
              raise ImportError('HDFStore requires PyTables, "{ex}" problem '
                                'importing'.format(ex=str(ex)))
  
          if complib not in (None, 'blosc', 'bzip2', 'lzo', 'zlib'):
              raise ValueError("complib only supports 'blosc', 'bzip2', lzo' "
                               "or 'zlib' compression.")
  
          self._path = path
          if mode is None:
              mode = 'a'
          self._mode = mode
          self._handle = None
          self._complevel = complevel
          self._complib = complib
          self._fletcher32 = fletcher32
          self._filters = None
          self.open(mode=mode, **kwargs)
  
      @property
      def root(self):
          """ return the root node """
          self._check_if_open()
          return self._handle.root
  
      @property
      def filename(self):
          return self._path
  
      def __getitem__(self, key):
          return self.get(key)
  
      def __setitem__(self, key, value):
          self.put(key, value)
  
      def __delitem__(self, key):
          return self.remove(key)
  
      def __getattr__(self, name):
          """ allow attribute access to get stores """
          self._check_if_open()
          try:
              return self.get(name)
          except:
              pass
          raise AttributeError("'%s' object has no attribute '%s'" %
                               (type(self).__name__, name))
  
      def __contains__(self, key):
          """ check for existance of this key
                can match the exact pathname or the pathnm w/o the leading '/'
                """
          node = self.get_node(key)
          if node is not None:
              name = node._v_pathname
              if name == key or name[1:] == key:
                  return True
          return False
  
      def __len__(self):
          return len(self.groups())
  
      def __unicode__(self):
          output = '%s\nFile path: %s\n' % (type(self), pprint_thing(self._path))
          if self.is_open:
              lkeys = sorted(list(self.keys()))
              if len(lkeys):
                  keys = []
                  values = []
  
                  for k in lkeys:
                      try:
                          s = self.get_storer(k)
                          if s is not None:
                              keys.append(pprint_thing(s.pathname or k))
                              values.append(
                                  pprint_thing(s or 'invalid_HDFStore node'))
                      except Exception as detail:
                          keys.append(k)
                          values.append("[invalid_HDFStore node: %s]"
                                        % pprint_thing(detail))
  
                  output += adjoin(12, keys, values)
              else:
                  output += 'Empty'
          else:
              output += "File is CLOSED"
  
          return output
  
      def __enter__(self):
          return self
  
      def __exit__(self, exc_type, exc_value, traceback):
          self.close()
  
      def keys(self):
          """
          Return a (potentially unordered) list of the keys corresponding to the
          objects stored in the HDFStore. These are ABSOLUTE path-names (e.g.
          have the leading '/'
          """
          return [n._v_pathname for n in self.groups()]
  
      def __iter__(self):
          return iter(self.keys())
  
      def items(self):
          """
          iterate on key->group
          """
          for g in self.groups():
              yield g._v_pathname, g
  
      iteritems = items
  
      def open(self, mode='a', **kwargs):
          """
          Open the file in the specified mode
  
          Parameters
          ----------
          mode : {'a', 'w', 'r', 'r+'}, default 'a'
              See HDFStore docstring or tables.open_file for info about modes
          """
          tables = _tables()
  
          if self._mode != mode:
  
              # if we are changing a write mode to read, ok
              if self._mode in ['a', 'w'] and mode in ['r', 'r+']:
                  pass
              elif mode in ['w']:
  
                  # this would truncate, raise here
                  if self.is_open:
                      raise PossibleDataLossError(
                          "Re-opening the file [{0}] with mode [{1}] "
                          "will delete the current file!"
                          .format(self._path, self._mode)
                      )
  
              self._mode = mode
  
          # close and reopen the handle
          if self.is_open:
              self.close()
  
          if self._complib is not None:
              if self._complevel is None:
                  self._complevel = 9
              self._filters = _tables().Filters(self._complevel,
                                                self._complib,
                                                fletcher32=self._fletcher32)
  
          try:
              self._handle = tables.open_file(self._path, self._mode, **kwargs)
          except (IOError) as e:  # pragma: no cover
              if 'can not be written' in str(e):
                  print('Opening %s in read-only mode' % self._path)
                  self._handle = tables.open_file(self._path, 'r', **kwargs)
              else:
                  raise
  
          except (ValueError) as e:
  
              # trap PyTables >= 3.1 FILE_OPEN_POLICY exception
              # to provide an updated message
              if 'FILE_OPEN_POLICY' in str(e):
                  e = ValueError(
                      "PyTables [{version}] no longer supports opening multiple "
                      "files\n"
                      "even in read-only mode on this HDF5 version "
                      "[{hdf_version}]. You can accept this\n"
                      "and not open the same file multiple times at once,\n"
                      "upgrade the HDF5 version, or downgrade to PyTables 3.0.0 "
                      "which allows\n"
                      "files to be opened multiple times at once\n"
                      .format(version=tables.__version__,
                              hdf_version=tables.get_hdf5_version()))
  
              raise e
  
          except (Exception) as e:
  
              # trying to read from a non-existant file causes an error which
              # is not part of IOError, make it one
              if self._mode == 'r' and 'Unable to open/create file' in str(e):
                  raise IOError(str(e))
              raise
  
      def close(self):
          """
          Close the PyTables file handle
          """
          if self._handle is not None:
              self._handle.close()
          self._handle = None
  
      @property
      def is_open(self):
          """
          return a boolean indicating whether the file is open
          """
          if self._handle is None:
              return False
          return bool(self._handle.isopen)
  
      def flush(self, fsync=False):
          """
          Force all buffered modifications to be written to disk.
  
          Parameters
          ----------
          fsync : bool (default False)
            call ``os.fsync()`` on the file handle to force writing to disk.
  
          Notes
          -----
          Without ``fsync=True``, flushing may not guarantee that the OS writes
          to disk. With fsync, the operation will block until the OS claims the
          file has been written; however, other caching layers may still
          interfere.
          """
          if self._handle is not None:
              self._handle.flush()
              if fsync:
                  try:
                      os.fsync(self._handle.fileno())
                  except:
                      pass
  
      def get(self, key):
          """
          Retrieve pandas object stored in file
  
          Parameters
          ----------
          key : object
  
          Returns
          -------
          obj : type of object stored in file
          """
          group = self.get_node(key)
          if group is None:
              raise KeyError('No object named %s in the file' % key)
          return self._read_group(group)
  
      def select(self, key, where=None, start=None, stop=None, columns=None,
                 iterator=False, chunksize=None, auto_close=False, **kwargs):
          """
          Retrieve pandas object stored in file, optionally based on where
          criteria
  
          Parameters
          ----------
          key : object
          where : list of Term (or convertable) objects, optional
          start : integer (defaults to None), row number to start selection
          stop  : integer (defaults to None), row number to stop selection
          columns : a list of columns that if not None, will limit the return
              columns
          iterator : boolean, return an iterator, default False
          chunksize : nrows to include in iteration, return an iterator
          auto_close : boolean, should automatically close the store when
              finished, default is False
  
          Returns
          -------
          The selected object
  
          """
          group = self.get_node(key)
          if group is None:
              raise KeyError('No object named %s in the file' % key)
  
          # create the storer and axes
          where = _ensure_term(where, scope_level=1)
          s = self._create_storer(group)
          s.infer_axes()
  
          # function to call on iteration
          def func(_start, _stop, _where):
              return s.read(start=_start, stop=_stop,
                            where=_where,
                            columns=columns, **kwargs)
  
          # create the iterator
          it = TableIterator(self, s, func, where=where, nrows=s.nrows,
                             start=start, stop=stop, iterator=iterator,
                             chunksize=chunksize, auto_close=auto_close)
  
          return it.get_result()
  
      def select_as_coordinates(
              self, key, where=None, start=None, stop=None, **kwargs):
          """
          return the selection as an Index
  
          Parameters
          ----------
          key : object
          where : list of Term (or convertable) objects, optional
          start : integer (defaults to None), row number to start selection
          stop  : integer (defaults to None), row number to stop selection
          """
          where = _ensure_term(where, scope_level=1)
          return self.get_storer(key).read_coordinates(where=where, start=start,
                                                       stop=stop, **kwargs)
  
      def select_column(self, key, column, **kwargs):
          """
          return a single column from the table. This is generally only useful to
          select an indexable
  
          Parameters
          ----------
          key : object
          column: the column of interest
  
          Exceptions
          ----------
          raises KeyError if the column is not found (or key is not a valid
              store)
          raises ValueError if the column can not be extracted individually (it
              is part of a data block)
  
          """
          return self.get_storer(key).read_column(column=column, **kwargs)
  
      def select_as_multiple(self, keys, where=None, selector=None, columns=None,
                             start=None, stop=None, iterator=False,
                             chunksize=None, auto_close=False, **kwargs):
          """ Retrieve pandas objects from multiple tables
  
          Parameters
          ----------
          keys : a list of the tables
          selector : the table to apply the where criteria (defaults to keys[0]
              if not supplied)
          columns : the columns I want back
          start : integer (defaults to None), row number to start selection
          stop  : integer (defaults to None), row number to stop selection
          iterator : boolean, return an iterator, default False
          chunksize : nrows to include in iteration, return an iterator
  
          Exceptions
          ----------
          raises KeyError if keys or selector is not found or keys is empty
          raises TypeError if keys is not a list or tuple
          raises ValueError if the tables are not ALL THE SAME DIMENSIONS
          """
  
          # default to single select
          where = _ensure_term(where, scope_level=1)
          if isinstance(keys, (list, tuple)) and len(keys) == 1:
              keys = keys[0]
          if isinstance(keys, string_types):
              return self.select(key=keys, where=where, columns=columns,
                                 start=start, stop=stop, iterator=iterator,
                                 chunksize=chunksize, **kwargs)
  
          if not isinstance(keys, (list, tuple)):
              raise TypeError("keys must be a list/tuple")
  
          if not len(keys):
              raise ValueError("keys must have a non-zero length")
  
          if selector is None:
              selector = keys[0]
  
          # collect the tables
          tbls = [self.get_storer(k) for k in keys]
          s = self.get_storer(selector)
  
          # validate rows
          nrows = None
          for t, k in itertools.chain([(s, selector)], zip(tbls, keys)):
              if t is None:
                  raise KeyError("Invalid table [%s]" % k)
              if not t.is_table:
                  raise TypeError(
                      "object [%s] is not a table, and cannot be used in all "
                      "select as multiple" % t.pathname
                  )
  
              if nrows is None:
                  nrows = t.nrows
              elif t.nrows != nrows:
                  raise ValueError(
                      "all tables must have exactly the same nrows!")
  
          # axis is the concentation axes
          axis = list(set([t.non_index_axes[0][0] for t in tbls]))[0]
  
          def func(_start, _stop, _where):
  
              # retrieve the objs, _where is always passed as a set of
              # coordinates here
              objs = [t.read(where=_where, columns=columns, **kwargs)
                      for t in tbls]
  
              # concat and return
              return concat(objs, axis=axis,
                            verify_integrity=False).consolidate()
  
          # create the iterator
          it = TableIterator(self, s, func, where=where, nrows=nrows,
                             start=start, stop=stop, iterator=iterator,
                             chunksize=chunksize, auto_close=auto_close)
  
          return it.get_result(coordinates=True)
  
      def put(self, key, value, format=None, append=False, **kwargs):
          """
          Store object in HDFStore
  
          Parameters
          ----------
          key      : object
          value    : {Series, DataFrame, Panel}
          format   : 'fixed(f)|table(t)', default is 'fixed'
              fixed(f) : Fixed format
                         Fast writing/reading. Not-appendable, nor searchable
              table(t) : Table format
                         Write as a PyTables Table structure which may perform
                         worse but allow more flexible operations like searching
                         / selecting subsets of the data
          append   : boolean, default False
              This will force Table format, append the input data to the
              existing.
          data_columns : list of columns to create as data columns, or True to
              use all columns. See
              `here <http://pandas.pydata.org/pandas-docs/stable/io.html#query-via-data-columns>`__ # noqa
          encoding : default None, provide an encoding for strings
          dropna   : boolean, default False, do not write an ALL nan row to
              the store settable by the option 'io.hdf.dropna_table'
          """
          if format is None:
              format = get_option("io.hdf.default_format") or 'fixed'
          kwargs = self._validate_format(format, kwargs)
          self._write_to_group(key, value, append=append, **kwargs)
  
      def remove(self, key, where=None, start=None, stop=None):
          """
          Remove pandas object partially by specifying the where condition
  
          Parameters
          ----------
          key : string
              Node to remove or delete rows from
          where : list of Term (or convertable) objects, optional
          start : integer (defaults to None), row number to start selection
          stop  : integer (defaults to None), row number to stop selection
  
          Returns
          -------
          number of rows removed (or None if not a Table)
  
          Exceptions
          ----------
          raises KeyError if key is not a valid store
  
          """
          where = _ensure_term(where, scope_level=1)
          try:
              s = self.get_storer(key)
          except:
  
              if where is not None:
                  raise ValueError(
                      "trying to remove a node with a non-None where clause!")
  
              # we are actually trying to remove a node (with children)
              s = self.get_node(key)
              if s is not None:
                  s._f_remove(recursive=True)
                  return None
  
          if s is None:
              raise KeyError('No object named %s in the file' % key)
  
          # remove the node
          if where is None and start is None and stop is None:
              s.group._f_remove(recursive=True)
  
          # delete from the table
          else:
              if not s.is_table:
                  raise ValueError(
                      'can only remove with where on objects written as tables')
              return s.delete(where=where, start=start, stop=stop)
  
      def append(self, key, value, format=None, append=True, columns=None,
                 dropna=None, **kwargs):
          """
          Append to Table in file. Node must already exist and be Table
          format.
  
          Parameters
          ----------
          key : object
          value : {Series, DataFrame, Panel, Panel4D}
          format: 'table' is the default
              table(t) : table format
                         Write as a PyTables Table structure which may perform
                         worse but allow more flexible operations like searching
                         / selecting subsets of the data
          append       : boolean, default True, append the input data to the
              existing
          data_columns :  list of columns, or True, default None
              List of columns to create as indexed data columns for on-disk
              queries, or True to use all columns. By default only the axes
              of the object are indexed. See `here
              <http://pandas.pydata.org/pandas-docs/stable/io.html#query-via-data-columns>`__.
          min_itemsize : dict of columns that specify minimum string sizes
          nan_rep      : string to use as string nan represenation
          chunksize    : size to chunk the writing
          expectedrows : expected TOTAL row size of this table
          encoding     : default None, provide an encoding for strings
          dropna       : boolean, default False, do not write an ALL nan row to
              the store settable by the option 'io.hdf.dropna_table'
  
          Notes
          -----
          Does *not* check if data being appended overlaps with existing
          data in the table, so be careful
          """
          if columns is not None:
              raise TypeError("columns is not a supported keyword in append, "
                              "try data_columns")
  
          if dropna is None:
              dropna = get_option("io.hdf.dropna_table")
          if format is None:
              format = get_option("io.hdf.default_format") or 'table'
          kwargs = self._validate_format(format, kwargs)
          self._write_to_group(key, value, append=append, dropna=dropna,
                               **kwargs)
  
      def append_to_multiple(self, d, value, selector, data_columns=None,
                             axes=None, dropna=False, **kwargs):
          """
          Append to multiple tables
  
          Parameters
          ----------
          d : a dict of table_name to table_columns, None is acceptable as the
              values of one node (this will get all the remaining columns)
          value : a pandas object
          selector : a string that designates the indexable table; all of its
              columns will be designed as data_columns, unless data_columns is
              passed, in which case these are used
          data_columns : list of columns to create as data columns, or True to
              use all columns
          dropna : if evaluates to True, drop rows from all tables if any single
                   row in each table has all NaN. Default False.
  
          Notes
          -----
          axes parameter is currently not accepted
  
          """
          if axes is not None:
              raise TypeError("axes is currently not accepted as a parameter to"
                              " append_to_multiple; you can create the "
                              "tables independently instead")
  
          if not isinstance(d, dict):
              raise ValueError(
                  "append_to_multiple must have a dictionary specified as the "
                  "way to split the value"
              )
  
          if selector not in d:
              raise ValueError(
                  "append_to_multiple requires a selector that is in passed dict"
              )
  
          # figure out the splitting axis (the non_index_axis)
          axis = list(set(range(value.ndim)) - set(_AXES_MAP[type(value)]))[0]
  
          # figure out how to split the value
          remain_key = None
          remain_values = []
          for k, v in d.items():
              if v is None:
                  if remain_key is not None:
                      raise ValueError(
                          "append_to_multiple can only have one value in d that "
                          "is None"
                      )
                  remain_key = k
              else:
                  remain_values.extend(v)
          if remain_key is not None:
              ordered = value.axes[axis]
              ordd = ordered.difference(Index(remain_values))
              ordd = sorted(ordered.get_indexer(ordd))
              d[remain_key] = ordered.take(ordd)
  
          # data_columns
          if data_columns is None:
              data_columns = d[selector]
  
          # ensure rows are synchronized across the tables
          if dropna:
              idxs = (value[cols].dropna(how='all').index for cols in d.values())
              valid_index = next(idxs)
              for index in idxs:
                  valid_index = valid_index.intersection(index)
              value = value.ix[valid_index]
  
          # append
          for k, v in d.items():
              dc = data_columns if k == selector else None
  
              # compute the val
              val = value.reindex_axis(v, axis=axis)
  
              self.append(k, val, data_columns=dc, **kwargs)
  
      def create_table_index(self, key, **kwargs):
          """ Create a pytables index on the table
          Parameters
          ----------
          key : object (the node to index)
  
          Exceptions
          ----------
          raises if the node is not a table
  
          """
  
          # version requirements
          _tables()
          s = self.get_storer(key)
          if s is None:
              return
  
          if not s.is_table:
              raise TypeError(
                  "cannot create table index on a Fixed format store")
          s.create_index(**kwargs)
  
      def groups(self):
          """return a list of all the top-level nodes (that are not themselves a
          pandas storage object)
          """
          _tables()
          self._check_if_open()
          return [
              g for g in self._handle.walk_nodes()
              if (getattr(g._v_attrs, 'pandas_type', None) or
                  getattr(g, 'table', None) or
                  (isinstance(g, _table_mod.table.Table) and
                   g._v_name != u('table')))
          ]
  
      def get_node(self, key):
          """ return the node with the key or None if it does not exist """
          self._check_if_open()
          try:
              if not key.startswith('/'):
                  key = '/' + key
              return self._handle.get_node(self.root, key)
          except:
              return None
  
      def get_storer(self, key):
          """ return the storer object for a key, raise if not in the file """
          group = self.get_node(key)
          if group is None:
              return None
          s = self._create_storer(group)
          s.infer_axes()
          return s
  
      def copy(self, file, mode='w', propindexes=True, keys=None, complib=None,
               complevel=None, fletcher32=False, overwrite=True):
          """ copy the existing store to a new file, upgrading in place
  
              Parameters
              ----------
              propindexes: restore indexes in copied file (defaults to True)
              keys       : list of keys to include in the copy (defaults to all)
              overwrite  : overwrite (remove and replace) existing nodes in the
                  new store (default is True)
              mode, complib, complevel, fletcher32 same as in HDFStore.__init__
  
              Returns
              -------
              open file handle of the new store
  
          """
          new_store = HDFStore(
              file,
              mode=mode,
              complib=complib,
              complevel=complevel,
              fletcher32=fletcher32)
          if keys is None:
              keys = list(self.keys())
          if not isinstance(keys, (tuple, list)):
              keys = [keys]
          for k in keys:
              s = self.get_storer(k)
              if s is not None:
  
                  if k in new_store:
                      if overwrite:
                          new_store.remove(k)
  
                  data = self.select(k)
                  if s.is_table:
  
                      index = False
                      if propindexes:
                          index = [a.name for a in s.axes if a.is_indexed]
                      new_store.append(
                          k, data, index=index,
                          data_columns=getattr(s, 'data_columns', None),
                          encoding=s.encoding
                      )
                  else:
                      new_store.put(k, data, encoding=s.encoding)
  
          return new_store
  
      # private methods ######
      def _check_if_open(self):
          if not self.is_open:
              raise ClosedFileError("{0} file is not open!".format(self._path))
  
      def _validate_format(self, format, kwargs):
          """ validate / deprecate formats; return the new kwargs """
          kwargs = kwargs.copy()
  
          # validate
          try:
              kwargs['format'] = _FORMAT_MAP[format.lower()]
          except:
              raise TypeError("invalid HDFStore format specified [{0}]"
                              .format(format))
  
          return kwargs
  
      def _create_storer(self, group, format=None, value=None, append=False,
                         **kwargs):
          """ return a suitable class to operate """
  
          def error(t):
              raise TypeError(
                  "cannot properly create the storer for: [%s] [group->%s,"
                  "value->%s,format->%s,append->%s,kwargs->%s]"
                  % (t, group, type(value), format, append, kwargs)
              )
  
          pt = _ensure_decoded(getattr(group._v_attrs, 'pandas_type', None))
          tt = _ensure_decoded(getattr(group._v_attrs, 'table_type', None))
  
          # infer the pt from the passed value
          if pt is None:
              if value is None:
  
                  _tables()
                  if (getattr(group, 'table', None) or
                          isinstance(group, _table_mod.table.Table)):
                      pt = u('frame_table')
                      tt = u('generic_table')
                  else:
                      raise TypeError(
                          "cannot create a storer if the object is not existing "
                          "nor a value are passed")
              else:
  
                  try:
                      pt = _TYPE_MAP[type(value)]
                  except:
                      error('_TYPE_MAP')
  
                  # we are actually a table
                  if format == 'table':
                      pt += u('_table')
  
          # a storer node
          if u('table') not in pt:
              try:
                  return globals()[_STORER_MAP[pt]](self, group, **kwargs)
              except:
                  error('_STORER_MAP')
  
          # existing node (and must be a table)
          if tt is None:
  
              # if we are a writer, determin the tt
              if value is not None:
  
                  if pt == u('series_table'):
                      index = getattr(value, 'index', None)
                      if index is not None:
                          if index.nlevels == 1:
                              tt = u('appendable_series')
                          elif index.nlevels > 1:
                              tt = u('appendable_multiseries')
                  elif pt == u('frame_table'):
                      index = getattr(value, 'index', None)
                      if index is not None:
                          if index.nlevels == 1:
                              tt = u('appendable_frame')
                          elif index.nlevels > 1:
                              tt = u('appendable_multiframe')
                  elif pt == u('wide_table'):
                      tt = u('appendable_panel')
                  elif pt == u('ndim_table'):
                      tt = u('appendable_ndim')
  
              else:
  
                  # distiguish between a frame/table
                  tt = u('legacy_panel')
                  try:
                      fields = group.table._v_attrs.fields
                      if len(fields) == 1 and fields[0] == u('value'):
                          tt = u('legacy_frame')
                  except:
                      pass
  
          try:
              return globals()[_TABLE_MAP[tt]](self, group, **kwargs)
          except:
              error('_TABLE_MAP')
  
      def _write_to_group(self, key, value, format, index=True, append=False,
                          complib=None, encoding=None, **kwargs):
          group = self.get_node(key)
  
          # remove the node if we are not appending
          if group is not None and not append:
              self._handle.remove_node(group, recursive=True)
              group = None
  
          # we don't want to store a table node at all if are object is 0-len
          # as there are not dtypes
          if getattr(value, 'empty', None) and (format == 'table' or append):
              return
  
          if group is None:
              paths = key.split('/')
  
              # recursively create the groups
              path = '/'
              for p in paths:
                  if not len(p):
                      continue
                  new_path = path
                  if not path.endswith('/'):
                      new_path += '/'
                  new_path += p
                  group = self.get_node(new_path)
                  if group is None:
                      group = self._handle.create_group(path, p)
                  path = new_path
  
          s = self._create_storer(group, format, value, append=append,
                                  encoding=encoding, **kwargs)
          if append:
              # raise if we are trying to append to a Fixed format,
              #       or a table that exists (and we are putting)
              if (not s.is_table or
                      (s.is_table and format == 'fixed' and s.is_exists)):
                  raise ValueError('Can only append to Tables')
              if not s.is_exists:
                  s.set_object_info()
          else:
              s.set_object_info()
  
          if not s.is_table and complib:
              raise ValueError(
                  'Compression not supported on Fixed format stores'
              )
  
          # write the object
          s.write(obj=value, append=append, complib=complib, **kwargs)
  
          if s.is_table and index:
              s.create_index(columns=index)
  
      def _read_group(self, group, **kwargs):
          s = self._create_storer(group)
          s.infer_axes()
          return s.read(**kwargs)
  
  
  def get_store(path, **kwargs):
      """ Backwards compatible alias for ``HDFStore``
      """
      return HDFStore(path, **kwargs)
  
  
  class TableIterator(object):
  
      """ define the iteration interface on a table
  
          Parameters
          ----------
  
          store : the reference store
          s     : the refered storer
          func  : the function to execute the query
          where : the where of the query
          nrows : the rows to iterate on
          start : the passed start value (default is None)
          stop  : the passed stop value (default is None)
          iterator : boolean, whether to use the default iterator
          chunksize : the passed chunking value (default is 50000)
          auto_close : boolean, automatically close the store at the end of
              iteration, default is False
          kwargs : the passed kwargs
          """
  
      def __init__(self, store, s, func, where, nrows, start=None, stop=None,
                   iterator=False, chunksize=None, auto_close=False):
          self.store = store
          self.s = s
          self.func = func
          self.where = where
  
          # set start/stop if they are not set if we are a table
          if self.s.is_table:
              if nrows is None:
                  nrows = 0
              if start is None:
                  start = 0
              if stop is None:
                  stop = nrows
              stop = min(nrows, stop)
  
          self.nrows = nrows
          self.start = start
          self.stop = stop
  
          self.coordinates = None
          if iterator or chunksize is not None:
              if chunksize is None:
                  chunksize = 100000
              self.chunksize = int(chunksize)
          else:
              self.chunksize = None
  
          self.auto_close = auto_close
  
      def __iter__(self):
  
          # iterate
          current = self.start
          while current < self.stop:
  
              stop = min(current + self.chunksize, self.stop)
              value = self.func(None, None, self.coordinates[current:stop])
              current = stop
              if value is None or not len(value):
                  continue
  
              yield value
  
          self.close()
  
      def close(self):
          if self.auto_close:
              self.store.close()
  
      def get_result(self, coordinates=False):
  
          #  return the actual iterator
          if self.chunksize is not None:
              if not self.s.is_table:
                  raise TypeError(
                      "can only use an iterator or chunksize on a table")
  
              self.coordinates = self.s.read_coordinates(where=self.where)
  
              return self
  
          # if specified read via coordinates (necessary for multiple selections
          if coordinates:
              where = self.s.read_coordinates(where=self.where)
          else:
              where = self.where
  
          # directly return the result
          results = self.func(self.start, self.stop, where)
          self.close()
          return results
  
  
  class IndexCol(StringMixin):
  
      """ an index column description class
  
          Parameters
          ----------
  
          axis   : axis which I reference
          values : the ndarray like converted values
          kind   : a string description of this type
          typ    : the pytables type
          pos    : the position in the pytables
  
          """
      is_an_indexable = True
      is_data_indexable = True
      _info_fields = ['freq', 'tz', 'index_name']
  
      def __init__(self, values=None, kind=None, typ=None, cname=None,
                   itemsize=None, name=None, axis=None, kind_attr=None,
                   pos=None, freq=None, tz=None, index_name=None, **kwargs):
          self.values = values
          self.kind = kind
          self.typ = typ
          self.itemsize = itemsize
          self.name = name
          self.cname = cname
          self.kind_attr = kind_attr
          self.axis = axis
          self.pos = pos
          self.freq = freq
          self.tz = tz
          self.index_name = index_name
          self.table = None
          self.meta = None
          self.metadata = None
  
          if name is not None:
              self.set_name(name, kind_attr)
          if pos is not None:
              self.set_pos(pos)
  
      def set_name(self, name, kind_attr=None):
          """ set the name of this indexer """
          self.name = name
          self.kind_attr = kind_attr or "%s_kind" % name
          if self.cname is None:
              self.cname = name
  
          return self
  
      def set_axis(self, axis):
          """ set the axis over which I index """
          self.axis = axis
  
          return self
  
      def set_pos(self, pos):
          """ set the position of this column in the Table """
          self.pos = pos
          if pos is not None and self.typ is not None:
              self.typ._v_pos = pos
          return self
  
      def set_table(self, table):
          self.table = table
          return self
  
      def __unicode__(self):
          temp = tuple(
              map(pprint_thing,
                      (self.name,
                       self.cname,
                       self.axis,
                       self.pos,
                       self.kind)))
          return "name->%s,cname->%s,axis->%s,pos->%s,kind->%s" % temp
  
      def __eq__(self, other):
          """ compare 2 col items """
          return all([getattr(self, a, None) == getattr(other, a, None)
                      for a in ['name', 'cname', 'axis', 'pos']])
  
      def __ne__(self, other):
          return not self.__eq__(other)
  
      @property
      def is_indexed(self):
          """ return whether I am an indexed column """
          try:
              return getattr(self.table.cols, self.cname).is_indexed
          except:
              False
  
      def copy(self):
          new_self = copy.copy(self)
          return new_self
  
      def infer(self, handler):
          """infer this column from the table: create and return a new object"""
          table = handler.table
          new_self = self.copy()
          new_self.set_table(table)
          new_self.get_attr()
          new_self.read_metadata(handler)
          return new_self
  
      def convert(self, values, nan_rep, encoding):
          """ set the values from this selection: take = take ownership """
  
          # values is a recarray
          if values.dtype.fields is not None:
              values = values[self.cname]
  
          values = _maybe_convert(values, self.kind, encoding)
  
          kwargs = dict()
          if self.freq is not None:
              kwargs['freq'] = _ensure_decoded(self.freq)
          if self.index_name is not None:
              kwargs['name'] = _ensure_decoded(self.index_name)
          try:
              self.values = Index(values, **kwargs)
          except:
  
              # if the output freq is different that what we recorded,
              # it should be None (see also 'doc example part 2')
              if 'freq' in kwargs:
                  kwargs['freq'] = None
              self.values = Index(values, **kwargs)
  
          self.values = _set_tz(self.values, self.tz)
  
          return self
  
      def take_data(self):
          """ return the values & release the memory """
          self.values, values = None, self.values
          return values
  
      @property
      def attrs(self):
          return self.table._v_attrs
  
      @property
      def description(self):
          return self.table.description
  
      @property
      def col(self):
          """ return my current col description """
          return getattr(self.description, self.cname, None)
  
      @property
      def cvalues(self):
          """ return my cython values """
          return self.values
  
      def __iter__(self):
          return iter(self.values)
  
      def maybe_set_size(self, min_itemsize=None, **kwargs):
          """ maybe set a string col itemsize:
                 min_itemsize can be an interger or a dict with this columns name
                 with an integer size """
          if _ensure_decoded(self.kind) == u('string'):
  
              if isinstance(min_itemsize, dict):
                  min_itemsize = min_itemsize.get(self.name)
  
              if min_itemsize is not None and self.typ.itemsize < min_itemsize:
                  self.typ = _tables(
                  ).StringCol(itemsize=min_itemsize, pos=self.pos)
  
      def validate(self, handler, append, **kwargs):
          self.validate_names()
  
      def validate_names(self):
          pass
  
      def validate_and_set(self, handler, append, **kwargs):
          self.set_table(handler.table)
          self.validate_col()
          self.validate_attr(append)
          self.validate_metadata(handler)
          self.write_metadata(handler)
          self.set_attr()
  
      def validate_col(self, itemsize=None):
          """ validate this column: return the compared against itemsize """
  
          # validate this column for string truncation (or reset to the max size)
          if _ensure_decoded(self.kind) == u('string'):
              c = self.col
              if c is not None:
                  if itemsize is None:
                      itemsize = self.itemsize
                  if c.itemsize < itemsize:
                      raise ValueError(
                          "Trying to store a string with len [%s] in [%s] "
                          "column but\nthis column has a limit of [%s]!\n"
                          "Consider using min_itemsize to preset the sizes on "
                          "these columns" % (itemsize, self.cname, c.itemsize))
                  return c.itemsize
  
          return None
  
      def validate_attr(self, append):
          # check for backwards incompatibility
          if append:
              existing_kind = getattr(self.attrs, self.kind_attr, None)
              if existing_kind is not None and existing_kind != self.kind:
                  raise TypeError("incompatible kind in col [%s - %s]" %
                                  (existing_kind, self.kind))
  
      def update_info(self, info):
          """ set/update the info for this indexable with the key/value
              if there is a conflict raise/warn as needed """
  
          for key in self._info_fields:
  
              value = getattr(self, key, None)
              idx = _get_info(info, self.name)
  
              existing_value = idx.get(key)
              if key in idx and value is not None and existing_value != value:
  
                  # frequency/name just warn
                  if key in ['freq', 'index_name']:
                      ws = attribute_conflict_doc % (key, existing_value, value)
                      warnings.warn(ws, AttributeConflictWarning, stacklevel=6)
  
                      # reset
                      idx[key] = None
                      setattr(self, key, None)
  
                  else:
                      raise ValueError(
                          "invalid info for [%s] for [%s], existing_value [%s] "
                          "conflicts with new value [%s]"
                          % (self.name, key, existing_value, value))
              else:
                  if value is not None or existing_value is not None:
                      idx[key] = value
  
          return self
  
      def set_info(self, info):
          """ set my state from the passed info """
          idx = info.get(self.name)
          if idx is not None:
              self.__dict__.update(idx)
  
      def get_attr(self):
          """ set the kind for this colummn """
          self.kind = getattr(self.attrs, self.kind_attr, None)
  
      def set_attr(self):
          """ set the kind for this colummn """
          setattr(self.attrs, self.kind_attr, self.kind)
  
      def read_metadata(self, handler):
          """ retrieve the metadata for this columns """
          self.metadata = handler.read_metadata(self.cname)
  
      def validate_metadata(self, handler):
          """ validate that kind=category does not change the categories """
          if self.meta == 'category':
              new_metadata = self.metadata
              cur_metadata = handler.read_metadata(self.cname)
              if new_metadata is not None and cur_metadata is not None \
                      and not array_equivalent(new_metadata, cur_metadata):
                  raise ValueError("cannot append a categorical with "
                                   "different categories to the existing")
  
      def write_metadata(self, handler):
          """ set the meta data """
          if self.metadata is not None:
              handler.write_metadata(self.cname, self.metadata)
  
  
  class GenericIndexCol(IndexCol):
  
      """ an index which is not represented in the data of the table """
  
      @property
      def is_indexed(self):
          return False
  
      def convert(self, values, nan_rep, encoding):
          """ set the values from this selection: take = take ownership """
  
          self.values = Int64Index(np.arange(self.table.nrows))
          return self
  
      def get_attr(self):
          pass
  
      def set_attr(self):
          pass
  
  
  class DataCol(IndexCol):
  
      """ a data holding column, by definition this is not indexable
  
          Parameters
          ----------
  
          data   : the actual data
          cname  : the column name in the table to hold the data (typically
                   values)
          meta   : a string description of the metadata
          metadata : the actual metadata
          """
      is_an_indexable = False
      is_data_indexable = False
      _info_fields = ['tz', 'ordered']
  
      @classmethod
      def create_for_block(
              cls, i=None, name=None, cname=None, version=None, **kwargs):
          """ return a new datacol with the block i """
  
          if cname is None:
              cname = name or 'values_block_%d' % i
          if name is None:
              name = cname
  
          # prior to 0.10.1, we named values blocks like: values_block_0 an the
          # name values_0
          try:
              if version[0] == 0 and version[1] <= 10 and version[2] == 0:
                  m = re.search("values_block_(\d+)", name)
                  if m:
                      name = "values_%s" % m.groups()[0]
          except:
              pass
  
          return cls(name=name, cname=cname, **kwargs)
  
      def __init__(self, values=None, kind=None, typ=None,
                   cname=None, data=None, meta=None, metadata=None,
                   block=None, **kwargs):
          super(DataCol, self).__init__(values=values, kind=kind, typ=typ,
                                        cname=cname, **kwargs)
          self.dtype = None
          self.dtype_attr = u("%s_dtype" % self.name)
          self.meta = meta
          self.meta_attr = u("%s_meta" % self.name)
          self.set_data(data)
          self.set_metadata(metadata)
  
      def __unicode__(self):
          temp = tuple(
              map(pprint_thing,
                      (self.name,
                       self.cname,
                       self.dtype,
                       self.kind,
                       self.shape)))
          return "name->%s,cname->%s,dtype->%s,kind->%s,shape->%s" % temp
  
      def __eq__(self, other):
          """ compare 2 col items """
          return all([getattr(self, a, None) == getattr(other, a, None)
                      for a in ['name', 'cname', 'dtype', 'pos']])
  
      def set_data(self, data, dtype=None):
          self.data = data
          if data is not None:
              if dtype is not None:
                  self.dtype = dtype
                  self.set_kind()
              elif self.dtype is None:
                  self.dtype = data.dtype.name
                  self.set_kind()
  
      def take_data(self):
          """ return the data & release the memory """
          self.data, data = None, self.data
          return data
  
      def set_metadata(self, metadata):
          """ record the metadata """
          if metadata is not None:
              metadata = np.array(metadata, copy=False).ravel()
          self.metadata = metadata
  
      def set_kind(self):
          # set my kind if we can
  
          if self.dtype is not None:
              dtype = _ensure_decoded(self.dtype)
  
              if dtype.startswith(u('string')) or dtype.startswith(u('bytes')):
                  self.kind = 'string'
              elif dtype.startswith(u('float')):
                  self.kind = 'float'
              elif dtype.startswith(u('complex')):
                  self.kind = 'complex'
              elif dtype.startswith(u('int')) or dtype.startswith(u('uint')):
                  self.kind = 'integer'
              elif dtype.startswith(u('date')):
                  self.kind = 'datetime'
              elif dtype.startswith(u('timedelta')):
                  self.kind = 'timedelta'
              elif dtype.startswith(u('bool')):
                  self.kind = 'bool'
              else:
                  raise AssertionError(
                      "cannot interpret dtype of [%s] in [%s]" % (dtype, self))
  
              # set my typ if we need
              if self.typ is None:
                  self.typ = getattr(self.description, self.cname, None)
  
      def set_atom(self, block, block_items, existing_col, min_itemsize,
                   nan_rep, info, encoding=None, **kwargs):
          """ create and setup my atom from the block b """
  
          self.values = list(block_items)
  
          # short-cut certain block types
          if block.is_categorical:
              return self.set_atom_categorical(block, items=block_items,
                                               info=info)
          elif block.is_datetimetz:
              return self.set_atom_datetime64tz(block, info=info)
          elif block.is_datetime:
              return self.set_atom_datetime64(block)
          elif block.is_timedelta:
              return self.set_atom_timedelta64(block)
          elif block.is_complex:
              return self.set_atom_complex(block)
  
          dtype = block.dtype.name
          inferred_type = lib.infer_dtype(block.values)
  
          if inferred_type == 'date':
              raise TypeError(
                  "[date] is not implemented as a table column")
          elif inferred_type == 'datetime':
              # after 8260
              # this only would be hit for a mutli-timezone dtype
              # which is an error
  
              raise TypeError(
                  "too many timezones in this block, create separate "
                  "data columns"
              )
          elif inferred_type == 'unicode':
              raise TypeError(
                  "[unicode] is not implemented as a table column")
  
          # this is basically a catchall; if say a datetime64 has nans then will
          # end up here ###
          elif inferred_type == 'string' or dtype == 'object':
              self.set_atom_string(
                  block, block_items,
                  existing_col,
                  min_itemsize,
                  nan_rep,
                  encoding)
  
          # set as a data block
          else:
              self.set_atom_data(block)
  
      def get_atom_string(self, block, itemsize):
          return _tables().StringCol(itemsize=itemsize, shape=block.shape[0])
  
      def set_atom_string(self, block, block_items, existing_col, min_itemsize,
                          nan_rep, encoding):
          # fill nan items with myself, don't disturb the blocks by
          # trying to downcast
          block = block.fillna(nan_rep, downcast=False)
          if isinstance(block, list):
              block = block[0]
          data = block.values
  
          # see if we have a valid string type
          inferred_type = lib.infer_dtype(data.ravel())
          if inferred_type != 'string':
  
              # we cannot serialize this data, so report an exception on a column
              # by column basis
              for i, item in enumerate(block_items):
  
                  col = block.iget(i)
                  inferred_type = lib.infer_dtype(col.ravel())
                  if inferred_type != 'string':
                      raise TypeError(
                          "Cannot serialize the column [%s] because\n"
                          "its data contents are [%s] object dtype"
                          % (item, inferred_type)
                      )
  
          # itemsize is the maximum length of a string (along any dimension)
          data_converted = _convert_string_array(data, encoding)
          itemsize = data_converted.itemsize
  
          # specified min_itemsize?
          if isinstance(min_itemsize, dict):
              min_itemsize = int(min_itemsize.get(
                  self.name) or min_itemsize.get('values') or 0)
          itemsize = max(min_itemsize or 0, itemsize)
  
          # check for column in the values conflicts
          if existing_col is not None:
              eci = existing_col.validate_col(itemsize)
              if eci > itemsize:
                  itemsize = eci
  
          self.itemsize = itemsize
          self.kind = 'string'
          self.typ = self.get_atom_string(block, itemsize)
          self.set_data(data_converted.astype('|S%d' % itemsize, copy=False))
  
      def get_atom_coltype(self, kind=None):
          """ return the PyTables column class for this column """
          if kind is None:
              kind = self.kind
          if self.kind.startswith('uint'):
              col_name = "UInt%sCol" % kind[4:]
          else:
              col_name = "%sCol" % kind.capitalize()
  
          return getattr(_tables(), col_name)
  
      def get_atom_data(self, block, kind=None):
          return self.get_atom_coltype(kind=kind)(shape=block.shape[0])
  
      def set_atom_complex(self, block):
          self.kind = block.dtype.name
          itemsize = int(self.kind.split('complex')[-1]) // 8
          self.typ = _tables().ComplexCol(
              itemsize=itemsize, shape=block.shape[0])
          self.set_data(block.values.astype(self.typ.type, copy=False))
  
      def set_atom_data(self, block):
          self.kind = block.dtype.name
          self.typ = self.get_atom_data(block)
          self.set_data(block.values.astype(self.typ.type, copy=False))
  
      def set_atom_categorical(self, block, items, info=None, values=None):
          # currently only supports a 1-D categorical
          # in a 1-D block
  
          values = block.values
          codes = values.codes
          self.kind = 'integer'
          self.dtype = codes.dtype.name
          if values.ndim > 1:
              raise NotImplementedError("only support 1-d categoricals")
          if len(items) > 1:
              raise NotImplementedError("only support single block categoricals")
  
          # write the codes; must be in a block shape
          self.ordered = values.ordered
          self.typ = self.get_atom_data(block, kind=codes.dtype.name)
          self.set_data(_block_shape(codes))
  
          # write the categories
          self.meta = 'category'
          self.set_metadata(block.values.categories)
  
          # update the info
          self.update_info(info)
  
      def get_atom_datetime64(self, block):
          return _tables().Int64Col(shape=block.shape[0])
  
      def set_atom_datetime64(self, block, values=None):
          self.kind = 'datetime64'
          self.typ = self.get_atom_datetime64(block)
          if values is None:
              values = block.values.view('i8')
          self.set_data(values, 'datetime64')
  
      def set_atom_datetime64tz(self, block, info, values=None):
  
          if values is None:
              values = block.values
  
          # convert this column to i8 in UTC, and save the tz
          values = values.asi8.reshape(block.shape)
  
          # store a converted timezone
          self.tz = _get_tz(block.values.tz)
          self.update_info(info)
  
          self.kind = 'datetime64'
          self.typ = self.get_atom_datetime64(block)
          self.set_data(values, 'datetime64')
  
      def get_atom_timedelta64(self, block):
          return _tables().Int64Col(shape=block.shape[0])
  
      def set_atom_timedelta64(self, block, values=None):
          self.kind = 'timedelta64'
          self.typ = self.get_atom_timedelta64(block)
          if values is None:
              values = block.values.view('i8')
          self.set_data(values, 'timedelta64')
  
      @property
      def shape(self):
          return getattr(self.data, 'shape', None)
  
      @property
      def cvalues(self):
          """ return my cython values """
          return self.data
  
      def validate_attr(self, append):
          """validate that we have the same order as the existing & same dtype"""
          if append:
              existing_fields = getattr(self.attrs, self.kind_attr, None)
              if (existing_fields is not None and
                      existing_fields != list(self.values)):
                  raise ValueError("appended items do not match existing items"
                                   " in table!")
  
              existing_dtype = getattr(self.attrs, self.dtype_attr, None)
              if (existing_dtype is not None and
                      existing_dtype != self.dtype):
                  raise ValueError("appended items dtype do not match existing "
                                   "items dtype in table!")
  
      def convert(self, values, nan_rep, encoding):
          """set the data from this selection (and convert to the correct dtype
          if we can)
          """
  
          # values is a recarray
          if values.dtype.fields is not None:
              values = values[self.cname]
  
          self.set_data(values)
  
          # use the meta if needed
          meta = _ensure_decoded(self.meta)
  
          # convert to the correct dtype
          if self.dtype is not None:
              dtype = _ensure_decoded(self.dtype)
  
              # reverse converts
              if dtype == u('datetime64'):
  
                  # recreate with tz if indicated
                  self.data = _set_tz(self.data, self.tz, coerce=True)
  
              elif dtype == u('timedelta64'):
                  self.data = np.asarray(self.data, dtype='m8[ns]')
              elif dtype == u('date'):
                  try:
                      self.data = np.asarray(
                          [date.fromordinal(v) for v in self.data], dtype=object)
                  except ValueError:
                      self.data = np.asarray(
                          [date.fromtimestamp(v) for v in self.data],
                          dtype=object)
              elif dtype == u('datetime'):
                  self.data = np.asarray(
                      [datetime.fromtimestamp(v) for v in self.data],
                      dtype=object)
  
              elif meta == u('category'):
  
                  # we have a categorical
                  categories = self.metadata
                  self.data = Categorical.from_codes(self.data.ravel(),
                                                     categories=categories,
                                                     ordered=self.ordered)
  
              else:
  
                  try:
                      self.data = self.data.astype(dtype, copy=False)
                  except:
                      self.data = self.data.astype('O', copy=False)
  
          # convert nans / decode
          if _ensure_decoded(self.kind) == u('string'):
              self.data = _unconvert_string_array(
                  self.data, nan_rep=nan_rep, encoding=encoding)
  
          return self
  
      def get_attr(self):
          """ get the data for this colummn """
          self.values = getattr(self.attrs, self.kind_attr, None)
          self.dtype = getattr(self.attrs, self.dtype_attr, None)
          self.meta = getattr(self.attrs, self.meta_attr, None)
          self.set_kind()
  
      def set_attr(self):
          """ set the data for this colummn """
          setattr(self.attrs, self.kind_attr, self.values)
          setattr(self.attrs, self.meta_attr, self.meta)
          if self.dtype is not None:
              setattr(self.attrs, self.dtype_attr, self.dtype)
  
  
  class DataIndexableCol(DataCol):
  
      """ represent a data column that can be indexed """
      is_data_indexable = True
  
      def validate_names(self):
          if not Index(self.values).is_object():
              raise ValueError("cannot have non-object label DataIndexableCol")
  
      def get_atom_string(self, block, itemsize):
          return _tables().StringCol(itemsize=itemsize)
  
      def get_atom_data(self, block, kind=None):
          return self.get_atom_coltype(kind=kind)()
  
      def get_atom_datetime64(self, block):
          return _tables().Int64Col()
  
      def get_atom_timedelta64(self, block):
          return _tables().Int64Col()
  
  
  class GenericDataIndexableCol(DataIndexableCol):
  
      """ represent a generic pytables data column """
  
      def get_attr(self):
          pass
  
  
  class Fixed(StringMixin):
  
      """ represent an object in my store
          facilitate read/write of various types of objects
          this is an abstract base class
  
          Parameters
          ----------
  
          parent : my parent HDFStore
          group  : the group node where the table resides
          """
      pandas_kind = None
      obj_type = None
      ndim = None
      is_table = False
  
      def __init__(self, parent, group, encoding=None, **kwargs):
          self.parent = parent
          self.group = group
          self.encoding = _ensure_encoding(encoding)
          self.set_version()
  
      @property
      def is_old_version(self):
          return (self.version[0] <= 0 and self.version[1] <= 10 and
                  self.version[2] < 1)
  
      def set_version(self):
          """ compute and set our version """
          version = _ensure_decoded(
              getattr(self.group._v_attrs, 'pandas_version', None))
          try:
              self.version = tuple([int(x) for x in version.split('.')])
              if len(self.version) == 2:
                  self.version = self.version + (0,)
          except:
              self.version = (0, 0, 0)
  
      @property
      def pandas_type(self):
          return _ensure_decoded(getattr(self.group._v_attrs,
                                         'pandas_type', None))
  
      @property
      def format_type(self):
          return 'fixed'
  
      def __unicode__(self):
          """ return a pretty representation of myself """
          self.infer_axes()
          s = self.shape
          if s is not None:
              if isinstance(s, (list, tuple)):
                  s = "[%s]" % ','.join([pprint_thing(x) for x in s])
              return "%-12.12s (shape->%s)" % (self.pandas_type, s)
          return self.pandas_type
  
      def set_object_info(self):
          """ set my pandas type & version """
          self.attrs.pandas_type = str(self.pandas_kind)
          self.attrs.pandas_version = str(_version)
          self.set_version()
  
      def copy(self):
          new_self = copy.copy(self)
          return new_self
  
      @property
      def storage_obj_type(self):
          return self.obj_type
  
      @property
      def shape(self):
          return self.nrows
  
      @property
      def pathname(self):
          return self.group._v_pathname
  
      @property
      def _handle(self):
          return self.parent._handle
  
      @property
      def _filters(self):
          return self.parent._filters
  
      @property
      def _complevel(self):
          return self.parent._complevel
  
      @property
      def _fletcher32(self):
          return self.parent._fletcher32
  
      @property
      def _complib(self):
          return self.parent._complib
  
      @property
      def attrs(self):
          return self.group._v_attrs
  
      def set_attrs(self):
          """ set our object attributes """
          pass
  
      def get_attrs(self):
          """ get our object attributes """
          pass
  
      @property
      def storable(self):
          """ return my storable """
          return self.group
  
      @property
      def is_exists(self):
          return False
  
      @property
      def nrows(self):
          return getattr(self.storable, 'nrows', None)
  
      def validate(self, other):
          """ validate against an existing storable """
          if other is None:
              return
          return True
  
      def validate_version(self, where=None):
          """ are we trying to operate on an old version? """
          return True
  
      def infer_axes(self):
          """ infer the axes of my storer
                return a boolean indicating if we have a valid storer or not """
  
          s = self.storable
          if s is None:
              return False
          self.get_attrs()
          return True
  
      def read(self, **kwargs):
          raise NotImplementedError(
              "cannot read on an abstract storer: subclasses should implement")
  
      def write(self, **kwargs):
          raise NotImplementedError(
              "cannot write on an abstract storer: sublcasses should implement")
  
      def delete(self, where=None, start=None, stop=None, **kwargs):
          """
          support fully deleting the node in its entirety (only) - where
          specification must be None
          """
          if where is None and start is None and stop is None:
              self._handle.remove_node(self.group, recursive=True)
              return None
  
          raise TypeError("cannot delete on an abstract storer")
  
  
  class GenericFixed(Fixed):
  
      """ a generified fixed version """
      _index_type_map = {DatetimeIndex: 'datetime', PeriodIndex: 'period'}
      _reverse_index_map = dict([(v, k)
                                 for k, v in compat.iteritems(_index_type_map)])
      attributes = []
  
      # indexer helpders
      def _class_to_alias(self, cls):
          return self._index_type_map.get(cls, '')
  
      def _alias_to_class(self, alias):
          if isinstance(alias, type):  # pragma: no cover
              # compat: for a short period of time master stored types
              return alias
          return self._reverse_index_map.get(alias, Index)
  
      def _get_index_factory(self, klass):
          if klass == DatetimeIndex:
              def f(values, freq=None, tz=None):
                  return DatetimeIndex._simple_new(values, None, freq=freq,
                                                   tz=tz)
              return f
          elif klass == PeriodIndex:
              def f(values, freq=None, tz=None):
                  return PeriodIndex._simple_new(values, None, freq=freq)
              return f
  
          return klass
  
      def validate_read(self, kwargs):
          """
          remove table keywords from kwargs and return
          raise if any keywords are passed which are not-None
          """
          kwargs = copy.copy(kwargs)
  
          columns = kwargs.pop('columns', None)
          if columns is not None:
              raise TypeError("cannot pass a column specification when reading "
                              "a Fixed format store. this store must be "
                              "selected in its entirety")
          where = kwargs.pop('where', None)
          if where is not None:
              raise TypeError("cannot pass a where specification when reading "
                              "from a Fixed format store. this store must be "
                              "selected in its entirety")
          return kwargs
  
      @property
      def is_exists(self):
          return True
  
      def set_attrs(self):
          """ set our object attributes """
          self.attrs.encoding = self.encoding
  
      def get_attrs(self):
          """ retrieve our attributes """
          self.encoding = _ensure_encoding(getattr(self.attrs, 'encoding', None))
          for n in self.attributes:
              setattr(self, n, _ensure_decoded(getattr(self.attrs, n, None)))
  
      def write(self, obj, **kwargs):
          self.set_attrs()
  
      def read_array(self, key, start=None, stop=None):
          """ read an array for the specified node (off of group """
          import tables
          node = getattr(self.group, key)
          data = node[start:stop]
          attrs = node._v_attrs
  
          transposed = getattr(attrs, 'transposed', False)
  
          if isinstance(node, tables.VLArray):
              ret = data[0]
          else:
              dtype = getattr(attrs, 'value_type', None)
              shape = getattr(attrs, 'shape', None)
  
              if shape is not None:
                  # length 0 axis
                  ret = np.empty(shape, dtype=dtype)
              else:
                  ret = data
  
              if dtype == u('datetime64'):
  
                  # reconstruct a timezone if indicated
                  ret = _set_tz(ret, getattr(attrs, 'tz', None), coerce=True)
  
              elif dtype == u('timedelta64'):
                  ret = np.asarray(ret, dtype='m8[ns]')
  
          if transposed:
              return ret.T
          else:
              return ret
  
      def read_index(self, key, **kwargs):
          variety = _ensure_decoded(getattr(self.attrs, '%s_variety' % key))
  
          if variety == u('multi'):
              return self.read_multi_index(key, **kwargs)
          elif variety == u('block'):
              return self.read_block_index(key, **kwargs)
          elif variety == u('sparseint'):
              return self.read_sparse_intindex(key, **kwargs)
          elif variety == u('regular'):
              _, index = self.read_index_node(getattr(self.group, key), **kwargs)
              return index
          else:  # pragma: no cover
              raise TypeError('unrecognized index variety: %s' % variety)
  
      def write_index(self, key, index):
          if isinstance(index, MultiIndex):
              setattr(self.attrs, '%s_variety' % key, 'multi')
              self.write_multi_index(key, index)
          elif isinstance(index, BlockIndex):
              setattr(self.attrs, '%s_variety' % key, 'block')
              self.write_block_index(key, index)
          elif isinstance(index, IntIndex):
              setattr(self.attrs, '%s_variety' % key, 'sparseint')
              self.write_sparse_intindex(key, index)
          else:
              setattr(self.attrs, '%s_variety' % key, 'regular')
              converted = _convert_index(index, self.encoding,
                                         self.format_type).set_name('index')
  
              self.write_array(key, converted.values)
  
              node = getattr(self.group, key)
              node._v_attrs.kind = converted.kind
              node._v_attrs.name = index.name
  
              if isinstance(index, (DatetimeIndex, PeriodIndex)):
                  node._v_attrs.index_class = self._class_to_alias(type(index))
  
              if hasattr(index, 'freq'):
                  node._v_attrs.freq = index.freq
  
              if hasattr(index, 'tz') and index.tz is not None:
                  node._v_attrs.tz = _get_tz(index.tz)
  
      def write_block_index(self, key, index):
          self.write_array('%s_blocs' % key, index.blocs)
          self.write_array('%s_blengths' % key, index.blengths)
          setattr(self.attrs, '%s_length' % key, index.length)
  
      def read_block_index(self, key, **kwargs):
          length = getattr(self.attrs, '%s_length' % key)
          blocs = self.read_array('%s_blocs' % key, **kwargs)
          blengths = self.read_array('%s_blengths' % key, **kwargs)
          return BlockIndex(length, blocs, blengths)
  
      def write_sparse_intindex(self, key, index):
          self.write_array('%s_indices' % key, index.indices)
          setattr(self.attrs, '%s_length' % key, index.length)
  
      def read_sparse_intindex(self, key, **kwargs):
          length = getattr(self.attrs, '%s_length' % key)
          indices = self.read_array('%s_indices' % key, **kwargs)
          return IntIndex(length, indices)
  
      def write_multi_index(self, key, index):
          setattr(self.attrs, '%s_nlevels' % key, index.nlevels)
  
          for i, (lev, lab, name) in enumerate(zip(index.levels,
                                                   index.labels,
                                                   index.names)):
              # write the level
              level_key = '%s_level%d' % (key, i)
              conv_level = _convert_index(lev, self.encoding,
                                          self.format_type).set_name(level_key)
              self.write_array(level_key, conv_level.values)
              node = getattr(self.group, level_key)
              node._v_attrs.kind = conv_level.kind
              node._v_attrs.name = name
  
              # write the name
              setattr(node._v_attrs, '%s_name%d' % (key, i), name)
  
              # write the labels
              label_key = '%s_label%d' % (key, i)
              self.write_array(label_key, lab)
  
      def read_multi_index(self, key, **kwargs):
          nlevels = getattr(self.attrs, '%s_nlevels' % key)
  
          levels = []
          labels = []
          names = []
          for i in range(nlevels):
              level_key = '%s_level%d' % (key, i)
              name, lev = self.read_index_node(getattr(self.group, level_key),
                                               **kwargs)
              levels.append(lev)
              names.append(name)
  
              label_key = '%s_label%d' % (key, i)
              lab = self.read_array(label_key, **kwargs)
              labels.append(lab)
  
          return MultiIndex(levels=levels, labels=labels, names=names,
                            verify_integrity=True)
  
      def read_index_node(self, node, start=None, stop=None):
          data = node[start:stop]
          # If the index was an empty array write_array_empty() will
          # have written a sentinel. Here we relace it with the original.
          if ('shape' in node._v_attrs and
                  self._is_empty_array(getattr(node._v_attrs, 'shape'))):
              data = np.empty(getattr(node._v_attrs, 'shape'),
                              dtype=getattr(node._v_attrs, 'value_type'))
          kind = _ensure_decoded(node._v_attrs.kind)
          name = None
  
          if 'name' in node._v_attrs:
              name = node._v_attrs.name
  
          index_class = self._alias_to_class(getattr(node._v_attrs,
                                                     'index_class', ''))
          factory = self._get_index_factory(index_class)
  
          kwargs = {}
          if u('freq') in node._v_attrs:
              kwargs['freq'] = node._v_attrs['freq']
  
          if u('tz') in node._v_attrs:
              kwargs['tz'] = node._v_attrs['tz']
  
          if kind in (u('date'), u('datetime')):
              index = factory(_unconvert_index(data, kind,
                                               encoding=self.encoding),
                              dtype=object, **kwargs)
          else:
              index = factory(_unconvert_index(data, kind,
                                               encoding=self.encoding), **kwargs)
  
          index.name = name
  
          return name, index
  
      def write_array_empty(self, key, value):
          """ write a 0-len array """
  
          # ugly hack for length 0 axes
          arr = np.empty((1,) * value.ndim)
          self._handle.create_array(self.group, key, arr)
          getattr(self.group, key)._v_attrs.value_type = str(value.dtype)
          getattr(self.group, key)._v_attrs.shape = value.shape
  
      def _is_empty_array(self, shape):
          """Returns true if any axis is zero length."""
          return any(x == 0 for x in shape)
  
      def write_array(self, key, value, items=None):
          if key in self.group:
              self._handle.remove_node(self.group, key)
  
          # Transform needed to interface with pytables row/col notation
          empty_array = self._is_empty_array(value.shape)
          transposed = False
  
          if is_categorical_dtype(value):
              raise NotImplementedError('Cannot store a category dtype in '
                                        'a HDF5 dataset that uses format='
                                        '"fixed". Use format="table".')
  
          if not empty_array:
              value = value.T
              transposed = True
  
          if self._filters is not None:
              atom = None
              try:
                  # get the atom for this datatype
                  atom = _tables().Atom.from_dtype(value.dtype)
              except ValueError:
                  pass
  
              if atom is not None:
                  # create an empty chunked array and fill it from value
                  if not empty_array:
                      ca = self._handle.create_carray(self.group, key, atom,
                                                      value.shape,
                                                      filters=self._filters)
                      ca[:] = value
                      getattr(self.group, key)._v_attrs.transposed = transposed
  
                  else:
                      self.write_array_empty(key, value)
  
                  return
  
          if value.dtype.type == np.object_:
  
              # infer the type, warn if we have a non-string type here (for
              # performance)
              inferred_type = lib.infer_dtype(value.ravel())
              if empty_array:
                  pass
              elif inferred_type == 'string':
                  pass
              else:
                  try:
                      items = list(items)
                  except:
                      pass
                  ws = performance_doc % (inferred_type, key, items)
                  warnings.warn(ws, PerformanceWarning, stacklevel=7)
  
              vlarr = self._handle.create_vlarray(self.group, key,
                                                  _tables().ObjectAtom())
              vlarr.append(value)
          else:
              if empty_array:
                  self.write_array_empty(key, value)
              else:
                  if is_datetime64_dtype(value.dtype):
                      self._handle.create_array(
                          self.group, key, value.view('i8'))
                      getattr(
                          self.group, key)._v_attrs.value_type = 'datetime64'
                  elif is_datetime64tz_dtype(value.dtype):
                      # store as UTC
                      # with a zone
                      self._handle.create_array(self.group, key,
                                                value.asi8)
  
                      node = getattr(self.group, key)
                      node._v_attrs.tz = _get_tz(value.tz)
                      node._v_attrs.value_type = 'datetime64'
                  elif is_timedelta64_dtype(value.dtype):
                      self._handle.create_array(
                          self.group, key, value.view('i8'))
                      getattr(
                          self.group, key)._v_attrs.value_type = 'timedelta64'
                  else:
                      self._handle.create_array(self.group, key, value)
  
          getattr(self.group, key)._v_attrs.transposed = transposed
  
  
  class LegacyFixed(GenericFixed):
  
      def read_index_legacy(self, key, start=None, stop=None):
          node = getattr(self.group, key)
          data = node[start:stop]
          kind = node._v_attrs.kind
          return _unconvert_index_legacy(data, kind, encoding=self.encoding)
  
  
  class LegacySeriesFixed(LegacyFixed):
  
      def read(self, **kwargs):
          kwargs = self.validate_read(kwargs)
          index = self.read_index_legacy('index')
          values = self.read_array('values')
          return Series(values, index=index)
  
  
  class LegacyFrameFixed(LegacyFixed):
  
      def read(self, **kwargs):
          kwargs = self.validate_read(kwargs)
          index = self.read_index_legacy('index')
          columns = self.read_index_legacy('columns')
          values = self.read_array('values')
          return DataFrame(values, index=index, columns=columns)
  
  
  class SeriesFixed(GenericFixed):
      pandas_kind = u('series')
      attributes = ['name']
  
      @property
      def shape(self):
          try:
              return len(getattr(self.group, 'values')),
          except:
              return None
  
      def read(self, **kwargs):
          kwargs = self.validate_read(kwargs)
          index = self.read_index('index', **kwargs)
          values = self.read_array('values', **kwargs)
          return Series(values, index=index, name=self.name)
  
      def write(self, obj, **kwargs):
          super(SeriesFixed, self).write(obj, **kwargs)
          self.write_index('index', obj.index)
          self.write_array('values', obj.values)
          self.attrs.name = obj.name
  
  
  class SparseFixed(GenericFixed):
  
      def validate_read(self, kwargs):
          """
          we don't support start, stop kwds in Sparse
          """
          kwargs = super(SparseFixed, self).validate_read(kwargs)
          if 'start' in kwargs or 'stop' in kwargs:
              raise NotImplementedError("start and/or stop are not supported "
                                        "in fixed Sparse reading")
          return kwargs
  
  
  class SparseSeriesFixed(SparseFixed):
      pandas_kind = u('sparse_series')
      attributes = ['name', 'fill_value', 'kind']
  
      def read(self, **kwargs):
          kwargs = self.validate_read(kwargs)
          index = self.read_index('index')
          sp_values = self.read_array('sp_values')
          sp_index = self.read_index('sp_index')
          return SparseSeries(sp_values, index=index, sparse_index=sp_index,
                              kind=self.kind or u('block'),
                              fill_value=self.fill_value,
                              name=self.name)
  
      def write(self, obj, **kwargs):
          super(SparseSeriesFixed, self).write(obj, **kwargs)
          self.write_index('index', obj.index)
          self.write_index('sp_index', obj.sp_index)
          self.write_array('sp_values', obj.sp_values)
          self.attrs.name = obj.name
          self.attrs.fill_value = obj.fill_value
          self.attrs.kind = obj.kind
  
  
  class SparseFrameFixed(SparseFixed):
      pandas_kind = u('sparse_frame')
      attributes = ['default_kind', 'default_fill_value']
  
      def read(self, **kwargs):
          kwargs = self.validate_read(kwargs)
          columns = self.read_index('columns')
          sdict = {}
          for c in columns:
              key = 'sparse_series_%s' % c
              s = SparseSeriesFixed(self.parent, getattr(self.group, key))
              s.infer_axes()
              sdict[c] = s.read()
          return SparseDataFrame(sdict, columns=columns,
                                 default_kind=self.default_kind,
                                 default_fill_value=self.default_fill_value)
  
      def write(self, obj, **kwargs):
          """ write it as a collection of individual sparse series """
          super(SparseFrameFixed, self).write(obj, **kwargs)
          for name, ss in compat.iteritems(obj):
              key = 'sparse_series_%s' % name
              if key not in self.group._v_children:
                  node = self._handle.create_group(self.group, key)
              else:
                  node = getattr(self.group, key)
              s = SparseSeriesFixed(self.parent, node)
              s.write(ss)
          self.attrs.default_fill_value = obj.default_fill_value
          self.attrs.default_kind = obj.default_kind
          self.write_index('columns', obj.columns)
  
  
  class BlockManagerFixed(GenericFixed):
      attributes = ['ndim', 'nblocks']
      is_shape_reversed = False
  
      @property
      def shape(self):
          try:
              ndim = self.ndim
  
              # items
              items = 0
              for i in range(self.nblocks):
                  node = getattr(self.group, 'block%d_items' % i)
                  shape = getattr(node, 'shape', None)
                  if shape is not None:
                      items += shape[0]
  
              # data shape
              node = getattr(self.group, 'block0_values')
              shape = getattr(node, 'shape', None)
              if shape is not None:
                  shape = list(shape[0:(ndim - 1)])
              else:
                  shape = []
  
              shape.append(items)
  
              # hacky - this works for frames, but is reversed for panels
              if self.is_shape_reversed:
                  shape = shape[::-1]
  
              return shape
          except:
              return None
  
      def read(self, start=None, stop=None, **kwargs):
          # start, stop applied to rows, so 0th axis only
  
          kwargs = self.validate_read(kwargs)
          select_axis = self.obj_type()._get_block_manager_axis(0)
  
          axes = []
          for i in range(self.ndim):
  
              _start, _stop = (start, stop) if i == select_axis else (None, None)
              ax = self.read_index('axis%d' % i, start=_start, stop=_stop)
              axes.append(ax)
  
          items = axes[0]
          blocks = []
          for i in range(self.nblocks):
  
              blk_items = self.read_index('block%d_items' % i)
              values = self.read_array('block%d_values' % i,
                                       start=_start, stop=_stop)
              blk = make_block(values,
                               placement=items.get_indexer(blk_items))
              blocks.append(blk)
  
          return self.obj_type(BlockManager(blocks, axes))
  
      def write(self, obj, **kwargs):
          super(BlockManagerFixed, self).write(obj, **kwargs)
          data = obj._data
          if not data.is_consolidated():
              data = data.consolidate()
  
          self.attrs.ndim = data.ndim
          for i, ax in enumerate(data.axes):
              if i == 0:
                  if not ax.is_unique:
                      raise ValueError(
                          "Columns index has to be unique for fixed format")
              self.write_index('axis%d' % i, ax)
  
          # Supporting mixed-type DataFrame objects...nontrivial
          self.attrs.nblocks = len(data.blocks)
          for i, blk in enumerate(data.blocks):
              # I have no idea why, but writing values before items fixed #2299
              blk_items = data.items.take(blk.mgr_locs)
              self.write_array('block%d_values' % i, blk.values, items=blk_items)
              self.write_index('block%d_items' % i, blk_items)
  
  
  class FrameFixed(BlockManagerFixed):
      pandas_kind = u('frame')
      obj_type = DataFrame
  
  
  class PanelFixed(BlockManagerFixed):
      pandas_kind = u('wide')
      obj_type = Panel
      is_shape_reversed = True
  
      def write(self, obj, **kwargs):
          obj._consolidate_inplace()
          return super(PanelFixed, self).write(obj, **kwargs)
  
  
  class Table(Fixed):
  
      """ represent a table:
            facilitate read/write of various types of tables
  
          Attrs in Table Node
          -------------------
          These are attributes that are store in the main table node, they are
          necessary to recreate these tables when read back in.
  
          index_axes    : a list of tuples of the (original indexing axis and
              index column)
          non_index_axes: a list of tuples of the (original index axis and
              columns on a non-indexing axis)
          values_axes   : a list of the columns which comprise the data of this
              table
          data_columns  : a list of the columns that we are allowing indexing
              (these become single columns in values_axes), or True to force all
              columns
          nan_rep       : the string to use for nan representations for string
              objects
          levels        : the names of levels
          metadata      : the names of the metadata columns
  
          """
      pandas_kind = u('wide_table')
      table_type = None
      levels = 1
      is_table = True
      is_shape_reversed = False
  
      def __init__(self, *args, **kwargs):
          super(Table, self).__init__(*args, **kwargs)
          self.index_axes = []
          self.non_index_axes = []
          self.values_axes = []
          self.data_columns = []
          self.metadata = []
          self.info = dict()
          self.nan_rep = None
          self.selection = None
  
      @property
      def table_type_short(self):
          return self.table_type.split('_')[0]
  
      @property
      def format_type(self):
          return 'table'
  
      def __unicode__(self):
          """ return a pretty representatgion of myself """
          self.infer_axes()
          dc = ",dc->[%s]" % ','.join(
              self.data_columns) if len(self.data_columns) else ''
  
          ver = ''
          if self.is_old_version:
              ver = "[%s]" % '.'.join([str(x) for x in self.version])
  
          return "%-12.12s%s (typ->%s,nrows->%s,ncols->%s,indexers->[%s]%s)" % (
              self.pandas_type, ver, self.table_type_short, self.nrows,
              self.ncols, ','.join([a.name for a in self.index_axes]), dc
          )
  
      def __getitem__(self, c):
          """ return the axis for c """
          for a in self.axes:
              if c == a.name:
                  return a
          return None
  
      def validate(self, other):
          """ validate against an existing table """
          if other is None:
              return
  
          if other.table_type != self.table_type:
              raise TypeError("incompatible table_type with existing [%s - %s]" %
                              (other.table_type, self.table_type))
  
          for c in ['index_axes', 'non_index_axes', 'values_axes']:
              sv = getattr(self, c, None)
              ov = getattr(other, c, None)
              if sv != ov:
  
                  # show the error for the specific axes
                  for i, sax in enumerate(sv):
                      oax = ov[i]
                      if sax != oax:
                          raise ValueError(
                              "invalid combinate of [%s] on appending data [%s] "
                              "vs current table [%s]" % (c, sax, oax))
  
                  # should never get here
                  raise Exception(
                      "invalid combinate of [%s] on appending data [%s] vs "
                      "current table [%s]" % (c, sv, ov))
  
      @property
      def is_multi_index(self):
          """the levels attribute is 1 or a list in the case of a multi-index"""
          return isinstance(self.levels, list)
  
      def validate_metadata(self, existing):
          """ create / validate metadata """
          self.metadata = [
              c.name for c in self.values_axes if c.metadata is not None]
  
      def validate_multiindex(self, obj):
          """validate that we can store the multi-index; reset and return the
          new object
          """
          levels = [l if l is not None else "level_{0}".format(i)
                    for i, l in enumerate(obj.index.names)]
          try:
              return obj.reset_index(), levels
          except ValueError:
              raise ValueError("duplicate names/columns in the multi-index when "
                               "storing as a table")
  
      @property
      def nrows_expected(self):
          """ based on our axes, compute the expected nrows """
          return np.prod([i.cvalues.shape[0] for i in self.index_axes])
  
      @property
      def is_exists(self):
          """ has this table been created """
          return u('table') in self.group
  
      @property
      def storable(self):
          return getattr(self.group, 'table', None)
  
      @property
      def table(self):
          """ return the table group (this is my storable) """
          return self.storable
  
      @property
      def dtype(self):
          return self.table.dtype
  
      @property
      def description(self):
          return self.table.description
  
      @property
      def axes(self):
          return itertools.chain(self.index_axes, self.values_axes)
  
      @property
      def ncols(self):
          """ the number of total columns in the values axes """
          return sum([len(a.values) for a in self.values_axes])
  
      @property
      def is_transposed(self):
          return False
  
      @property
      def data_orientation(self):
          """return a tuple of my permutated axes, non_indexable at the front"""
          return tuple(itertools.chain([int(a[0]) for a in self.non_index_axes],
                                       [int(a.axis) for a in self.index_axes]))
  
      def queryables(self):
          """ return a dict of the kinds allowable columns for this object """
  
          # compute the values_axes queryables
          return dict(
              [(a.cname, a) for a in self.index_axes] +
              [(self.storage_obj_type._AXIS_NAMES[axis], None)
               for axis, values in self.non_index_axes] +
              [(v.cname, v) for v in self.values_axes
               if v.name in set(self.data_columns)]
          )
  
      def index_cols(self):
          """ return a list of my index cols """
          return [(i.axis, i.cname) for i in self.index_axes]
  
      def values_cols(self):
          """ return a list of my values cols """
          return [i.cname for i in self.values_axes]
  
      def _get_metadata_path(self, key):
          """ return the metadata pathname for this key """
          return "{group}/meta/{key}/meta".format(group=self.group._v_pathname,
                                                  key=key)
  
      def write_metadata(self, key, values):
          """
          write out a meta data array to the key as a fixed-format Series
  
          Parameters
          ----------
          key : string
          values : ndarray
  
          """
          values = Series(values)
          self.parent.put(self._get_metadata_path(key), values, format='table',
                          encoding=self.encoding, nan_rep=self.nan_rep)
  
      def read_metadata(self, key):
          """ return the meta data array for this key """
          if getattr(getattr(self.group, 'meta', None), key, None) is not None:
              return self.parent.select(self._get_metadata_path(key))
          return None
  
      def set_info(self):
          """ update our table index info """
          self.attrs.info = self.info
  
      def set_attrs(self):
          """ set our table type & indexables """
          self.attrs.table_type = str(self.table_type)
          self.attrs.index_cols = self.index_cols()
          self.attrs.values_cols = self.values_cols()
          self.attrs.non_index_axes = self.non_index_axes
          self.attrs.data_columns = self.data_columns
          self.attrs.nan_rep = self.nan_rep
          self.attrs.encoding = self.encoding
          self.attrs.levels = self.levels
          self.attrs.metadata = self.metadata
          self.set_info()
  
      def get_attrs(self):
          """ retrieve our attributes """
          self.non_index_axes = getattr(
              self.attrs, 'non_index_axes', None) or []
          self.data_columns = getattr(
              self.attrs, 'data_columns', None) or []
          self.info = getattr(
              self.attrs, 'info', None) or dict()
          self.nan_rep = getattr(self.attrs, 'nan_rep', None)
          self.encoding = _ensure_encoding(
              getattr(self.attrs, 'encoding', None))
          self.levels = getattr(
              self.attrs, 'levels', None) or []
          self.index_axes = [
              a.infer(self) for a in self.indexables if a.is_an_indexable
          ]
          self.values_axes = [
              a.infer(self) for a in self.indexables if not a.is_an_indexable
          ]
          self.metadata = getattr(
              self.attrs, 'metadata', None) or []
  
      def validate_version(self, where=None):
          """ are we trying to operate on an old version? """
          if where is not None:
              if (self.version[0] <= 0 and self.version[1] <= 10 and
                      self.version[2] < 1):
                  ws = incompatibility_doc % '.'.join(
                      [str(x) for x in self.version])
                  warnings.warn(ws, IncompatibilityWarning)
  
      def validate_min_itemsize(self, min_itemsize):
          """validate the min_itemisze doesn't contain items that are not in the
          axes this needs data_columns to be defined
          """
          if min_itemsize is None:
              return
          if not isinstance(min_itemsize, dict):
              return
  
          q = self.queryables()
          for k, v in min_itemsize.items():
  
              # ok, apply generally
              if k == 'values':
                  continue
              if k not in q:
                  raise ValueError(
                      "min_itemsize has the key [%s] which is not an axis or "
                      "data_column" % k)
  
      @property
      def indexables(self):
          """ create/cache the indexables if they don't exist """
          if self._indexables is None:
  
              self._indexables = []
  
              # index columns
              self._indexables.extend([
                  IndexCol(name=name, axis=axis, pos=i)
                  for i, (axis, name) in enumerate(self.attrs.index_cols)
              ])
  
              # values columns
              dc = set(self.data_columns)
              base_pos = len(self._indexables)
  
              def f(i, c):
                  klass = DataCol
                  if c in dc:
                      klass = DataIndexableCol
                  return klass.create_for_block(i=i, name=c, pos=base_pos + i,
                                                version=self.version)
  
              self._indexables.extend(
                  [f(i, c) for i, c in enumerate(self.attrs.values_cols)])
  
          return self._indexables
  
      def create_index(self, columns=None, optlevel=None, kind=None):
          """
          Create a pytables index on the specified columns
            note: cannot index Time64Col() or ComplexCol currently;
            PyTables must be >= 3.0
  
          Parameters
          ----------
          columns : False (don't create an index), True (create all columns
              index), None or list_like (the indexers to index)
          optlevel: optimization level (defaults to 6)
          kind    : kind of index (defaults to 'medium')
  
          Exceptions
          ----------
          raises if the node is not a table
  
          """
  
          if not self.infer_axes():
              return
          if columns is False:
              return
  
          # index all indexables and data_columns
          if columns is None or columns is True:
              columns = [a.cname for a in self.axes if a.is_data_indexable]
          if not isinstance(columns, (tuple, list)):
              columns = [columns]
  
          kw = dict()
          if optlevel is not None:
              kw['optlevel'] = optlevel
          if kind is not None:
              kw['kind'] = kind
  
          table = self.table
          for c in columns:
              v = getattr(table.cols, c, None)
              if v is not None:
  
                  # remove the index if the kind/optlevel have changed
                  if v.is_indexed:
                      index = v.index
                      cur_optlevel = index.optlevel
                      cur_kind = index.kind
  
                      if kind is not None and cur_kind != kind:
                          v.remove_index()
                      else:
                          kw['kind'] = cur_kind
  
                      if optlevel is not None and cur_optlevel != optlevel:
                          v.remove_index()
                      else:
                          kw['optlevel'] = cur_optlevel
  
                  # create the index
                  if not v.is_indexed:
                      if v.type.startswith('complex'):
                          raise TypeError(
                              'Columns containing complex values can be stored '
                              'but cannot'
                              ' be indexed when using table format. Either use '
                              'fixed format, set index=False, or do not include '
                              'the columns containing complex values to '
                              'data_columns when initializing the table.')
                      v.create_index(**kw)
  
      def read_axes(self, where, **kwargs):
          """create and return the axes sniffed from the table: return boolean
          for success
          """
  
          # validate the version
          self.validate_version(where)
  
          # infer the data kind
          if not self.infer_axes():
              return False
  
          # create the selection
          self.selection = Selection(self, where=where, **kwargs)
          values = self.selection.select()
  
          # convert the data
          for a in self.axes:
              a.set_info(self.info)
              a.convert(values, nan_rep=self.nan_rep, encoding=self.encoding)
  
          return True
  
      def get_object(self, obj):
          """ return the data for this obj """
          return obj
  
      def validate_data_columns(self, data_columns, min_itemsize):
          """take the input data_columns and min_itemize and create a data
          columns spec
          """
  
          if not len(self.non_index_axes):
              return []
  
          axis, axis_labels = self.non_index_axes[0]
          info = self.info.get(axis, dict())
          if info.get('type') == 'MultiIndex' and data_columns:
              raise ValueError("cannot use a multi-index on axis [{0}] with "
                               "data_columns {1}".format(axis, data_columns))
  
          # evaluate the passed data_columns, True == use all columns
          # take only valide axis labels
          if data_columns is True:
              data_columns = axis_labels
          elif data_columns is None:
              data_columns = []
  
          # if min_itemsize is a dict, add the keys (exclude 'values')
          if isinstance(min_itemsize, dict):
  
              existing_data_columns = set(data_columns)
              data_columns.extend([
                  k for k in min_itemsize.keys()
                  if k != 'values' and k not in existing_data_columns
              ])
  
          # return valid columns in the order of our axis
          return [c for c in data_columns if c in axis_labels]
  
      def create_axes(self, axes, obj, validate=True, nan_rep=None,
                      data_columns=None, min_itemsize=None, **kwargs):
          """ create and return the axes
          leagcy tables create an indexable column, indexable index,
          non-indexable fields
  
              Parameters:
              -----------
              axes: a list of the axes in order to create (names or numbers of
                  the axes)
              obj : the object to create axes on
              validate: validate the obj against an existing object already
                  written
              min_itemsize: a dict of the min size for a column in bytes
              nan_rep : a values to use for string column nan_rep
              encoding : the encoding for string values
              data_columns : a list of columns that we want to create separate to
                  allow indexing (or True will force all columns)
  
          """
  
          # set the default axes if needed
          if axes is None:
              try:
                  axes = _AXES_MAP[type(obj)]
              except:
                  raise TypeError("cannot properly create the storer for: "
                                  "[group->%s,value->%s]"
                                  % (self.group._v_name, type(obj)))
  
          # map axes to numbers
          axes = [obj._get_axis_number(a) for a in axes]
  
          # do we have an existing table (if so, use its axes & data_columns)
          if self.infer_axes():
              existing_table = self.copy()
              existing_table.infer_axes()
              axes = [a.axis for a in existing_table.index_axes]
              data_columns = existing_table.data_columns
              nan_rep = existing_table.nan_rep
              self.encoding = existing_table.encoding
              self.info = copy.copy(existing_table.info)
          else:
              existing_table = None
  
          # currently support on ndim-1 axes
          if len(axes) != self.ndim - 1:
              raise ValueError(
                  "currently only support ndim-1 indexers in an AppendableTable")
  
          # create according to the new data
          self.non_index_axes = []
          self.data_columns = []
  
          # nan_representation
          if nan_rep is None:
              nan_rep = 'nan'
  
          self.nan_rep = nan_rep
  
          # create axes to index and non_index
          index_axes_map = dict()
          for i, a in enumerate(obj.axes):
  
              if i in axes:
                  name = obj._AXIS_NAMES[i]
                  index_axes_map[i] = _convert_index(
                      a, self.encoding, self.format_type
                  ).set_name(name).set_axis(i)
              else:
  
                  # we might be able to change the axes on the appending data if
                  # necessary
                  append_axis = list(a)
                  if existing_table is not None:
                      indexer = len(self.non_index_axes)
                      exist_axis = existing_table.non_index_axes[indexer][1]
                      if append_axis != exist_axis:
  
                          # ahah! -> reindex
                          if sorted(append_axis) == sorted(exist_axis):
                              append_axis = exist_axis
  
                  # the non_index_axes info
                  info = _get_info(self.info, i)
                  info['names'] = list(a.names)
                  info['type'] = a.__class__.__name__
  
                  self.non_index_axes.append((i, append_axis))
  
          # set axis positions (based on the axes)
          self.index_axes = [
              index_axes_map[a].set_pos(j).update_info(self.info)
              for j, a in enumerate(axes)
          ]
          j = len(self.index_axes)
  
          # check for column conflicts
          if validate:
              for a in self.axes:
                  a.maybe_set_size(min_itemsize=min_itemsize)
  
          # reindex by our non_index_axes & compute data_columns
          for a in self.non_index_axes:
              obj = _reindex_axis(obj, a[0], a[1])
  
          def get_blk_items(mgr, blocks):
              return [mgr.items.take(blk.mgr_locs) for blk in blocks]
  
          # figure out data_columns and get out blocks
          block_obj = self.get_object(obj).consolidate()
          blocks = block_obj._data.blocks
          blk_items = get_blk_items(block_obj._data, blocks)
          if len(self.non_index_axes):
              axis, axis_labels = self.non_index_axes[0]
              data_columns = self.validate_data_columns(
                  data_columns, min_itemsize)
              if len(data_columns):
                  mgr = block_obj.reindex_axis(
                      Index(axis_labels).difference(Index(data_columns)),
                      axis=axis
                  )._data
  
                  blocks = list(mgr.blocks)
                  blk_items = get_blk_items(mgr, blocks)
                  for c in data_columns:
                      mgr = block_obj.reindex_axis([c], axis=axis)._data
                      blocks.extend(mgr.blocks)
                      blk_items.extend(get_blk_items(mgr, mgr.blocks))
  
          # reorder the blocks in the same order as the existing_table if we can
          if existing_table is not None:
              by_items = dict([(tuple(b_items.tolist()), (b, b_items))
                               for b, b_items in zip(blocks, blk_items)])
              new_blocks = []
              new_blk_items = []
              for ea in existing_table.values_axes:
                  items = tuple(ea.values)
                  try:
                      b, b_items = by_items.pop(items)
                      new_blocks.append(b)
                      new_blk_items.append(b_items)
                  except:
                      raise ValueError(
                          "cannot match existing table structure for [%s] on "
                          "appending data" % ','.join(pprint_thing(item) for
                                                      item in items))
              blocks = new_blocks
              blk_items = new_blk_items
  
          # add my values
          self.values_axes = []
          for i, (b, b_items) in enumerate(zip(blocks, blk_items)):
  
              # shape of the data column are the indexable axes
              klass = DataCol
              name = None
  
              # we have a data_column
              if (data_columns and len(b_items) == 1 and
                      b_items[0] in data_columns):
                  klass = DataIndexableCol
                  name = b_items[0]
                  self.data_columns.append(name)
  
              # make sure that we match up the existing columns
              # if we have an existing table
              if existing_table is not None and validate:
                  try:
                      existing_col = existing_table.values_axes[i]
                  except:
                      raise ValueError("Incompatible appended table [%s] with "
                                       "existing table [%s]"
                                       % (blocks, existing_table.values_axes))
              else:
                  existing_col = None
  
              try:
                  col = klass.create_for_block(
                      i=i, name=name, version=self.version)
                  col.set_atom(block=b, block_items=b_items,
                               existing_col=existing_col,
                               min_itemsize=min_itemsize,
                               nan_rep=nan_rep,
                               encoding=self.encoding,
                               info=self.info,
                               **kwargs)
                  col.set_pos(j)
  
                  self.values_axes.append(col)
              except (NotImplementedError, ValueError, TypeError) as e:
                  raise e
              except Exception as detail:
                  raise Exception(
                      "cannot find the correct atom type -> "
                      "[dtype->%s,items->%s] %s"
                      % (b.dtype.name, b_items, str(detail))
                  )
              j += 1
  
          # validate our min_itemsize
          self.validate_min_itemsize(min_itemsize)
  
          # validate our metadata
          self.validate_metadata(existing_table)
  
          # validate the axes if we have an existing table
          if validate:
              self.validate(existing_table)
  
      def process_axes(self, obj, columns=None):
          """ process axes filters """
  
          # make a copy to avoid side effects
          if columns is not None:
              columns = list(columns)
  
          # make sure to include levels if we have them
          if columns is not None and self.is_multi_index:
              for n in self.levels:
                  if n not in columns:
                      columns.insert(0, n)
  
          # reorder by any non_index_axes & limit to the select columns
          for axis, labels in self.non_index_axes:
              obj = _reindex_axis(obj, axis, labels, columns)
  
          # apply the selection filters (but keep in the same order)
          if self.selection.filter is not None:
              for field, op, filt in self.selection.filter.format():
  
                  def process_filter(field, filt):
  
                      for axis_name in obj._AXIS_NAMES.values():
                          axis_number = obj._get_axis_number(axis_name)
                          axis_values = obj._get_axis(axis_name)
  
                          # see if the field is the name of an axis
                          if field == axis_name:
  
                              # if we have a multi-index, then need to include
                              # the levels
                              if self.is_multi_index:
                                  filt = filt.union(Index(self.levels))
  
                              takers = op(axis_values, filt)
                              return obj.ix._getitem_axis(takers,
                                                          axis=axis_number)
  
                          # this might be the name of a file IN an axis
                          elif field in axis_values:
  
                              # we need to filter on this dimension
                              values = _ensure_index(getattr(obj, field).values)
                              filt = _ensure_index(filt)
  
                              # hack until we support reversed dim flags
                              if isinstance(obj, DataFrame):
                                  axis_number = 1 - axis_number
                              takers = op(values, filt)
                              return obj.ix._getitem_axis(takers,
                                                          axis=axis_number)
  
                      raise ValueError(
                          "cannot find the field [%s] for filtering!" % field)
  
                  obj = process_filter(field, filt)
  
          return obj
  
      def create_description(self, complib=None, complevel=None,
                             fletcher32=False, expectedrows=None):
          """ create the description of the table from the axes & values """
  
          # provided expected rows if its passed
          if expectedrows is None:
              expectedrows = max(self.nrows_expected, 10000)
  
          d = dict(name='table', expectedrows=expectedrows)
  
          # description from the axes & values
          d['description'] = dict([(a.cname, a.typ) for a in self.axes])
  
          if complib:
              if complevel is None:
                  complevel = self._complevel or 9
              filters = _tables().Filters(
                  complevel=complevel, complib=complib,
                  fletcher32=fletcher32 or self._fletcher32)
              d['filters'] = filters
          elif self._filters is not None:
              d['filters'] = self._filters
  
          return d
  
      def read_coordinates(self, where=None, start=None, stop=None, **kwargs):
          """select coordinates (row numbers) from a table; return the
          coordinates object
          """
  
          # validate the version
          self.validate_version(where)
  
          # infer the data kind
          if not self.infer_axes():
              return False
  
          # create the selection
          self.selection = Selection(
              self, where=where, start=start, stop=stop, **kwargs)
          coords = self.selection.select_coords()
          if self.selection.filter is not None:
              for field, op, filt in self.selection.filter.format():
                  data = self.read_column(
                      field, start=coords.min(), stop=coords.max() + 1)
                  coords = coords[
                      op(data.iloc[coords - coords.min()], filt).values]
  
          return Index(coords)
  
      def read_column(self, column, where=None, start=None, stop=None, **kwargs):
          """return a single column from the table, generally only indexables
          are interesting
          """
  
          # validate the version
          self.validate_version()
  
          # infer the data kind
          if not self.infer_axes():
              return False
  
          if where is not None:
              raise TypeError("read_column does not currently accept a where "
                              "clause")
  
          # find the axes
          for a in self.axes:
              if column == a.name:
  
                  if not a.is_data_indexable:
                      raise ValueError(
                          "column [%s] can not be extracted individually; it is "
                          "not data indexable" % column)
  
                  # column must be an indexable or a data column
                  c = getattr(self.table.cols, column)
                  a.set_info(self.info)
                  return Series(_set_tz(a.convert(c[start:stop],
                                                  nan_rep=self.nan_rep,
                                                  encoding=self.encoding
                                                  ).take_data(),
                                        a.tz, True), name=column)
  
          raise KeyError("column [%s] not found in the table" % column)
  
  
  class WORMTable(Table):
  
      """ a write-once read-many table: this format DOES NOT ALLOW appending to a
           table. writing is a one-time operation the data are stored in a format
           that allows for searching the data on disk
           """
      table_type = u('worm')
  
      def read(self, **kwargs):
          """ read the indicies and the indexing array, calculate offset rows and
          return """
          raise NotImplementedError("WORMTable needs to implement read")
  
      def write(self, **kwargs):
          """ write in a format that we can search later on (but cannot append
                 to): write out the indicies and the values using _write_array
                 (e.g. a CArray) create an indexing table so that we can search
          """
          raise NotImplementedError("WORKTable needs to implement write")
  
  
  class LegacyTable(Table):
  
      """ an appendable table: allow append/query/delete operations to a
            (possibily) already existing appendable table this table ALLOWS
            append (but doesn't require them), and stores the data in a format
            that can be easily searched
  
      """
      _indexables = [
          IndexCol(name='index', axis=1, pos=0),
          IndexCol(name='column', axis=2, pos=1, index_kind='columns_kind'),
          DataCol(name='fields', cname='values', kind_attr='fields', pos=2)
      ]
      table_type = u('legacy')
      ndim = 3
  
      def write(self, **kwargs):
          raise TypeError("write operations are not allowed on legacy tables!")
  
      def read(self, where=None, columns=None, **kwargs):
          """we have n indexable columns, with an arbitrary number of data
          axes
          """
  
          if not self.read_axes(where=where, **kwargs):
              return None
  
          factors = [Categorical.from_array(
              a.values, ordered=True) for a in self.index_axes]
          levels = [f.categories for f in factors]
          N = [len(f.categories) for f in factors]
          labels = [f.codes for f in factors]
  
          # compute the key
          key = _factor_indexer(N[1:], labels)
  
          objs = []
          if len(unique(key)) == len(key):
  
              sorter, _ = algos.groupsort_indexer(
                  _ensure_int64(key), np.prod(N))
              sorter = _ensure_platform_int(sorter)
  
              # create the objs
              for c in self.values_axes:
  
                  # the data need to be sorted
                  sorted_values = c.take_data().take(sorter, axis=0)
                  if sorted_values.ndim == 1:
                      sorted_values = sorted_values.reshape(
                          (sorted_values.shape[0], 1))
  
                  take_labels = [l.take(sorter) for l in labels]
                  items = Index(c.values)
                  block = _block2d_to_blocknd(
                      values=sorted_values, placement=np.arange(len(items)),
                      shape=tuple(N), labels=take_labels, ref_items=items)
  
                  # create the object
                  mgr = BlockManager([block], [items] + levels)
                  obj = self.obj_type(mgr)
  
                  # permute if needed
                  if self.is_transposed:
                      obj = obj.transpose(
                          *tuple(Series(self.data_orientation).argsort()))
  
                  objs.append(obj)
  
          else:
              warnings.warn(duplicate_doc, DuplicateWarning, stacklevel=5)
  
              # reconstruct
              long_index = MultiIndex.from_arrays(
                  [i.values for i in self.index_axes])
  
              for c in self.values_axes:
                  lp = DataFrame(c.data, index=long_index, columns=c.values)
  
                  # need a better algorithm
                  tuple_index = long_index._tuple_index
  
                  unique_tuples = lib.fast_unique(tuple_index.values)
                  unique_tuples = _asarray_tuplesafe(unique_tuples)
  
                  indexer = match(unique_tuples, tuple_index)
                  indexer = _ensure_platform_int(indexer)
  
                  new_index = long_index.take(indexer)
                  new_values = lp.values.take(indexer, axis=0)
  
                  lp = DataFrame(new_values, index=new_index, columns=lp.columns)
                  objs.append(lp.to_panel())
  
          # create the composite object
          if len(objs) == 1:
              wp = objs[0]
          else:
              wp = concat(objs, axis=0, verify_integrity=False).consolidate()
  
          # apply the selection filters & axis orderings
          wp = self.process_axes(wp, columns=columns)
  
          return wp
  
  
  class LegacyFrameTable(LegacyTable):
  
      """ support the legacy frame table """
      pandas_kind = u('frame_table')
      table_type = u('legacy_frame')
      obj_type = Panel
  
      def read(self, *args, **kwargs):
          return super(LegacyFrameTable, self).read(*args, **kwargs)['value']
  
  
  class LegacyPanelTable(LegacyTable):
  
      """ support the legacy panel table """
      table_type = u('legacy_panel')
      obj_type = Panel
  
  
  class AppendableTable(LegacyTable):
  
      """ suppor the new appendable table formats """
      _indexables = None
      table_type = u('appendable')
  
      def write(self, obj, axes=None, append=False, complib=None,
                complevel=None, fletcher32=None, min_itemsize=None,
                chunksize=None, expectedrows=None, dropna=False, **kwargs):
  
          if not append and self.is_exists:
              self._handle.remove_node(self.group, 'table')
  
          # create the axes
          self.create_axes(axes=axes, obj=obj, validate=append,
                           min_itemsize=min_itemsize,
                           **kwargs)
  
          for a in self.axes:
              a.validate(self, append)
  
          if not self.is_exists:
  
              # create the table
              options = self.create_description(complib=complib,
                                                complevel=complevel,
                                                fletcher32=fletcher32,
                                                expectedrows=expectedrows)
  
              # set the table attributes
              self.set_attrs()
  
              # create the table
              self._handle.create_table(self.group, **options)
          else:
              pass
              # table = self.table
  
          # update my info
          self.set_info()
  
          # validate the axes and set the kinds
          for a in self.axes:
              a.validate_and_set(self, append)
  
          # add the rows
          self.write_data(chunksize, dropna=dropna)
  
      def write_data(self, chunksize, dropna=False):
          """ we form the data into a 2-d including indexes,values,mask
              write chunk-by-chunk """
  
          names = self.dtype.names
          nrows = self.nrows_expected
  
          # if dropna==True, then drop ALL nan rows
          masks = []
          if dropna:
  
              for a in self.values_axes:
  
                  # figure the mask: only do if we can successfully process this
                  # column, otherwise ignore the mask
                  mask = isnull(a.data).all(axis=0)
                  if isinstance(mask, np.ndarray):
                      masks.append(mask.astype('u1', copy=False))
  
          # consolidate masks
          if len(masks):
              mask = masks[0]
              for m in masks[1:]:
                  mask = mask & m
              mask = mask.ravel()
          else:
              mask = None
  
          # broadcast the indexes if needed
          indexes = [a.cvalues for a in self.index_axes]
          nindexes = len(indexes)
          bindexes = []
          for i, idx in enumerate(indexes):
  
              # broadcast to all other indexes except myself
              if i > 0 and i < nindexes:
                  repeater = np.prod(
                      [indexes[bi].shape[0] for bi in range(0, i)])
                  idx = np.tile(idx, repeater)
  
              if i < nindexes - 1:
                  repeater = np.prod([indexes[bi].shape[0]
                                      for bi in range(i + 1, nindexes)])
                  idx = np.repeat(idx, repeater)
  
              bindexes.append(idx)
  
          # transpose the values so first dimension is last
          # reshape the values if needed
          values = [a.take_data() for a in self.values_axes]
          values = [v.transpose(np.roll(np.arange(v.ndim), v.ndim - 1))
                    for v in values]
          bvalues = []
          for i, v in enumerate(values):
              new_shape = (nrows,) + self.dtype[names[nindexes + i]].shape
              bvalues.append(values[i].reshape(new_shape))
  
          # write the chunks
          if chunksize is None:
              chunksize = 100000
  
          rows = np.empty(min(chunksize, nrows), dtype=self.dtype)
          chunks = int(nrows / chunksize) + 1
          for i in range(chunks):
              start_i = i * chunksize
              end_i = min((i + 1) * chunksize, nrows)
              if start_i >= end_i:
                  break
  
              self.write_data_chunk(
                  rows,
                  indexes=[a[start_i:end_i] for a in bindexes],
                  mask=mask[start_i:end_i] if mask is not None else None,
                  values=[v[start_i:end_i] for v in bvalues])
  
      def write_data_chunk(self, rows, indexes, mask, values):
          """
          Parameters
          ----------
          rows : an empty memory space where we are putting the chunk
          indexes : an array of the indexes
          mask : an array of the masks
          values : an array of the values
          """
  
          # 0 len
          for v in values:
              if not np.prod(v.shape):
                  return
  
          try:
              nrows = indexes[0].shape[0]
              if nrows != len(rows):
                  rows = np.empty(nrows, dtype=self.dtype)
              names = self.dtype.names
              nindexes = len(indexes)
  
              # indexes
              for i, idx in enumerate(indexes):
                  rows[names[i]] = idx
  
              # values
              for i, v in enumerate(values):
                  rows[names[i + nindexes]] = v
  
              # mask
              if mask is not None:
                  m = ~mask.ravel().astype(bool, copy=False)
                  if not m.all():
                      rows = rows[m]
  
          except Exception as detail:
              raise Exception("cannot create row-data -> %s" % detail)
  
          try:
              if len(rows):
                  self.table.append(rows)
                  self.table.flush()
          except Exception as detail:
              raise TypeError("tables cannot write this data -> %s" % detail)
  
      def delete(self, where=None, start=None, stop=None, **kwargs):
  
          # delete all rows (and return the nrows)
          if where is None or not len(where):
              if start is None and stop is None:
                  nrows = self.nrows
                  self._handle.remove_node(self.group, recursive=True)
              else:
                  # pytables<3.0 would remove a single row with stop=None
                  if stop is None:
                      stop = self.nrows
                  nrows = self.table.remove_rows(start=start, stop=stop)
                  self.table.flush()
              return nrows
  
          # infer the data kind
          if not self.infer_axes():
              return None
  
          # create the selection
          table = self.table
          self.selection = Selection(
              self, where, start=start, stop=stop, **kwargs)
          values = self.selection.select_coords()
  
          # delete the rows in reverse order
          l = Series(values).sort_values()
          ln = len(l)
  
          if ln:
  
              # construct groups of consecutive rows
              diff = l.diff()
              groups = list(diff[diff > 1].index)
  
              # 1 group
              if not len(groups):
                  groups = [0]
  
              # final element
              if groups[-1] != ln:
                  groups.append(ln)
  
              # initial element
              if groups[0] != 0:
                  groups.insert(0, 0)
  
              # we must remove in reverse order!
              pg = groups.pop()
              for g in reversed(groups):
                  rows = l.take(lrange(g, pg))
                  table.remove_rows(start=rows[rows.index[0]
                                               ], stop=rows[rows.index[-1]] + 1)
                  pg = g
  
              self.table.flush()
  
          # return the number of rows removed
          return ln
  
  
  class AppendableFrameTable(AppendableTable):
  
      """ suppor the new appendable table formats """
      pandas_kind = u('frame_table')
      table_type = u('appendable_frame')
      ndim = 2
      obj_type = DataFrame
  
      @property
      def is_transposed(self):
          return self.index_axes[0].axis == 1
  
      def get_object(self, obj):
          """ these are written transposed """
          if self.is_transposed:
              obj = obj.T
          return obj
  
      def read(self, where=None, columns=None, **kwargs):
  
          if not self.read_axes(where=where, **kwargs):
              return None
  
          info = (self.info.get(self.non_index_axes[0][0], dict())
                  if len(self.non_index_axes) else dict())
          index = self.index_axes[0].values
          frames = []
          for a in self.values_axes:
  
              # we could have a multi-index constructor here
              # _ensure_index doesn't recognized our list-of-tuples here
              if info.get('type') == 'MultiIndex':
                  cols = MultiIndex.from_tuples(a.values)
              else:
                  cols = Index(a.values)
              names = info.get('names')
              if names is not None:
                  cols.set_names(names, inplace=True)
  
              if self.is_transposed:
                  values = a.cvalues
                  index_ = cols
                  cols_ = Index(index, name=getattr(index, 'name', None))
              else:
                  values = a.cvalues.T
                  index_ = Index(index, name=getattr(index, 'name', None))
                  cols_ = cols
  
              # if we have a DataIndexableCol, its shape will only be 1 dim
              if values.ndim == 1 and isinstance(values, np.ndarray):
                  values = values.reshape((1, values.shape[0]))
  
              block = make_block(values, placement=np.arange(len(cols_)))
              mgr = BlockManager([block], [cols_, index_])
              frames.append(DataFrame(mgr))
  
          if len(frames) == 1:
              df = frames[0]
          else:
              df = concat(frames, axis=1)
  
          # apply the selection filters & axis orderings
          df = self.process_axes(df, columns=columns)
  
          return df
  
  
  class AppendableSeriesTable(AppendableFrameTable):
      """ support the new appendable table formats """
      pandas_kind = u('series_table')
      table_type = u('appendable_series')
      ndim = 2
      obj_type = Series
      storage_obj_type = DataFrame
  
      @property
      def is_transposed(self):
          return False
  
      def get_object(self, obj):
          return obj
  
      def write(self, obj, data_columns=None, **kwargs):
          """ we are going to write this as a frame table """
          if not isinstance(obj, DataFrame):
              name = obj.name or 'values'
              obj = DataFrame({name: obj}, index=obj.index)
              obj.columns = [name]
          return super(AppendableSeriesTable, self).write(
              obj=obj, data_columns=obj.columns, **kwargs)
  
      def read(self, columns=None, **kwargs):
  
          is_multi_index = self.is_multi_index
          if columns is not None and is_multi_index:
              for n in self.levels:
                  if n not in columns:
                      columns.insert(0, n)
          s = super(AppendableSeriesTable, self).read(columns=columns, **kwargs)
          if is_multi_index:
              s.set_index(self.levels, inplace=True)
  
          s = s.iloc[:, 0]
  
          # remove the default name
          if s.name == 'values':
              s.name = None
          return s
  
  
  class AppendableMultiSeriesTable(AppendableSeriesTable):
      """ support the new appendable table formats """
      pandas_kind = u('series_table')
      table_type = u('appendable_multiseries')
  
      def write(self, obj, **kwargs):
          """ we are going to write this as a frame table """
          name = obj.name or 'values'
          obj, self.levels = self.validate_multiindex(obj)
          cols = list(self.levels)
          cols.append(name)
          obj.columns = cols
          return super(AppendableMultiSeriesTable, self).write(obj=obj, **kwargs)
  
  
  class GenericTable(AppendableFrameTable):
      """ a table that read/writes the generic pytables table format """
      pandas_kind = u('frame_table')
      table_type = u('generic_table')
      ndim = 2
      obj_type = DataFrame
  
      @property
      def pandas_type(self):
          return self.pandas_kind
  
      @property
      def storable(self):
          return getattr(self.group, 'table', None) or self.group
  
      def get_attrs(self):
          """ retrieve our attributes """
          self.non_index_axes = []
          self.nan_rep = None
          self.levels = []
  
          self.index_axes = [a.infer(self)
                             for a in self.indexables if a.is_an_indexable]
          self.values_axes = [a.infer(self)
                              for a in self.indexables if not a.is_an_indexable]
          self.data_columns = [a.name for a in self.values_axes]
  
      @property
      def indexables(self):
          """ create the indexables from the table description """
          if self._indexables is None:
  
              d = self.description
  
              # the index columns is just a simple index
              self._indexables = [GenericIndexCol(name='index', axis=0)]
  
              for i, n in enumerate(d._v_names):
  
                  dc = GenericDataIndexableCol(
                      name=n, pos=i, values=[n], version=self.version)
                  self._indexables.append(dc)
  
          return self._indexables
  
      def write(self, **kwargs):
          raise NotImplementedError("cannot write on an generic table")
  
  
  class AppendableMultiFrameTable(AppendableFrameTable):
  
      """ a frame with a multi-index """
      table_type = u('appendable_multiframe')
      obj_type = DataFrame
      ndim = 2
      _re_levels = re.compile("^level_\d+$")
  
      @property
      def table_type_short(self):
          return u('appendable_multi')
  
      def write(self, obj, data_columns=None, **kwargs):
          if data_columns is None:
              data_columns = []
          elif data_columns is True:
              data_columns = obj.columns[:]
          obj, self.levels = self.validate_multiindex(obj)
          for n in self.levels:
              if n not in data_columns:
                  data_columns.insert(0, n)
          return super(AppendableMultiFrameTable, self).write(
              obj=obj, data_columns=data_columns, **kwargs)
  
      def read(self, **kwargs):
  
          df = super(AppendableMultiFrameTable, self).read(**kwargs)
          df = df.set_index(self.levels)
  
          # remove names for 'level_%d'
          df.index = df.index.set_names([
              None if self._re_levels.search(l) else l for l in df.index.names
          ])
  
          return df
  
  
  class AppendablePanelTable(AppendableTable):
  
      """ suppor the new appendable table formats """
      table_type = u('appendable_panel')
      ndim = 3
      obj_type = Panel
  
      def get_object(self, obj):
          """ these are written transposed """
          if self.is_transposed:
              obj = obj.transpose(*self.data_orientation)
          return obj
  
      @property
      def is_transposed(self):
          return self.data_orientation != tuple(range(self.ndim))
  
  
  class AppendableNDimTable(AppendablePanelTable):
  
      """ suppor the new appendable table formats """
      table_type = u('appendable_ndim')
      ndim = 4
      obj_type = Panel4D
  
  
  def _reindex_axis(obj, axis, labels, other=None):
      ax = obj._get_axis(axis)
      labels = _ensure_index(labels)
  
      # try not to reindex even if other is provided
      # if it equals our current index
      if other is not None:
          other = _ensure_index(other)
      if (other is None or labels.equals(other)) and labels.equals(ax):
          return obj
  
      labels = _ensure_index(labels.unique())
      if other is not None:
          labels = labels & _ensure_index(other.unique())
      if not labels.equals(ax):
          slicer = [slice(None, None)] * obj.ndim
          slicer[axis] = labels
          obj = obj.loc[tuple(slicer)]
      return obj
  
  
  def _get_info(info, name):
      """ get/create the info for this name """
      try:
          idx = info[name]
      except:
          idx = info[name] = dict()
      return idx
  
  # tz to/from coercion
  
  
  def _get_tz(tz):
      """ for a tz-aware type, return an encoded zone """
      zone = tslib.get_timezone(tz)
      if zone is None:
          zone = tslib.tot_seconds(tz.utcoffset())
      return zone
  
  
  def _set_tz(values, tz, preserve_UTC=False, coerce=False):
      """
      coerce the values to a DatetimeIndex if tz is set
      preserve the input shape if possible
  
      Parameters
      ----------
      values : ndarray
      tz : string/pickled tz object
      preserve_UTC : boolean,
          preserve the UTC of the result
      coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray
      """
      if tz is not None:
          name = getattr(values, 'name', None)
          values = values.ravel()
          tz = tslib.get_timezone(_ensure_decoded(tz))
          values = DatetimeIndex(values, name=name)
          if values.tz is None:
              values = values.tz_localize('UTC').tz_convert(tz)
          if preserve_UTC:
              if tz == 'UTC':
                  values = list(values)
      elif coerce:
          values = np.asarray(values, dtype='M8[ns]')
  
      return values
  
  
  def _convert_index(index, encoding=None, format_type=None):
      index_name = getattr(index, 'name', None)
  
      if isinstance(index, DatetimeIndex):
          converted = index.asi8
          return IndexCol(converted, 'datetime64', _tables().Int64Col(),
                          freq=getattr(index, 'freq', None),
                          tz=getattr(index, 'tz', None),
                          index_name=index_name)
      elif isinstance(index, TimedeltaIndex):
          converted = index.asi8
          return IndexCol(converted, 'timedelta64', _tables().Int64Col(),
                          freq=getattr(index, 'freq', None),
                          index_name=index_name)
      elif isinstance(index, (Int64Index, PeriodIndex)):
          atom = _tables().Int64Col()
          # avoid to store ndarray of Period objects
          return IndexCol(index._values, 'integer', atom,
                          freq=getattr(index, 'freq', None),
                          index_name=index_name)
  
      if isinstance(index, MultiIndex):
          raise TypeError('MultiIndex not supported here!')
  
      inferred_type = lib.infer_dtype(index)
  
      values = np.asarray(index)
  
      if inferred_type == 'datetime64':
          converted = values.view('i8')
          return IndexCol(converted, 'datetime64', _tables().Int64Col(),
                          freq=getattr(index, 'freq', None),
                          tz=getattr(index, 'tz', None),
                          index_name=index_name)
      elif inferred_type == 'timedelta64':
          converted = values.view('i8')
          return IndexCol(converted, 'timedelta64', _tables().Int64Col(),
                          freq=getattr(index, 'freq', None),
                          index_name=index_name)
      elif inferred_type == 'datetime':
          converted = np.asarray([(time.mktime(v.timetuple()) +
                                   v.microsecond / 1E6) for v in values],
                                 dtype=np.float64)
          return IndexCol(converted, 'datetime', _tables().Time64Col(),
                          index_name=index_name)
      elif inferred_type == 'date':
          converted = np.asarray([v.toordinal() for v in values],
                                 dtype=np.int32)
          return IndexCol(converted, 'date', _tables().Time32Col(),
                          index_name=index_name)
      elif inferred_type == 'string':
          # atom = _tables().ObjectAtom()
          # return np.asarray(values, dtype='O'), 'object', atom
  
          converted = _convert_string_array(values, encoding)
          itemsize = converted.dtype.itemsize
          return IndexCol(
              converted, 'string', _tables().StringCol(itemsize),
              itemsize=itemsize, index_name=index_name
          )
      elif inferred_type == 'unicode':
          if format_type == 'fixed':
              atom = _tables().ObjectAtom()
              return IndexCol(np.asarray(values, dtype='O'), 'object', atom,
                              index_name=index_name)
          raise TypeError(
              "[unicode] is not supported as a in index type for [{0}] formats"
              .format(format_type)
          )
  
      elif inferred_type == 'integer':
          # take a guess for now, hope the values fit
          atom = _tables().Int64Col()
          return IndexCol(np.asarray(values, dtype=np.int64), 'integer', atom,
                          index_name=index_name)
      elif inferred_type == 'floating':
          atom = _tables().Float64Col()
          return IndexCol(np.asarray(values, dtype=np.float64), 'float', atom,
                          index_name=index_name)
      else:  # pragma: no cover
          atom = _tables().ObjectAtom()
          return IndexCol(np.asarray(values, dtype='O'), 'object', atom,
                          index_name=index_name)
  
  
  def _unconvert_index(data, kind, encoding=None):
      kind = _ensure_decoded(kind)
      if kind == u('datetime64'):
          index = DatetimeIndex(data)
      elif kind == u('timedelta64'):
          index = TimedeltaIndex(data)
      elif kind == u('datetime'):
          index = np.asarray([datetime.fromtimestamp(v) for v in data],
                             dtype=object)
      elif kind == u('date'):
          try:
              index = np.asarray(
                  [date.fromordinal(v) for v in data], dtype=object)
          except (ValueError):
              index = np.asarray(
                  [date.fromtimestamp(v) for v in data], dtype=object)
      elif kind in (u('integer'), u('float')):
          index = np.asarray(data)
      elif kind in (u('string')):
          index = _unconvert_string_array(data, nan_rep=None, encoding=encoding)
      elif kind == u('object'):
          index = np.asarray(data[0])
      else:  # pragma: no cover
          raise ValueError('unrecognized index type %s' % kind)
      return index
  
  
  def _unconvert_index_legacy(data, kind, legacy=False, encoding=None):
      kind = _ensure_decoded(kind)
      if kind == u('datetime'):
          index = lib.time64_to_datetime(data)
      elif kind in (u('integer')):
          index = np.asarray(data, dtype=object)
      elif kind in (u('string')):
          index = _unconvert_string_array(data, nan_rep=None, encoding=encoding)
      else:  # pragma: no cover
          raise ValueError('unrecognized index type %s' % kind)
      return index
  
  
  def _convert_string_array(data, encoding, itemsize=None):
      """
      we take a string-like that is object dtype and coerce to a fixed size
      string type
  
      Parameters
      ----------
      data : a numpy array of object dtype
      encoding : None or string-encoding
      itemsize : integer, optional, defaults to the max length of the strings
  
      Returns
      -------
      data in a fixed-length string dtype, encoded to bytes if needed
      """
  
      # encode if needed
      if encoding is not None and len(data):
          data = Series(data.ravel()).str.encode(
              encoding).values.reshape(data.shape)
  
      # create the sized dtype
      if itemsize is None:
          itemsize = lib.max_len_string_array(_ensure_object(data.ravel()))
  
      data = np.asarray(data, dtype="S%d" % itemsize)
      return data
  
  
  def _unconvert_string_array(data, nan_rep=None, encoding=None):
      """
      inverse of _convert_string_array
  
      Parameters
      ----------
      data : fixed length string dtyped array
      nan_rep : the storage repr of NaN, optional
      encoding : the encoding of the data, optional
  
      Returns
      -------
      an object array of the decoded data
  
      """
      shape = data.shape
      data = np.asarray(data.ravel(), dtype=object)
  
      # guard against a None encoding in PY3 (because of a legacy
      # where the passed encoding is actually None)
      encoding = _ensure_encoding(encoding)
      if encoding is not None and len(data):
  
          itemsize = lib.max_len_string_array(_ensure_object(data))
          if compat.PY3:
              dtype = "U{0}".format(itemsize)
          else:
              dtype = "S{0}".format(itemsize)
  
          if isinstance(data[0], compat.binary_type):
              data = Series(data).str.decode(encoding).values
          else:
              data = data.astype(dtype, copy=False).astype(object, copy=False)
  
      if nan_rep is None:
          nan_rep = 'nan'
  
      data = lib.string_array_replace_from_nan_rep(data, nan_rep)
      return data.reshape(shape)
  
  
  def _maybe_convert(values, val_kind, encoding):
      if _need_convert(val_kind):
          conv = _get_converter(val_kind, encoding)
          # conv = np.frompyfunc(conv, 1, 1)
          values = conv(values)
      return values
  
  
  def _get_converter(kind, encoding):
      kind = _ensure_decoded(kind)
      if kind == 'datetime64':
          return lambda x: np.asarray(x, dtype='M8[ns]')
      elif kind == 'datetime':
          return lib.convert_timestamps
      elif kind == 'string':
          return lambda x: _unconvert_string_array(x, encoding=encoding)
      else:  # pragma: no cover
          raise ValueError('invalid kind %s' % kind)
  
  
  def _need_convert(kind):
      kind = _ensure_decoded(kind)
      if kind in (u('datetime'), u('datetime64'), u('string')):
          return True
      return False
  
  
  class Selection(object):
  
      """
      Carries out a selection operation on a tables.Table object.
  
      Parameters
      ----------
      table : a Table object
      where : list of Terms (or convertable to)
      start, stop: indicies to start and/or stop selection
  
      """
  
      def __init__(self, table, where=None, start=None, stop=None, **kwargs):
          self.table = table
          self.where = where
          self.start = start
          self.stop = stop
          self.condition = None
          self.filter = None
          self.terms = None
          self.coordinates = None
  
          if is_list_like(where):
  
              # see if we have a passed coordinate like
              try:
                  inferred = lib.infer_dtype(where)
                  if inferred == 'integer' or inferred == 'boolean':
                      where = np.asarray(where)
                      if where.dtype == np.bool_:
                          start, stop = self.start, self.stop
                          if start is None:
                              start = 0
                          if stop is None:
                              stop = self.table.nrows
                          self.coordinates = np.arange(start, stop)[where]
                      elif issubclass(where.dtype.type, np.integer):
                          if ((self.start is not None and
                                  (where < self.start).any()) or
                              (self.stop is not None and
                                  (where >= self.stop).any())):
                              raise ValueError(
                                  "where must have index locations >= start and "
                                  "< stop"
                              )
                          self.coordinates = where
  
              except:
                  pass
  
          if self.coordinates is None:
  
              self.terms = self.generate(where)
  
              # create the numexpr & the filter
              if self.terms is not None:
                  self.condition, self.filter = self.terms.evaluate()
  
      def generate(self, where):
          """ where can be a : dict,list,tuple,string """
          if where is None:
              return None
  
          q = self.table.queryables()
          try:
              return Expr(where, queryables=q, encoding=self.table.encoding)
          except NameError:
              # raise a nice message, suggesting that the user should use
              # data_columns
              raise ValueError(
                  "The passed where expression: {0}\n"
                  "            contains an invalid variable reference\n"
                  "            all of the variable refrences must be a "
                  "reference to\n"
                  "            an axis (e.g. 'index' or 'columns'), or a "
                  "data_column\n"
                  "            The currently defined references are: {1}\n"
                  .format(where, ','.join(q.keys()))
              )
  
      def select(self):
          """
          generate the selection
          """
          if self.condition is not None:
              return self.table.table.read_where(self.condition.format(),
                                                 start=self.start,
                                                 stop=self.stop)
          elif self.coordinates is not None:
              return self.table.table.read_coordinates(self.coordinates)
          return self.table.table.read(start=self.start, stop=self.stop)
  
      def select_coords(self):
          """
          generate the selection
          """
          start, stop = self.start, self.stop
          nrows = self.table.nrows
          if start is None:
              start = 0
          elif start < 0:
              start += nrows
          if self.stop is None:
              stop = nrows
          elif stop < 0:
              stop += nrows
  
          if self.condition is not None:
              return self.table.table.get_where_list(self.condition.format(),
                                                     start=start, stop=stop,
                                                     sort=True)
          elif self.coordinates is not None:
              return self.coordinates
  
          return np.arange(start, stop)
  
  # utilities ###
  
  
  def timeit(key, df, fn=None, remove=True, **kwargs):
      if fn is None:
          fn = 'timeit.h5'
      store = HDFStore(fn, mode='w')
      store.append(key, df, **kwargs)
      store.close()
  
      if remove:
          os.remove(fn)
  
apis: pandas.types.common.is_categorical_dtype pandas.types.common._ensure_int64 pandas.Series pandas.compat.iteritems pandas.formats.printing.adjoin pandas.core.algorithms.unique pandas.compat.u_safe numpy.asarray pandas.tslib.get_timezone pandas.io.common._stringify_path pandas.core.categorical.Categorical.from_array pandas.computation.pytables.Expr pandas.core.common._asarray_tuplesafe pandas.core.config.config_prefix pandas.types.common._ensure_object pandas.core.config.get_option pandas.sparse.array.BlockIndex pandas.types.common.is_datetime64tz_dtype pandas.types.common.is_list_like pandas.lib.time64_to_datetime pandas.tseries.api.DatetimeIndex pandas.lib.infer_dtype pandas.tseries.api.PeriodIndex._simple_new pandas.computation.pytables.maybe_expression pandas.lib.string_array_replace_from_nan_rep pandas.isnull pandas.types.missing.array_equivalent pandas.core.index._ensure_index numpy.tile pandas.tseries.api.DatetimeIndex._simple_new pandas.types.common.is_timedelta64_dtype pandas.tools.merge.concat pandas.compat.filter pandas.types.common.is_datetime64_dtype numpy.repeat numpy.arange pandas.MultiIndex.from_tuples pandas.formats.printing.pprint_thing numpy.prod pandas.types.common._ensure_platform_int pandas.Index pandas.core.config.is_one_of_factory pandas.lib.fast_unique numpy.array pandas.core.algorithms.match pandas.core.config.register_option numpy.empty pandas.MultiIndex.from_arrays pandas.sparse.array.IntIndex pandas.core.internals._factor_indexer pandas.DataFrame pandas.compat.lrange pandas.sparse.api.SparseDataFrame pandas.compat.range pandas.core.internals._block_shape pandas.core.internals.BlockManager pandas.tseries.tdi.TimedeltaIndex pandas.MultiIndex