content: """miscellaneous file manipulation utilities
  
  """
  
  import numpy as np
  import pysal as ps
  import pandas as pd
  
  def check_dups(li):
      """checks duplicates in list of ID values
         ID values must be read in as a list
  
         __author__  = "Luc Anselin <luc.anselin@asu.edu> "
         
         Arguments
         ---------
         li      : list of ID values
         
         Returns
         -------
         a list with the duplicate IDs
      """
      return list(set([x for x in li if li.count(x) > 1]))
      
      
  def dbfdups(dbfpath,idvar):
      """checks duplicates in a dBase file
         ID variable must be specified correctly
  
         __author__  = "Luc Anselin <luc.anselin@asu.edu> " 
         
         Arguments
         ---------
         dbfpath  : file path to dBase file
         idvar    : ID variable in dBase file
         
         Returns
         -------
         a list with the duplicate IDs
      """
      db = ps.open(dbfpath,'r')
      li = db.by_col(idvar)
      return list(set([x for x in li if li.count(x) > 1]))   
      
  
  def df2dbf(df, dbf_path, my_specs=None):
      '''
      Convert a pandas.DataFrame into a dbf.
  
      __author__  = "Dani Arribas-Bel <darribas@asu.edu>, Luc Anselin <luc.anselin@asu.edu>"
      ...
  
      Arguments
      ---------
      df          : DataFrame
                    Pandas dataframe object to be entirely written out to a dbf
      dbf_path    : str
                    Path to the output dbf. It is also returned by the function
      my_specs    : list
                    List with the field_specs to use for each column.
                    Defaults to None and applies the following scheme:
                      * int: ('N', 14, 0) - for all ints
                      * float: ('N', 14, 14) - for all floats
                      * str: ('C', 14, 0) - for string, object and category
                    with all variants for different type sizes
                    
      Note: use of dtypes.name may not be fully robust, but preferred apprach of using
      isinstance seems too clumsy
      '''
      if my_specs:
          specs = my_specs
      else:
          """
          type2spec = {int: ('N', 20, 0),
                       np.int64: ('N', 20, 0),
                       np.int32: ('N', 20, 0),
                       np.int16: ('N', 20, 0),
                       np.int8: ('N', 20, 0),
                       float: ('N', 36, 15),
                       np.float64: ('N', 36, 15),
                       np.float32: ('N', 36, 15),
                       str: ('C', 14, 0)
                       }
          types = [type(df[i].iloc[0]) for i in df.columns]
          """
          # new approach using dtypes.name to avoid numpy name issue in type
          type2spec = {'int': ('N', 20, 0),
                       'int8': ('N', 20, 0),
                       'int16': ('N', 20, 0),
                       'int32': ('N', 20, 0),
                       'int64': ('N', 20, 0),
                       'float': ('N', 36, 15),
                       'float32': ('N', 36, 15),
                       'float64': ('N', 36, 15),
                       'str': ('C', 14, 0),
                       'object': ('C', 14, 0),
                       'category': ('C', 14, 0)
                       }
          types = [df[i].dtypes.name for i in df.columns]
          specs = [type2spec[t] for t in types]
      db = ps.open(dbf_path, 'w')
      db.header = list(df.columns)
      db.field_spec = specs
      for i, row in df.T.iteritems():
          db.write(row)
      db.close()
      return dbf_path
  
  def dbf2df(dbf_path, index=None, cols=False, incl_index=False):
      '''
      Read a dbf file as a pandas.DataFrame, optionally selecting the index
      variable and which columns are to be loaded.
  
      __author__  = "Dani Arribas-Bel <darribas@asu.edu> "
      ...
  
      Arguments
      ---------
      dbf_path    : str
                    Path to the DBF file to be read
      index       : str
                    Name of the column to be used as the index of the DataFrame
      cols        : list
                    List with the names of the columns to be read into the
                    DataFrame. Defaults to False, which reads the whole dbf
      incl_index  : Boolean
                    If True index is included in the DataFrame as a
                    column too. Defaults to False
  
      Returns
      -------
      df          : DataFrame
                    pandas.DataFrame object created
      '''
      db = ps.open(dbf_path)
      if cols:
          if incl_index:
              cols.append(index)
          vars_to_read = cols
      else:
          vars_to_read = db.header
      data = dict([(var, db.by_col(var)) for var in vars_to_read])
      if index:
          index = db.by_col(index)
          db.close()
          return pd.DataFrame(data, index=index, columns=vars_to_read)
      else:
          db.close()
          return pd.DataFrame(data,columns=vars_to_read)
  
      
  def dbfjoin(dbf1_path,dbf2_path,out_path,joinkey1,joinkey2):
      '''
      Wrapper function to merge two dbf files into a new dbf file. 
  
      __author__  = "Luc Anselin <luc.anselin@asu.edu> "
      
      Uses dbf2df and df2dbf to read and write the dbf files into a pandas
      DataFrame. Uses all default settings for dbf2df and df2dbf (see docs
      for specifics).
      ...
  
      Arguments
      ---------
  
      dbf1_path   : str
                    Path to the first (left) dbf file
      dbf2_path   : str
                    Path to the second (right) dbf file
      out_path    : str
                    Path to the output dbf file (returned by the function)
      joinkey1    : str
                    Variable name for the key in the first dbf. Must be specified.
                    Key must take unique values.
      joinkey2    : str
                    Variable name for the key in the second dbf. Must be specified.
                    Key must take unique values.
                    
      Returns
      -------
      dbfpath     : path to output file
      
      '''
      df1 = dbf2df(dbf1_path,index=joinkey1)
      df2 = dbf2df(dbf2_path,index=joinkey2)
      dfbig = pd.merge(df1,df2,left_on=joinkey1,right_on=joinkey2,sort=False)
      dp = df2dbf(dfbig,out_path)
      return dp
  
  
  def dta2dbf(dta_path,dbf_path):
      """
      Wrapper function to convert a stata dta file into a dbf file. 
  
      __author__  = "Luc Anselin <luc.anselin@asu.edu> "
      
      Uses df2dbf to write the dbf files from a pandas
      DataFrame. Uses all default settings for df2dbf (see docs
      for specifics).
      ...
  
      Arguments
      ---------
  
      dta_path    : str
                    Path to the Stata dta file
      dbf_path    : str
                    Path to the output dbf file
                    
      Returns
      -------
      dbf_path    : path to output file
      """
      
      db = pd.read_stata(dta_path)
      dp = df2dbf(db,dbf_path)
      return dp    
  