merge_machine/normalizer.py
def read_csv(self, file):
        '''
        Read CSV and perform inference on streaming file
        
        /!\ Use only on upload. Otherwise, look into _static_load_data in parent
        class        
        '''
        ENCODINGS = ['utf-8', 'ISO-8859-1', 'windows-1252']
        
        first_lines = b''.join([file.readline() for _ in range(self.CHUNKSIZE)])
        could_read = False    
        #for sep_arg in [None, ',']: # Fix for pandas that can't find separators for single columns
        for encoding in ENCODINGS:
            best_sep = None
            best_sep_num_cols = 0    
            for sep in [';', ',', '\t']:
                try:
                    first_lines_io = io.BytesIO(first_lines)
                    tab_part = pd.read_csv(first_lines_io, 
                                           sep=sep, 
                                           encoding=encoding, 
                                           dtype=str)
                    columns = tab_part.columns
                    print(len(columns))
                    
                    if len(columns) >= best_sep_num_cols:
                        best_sep = sep
                        best_columns = columns
                        best_sep_num_cols = len(columns)
                    could_read = True
                    
                except Exception as e:
                    logging.info(e)
                    
            # If one or more separators were valid with this encoding
            if could_read:    
                break
       
        else:
            raise Exception('Separator and/or Encoding not detected. Try uploading' \
                          + ' a csv with "," as separator with utf-8 encoding')                

        if sep != best_sep:
            first_lines_io = io.BytesIO(first_lines)
            tab_part = pd.read_csv(first_lines_io, 
                                   sep=best_sep, 
                                   encoding=encoding, 
                                   dtype=str)   
        try:
            
         
            tab_next = pd.read_csv(file, 
                                 sep=best_sep, 
                                 encoding=encoding,
                                 dtype=str,
                                 header=None,
                                 chunksize=self.CHUNKSIZE)
            tab = itertools.chain([tab_part], tab_next)
        except:
            tab = itertools.chain([tab_part])  
        
        
        
        tab = (self._clean_header(self.assign_columns(tab_part, best_columns)) for tab_part in tab)
        
        print(tab, best_sep, encoding, self._clean_column_names(best_columns))
        return tab, best_sep, encoding, self._clean_column_names(best_columns)

        Read CSV and perform inference on streaming file
        
        /!\ Use only on upload. Otherwise, look into _static_load_data in parent
        class        
        
Read CSV and perform inference on streaming file
\ Use only on upload. Otherwise, look into _static_load_data in parent
class
Read CSV and perform inference on streaming file \ Use only on upload . Otherwise look into _static_load_data in parent class
def read_csv(self, file):
        ENCODINGS = ['utf-8', 'ISO-8859-1', 'windows-1252']
        first_lines = b''.join([file.readline() for _ in range(self.CHUNKSIZE)])
        could_read = False    
        for encoding in ENCODINGS:
            best_sep = None
            best_sep_num_cols = 0    
            for sep in [';', ',', '\t']:
                try:
                    first_lines_io = io.BytesIO(first_lines)
                    tab_part = pd.read_csv(first_lines_io, 
                                           sep=sep, 
                                           encoding=encoding, 
                                           dtype=str)
                    columns = tab_part.columns
                    print(len(columns))
                    if len(columns) >= best_sep_num_cols:
                        best_sep = sep
                        best_columns = columns
                        best_sep_num_cols = len(columns)
                    could_read = True
                except Exception as e:
                    logging.info(e)
            if could_read:    
                break
        else:
            raise Exception('Separator and/or Encoding not detected. Try uploading' \
                          + ' a csv with "," as separator with utf-8 encoding')                
        if sep != best_sep:
            first_lines_io = io.BytesIO(first_lines)
            tab_part = pd.read_csv(first_lines_io, 
                                   sep=best_sep, 
                                   encoding=encoding, 
                                   dtype=str)   
        try:
            tab_next = pd.read_csv(file, 
                                 sep=best_sep, 
                                 encoding=encoding,
                                 dtype=str,
                                 header=None,
                                 chunksize=self.CHUNKSIZE)
            tab = itertools.chain([tab_part], tab_next)
        except:
            tab = itertools.chain([tab_part])  
        tab = (self._clean_header(self.assign_columns(tab_part, best_columns)) for tab_part in tab)
        print(tab, best_sep, encoding, self._clean_column_names(best_columns))
        return tab, best_sep, encoding, self._clean_column_names(best_columns)
def read_csv ( self , file ) : ENCODINGS = [ 'utf-8' , 'ISO-8859-1' , 'windows-1252' ] first_lines = b'' . join ( [ file . readline ( ) for _ in range ( self . CHUNKSIZE ) ] ) could_read = False for encoding in ENCODINGS : best_sep = None best_sep_num_cols = 0 for sep in [ ';' , ',' , '\t' ] : try : first_lines_io = io . BytesIO ( first_lines ) tab_part = pd . read_csv ( first_lines_io , sep = sep , encoding = encoding , dtype = str ) columns = tab_part . columns print ( len ( columns ) ) if len ( columns ) >= best_sep_num_cols : best_sep = sep best_columns = columns best_sep_num_cols = len ( columns ) could_read = True except Exception as e : logging . info ( e ) if could_read : break else : raise Exception ( 'Separator and/or Encoding not detected. Try uploading' + ' a csv with "," as separator with utf-8 encoding' ) if sep != best_sep : first_lines_io = io . BytesIO ( first_lines ) tab_part = pd . read_csv ( first_lines_io , sep = best_sep , encoding = encoding , dtype = str ) try : tab_next = pd . read_csv ( file , sep = best_sep , encoding = encoding , dtype = str , header = None , chunksize = self . CHUNKSIZE ) tab = itertools . chain ( [ tab_part ] , tab_next ) except : tab = itertools . chain ( [ tab_part ] ) tab = ( self . _clean_header ( self . assign_columns ( tab_part , best_columns ) ) for tab_part in tab ) print ( tab , best_sep , encoding , self . _clean_column_names ( best_columns ) ) return tab , best_sep , encoding , self . _clean_column_names ( best_columns )
Read CSV and perform inference on streaming file
\ Use only on upload.
Read CSV and perform inference on streaming file \ Use only on upload .
'''
        Read CSV and perform inference on streaming file
        
        /!\ Use only on upload. Otherwise, look into _static_load_data in parent
        class        
        ''' #for sep_arg in [None, ',']: # Fix for pandas that can't find separators for single columns # If one or more separators were valid with this encoding
self
file