merge_machine/abstract_data_project.py
def _ES_res_to_pandas(self, res, columns, thresh=None):
        """Return the result of an elasticsearch query as a pandas DataFrame.
        
        Parameters
        ----------
        res: list
            Result of a bulk Elasticsearch query.
        columns: list of str
            Columns to load
        thresh: None or float
             If thresh is not None, filter out all data from the referential if
             the __CONFIDENCE is not above the threshold
        
        """
        
        sources = [x['hits']['hits'][0]['_source'] for x in res['responses']]
        if columns is not None:
            if isinstance(columns, list):
                sources = [{key: val for key, val in x.items() if key in columns} for x in sources]
            elif callable(columns):
                sources = [{key: val for key, val in x.items() if columns(key)} for x in sources]
                assert sources
                columns = list(sources[0].keys())
            else:
                raise TypeError('Variable "columns" should be list or callable or None.')
                
        dtype = {col: self._choose_dtype(col) for col in columns}
        
        ids = [x['hits']['hits'][0]['_id'] for x in res['responses']]

        tab = pd.DataFrame(sources, index=ids)[columns]
        
        # Workaround for pandas bug: https://stackoverflow.com/a/38750433/7856919
        for k, v in dtype.items():
            if v == bool:
                tab[k] = tab[k].astype(str) == 'True'
            else:
                tab[k] = tab[k].astype(v)
        
        if thresh is not None:
            # Select rows that are not above the threshold 
            sel = ~(tab['__CONFIDENCE'] >= thresh)
            columns_to_remove = [x for x in tab.columns if '__' in x]
            tab.loc[sel, columns_to_remove] = pd.np.nan
            
        # Dirty fix for np.nan that transforms dtype bool into float.
        tab['__IS_MATCH'].fillna(False, inplace=True)
        tab['__IS_MATCH'] = tab['__IS_MATCH'].astype(bool)    

        return tab
Return the result of an elasticsearch query as a pandas DataFrame.
        
        Parameters
        ----------
        res: list
            Result of a bulk Elasticsearch query.
        columns: list of str
            Columns to load
        thresh: None or float
             If thresh is not None, filter out all data from the referential if
             the __CONFIDENCE is not above the threshold
        
        
Return the result of an elasticsearch query as a pandas DataFrame.
Parameters

list
Result of a bulk Elasticsearch query.
columns: list of str
Columns to load
thresh: None or float
If thresh is not None, filter out all data from the referential if
the __CONFIDENCE is not above the threshold
Return the result of an elasticsearch query as a pandas DataFrame . Parameters list Result of a bulk Elasticsearch query . columns : list of str Columns to load thresh : None or float If thresh is not None filter out all data from the referential if the __CONFIDENCE is not above the threshold
def _ES_res_to_pandas(self, res, columns, thresh=None):
        sources = [x['hits']['hits'][0]['_source'] for x in res['responses']]
        if columns is not None:
            if isinstance(columns, list):
                sources = [{key: val for key, val in x.items() if key in columns} for x in sources]
            elif callable(columns):
                sources = [{key: val for key, val in x.items() if columns(key)} for x in sources]
                assert sources
                columns = list(sources[0].keys())
            else:
                raise TypeError('Variable "columns" should be list or callable or None.')
        dtype = {col: self._choose_dtype(col) for col in columns}
        ids = [x['hits']['hits'][0]['_id'] for x in res['responses']]
        tab = pd.DataFrame(sources, index=ids)[columns]
        for k, v in dtype.items():
            if v == bool:
                tab[k] = tab[k].astype(str) == 'True'
            else:
                tab[k] = tab[k].astype(v)
        if thresh is not None:
            sel = ~(tab['__CONFIDENCE'] >= thresh)
            columns_to_remove = [x for x in tab.columns if '__' in x]
            tab.loc[sel, columns_to_remove] = pd.np.nan
        tab['__IS_MATCH'].fillna(False, inplace=True)
        tab['__IS_MATCH'] = tab['__IS_MATCH'].astype(bool)    
        return tab
def _ES_res_to_pandas ( self , res , columns , thresh = None ) : sources = [ x [ 'hits' ] [ 'hits' ] [ 0 ] [ '_source' ] for x in res [ 'responses' ] ] if columns is not None : if isinstance ( columns , list ) : sources = [ { key : val for key , val in x . items ( ) if key in columns } for x in sources ] elif callable ( columns ) : sources = [ { key : val for key , val in x . items ( ) if columns ( key ) } for x in sources ] assert sources columns = list ( sources [ 0 ] . keys ( ) ) else : raise TypeError ( 'Variable "columns" should be list or callable or None.' ) dtype = { col : self . _choose_dtype ( col ) for col in columns } ids = [ x [ 'hits' ] [ 'hits' ] [ 0 ] [ '_id' ] for x in res [ 'responses' ] ] tab = pd . DataFrame ( sources , index = ids ) [ columns ] for k , v in dtype . items ( ) : if v == bool : tab [ k ] = tab [ k ] . astype ( str ) == 'True' else : tab [ k ] = tab [ k ] . astype ( v ) if thresh is not None : sel = ~ ( tab [ '__CONFIDENCE' ] >= thresh ) columns_to_remove = [ x for x in tab . columns if '__' in x ] tab . loc [ sel , columns_to_remove ] = pd . np . nan tab [ '__IS_MATCH' ] . fillna ( False , inplace = True ) tab [ '__IS_MATCH' ] = tab [ '__IS_MATCH' ] . astype ( bool ) return tab
Return the result of an elasticsearch query as a pandas DataFrame.
Return the result of an elasticsearch query as a pandas DataFrame .
"""Return the result of an elasticsearch query as a pandas DataFrame.
        
        Parameters
        ----------
        res: list
            Result of a bulk Elasticsearch query.
        columns: list of str
            Columns to load
        thresh: None or float
             If thresh is not None, filter out all data from the referential if
             the __CONFIDENCE is not above the threshold
        
        """ # Workaround for pandas bug: https://stackoverflow.com/a/38750433/7856919 # Select rows that are not above the threshold  # Dirty fix for np.nan that transforms dtype bool into float.
self
res
columns
thresh