merge_machine/linker.py
def gen_default_columns_to_index(self):
        '''Generate the dict specifying the analyzers to use for each column 
        while indexing in Elasticsearch. 
        
        This method only takes into account the reference file as to avoid 
        re-indexing when using the same reference with a different source. This 
        could change if partial re-indexing is implemented.
            
        Returns
        -------
        columns_to_index: dict associating sets of str (values) to str (keys)
            A dict indicating what Elasticsearch analyzers to use on each column
            type during indexing.
        '''
        INDEX_ALL = False # Whether or not to index all selected columns of the file
        
        def temp(column_types, col):
            """Return the type specific default analyzer for a column or return 
            all default analyzers if type is not specified or could not be found.
            """
            return DEFAULT_ANALYZERS_TYPE.get(column_types.get(col), DEFAULT_ANALYZERS)
        
        # Try fetching referential column types
        # TODO: dangerous if config was not confirmed by user...
        column_types = self.ref.read_config_data('recode_types', 'infered_config.json')

        # Read column match data
        column_matches = self.read_config_data('es_linker', 'column_matches.json')  
        if not column_matches:
            raise RuntimeError('No column matches to read from')

        # Add default analyzer for columns that are exact matches
        
        if INDEX_ALL:
            list_of_columns_exact = self.ref.metadata['column_tracker']['selected']
            list_of_columns_exact = {x for x in list_of_columns_exact if '__' not in x}
        else:
            exact_matches = filter(lambda m: m.get('exact_only', False), column_matches)
            list_of_columns_exact = {y for z in [[m['ref']] if isinstance(m['ref'], str) \
                                    else m['ref'] for m in exact_matches] for y in z}
            
        columns_to_index = {col: {} for col in list_of_columns_exact}
        
        # Add analyzers for columns that are non-exact matches
        # NB: Preserve order to not overwrite columns_to_index of non-exact
        non_exact_matches = filter(lambda m: not m.get('exact_only', False), column_matches)
        list_of_columns_non_exact = {y for z in [[m['ref']] if isinstance(m['ref'], str) \
                                else m['ref'] for m in non_exact_matches] for y in z}
        columns_to_index.update({col: temp(column_types, col) for col in list_of_columns_non_exact})
        
        # Add all columns that were selected
        for col in self.ref.metadata['column_tracker']['selected']:
            columns_to_index.setdefault(col, {})
        
        print('columns_to_index:')
        print(columns_to_index)
        
        return columns_to_index
Generate the dict specifying the analyzers to use for each column 
        while indexing in Elasticsearch. 
        
        This method only takes into account the reference file as to avoid 
        re-indexing when using the same reference with a different source. This 
        could change if partial re-indexing is implemented.
            
        Returns
        -------
        columns_to_index: dict associating sets of str (values) to str (keys)
            A dict indicating what Elasticsearch analyzers to use on each column
            type during indexing.
        
Generate the dict specifying the analyzers to use for each column
while indexing in Elasticsearch.

This method only takes into account the reference file as to avoid
re-indexing when using the same reference with a different source. This
could change if partial re-indexing is implemented.

Returns

dict associating sets of str (values) to str (keys)
A dict indicating what Elasticsearch analyzers to use on each column
type during indexing.
Generate the dict specifying the analyzers to use for each column while indexing in Elasticsearch . This method only takes into account the reference file as to avoid re - indexing when using the same reference with a different source . This could change if partial re - indexing is implemented . Returns dict associating sets of str ( values ) to str ( keys ) A dict indicating what Elasticsearch analyzers to use on each column type during indexing .
def gen_default_columns_to_index(self):
        INDEX_ALL = False 
        def temp(column_types, col):
            return DEFAULT_ANALYZERS_TYPE.get(column_types.get(col), DEFAULT_ANALYZERS)
        column_types = self.ref.read_config_data('recode_types', 'infered_config.json')
        column_matches = self.read_config_data('es_linker', 'column_matches.json')  
        if not column_matches:
            raise RuntimeError('No column matches to read from')
        if INDEX_ALL:
            list_of_columns_exact = self.ref.metadata['column_tracker']['selected']
            list_of_columns_exact = {x for x in list_of_columns_exact if '__' not in x}
        else:
            exact_matches = filter(lambda m: m.get('exact_only', False), column_matches)
            list_of_columns_exact = {y for z in [[m['ref']] if isinstance(m['ref'], str) \
                                    else m['ref'] for m in exact_matches] for y in z}
        columns_to_index = {col: {} for col in list_of_columns_exact}
        non_exact_matches = filter(lambda m: not m.get('exact_only', False), column_matches)
        list_of_columns_non_exact = {y for z in [[m['ref']] if isinstance(m['ref'], str) \
                                else m['ref'] for m in non_exact_matches] for y in z}
        columns_to_index.update({col: temp(column_types, col) for col in list_of_columns_non_exact})
        for col in self.ref.metadata['column_tracker']['selected']:
            columns_to_index.setdefault(col, {})
        print('columns_to_index:')
        print(columns_to_index)
        return columns_to_index
def gen_default_columns_to_index ( self ) : INDEX_ALL = False def temp ( column_types , col ) : """Return the type specific default analyzer for a column or return 
            all default analyzers if type is not specified or could not be found.
            """ return DEFAULT_ANALYZERS_TYPE . get ( column_types . get ( col ) , DEFAULT_ANALYZERS ) column_types = self . ref . read_config_data ( 'recode_types' , 'infered_config.json' ) column_matches = self . read_config_data ( 'es_linker' , 'column_matches.json' ) if not column_matches : raise RuntimeError ( 'No column matches to read from' ) if INDEX_ALL : list_of_columns_exact = self . ref . metadata [ 'column_tracker' ] [ 'selected' ] list_of_columns_exact = { x for x in list_of_columns_exact if '__' not in x } else : exact_matches = filter ( lambda m : m . get ( 'exact_only' , False ) , column_matches ) list_of_columns_exact = { y for z in [ [ m [ 'ref' ] ] if isinstance ( m [ 'ref' ] , str ) else m [ 'ref' ] for m in exact_matches ] for y in z } columns_to_index = { col : { } for col in list_of_columns_exact } non_exact_matches = filter ( lambda m : not m . get ( 'exact_only' , False ) , column_matches ) list_of_columns_non_exact = { y for z in [ [ m [ 'ref' ] ] if isinstance ( m [ 'ref' ] , str ) else m [ 'ref' ] for m in non_exact_matches ] for y in z } columns_to_index . update ( { col : temp ( column_types , col ) for col in list_of_columns_non_exact } ) for col in self . ref . metadata [ 'column_tracker' ] [ 'selected' ] : columns_to_index . setdefault ( col , { } ) print ( 'columns_to_index:' ) print ( columns_to_index ) return columns_to_index
Generate the dict specifying the analyzers to use for each column
while indexing in Elasticsearch.
Generate the dict specifying the analyzers to use for each column while indexing in Elasticsearch .
'''Generate the dict specifying the analyzers to use for each column 
        while indexing in Elasticsearch. 
        
        This method only takes into account the reference file as to avoid 
        re-indexing when using the same reference with a different source. This 
        could change if partial re-indexing is implemented.
            
        Returns
        -------
        columns_to_index: dict associating sets of str (values) to str (keys)
            A dict indicating what Elasticsearch analyzers to use on each column
            type during indexing.
        ''' # Whether or not to index all selected columns of the file """Return the type specific default analyzer for a column or return 
            all default analyzers if type is not specified or could not be found.
            """ # Try fetching referential column types # TODO: dangerous if config was not confirmed by user... # Read column match data # Add default analyzer for columns that are exact matches # Add analyzers for columns that are non-exact matches # NB: Preserve order to not overwrite columns_to_index of non-exact # Add all columns that were selected
self