nl_tokens: filterNames () will process a dist / names_all . json file splitting the data up into 2 files : dist / names_keep . json - candidates for suggestion presets dist / names_discard . json - everything else The file format is identical to the names_all . json file : key / value|name : count shop / coffee|Starbucks : 8284
pl_tokens: function filterNames ( ) { console . <mask> ( '\nfiltering names' ) ; console . time ( colors . green ( 'names filtered' ) ) ; // Start clean shell . rm ( '-f' , [ 'dist/names_keep.json' , 'dist/names_discard.json' ] ) ; // filter by keepTags (move from discard -> keep) filters . keepTags . forEach ( s => { let re = new RegExp ( s , 'i' ) ; for ( let key in discard ) { let tag = key . split ( '|' , 2 ) [ 0 ] ; if ( re . test ( tag ) ) { keep [ key ] = discard [ key ] ; delete discard [ key ] ; } } } ) ; // filter by discardKeys (move from keep -> discard) filters . discardKeys . forEach ( s => { let re = new RegExp ( s , 'i' ) ; for ( let key in keep ) { if ( re . test ( key ) ) { discard [ key ] = keep [ key ] ; delete keep [ key ] ; } } } ) ; // filter by discardNames (move from keep -> discard) filters . discardNames . forEach ( s => { let re = new RegExp ( s , 'i' ) ; for ( let key in keep ) { let name = key . split ( '|' , 2 ) [ 1 ] ; if ( re . test ( name ) ) { discard [ key ] = keep [ key ] ; delete keep [ key ] ; } } } ) ; fs . writeFileSync ( 'dist/names_discard.json' , stringify ( sort ( discard ) ) ) ; fs . writeFileSync ( 'dist/names_keep.json' , stringify ( sort ( keep ) ) ) ; console . timeEnd ( colors . green ( 'names filtered' ) ) ; }