nl_tokens: protected function : Strategy of classifying the longest segments with a single class .
pl_tokens: function ( words , accumulatedClasses , explain , explanations ) { var currentStart = 0 ; var segment = null ; var segmentClassesWithExplain = null ; var segmentClasses = null ; for ( var currentEnd = 1 ; currentEnd <= words . length ; ++ currentEnd ) { var nextSegment = words . slice ( currentStart , currentEnd ) . join ( " " ) ; var nextSegmentClassesWithExplain = this . classifySegment ( nextSegment , explain ) ; var nextSegmentClasses = ( nextSegmentClassesWithExplain . classes ? nextSegmentClassesWithExplain . classes : nextSegmentClassesWithExplain ) ; //console.log("\t"+JSON.stringify(nextSegment) +" -> "+nextSegmentClasses) nextSegmentClasses . sort ( ) ; if ( segmentClasses && segmentClasses . length == 1 && ( nextSegmentClasses . length > 1 || ! _ ( nextSegmentClasses ) . isEqual ( segmentClasses ) ) ) { // greedy algorithm: found a section with a single class - cut it and go on accumulatedClasses [ segmentClasses [ 0 ] ] = true ; currentStart = currentEnd - 1 ; if ( explain > 0 ) { explanations . push ( segment ) ; explanations . push ( segmentClassesWithExplain . explanation ) ; } ; } segment = nextSegment ; segmentClassesWithExplain = nextSegmentClassesWithExplain ; segmentClasses = nextSegmentClasses ; } // add the classes of the last section: for ( var i in segmentClasses ) accumulatedClasses [ segmentClasses [ i ] ] = true ; if ( explain > 0 ) { explanations . <mask> ( segment ) ; explanations . push ( segmentClassesWithExplain . explanation ) ; } ; /*if (words.length>20)  {
  			console.dir(explanations);
  			process.exit(1);
  		}*/ }