nl_tokens: protected function : Strategy of classifying the shortest segments with a single class .
pl_tokens: function ( words , accumulatedClasses , explain , explanations ) { var currentStart = 0 ; for ( var currentEnd = 1 ; currentEnd <= words . length ; ++ currentEnd ) { var segment = words . slice ( currentStart , currentEnd ) . join ( " " ) ; var segmentClassesWithExplain = this . classifySegment ( segment , explain ) ; var segmentClasses = ( segmentClassesWithExplain . classes ? segmentClassesWithExplain . classes : segmentClassesWithExplain ) ; if ( segmentClasses . length == 1 ) { // greedy algorithm: found a section with a single class - cut it and go on accumulatedClasses [ segmentClasses [ 0 ] ] = <mask> ; currentStart = currentEnd ; if ( explain > 0 ) { explanations . push ( segment ) ; explanations . push ( segmentClassesWithExplain . explanation ) ; } ; } } }