nl_tokens: protected function : Strategy of finding the cheapest segmentation ( - most probable segmentation ) using a dynamic programming algorithm . Based on : Morbini Fabrizio Sagae Kenji . Joint Identification and Segmentation of Domain - Specific Dialogue Acts for Conversational Dialogue Systems . ACL - HLT 2011 http : // www . citeulike . org / user / erelsegal - halevi / article / 10259046
pl_tokens: function ( words , accumulatedClasses , explain , explanations ) { // Calculate the cost of classification of the segment from i to j. // (Cost = - log probability). var segmentClassificationCosts = [ ] ; // best cost to classify segment [i,j] for ( var start = 0 ; start <= words . length ; ++ start ) { segmentClassificationCosts [ start ] = [ ] ; for ( var end = 0 ; end < start ; ++ end ) segmentClassificationCosts [ start ] [ end ] = Infinity ; segmentClassificationCosts [ start ] [ start ] = 0 ; for ( var end = start + 1 ; end <= words . length ; ++ end ) { var segment = words . slice ( start , end ) . join ( " " ) ; var classification = this . bestClassOfSegment ( segment ) ; segmentClassificationCosts [ start ] [ end ] = - Math . log ( classification . probability ) ; } } //console.log(words+":  ");		console.log("segmentClassificationCosts");		console.dir(segmentClassificationCosts); var cheapest_paths = require ( "graph-paths" ) . cheapest_paths ; cheapestSegmentClassificationCosts = cheapest_paths ( segmentClassificationCosts , 0 ) ; cheapestSentenceClassificationCost = cheapestSegmentClassificationCosts [ words . length ] ; if ( ! cheapestSentenceClassificationCost ) throw new Error ( "cheapestSegmentClassificationCosts[" + words . <mask> + "] is empty" ) ; //console.log("cheapestSentenceClassificationCost");		console.dir(cheapestSentenceClassificationCost); var cheapestClassificationPath = cheapestSentenceClassificationCost . path ; explanations . push ( cheapestSentenceClassificationCost ) ; for ( var i = 0 ; i < cheapestClassificationPath . length - 1 ; ++ i ) { var segment = words . slice ( cheapestClassificationPath [ i ] , cheapestClassificationPath [ i + 1 ] ) . join ( " " ) ; //console.log(segment+":  "); var segmentCategoryWithExplain = this . classifySegment ( segment , explain ) ; //console.dir(segmentCategoryWithExplain); var segmentCategory = ( segmentCategoryWithExplain . category ? segmentCategoryWithExplain . category : segmentCategoryWithExplain ) ; accumulatedClasses [ segmentCategory ] = true ; if ( explain > 0 ) { explanations . push ( segment ) ; explanations . push ( segmentCategoryWithExplain . explanation ) ; } ; } }