def _get_grammar_errors(self,pos,text,tokens):
word_counts = [max(len(t),<NUM_LIT:1>) for t in tokens]<EOL>good_pos_tags = []<EOL>min_pos_seq=<NUM_LIT:2><EOL>max_pos_seq=<NUM_LIT:4><EOL>bad_pos_positions=[]<EOL>for i in xrange(<NUM_LIT:0>, len(text)):<EOL><INDENT>pos_seq = [tag[<NUM_LIT:1>] for tag in pos[i]]<EOL>pos_ngrams = util_functions.ngrams(pos_seq, min_pos_seq, max_pos_seq)<EOL>long_pos_ngrams=[z for z in pos_ngrams if z.count('<STR_LIT:U+0020>')==(max_pos_seq-<NUM_LIT:1>)]<EOL>bad_pos_tuples=[[z,z+max_pos_seq] for z in xrange(<NUM_LIT:0>,len(long_pos_ngrams)) if long_pos_ngrams[z] not in self._good_pos_ngrams]<EOL>bad_pos_tuples.sort(key=operator.itemgetter(<NUM_LIT:1>))<EOL>to_delete=[]<EOL>for m in reversed(xrange(len(bad_pos_tuples)-<NUM_LIT:1>)):<EOL><INDENT>start, end = bad_pos_tuples[m]<EOL>for j in xrange(m+<NUM_LIT:1>, len(bad_pos_tuples)):<EOL><INDENT>lstart, lend = bad_pos_tuples[j]<EOL>if lstart >= start and lstart <= end:<EOL><INDENT>bad_pos_tuples[m][<NUM_LIT:1>]=bad_pos_tuples[j][<NUM_LIT:1>]<EOL>to_delete.append(j)<EOL><DEDENT><DEDENT><DEDENT>fixed_bad_pos_tuples=[bad_pos_tuples[z] for z in xrange(<NUM_LIT:0>,len(bad_pos_tuples)) if z not in to_delete]<EOL>bad_pos_positions.append(fixed_bad_pos_tuples)<EOL>overlap_ngrams = [z for z in pos_ngrams if z in self._good_pos_ngrams]<EOL>if (len(pos_ngrams)-len(overlap_ngrams))><NUM_LIT:0>:<EOL><INDENT>divisor=len(pos_ngrams)/len(pos_seq)<EOL><DEDENT>else:<EOL><INDENT>divisor=<NUM_LIT:1><EOL><DEDENT>if divisor == <NUM_LIT:0>:<EOL><INDENT>divisor=<NUM_LIT:1><EOL><DEDENT>good_grammar_ratio = (len(pos_ngrams)-len(overlap_ngrams))/divisor<EOL>good_pos_tags.append(good_grammar_ratio)<EOL><DEDENT>return good_pos_tags,bad_pos_positions<EOL>
Internal function to get the number of grammar errors in given text
pos - part of speech tagged text (list)
text - normal text (list)
tokens - list of lists of tokenized text