M1-Arborator-NaijaSUD/arborator/lib/treebankfiles.py
def correcttokens(tree, origdic, strict=True):
	"""
	tree is checked
	origdic is the parser's annotation whose tokens are relevant
	strict: first trying to match exactly the parser's tokens
	"""
	if debug: 
		print "\n=================== start correcttokens tree"
		if strict:print "strict!!!"
		else: print "not strict!!!!"
		for i,d in  tree.iteritems():print i,d[tokenname],d 
		print "with origdic"
		for i,d in  origdic.iteritems():print i,d[tokenname],d 
	
	newtree={}
	try:
		for nr in sorted(origdic):
			#print origdic[nr]
			#print nr, origdic.get(nr,{tokenname:"___________verystrange"})[tokenname],tree.get(nr,{tokenname:"___________verystrange"})[tokenname]
			
			if nr in tree and tree[nr][tokenname]==origdic[nr][tokenname]: # found perfect matching token, copying everything to the new node
				#print "ok"
				newtree[nr]=tree[nr]
			else:
				#print "___problem"
				i=0
				intdic,normal2ins={},{}
				
				smalltoks=tree[nr][tokenname].encode("utf-8").split()
				if strict:
					
					while tree[nr][tokenname].startswith(" ".join([origdic[ii][tokenname] for ii in range(nr,nr+i+1)])): # making the insertion dict if words fit on one another:
						intdic["i"+str(i+1)]=origdic[nr+i]
						normal2ins[nr+i]="i"+str(i+1)
						i+=1
				else: 
					while i<len(smalltoks) and jellyfish.levenshtein_distance(    origdic[nr+i][tokenname].encode("utf-8"), smalltoks[i]   ) / len ( smalltoks[i] ) <= .5:
						intdic["i"+str(i+1)]=origdic[nr+i]
						normal2ins[nr+i]="i"+str(i)
						i+=1
				#print intdic
				
				if len(intdic)>1: # we found a token that has to be separated
					for ind in intdic: # copy good internal governors
						newgov={}
						for govi in intdic[ind]["gov"]:
							if govi in normal2ins: newgov[normal2ins[govi]]=intdic[ind]["gov"][govi]
						intdic[ind]["gov"]=newgov
					intdic["i1"]["gov"]={"i0":"root"} 
					intdic["i"+str(i)]["child"]=True
					#print intdic
					newtree=integrate(tree,nr,intdic)
					return correcttokens(newtree, origdic, strict) # start over because indeces changed
					
				else: # just wrong token for example ce != c'
					#print "changed token from",tree[nr][t],"to",origdic[nr][t]
					newtree[nr]=tree[nr]
					newtree[nr][tokenname]=origdic[nr][tokenname]
	except Exception, err:
		if debug:print traceback.format_exc()
		if logfile:
			f=codecs.open(logfile,"a","utf-8")
			f.write(traceback.format_exc()+"\n\n\n")
			f.close()
		#1/0
		return {}
	return newtree

	tree is checked
	origdic is the parser's annotation whose tokens are relevant
	strict: first trying to match exactly the parser's tokens
	
tree is checked
origdic is the parser's annotation whose tokens are relevant
strict: first trying to match exactly the parser's tokens
tree is checked origdic is the parser ' s annotation whose tokens are relevant strict : first trying to match exactly the parser ' s tokens
def correcttokens(tree, origdic, strict=True):
	if debug: 
		print "\n=================== start correcttokens tree"
		if strict:print "strict!!!"
		else: print "not strict!!!!"
		for i,d in  tree.iteritems():print i,d[tokenname],d 
		print "with origdic"
		for i,d in  origdic.iteritems():print i,d[tokenname],d 
	newtree={}
	try:
		for nr in sorted(origdic):
			if nr in tree and tree[nr][tokenname]==origdic[nr][tokenname]: 
				newtree[nr]=tree[nr]
			else:
				i=0
				intdic,normal2ins={},{}
				smalltoks=tree[nr][tokenname].encode("utf-8").split()
				if strict:
					while tree[nr][tokenname].startswith(" ".join([origdic[ii][tokenname] for ii in range(nr,nr+i+1)])): 
						intdic["i"+str(i+1)]=origdic[nr+i]
						normal2ins[nr+i]="i"+str(i+1)
						i+=1
				else: 
					while i<len(smalltoks) and jellyfish.levenshtein_distance(    origdic[nr+i][tokenname].encode("utf-8"), smalltoks[i]   ) / len ( smalltoks[i] ) <= .5:
						intdic["i"+str(i+1)]=origdic[nr+i]
						normal2ins[nr+i]="i"+str(i)
						i+=1
				if len(intdic)>1: 
					for ind in intdic: 
						newgov={}
						for govi in intdic[ind]["gov"]:
							if govi in normal2ins: newgov[normal2ins[govi]]=intdic[ind]["gov"][govi]
						intdic[ind]["gov"]=newgov
					intdic["i1"]["gov"]={"i0":"root"} 
					intdic["i"+str(i)]["child"]=True
					newtree=integrate(tree,nr,intdic)
					return correcttokens(newtree, origdic, strict) 
				else: 
					newtree[nr]=tree[nr]
					newtree[nr][tokenname]=origdic[nr][tokenname]
	except Exception, err:
		if debug:print traceback.format_exc()
		if logfile:
			f=codecs.open(logfile,"a","utf-8")
			f.write(traceback.format_exc()+"\n\n\n")
			f.close()
		return {}
	return newtree
def correcttokens ( tree , origdic , strict = True ) : if debug : print "\n=================== start correcttokens tree" if strict : print "strict!!!" else : print "not strict!!!!" for i , d in tree . iteritems ( ) : print i , d [ tokenname ] , d print "with origdic" for i , d in origdic . iteritems ( ) : print i , d [ tokenname ] , d newtree = { } try : for nr in sorted ( origdic ) : if nr in tree and tree [ nr ] [ tokenname ] == origdic [ nr ] [ tokenname ] : newtree [ nr ] = tree [ nr ] else : i = 0 intdic , normal2ins = { } , { } smalltoks = tree [ nr ] [ tokenname ] . encode ( "utf-8" ) . split ( ) if strict : while tree [ nr ] [ tokenname ] . startswith ( " " . join ( [ origdic [ ii ] [ tokenname ] for ii in range ( nr , nr + i + 1 ) ] ) ) : intdic [ "i" + str ( i + 1 ) ] = origdic [ nr + i ] normal2ins [ nr + i ] = "i" + str ( i + 1 ) i += 1 else : while i < len ( smalltoks ) and jellyfish . levenshtein_distance ( origdic [ nr + i ] [ tokenname ] . encode ( "utf-8" ) , smalltoks [ i ] ) / len ( smalltoks [ i ] ) <= .5 : intdic [ "i" + str ( i + 1 ) ] = origdic [ nr + i ] normal2ins [ nr + i ] = "i" + str ( i ) i += 1 if len ( intdic ) > 1 : for ind in intdic : newgov = { } for govi in intdic [ ind ] [ "gov" ] : if govi in normal2ins : newgov [ normal2ins [ govi ] ] = intdic [ ind ] [ "gov" ] [ govi ] intdic [ ind ] [ "gov" ] = newgov intdic [ "i1" ] [ "gov" ] = { "i0" : "root" } intdic [ "i" + str ( i ) ] [ "child" ] = True newtree = integrate ( tree , nr , intdic ) return correcttokens ( newtree , origdic , strict ) else : newtree [ nr ] = tree [ nr ] newtree [ nr ] [ tokenname ] = origdic [ nr ] [ tokenname ] except Exception , err : if debug : print traceback . format_exc ( ) if logfile : f = codecs . open ( logfile , "a" , "utf-8" ) f . write ( traceback . format_exc ( ) + "\n\n\n" ) f . close ( ) return { } return newtree
tree is checked
origdic is the parser's annotation whose tokens are relevant
strict: first trying to match exactly the parser's tokens
tree is checked origdic is the parser ' s annotation whose tokens are relevant strict : first trying to match exactly the parser ' s tokens
"""
	tree is checked
	origdic is the parser's annotation whose tokens are relevant
	strict: first trying to match exactly the parser's tokens
	""" #print origdic[nr] #print nr, origdic.get(nr,{tokenname:"___________verystrange"})[tokenname],tree.get(nr,{tokenname:"___________verystrange"})[tokenname] # found perfect matching token, copying everything to the new node #print "ok" #print "___problem" # making the insertion dict if words fit on one another: #print intdic # we found a token that has to be separated # copy good internal governors #print intdic # start over because indeces changed # just wrong token for example ce != c' #print "changed token from",tree[nr][t],"to",origdic[nr][t] #1/0
tree
origdic
strict