M1-Arborator-NaijaSUD/arborator/lib/treebankfiles.py
def readInTestResults(projectname,conllfilename,gold="gold",parser="parser"):
	"""
	reads in the result of the mate transition parser into visually comparable results
	the result file is a conll 14 files with double annotation (lemma + lemma2, ...)
	"""
	print "________________________________________current file is: " + conllfilename
	shortname = conllfilename.split("/")[-1]
	if projectname: # really modifying the database
		
		sql = database.SQL(projectname)
		db,cursor=sql.open()
		textid = sql.enter(cursor, "texts",["textname"],(shortname,))
		goldid = sql.enter(cursor, "users",["user","realname"],(gold,gold,))
		parserid = sql.enter(cursor, "users",["user","realname"],(parser,parser,))
		
		
		sents={} # score of correct functions -> list of trees having that score. allows to sort by annotation quality. the list of trees are triples: ( goldtree,parsetree,info )
		goldsentences,parsesentences,sentencefeatures=[],[],[]
		l,t,g,f,length=0.0,0.0,0.0,0.0,0
		allcats,allfuncs={},{}
		
		sentences=conll.conllFile2trees(conllfilename)
		#print len(sentences)
		for tree in sentences:
			tl,tt,tg,tf,tlength=computeDifference(tree)
			#"lemma precision:"+str(round(tl/tlength,2))+
			info = " LAS:"+str(round(tf/tlength,3))+" UAS:"+str(round(tg/tlength,3))+" pos precision:"+str(round(tt/tlength,3))+u" n° tokens:"+str(tlength)
			
			
			goldtree,parsetree={},{}
			for i in tree:
				node=tree[i]
				goldtree[i]= {"id":node["id"],"t":node["t"],"lemma":node["lemma"],"tag":node["tag"],"morph":node["morph"],"gov":node["gov"]}
				parsetree[i]={"id":node["id"],"t":node["t"],"lemma":node["lemma2"],"tag":node["tag2"],"morph":node["morph2"],"gov":node["gov2"]}
				allcats[node["tag"]]=None
				allcats[node["tag2"]]=None
				allfuncs[node["gov"].values()[0]]=None
				allfuncs[node["gov2"].values()[0]]=None
				
			sents[tf/tlength]=sents.get(tf/tlength,[])+[ ( goldtree,parsetree,info ) ]
			(l,t,g,f,length)=map(sum,zip((l,t,g,f,length),(tl,tt,tg,tf,tlength) ))
		#	
		globalinfo = "lemma precision:"+str(round(l/length,2))+" pos precision:"+str(round(t/length,2))+" UAS:"+str(round(g/length,2))+" LAS:"+str(round(f/length,2))+u" n° tokens:"+str(length)+u" n° sentences:"+str(len(sentences))	
		print globalinfo
		treec=0
		for f in sorted(sents):
			for ( goldtree,parsetree,info ) in sents[f]:
				#print f, treec
				#print f, goldtree
				goldsentences+=[goldtree]
				parsesentences+=[parsetree]
				sentencefeatures+=[{"info":info}]
				treec+=1
		print "entering",len(parsesentences),"sentences"
		enterSentences(sql,cursor,goldsentences,shortname, textid,gold,eraseAllAnnos=True, sentencefeatures=sentencefeatures,tokname="t" )
		enterSentences(sql,cursor,parsesentences,shortname, textid,parser,eraseAllAnnos=False, defaultAnnotatorName="gold", tokname="t" )
		db.commit()
		db.close()
		
		writeConfigs(projectname,allcats,allfuncs)
		
		
	else:
		#only testing
		sentences=conll.conllFile2trees(conllfilename)
		l,t,g,f,length=0.0,0.0,0.0,0.0,0
		
		for s in sentences:
			(l,t,g,f,length)=map(sum,zip((l,t,g,f,length),computeDifference(s)))
			
		print "____"
		print "lemma precision:",l/length,"tag precision:",t/length,"UAS:",g/length,"LAS:",f/length
		print "n° tokens:",length,"n° sentences:",len(sentences)
		#print computeDifference(sentences[1])
		print sentences[1]

	reads in the result of the mate transition parser into visually comparable results
	the result file is a conll 14 files with double annotation (lemma + lemma2, ...)
	
reads in the result of the mate transition parser into visually comparable results
the result file is a conll 14 files with double annotation (lemma + lemma2, ...)
reads in the result of the mate transition parser into visually comparable results the result file is a conll 14 files with double annotation ( lemma + lemma2 ... )
def readInTestResults(projectname,conllfilename,gold="gold",parser="parser"):
	print "________________________________________current file is: " + conllfilename
	shortname = conllfilename.split("/")[-1]
	if projectname: 
		sql = database.SQL(projectname)
		db,cursor=sql.open()
		textid = sql.enter(cursor, "texts",["textname"],(shortname,))
		goldid = sql.enter(cursor, "users",["user","realname"],(gold,gold,))
		parserid = sql.enter(cursor, "users",["user","realname"],(parser,parser,))
		sents={} 
		goldsentences,parsesentences,sentencefeatures=[],[],[]
		l,t,g,f,length=0.0,0.0,0.0,0.0,0
		allcats,allfuncs={},{}
		sentences=conll.conllFile2trees(conllfilename)
		for tree in sentences:
			tl,tt,tg,tf,tlength=computeDifference(tree)
			info = " LAS:"+str(round(tf/tlength,3))+" UAS:"+str(round(tg/tlength,3))+" pos precision:"+str(round(tt/tlength,3))+u" n° tokens:"+str(tlength)
			goldtree,parsetree={},{}
			for i in tree:
				node=tree[i]
				goldtree[i]= {"id":node["id"],"t":node["t"],"lemma":node["lemma"],"tag":node["tag"],"morph":node["morph"],"gov":node["gov"]}
				parsetree[i]={"id":node["id"],"t":node["t"],"lemma":node["lemma2"],"tag":node["tag2"],"morph":node["morph2"],"gov":node["gov2"]}
				allcats[node["tag"]]=None
				allcats[node["tag2"]]=None
				allfuncs[node["gov"].values()[0]]=None
				allfuncs[node["gov2"].values()[0]]=None
			sents[tf/tlength]=sents.get(tf/tlength,[])+[ ( goldtree,parsetree,info ) ]
			(l,t,g,f,length)=map(sum,zip((l,t,g,f,length),(tl,tt,tg,tf,tlength) ))
		globalinfo = "lemma precision:"+str(round(l/length,2))+" pos precision:"+str(round(t/length,2))+" UAS:"+str(round(g/length,2))+" LAS:"+str(round(f/length,2))+u" n° tokens:"+str(length)+u" n° sentences:"+str(len(sentences))	
		print globalinfo
		treec=0
		for f in sorted(sents):
			for ( goldtree,parsetree,info ) in sents[f]:
				goldsentences+=[goldtree]
				parsesentences+=[parsetree]
				sentencefeatures+=[{"info":info}]
				treec+=1
		print "entering",len(parsesentences),"sentences"
		enterSentences(sql,cursor,goldsentences,shortname, textid,gold,eraseAllAnnos=True, sentencefeatures=sentencefeatures,tokname="t" )
		enterSentences(sql,cursor,parsesentences,shortname, textid,parser,eraseAllAnnos=False, defaultAnnotatorName="gold", tokname="t" )
		db.commit()
		db.close()
		writeConfigs(projectname,allcats,allfuncs)
	else:
		sentences=conll.conllFile2trees(conllfilename)
		l,t,g,f,length=0.0,0.0,0.0,0.0,0
		for s in sentences:
			(l,t,g,f,length)=map(sum,zip((l,t,g,f,length),computeDifference(s)))
		print "____"
		print "lemma precision:",l/length,"tag precision:",t/length,"UAS:",g/length,"LAS:",f/length
		print "n° tokens:",length,"n° sentences:",len(sentences)
		print sentences[1]
def readInTestResults ( projectname , conllfilename , gold = "gold" , parser = "parser" ) : print "________________________________________current file is: " + conllfilename shortname = conllfilename . split ( "/" ) [ - 1 ] if projectname : sql = database . SQL ( projectname ) db , cursor = sql . open ( ) textid = sql . enter ( cursor , "texts" , [ "textname" ] , ( shortname , ) ) goldid = sql . enter ( cursor , "users" , [ "user" , "realname" ] , ( gold , gold , ) ) parserid = sql . enter ( cursor , "users" , [ "user" , "realname" ] , ( parser , parser , ) ) sents = { } goldsentences , parsesentences , sentencefeatures = [ ] , [ ] , [ ] l , t , g , f , length = 0.0 , 0.0 , 0.0 , 0.0 , 0 allcats , allfuncs = { } , { } sentences = conll . conllFile2trees ( conllfilename ) for tree in sentences : tl , tt , tg , tf , tlength = computeDifference ( tree ) info = " LAS:" + str ( round ( tf / tlength , 3 ) ) + " UAS:" + str ( round ( tg / tlength , 3 ) ) + " pos precision:" + str ( round ( tt / tlength , 3 ) ) + u" n° tokens:"+ s tr( t length) goldtree , parsetree = { } , { } for i in tree : node = tree [ i ] goldtree [ i ] = { "id" : node [ "id" ] , "t" : node [ "t" ] , "lemma" : node [ "lemma" ] , "tag" : node [ "tag" ] , "morph" : node [ "morph" ] , "gov" : node [ "gov" ] } parsetree [ i ] = { "id" : node [ "id" ] , "t" : node [ "t" ] , "lemma" : node [ "lemma2" ] , "tag" : node [ "tag2" ] , "morph" : node [ "morph2" ] , "gov" : node [ "gov2" ] } allcats [ node [ "tag" ] ] = None allcats [ node [ "tag2" ] ] = None allfuncs [ node [ "gov" ] . values ( ) [ 0 ] ] = None allfuncs [ node [ "gov2" ] . values ( ) [ 0 ] ] = None sents [ tf / tlength ] = sents . get ( tf / tlength , [ ] ) + [ ( goldtree , parsetree , info ) ] ( l , t , g , f , length ) = map ( sum , zip ( ( l , t , g , f , length ) , ( tl , tt , tg , tf , tlength ) ) ) globalinfo = "lemma precision:" + str ( round ( l / length , 2 ) ) + " pos precision:" + str ( round ( t / length , 2 ) ) + " UAS:" + str ( round ( g / length , 2 ) ) + " LAS:" + str ( round ( f / length , 2 ) ) + u" n° tokens:"+ s tr( l ength) + u " n° sentences:"+s t r(l e n(s e ntences)) print globalinfo treec = 0 for f in sorted ( sents ) : for ( goldtree , parsetree , info ) in sents [ f ] : goldsentences += [ goldtree ] parsesentences += [ parsetree ] sentencefeatures += [ { "info" : info } ] treec += 1 print "entering" , len ( parsesentences ) , "sentences" enterSentences ( sql , cursor , goldsentences , shortname , textid , gold , eraseAllAnnos = True , sentencefeatures = sentencefeatures , tokname = "t" ) enterSentences ( sql , cursor , parsesentences , shortname , textid , parser , eraseAllAnnos = False , defaultAnnotatorName = "gold" , tokname = "t" ) db . commit ( ) db . close ( ) writeConfigs ( projectname , allcats , allfuncs ) else : sentences = conll . conllFile2trees ( conllfilename ) l , t , g , f , length = 0.0 , 0.0 , 0.0 , 0.0 , 0 for s in sentences : ( l , t , g , f , length ) = map ( sum , zip ( ( l , t , g , f , length ) , computeDifference ( s ) ) ) print "____" print "lemma precision:" , l / length , "tag precision:" , t / length , "UAS:" , g / length , "LAS:" , f / length print "n° tokens:", l ength, " n° sentences:",l e n(s e ntences) print sentences [ 1 ]
reads in the result of the mate transition parser into visually comparable results
the result file is a conll 14 files with double annotation (lemma + lemma2, ...)
reads in the result of the mate transition parser into visually comparable results the result file is a conll 14 files with double annotation ( lemma + lemma2 ... )
"""
	reads in the result of the mate transition parser into visually comparable results
	the result file is a conll 14 files with double annotation (lemma + lemma2, ...)
	""" # really modifying the database # score of correct functions -> list of trees having that score. allows to sort by annotation quality. the list of trees are triples: ( goldtree,parsetree,info ) #print len(sentences) #"lemma precision:"+str(round(tl/tlength,2))+ #	 #print f, treec #print f, goldtree #only testing #print computeDifference(sentences[1])
projectname
conllfilename
gold
parser