M1-Arborator-NaijaSUD/arborator/lib/trees2train.py
def fusionForgottenTrees(project="Platinum",fusdir="../projects/OrfeoGold2016/platinum/*", annotators=["admin"]):
	"""
	takes trees from project ordered by annotators. if they exist fuse them into the fusdir
	result has the extension "cool.conll"
	,"Sy","Marion"
	"""
	
	#print lemmacorrection
	sys.path.insert(0, '../tools')
	import difflib
	outfiles=[]
	sql = SQL(project)
	db,cursor=sql.open()
	goodTexts={}
	outdir=os.path.join("..","projects",project,"exportcool")
	try: os.mkdir(outdir)	
	except OSError: pass
	for annotator in annotators:
		print [list(cursor.execute("select rowid from users where user =?;",(annotator,) ))]
	annotatorIds=tuple(a for (a,) in [list(cursor.execute("select rowid from users where user =?;",(annotator,) ))[0] for annotator in annotators])
	print annotators, annotatorIds
	
	for textid, textname, nrtokens in list(cursor.execute("select rowid, * from texts;" )):  # for each text
		print "\n__________________________doing",textname,"with", nrtokens, "tokens"
		nrutids={}
		for nr,userid,treeid in list(cursor.execute("select nr,userid,trees.rowid as treeid from trees, sentences where sentenceid=sentences.rowid and userid in {annotatorIds} and  textid = ? order by nr;".format(annotatorIds=annotatorIds),(textid,))):
			nrutids[nr]=nrutids.get(nr,{})
			nrutids[nr][userid]=treeid
		trees={}
		for nr in sorted(nrutids): # for each sentence
			tree=None
			for aid in annotatorIds: # for each interesting annotator id
				if aid in nrutids[nr]:
					tree=sql.gettree(treeid=nrutids[nr][aid], indb=db, incursor=cursor)["tree"]
					trees[nr]=tree
					#print "atree:",tree
					break
			#if not tree:
				#print "problem: no tree for nr",nr,"type",type(nr)
				#print "annotatorIds",annotatorIds				
				#raise Exception('no tree', nr)
		#print trees
		print len(trees),"trees from", project
		print textname,textname.split(".")[0]
		btextname=os.path.basename(textname).split(".")[0] 
		if btextname.endswith("-one-word-per-line"): btextname=btextname[:-len("-one-word-per-line")]
		#print glob.glob(fusdir),[os.path.basename(fi).split(".")[0] for fi in glob.glob(fusdir)]
		cooltrees=[]
		ptrees,ftrees=0,0
		for fi in glob.glob(fusdir):
			if btextname == os.path.basename(fi).split(".")[0] :
				print "yes",btextname
				fustrees=conll.conllFile2trees(fi)
				print len(fustrees),"ftrees",fi
				for nr,ftree in enumerate(fustrees):
					if nr+1 in trees:
						#print "added tree",nr+1,"from database"
						#ptree=platinum(trees[nr+1])
						ptree=trees[nr+1]
						for iii in ptree: 
							ptree[iii]["tag2"]="_"
							if ptree[iii]["lemma"] in lemmacorrection:
								ptree[iii]["lemma"]=lemmacorrection[ptree[iii]["lemma"]]
						cooltrees+=[ptree]
						#print nr+1,"tree from",project#,tree
						ptrees+=1
						if ftree.sentence() != u" ".join([ptree[i].get("t","") for i in sorted(ptree)]):
							print "\n_________",nr+1
							print ftree.sentence() 
							print u" ".join([ptree[i].get("t","") for i in sorted(ptree)])
							#for l in difflib.context_diff(ftree.sentence() ,u" ".join([ptree[i].get("t","") for i in sorted(ptree)])):print l
							
						#print "dbtree",platinum(trees[nr+1])
					else:						
						for iii in ftree: 
							ftree[iii]["tag2"]="_"
							if ftree[iii]["lemma"] in lemmacorrection:
								ftree[iii]["lemma"]=lemmacorrection[ftree[iii]["lemma"]]
						#print nr+1,"tree from",fusdir#,tree
						ftrees+=1
						cooltrees+=[ftree]
						#print "added tree",nr+1,"from fustrees",fi
				outfile=os.path.join(outdir,textname+".cool.conll")
				conll.trees2conllFile(cooltrees, outfile=outfile, columns=10)
				print "wrote",outfile
				print ptrees,"ptrees, ",ftrees,"ftrees"
				break
		if len(cooltrees)==0:print "nothing for",btextname
		outfiles+=[outfile]
		#qsdf
	return outfiles

	takes trees from project ordered by annotators. if they exist fuse them into the fusdir
	result has the extension "cool.conll"
	,"Sy","Marion"
	
takes trees from project ordered by annotators.
takes trees from project ordered by annotators .
def fusionForgottenTrees(project="Platinum",fusdir="../projects/OrfeoGold2016/platinum/*", annotators=["admin"]):
	sys.path.insert(0, '../tools')
	import difflib
	outfiles=[]
	sql = SQL(project)
	db,cursor=sql.open()
	goodTexts={}
	outdir=os.path.join("..","projects",project,"exportcool")
	try: os.mkdir(outdir)	
	except OSError: pass
	for annotator in annotators:
		print [list(cursor.execute("select rowid from users where user =?;",(annotator,) ))]
	annotatorIds=tuple(a for (a,) in [list(cursor.execute("select rowid from users where user =?;",(annotator,) ))[0] for annotator in annotators])
	print annotators, annotatorIds
	for textid, textname, nrtokens in list(cursor.execute("select rowid, * from texts;" )):  
		print "\n__________________________doing",textname,"with", nrtokens, "tokens"
		nrutids={}
		for nr,userid,treeid in list(cursor.execute("select nr,userid,trees.rowid as treeid from trees, sentences where sentenceid=sentences.rowid and userid in {annotatorIds} and  textid = ? order by nr;".format(annotatorIds=annotatorIds),(textid,))):
			nrutids[nr]=nrutids.get(nr,{})
			nrutids[nr][userid]=treeid
		trees={}
		for nr in sorted(nrutids): 
			tree=None
			for aid in annotatorIds: 
				if aid in nrutids[nr]:
					tree=sql.gettree(treeid=nrutids[nr][aid], indb=db, incursor=cursor)["tree"]
					trees[nr]=tree
					break
		print len(trees),"trees from", project
		print textname,textname.split(".")[0]
		btextname=os.path.basename(textname).split(".")[0] 
		if btextname.endswith("-one-word-per-line"): btextname=btextname[:-len("-one-word-per-line")]
		cooltrees=[]
		ptrees,ftrees=0,0
		for fi in glob.glob(fusdir):
			if btextname == os.path.basename(fi).split(".")[0] :
				print "yes",btextname
				fustrees=conll.conllFile2trees(fi)
				print len(fustrees),"ftrees",fi
				for nr,ftree in enumerate(fustrees):
					if nr+1 in trees:
						ptree=trees[nr+1]
						for iii in ptree: 
							ptree[iii]["tag2"]="_"
							if ptree[iii]["lemma"] in lemmacorrection:
								ptree[iii]["lemma"]=lemmacorrection[ptree[iii]["lemma"]]
						cooltrees+=[ptree]
						ptrees+=1
						if ftree.sentence() != u" ".join([ptree[i].get("t","") for i in sorted(ptree)]):
							print "\n_________",nr+1
							print ftree.sentence() 
							print u" ".join([ptree[i].get("t","") for i in sorted(ptree)])
					else:						
						for iii in ftree: 
							ftree[iii]["tag2"]="_"
							if ftree[iii]["lemma"] in lemmacorrection:
								ftree[iii]["lemma"]=lemmacorrection[ftree[iii]["lemma"]]
						ftrees+=1
						cooltrees+=[ftree]
				outfile=os.path.join(outdir,textname+".cool.conll")
				conll.trees2conllFile(cooltrees, outfile=outfile, columns=10)
				print "wrote",outfile
				print ptrees,"ptrees, ",ftrees,"ftrees"
				break
		if len(cooltrees)==0:print "nothing for",btextname
		outfiles+=[outfile]
	return outfiles
def fusionForgottenTrees ( project = "Platinum" , fusdir = "../projects/OrfeoGold2016/platinum/*" , annotators = [ "admin" ] ) : sys . path . insert ( 0 , '../tools' ) import difflib outfiles = [ ] sql = SQL ( project ) db , cursor = sql . open ( ) goodTexts = { } outdir = os . path . join ( ".." , "projects" , project , "exportcool" ) try : os . mkdir ( outdir ) except OSError : pass for annotator in annotators : print [ list ( cursor . execute ( "select rowid from users where user =?;" , ( annotator , ) ) ) ] annotatorIds = tuple ( a for ( a , ) in [ list ( cursor . execute ( "select rowid from users where user =?;" , ( annotator , ) ) ) [ 0 ] for annotator in annotators ] ) print annotators , annotatorIds for textid , textname , nrtokens in list ( cursor . execute ( "select rowid, * from texts;" ) ) : print "\n__________________________doing" , textname , "with" , nrtokens , "tokens" nrutids = { } for nr , userid , treeid in list ( cursor . execute ( "select nr,userid,trees.rowid as treeid from trees, sentences where sentenceid=sentences.rowid and userid in {annotatorIds} and  textid = ? order by nr;" . format ( annotatorIds = annotatorIds ) , ( textid , ) ) ) : nrutids [ nr ] = nrutids . get ( nr , { } ) nrutids [ nr ] [ userid ] = treeid trees = { } for nr in sorted ( nrutids ) : tree = None for aid in annotatorIds : if aid in nrutids [ nr ] : tree = sql . gettree ( treeid = nrutids [ nr ] [ aid ] , indb = db , incursor = cursor ) [ "tree" ] trees [ nr ] = tree break print len ( trees ) , "trees from" , project print textname , textname . split ( "." ) [ 0 ] btextname = os . path . basename ( textname ) . split ( "." ) [ 0 ] if btextname . endswith ( "-one-word-per-line" ) : btextname = btextname [ : - len ( "-one-word-per-line" ) ] cooltrees = [ ] ptrees , ftrees = 0 , 0 for fi in glob . glob ( fusdir ) : if btextname == os . path . basename ( fi ) . split ( "." ) [ 0 ] : print "yes" , btextname fustrees = conll . conllFile2trees ( fi ) print len ( fustrees ) , "ftrees" , fi for nr , ftree in enumerate ( fustrees ) : if nr + 1 in trees : ptree = trees [ nr + 1 ] for iii in ptree : ptree [ iii ] [ "tag2" ] = "_" if ptree [ iii ] [ "lemma" ] in lemmacorrection : ptree [ iii ] [ "lemma" ] = lemmacorrection [ ptree [ iii ] [ "lemma" ] ] cooltrees += [ ptree ] ptrees += 1 if ftree . sentence ( ) != u" " . join ( [ ptree [ i ] . get ( "t" , "" ) for i in sorted ( ptree ) ] ) : print "\n_________" , nr + 1 print ftree . sentence ( ) print u" " . join ( [ ptree [ i ] . get ( "t" , "" ) for i in sorted ( ptree ) ] ) else : for iii in ftree : ftree [ iii ] [ "tag2" ] = "_" if ftree [ iii ] [ "lemma" ] in lemmacorrection : ftree [ iii ] [ "lemma" ] = lemmacorrection [ ftree [ iii ] [ "lemma" ] ] ftrees += 1 cooltrees += [ ftree ] outfile = os . path . join ( outdir , textname + ".cool.conll" ) conll . trees2conllFile ( cooltrees , outfile = outfile , columns = 10 ) print "wrote" , outfile print ptrees , "ptrees, " , ftrees , "ftrees" break if len ( cooltrees ) == 0 : print "nothing for" , btextname outfiles += [ outfile ] return outfiles
takes trees from project ordered by annotators.
takes trees from project ordered by annotators .
"""
	takes trees from project ordered by annotators. if they exist fuse them into the fusdir
	result has the extension "cool.conll"
	,"Sy","Marion"
	""" #print lemmacorrection # for each text # for each sentence # for each interesting annotator id #print "atree:",tree #if not tree: #print "problem: no tree for nr",nr,"type",type(nr) #print "annotatorIds",annotatorIds				 #raise Exception('no tree', nr) #print trees #print glob.glob(fusdir),[os.path.basename(fi).split(".")[0] for fi in glob.glob(fusdir)] #print "added tree",nr+1,"from database" #ptree=platinum(trees[nr+1]) #print nr+1,"tree from",project#,tree #for l in difflib.context_diff(ftree.sentence() ,u" ".join([ptree[i].get("t","") for i in sorted(ptree)])):print l #print "dbtree",platinum(trees[nr+1]) #print nr+1,"tree from",fusdir#,tree #print "added tree",nr+1,"from fustrees",fi #qsdf
project
fusdir
annotators