M1-Arborator-NaijaSUD/arborator/lib/database.py
def exportLastBestAnnotations(self, textid, textname):
		"""
		Conll export function (customized for Naija)

		Take the most recent tree and add the features and misc from a trusted user (here maigida, hardcoded in the function).
		If no misc (which contains timestamps info) can be found, copy those from parser.
		Also provides the rank of the tree in the text, to reorder them later on.

		input: Int, Str
		does: creates a file in projects/<project-name>/<textname>.most.recent.trees.with.feats.conllu
		output: List, Int
		"""
		print("exportingAnnotations")
		db,cursor=self.open()

		scounter, users=0,[]
		doublegovs=False
		renolet=re.compile("\W",re.U+re.I)
		basefilename=renolet.sub(".",textname.strip().replace(" ","_").replace("?",""))
		if not os.path.exists(self.exportpath.encode("utf-8")): 
			os.makedirs(self.exportpath.encode("utf-8"))
			os.chmod(self.exportpath.encode("utf-8"),0777)
			with codecs.open(".htaccess","w","utf-8") as htfile:
				htfile.write("Options Indexes\n")

		r = self.getall(cursor, "sentences",["textid"], [textid])

		users=[]
		trees = []
		if r:
			self.exportpath+u"é"
			basefilename+u"é"
			filename = self.exportpath+basefilename + u".most.recent.trees.with.feats.conllu"
			for sentenceid,nr,sentence,textid in r: # for each sentence of the text:

				treeDict=dict([ (user, (treeid,uid,realname,timestamp)) for treeid,uid,user,realname,timestamp in self.treesForSentence(sentenceid)])
				# order by timestamp
				timeDict=dict([ (timestamp,user) for (user, (treeid,uid,realname,timestamp)) in treeDict.iteritems() ])
				lastuser=timeDict[sorted(timeDict,reverse=True)[0]]
				treeid,uid,realname,timestamp=treeDict[lastuser]
				if lastuser not in users:
					users+=[lastuser]

				tree=self.conllSentenceExport(sentenceid, cursor, treeid, None)
				maigida_tree = self.gettree(sid=sentenceid, username="maigida")
				try:
					new_tree = addFeatsMiscs2Tree(tree, maigida_tree)
				except BaseException:
					print("problem with", maigida_tree)
					continue
				# print(maigida_tree)
				new_tree.sentencefeatures["nr"] = str(nr)

				if not hasMisc(new_tree):
					print("at least get timestamps from parser")
					base_tree = self.gettree(sid=sentenceid, username="parser")
					try:
						new_tree = addFeatsMiscs2Tree(tree, base_tree, miscOnly=True)
					except BaseException:
						print(tree, "\n\n", base_tree)
					new_tree.sentencefeatures["nr"] = str(nr)

				trees.append(new_tree)
				scounter+=1

		print("all features and misc added")
		# users=["most recent modification (including trees from "+", ".join(users)+")"]
		conll.trees2conllFile(trees, filename)
		print("all done")
		return users,scounter

		Conll export function (customized for Naija)

		Take the most recent tree and add the features and misc from a trusted user (here maigida, hardcoded in the function).
		If no misc (which contains timestamps info) can be found, copy those from parser.
		Also provides the rank of the tree in the text, to reorder them later on.

		input: Int, Str
		does: creates a file in projects/<project-name>/<textname>.most.recent.trees.with.feats.conllu
		output: List, Int
		
Conll export function (customized for Naija)
Take the most recent tree and add the features and misc from a trusted user (here maigida, hardcoded in the function).
If no misc (which contains timestamps info) can be found, copy those from parser.
Also provides the rank of the tree in the text, to reorder them later on.

Int, Str
does: creates a file in projects//.most.recent.trees.with.feats.conllu
output: List, Int
Conll export function ( customized for Naija ) Take the most recent tree and add the features and misc from a trusted user ( here maigida hardcoded in the function ) . If no misc ( which contains timestamps info ) can be found copy those from parser . Also provides the rank of the tree in the text to reorder them later on . Int Str does : creates a file in projects // . most . recent . trees . with . feats . conllu output : List Int
def exportLastBestAnnotations(self, textid, textname):
		print("exportingAnnotations")
		db,cursor=self.open()
		scounter, users=0,[]
		doublegovs=False
		renolet=re.compile("\W",re.U+re.I)
		basefilename=renolet.sub(".",textname.strip().replace(" ","_").replace("?",""))
		if not os.path.exists(self.exportpath.encode("utf-8")): 
			os.makedirs(self.exportpath.encode("utf-8"))
			os.chmod(self.exportpath.encode("utf-8"),0777)
			with codecs.open(".htaccess","w","utf-8") as htfile:
				htfile.write("Options Indexes\n")
		r = self.getall(cursor, "sentences",["textid"], [textid])
		users=[]
		trees = []
		if r:
			self.exportpath+u"é"
			basefilename+u"é"
			filename = self.exportpath+basefilename + u".most.recent.trees.with.feats.conllu"
			for sentenceid,nr,sentence,textid in r: 
				treeDict=dict([ (user, (treeid,uid,realname,timestamp)) for treeid,uid,user,realname,timestamp in self.treesForSentence(sentenceid)])
				timeDict=dict([ (timestamp,user) for (user, (treeid,uid,realname,timestamp)) in treeDict.iteritems() ])
				lastuser=timeDict[sorted(timeDict,reverse=True)[0]]
				treeid,uid,realname,timestamp=treeDict[lastuser]
				if lastuser not in users:
					users+=[lastuser]
				tree=self.conllSentenceExport(sentenceid, cursor, treeid, None)
				maigida_tree = self.gettree(sid=sentenceid, username="maigida")
				try:
					new_tree = addFeatsMiscs2Tree(tree, maigida_tree)
				except BaseException:
					print("problem with", maigida_tree)
					continue
				new_tree.sentencefeatures["nr"] = str(nr)
				if not hasMisc(new_tree):
					print("at least get timestamps from parser")
					base_tree = self.gettree(sid=sentenceid, username="parser")
					try:
						new_tree = addFeatsMiscs2Tree(tree, base_tree, miscOnly=True)
					except BaseException:
						print(tree, "\n\n", base_tree)
					new_tree.sentencefeatures["nr"] = str(nr)
				trees.append(new_tree)
				scounter+=1
		print("all features and misc added")
		conll.trees2conllFile(trees, filename)
		print("all done")
		return users,scounter
def exportLastBestAnnotations ( self , textid , textname ) : print ( "exportingAnnotations" ) db , cursor = self . open ( ) scounter , users = 0 , [ ] doublegovs = False renolet = re . compile ( "\W" , re . U + re . I ) basefilename = renolet . sub ( "." , textname . strip ( ) . replace ( " " , "_" ) . replace ( "?" , "" ) ) if not os . path . exists ( self . exportpath . encode ( "utf-8" ) ) : os . makedirs ( self . exportpath . encode ( "utf-8" ) ) os . chmod ( self . exportpath . encode ( "utf-8" ) , 0777 ) with codecs . open ( ".htaccess" , "w" , "utf-8" ) as htfile : htfile . write ( "Options Indexes\n" ) r = self . getall ( cursor , "sentences" , [ "textid" ] , [ textid ] ) users = [ ] trees = [ ] if r : self . exportpath + u"é" basefilename + u"é" filename = self . exportpath + basefilename + u".most.recent.trees.with.feats.conllu" for sentenceid , nr , sentence , textid in r : treeDict = dict ( [ ( user , ( treeid , uid , realname , timestamp ) ) for treeid , uid , user , realname , timestamp in self . treesForSentence ( sentenceid ) ] ) timeDict = dict ( [ ( timestamp , user ) for ( user , ( treeid , uid , realname , timestamp ) ) in treeDict . iteritems ( ) ] ) lastuser = timeDict [ sorted ( timeDict , reverse = True ) [ 0 ] ] treeid , uid , realname , timestamp = treeDict [ lastuser ] if lastuser not in users : users += [ lastuser ] tree = self . conllSentenceExport ( sentenceid , cursor , treeid , None ) maigida_tree = self . gettree ( sid = sentenceid , username = "maigida" ) try : new_tree = addFeatsMiscs2Tree ( tree , maigida_tree ) except BaseException : print ( "problem with" , maigida_tree ) continue new_tree . sentencefeatures [ "nr" ] = str ( nr ) if not hasMisc ( new_tree ) : print ( "at least get timestamps from parser" ) base_tree = self . gettree ( sid = sentenceid , username = "parser" ) try : new_tree = addFeatsMiscs2Tree ( tree , base_tree , miscOnly = True ) except BaseException : print ( tree , "\n\n" , base_tree ) new_tree . sentencefeatures [ "nr" ] = str ( nr ) trees . append ( new_tree ) scounter += 1 print ( "all features and misc added" ) conll . trees2conllFile ( trees , filename ) print ( "all done" ) return users , scounter
Conll export function (customized for Naija)
Take the most recent tree and add the features and misc from a trusted user (here maigida, hardcoded in the function).
Conll export function ( customized for Naija ) Take the most recent tree and add the features and misc from a trusted user ( here maigida hardcoded in the function ) .
"""
		Conll export function (customized for Naija)

		Take the most recent tree and add the features and misc from a trusted user (here maigida, hardcoded in the function).
		If no misc (which contains timestamps info) can be found, copy those from parser.
		Also provides the rank of the tree in the text, to reorder them later on.

		input: Int, Str
		does: creates a file in projects/<project-name>/<textname>.most.recent.trees.with.feats.conllu
		output: List, Int
		""" # for each sentence of the text: # order by timestamp # print(maigida_tree) # users=["most recent modification (including trees from "+", ".join(users)+")"]
self
textid
textname