M1-Arborator-NaijaSUD/arborator/lib/treebankfiles.py
def enterSentences(sql, cursor, sentences, filename, textid, annotatorName, eraseAllAnnos, notenter=["order"], sentencefeatures={}, defaultAnnotatorName="parser", tokname=None):
	"""
	takes a list of nodedics (sentences)
	
	puts them into the database
	
	"""
	#print "found",len(sentences),"sentences to enter"
	
	if debug: print "entering",filename,"as annotation by",annotatorName, "into annotations of textid",textid
	
	if not tokname:tokname=tokenname
	
	shortname = filename.split("/")[-1]
	
	userid = sql.enter(cursor, "users",["user"],(annotatorName,))
	sql.realname(annotatorName, cursor)
	if not userid:
		print "the user is not in the database"
		return
	
	
	scounter,wcounter=0,0
	ti = time()
	if eraseAllAnnos:
		# TODO : remove trees completely with all features!
		sql.clean(cursor, "sentences",["textid"],(textid,)) # erase all existing sentences for this textid
		sql.clean(cursor, "sentencesearch",["textid"],(textid,)) # erase all existing sentences for this textid
		
		#{ 1: {'head': 2, 'word': u'``', 'deps': [], 'lemma': u'--', 'tag': u'$(', 'rel': u'-PUNCT-', 'id': 1}, 2: {'head': 4, 'word': u'Ross', 'deps': [1, 3], 'lemma': u'--', 'tag': u'NE', 'rel': u'SUBJ', 'id': 2}, 3: {'head': 2, 'word': u'Perot', 'deps': [], 'lemma': u'--', 'tag': u'NE', 'rel': u'APP', 'id': 3}, 4: {'head': 0, 'word': u'w\xe4re', 'deps': [2, 5, 8], 'lemma': u'--', 'tag': u'VAFIN', 'rel': u'ROOT', 'id': 4}, 5: {'head': 4, 'word': u'vielleicht', 'deps': [], 'lemma': u'--', 'tag': u'ADV', 'rel': u'ADV', 'id': 5}, 6: {'head': 8, 'word': u'ein', 'deps': [], 'lemma': u'--', 'tag': u'ART', 'rel': u'DET', 'id': 6}, 7: {'head': 8, 'word': u'pr\xe4chtiger', 'deps': [], 'lemma': u'--', 'tag': u'ADJA', 'rel': u'ATTR', 'id': 7}, 8: {'head': 4, 'word': u'Diktator', 'deps': [6, 7, 9], 'lemma': u'--', 'tag': u'NN', 'rel': u'PRED', 'id': 8}, 9: {'head': 8, 'word': u"''", 'deps': [], 'lemma': u'--', 'tag': u'$(', 'rel': u'-PUNCT-', 'id': 9}}
		
		
		#for snode in sentences:
			#print snode

		for i,tree in enumerate(sentences): # for every sentence
			#print "ooooooooooooooooo",len(tree)
			
			if len(tree)<maxlength:
				scounter+=1
				
				sentence=" ".join([tree[j].get(tokname,"") for j in sorted(tree)])
				sentenceid = sql.enter(cursor, "sentences",["nr","sentence","textid"],(scounter,sentence,textid,))
				sql.enter(cursor, "sentencesearch",["nr","sentence","textid"],(scounter,sentence,textid,))
				#print sentencefeatures
				if tree.sentencefeatures:
					for a,v in tree.sentencefeatures.iteritems():
						sql.enter(cursor, "sentencefeatures",["sentenceid","attr","value"],(sentenceid,a,v,))
				
				# enter first tree for sentencewcounter, sent, treeid
				ws, sent, treeid = sql.enterTree(cursor, tree, sentenceid, userid, notenter=notenter, intokenname=tokname)
				wcounter+=ws
				
				if debug:print "----",scounter,"of",len(sentences),sent.encode("utf-8")," - it took",time()-ti,"seconds up to now for",wcounter, "words - ", float(wcounter)/(time()-ti),'words per second.'
				#' expected time remaining', float(scounter)/(time()-ti)*(len(sentences)-scounter)
			else:
				print "ooooooh"
				if logfile:
					f=codecs.open("toolong.txt","a","utf-8")
					f.write("\t".join([filename, str(len(nodes)),unicode(tree)])+"\n")
					f.close()
				
					f=codecs.open(logfile,"a","utf-8")
					f.write("tooooooooooooooooolong: \n"+ filename+"\n"+unicode(tree)+"\n\n\n")
					f.close()

	else: # not erasing, adding
		#print "enterSentences qqq"
		
		r = sql.getall(cursor, "sentences",["textid"], [textid])

		mini=0
		
		for tree in sentences: # for every sentence
			if len(tree)<maxlength:
				
				
				#print tree
				
				distas=[]
				
				scounter+=1
				inssentence=" ".join([tree[j].get(tokname,"") for j in sorted(tree)])
				entered=False
				for sentenceid,nr,sentence,textid in r[mini:]: 
					#print "found:",sentence
					#print "looking for:",inssentence
					#print len(tree),len(sentence.split()),len(tree)==len(sentence.split())
					if sentence==inssentence:
						mini=nr
						origdic=getorigdic(sql,cursor,sentenceid,defaultAnnotatorName)
						#origdic=sql.gettree(sid=sentenceid,username=defaultAnnotatorName)["tree"]
						if len(origdic)==len(tree):
							if debug: print "standard entry",inssentence
							ws, sent, treeid = sql.enterTree(cursor, tree, sentenceid, userid, notenter=notenter, intokenname=tokname)
						else:
							if debug: print "try fuzzy entry",inssentence
							ws, sent, treeid = fuzzyEnterSentence(tree,origdic,sql, cursor, filename, textid, annotatorName, scounter,sentence, sentenceid, inssentence, userid, notenter=notenter)
						wcounter+=ws
						if debug :
							if sent:print "---",sent.encode("utf-8"),"sentence"
							else:
								print "ooooooh"
								if logfile:
									
									f=codecs.open(logfile,"a","utf-8")
									f.write("no sentence: \n"+ filename+"\n"+unicode(tree)+"\n\n\n")
									f.close()
								
						entered=True
						break
					else: 
						distas+=[(jellyfish.levenshtein_distance(unicode(sentence).encode("utf-8"), unicode(inssentence).encode("utf-8")), (tree, sentenceid, sentence))]
					
				if not entered: 
					print "couldn't find '"+inssentence+"' in", shortname,"! Haven't yet entered the annotation"
					if distas:
						tree, sentenceid, sentence = sorted(distas)[0][1] # get best sentence
						print "will try to enter instead\n",inssentence,"\ninto\n",sentence
						origdic=getorigdic(sql,cursor,sentenceid,defaultAnnotatorName)
						ws, sent, treeid = fuzzyEnterSentence(tree,origdic,sql, cursor, filename, textid, annotatorName, scounter,sentence, sentenceid, inssentence, userid, notenter=notenter)
						wcounter+=ws
					else:
						print "no data under that name:",shortname
						#enterSentences(sql, cursor, sentences, filename, textid, annotatorName, eraseAllAnnos=True, notenter=notenter, sentencefeatures=sentencefeatures, defaultAnnotatorName=defaultAnnotatorName, tokname=tokname)
						
						
			
			else:
				print "ooooooh"
				if logfile:
					f=codecs.open("toolong.txt","a","utf-8")
					f.write("\t".join([filename, str(len(nodes)),unicode(tree)])+"\n")
					f.close()
				
					f=codecs.open(logfile,"a","utf-8")
					f.write("tooooooooooooooooolong: \n"+ filename+"\n"+unicode(tree)+"\n\n\n")
					f.close()
			
			
	#db.commit()
	if debug:print "it took",time()-ti,"seconds"

	takes a list of nodedics (sentences)
	
	puts them into the database
	
	
takes a list of nodedics (sentences)
puts them into the database
takes a list of nodedics ( sentences ) puts them into the database
def enterSentences(sql, cursor, sentences, filename, textid, annotatorName, eraseAllAnnos, notenter=["order"], sentencefeatures={}, defaultAnnotatorName="parser", tokname=None):
	if debug: print "entering",filename,"as annotation by",annotatorName, "into annotations of textid",textid
	if not tokname:tokname=tokenname
	shortname = filename.split("/")[-1]
	userid = sql.enter(cursor, "users",["user"],(annotatorName,))
	sql.realname(annotatorName, cursor)
	if not userid:
		print "the user is not in the database"
		return
	scounter,wcounter=0,0
	ti = time()
	if eraseAllAnnos:
		sql.clean(cursor, "sentences",["textid"],(textid,)) 
		sql.clean(cursor, "sentencesearch",["textid"],(textid,)) 
		for i,tree in enumerate(sentences): 
			if len(tree)<maxlength:
				scounter+=1
				sentence=" ".join([tree[j].get(tokname,"") for j in sorted(tree)])
				sentenceid = sql.enter(cursor, "sentences",["nr","sentence","textid"],(scounter,sentence,textid,))
				sql.enter(cursor, "sentencesearch",["nr","sentence","textid"],(scounter,sentence,textid,))
				if tree.sentencefeatures:
					for a,v in tree.sentencefeatures.iteritems():
						sql.enter(cursor, "sentencefeatures",["sentenceid","attr","value"],(sentenceid,a,v,))
				ws, sent, treeid = sql.enterTree(cursor, tree, sentenceid, userid, notenter=notenter, intokenname=tokname)
				wcounter+=ws
				if debug:print "----",scounter,"of",len(sentences),sent.encode("utf-8")," - it took",time()-ti,"seconds up to now for",wcounter, "words - ", float(wcounter)/(time()-ti),'words per second.'
			else:
				print "ooooooh"
				if logfile:
					f=codecs.open("toolong.txt","a","utf-8")
					f.write("\t".join([filename, str(len(nodes)),unicode(tree)])+"\n")
					f.close()
					f=codecs.open(logfile,"a","utf-8")
					f.write("tooooooooooooooooolong: \n"+ filename+"\n"+unicode(tree)+"\n\n\n")
					f.close()
	else: 
		r = sql.getall(cursor, "sentences",["textid"], [textid])
		mini=0
		for tree in sentences: 
			if len(tree)<maxlength:
				distas=[]
				scounter+=1
				inssentence=" ".join([tree[j].get(tokname,"") for j in sorted(tree)])
				entered=False
				for sentenceid,nr,sentence,textid in r[mini:]: 
					if sentence==inssentence:
						mini=nr
						origdic=getorigdic(sql,cursor,sentenceid,defaultAnnotatorName)
						if len(origdic)==len(tree):
							if debug: print "standard entry",inssentence
							ws, sent, treeid = sql.enterTree(cursor, tree, sentenceid, userid, notenter=notenter, intokenname=tokname)
						else:
							if debug: print "try fuzzy entry",inssentence
							ws, sent, treeid = fuzzyEnterSentence(tree,origdic,sql, cursor, filename, textid, annotatorName, scounter,sentence, sentenceid, inssentence, userid, notenter=notenter)
						wcounter+=ws
						if debug :
							if sent:print "---",sent.encode("utf-8"),"sentence"
							else:
								print "ooooooh"
								if logfile:
									f=codecs.open(logfile,"a","utf-8")
									f.write("no sentence: \n"+ filename+"\n"+unicode(tree)+"\n\n\n")
									f.close()
						entered=True
						break
					else: 
						distas+=[(jellyfish.levenshtein_distance(unicode(sentence).encode("utf-8"), unicode(inssentence).encode("utf-8")), (tree, sentenceid, sentence))]
				if not entered: 
					print "couldn't find '"+inssentence+"' in", shortname,"! Haven't yet entered the annotation"
					if distas:
						tree, sentenceid, sentence = sorted(distas)[0][1] 
						print "will try to enter instead\n",inssentence,"\ninto\n",sentence
						origdic=getorigdic(sql,cursor,sentenceid,defaultAnnotatorName)
						ws, sent, treeid = fuzzyEnterSentence(tree,origdic,sql, cursor, filename, textid, annotatorName, scounter,sentence, sentenceid, inssentence, userid, notenter=notenter)
						wcounter+=ws
					else:
						print "no data under that name:",shortname
			else:
				print "ooooooh"
				if logfile:
					f=codecs.open("toolong.txt","a","utf-8")
					f.write("\t".join([filename, str(len(nodes)),unicode(tree)])+"\n")
					f.close()
					f=codecs.open(logfile,"a","utf-8")
					f.write("tooooooooooooooooolong: \n"+ filename+"\n"+unicode(tree)+"\n\n\n")
					f.close()
	if debug:print "it took",time()-ti,"seconds"
def enterSentences ( sql , cursor , sentences , filename , textid , annotatorName , eraseAllAnnos , notenter = [ "order" ] , sentencefeatures = { } , defaultAnnotatorName = "parser" , tokname = None ) : if debug : print "entering" , filename , "as annotation by" , annotatorName , "into annotations of textid" , textid if not tokname : tokname = tokenname shortname = filename . split ( "/" ) [ - 1 ] userid = sql . enter ( cursor , "users" , [ "user" ] , ( annotatorName , ) ) sql . realname ( annotatorName , cursor ) if not userid : print "the user is not in the database" return scounter , wcounter = 0 , 0 ti = time ( ) if eraseAllAnnos : sql . clean ( cursor , "sentences" , [ "textid" ] , ( textid , ) ) sql . clean ( cursor , "sentencesearch" , [ "textid" ] , ( textid , ) ) for i , tree in enumerate ( sentences ) : if len ( tree ) < maxlength : scounter += 1 sentence = " " . join ( [ tree [ j ] . get ( tokname , "" ) for j in sorted ( tree ) ] ) sentenceid = sql . enter ( cursor , "sentences" , [ "nr" , "sentence" , "textid" ] , ( scounter , sentence , textid , ) ) sql . enter ( cursor , "sentencesearch" , [ "nr" , "sentence" , "textid" ] , ( scounter , sentence , textid , ) ) if tree . sentencefeatures : for a , v in tree . sentencefeatures . iteritems ( ) : sql . enter ( cursor , "sentencefeatures" , [ "sentenceid" , "attr" , "value" ] , ( sentenceid , a , v , ) ) ws , sent , treeid = sql . enterTree ( cursor , tree , sentenceid , userid , notenter = notenter , intokenname = tokname ) wcounter += ws if debug : print "----" , scounter , "of" , len ( sentences ) , sent . encode ( "utf-8" ) , " - it took" , time ( ) - ti , "seconds up to now for" , wcounter , "words - " , float ( wcounter ) / ( time ( ) - ti ) , 'words per second.' else : print "ooooooh" if logfile : f = codecs . open ( "toolong.txt" , "a" , "utf-8" ) f . write ( "\t" . join ( [ filename , str ( len ( nodes ) ) , unicode ( tree ) ] ) + "\n" ) f . close ( ) f = codecs . open ( logfile , "a" , "utf-8" ) f . write ( "tooooooooooooooooolong: \n" + filename + "\n" + unicode ( tree ) + "\n\n\n" ) f . close ( ) else : r = sql . getall ( cursor , "sentences" , [ "textid" ] , [ textid ] ) mini = 0 for tree in sentences : if len ( tree ) < maxlength : distas = [ ] scounter += 1 inssentence = " " . join ( [ tree [ j ] . get ( tokname , "" ) for j in sorted ( tree ) ] ) entered = False for sentenceid , nr , sentence , textid in r [ mini : ] : if sentence == inssentence : mini = nr origdic = getorigdic ( sql , cursor , sentenceid , defaultAnnotatorName ) if len ( origdic ) == len ( tree ) : if debug : print "standard entry" , inssentence ws , sent , treeid = sql . enterTree ( cursor , tree , sentenceid , userid , notenter = notenter , intokenname = tokname ) else : if debug : print "try fuzzy entry" , inssentence ws , sent , treeid = fuzzyEnterSentence ( tree , origdic , sql , cursor , filename , textid , annotatorName , scounter , sentence , sentenceid , inssentence , userid , notenter = notenter ) wcounter += ws if debug : if sent : print "---" , sent . encode ( "utf-8" ) , "sentence" else : print "ooooooh" if logfile : f = codecs . open ( logfile , "a" , "utf-8" ) f . write ( "no sentence: \n" + filename + "\n" + unicode ( tree ) + "\n\n\n" ) f . close ( ) entered = True break else : distas += [ ( jellyfish . levenshtein_distance ( unicode ( sentence ) . encode ( "utf-8" ) , unicode ( inssentence ) . encode ( "utf-8" ) ) , ( tree , sentenceid , sentence ) ) ] if not entered : print "couldn't find '" + inssentence + "' in" , shortname , "! Haven't yet entered the annotation" if distas : tree , sentenceid , sentence = sorted ( distas ) [ 0 ] [ 1 ] print "will try to enter instead\n" , inssentence , "\ninto\n" , sentence origdic = getorigdic ( sql , cursor , sentenceid , defaultAnnotatorName ) ws , sent , treeid = fuzzyEnterSentence ( tree , origdic , sql , cursor , filename , textid , annotatorName , scounter , sentence , sentenceid , inssentence , userid , notenter = notenter ) wcounter += ws else : print "no data under that name:" , shortname else : print "ooooooh" if logfile : f = codecs . open ( "toolong.txt" , "a" , "utf-8" ) f . write ( "\t" . join ( [ filename , str ( len ( nodes ) ) , unicode ( tree ) ] ) + "\n" ) f . close ( ) f = codecs . open ( logfile , "a" , "utf-8" ) f . write ( "tooooooooooooooooolong: \n" + filename + "\n" + unicode ( tree ) + "\n\n\n" ) f . close ( ) if debug : print "it took" , time ( ) - ti , "seconds"
takes a list of nodedics (sentences)
puts them into the database
takes a list of nodedics ( sentences ) puts them into the database
"""
	takes a list of nodedics (sentences)
	
	puts them into the database
	
	""" #print "found",len(sentences),"sentences to enter" # TODO : remove trees completely with all features! # erase all existing sentences for this textid # erase all existing sentences for this textid #{ 1: {'head': 2, 'word': u'``', 'deps': [], 'lemma': u'--', 'tag': u'$(', 'rel': u'-PUNCT-', 'id': 1}, 2: {'head': 4, 'word': u'Ross', 'deps': [1, 3], 'lemma': u'--', 'tag': u'NE', 'rel': u'SUBJ', 'id': 2}, 3: {'head': 2, 'word': u'Perot', 'deps': [], 'lemma': u'--', 'tag': u'NE', 'rel': u'APP', 'id': 3}, 4: {'head': 0, 'word': u'w\xe4re', 'deps': [2, 5, 8], 'lemma': u'--', 'tag': u'VAFIN', 'rel': u'ROOT', 'id': 4}, 5: {'head': 4, 'word': u'vielleicht', 'deps': [], 'lemma': u'--', 'tag': u'ADV', 'rel': u'ADV', 'id': 5}, 6: {'head': 8, 'word': u'ein', 'deps': [], 'lemma': u'--', 'tag': u'ART', 'rel': u'DET', 'id': 6}, 7: {'head': 8, 'word': u'pr\xe4chtiger', 'deps': [], 'lemma': u'--', 'tag': u'ADJA', 'rel': u'ATTR', 'id': 7}, 8: {'head': 4, 'word': u'Diktator', 'deps': [6, 7, 9], 'lemma': u'--', 'tag': u'NN', 'rel': u'PRED', 'id': 8}, 9: {'head': 8, 'word': u"''", 'deps': [], 'lemma': u'--', 'tag': u'$(', 'rel': u'-PUNCT-', 'id': 9}} #for snode in sentences: #print snode # for every sentence #print "ooooooooooooooooo",len(tree) #print sentencefeatures # enter first tree for sentencewcounter, sent, treeid #' expected time remaining', float(scounter)/(time()-ti)*(len(sentences)-scounter) # not erasing, adding #print "enterSentences qqq" # for every sentence #print tree #print "found:",sentence #print "looking for:",inssentence #print len(tree),len(sentence.split()),len(tree)==len(sentence.split()) #origdic=sql.gettree(sid=sentenceid,username=defaultAnnotatorName)["tree"] # get best sentence #enterSentences(sql, cursor, sentences, filename, textid, annotatorName, eraseAllAnnos=True, notenter=notenter, sentencefeatures=sentencefeatures, defaultAnnotatorName=defaultAnnotatorName, tokname=tokname) #db.commit()
sql
cursor
sentences
filename
textid
annotatorName
eraseAllAnnos
notenter
sentencefeatures
defaultAnnotatorName
tokname