M1-Arborator-NaijaSUD/arborator/lib/treebankfiles.py
def simpleEnterSentences(sql, trees, dbtextname, annotatorName, eraseAllAnnos, sentencefeatures={}, preserveSampleWithSameName=False):
	"""
	takes a list of trees (nodedics)
	puts them into the database
	if preserveSampleWithSameName: add _ to names to create new name of sample
	"""
	ti = time()
	db,cursor=sql.open()
	if preserveSampleWithSameName:
		while sql.getUniqueId(cursor, "texts", ["textname"],[dbtextname])!=None:
			dbtextname+="_"
	textid = sql.enter(cursor, "texts",["textname"],(dbtextname,))
	print "textid",textid, 
	if eraseAllAnnos: 
		db.close()
		sql.removeText(textid)
		db,cursor=sql.open()
		textid = sql.enter(cursor, "texts",["textname"],(dbtextname,))
		print "textid",textid
	if debug: 
		print "found",len(trees),"trees to enter"
		print "entering an annotation by",annotatorName, "into annotations of textid",textid
			
	userid = sql.enter(cursor, "users",["user"],(annotatorName,))
	sql.realname(annotatorName, cursor)
	if not userid:
		print "the user is not in the database"
		return
	
	scounter,wcounter=0,0

	for i,tree in enumerate(trees): # for every sentence
		scounter+=1
		sentence=" ".join([tree[j].get("t","") for j in sorted(tree)])
		#cursor.execute("select rowid from sentences,texts  where nr=? and sentence=?;",
		sentenceid = sql.getUniqueId(cursor, "sentences", ["nr","sentence","textid"], [scounter,sentence,textid])
		#print "found sentenceid",sentenceid
		if not sentenceid:
			sentenceid = sql.enter(cursor, "sentences",["nr","sentence","textid"],(scounter,sentence,textid,))
			#print "made sentenceid",sentenceid
		sql.enter(cursor, "sentencesearch",["nr","sentence","textid"],(scounter,sentence,textid,))
		if sentencefeatures == True:
			for a,v in tree.sentencefeatures.iteritems():
				sql.enter(cursor, "sentencefeatures",["sentenceid","attr","value"],(sentenceid,a,v,))
		elif sentencefeatures:	
			for a,v in sentencefeatures[i].iteritems():
				sql.enter(cursor, "sentencefeatures",["sentenceid","attr","value"],(sentenceid,a,v,))
		#print tree
		ws, sent, treeid = sql.enterTree(cursor, tree, sentenceid, userid)
		#print "ws, sent, treeid",ws, sent, treeid
		wcounter+=ws
		#if debug:print "----",scounter,"of",len(tree),sent.encode("utf-8")," - it took",time()-ti,"seconds up to now for",wcounter, "words - ", float(wcounter)/(time()-ti),'words per second.'
	
	db.commit()
	db.close()	
	print "entered",sql.getNumberTokensPerText(textid, recompute=True),"tokens"
	if debug:print "it took",time()-ti,"seconds"
	return textid

	takes a list of trees (nodedics)
	puts them into the database
	if preserveSampleWithSameName: add _ to names to create new name of sample
	
takes a list of trees (nodedics)
puts them into the database
if preserveSampleWithSameName: add _ to names to create new name of sample
takes a list of trees ( nodedics ) puts them into the database if preserveSampleWithSameName : add _ to names to create new name of sample
def simpleEnterSentences(sql, trees, dbtextname, annotatorName, eraseAllAnnos, sentencefeatures={}, preserveSampleWithSameName=False):
	ti = time()
	db,cursor=sql.open()
	if preserveSampleWithSameName:
		while sql.getUniqueId(cursor, "texts", ["textname"],[dbtextname])!=None:
			dbtextname+="_"
	textid = sql.enter(cursor, "texts",["textname"],(dbtextname,))
	print "textid",textid, 
	if eraseAllAnnos: 
		db.close()
		sql.removeText(textid)
		db,cursor=sql.open()
		textid = sql.enter(cursor, "texts",["textname"],(dbtextname,))
		print "textid",textid
	if debug: 
		print "found",len(trees),"trees to enter"
		print "entering an annotation by",annotatorName, "into annotations of textid",textid
	userid = sql.enter(cursor, "users",["user"],(annotatorName,))
	sql.realname(annotatorName, cursor)
	if not userid:
		print "the user is not in the database"
		return
	scounter,wcounter=0,0
	for i,tree in enumerate(trees): 
		scounter+=1
		sentence=" ".join([tree[j].get("t","") for j in sorted(tree)])
		sentenceid = sql.getUniqueId(cursor, "sentences", ["nr","sentence","textid"], [scounter,sentence,textid])
		if not sentenceid:
			sentenceid = sql.enter(cursor, "sentences",["nr","sentence","textid"],(scounter,sentence,textid,))
		sql.enter(cursor, "sentencesearch",["nr","sentence","textid"],(scounter,sentence,textid,))
		if sentencefeatures == True:
			for a,v in tree.sentencefeatures.iteritems():
				sql.enter(cursor, "sentencefeatures",["sentenceid","attr","value"],(sentenceid,a,v,))
		elif sentencefeatures:	
			for a,v in sentencefeatures[i].iteritems():
				sql.enter(cursor, "sentencefeatures",["sentenceid","attr","value"],(sentenceid,a,v,))
		ws, sent, treeid = sql.enterTree(cursor, tree, sentenceid, userid)
		wcounter+=ws
	db.commit()
	db.close()	
	print "entered",sql.getNumberTokensPerText(textid, recompute=True),"tokens"
	if debug:print "it took",time()-ti,"seconds"
	return textid
def simpleEnterSentences ( sql , trees , dbtextname , annotatorName , eraseAllAnnos , sentencefeatures = { } , preserveSampleWithSameName = False ) : ti = time ( ) db , cursor = sql . open ( ) if preserveSampleWithSameName : while sql . getUniqueId ( cursor , "texts" , [ "textname" ] , [ dbtextname ] ) != None : dbtextname += "_" textid = sql . enter ( cursor , "texts" , [ "textname" ] , ( dbtextname , ) ) print "textid" , textid , if eraseAllAnnos : db . close ( ) sql . removeText ( textid ) db , cursor = sql . open ( ) textid = sql . enter ( cursor , "texts" , [ "textname" ] , ( dbtextname , ) ) print "textid" , textid if debug : print "found" , len ( trees ) , "trees to enter" print "entering an annotation by" , annotatorName , "into annotations of textid" , textid userid = sql . enter ( cursor , "users" , [ "user" ] , ( annotatorName , ) ) sql . realname ( annotatorName , cursor ) if not userid : print "the user is not in the database" return scounter , wcounter = 0 , 0 for i , tree in enumerate ( trees ) : scounter += 1 sentence = " " . join ( [ tree [ j ] . get ( "t" , "" ) for j in sorted ( tree ) ] ) sentenceid = sql . getUniqueId ( cursor , "sentences" , [ "nr" , "sentence" , "textid" ] , [ scounter , sentence , textid ] ) if not sentenceid : sentenceid = sql . enter ( cursor , "sentences" , [ "nr" , "sentence" , "textid" ] , ( scounter , sentence , textid , ) ) sql . enter ( cursor , "sentencesearch" , [ "nr" , "sentence" , "textid" ] , ( scounter , sentence , textid , ) ) if sentencefeatures == True : for a , v in tree . sentencefeatures . iteritems ( ) : sql . enter ( cursor , "sentencefeatures" , [ "sentenceid" , "attr" , "value" ] , ( sentenceid , a , v , ) ) elif sentencefeatures : for a , v in sentencefeatures [ i ] . iteritems ( ) : sql . enter ( cursor , "sentencefeatures" , [ "sentenceid" , "attr" , "value" ] , ( sentenceid , a , v , ) ) ws , sent , treeid = sql . enterTree ( cursor , tree , sentenceid , userid ) wcounter += ws db . commit ( ) db . close ( ) print "entered" , sql . getNumberTokensPerText ( textid , recompute = True ) , "tokens" if debug : print "it took" , time ( ) - ti , "seconds" return textid
takes a list of trees (nodedics)
puts them into the database
if preserveSampleWithSameName: add _ to names to create new name of sample
takes a list of trees ( nodedics ) puts them into the database if preserveSampleWithSameName : add _ to names to create new name of sample
"""
	takes a list of trees (nodedics)
	puts them into the database
	if preserveSampleWithSameName: add _ to names to create new name of sample
	""" # for every sentence #cursor.execute("select rowid from sentences,texts  where nr=? and sentence=?;", #print "found sentenceid",sentenceid #print "made sentenceid",sentenceid #print tree #print "ws, sent, treeid",ws, sent, treeid #if debug:print "----",scounter,"of",len(tree),sent.encode("utf-8")," - it took",time()-ti,"seconds up to now for",wcounter, "words - ", float(wcounter)/(time()-ti),'words per second.'
sql
trees
dbtextname
annotatorName
eraseAllAnnos
sentencefeatures
preserveSampleWithSameName