M1-Arborator-NaijaSUD/arborator/lib/trees2train.py
def exportUniqueSentences(project, mode="lasttree",pattern=False):
	"""
	exports one tree per sentences: the first time the sentence is found, the newest tree
	"""
	sql = SQL(project)
	db,cursor=sql.open()
	sentences={} # toks -> tree
	outdir=os.path.join("..","projects",project,"export")
	try: os.mkdir(outdir)	
	except OSError: pass
	outfile=os.path.join(outdir,"allSentences.conll")
	if pattern:
		command="""select trees.rowid,userid,max(timestamp) from trees, sentences, texts where texts.rowid=sentences.textid and sentences.rowid=trees.sentenceid 
		and textname like "{pattern}"
		group by sentenceid order by trees.rowid;""".format(pattern=pattern)
	else:
		command="""select trees.rowid,userid,max(timestamp) from trees, sentences, texts where texts.rowid=sentences.textid and sentences.rowid=trees.sentenceid 
		group by sentenceid order by trees.rowid;"""
	for i,(treeid,userid,timestamp,) in enumerate(cursor.execute(command).fetchall()):
		tree=sql.gettree(treeid=treeid, indb=db, incursor=cursor)["tree"]
		toks=tuple(tree[i]["t"] for i in tree)
		print "___",i, "\r",
		if toks not in sentences:
			sentences[toks]=tree
	print "writing file with",len(sentences),"sentences..."
	conll.trees2conllFile([sentences[toks] for toks in sorted(sentences)], outfile=outfile, columns=10)
	return outfile

	exports one tree per sentences: the first time the sentence is found, the newest tree
	
exports one tree per sentences: the first time the sentence is found, the newest tree
exports one tree per sentences : the first time the sentence is found the newest tree
def exportUniqueSentences(project, mode="lasttree",pattern=False):
	sql = SQL(project)
	db,cursor=sql.open()
	sentences={} 
	outdir=os.path.join("..","projects",project,"export")
	try: os.mkdir(outdir)	
	except OSError: pass
	outfile=os.path.join(outdir,"allSentences.conll")
	if pattern:
		command="""select trees.rowid,userid,max(timestamp) from trees, sentences, texts where texts.rowid=sentences.textid and sentences.rowid=trees.sentenceid 
		and textname like "{pattern}"
		group by sentenceid order by trees.rowid;""".format(pattern=pattern)
	else:
		command="""select trees.rowid,userid,max(timestamp) from trees, sentences, texts where texts.rowid=sentences.textid and sentences.rowid=trees.sentenceid 
		group by sentenceid order by trees.rowid;"""
	for i,(treeid,userid,timestamp,) in enumerate(cursor.execute(command).fetchall()):
		tree=sql.gettree(treeid=treeid, indb=db, incursor=cursor)["tree"]
		toks=tuple(tree[i]["t"] for i in tree)
		print "___",i, "\r",
		if toks not in sentences:
			sentences[toks]=tree
	print "writing file with",len(sentences),"sentences..."
	conll.trees2conllFile([sentences[toks] for toks in sorted(sentences)], outfile=outfile, columns=10)
	return outfile
def exportUniqueSentences ( project , mode = "lasttree" , pattern = False ) : sql = SQL ( project ) db , cursor = sql . open ( ) sentences = { } outdir = os . path . join ( ".." , "projects" , project , "export" ) try : os . mkdir ( outdir ) except OSError : pass outfile = os . path . join ( outdir , "allSentences.conll" ) if pattern : command = """select trees.rowid,userid,max(timestamp) from trees, sentences, texts where texts.rowid=sentences.textid and sentences.rowid=trees.sentenceid 
		and textname like "{pattern}"
		group by sentenceid order by trees.rowid;""" . format ( pattern = pattern ) else : command = """select trees.rowid,userid,max(timestamp) from trees, sentences, texts where texts.rowid=sentences.textid and sentences.rowid=trees.sentenceid 
		group by sentenceid order by trees.rowid;""" for i , ( treeid , userid , timestamp , ) in enumerate ( cursor . execute ( command ) . fetchall ( ) ) : tree = sql . gettree ( treeid = treeid , indb = db , incursor = cursor ) [ "tree" ] toks = tuple ( tree [ i ] [ "t" ] for i in tree ) print "___" , i , "\r" , if toks not in sentences : sentences [ toks ] = tree print "writing file with" , len ( sentences ) , "sentences..." conll . trees2conllFile ( [ sentences [ toks ] for toks in sorted ( sentences ) ] , outfile = outfile , columns = 10 ) return outfile
exports one tree per sentences: the first time the sentence is found, the newest tree
exports one tree per sentences : the first time the sentence is found the newest tree
"""
	exports one tree per sentences: the first time the sentence is found, the newest tree
	""" # toks -> tree
project
mode
pattern