M1-Arborator-NaijaSUD/arborator/lib/treebankfiles.py
def vietspacetomorph(conllfilename):
	"""
	puts tokens that have spaces into separate tokens connected by "morph" links to the previous token
	"""
	trees=conll.conllFile2trees(conllfilename)
	print len(trees),"sentences read from",conllfilename
	newtrees = []
	
	for tree in trees:
		newtree = tree
		#print '\n\ndoing',tree.dictrepr()
		addedpos = 0
		for i in tree:
			
			t = tree[i]['t']
			if " " in t:
				#print "t:",t
				#insdic = {}
				insdic = conll.Tree()
				
				for j,m in enumerate(t.split()):
					#print j,m
					iind = 'i'+str(j+1)
					insdic[iind]=copy.deepcopy(tree[i]) # copy all old features by default
					insdic[iind]['t']=m
					insdic[iind]['lemma']=m
					insdic[iind]['id']=iind
					insdic[iind]['gov']={'i'+str(j):'morph'}
				
				#print 'insdic',insdic.dictrepr(),"to put at position",i #str(insdic).encode("utf-8")	
				newtree = integrate(copy.deepcopy(newtree),i+addedpos,insdic)
				#print 'newtree',newtree.dictrepr()
				addedpos+=len(t.split())-1
				#print "addedpos",addedpos,t.split()
		print tree.sentencefeatures
		newtree.sentencefeatures=tree.sentencefeatures
		newtrees+=[newtree]
		#break
	conll.trees2conllFile(newtrees, conllfilename+'.morphed.conllu')

	puts tokens that have spaces into separate tokens connected by "morph" links to the previous token
	
puts tokens that have spaces into separate tokens connected by "morph" links to the previous token
puts tokens that have spaces into separate tokens connected by " morph " links to the previous token
def vietspacetomorph(conllfilename):
	trees=conll.conllFile2trees(conllfilename)
	print len(trees),"sentences read from",conllfilename
	newtrees = []
	for tree in trees:
		newtree = tree
		addedpos = 0
		for i in tree:
			t = tree[i]['t']
			if " " in t:
				insdic = conll.Tree()
				for j,m in enumerate(t.split()):
					iind = 'i'+str(j+1)
					insdic[iind]=copy.deepcopy(tree[i]) 
					insdic[iind]['t']=m
					insdic[iind]['lemma']=m
					insdic[iind]['id']=iind
					insdic[iind]['gov']={'i'+str(j):'morph'}
				newtree = integrate(copy.deepcopy(newtree),i+addedpos,insdic)
				addedpos+=len(t.split())-1
		print tree.sentencefeatures
		newtree.sentencefeatures=tree.sentencefeatures
		newtrees+=[newtree]
	conll.trees2conllFile(newtrees, conllfilename+'.morphed.conllu')
def vietspacetomorph ( conllfilename ) : trees = conll . conllFile2trees ( conllfilename ) print len ( trees ) , "sentences read from" , conllfilename newtrees = [ ] for tree in trees : newtree = tree addedpos = 0 for i in tree : t = tree [ i ] [ 't' ] if " " in t : insdic = conll . Tree ( ) for j , m in enumerate ( t . split ( ) ) : iind = 'i' + str ( j + 1 ) insdic [ iind ] = copy . deepcopy ( tree [ i ] ) insdic [ iind ] [ 't' ] = m insdic [ iind ] [ 'lemma' ] = m insdic [ iind ] [ 'id' ] = iind insdic [ iind ] [ 'gov' ] = { 'i' + str ( j ) : 'morph' } newtree = integrate ( copy . deepcopy ( newtree ) , i + addedpos , insdic ) addedpos += len ( t . split ( ) ) - 1 print tree . sentencefeatures newtree . sentencefeatures = tree . sentencefeatures newtrees += [ newtree ] conll . trees2conllFile ( newtrees , conllfilename + '.morphed.conllu' )
puts tokens that have spaces into separate tokens connected by "morph" links to the previous token
puts tokens that have spaces into separate tokens connected by " morph " links to the previous token
"""
	puts tokens that have spaces into separate tokens connected by "morph" links to the previous token
	""" #print '\n\ndoing',tree.dictrepr() #print "t:",t #insdic = {} #print j,m # copy all old features by default #print 'insdic',insdic.dictrepr(),"to put at position",i #str(insdic).encode("utf-8")	 #print 'newtree',newtree.dictrepr() #print "addedpos",addedpos,t.split() #break
conllfilename