<filename>tools/extvec.py
#encoding: utf-8

import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )

from random import random

def ldvocab(fname):
	rs = {}
	curid = 1
	with open(fname) as frd:
		for line in frd:
			tmp = line.rstrip("\r\n")
			if tmp:
				tmp = tmp.decode("utf-8")
				if not tmp in rs:
					rs[tmp] = curid
					curid += 1
				else:
					print("Something wrong in your vocab file")
					print(tmp+" appeared at least twice")
	return rs

# mpf : vocab.txt generated by vocab
# vecf : pre-defined vectors generated by GloVe or something
# rsf : result file(extracted vectors) will be loaded into load_glove
# vecsize : the dimension of word vectors
def extvec(mpf,vecf,rsf,vecsize):
	wdm = ldvocab(mpf)
	rsd = {}
	with open(vecf) as frd:
		for line in frd:
			tmp = line.strip()
			if tmp:
				tmp = tmp.decode("utf-8")
				ind = tmp.find(" ")
				wd = tmp[:ind]
				if wd == "<unk>":
					unkvec = tmp[ind+1:].split(" ")
				if wd in wdm:
					rsd[wdm[wd]] = tmp[ind+1:].split(" ")
	if not unkvec:
		unkvec = " ".join([str((random()-0.5)/vecsize) for i in xrange(vecsize)])
	rs = []
	for i in xrange(1,len(wdm)+1):
		if i in rsd:
			rs.append(rsd[i])
		else:
			rs.append(unkvec)
	rs = map(list, zip(*rs))# this line for load_glove, to tranpose vocabsize*vecsize to vecsize*vocabsize, so it could be correctly loaded.
	rs = "\n".join([" ".join(rsu) for rsu in rs])
	with open(rsf, "w") as fwrt:
		fwrt.write(rs.encode("utf-8"))

if __name__ == "__main__":
	extvec(sys.argv[1].decode("utf-8"),sys.argv[2].decode("utf-8"),sys.argv[3].decode("utf-8"),int(sys.argv[4].decode("utf-8")))
