from gensim.corpora.wikicorpus import WikiCorpus
from gensim.models.word2vec import Word2Vec, LineSentence
from pprint import pprint
from copy import deepcopy
from multiprocessing import cpu_count

"""
Explanation: Online word2vec tutorial
So far, word2vec cannot increase the size of vocabulary after initial training. To handle unknown words, not in word2vec vocaburary, you must  retrain updated documents over again.
In this tutorial, we introduce gensim new feature, online vocaburary update. This additional feature overcomes the unknown word problems. Despite after initial training, we can continuously add new vocaburary to the pre-trained word2vec model using this online feature.
End of explanation
"""


%%bash
wget https://dumps.wikimedia.org/archive/2010/2010-11/enwiki/20101011/enwiki-20101011-pages-articles.xml.bz2
wget https://dumps.wikimedia.org/enwiki/20160820/enwiki-20160820-pages-articles.xml.bz2

"""
Explanation: Download wikipedia dump files
We use the past and the current version of wiki dump files as online training.
End of explanation
"""


old, new = [WikiCorpus('enwiki-{}-pages-articles.xml.bz2'.format(ymd)) for ymd in ['20101011', '20160820']]

def write_wiki(wiki, name, titles = []):
    with open('{}.wiki'.format(name), 'wb') as f:
        wiki.metadata = True
        for text, (page_id, title) in wiki.get_texts():
            if title not in titles:
                f.write(b' '.join(text)+b'\n')
                titles.append(title)
    return titles

old_titles = write_wiki(old, 'old')
all_titles = write_wiki(new, 'new', old_titles)

oldwiki, newwiki = [LineSentence(f+'.wiki') for f in ['old', 'new']]

"""
Explanation: Convert two wikipedia dump files
To avoid alert when convert old verision of wikipedia dump, you should download alternative wikicorpus.py in my repo.
End of explanation
"""


%%time
model = Word2Vec(oldwiki, min_count = 0, workers=cpu_count())
# model = Word2Vec.load('oldmodel')
oldmodel = deepcopy(model)
oldmodel.save('oldmodel')

"""
Explanation: Initial training
At first we train word2vec using "enwiki-20101011-pages-articles.xml.bz2". After that, we update model using "enwiki-20160820-pages-articles.xml.bz2".
End of explanation
"""


try:
    print(oldmodel.most_similar('babymetal'))
except KeyError as e:
    print(e)

"""
Explanation: Japanese new idol group, "Babymetal", weren't known worldwide in 2010, so that the word, "babymetal", is not in oldmodel vocaburary.
Note: In recent years, they became the famous idol group not only in Japan. They won many music awards and run world tour.
End of explanation
"""


%%time
model.build_vocab(newwiki, update=True)
model.train(newwiki)
model.save('newmodel')
# model = Word2Vec.load('newmodel')

"""
Explanation: Online update
To use online word2vec feature, set update=True when you use build_vocab using new documents.
End of explanation
"""


for m in ['oldmodel', 'model']:
    print('The vocabulary size of the', m, 'is', len(eval(m).vocab))

"""
Explanation: Model Comparison
By the online training, the size of vocaburaries are increased about 3 millions.
End of explanation
"""


try:
    pprint(model.most_similar('babymetal'))
except KeyError as e:
    print(e)

"""
Explanation: After online training, the word, "babymetal", is added in model. This word is simillar with rock and metal bands.
End of explanation
"""


w = 'zootopia'
for m in ['oldmodel', 'model']:
    print('The count of the word,'+w+', is', eval(m).vocab[w].count, 'in', m)
    pprint(eval(m).most_similar(w))
    print('')

"""
Explanation: The word, "Zootopia", become disney movie through the years.
In the past, the word, "Zootopia", was used just for an annual summer concert put on by New York top-40 radio station Z100, so that the word, "zootopia", is simillar with music festival.
In 2016, Zootopia is a American 3D computer-animated comedy film released by Walt Disney Pictures. As a result, the word, "zootopia", was often used as Animation films.
End of explanation
"""


