prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  wiki examples: sample code to get from tfidf doc to wikipedia title/uri and vice versa
  I very much like the LSI and LDA [wiki examples](https://github.com/RaRe-Technologies/gensim/blob/develop/docs/src/wiki.rst), but one aspect that i think is missing is: how to get from tf-idf doc vectors (or later LSI / LDA vecs) back to Wikipedia URIs (or titles if easier) and vice versa?
  
  Am i missing something obvious, or do i have to run another pass over the wiki dump, as the titles aren't saved anywhere?
  
  I'll happily make a PR to extend the examples with this...
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `gensim/scripts/make_wikicorpus.py`
  Content:
  ```
  1 #!/usr/bin/env python
  2 # -*- coding: utf-8 -*-
  3 #
  4 # Copyright (C) 2010 Radim Rehurek <radimrehurek@seznam.cz>
  5 # Copyright (C) 2012 Lars Buitinck <larsmans@gmail.com>
  6 # Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
  7 
  8 
  9 """
  10 USAGE: %(program)s WIKI_XML_DUMP OUTPUT_PREFIX [VOCABULARY_SIZE]
  11 
  12 Convert articles from a Wikipedia dump to (sparse) vectors. The input is a
  13 bz2-compressed dump of Wikipedia articles, in XML format.
  14 
  15 This actually creates three files:
  16 
  17 * `OUTPUT_PREFIX_wordids.txt`: mapping between words and their integer ids
  18 * `OUTPUT_PREFIX_bow.mm`: bag-of-words (word counts) representation, in
  19   Matrix Matrix format
  20 * `OUTPUT_PREFIX_tfidf.mm`: TF-IDF representation
  21 * `OUTPUT_PREFIX.tfidf_model`: TF-IDF model dump
  22 
  23 The output Matrix Market files can then be compressed (e.g., by bzip2) to save
  24 disk space; gensim's corpus iterators can work with compressed input, too.
  25 
  26 `VOCABULARY_SIZE` controls how many of the most frequent words to keep (after
  27 removing tokens that appear in more than 10%% of all documents). Defaults to
  28 100,000.
  29 
  30 If you have the `pattern` package installed, this script will use a fancy
  31 lemmatization to get a lemma of each token (instead of plain alphabetic
  32 tokenizer). The package is available at https://github.com/clips/pattern .
  33 
  34 Example:
  35   python -m gensim.scripts.make_wikicorpus ~/gensim/results/enwiki-latest-pages-articles.xml.bz2 ~/gensim/results/wiki
  36 """
  37 
  38 
  39 import logging
  40 import os.path
  41 import sys
  42 
  43 from gensim.corpora import Dictionary, HashDictionary, MmCorpus, WikiCorpus
  44 from gensim.models import TfidfModel
  45 
  46 
  47 # Wiki is first scanned for all distinct word types (~7M). The types that
  48 # appear in more than 10% of articles are removed and from the rest, the
  49 # DEFAULT_DICT_SIZE most frequent types are kept.
  50 DEFAULT_DICT_SIZE = 100000
  51 
  52 
  53 if __name__ == '__main__':
  54     program = os.path.basename(sys.argv[0])
  55     logger = logging.getLogger(program)
  56 
  57     logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')
  58     logging.root.setLevel(level=logging.INFO)
  59     logger.info("running %s", ' '.join(sys.argv))
  60 
  61     # check and process input arguments
  62     if len(sys.argv) < 3:
  63         print(globals()['__doc__'] % locals())
  64         sys.exit(1)
  65     inp, outp = sys.argv[1:3]
  66 
  67     if not os.path.isdir(os.path.dirname(outp)):
  68         raise SystemExit("Error: The output directory does not exist. Create the directory and try again.")
  69 
  70     if len(sys.argv) > 3:
  71         keep_words = int(sys.argv[3])
  72     else:
  73         keep_words = DEFAULT_DICT_SIZE
  74     online = 'online' in program
  75     lemmatize = 'lemma' in program
  76     debug = 'nodebug' not in program
  77 
  78     if online:
  79         dictionary = HashDictionary(id_range=keep_words, debug=debug)
  80         dictionary.allow_update = True  # start collecting document frequencies
  81         wiki = WikiCorpus(inp, lemmatize=lemmatize, dictionary=dictionary)
  82         # ~4h on my macbook pro without lemmatization, 3.1m articles (august 2012)
  83         MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000)
  84         # with HashDictionary, the token->id mapping is only fully instantiated now, after `serialize`
  85         dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)
  86         dictionary.save_as_text(outp + '_wordids.txt.bz2')
  87         wiki.save(outp + '_corpus.pkl.bz2')
  88         dictionary.allow_update = False
  89     else:
  90         wiki = WikiCorpus(inp, lemmatize=lemmatize)  # takes about 9h on a macbook pro, for 3.5m articles (june 2011)
  91         # only keep the most frequent words (out of total ~8.2m unique tokens)
  92         wiki.dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)
  93         # save dictionary and bag-of-words (term-document frequency matrix)
  94         MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000)  # another ~9h
  95         wiki.dictionary.save_as_text(outp + '_wordids.txt.bz2')
  96         # load back the id->word mapping directly from file
  97         # this seems to save more memory, compared to keeping the wiki.dictionary object from above
  98         dictionary = Dictionary.load_from_text(outp + '_wordids.txt.bz2')
  99     del wiki
  100 
  101     # initialize corpus reader and word->id mapping
  102     mm = MmCorpus(outp + '_bow.mm')
  103 
  104     # build tfidf, ~50min
  105     tfidf = TfidfModel(mm, id2word=dictionary, normalize=True)
  106     tfidf.save(outp + '.tfidf_model')
  107 
  108     # save tfidf vectors in matrix market format
  109     # ~4h; result file is 15GB! bzip2'ed down to 4.5GB
  110     MmCorpus.serialize(outp + '_tfidf.mm', tfidf[mm], progress_cnt=10000)
  111 
  112     logger.info("finished running %s", program)
  113
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/gensim/scripts/make_wikicorpus.py b/gensim/scripts/make_wikicorpus.py
  --- a/gensim/scripts/make_wikicorpus.py
  +++ b/gensim/scripts/make_wikicorpus.py
  @@ -12,13 +12,15 @@
   Convert articles from a Wikipedia dump to (sparse) vectors. The input is a
   bz2-compressed dump of Wikipedia articles, in XML format.
   
  -This actually creates three files:
  +This actually creates several files:
   
  -* `OUTPUT_PREFIX_wordids.txt`: mapping between words and their integer ids
  -* `OUTPUT_PREFIX_bow.mm`: bag-of-words (word counts) representation, in
  -  Matrix Matrix format
  -* `OUTPUT_PREFIX_tfidf.mm`: TF-IDF representation
  -* `OUTPUT_PREFIX.tfidf_model`: TF-IDF model dump
  +* `OUTPUT_PREFIX_wordids.txt.bz2`: mapping between words and their integer ids
  +* `OUTPUT_PREFIX_bow.mm`: bag-of-words (word counts) representation in Matrix Market format
  +* `OUTPUT_PREFIX_bow.mm.index`: index for `OUTPUT_PREFIX_bow.mm`
  +* `OUTPUT_PREFIX_bow.mm.metadata.cpickle`: titles of documents
  +* `OUTPUT_PREFIX_tfidf.mm`: TF-IDF representation in Matix Market format
  +* `OUTPUT_PREFIX_tfidf.mm.index`: index for `OUTPUT_PREFIX_tfidf.mm`
  +* `OUTPUT_PREFIX.tfidf_model`: TF-IDF model
   
   The output Matrix Market files can then be compressed (e.g., by bzip2) to save
   disk space; gensim's corpus iterators can work with compressed input, too.
  @@ -80,7 +82,7 @@
           dictionary.allow_update = True  # start collecting document frequencies
           wiki = WikiCorpus(inp, lemmatize=lemmatize, dictionary=dictionary)
           # ~4h on my macbook pro without lemmatization, 3.1m articles (august 2012)
  -        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000)
  +        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000, metadata=True)
           # with HashDictionary, the token->id mapping is only fully instantiated now, after `serialize`
           dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)
           dictionary.save_as_text(outp + '_wordids.txt.bz2')
  @@ -91,7 +93,7 @@
           # only keep the most frequent words (out of total ~8.2m unique tokens)
           wiki.dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)
           # save dictionary and bag-of-words (term-document frequency matrix)
  -        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000)  # another ~9h
  +        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000, metadata=True)  # another ~9h
           wiki.dictionary.save_as_text(outp + '_wordids.txt.bz2')
           # load back the id->word mapping directly from file
           # this seems to save more memory, compared to keeping the wiki.dictionary object from above
  
verification_info: {"golden_diff": "diff --git a/gensim/scripts/make_wikicorpus.py b/gensim/scripts/make_wikicorpus.py\n--- a/gensim/scripts/make_wikicorpus.py\n+++ b/gensim/scripts/make_wikicorpus.py\n@@ -12,13 +12,15 @@\n Convert articles from a Wikipedia dump to (sparse) vectors. The input is a\n bz2-compressed dump of Wikipedia articles, in XML format.\n \n-This actually creates three files:\n+This actually creates several files:\n \n-* `OUTPUT_PREFIX_wordids.txt`: mapping between words and their integer ids\n-* `OUTPUT_PREFIX_bow.mm`: bag-of-words (word counts) representation, in\n-  Matrix Matrix format\n-* `OUTPUT_PREFIX_tfidf.mm`: TF-IDF representation\n-* `OUTPUT_PREFIX.tfidf_model`: TF-IDF model dump\n+* `OUTPUT_PREFIX_wordids.txt.bz2`: mapping between words and their integer ids\n+* `OUTPUT_PREFIX_bow.mm`: bag-of-words (word counts) representation in Matrix Market format\n+* `OUTPUT_PREFIX_bow.mm.index`: index for `OUTPUT_PREFIX_bow.mm`\n+* `OUTPUT_PREFIX_bow.mm.metadata.cpickle`: titles of documents\n+* `OUTPUT_PREFIX_tfidf.mm`: TF-IDF representation in Matix Market format\n+* `OUTPUT_PREFIX_tfidf.mm.index`: index for `OUTPUT_PREFIX_tfidf.mm`\n+* `OUTPUT_PREFIX.tfidf_model`: TF-IDF model\n \n The output Matrix Market files can then be compressed (e.g., by bzip2) to save\n disk space; gensim's corpus iterators can work with compressed input, too.\n@@ -80,7 +82,7 @@\n         dictionary.allow_update = True  # start collecting document frequencies\n         wiki = WikiCorpus(inp, lemmatize=lemmatize, dictionary=dictionary)\n         # ~4h on my macbook pro without lemmatization, 3.1m articles (august 2012)\n-        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000)\n+        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000, metadata=True)\n         # with HashDictionary, the token->id mapping is only fully instantiated now, after `serialize`\n         dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)\n         dictionary.save_as_text(outp + '_wordids.txt.bz2')\n@@ -91,7 +93,7 @@\n         # only keep the most frequent words (out of total ~8.2m unique tokens)\n         wiki.dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)\n         # save dictionary and bag-of-words (term-document frequency matrix)\n-        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000)  # another ~9h\n+        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000, metadata=True)  # another ~9h\n         wiki.dictionary.save_as_text(outp + '_wordids.txt.bz2')\n         # load back the id->word mapping directly from file\n         # this seems to save more memory, compared to keeping the wiki.dictionary object from above\n", "issue": "wiki examples: sample code to get from tfidf doc to wikipedia title/uri and vice versa\nI very much like the LSI and LDA [wiki examples](https://github.com/RaRe-Technologies/gensim/blob/develop/docs/src/wiki.rst), but one aspect that i think is missing is: how to get from tf-idf doc vectors (or later LSI / LDA vecs) back to Wikipedia URIs (or titles if easier) and vice versa?\r\n\r\nAm i missing something obvious, or do i have to run another pass over the wiki dump, as the titles aren't saved anywhere?\r\n\r\nI'll happily make a PR to extend the examples with this...\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) 2010 Radim Rehurek <radimrehurek@seznam.cz>\n# Copyright (C) 2012 Lars Buitinck <larsmans@gmail.com>\n# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html\n\n\n\"\"\"\nUSAGE: %(program)s WIKI_XML_DUMP OUTPUT_PREFIX [VOCABULARY_SIZE]\n\nConvert articles from a Wikipedia dump to (sparse) vectors. The input is a\nbz2-compressed dump of Wikipedia articles, in XML format.\n\nThis actually creates three files:\n\n* `OUTPUT_PREFIX_wordids.txt`: mapping between words and their integer ids\n* `OUTPUT_PREFIX_bow.mm`: bag-of-words (word counts) representation, in\n  Matrix Matrix format\n* `OUTPUT_PREFIX_tfidf.mm`: TF-IDF representation\n* `OUTPUT_PREFIX.tfidf_model`: TF-IDF model dump\n\nThe output Matrix Market files can then be compressed (e.g., by bzip2) to save\ndisk space; gensim's corpus iterators can work with compressed input, too.\n\n`VOCABULARY_SIZE` controls how many of the most frequent words to keep (after\nremoving tokens that appear in more than 10%% of all documents). Defaults to\n100,000.\n\nIf you have the `pattern` package installed, this script will use a fancy\nlemmatization to get a lemma of each token (instead of plain alphabetic\ntokenizer). The package is available at https://github.com/clips/pattern .\n\nExample:\n  python -m gensim.scripts.make_wikicorpus ~/gensim/results/enwiki-latest-pages-articles.xml.bz2 ~/gensim/results/wiki\n\"\"\"\n\n\nimport logging\nimport os.path\nimport sys\n\nfrom gensim.corpora import Dictionary, HashDictionary, MmCorpus, WikiCorpus\nfrom gensim.models import TfidfModel\n\n\n# Wiki is first scanned for all distinct word types (~7M). The types that\n# appear in more than 10% of articles are removed and from the rest, the\n# DEFAULT_DICT_SIZE most frequent types are kept.\nDEFAULT_DICT_SIZE = 100000\n\n\nif __name__ == '__main__':\n    program = os.path.basename(sys.argv[0])\n    logger = logging.getLogger(program)\n\n    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')\n    logging.root.setLevel(level=logging.INFO)\n    logger.info(\"running %s\", ' '.join(sys.argv))\n\n    # check and process input arguments\n    if len(sys.argv) < 3:\n        print(globals()['__doc__'] % locals())\n        sys.exit(1)\n    inp, outp = sys.argv[1:3]\n\n    if not os.path.isdir(os.path.dirname(outp)):\n        raise SystemExit(\"Error: The output directory does not exist. Create the directory and try again.\")\n\n    if len(sys.argv) > 3:\n        keep_words = int(sys.argv[3])\n    else:\n        keep_words = DEFAULT_DICT_SIZE\n    online = 'online' in program\n    lemmatize = 'lemma' in program\n    debug = 'nodebug' not in program\n\n    if online:\n        dictionary = HashDictionary(id_range=keep_words, debug=debug)\n        dictionary.allow_update = True  # start collecting document frequencies\n        wiki = WikiCorpus(inp, lemmatize=lemmatize, dictionary=dictionary)\n        # ~4h on my macbook pro without lemmatization, 3.1m articles (august 2012)\n        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000)\n        # with HashDictionary, the token->id mapping is only fully instantiated now, after `serialize`\n        dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)\n        dictionary.save_as_text(outp + '_wordids.txt.bz2')\n        wiki.save(outp + '_corpus.pkl.bz2')\n        dictionary.allow_update = False\n    else:\n        wiki = WikiCorpus(inp, lemmatize=lemmatize)  # takes about 9h on a macbook pro, for 3.5m articles (june 2011)\n        # only keep the most frequent words (out of total ~8.2m unique tokens)\n        wiki.dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)\n        # save dictionary and bag-of-words (term-document frequency matrix)\n        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000)  # another ~9h\n        wiki.dictionary.save_as_text(outp + '_wordids.txt.bz2')\n        # load back the id->word mapping directly from file\n        # this seems to save more memory, compared to keeping the wiki.dictionary object from above\n        dictionary = Dictionary.load_from_text(outp + '_wordids.txt.bz2')\n    del wiki\n\n    # initialize corpus reader and word->id mapping\n    mm = MmCorpus(outp + '_bow.mm')\n\n    # build tfidf, ~50min\n    tfidf = TfidfModel(mm, id2word=dictionary, normalize=True)\n    tfidf.save(outp + '.tfidf_model')\n\n    # save tfidf vectors in matrix market format\n    # ~4h; result file is 15GB! bzip2'ed down to 4.5GB\n    MmCorpus.serialize(outp + '_tfidf.mm', tfidf[mm], progress_cnt=10000)\n\n    logger.info(\"finished running %s\", program)\n", "path": "gensim/scripts/make_wikicorpus.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) 2010 Radim Rehurek <radimrehurek@seznam.cz>\n# Copyright (C) 2012 Lars Buitinck <larsmans@gmail.com>\n# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html\n\n\n\"\"\"\nUSAGE: %(program)s WIKI_XML_DUMP OUTPUT_PREFIX [VOCABULARY_SIZE]\n\nConvert articles from a Wikipedia dump to (sparse) vectors. The input is a\nbz2-compressed dump of Wikipedia articles, in XML format.\n\nThis actually creates several files:\n\n* `OUTPUT_PREFIX_wordids.txt.bz2`: mapping between words and their integer ids\n* `OUTPUT_PREFIX_bow.mm`: bag-of-words (word counts) representation in Matrix Market format\n* `OUTPUT_PREFIX_bow.mm.index`: index for `OUTPUT_PREFIX_bow.mm`\n* `OUTPUT_PREFIX_bow.mm.metadata.cpickle`: titles of documents\n* `OUTPUT_PREFIX_tfidf.mm`: TF-IDF representation in Matix Market format\n* `OUTPUT_PREFIX_tfidf.mm.index`: index for `OUTPUT_PREFIX_tfidf.mm`\n* `OUTPUT_PREFIX.tfidf_model`: TF-IDF model\n\nThe output Matrix Market files can then be compressed (e.g., by bzip2) to save\ndisk space; gensim's corpus iterators can work with compressed input, too.\n\n`VOCABULARY_SIZE` controls how many of the most frequent words to keep (after\nremoving tokens that appear in more than 10%% of all documents). Defaults to\n100,000.\n\nIf you have the `pattern` package installed, this script will use a fancy\nlemmatization to get a lemma of each token (instead of plain alphabetic\ntokenizer). The package is available at https://github.com/clips/pattern .\n\nExample:\n  python -m gensim.scripts.make_wikicorpus ~/gensim/results/enwiki-latest-pages-articles.xml.bz2 ~/gensim/results/wiki\n\"\"\"\n\n\nimport logging\nimport os.path\nimport sys\n\nfrom gensim.corpora import Dictionary, HashDictionary, MmCorpus, WikiCorpus\nfrom gensim.models import TfidfModel\n\n\n# Wiki is first scanned for all distinct word types (~7M). The types that\n# appear in more than 10% of articles are removed and from the rest, the\n# DEFAULT_DICT_SIZE most frequent types are kept.\nDEFAULT_DICT_SIZE = 100000\n\n\nif __name__ == '__main__':\n    program = os.path.basename(sys.argv[0])\n    logger = logging.getLogger(program)\n\n    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')\n    logging.root.setLevel(level=logging.INFO)\n    logger.info(\"running %s\", ' '.join(sys.argv))\n\n    # check and process input arguments\n    if len(sys.argv) < 3:\n        print(globals()['__doc__'] % locals())\n        sys.exit(1)\n    inp, outp = sys.argv[1:3]\n\n    if not os.path.isdir(os.path.dirname(outp)):\n        raise SystemExit(\"Error: The output directory does not exist. Create the directory and try again.\")\n\n    if len(sys.argv) > 3:\n        keep_words = int(sys.argv[3])\n    else:\n        keep_words = DEFAULT_DICT_SIZE\n    online = 'online' in program\n    lemmatize = 'lemma' in program\n    debug = 'nodebug' not in program\n\n    if online:\n        dictionary = HashDictionary(id_range=keep_words, debug=debug)\n        dictionary.allow_update = True  # start collecting document frequencies\n        wiki = WikiCorpus(inp, lemmatize=lemmatize, dictionary=dictionary)\n        # ~4h on my macbook pro without lemmatization, 3.1m articles (august 2012)\n        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000, metadata=True)\n        # with HashDictionary, the token->id mapping is only fully instantiated now, after `serialize`\n        dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)\n        dictionary.save_as_text(outp + '_wordids.txt.bz2')\n        wiki.save(outp + '_corpus.pkl.bz2')\n        dictionary.allow_update = False\n    else:\n        wiki = WikiCorpus(inp, lemmatize=lemmatize)  # takes about 9h on a macbook pro, for 3.5m articles (june 2011)\n        # only keep the most frequent words (out of total ~8.2m unique tokens)\n        wiki.dictionary.filter_extremes(no_below=20, no_above=0.1, keep_n=DEFAULT_DICT_SIZE)\n        # save dictionary and bag-of-words (term-document frequency matrix)\n        MmCorpus.serialize(outp + '_bow.mm', wiki, progress_cnt=10000, metadata=True)  # another ~9h\n        wiki.dictionary.save_as_text(outp + '_wordids.txt.bz2')\n        # load back the id->word mapping directly from file\n        # this seems to save more memory, compared to keeping the wiki.dictionary object from above\n        dictionary = Dictionary.load_from_text(outp + '_wordids.txt.bz2')\n    del wiki\n\n    # initialize corpus reader and word->id mapping\n    mm = MmCorpus(outp + '_bow.mm')\n\n    # build tfidf, ~50min\n    tfidf = TfidfModel(mm, id2word=dictionary, normalize=True)\n    tfidf.save(outp + '.tfidf_model')\n\n    # save tfidf vectors in matrix market format\n    # ~4h; result file is 15GB! bzip2'ed down to 4.5GB\n    MmCorpus.serialize(outp + '_tfidf.mm', tfidf[mm], progress_cnt=10000)\n\n    logger.info(\"finished running %s\", program)\n", "path": "gensim/scripts/make_wikicorpus.py"}]}