text: import os
  import gensim
  import smart_open
  
  test_data_dir = os.path.join(gensim.__path__[0], 'test', 'test_data')
  lee_train_file = os.path.join(test_data_dir, 'lee_background.cor')
  lee_test_file = os.path.join(test_data_dir, 'lee.cor')
  
  def read_corpus(fname, tokens_only=False):
      with smart_open.open(fname, encoding="iso-8859-1") as f:
          for i, line in enumerate(f):
              tokens = gensim.utils.simple_preprocess(line)
              if tokens_only:
                  yield tokens
              else:
                  # For training data, add tags
                  yield gensim.models.doc2vec.TaggedDocument(tokens, [i])
  
  train_corpus = list(read_corpus(lee_train_file))
  test_corpus = list(read_corpus(lee_test_file, tokens_only=True))
  
  print(train_corpus[:2])
  
  model = gensim.models.doc2vec.Doc2Vec(vector_size=50, min_count=2, epochs=40)
  model.build_vocab(train_corpus)
  
  print(f"Word 'penalty' appeared {model.wv.get_vecattr('penalty', 'count')} times in the training corpus.")
  
  
  model.train(train_corpus, total_examples=model.corpus_count, epochs=model.epochs)
  
  vector = model.infer_vector(['only', 'you', 'can', 'prevent', 'forest', 'fires'])
  print(vector)
  
  ranks = []
  second_ranks = []
  for doc_id in range(len(train_corpus)):
      inferred_vector = model.infer_vector(train_corpus[doc_id].words)
      sims = model.dv.most_similar([inferred_vector], topn=len(model.dv))
      rank = [docid for docid, sim in sims].index(doc_id)
      ranks.append(rank)
  
      second_ranks.append(sims[1])