  
          doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
  
          for i, true_entity in enumerate(true_entity_list):
              false_entities = false_entities_list[i]
  
              true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
              # print("encoding dim", len(true_entity_encoding[0]))
  
              consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)
  
              doc_mse, doc_diff = self._calculate_similarity(doc_encoding, consensus_encoding)
  
              entity_mses = list()
  
              true_mse, true_diffs = self._calculate_similarity(true_entity_encoding, consensus_encoding)
              # print("true_mse", true_mse)
              # print("true_diffs", true_diffs)
              entity_mses.append(true_mse)
              # true_exp = np.exp(true_entity_encoding.dot(consensus_encoding_t))
              # print("true_exp", true_exp)
  
              # false_exp_sum = 0
  
              if doc_diffs is not None:
                  doc_diffs += doc_diff
                  entity_diffs += true_diffs
              else:
                  doc_diffs = doc_diff
                  entity_diffs = true_diffs
  
              for false_entity in false_entities:
                  false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                  false_mse, false_diffs = self._calculate_similarity(false_entity_encoding, consensus_encoding)
                  # print("false_mse", false_mse)
                  # false_exp = np.exp(false_entity_encoding.dot(consensus_encoding_t))
                  # print("false_exp", false_exp)
                  # print("false_diffs", false_diffs)
                  entity_mses.append(false_mse)
                  # if false_mse > true_mse:
                      # true_diffs = true_diffs - false_diffs ???
                  # false_exp_sum += false_exp
  
              # prob = true_exp / false_exp_sum
              # print("prob", prob)
  
              entity_mses = sorted(entity_mses)
              # mse_sum = sum(entity_mses)
              # entity_probs = [1 - x/mse_sum for x in entity_mses]
              # print("entity_mses", entity_mses)
              # print("entity_probs", entity_probs)
              true_index = entity_mses.index(true_mse)
              # print("true index", true_index)
              # print("true prob", entity_probs[true_index])
  
              # print("training loss", true_mse)
  
              # print()
  
          # TODO: proper backpropagation taking ranking of elements into account ?
          # TODO backpropagation also for negative examples
  
          if doc_diffs is not None:
              doc_diffs = doc_diffs / len(true_entity_list)
  
              true_entity_bp(entity_diffs, sgd=self.sgd_entity)
              article_bp(doc_diffs, sgd=self.sgd_article)
  
  
      # TODO delete ?
      def _simple_cnn_model(self, internal_dim):
          nr_class = len(self.labels)
          with Model.define_operators({">>": chain}):
              model_entity = SpacyVectors >> flatten_add_lengths >> Pooling(mean_pool)    # entity encoding
              model_doc = SpacyVectors >> flatten_add_lengths >> Pooling(mean_pool)       # doc encoding
              output_layer = Softmax(nr_class, internal_dim*2)
              model = (model_entity | model_doc) >> output_layer
          # model.tok2vec = chain(tok2vec, flatten)
          model.nO = nr_class
          return model
  
      def predict(self, entity_doc, article_doc):
          entity_encoding = self.entity_encoder(entity_doc)
          doc_encoding = self.article_encoder(article_doc)
  
          print("entity_encodings", len(entity_encoding), entity_encoding)
          print("doc_encodings", len(doc_encoding), doc_encoding)
          mse, diffs = self._calculate_similarity(entity_encoding, doc_encoding)
          print("mse", mse)
  
          return mse
  
      # TODO: expand to more than 2 vectors
      def _calculate_consensus(self, vector1, vector2):
          if len(vector1) != len(vector2):
              raise ValueError("To calculate consenus, both vectors should be of equal length")
  
          avg = (vector2 + vector1) / 2
 
</s> remove         # TODO: one call only to begin_update ?

        entity_diffs = None
        doc_diffs = None
 </s> add  </s> remove         diffs = (vector1 - vector2)
        error_sum = (diffs ** 2).sum()
        mean_square_error = error_sum / len(vector1)
        return float(mean_square_error), diffs
 </s> add         vector1_t = vector1.transpose()
        e = self._calculate_dot_exp(vector2, vector1_t)
        e_sum = 0
        for v in allvectors:
            e_sum += self._calculate_dot_exp(v, vector1_t)

        return float(e / e_sum)

    @staticmethod
    def _calculate_loss(true_prob, all_probs):
        """ all_probs should include true_prob ! """
        return -1 * np.log(true_prob / sum(all_probs))

    @staticmethod
    def _calculate_doc_gradient(loss, doc_vector, true_vector, false_vectors):
        gradient = np.zeros(len(doc_vector))
        for i in range(len(doc_vector)):
            min_false = min(x[i] for x in false_vectors)
            max_false = max(x[i] for x in false_vectors)

            if true_vector[i] > max_false:
                if doc_vector[i] > 0:
                    gradient[i] = 0
                else:
                    gradient[i] = -loss
            elif true_vector[i] < min_false:
                if doc_vector[i] > 0:
                    gradient[i] = loss
                if doc_vector[i] < 0:
                    gradient[i] = 0
            else:
                target = 0  # non-distinctive vector positions should convert to 0
                gradient[i] = doc_vector[i] - target

        return gradient </s> remove     def _calculate_similarity(self, vector1, vector2):
 </s> add     def _calculate_probability(self, vector1, vector2, allvectors):
        """ Make sure that vector2 is included in allvectors """ </s> remove         doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
 </s> add  </s> remove                     best_entity, lowest_mse = self._predict(examples, article_doc)
 </s> add                     best_entity, highest_prob = self._predict(examples, article_doc)
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
  
          doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
  
          for i, true_entity in enumerate(true_entity_list):
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
            for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX
      def _calculate_consensus(self, vector1, vector2):
          if len(vector1) != len(vector2):
              raise ValueError("To calculate consenus, both vectors should be of equal length")
  
          avg = (vector2 + vector1) / 2
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 implement loss function using dot product and prob estimate per candidate cluster 
examples/pipeline/wiki_entity_linking/train_el.py