      if run_training:
          print("STEP 6: training ", datetime.datetime.now())
          my_nlp = spacy.load('en_core_web_md')
          trainer = EL_Model(kb=my_kb, nlp=my_nlp)
          trainer.train_model(training_dir=TRAINING_DIR, entity_descr_output=ENTITY_DESCR, trainlimit=50, devlimit=50)
          print()
  
      # STEP 7: apply the EL algorithm on the dev dataset
      if apply_to_dev:
          my_nlp = spacy.load('en_core_web_md')
 
</s> remove             false_entities = false_entities_list[i]

            true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
            # print("encoding dim", len(true_entity_encoding[0]))

            consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

            doc_mse, doc_diff = self._calculate_similarity(doc_encoding, consensus_encoding)

            entity_mses = list()

            true_mse, true_diffs = self._calculate_similarity(true_entity_encoding, consensus_encoding)
            # print("true_mse", true_mse)
            # print("true_diffs", true_diffs)
            entity_mses.append(true_mse)
            # true_exp = np.exp(true_entity_encoding.dot(consensus_encoding_t))
            # print("true_exp", true_exp)

            # false_exp_sum = 0

            if doc_diffs is not None:
                doc_diffs += doc_diff
                entity_diffs += true_diffs
            else:
                doc_diffs = doc_diff
                entity_diffs = true_diffs

            for false_entity in false_entities:
                false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                false_mse, false_diffs = self._calculate_similarity(false_entity_encoding, consensus_encoding)
                # print("false_mse", false_mse)
                # false_exp = np.exp(false_entity_encoding.dot(consensus_encoding_t))
                # print("false_exp", false_exp)
                # print("false_diffs", false_diffs)
                entity_mses.append(false_mse)
                # if false_mse > true_mse:
                    # true_diffs = true_diffs - false_diffs ???
                # false_exp_sum += false_exp

            # prob = true_exp / false_exp_sum
            # print("prob", prob)

            entity_mses = sorted(entity_mses)
            # mse_sum = sum(entity_mses)
            # entity_probs = [1 - x/mse_sum for x in entity_mses]
            # print("entity_mses", entity_mses)
            # print("entity_probs", entity_probs)
            true_index = entity_mses.index(true_mse)
            # print("true index", true_index)
            # print("true prob", entity_probs[true_index])

            # print("training loss", true_mse)

            # print()

        # TODO: proper backpropagation taking ranking of elements into account ?
        # TODO backpropagation also for negative examples

        if doc_diffs is not None:
            doc_diffs = doc_diffs / len(true_entity_list)

            true_entity_bp(entity_diffs, sgd=self.sgd_entity)
            article_bp(doc_diffs, sgd=self.sgd_article)


    # TODO delete ?
    def _simple_cnn_model(self, internal_dim):
        nr_class = len(self.labels)
        with Model.define_operators({">>": chain}):
            model_entity = SpacyVectors >> flatten_add_lengths >> Pooling(mean_pool)    # entity encoding
            model_doc = SpacyVectors >> flatten_add_lengths >> Pooling(mean_pool)       # doc encoding
            output_layer = Softmax(nr_class, internal_dim*2)
            model = (model_entity | model_doc) >> output_layer
        # model.tok2vec = chain(tok2vec, flatten)
        model.nO = nr_class
        return model

    def predict(self, entity_doc, article_doc):
        entity_encoding = self.entity_encoder(entity_doc)
        doc_encoding = self.article_encoder(article_doc)

        print("entity_encodings", len(entity_encoding), entity_encoding)
        print("doc_encodings", len(doc_encoding), doc_encoding)
        mse, diffs = self._calculate_similarity(entity_encoding, doc_encoding)
        print("mse", mse)

        return mse

    # TODO: expand to more than 2 vectors
 </s> add             for cnt in range(10):
            #try:
                false_vectors = list()
                false_entities = false_entities_list[i]
                if len(false_entities) > 0:
                    # TODO: batch per doc
                    doc_encoding, article_bp = self.article_encoder.begin_update([article_doc], drop=drop)
                    doc_encoding = doc_encoding[0]
                    print()
                    print(cnt)
                    print("doc", doc_encoding)

                    for false_entity in false_entities:
                        # TODO: one call only to begin_update ?
                        false_entity_encoding, false_entity_bp = self.entity_encoder.begin_update([false_entity], drop=drop)
                        false_entity_encoding = false_entity_encoding[0]
                        false_vectors.append(false_entity_encoding)

                    true_entity_encoding, true_entity_bp = self.entity_encoder.begin_update([true_entity], drop=drop)
                    true_entity_encoding = true_entity_encoding[0]

                    all_vectors = [true_entity_encoding]
                    all_vectors.extend(false_vectors)

                    # consensus_encoding = self._calculate_consensus(doc_encoding, true_entity_encoding)

                    true_prob = self._calculate_probability(doc_encoding, true_entity_encoding, all_vectors)
                    print("true", true_prob, true_entity_encoding)

                    all_probs = [true_prob]
                    for false_vector in false_vectors:
                        false_prob = self._calculate_probability(doc_encoding, false_vector, all_vectors)
                        print("false", false_prob, false_vector)
                        all_probs.append(false_prob)

                    loss = self._calculate_loss(true_prob, all_probs).astype(np.float32)
                    if self.PRINT_LOSS:
                        print("loss", round(loss, 5))

                    doc_gradient = self._calculate_doc_gradient(loss, doc_encoding, true_entity_encoding, false_vectors)
                    print("doc_gradient", doc_gradient)
                    article_bp([doc_gradient.astype(np.float32)], sgd=self.sgd_article)
            #except Exception as e:
                #pass


    # TODO: FIX </s> remove         diffs = (vector1 - vector2)
        error_sum = (diffs ** 2).sum()
        mean_square_error = error_sum / len(vector1)
        return float(mean_square_error), diffs
 </s> add         vector1_t = vector1.transpose()
        e = self._calculate_dot_exp(vector2, vector1_t)
        e_sum = 0
        for v in allvectors:
            e_sum += self._calculate_dot_exp(v, vector1_t)

        return float(e / e_sum)

    @staticmethod
    def _calculate_loss(true_prob, all_probs):
        """ all_probs should include true_prob ! """
        return -1 * np.log(true_prob / sum(all_probs))

    @staticmethod
    def _calculate_doc_gradient(loss, doc_vector, true_vector, false_vectors):
        gradient = np.zeros(len(doc_vector))
        for i in range(len(doc_vector)):
            min_false = min(x[i] for x in false_vectors)
            max_false = max(x[i] for x in false_vectors)

            if true_vector[i] > max_false:
                if doc_vector[i] > 0:
                    gradient[i] = 0
                else:
                    gradient[i] = -loss
            elif true_vector[i] < min_false:
                if doc_vector[i] > 0:
                    gradient[i] = loss
                if doc_vector[i] < 0:
                    gradient[i] = 0
            else:
                target = 0  # non-distinctive vector positions should convert to 0
                gradient[i] = doc_vector[i] - target

        return gradient </s> remove             entity_encoding = self.entity_encoder([entity])
            mse, _ = self._calculate_similarity(doc_encoding, entity_encoding)
            if not best_entity or mse < lowest_mse:
                lowest_mse = mse
 </s> add             entity_encoding = entity_to_vector[entity]
            prob = self._calculate_probability(doc_encoding, entity_encoding, entity_to_vector.values())
            if not best_entity or prob > highest_prob:
                highest_prob = prob
        trainer.train_model(training_dir=TRAINING_DIR, entity_descr_output=ENTITY_DESCR, trainlimit=1, devlimit=5)
      if run_training:
          print("STEP 6: training ", datetime.datetime.now())
          my_nlp = spacy.load('en_core_web_md')
          trainer = EL_Model(kb=my_kb, nlp=my_nlp)
        trainer.train_model(training_dir=TRAINING_DIR, entity_descr_output=ENTITY_DESCR, trainlimit=1, devlimit=5)
          print()
  
      # STEP 7: apply the EL algorithm on the dev dataset
      if apply_to_dev:
          my_nlp = spacy.load('en_core_web_md')
keep keep keep keep replace keep keep keep keep keep
 implement loss function using dot product and prob estimate per candidate cluster 
examples/pipeline/wiki_entity_linking/wiki_nel_pipeline.py