code: def fit_s3e_on_corpus(
      processor,
      model,
      corpus,
      n_clusters=10,
      mean_removal=True,
      pca_removal=True,
      pca_n_components=300,
      pca_n_top_components=10,
      default_token_weight=1,
      min_token_occurrences=0,
      svd_postprocessing=False,
      use_gpu=False,
      batch_size=50,
  ):
      """
      Pooling of word/token embeddings as described by Wang et al in the paper
      "Efficient Sentence Embedding via Semantic Subspace Analysis"
      (https://arxiv.org/abs/2002.09620)
      Adjusted their implementation from here: https://github.com/BinWang28/Sentence-Embedding-S3E
  
      This method fits the "model" on a custom corpus. This includes the derivation of token_weights depending on
      token occurences in the corpus, creation of the semantic clusters via k-means and a couple of
      pre-/post-processing steps to normalize the embeddings.
  
      The resulting objects can be saved or directly passed to the Inferencer to get the actual embeddings for your sentences.
      Note: Some operations like `mean_removal` imply changes on the AdaptiveModel or Processor. That's why we return them.
  
      :param processor: FARM Processor with a Tokenizer used for reading the corpus (e.g. Inference Processor)
      :param model: FARM AdaptiveModel with an embedding layer in the LM (currently only supporting 'WordEmbedding_LM' as a language model)
      :param corpus: Path to a text file or a str
      :param n_clusters: Number of clusters for S3E. The more clusters, the higher the dimensionality of the resulting embeddings.
      :param mean_removal: Bool, whether to remove the mean from the token embeddings (preprocessing)
      :param pca_removal: Bool, whether to remove pca components from the token embeddings (preprocessing)
      :param pca_n_components: int, how many PCA components to fit if `pca_removal` is enabled
      :param pca_n_top_components: int, how many top PCA components to remove if `pca_removal` is enabled
      :param default_token_weight: float, what weight to assign for tokens that are in vocab but not in corpus
      :param min_token_occurrences: int, mininum number of token occurrences in the corpus for keeping it in the vocab.
                                    Helps to shrink the model & speed it up.
      :param svd_postprocessing: Bool, whether to remove the top truncated SVD / LSA components from the sentence embeddings (postprocessing).
                                 Note: Requires creating all sentence embeddings once for the corpus slowing down this method substantially.
                                       Doesn't impact later inference speed though.
      :param use_gpu: bool, whether to use a GPU
      :param batch_size: int, size of batch for the inferencer (only needed when `svd_postprocessing` is enabled)
      :return: model, processor, s3e_stats
      """
  
      from farm.infer import Inferencer
      from farm.modeling.tokenization import tokenize_with_metadata
  
      # Get tokens of corpus
      if isinstance(corpus, Path):
          logger.info("Reading corpus for fitting S3E ")
          with open(corpus, "r") as f:
              corpus = f.read()
      else:
          assert (
              type(corpus) == str
          ), "`corpus` must be of type str or Path()"
  
      tokenized_corpus = tokenize_with_metadata(
          corpus, processor.tokenizer
      )["tokens"]
      token_counts = dict(Counter(tokenized_corpus))
      n_tokens = sum(token_counts.values())
  
      # Trim vocab & embeddings to most frequent tokens (only to improve speed & ram consumption)
      model.language_model.trim_vocab(
          token_counts, processor, min_threshold=min_token_occurrences
      )
  
      # Normalize embeddings
      model.language_model.normalize_embeddings(
          zero_mean=mean_removal,
          pca_removal=pca_removal,
          pca_n_components=pca_n_components,
          pca_n_top_components=pca_n_top_components,
      )
      normalized_word_embs = (
          model.language_model.model.embeddings.cpu().numpy()
      )
  
      # Get token weights
      token_weights = {}
      eps = 1e-3
      for word, id in processor.tokenizer.vocab.items():
          if word in token_counts:
              token_weights[id] = eps / (
                  eps + token_counts[word] / n_tokens
              )
          else:
              # words that are in vocab but not present in corpus get the default weight
              token_weights[id] = default_token_weight
  
      # Construct Cluster
      weight_list = np.array(list(token_weights.values()))
      logger.info("Creating clusters for S3E embeddings")
      kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(
          normalized_word_embs, sample_weight=weight_list
      )
  
      s3e_stats = {
          "token_to_cluster": kmeans.labels_,
          "centroids": kmeans.cluster_centers_,
          "token_weights": token_weights,
          "svd_components": None,
      }
  
      if svd_postprocessing:
          logger.info(
              "Post processing sentence embeddings using principal component removal"
          )
  
          # Input
          sentences = [
              {"text": s}
              for s in corpus.split("\n")
              if len(s.strip()) > 0
          ]
  
          # Get embeddings
          try:
              inferencer = Inferencer(
                  model=model,
                  processor=processor,
                  task_type="embeddings",
                  gpu=use_gpu,
                  batch_size=batch_size,
                  extraction_strategy="s3e",
                  extraction_layer=-1,
                  s3e_stats=s3e_stats,
              )
              result = inferencer.inference_from_dicts(dicts=sentences)
          finally:
              inferencer.close_multiprocessing_pool()
          sentence_embeddings = [s["vec"] for s in result]
          sentence_embeddings = np.vstack(sentence_embeddings)
  
          # Principal Component Removal
          svd = TruncatedSVD(n_components=1, n_iter=7, random_state=0)
          svd.fit(sentence_embeddings)
          s3e_stats["svd_components"] = svd.components_
  
      return model, processor, s3e_stats
  
imports: from __future__ import absolute_import, division, print_function, unicode_literals
  import io
  import json
  import logging
  import os
  import unicodedata
  from pathlib import Path
  import numpy
  import pandas
  from tqdm import tqdm
  from transformers import BertTokenizer
  from sklearn.decomposition import TruncatedSVD
  from sklearn.cluster import KMeans
  from collections import Counter
  from farm.file_utils import load_from_cache
label_desc: Use image node_id 10 for calling a global function with example usage: fit_s3e_on_corpus(processor, model, corpus, n_clusters, mean_removal, pca_removal, pca_n_components, pca_n_top_components, default_token_weight, min_token_occurrences, svd_postprocessing, use_gpu, batch_size) and returns: model,  processor,  s3e_stats