code: def paraphrase_mining_embeddings(
      embeddings: Tensor,
      query_chunk_size: int = 5000,
      corpus_chunk_size: int = 100000,
      max_pairs: int = 500000,
      top_k: int = 100,
      score_function: Callable[[Tensor, Tensor], Tensor] = cos_sim,
  ) -> List[List[Union[float, int]]]:
      """
      Given a list of sentences / texts, this function performs paraphrase mining. It compares all sentences against all
      other sentences and returns a list with the pairs that have the highest cosine similarity score.
  
      :param embeddings: A tensor with the embeddings
      :param query_chunk_size: Search for most similar pairs for #query_chunk_size at the same time. Decrease, to lower memory footprint (increases run-time).
      :param corpus_chunk_size: Compare a sentence simultaneously against #corpus_chunk_size other sentences. Decrease, to lower memory footprint (increases run-time).
      :param max_pairs: Maximal number of text pairs returned.
      :param top_k: For each sentence, we retrieve up to top_k other sentences
      :param score_function: Function for computing scores. By default, cosine similarity.
      :return: Returns a list of triplets with the format [score, id1, id2]
      """
  
      top_k += 1  # A sentence has the highest similarity to itself. Increase +1 as we are interest in distinct pairs
  
      # Mine for duplicates
      pairs = queue.PriorityQueue()
      min_score = -1
      num_added = 0
  
      for corpus_start_idx in range(
          0, len(embeddings), corpus_chunk_size
      ):
          for query_start_idx in range(
              0, len(embeddings), query_chunk_size
          ):
              scores = score_function(
                  embeddings[
                      query_start_idx : query_start_idx
                      + query_chunk_size
                  ],
                  embeddings[
                      corpus_start_idx : corpus_start_idx
                      + corpus_chunk_size
                  ],
              )
  
              scores_top_k_values, scores_top_k_idx = torch.topk(
                  scores,
                  min(top_k, len(scores[0])),
                  dim=1,
                  largest=True,
                  sorted=False,
              )
              scores_top_k_values = scores_top_k_values.cpu().tolist()
              scores_top_k_idx = scores_top_k_idx.cpu().tolist()
  
              for query_itr in range(len(scores)):
                  for top_k_idx, corpus_itr in enumerate(
                      scores_top_k_idx[query_itr]
                  ):
                      i = query_start_idx + query_itr
                      j = corpus_start_idx + corpus_itr
  
                      if (
                          i != j
                          and scores_top_k_values[query_itr][top_k_idx]
                          > min_score
                      ):
                          pairs.put(
                              (
                                  scores_top_k_values[query_itr][
                                      top_k_idx
                                  ],
                                  i,
                                  j,
                              )
                          )
                          num_added += 1
  
                          if num_added >= max_pairs:
                              entry = pairs.get()
                              min_score = entry[0]
  
      # Get the pairs
      added_pairs = set()  # Used for duplicate detection
      pairs_list = []
      while not pairs.empty():
          score, i, j = pairs.get()
          sorted_i, sorted_j = sorted([i, j])
  
          if (
              sorted_i != sorted_j
              and (sorted_i, sorted_j) not in added_pairs
          ):
              added_pairs.add((sorted_i, sorted_j))
              pairs_list.append([score, sorted_i, sorted_j])
  
      # Highest scores first
      pairs_list = sorted(pairs_list, key=lambda x: x[0], reverse=True)
      return pairs_list
  
imports: import functools
  import requests
  from torch import Tensor, device
  from typing import List, Callable
  from tqdm.autonotebook import tqdm
  import sys
  import importlib
  import os
  import torch
  import numpy
  import queue
  import logging
  from typing import Dict, Optional, Union
  from huggingface_hub import snapshot_download, hf_hub_download
  import heapq
label_desc: Use image node_id 8 for calling a global function with example usage: paraphrase_mining_embeddings(embeddings, query_chunk_size, corpus_chunk_size, max_pairs, top_k, score_function) and returns: pairs_list