class_docstr: CTCBaseSearcher class to be inherited by other
  CTC beam searchers.
  
  This class provides the basic functionalities for
  CTC beam search decoding.
  
  The space_token is required with a non-sentencepiece vocabulary list
  if your transcription is expecting to contain spaces.
  
  Arguments
  ---------
  blank_index : int
      The index of the blank token.
  vocab_list : list
      The list of the vocabulary tokens.
  space_token : int, optional
      The index of the space token. (default: -1)
  kenlm_model_path : str, optional
      The path to the kenlm model. Use .bin for a faster loading.
      If None, no language model will be used. (default: None)
  unigrams : list, optional
      The list of known word unigrams. (default: None)
  alpha : float
      Weight for language model during shallow fusion. (default: 0.5)
  beta : float
      Weight for length score adjustment of during scoring. (default: 1.5)
  unk_score_offset : float
      Amount of log score offset for unknown tokens. (default: -10.0)
  score_boundary : bool
      Whether to have kenlm respect boundaries when scoring. (default: True)
  beam_size : int, optional
      The width of the beam. (default: 100)
  beam_prune_logp : float, optional
      The pruning threshold for the beam. (default: -10.0)
  token_prune_min_logp : float, optional
      The pruning threshold for the tokens. (default: -5.0)
  prune_history : bool, optional
      Whether to prune the history. (default: True)
      Note: when using topk > 1, this should be set to False as
      it is pruning a lot of beams.
  blank_skip_threshold : float, optional
      Skip frames if log_prob(blank) > log(blank_skip_threshold), to speed up decoding.
      Note: This is only used when using the CUDA decoder, and it might worsen the WER/CER results. Use it at your own risk. (default: 1.0)
  topk : int, optional
      The number of top hypotheses to return. (default: 1)
  spm_token: str, optional
      The sentencepiece token. (default: "▁")
  
  Example
  -------
  >>> blank_index = 0
  >>> vocab_list = ['blank', 'a', 'b', 'c', ' ']
  >>> space_token = ' '
  >>> kenlm_model_path = None
  >>> unigrams = None
  >>> beam_size = 100
  >>> beam_prune_logp = -10.0
  >>> token_prune_min_logp = -5.0
  >>> prune_history = True
  >>> blank_skip_threshold = 1.0
  >>> topk = 1
  >>> searcher = CTCBaseSearcher(
  ...     blank_index=blank_index,
  ...     vocab_list=vocab_list,
  ...     space_token=space_token,
  ...     kenlm_model_path=kenlm_model_path,
  ...     unigrams=unigrams,
  ...     beam_size=beam_size,
  ...     beam_prune_logp=beam_prune_logp,
  ...     token_prune_min_logp=token_prune_min_logp,
  ...     prune_history=prune_history,
  ...     blank_skip_threshold=blank_skip_threshold,
  ...     topk=topk,
  ... )
class_docstr_tok: ["CTCBaseSearcher","class","to","be","inherited","by","other","CTC","beam","searchers",".","This","class","provides","the","basic","functionalities","for","CTC","beam","search","decoding",".","The","space_token","is","required","with","a","non-sentencepiece","vocabulary","list","if","your","transcription","is","expecting","to","contain","spaces",".","Arguments","--","--","--","--","-","blank_index",":","int","The","index","of","the","blank","token",".","vocab_list",":","list","The","list","of","the","vocabulary","tokens",".","space_token",":","int",",","optional","The","index","of","the","space","token",".","(","default",":","-1",")","kenlm_model_path",":","str",",","optional","The","path","to","the","kenlm","model",".","Use",".bin","for","a","faster","loading",".","If","None",",","no","language","model","will","be","used",".","(","default",":","None",")","unigrams",":","list",",","optional","The","list","of","known","word","unigrams",".","(","default",":","None",")","alpha",":","float","Weight","for","language","model","during","shallow","fusion",".","(","default",":","0.5",")","beta",":","float","Weight","for","length","score","adjustment","of","during","scoring",".","(","default",":","1.5",")","unk_score_offset",":","float","Amount","of","log","score","offset","for","unknown","tokens",".","(","default",":","-10.0",")","score_boundary",":","bool","Whether","to","have","kenlm","respect","boundaries","when","scoring",".","(","default",":","True",")","beam_size",":","int",",","optional","The","width","of","the","beam",".","(","default",":","100",")","beam_prune_logp",":","float",",","optional","The","pruning","threshold","for","the","beam",".","(","default",":","-10.0",")","token_prune_min_logp",":","float",",","optional","The","pruning","threshold","for","the","tokens",".","(","default",":","-5.0",")","prune_history",":","bool",",","optional","Whether","to","prune","the","history",".","(","default",":","True",")","Note",":","when","using","topk",">","1",",","this","should","be","set","to","False","as","it","is","pruning","a","lot","of","beams",".","blank_skip_threshold",":","float",",","optional","Skip","frames","if","log_prob","(","blank",")",">","log","(","blank_skip_threshold",")",",","to","speed","up","decoding",".","Note",":","This","is","only","used","when","using","the","CUDA","decoder",",","and","it","might","worsen","the","WER\/CER","results",".","Use","it","at","your","own","risk",".","(","default",":","1.0",")","topk",":","int",",","optional","The","number","of","top","hypotheses","to","return",".","(","default",":","1",")","spm_token",":","str",",","optional","The","sentencepiece","token",".","(","default",":","``","\u2581","''",")","Example","--","--","--","-",">",">",">","blank_index","=","0",">",">",">","vocab_list","=","[","'blank","'",",","'","a","'",",","'","b","'",",","'","c","'",",","'","'","]",">",">",">","space_token","=","'","'",">",">",">","kenlm_model_path","=","None",">",">",">","unigrams","=","None",">",">",">","beam_size","=","100",">",">",">","beam_prune_logp","=","-10.0",">",">",">","token_prune_min_logp","=","-5.0",">",">",">","prune_history","=","True",">",">",">","blank_skip_threshold","=","1.0",">",">",">","topk","=","1",">",">",">","searcher","=","CTCBaseSearcher","(","...","blank_index=blank_index",",","...","vocab_list=vocab_list",",","...","space_token=space_token",",","...","kenlm_model_path=kenlm_model_path",",","...","unigrams=unigrams",",","...","beam_size=beam_size",",","...","beam_prune_logp=beam_prune_logp",",","...","token_prune_min_logp=token_prune_min_logp",",","...","prune_history=prune_history",",","...","blank_skip_threshold=blank_skip_threshold",",","...","topk=topk",",","...",")"]
docstr: Filter out beams that are the same over max_ngram history.
  
  Since n-gram language models have a finite history when scoring a new token, we can use that
  fact to prune beams that only differ early on (more than n tokens in the past) and keep only the
  higher scoring ones. Note that this helps speed up the decoding process but comes at the cost of
  some amount of beam diversity. If more than the top beam is used in the output it should
  potentially be disabled.
  
  Taken from: https://github.com/kensho-technologies/pyctcdecode
  
  Arguments
  ---------
  beams : list
      The list of the beams.
  lm_order : int
      The order of the language model.
  
  Returns
  -------
  list
      The list of CTCBeam.
docstr_tok: ["Filter","out","beams","that","are","the","same","over","max_ngram","history",".","Since","n-gram","language","models","have","a","finite","history","when","scoring","a","new","token",",","we","can","use","that","fact","to","prune","beams","that","only","differ","early","on","(","more","than","n","tokens","in","the","past",")","and","keep","only","the","higher","scoring","ones",".","Note","that","this","helps","speed","up","the","decoding","process","but","comes","at","the","cost","of","some","amount","of","beam","diversity",".","If","more","than","the","top","beam","is","used","in","the","output","it","should","potentially","be","disabled",".","Taken","from",":","https",":","\/\/github.com\/kensho-technologies\/pyctcdecode","Arguments","--","--","--","--","-","beams",":","list","The","list","of","the","beams",".","lm_order",":","int","The","order","of","the","language","model",".","Returns","--","--","--","-","list","The","list","of","CTCBeam","."]
code: def _prune_history(
      self, beams: List[CTCBeam], lm_order: int
  ) -> List[CTCBeam]:
      """Filter out beams that are the same over max_ngram history.
  
      Since n-gram language models have a finite history when scoring a new token, we can use that
      fact to prune beams that only differ early on (more than n tokens in the past) and keep only the
      higher scoring ones. Note that this helps speed up the decoding process but comes at the cost of
      some amount of beam diversity. If more than the top beam is used in the output it should
      potentially be disabled.
  
      Taken from: https://github.com/kensho-technologies/pyctcdecode
  
      Arguments
      ---------
      beams : list
          The list of the beams.
      lm_order : int
          The order of the language model.
  
      Returns
      -------
      list
          The list of CTCBeam.
      """
      # let's keep at least 1 word of history
      min_n_history = max(1, lm_order - 1)
      seen_hashes = set()
      filtered_beams = []
      # for each beam after this, check if we need to add it
      for lm_beam in beams:
          # hash based on history that can still affect lm scoring going forward
          hash_idx = (
              tuple(lm_beam.text.split()[-min_n_history:]),
              lm_beam.partial_word,
              lm_beam.last_token,
          )
          if hash_idx not in seen_hashes:
              filtered_beams.append(CTCBeam.from_lm_beam(lm_beam))
              seen_hashes.add(hash_idx)
      return filtered_beams
  
imports: from itertools import groupby
  from speechbrain.dataio.dataio import length_to_mask
  import math
  import dataclasses
  import numpy
  import heapq
  import logging
  import torch
  import warnings
  from typing import Dict, List, Optional, Union, Any, Tuple
label_desc: Use image node_id 7 for calling the CTCBaseSearcher obj's underlying member method code with example usage: obj._prune_history(beams, lm_order) and returns: filtered_beams