class_docstr: Align two similiar tokenizations.
  
  Args:
      source (Union[Iterable[str], str]): Source text tokens or string.
      target (Union[Iterable[str], str]): Target text tokens or string.
  
  Usage:
      ta = TokenAligner(source_tokens, target_tokens)
      target_span = ta.project_span(*source_span)
  
  Uses Levenshtein distance to align two similar tokenizations, and provide facilities to project
  indices and spans from the source tokenization to the target.
  
  Let source contain m tokens and M chars, and target contain n tokens and N chars. The token
  alignment is treated as a (sparse) m x n adjacency matrix T representing the bipartite graph
  between the source and target tokens.
  
  This is constructed by performing a character-level alignment using Levenshtein distance to
  obtain a (M x N) character adjacency matrix C. We then construct token-to-character matricies
  U (m x M) and V (n x N) and construct T as:
      T = (U C V')
  where V' denotes the transpose.
  
  Spans of non-aligned bytes are assumed to contain a many-to-many alignment of all chars in that
  range. This can lead to unwanted alignments if, for example, two consecutive tokens are mapped
  to escape sequences:
      source: ["'s", "["]
      target: ["&apos;", "s", "&#91;"]
  In the above case, "'s'" may be spuriously aligned to "&apos;" while "[" has no character match
  with "s" or "&#91;", and so is aligned to both. To make a correct alignment more likely, ensure
  that the characters in target correspond as closely as possible to those in source. For example,
  the following will align correctly:
      source: ["'s", "["]
      target: ["'", "s", "["]
class_docstr_tok: ["Align","two","similiar","tokenizations",".","Args",":","source","(","Union","[","Iterable","[","str","]",",","str","]",")",":","Source","text","tokens","or","string",".","target","(","Union","[","Iterable","[","str","]",",","str","]",")",":","Target","text","tokens","or","string",".","Usage",":","ta","=","TokenAligner","(","source_tokens",",","target_tokens",")","target_span","=","ta.project_span","(","*","source_span",")","Uses","Levenshtein","distance","to","align","two","similar","tokenizations",",","and","provide","facilities","to","project","indices","and","spans","from","the","source","tokenization","to","the","target",".","Let","source","contain","m","tokens","and","M","chars",",","and","target","contain","n","tokens","and","N","chars",".","The","token","alignment","is","treated","as","a","(","sparse",")","m","x","n","adjacency","matrix","T","representing","the","bipartite","graph","between","the","source","and","target","tokens",".","This","is","constructed","by","performing","a","character-level","alignment","using","Levenshtein","distance","to","obtain","a","(","M","x","N",")","character","adjacency","matrix","C.","We","then","construct","token-to-character","matricies","U","(","m","x","M",")","and","V","(","n","x","N",")","and","construct","T","as",":","T","=","(","U","C","V","'",")","where","V","'","denotes","the","transpose",".","Spans","of","non-aligned","bytes","are","assumed","to","contain","a","many-to-many","alignment","of","all","chars","in","that","range",".","This","can","lead","to","unwanted","alignments","if",",","for","example",",","two","consecutive","tokens","are","mapped","to","escape","sequences",":","source",":","[","``","'s","''",",","``","[","``","]","target",":","[","``","&","apos",";","''",",","``","s","''",",","``","&","#","91",";","''","]","In","the","above","case",",","``","'s","'","''","may","be","spuriously","aligned","to","``","&","apos",";","''","while","``","[","``","has","no","character","match","with","``","s","''","or","``","&","#","91",";","''",",","and","so","is","aligned","to","both",".","To","make","a","correct","alignment","more","likely",",","ensure","that","the","characters","in","target","correspond","as","closely","as","possible","to","those","in","source",".","For","example",",","the","following","will","align","correctly",":","source",":","[","``","'s","''",",","``","[","``","]","target",":","[","``","'","''",",","``","s","''",",","``","[","``","]"]
docstr: Project a span from source to target token sequence.
  
  Notes:
      When param inclusive=False, the end index is interpreted as exclusive,
      and the end of the span returned by the function will also be exclusive.
      When param inclusive=True, both start and end indexes are interpreted as inclusive,
      and the span returned by the function will also be inclusive.
  
  Examples:
      >>> source_tokens = ['abc', 'def', 'ghi', 'jkl']
      >>> target_tokens = ['abc', 'd', 'ef', 'ghi', 'jkl']
      >>> ta = TokenAligner(source_tokens, target_tokens)
      >>> start, end = 0, 2
      >>> print(ta.project_token_span(start, end))
      (0, 3)
  
  Raise:
      When target span is empty
  
  Returns:
      Tuple[int, int] representing the target span corresponding to the source span.
docstr_tok: ["Project","a","span","from","source","to","target","token","sequence",".","Notes",":","When","param","inclusive=False",",","the","end","index","is","interpreted","as","exclusive",",","and","the","end","of","the","span","returned","by","the","function","will","also","be","exclusive",".","When","param","inclusive=True",",","both","start","and","end","indexes","are","interpreted","as","inclusive",",","and","the","span","returned","by","the","function","will","also","be","inclusive",".","Examples",":",">",">",">","source_tokens","=","[","'abc","'",",","'def","'",",","'ghi","'",",","'jkl","'","]",">",">",">","target_tokens","=","[","'abc","'",",","'d","'",",","'ef","'",",","'ghi","'",",","'jkl","'","]",">",">",">","ta","=","TokenAligner","(","source_tokens",",","target_tokens",")",">",">",">","start",",","end","=","0",",","2",">",">",">","print","(","ta.project_token_span","(","start",",","end",")",")","(","0",",","3",")","Raise",":","When","target","span","is","empty","Returns",":","Tuple","[","int",",","int","]","representing","the","target","span","corresponding","to","the","source","span","."]
code: def project_token_span(
      self, start, end, inclusive=False
  ) -> Tuple[int, int]:
      """Project a span from source to target token sequence.
  
      Notes:
          When param inclusive=False, the end index is interpreted as exclusive,
          and the end of the span returned by the function will also be exclusive.
          When param inclusive=True, both start and end indexes are interpreted as inclusive,
          and the span returned by the function will also be inclusive.
  
      Examples:
          >>> source_tokens = ['abc', 'def', 'ghi', 'jkl']
          >>> target_tokens = ['abc', 'd', 'ef', 'ghi', 'jkl']
          >>> ta = TokenAligner(source_tokens, target_tokens)
          >>> start, end = 0, 2
          >>> print(ta.project_token_span(start, end))
          (0, 3)
  
      Raise:
          When target span is empty
  
      Returns:
          Tuple[int, int] representing the target span corresponding to the source span.
      """
      return self._project_span(
          mat=self.source_token_idx_to_target_token_idx,
          start=start,
          end=end,
          inclusive=inclusive,
      )
  
file_path: jiant/jiant/utils/retokenize.py
imports: from typing import Iterable, Sequence, Tuple, Union
  from Levenshtein.StringMatcher import StringMatcher
  from nltk.tokenize.util import string_span_tokenize
  import numpy
label_desc: Use image node_id 4 for calling the TokenAligner obj's underlying member method code with example usage: obj.project_token_span(start, end, inclusive) and returns: self