class_docstr: Class for augmenting source and target sentences
  with inline tags (placeholders).
  
  It requires a prepared tab-delimited dictionary of source-target
  words and phrases. A dictionary can be created with
  tools such as fastalign and it should ideally contain enough long
  phrases (of 3-4 words or more) for more realistic applications
  of start and end tags. A dictionary with 20.000-30.000 entries
  should give sufficient number of matches.
class_docstr_tok: ["Class","for","augmenting","source","and","target","sentences","with","inline","tags","(","placeholders",")",".","It","requires","a","prepared","tab-delimited","dictionary","of","source-target","words","and","phrases",".","A","dictionary","can","be","created","with","tools","such","as","fastalign","and","it","should","ideally","contain","enough","long","phrases","(","of","3-4","words","or","more",")","for","more","realistic","applications","of","start","and","end","tags",".","A","dictionary","with","20.000-30.000","entries","should","give","sufficient","number","of","matches","."]
docstr: Uses the dictionary to find exact source matches with corresponding
  target matches and adds both paired tags and standalone tags.
docstr_tok: ["Uses","the","dictionary","to","find","exact","source","matches","with","corresponding","target","matches","and","adds","both","paired","tags","and","standalone","tags","."]
code: def _tagged_src_tgt(self, src_example, tgt_example) -> tuple:
      """Uses the dictionary to find exact source matches with corresponding
      target matches and adds both paired tags and standalone tags."""
  
      maybe_augmented = src_example.split(self.src_delimiter)
      source_only = maybe_augmented[0].strip()
  
      augmented_part = (
          maybe_augmented[1].strip()
          if len(maybe_augmented) > 1
          else None
      )
  
      tokenized_source_string = source_only.split()
      tokenized_target_string = tgt_example.split()
  
      src_offset, tgt_offset = 0, 0
      src_with_tags, tgt_with_tags = list(), list()
  
      # We set the start number of tags to a random number from 1
      # to 12 + the number of subsequent tags that
      # will be added. We also apply weights to this choice so tags
      # are more probable to start from 1, then from 2, etc.
      # This way we cover most scenarios met in real usage and
      # the system will learn to handle a fairly large number of
      # numbered tags (but not an excessively large number)
      paired_tag_start_num = random.choices(
          range(1, self.max_tags + 1),
          weights=range(self.max_tags, 0, -1),
          k=1,
      )[0]
      single_tag_start_num = random.choices(
          range(1, self.max_tags + 1),
          weights=range(self.max_tags, 0, -1),
          k=1,
      )[0]
  
      is_match = False
      tag_counter = 0
  
      for src_match_end, pair in self.automaton.iter_long(source_only):
          if tag_counter == self.max_tags:
              break
  
          src_match_start = src_match_end - len(pair[0]) + 1
          tgt_match_start = tgt_example.find(pair[1], tgt_offset)
          tgt_match_end = tgt_match_start + len(pair[1])
  
          # Make sure we only search for exact matches (we don't want
          # to match part of words) and perform some bound checking
          if (
              (pair[1] not in " ".join(tokenized_target_string))
              or (
                  len(source_only) != src_match_end + 1
                  and not (
                      source_only[src_match_end + 1].isspace()
                      or source_only[src_match_end + 1]
                      in string.punctuation
                  )
              )
              or (
                  not source_only[src_match_start - 1].isspace()
                  and src_match_start != 0
              )
          ):
              continue
          else:
              source_index = 0
              for i, w in enumerate(tokenized_source_string):
                  if source_index == src_match_start:
                      source_index = i
                      break
                  else:
                      source_index += len(w) + 1
  
              target_index = 0
              for i, w in enumerate(tokenized_target_string):
                  if target_index == tgt_match_start:
                      target_index = i
                      break
                  else:
                      target_index += len(w) + 1
  
              src_term = " ".join(
                  tokenized_source_string[
                      source_index : source_index + len(pair[0].split())
                  ]
              )
              tgt_term = " ".join(
                  tokenized_target_string[
                      target_index : target_index + len(pair[1].split())
                  ]
              )
  
              # Create all possible tag forms. We inject a special
              # unicode char (∥) as a placeholder for whitespace in order
              # to keep the indices unaltered. This char is replaced with
              # spaces before we return the augmented examples.
              src_single_tags = (
                  f"{source_only[src_offset: src_match_start]}"
                  f"{self.isolated_tag_prefix}{single_tag_start_num}"
                  f"{self.isolated_tag_suffix}∥{src_term}∥"
              )
              src_paired_tags = (
                  f"{source_only[src_offset: src_match_start]}"
                  f"{self.paired_stag_prefix}{paired_tag_start_num}"
                  f"{self.paired_stag_suffix}∥{src_term}∥"
                  f"{self.paired_etag_prefix}{paired_tag_start_num}"
                  f"{self.paired_etag_suffix}"
              )
  
              tgt_single_tags = (
                  f"{tgt_example[tgt_offset: tgt_match_start]}"
                  f"{self.isolated_tag_prefix}{single_tag_start_num}"
                  f"{self.isolated_tag_suffix}∥{tgt_term}∥"
              )
              tgt_paired_tags = (
                  f"{tgt_example[tgt_offset: tgt_match_start]}"
                  f"{self.paired_stag_prefix}{paired_tag_start_num}"
                  f"{self.paired_stag_suffix}∥{tgt_term}∥"
                  f"{self.paired_etag_prefix}{paired_tag_start_num}"
                  f"{self.paired_etag_suffix}∥"
              )
  
              # Make a weighted choice between paired tags or single tags.
              # We usually encounter, and thus here we favor, paired tags
              # with a ratio 1/3.
              choice = random.choices(
                  [src_single_tags, src_paired_tags],
                  weights=(1, 3),
                  k=1,
              )
  
              src_with_tags.append(choice[0])
              src_offset = src_match_end + 1
  
              if choice[0] is src_single_tags:
                  tgt_with_tags.append(tgt_single_tags)
                  single_tag_start_num += 1
              else:
                  tgt_with_tags.append(tgt_paired_tags)
                  paired_tag_start_num += 1
  
              tgt_offset = tgt_match_end + 1
              tag_counter += 1
              is_match = True
      if is_match:
          if augmented_part is not None:
              src_with_tags.append(
                  source_only[src_offset:]
                  + self.src_delimiter
                  + augmented_part
              )
          else:
              src_with_tags.append(source_only[src_offset:])
  
          tgt_with_tags.append(tgt_example[tgt_offset:])
  
          return (
              "".join(src_with_tags).replace("∥", " ").split(),
              "".join(tgt_with_tags).replace("∥", " ").split(),
          ), is_match
      else:
          return (src_example.split(), tgt_example.split()), is_match
  
imports: from onmt.utils.logging import logger
  from onmt.transforms import register_transform
  from .transform import Transform
  import random
  import ahocorasick
  import string
label_desc: Use image node_id 4 for calling the InlineTagger obj's underlying member method code with example usage: obj._tagged_src_tgt(src_example, tgt_example) and returns: str,  str,  is_match, src_example,  tgt_example,  is_match