code: def clean_sentence(line):
      """
      Sentence cleaning from LSTM-LID
      GitHub: https://github.com/AU-DIS/LSTM_langid/blob/main/src/dataset_creator.py
      """
      # We remove some special characters and fix small errors in the data, to improve the quality of the data
      line = line.replace(
          "\n", ""
      )  # {"text": "- Mor.\n", "label": "da"}
      line = line.replace("- ", "")  # {"text": "- Mor.", "label": "da"}
      line = line.replace("_", "")  # {"text": "- Mor.", "label": "da"}
      line = line.replace("\\", "")
      line = line.replace('"', "")
      line = line.replace("  ", " ")
      remove_digits = str.maketrans("", "", digits)
      line = line.translate(remove_digits)
      words = line.split()
      new_words = []
      # Below fixes large I instead of l. Does not catch everything, but should also not really make any mistakes either
      for word in words:
          clean_word = word
          s = clean_word
          if clean_word[1:].__contains__("I"):
              indices = find(clean_word, "I")
              for indx in indices:
                  if clean_word[indx - 1].islower():
                      if len(clean_word) > indx + 1:
                          if clean_word[indx + 1].islower():
                              s = s[:indx] + "l" + s[indx + 1 :]
                      else:
                          s = s[:indx] + "l" + s[indx + 1 :]
          new_words.append(s)
      new_line = " ".join(new_words)
      return new_line
  
imports: import argparse
  import json
  import logging
  import os
  import re
  import sys
  from pathlib import Path
  from random import randint, random, shuffle
  from string import digits
  from tqdm import tqdm
  from stanza.models.common.constant import treebank_to_langid
label_desc: Use image node_id 10 for calling a global function with example usage: clean_sentence(line) and returns: new_line