code: def get_scores(lines):
      scorer = PerspectiveApiScorer()
      all_data = []
      for i, line in enumerate(tqdm(lines)):
          data = json.loads(line)
          text = data["text"]
          if not text.strip():
              data["score"] = None
              all_data.append(json.dumps(data))
              continue
          encoded_text = text.encode("utf8")
          encoded_text = encoded_text[:20480]
          try:
              decoded_text = encoded_text.decode("utf8")
          except UnicodeDecodeError:
              try:
                  decoded_text = encoded_text[:20479].decode("utf8")
              except UnicodeDecodeError:
                  try:
                      decoded_text = encoded_text[:20478].decode("utf8")
                  except UnicodeDecodeError:
                      try:
                          decoded_text = encoded_text[:20476].decode(
                              "utf8"
                          )
                      except:
                          print("Error occurred")
                          data["score"] = None
                          all_data.append(json.dumps(data))
                          continue
          data["score"] = scorer.get_scores(decoded_text)
          all_data.append(json.dumps(data))
      return all_data
  
file_path: megatron-lm/examples/detxoify_lm/annotations/perspective_api_annotate.py
imports: import json
  import time
  from typing import Dict, Optional, List
  import joblib
  from googleapiclient import discovery
  from googleapiclient.errors import HttpError
  import argparse
  from tqdm import tqdm
  from joblib import Parallel, delayed
label_desc: Use image node_id 4 for calling a global function with example usage: get_scores(lines) and returns: all_data