code: def test_assemble_and_detox():
      import re
      from profanity_check import predict_prob
  
      df_list = []
      for data in useful_oig_files:
          print("Processing %s" % data, flush=True)
          df = pd.read_parquet(data)
          df = df.reset_index(drop=True)
          # chop up into human/bot interactions of no more than 10kB per row
          text_list = df[["text"]].values.ravel().tolist()
          new_text = []
          max_len = 2048  # uber cutoff
          MAX_LEN = 2048 // 2 - 30  # max len per question/answer
          for text in tqdm(text_list):
              human_starts = [
                  m.start() for m in re.finditer("<human>: ", text)
              ]
              if len(human_starts) == 1:
                  human_starts = [
                      0,
                      len(text),
                  ]  # always go into for loop below
              blurb = ""
              for i in range(len(human_starts) - 1):
                  interaction = text[
                      human_starts[i] : human_starts[i + 1]
                  ][:max_len]
                  blurb += interaction
                  if len(blurb) >= MAX_LEN:
                      blurb = get_sentences(blurb, length=MAX_LEN)[0]
                      new_text.append(blurb + "\n<human>:")
                      blurb = ""
              if blurb:
                  blurb = get_sentences(blurb, length=MAX_LEN)[0]
                  new_text.append(blurb + "\n<human>:")
  
          if len(new_text) > len(text_list):
              print(
                  "Added %d new rows (before: %d)"
                  % (len(new_text) - df.shape[0], df.shape[0])
              )
          df = pd.DataFrame(
              {"text": new_text, "source": [data] * len(new_text)}
          )
          df = df.drop_duplicates(keep="first")
          print(df["text"].apply(lambda x: len(x)).describe())
          assert df["text"].apply(lambda x: len(x)).max() <= 2 * max_len
  
          # faster than better_profanity, do early
          df["profanity"] = predict_prob(df["text"])
          before_rows = df.shape[0]
          df = df[df["profanity"] < 0.25]  # drop any low quality stuff
          after_rows = df.shape[0]
          print(
              "Dropped %d rows out of %d due to alt-profanity-check"
              % (before_rows - after_rows, before_rows)
          )
          df_list.append(df)
          print(
              "Done processing %s -> %s rows" % (data, df.shape[0]),
              flush=True,
          )
          print("So far have %d rows" % sum([len(x) for x in df_list]))
      df_final = pd.concat(df_list)
      df_final = df_final.sample(frac=1, random_state=1234).reset_index(
          drop=True
      )
      df_final.to_parquet(
          "h2oGPT.cleaned.human_bot.shorter.parquet", index=False
      )
  
imports: import ast
  import concurrent.futures
  import contextlib
  import hashlib
  import json
  import os
  import shutil
  import signal
  import sys
  import traceback
  from concurrent.futures import ProcessPoolExecutor
  import psutil
  import pytest
  import pandas
  import numpy
  from tqdm import tqdm
  from utils import flatten_list, remove
  from joblib import Parallel, delayed, effective_n_jobs
  from sklearn.utils import gen_even_slices
  from sklearn.utils.validation import _num_samples
label_desc: Use image node_id 23 for calling a global function with example usage: test_assemble_and_detox() without return types