code: def test_check_unhelpful():
      # file = '/home/jon/Downloads/openassistant_oasst1_h2ogpt_graded.json'
      file = (
          "/home/jon/Downloads/openassistant_oasst1_h2ogpt_grades.json"
      )
      # file = 'h2ogpt-oig-oasst1-instruct-cleaned-v2.json'
  
      unhelpful = get_unhelpful_list()
      # data = json.load(open(file, 'rt'))
      df = pd.read_json(file)
  
      use_reward_score_threshold = False
      use_bleu_threshold = False
      use_sentence_sim = True
  
      from sacrebleu.metrics import BLEU
  
      bleu = BLEU()
      from nltk.translate.bleu_score import sentence_bleu
  
      def get_bleu(actual, expected_list):
          # return bleu.sentence_score(actual, expected_list).score
          return sentence_bleu(expected_list, actual)
  
      threshold = 0.0
      if use_reward_score_threshold:
          df = df[df["grade_deberta"] > threshold]
  
      # back to as if original json load
      data = df.to_dict(orient="records")
      bads = {}
      string_all = str(data)
      for sub in unhelpful:
          bads[sub] = string_all.count(sub)
      bads = {k: v for k, v in bads.items() if v > 0}
      import pprint
  
      pp = pprint.PrettyPrinter(indent=4)
      pp.pprint(bads)
  
      total_bads = sum(list(bads.values()))
      print("total_bads: %s" % total_bads, flush=True)
  
      # check just bot
      import re
  
      convs = [
          [
              x.strip()
              for x in re.split(r"%s|%s" % (human, bot), y["input"])
              if x.strip()
          ]
          for y in data
      ]
      humans = [
          [x for i, x in enumerate(y) if i % 2 == 0] for y in convs
      ]
      bots = [[x for i, x in enumerate(y) if i % 2 == 1] for y in convs]
  
      # FIXME: apply back to json etc., just see for now
      bleu_threshold = 0.9
      if use_bleu_threshold:
          bots = [
              [x for x in y if get_bleu(x, unhelpful) < bleu_threshold]
              for y in tqdm(bots)
          ]
  
      cosine_sim_threshold = 0.8
      if use_sentence_sim:
          # pip install sentence_transformers-2.2.2
          from sentence_transformers import SentenceTransformer
  
          # sent_model = 'bert-base-nli-mean-tokens'
          # sent_model = 'nli-distilroberta-base-v2'
          sent_model = "all-MiniLM-L6-v2"
          model = SentenceTransformer(sent_model)
          sentence_embeddings = model.encode(unhelpful)
          from sklearn.metrics.pairwise import cosine_similarity
  
          bots = [
              x
              for x in tqdm(bots)
              if np.max(
                  cosine_similarity(
                      model.encode(x), sentence_embeddings
                  )
              )
              < cosine_sim_threshold
          ]
  
      bads_bots = {}
      string_all = str(bots)
      for sub in unhelpful:
          bads_bots[sub] = string_all.count(sub)
      bads_bots = {k: v for k, v in bads_bots.items() if v > 0}
      import pprint
  
      pp = pprint.PrettyPrinter(indent=4)
      pp.pprint(bads_bots)
  
      total_bads_bots = sum(list(bads_bots.values()))
      print(
          "threshold: %g use_bleu_threshold: %g total_bads_bots: %s total_bots: %s total_humans: %s"
          % (
              threshold,
              use_bleu_threshold,
              total_bads_bots,
              len(bots),
              len(humans),
          ),
          flush=True,
      )
  
      # assert len(bads) == 0, bads
      assert len(bads_bots) == 0, bads_bots
  
imports: import ast
  import concurrent.futures
  import contextlib
  import hashlib
  import json
  import os
  import shutil
  import signal
  import sys
  import traceback
  from concurrent.futures import ProcessPoolExecutor
  import psutil
  import pytest
  import pandas
  import numpy
  from tqdm import tqdm
  from utils import flatten_list, remove
  from joblib import Parallel, delayed, effective_n_jobs
  from sklearn.utils import gen_even_slices
  from sklearn.utils.validation import _num_samples
label_desc: Use image node_id 37 for calling a global function with example usage: test_check_unhelpful() and returns: sentence_bleu