code: def run_eval(  # for local function:
      base_model=None,
      lora_weights=None,
      inference_server=None,
      regenerate_clients=None,
      prompt_type=None,
      prompt_dict=None,
      system_prompt=None,
      debug=None,
      chat=False,
      stream_output=None,
      async_output=None,
      num_async=None,
      eval_filename=None,
      eval_prompts_only_num=None,
      eval_prompts_only_seed=None,
      eval_as_output=None,
      examples=None,
      memory_restriction_level=None,
      # for get_model:
      score_model=None,
      load_8bit=None,
      load_4bit=None,
      low_bit_mode=None,
      load_half=None,
      use_flash_attention_2=None,
      load_gptq=None,
      use_autogptq=None,
      load_awq=None,
      load_exllama=None,
      use_safetensors=None,
      revision=None,
      use_gpu_id=None,
      tokenizer_base_model=None,
      gpu_id=None,
      n_jobs=None,
      n_gpus=None,
      local_files_only=None,
      resume_download=None,
      use_auth_token=None,
      trust_remote_code=None,
      offload_folder=None,
      rope_scaling=None,
      max_seq_len=None,
      compile_model=None,
      llamacpp_dict=None,
      llamacpp_path=None,
      exllama_dict=None,
      gptq_dict=None,
      attention_sinks=None,
      sink_dict=None,
      hf_model_dict=None,
      truncation_generation=None,
      use_pymupdf=None,
      use_unstructured_pdf=None,
      use_pypdf=None,
      enable_pdf_ocr=None,
      enable_pdf_doctr=None,
      enable_imagegen_high_sd=None,
      try_pdf_as_html=None,
      # for evaluate args beyond what's already above, or things that are always dynamic and locally created
      temperature=None,
      top_p=None,
      top_k=None,
      penalty_alpha=None,
      num_beams=None,
      max_new_tokens=None,
      min_new_tokens=None,
      early_stopping=None,
      max_time=None,
      repetition_penalty=None,
      num_return_sequences=None,
      do_sample=None,
      langchain_mode=None,
      langchain_action=None,
      langchain_agents=[],
      top_k_docs=None,
      chunk=None,
      chunk_size=None,
      document_subset=None,
      document_choice=None,
      document_source_substrings=None,
      document_source_substrings_op=None,
      document_content_substrings=None,
      document_content_substrings_op=None,
      pre_prompt_query=None,
      prompt_query=None,
      pre_prompt_summary=None,
      prompt_summary=None,
      hyde_llm_prompt=None,
      image_audio_loaders=None,
      pdf_loaders=None,
      url_loaders=None,
      jq_schema=None,
      extract_frames=None,
      llava_prompt=None,
      visible_models=None,
      h2ogpt_key=None,
      add_search_to_context=None,
      chat_conversation=None,
      text_context_list=None,
      docs_ordering_type=None,
      min_max_new_tokens=None,
      max_input_tokens=None,
      max_total_input_tokens=None,
      docs_token_handling=None,
      docs_joiner=None,
      hyde_level=None,
      hyde_template=None,
      hyde_show_only_final=None,
      hyde_show_intermediate_in_accordion=None,
      doc_json_mode=None,
      chatbot_role=None,
      speaker=None,
      tts_language=None,
      tts_speed=None,
      # for evaluate kwargs:
      captions_model=None,
      caption_loader=None,
      doctr_loader=None,
      pix2struct_loader=None,
      llava_model=None,
      image_gen_loader=None,
      image_gen_loader_high=None,
      image_change_loader=None,
      asr_model=None,
      asr_loader=None,
      image_audio_loaders_options0=None,
      pdf_loaders_options0=None,
      url_loaders_options0=None,
      jq_schema0=None,
      keep_sources_in_context=None,
      gradio_errors_to_chatbot=None,
      allow_chat_system_prompt=None,
      src_lang=None,
      tgt_lang=None,
      concurrency_count=None,
      save_dir=None,
      sanitize_bot_response=None,
      model_state0=None,
      max_max_new_tokens=None,
      is_public=None,
      max_max_time=None,
      raise_generate_gpu_exceptions=None,
      load_db_if_exists=None,
      use_llm_if_no_docs=None,
      my_db_state0=None,
      selection_docs_state0=None,
      dbs=None,
      langchain_modes=None,
      langchain_mode_paths=None,
      detect_user_path_changes_every_query=None,
      use_openai_embedding=None,
      use_openai_model=None,
      hf_embedding_model=None,
      migrate_embedding_model=None,
      auto_migrate_db=None,
      cut_distance=None,
      answer_with_sources=None,
      append_sources_to_answer=None,
      append_sources_to_chat=None,
      show_accordions=None,
      top_k_docs_max_show=None,
      show_link_in_sources=None,
      langchain_instruct_mode=None,
      add_chat_history_to_context=None,
      context=None,
      iinput=None,
      db_type=None,
      first_para=None,
      text_limit=None,
      verbose=None,
      gradio=None,
      cli=None,
      use_cache=None,
      auto_reduce_chunks=None,
      max_chunks=None,
      headsize=None,
      model_lock=None,
      force_langchain_evaluate=None,
      model_state_none=None,
  ):
      from_ui = False
      # makes no sense to evaluate document content for langchain case
      answer_with_sources = False
      show_link_in_sources = False
      append_sources_to_answer = False
      append_sources_to_chat = False
  
      check_locals(**locals())
  
      if not context:
          context = ""
  
      if eval_prompts_only_num > 0:
          np.random.seed(eval_prompts_only_seed)
          example1 = examples[-1]  # pick reference example
          examples = []
          responses = []
          if eval_filename is None:
              # override default examples with shareGPT ones for human-level eval purposes only
              eval_filename = (
                  "ShareGPT_V3_unfiltered_cleaned_split_no_imsorry.json"
              )
              if not os.path.isfile(eval_filename):
                  os.system(
                      "wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/%s"
                      % eval_filename
                  )
              import json
  
              with open(eval_filename, "r", encoding="utf-8") as f:
                  data = json.load(f)
              # focus on data that starts with human, else likely chopped from other data
              turn_start = 0  # odd in general
              data = [
                  x
                  for x in data
                  if len(x["conversations"]) > turn_start + 1
                  and x["conversations"][turn_start]["from"] == "human"
                  and x["conversations"][turn_start + 1]["from"]
                  == "gpt"
              ]
              for i in sorted(
                  np.random.randint(
                      0, len(data), size=eval_prompts_only_num
                  )
              ):
                  assert (
                      data[i]["conversations"][turn_start]["from"]
                      == "human"
                  )
                  instruction = data[i]["conversations"][turn_start][
                      "value"
                  ]
                  assert (
                      data[i]["conversations"][turn_start + 1]["from"]
                      == "gpt"
                  )
                  output = data[i]["conversations"][turn_start + 1][
                      "value"
                  ]
                  examplenew = example1.copy()
                  assert (
                      not chat
                  ), "No gradio must use chat=False, uses nochat instruct"
                  examplenew[
                      eval_func_param_names.index("instruction_nochat")
                  ] = instruction
                  examplenew[
                      eval_func_param_names.index("iinput_nochat")
                  ] = iinput
                  examplenew[
                      eval_func_param_names.index("context")
                  ] = context
                  examples.append(examplenew)
                  responses.append(output)
          else:
              # get data, assume in correct format: json of rows of dict of instruction and output
              # only instruction is required
              import json
  
              with open(eval_filename, "r", encoding="utf-8") as f:
                  data = json.load(f)
              for i in sorted(
                  np.random.randint(
                      0, len(data), size=eval_prompts_only_num
                  )
              ):
                  examplenew = example1.copy()
                  instruction = data[i]["instruction"]
                  output = data[i].get("output", "")  # not required
                  assert (
                      not chat
                  ), "No gradio must use chat=False, uses nochat instruct"
                  examplenew[
                      eval_func_param_names.index("instruction_nochat")
                  ] = instruction
                  examplenew[
                      eval_func_param_names.index("iinput_nochat")
                  ] = iinput
                  examplenew[
                      eval_func_param_names.index("context")
                  ] = context
                  examples.append(examplenew)
                  responses.append(output)
  
      num_examples = len(examples)
      scoring_path = "scoring"
      # if no permissions, assume may not want files, put into temp
      scoring_path = makedirs(scoring_path, tmp_ok=True, use_base=True)
      if eval_as_output:
          used_base_model = "gpt35"
          used_lora_weights = ""
          used_inference_server = ""
      else:
          used_base_model = str(base_model.split("/")[-1])
          used_lora_weights = str(lora_weights.split("/")[-1])
          used_inference_server = str(inference_server.split("/")[-1])
      eval_out_filename = "df_scores_%s_%s_%s_%s_%s_%s_%s.parquet" % (
          num_examples,
          eval_prompts_only_num,
          eval_prompts_only_seed,
          eval_as_output,
          used_base_model,
          used_lora_weights,
          used_inference_server,
      )
      eval_out_filename = os.path.join(scoring_path, eval_out_filename)
  
      # torch.device("cuda") leads to cuda:x cuda:y mismatches for multi-GPU consistently
      n_gpus = (
          torch.cuda.device_count() if torch.cuda.is_available() else 0
      )
      device = "cpu" if n_gpus == 0 else "cuda"
      context_class = (
          NullContext if n_gpus > 1 or n_gpus == 0 else torch.device
      )
  
      with context_class(device):
          # ensure was set right above before examples generated
          assert (
              not stream_output
          ), "stream_output=True does not make sense with example loop"
          import time
          from functools import partial
  
          # get score model
          smodel, stokenizer, sdevice = get_score_model(
              reward_type=True,
              **get_kwargs(
                  get_score_model,
                  exclude_names=["reward_type"],
                  **locals(),
              ),
          )
  
          if not eval_as_output:
              model, tokenizer, device = get_model_retry(
                  reward_type=False,
                  **get_kwargs(
                      get_model,
                      exclude_names=["reward_type"],
                      **locals(),
                  ),
              )
              model_dict = dict(
                  base_model=base_model,
                  tokenizer_base_model=tokenizer_base_model,
                  lora_weights=lora_weights,
                  inference_server=inference_server,
                  prompt_type=prompt_type,
                  prompt_dict=prompt_dict,
                  visible_models=None,
                  h2ogpt_key=None,
              )
              model_state = dict(
                  model=model, tokenizer=tokenizer, device=device
              )
              model_state.update(model_dict)
              requests_state0 = {}
              roles_state0 = None
              args = (
                  model_state,
                  my_db_state0,
                  selection_docs_state0,
                  requests_state0,
                  roles_state0,
              )
              assert len(args) == len(input_args_list)
              fun = partial(
                  evaluate,
                  *args,
                  **get_kwargs(
                      evaluate,
                      exclude_names=input_args_list
                      + eval_func_param_names,
                      **locals(),
                  ),
              )
          else:
              assert eval_prompts_only_num > 0
  
              def get_response(*args, exi=0):
                  # assumes same ordering of examples and responses
                  yield responses[exi]
  
              fun = get_response
          t0 = time.time()
          score_dump = []
          score_avg = 0
          score_median = 0
  
          for exi, ex in enumerate(examples):
              clear_torch_cache(allow_skip=True)
  
              instruction = ex[
                  eval_func_param_names.index("instruction_nochat")
              ]
              iinput = ex[eval_func_param_names.index("iinput_nochat")]
              context = ex[eval_func_param_names.index("context")]
              clear_torch_cache(allow_skip=True)
              print("")
              print("START" + "=" * 100)
              print(
                  "Question: %s %s"
                  % (
                      instruction,
                      ("input=%s" % iinput if iinput else ""),
                  )
              )
              print("-" * 105)
              # fun yields as generator, so have to iterate over it
              # Also means likely do NOT want --stream_output=True, else would show all generations
              t1 = time.time()
  
              # grab other parameters, like langchain_mode
              eval_vars = ex.copy()
              for k in eval_func_param_names:
                  if k in locals():
                      eval_vars[
                          eval_func_param_names.index(k)
                      ] = locals()[k]
  
              gener = (
                  fun(*tuple(eval_vars), exi=exi)
                  if eval_as_output
                  else fun(*tuple(eval_vars))
              )
              for res_fun in gener:
                  res = res_fun["response"]
                  sources = res_fun.get(
                      "sources", "Failure of Generation"
                  )
                  print(res)
                  if smodel:
                      score_with_prompt = False
                      if score_with_prompt:
                          data_point = dict(
                              instruction=instruction,
                              input=iinput,
                              context=context,
                          )
                          prompter = Prompter(
                              prompt_type,
                              prompt_dict,
                              debug=debug,
                              stream_output=stream_output,
                          )
                          prompt = prompter.generate_prompt(
                              data_point, context_from_history=False
                          )
                      else:
                          # just raw input and output
                          if eval_prompts_only_num > 0:
                              # only our own examples have this filled at moment
                              assert iinput in [
                                  None,
                                  "",
                              ], iinput  # should be no iinput
                          prompt = instruction
                      if memory_restriction_level > 0:
                          cutoff_len = (
                              768
                              if memory_restriction_level <= 2
                              else 512
                          )
                      else:
                          cutoff_len = tokenizer.model_max_length
                      inputs = stokenizer(
                          prompt,
                          res,
                          return_tensors="pt",
                          truncation=True,
                          max_length=cutoff_len,
                      )
                      try:
                          score = (
                              torch.sigmoid(
                                  smodel(**inputs.to(smodel.device))
                                  .logits[0]
                                  .float()
                              )
                              .cpu()
                              .detach()
                              .numpy()[0]
                          )
                      except torch.cuda.OutOfMemoryError as e:
                          print(
                              "GPU OOM 1: question: %s answer: %s exception: %s"
                              % (prompt, res, str(e)),
                              flush=True,
                          )
                          traceback.print_exc()
                          score = 0.0
                          clear_torch_cache()
                      except (Exception, RuntimeError) as e:
                          if (
                              "Expected all tensors to be on the same device"
                              in str(e)
                              or "expected scalar type Half but found Float"
                              in str(e)
                              or "probability tensor contains either"
                              in str(e)
                              or "cublasLt ran into an error!" in str(e)
                          ):
                              print(
                                  "GPU error: question: %s answer: %s exception: %s"
                                  % (prompt, res, str(e)),
                                  flush=True,
                              )
                              traceback.print_exc()
                              score = 0.0
                              clear_torch_cache()
                          else:
                              raise
                      score_dump.append(ex + [prompt, res, score])
                      # dump every score in case abort
                      df_scores = pd.DataFrame(
                          score_dump,
                          columns=eval_func_param_names
                          + eval_extra_columns,
                      )
                      df_scores.to_parquet(
                          eval_out_filename, index=False
                      )
                      # plot histogram so far
                      plt.figure(figsize=(10, 10))
                      plt.hist(df_scores["score"], bins=20)
                      score_avg = np.mean(df_scores["score"])
                      score_median = np.median(df_scores["score"])
                      print(
                          "SCORE %s: %s  So far: AVG: %s MEDIAN: %s"
                          % (exi, score, score_avg, score_median),
                          flush=True,
                      )
                      plt.title(
                          "Score avg: %s median: %s"
                          % (score_avg, score_median)
                      )
                      plt.savefig(
                          eval_out_filename.replace(".parquet", ".png")
                      )
                      plt.close()
  
              print("END" + "=" * 102)
              print("")
              t2 = time.time()
              print(
                  "Time taken for example: %s Time taken so far: %.4f about %.4g per example"
                  % (t2 - t1, t2 - t0, (t2 - t0) / (1 + exi))
              )
          t1 = time.time()
          print(
              "Total time taken: %.4f about %.4g per example"
              % (t1 - t0, (t1 - t0) / num_examples)
          )
          print(
              "Score avg: %s median: %s" % (score_avg, score_median),
              flush=True,
          )
      return eval_out_filename
  
file_path: h2ogpt/src/eval.py
imports: import os
  import traceback
  import numpy
  import pandas
  import torch
  from matplotlib import pyplot
  from evaluate_params import eval_func_param_names, eval_extra_columns, input_args_list
  from gen import get_score_model, get_model, evaluate, check_locals, get_model_retry
  from prompter import Prompter
  from utils import clear_torch_cache, NullContext, get_kwargs, makedirs
label_desc: Use image node_id 1 for calling a global function with example usage: run_eval(base_model, lora_weights, inference_server, regenerate_clients, prompt_type, prompt_dict, system_prompt, debug, chat, stream_output, async_output, num_async, eval_filename, eval_prompts_only_num, eval_prompts_only_seed, eval_as_output, examples, memory_restriction_level, score_model, load_8bit, load_4bit, low_bit_mode, load_half, use_flash_attention_2, load_gptq, use_autogptq, load_awq, load_exllama, use_safetensors, revision, use_gpu_id, tokenizer_base_model, gpu_id, n_jobs, n_gpus, local_files_only, resume_download, use_auth_token, trust_remote_code, offload_folder, rope_scaling, max_seq_len, compile_model, llamacpp_dict, llamacpp_path, exllama_dict, gptq_dict, attention_sinks, sink_dict, hf_model_dict, truncation_generation, use_pymupdf, use_unstructured_pdf, use_pypdf, enable_pdf_ocr, enable_pdf_doctr, enable_imagegen_high_sd, try_pdf_as_html, temperature, top_p, top_k, penalty_alpha, num_beams, max_new_tokens, min_new_tokens, early_stopping, max_time, repetition_penalty, num_return_sequences, do_sample, langchain_mode, langchain_action, langchain_agents, top_k_docs, chunk, chunk_size, document_subset, document_choice, document_source_substrings, document_source_substrings_op, document_content_substrings, document_content_substrings_op, pre_prompt_query, prompt_query, pre_prompt_summary, prompt_summary, hyde_llm_prompt, image_audio_loaders, pdf_loaders, url_loaders, jq_schema, extract_frames, llava_prompt, visible_models, h2ogpt_key, add_search_to_context, chat_conversation, text_context_list, docs_ordering_type, min_max_new_tokens, max_input_tokens, max_total_input_tokens, docs_token_handling, docs_joiner, hyde_level, hyde_template, hyde_show_only_final, hyde_show_intermediate_in_accordion, doc_json_mode, chatbot_role, speaker, tts_language, tts_speed, captions_model, caption_loader, doctr_loader, pix2struct_loader, llava_model, image_gen_loader, image_gen_loader_high, image_change_loader, asr_model, asr_loader, image_audio_loaders_options0, pdf_loaders_options0, url_loaders_options0, jq_schema0, keep_sources_in_context, gradio_errors_to_chatbot, allow_chat_system_prompt, src_lang, tgt_lang, concurrency_count, save_dir, sanitize_bot_response, model_state0, max_max_new_tokens, is_public, max_max_time, raise_generate_gpu_exceptions, load_db_if_exists, use_llm_if_no_docs, my_db_state0, selection_docs_state0, dbs, langchain_modes, langchain_mode_paths, detect_user_path_changes_every_query, use_openai_embedding, use_openai_model, hf_embedding_model, migrate_embedding_model, auto_migrate_db, cut_distance, answer_with_sources, append_sources_to_answer, append_sources_to_chat, show_accordions, top_k_docs_max_show, show_link_in_sources, langchain_instruct_mode, add_chat_history_to_context, context, iinput, db_type, first_para, text_limit, verbose, gradio, cli, use_cache, auto_reduce_chunks, max_chunks, headsize, model_lock, force_langchain_evaluate, model_state_none) and returns: eval_out_filename