code: def test_evaluation():
      ##########################
      ########## Settings
      ##########################
      lang_model = "deepset/roberta-base-squad2"
      do_lower_case = False
  
      test_assertions = False
  
      data_dir = Path("testsave/data/squad20")
      evaluation_filename = "dev-v2.0.json"
  
      device, n_gpu = initialize_device_settings(use_cuda=True)
  
      # loading models and evals
      model = AdaptiveModel.convert_from_transformers(
          lang_model, device=device, task_type="question_answering"
      )
      model.prediction_heads[0].no_ans_boost = 0
      model.prediction_heads[0].n_best = 1
      model.prediction_heads[0].n_best_per_sample = 1
  
      tokenizer = Tokenizer.load(
          pretrained_model_name_or_path=lang_model,
          do_lower_case=do_lower_case,
      )
      processor = SquadProcessor(
          tokenizer=tokenizer,
          max_seq_len=256,
          label_list=["start_token", "end_token"],
          metric="squad",
          train_filename=None,
          dev_filename=None,
          dev_split=0,
          test_filename=evaluation_filename,
          data_dir=data_dir,
          doc_stride=128,
      )
  
      starttime = time()
  
      data_silo = DataSilo(
          processor=processor, batch_size=40 * n_gpu_factor
      )
      model.connect_heads_with_processor(
          data_silo.processor.tasks, require_labels=True
      )
      model, _ = optimize_model(
          model=model,
          device=device,
          local_rank=-1,
          optimizer=None,
          distributed=False,
          use_amp=None,
      )
  
      evaluator = Evaluator(
          data_loader=data_silo.get_data_loader("test"),
          tasks=data_silo.processor.tasks,
          device=device,
      )
  
      # 1. Test FARM internal evaluation
      results = evaluator.eval(model)
      f1_score = results[0]["f1"] * 100
      em_score = results[0]["EM"] * 100
      tnacc = results[0]["top_n_accuracy"] * 100
      elapsed = time() - starttime
      print(results)
      print(elapsed)
  
      gold_EM = 78.4721
      gold_f1 = 82.6671
      gold_tnacc = 84.3594  # top 1 recall
      gold_elapsed = 40  # 4x V100
      if test_assertions:
          np.testing.assert_allclose(
              em_score,
              gold_EM,
              rtol=0.001,
              err_msg=f"FARM Eval changed for EM by: {em_score-gold_EM}",
          )
          np.testing.assert_allclose(
              f1_score,
              gold_f1,
              rtol=0.001,
              err_msg=f"FARM Eval changed for f1 score by: {f1_score-gold_f1}",
          )
          np.testing.assert_allclose(
              tnacc,
              gold_tnacc,
              rtol=0.001,
              err_msg=f"FARM Eval changed for top 1 accuracy by: {tnacc-gold_tnacc}",
          )
          np.testing.assert_allclose(
              elapsed,
              gold_elapsed,
              rtol=0.1,
              err_msg=f"FARM Eval speed changed significantly by: {elapsed - gold_elapsed} seconds",
          )
  
      if not np.allclose(f1_score, gold_f1, rtol=0.001):
          error_messages.append(
              f"FARM Eval changed for f1 score by: {round(f1_score - gold_f1, 4)}"
          )
      if not np.allclose(em_score, gold_EM, rtol=0.001):
          error_messages.append(
              f"FARM Eval changed for EM by: {round(em_score - gold_EM, 4)}"
          )
      if not np.allclose(tnacc, gold_tnacc, rtol=0.001):
          error_messages.append(
              f"FARM Eval changed for top 1 accuracy by: {round(tnacc-gold_tnacc, 4)}"
          )
      if not np.allclose(elapsed, gold_elapsed, rtol=0.1):
          error_messages.append(
              f"FARM Eval speed changed significantly by: {round(elapsed - gold_elapsed, 4)} seconds"
          )
  
      benchmark_result = [
          {
              "run": "FARM internal evaluation",
              "f1_change": round(f1_score - gold_f1, 4),
              "em_change": round(em_score - gold_EM, 4),
              "tnacc_change": round(tnacc - gold_tnacc, 4),
              "elapsed_change": round(elapsed - gold_elapsed, 4),
              "f1": f1_score,
              "em": em_score,
              "tnacc": round(tnacc, 4),
              "elapsed": elapsed,
              "f1_gold": gold_f1,
              "em_gold": gold_EM,
              "tnacc_gold": gold_tnacc,
              "elapsed_gold": gold_elapsed,
          }
      ]
      logger.info("\n\n" + pformat(benchmark_result[0]) + "\n")
  
      # # 2. Test FARM predictions with outside eval script
      starttime = time()
      model = Inferencer(
          model=model,
          processor=processor,
          task_type="question_answering",
          batch_size=40 * n_gpu_factor,
          gpu=device.type == "cuda",
      )
      filename = data_dir / evaluation_filename
      result = model.inference_from_file(
          file=filename, return_json=False, multiprocessing_chunksize=80
      )
      results_squad = [x.to_squad_eval() for x in result]
      model.close_multiprocessing_pool()
  
      elapsed = time() - starttime
  
      os.makedirs("../testsave", exist_ok=True)
      write_squad_predictions(
          predictions=results_squad,
          predictions_filename=filename,
          out_filename="testsave/predictions.json",
      )
      script_params = {
          "data_file": filename,
          "pred_file": "testsave/predictions.json",
          "na_prob_thresh": 1,
          "na_prob_file": False,
          "out_file": False,
      }
      results_official = squad_evaluation.main(
          OPTS=DotMap(script_params)
      )
      f1_score = results_official["f1"]
      em_score = results_official["exact"]
  
      gold_EM = 79.878
      gold_f1 = 82.917
      gold_elapsed = 27  # 4x V100
      print(elapsed)
      if test_assertions:
          np.testing.assert_allclose(
              em_score,
              gold_EM,
              rtol=0.001,
              err_msg=f"Eval with official script changed for EM by: {em_score - gold_EM}",
          )
          np.testing.assert_allclose(
              f1_score,
              gold_f1,
              rtol=0.001,
              err_msg=f"Eval with official script changed for f1 score by: {f1_score - gold_f1}",
          )
          np.testing.assert_allclose(
              elapsed,
              gold_elapsed,
              rtol=0.1,
              err_msg=f"Inference speed changed significantly by: {elapsed - gold_elapsed} seconds",
          )
      if not np.allclose(f1_score, gold_f1, rtol=0.001):
          error_messages.append(
              f"Eval with official script changed for f1 score by: {round(f1_score - gold_f1, 4)}"
          )
      if not np.allclose(em_score, gold_EM, rtol=0.001):
          error_messages.append(
              f"Eval with official script changed for EM by: {round(em_score - gold_EM, 4)}"
          )
      if not np.allclose(elapsed, gold_elapsed, rtol=0.1):
          error_messages.append(
              f"Inference speed changed significantly by: {round(elapsed - gold_elapsed,4)} seconds"
          )
  
      benchmark_result.append(
          {
              "run": "outside eval script",
              "f1_change": round(f1_score - gold_f1, 4),
              "em_change": round(em_score - gold_EM, 4),
              "tnacc_change": "-",
              "elapsed_change": round(elapsed - gold_elapsed, 4),
              "f1": f1_score,
              "em": em_score,
              "tnacc": "-",
              "elapsed": elapsed,
              "f1_gold": gold_f1,
              "em_gold": gold_EM,
              "tnacc_gold": "-",
              "elapsed_gold": gold_elapsed,
          }
      )
      logger.info("\n\n" + pformat(benchmark_result[1]) + "\n")
      return benchmark_result
  
imports: import logging
  import os
  from pathlib import Path
  from time import time
  import numpy
  from pprint import pformat
  import pandas
  from dotmap import DotMap
  from farm.data_handler.data_silo import DataSilo
  from farm.data_handler.processor import SquadProcessor
  from farm.data_handler.utils import write_squad_predictions
  from farm.eval import Evaluator
  from farm.evaluation import squad_evaluation
  from farm.infer import Inferencer
  from farm.modeling.adaptive_model import AdaptiveModel
  from farm.modeling.language_model import LanguageModel
  from farm.modeling.optimization import initialize_optimizer, optimize_model
  from farm.modeling.prediction_head import QuestionAnsweringHead
  from farm.modeling.tokenization import Tokenizer
  from farm.train import Trainer
  from farm.utils import set_all_seeds, initialize_device_settings
label_desc: Use image node_id 1 for calling a global function with example usage: test_evaluation() and returns: benchmark_result