class_docstr: Bag-level relation extraction dataset. Note that relation of NA should be named as 'NA'.
docstr: Args:
      pred_result: a list with dict {'entpair': (head_id, tail_id), 'relation': rel, 'score': score}.
          Note that relation of NA should be excluded.
  Return:
      {'prec': narray[...], 'rec': narray[...], 'mean_prec': xx, 'f1': xx, 'auc': xx}
          prec (precision) and rec (recall) are in micro style.
          prec (precision) and rec (recall) are sorted in the decreasing order of the score.
          f1 is the max f1 score of those precison-recall points
docstr_tok: ["Args",":","pred_result",":","a","list","with","dict","{","'entpair","'",":","(","head_id",",","tail_id",")",",","'relation","'",":","rel",",","'score","'",":","score","}",".","Note","that","relation","of","NA","should","be","excluded",".","Return",":","{","'prec","'",":","narray","[","...","]",",","'rec","'",":","narray","[","...","]",",","'mean_prec","'",":","xx",",","'f1","'",":","xx",",","'auc","'",":","xx","}","prec","(","precision",")","and","rec","(","recall",")","are","in","micro","style",".","prec","(","precision",")","and","rec","(","recall",")","are","sorted","in","the","decreasing","order","of","the","score",".","f1","is","the","max","f1","score","of","those","precison-recall","points"]
code: def eval(self, pred_result, threshold=0.5):
      """
      Args:
          pred_result: a list with dict {'entpair': (head_id, tail_id), 'relation': rel, 'score': score}.
              Note that relation of NA should be excluded.
      Return:
          {'prec': narray[...], 'rec': narray[...], 'mean_prec': xx, 'f1': xx, 'auc': xx}
              prec (precision) and rec (recall) are in micro style.
              prec (precision) and rec (recall) are sorted in the decreasing order of the score.
              f1 is the max f1 score of those precison-recall points
      """
      sorted_pred_result = sorted(
          pred_result, key=lambda x: x["score"], reverse=True
      )
      prec = []
      rec = []
      correct = 0
      total = len(self.facts)
  
      entpair = {}
  
      for i, item in enumerate(sorted_pred_result):
          # Save entpair label and result for later calculating F1
          idtf = item["entpair"][0] + "#" + item["entpair"][1]
          if idtf not in entpair:
              entpair[idtf] = {
                  "label": np.zeros((len(self.rel2id)), dtype=np.int),
                  "pred": np.zeros((len(self.rel2id)), dtype=np.int),
                  "score": np.zeros((len(self.rel2id)), dtype=np.float),
              }
          if (
              item["entpair"][0],
              item["entpair"][1],
              item["relation"],
          ) in self.facts:
              correct += 1
              entpair[idtf]["label"][self.rel2id[item["relation"]]] = 1
          if item["score"] >= threshold:
              entpair[idtf]["pred"][self.rel2id[item["relation"]]] = 1
          entpair[idtf]["score"][self.rel2id[item["relation"]]] = item[
              "score"
          ]
  
          prec.append(float(correct) / float(i + 1))
          rec.append(float(correct) / float(total))
  
      auc = sklearn.metrics.auc(x=rec, y=prec)
      np_prec = np.array(prec)
      np_rec = np.array(rec)
      max_micro_f1 = (
          2 * np_prec * np_rec / (np_prec + np_rec + 1e-20)
      ).max()
      best_threshold = sorted_pred_result[
          (2 * np_prec * np_rec / (np_prec + np_rec + 1e-20)).argmax()
      ]["score"]
      mean_prec = np_prec.mean()
  
      label_vec = []
      pred_result_vec = []
      score_vec = []
      for ep in entpair:
          label_vec.append(entpair[ep]["label"])
          pred_result_vec.append(entpair[ep]["pred"])
          score_vec.append(entpair[ep]["score"])
      label_vec = np.stack(label_vec, 0)
      pred_result_vec = np.stack(pred_result_vec, 0)
      score_vec = np.stack(score_vec, 0)
  
      micro_p = sklearn.metrics.precision_score(
          label_vec,
          pred_result_vec,
          labels=list(range(1, len(self.rel2id))),
          average="micro",
      )
      micro_r = sklearn.metrics.recall_score(
          label_vec,
          pred_result_vec,
          labels=list(range(1, len(self.rel2id))),
          average="micro",
      )
      micro_f1 = sklearn.metrics.f1_score(
          label_vec,
          pred_result_vec,
          labels=list(range(1, len(self.rel2id))),
          average="micro",
      )
  
      macro_p = sklearn.metrics.precision_score(
          label_vec,
          pred_result_vec,
          labels=list(range(1, len(self.rel2id))),
          average="macro",
      )
      macro_r = sklearn.metrics.recall_score(
          label_vec,
          pred_result_vec,
          labels=list(range(1, len(self.rel2id))),
          average="macro",
      )
      macro_f1 = sklearn.metrics.f1_score(
          label_vec,
          pred_result_vec,
          labels=list(range(1, len(self.rel2id))),
          average="macro",
      )
  
      pred_result_vec = score_vec >= best_threshold
      max_macro_f1 = sklearn.metrics.f1_score(
          label_vec,
          pred_result_vec,
          labels=list(range(1, len(self.rel2id))),
          average="macro",
      )
      max_micro_f1_each_relation = {}
      for rel in self.rel2id:
          if rel != "NA":
              max_micro_f1_each_relation[
                  rel
              ] = sklearn.metrics.f1_score(
                  label_vec,
                  pred_result_vec,
                  labels=[self.rel2id[rel]],
                  average="micro",
              )
  
      return {
          "np_prec": np_prec,
          "np_rec": np_rec,
          "max_micro_f1": max_micro_f1,
          "max_macro_f1": max_macro_f1,
          "auc": auc,
          "p@100": np_prec[99],
          "p@200": np_prec[199],
          "p@300": np_prec[299],
          "avg_p300": (np_prec[99] + np_prec[199] + np_prec[299]) / 3,
          "micro_f1": micro_f1,
          "macro_f1": macro_f1,
          "max_micro_f1_each_relation": max_micro_f1_each_relation,
      }
  
imports: import torch
  import torch.utils.data
  import os, random, json, logging
  import numpy
  import sklearn.metrics
label_desc: Use image node_id 6 for calling the BagREDataset obj's underlying member method code with example usage: obj.eval(pred_result, threshold) and returns: dict