text: #!/usr/bin/env python3
  # -*- coding: utf-8 -*-
  """
  Created on Thu Nov 28 09:37:26 2019
  
  @author: weetee
  """
  import os
  import math
  import torch
  import torch.nn as nn
  from ..misc import save_as_pickle, load_pickle
  from seqeval.metrics import precision_score, recall_score, f1_score
  import logging
  from tqdm import tqdm
  
  logging.basicConfig(format='%(asctime)s [%(levelname)s]: %(message)s', \
                      datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO)
  logger = logging.getLogger(__file__)
  
  def load_state(net, optimizer, scheduler, args, load_best=False):
      """ Loads saved model and optimizer states if exists """
      base_path = "./data/"
      amp_checkpoint = None
      checkpoint_path = os.path.join(base_path,"task_test_checkpoint_%d.pth.tar" % args.model_no)
      best_path = os.path.join(base_path,"task_test_model_best_%d.pth.tar" % args.model_no)
      start_epoch, best_pred, checkpoint = 0, 0, None
      if (load_best == True) and os.path.isfile(best_path):
          checkpoint = torch.load(best_path)
          logger.info("Loaded best model.")
      elif os.path.isfile(checkpoint_path):
          checkpoint = torch.load(checkpoint_path)
          logger.info("Loaded checkpoint model.")
      if checkpoint != None:
          start_epoch = checkpoint['epoch']
          best_pred = checkpoint['best_acc']
          net.load_state_dict(checkpoint['state_dict'])
          if optimizer is not None:
              optimizer.load_state_dict(checkpoint['optimizer'])
          if scheduler is not None:
              scheduler.load_state_dict(checkpoint['scheduler'])
          amp_checkpoint = checkpoint['amp']
          logger.info("Loaded model and optimizer.")    
      return start_epoch, best_pred, amp_checkpoint
  
  def load_results(model_no=0):
      """ Loads saved results if exists """
      losses_path = "./data/task_test_losses_per_epoch_%d.pkl" % model_no
      accuracy_path = "./data/task_train_accuracy_per_epoch_%d.pkl" % model_no
      f1_path = "./data/task_test_f1_per_epoch_%d.pkl" % model_no
      if os.path.isfile(losses_path) and os.path.isfile(accuracy_path) and os.path.isfile(f1_path):
          losses_per_epoch = load_pickle("task_test_losses_per_epoch_%d.pkl" % model_no)
          accuracy_per_epoch = load_pickle("task_train_accuracy_per_epoch_%d.pkl" % model_no)
          f1_per_epoch = load_pickle("task_test_f1_per_epoch_%d.pkl" % model_no)
          logger.info("Loaded results buffer")
      else:
          losses_per_epoch, accuracy_per_epoch, f1_per_epoch = [], [], []
      return losses_per_epoch, accuracy_per_epoch, f1_per_epoch
  
  
  def evaluate_(output, labels, ignore_idx):
      ### ignore index 0 (padding) when calculating accuracy
      idxs = (labels != ignore_idx).squeeze()
      o_labels = torch.softmax(output, dim=1).max(1)[1]
      l = labels.squeeze()[idxs]; o = o_labels[idxs]
  
      '''
      print(idxs, "idxs")
      tensor([False,  True,  True, False,  True,  True, False,  True,  True, False,
           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,
           True,  True,  True,  True,  True,  True,  True,  True, False,  True,
           True,  True, False, False,  True,  True, False,  True,  True,  True,
           True,  True,  True,  True,  True,  True, False, False, False,  True,
           True, False,  True,  True, False, False,  True,  True,  True,  True,
           True,  True,  True,  True], device='cuda:0')
      '''
  
      # TODO 1. classification_logits 看看形状 2. idxs true 是不是太多了
  
      if len(idxs) > 1:
          acc = (l == o).sum().item()/len(idxs)
      else:
          acc = (l == o).sum().item()
      l = l.cpu().numpy().tolist() if l.is_cuda else l.numpy().tolist()
      o = o.cpu().numpy().tolist() if o.is_cuda else o.numpy().tolist()
  
      # print(idxs, "idxs")
      # print(o)
      # print(l)
  
      return acc, (o, l)
  
  # 这个函数还没使用，也还不知道对错，TODO
  def cal_prf(true_labels, out_labels):
      tp, tn, fp, fn = 0, 0, 0, 0
      for i in range(len(true_labels)):
          if true_labels[i] == out_labels[i]:
              if true_labels[i] != '0':
                  tp += 1
              else:
                  tn += 1
          else:
              if true_labels[i] != '0':
                  fn += 1
              elif out_labels[i] != '0':
                  fp += 1
      p = tp / (tp+fp)
      r = tp / (tp+fn)
      f = 2 * p * r / (p+r)
  
      return p, r, f
  
  
  def evaluate_results(net, test_loader, pad_id, cuda):
      logger.info("Evaluating test samples...")
      acc = 0; out_labels = []; true_labels = []
      net.eval()
      with torch.no_grad():
          for i, data in tqdm(enumerate(test_loader), total=len(test_loader)):
              x, e1_e2_start, labels, _,_,_ = data
              # print(x, e1_e2_start, labels)
              attention_mask = (x != pad_id).float()
              # print(attention_mask)
              token_type_ids = torch.zeros((x.shape[0], x.shape[1])).long()
  
              if cuda:
                  x = x.cuda()
                  labels = labels.cuda()
                  attention_mask = attention_mask.cuda()
                  token_type_ids = token_type_ids.cuda()
                  
              classification_logits = net(x, token_type_ids=token_type_ids, attention_mask=attention_mask, Q=None,\
                            e1_e2_start=e1_e2_start)
              
              accuracy, (o, l) = evaluate_(classification_logits, labels, ignore_idx=-1)
              out_labels.append([str(i) for i in o]); true_labels.append([str(i) for i in l])
              # out_labels += [str(i) for i in o]; true_labels += [str(i) for i in l]
              acc += accuracy
          print(accuracy, out_labels, true_labels)
      
      accuracy = acc/(i + 1)
      results = {
          "accuracy": accuracy,
          "precision": precision_score(true_labels, out_labels),
          "recall": recall_score(true_labels, out_labels),
          "f1": f1_score(true_labels, out_labels)
      }
      logger.info("***** Eval results *****")
      for key in sorted(results.keys()):
          logger.info("  %s = %s", key, str(results[key]))
      
      return results
      