text: import sys, os, io
  import random
  import json
  import torch
  import logging
  import argparse
  import gensim
  import struct
  import math
  import itertools
  import copy
  
  if sys.version_info[0] == 2:
      import cPickle as pickle
  else:
      import pickle
  
  import numpy as np
  import torch.nn as nn
  import torch.nn.functional as F
  import torch.optim as optim
  import torch.autograd as autograd
  
  from torch.autograd import Variable
  from sklearn.utils import shuffle
  from pytorch_pretrained_bert import BertTokenizer, BertModel
  
  from load_pretrained_word_embeddings import *
  from MIMO import *
  
  class Dictionary(object):
  	def __init__(self):
  		self.word2idx = {}
  		self.idx2word = []
  
  	def add_word(self, word):
  		if word not in self.word2idx:
  			self.idx2word.append(word)
  			self.word2idx[word] = len(self.idx2word) - 1
  		return self.word2idx[word]
  
  	def __len__(self):
  		return len(self.idx2word)
  
  class Corpus(object):
  	def __init__(self):
  		self.dictionary = Dictionary()
  		self.load_dictionary()
  
  	def load_dictionary(self):
  		print('loading dictionary for language model ...')
  		index = 0
  		filepath1 = './resources/LM_dictionary1.txt'
  		filepath2 = './resources/LM_dictionary2.txt'
  		filepath3 = './resources/LM_dictionary3.txt'
  		with io.open(filepath1, 'r', encoding="utf8") as f:
  			tokens = 0
  			for line in f:
  				words = line.split() + ['<eos>']
  				tokens += len(words)
  				for word in words:
  					self.dictionary.add_word(word)
  				index += 1
  				if index%100000 == 0:
  					print(index, 'done.')
  		with io.open(filepath2, 'r', encoding="utf8") as f:
  			tokens = 0
  			for line in f:
  				words = line.split() + ['<eos>']
  				tokens += len(words)
  				for word in words:
  					self.dictionary.add_word(word)
  				index += 1
  				if index%10000 == 0:
  					print(index, 'done.')
  		with io.open(filepath3, 'r', encoding="utf8") as f:
  			tokens = 0
  			for line in f:
  				words = line.split() + ['<eos>']
  				tokens += len(words)
  				for word in words:
  					self.dictionary.add_word(word)
  				index += 1
  				if index%10000 == 0:
  					print(index, 'done.')
  		print('done.')
  
  	def tokenize(self, seqword_list):
  		seq_len = len(seqword_list[0])
  		ids_list = []
  		index = 0
  		word_set = set()
  		for seqword in seqword_list:
  			ids = []
  			for word in seqword:
  				if word not in self.dictionary.word2idx:
  					ids.append(self.dictionary.word2idx['<eos>'])
  					word_set.add(word)
  				else:
  					ids.append(self.dictionary.word2idx[word])
  			while len(ids) != seq_len:
  				ids.append(self.dictionary.word2idx['<eos>'])
  			ids_list.append(ids)
  			index += 1
  			if index%10000 == 0:
  				print(index, 'done.')
  		return ids_list
  
  def getStmtLabel(docid_filelabel):
  
  	# load concepts, attributes, predicates, and stmts
  
  	docid2struc = {}
  
  	for [docid,filelabel] in docid_filelabel:
  
  		nid2tuple = {}
  		hid2tuple = {}
  		fid2tuple = {}
  		cid2tuple = {}
  		sid2stmts = {}
  
  		fr = open(filelabel,'r')
  		for line in fr:
  			text = line.strip()
  			if text == '': continue
  			head = text[0]
  			if head == '#':
  				continue
  			elif head == 'n':
  				pos = text.find('[')
  				arr = text[pos+1:-1].split(',')
  				assert(len(arr) == 3 and arr[1] == 'as')
  				_id = text[:pos-1]
  				assert(not _id in nid2tuple)
  				nid2tuple[_id] = [['C',arr[0]],arr[1],['C',arr[2]]]
  			elif head == 'h':
  				pos = text.find('[')
  				arr = text[pos+1:-1].split(',')
  				assert(len(arr) == 3 and arr[1] == 'contain')
  				_id = text[:pos-1]
  				assert(not _id in hid2tuple)		
  				hid2tuple[_id] = [['C',arr[0]],arr[1],['C',arr[2]]]
  			elif head == 'f':
  				pos = text.find('[')
  				arr = text[pos+1:-1].split(',')
  				assert(len(arr) == 3)
  				_tuple = [[],'',[]]
  				for i in [0,2]:
  					if ':' in arr[i]:
  						_arr = arr[i][1:-1].split(':')
  						assert(len(_arr) == 2)
  						_tuple[i] = ['A',_arr[0],_arr[1]]
  					else:
  						if arr[i] == 'NIL':
  							_tuple[i] = ['N',arr[i]]
  						else:
  							_tuple[i] = ['C',arr[i]]
  				_tuple[1] = arr[1]
  				_id = text[:pos-1]
  				try:
  					assert(not _id in fid2tuple)  
  				except:
  					print(text)
  					sys.exit(1)
  				fid2tuple[_id] = _tuple
  			elif head == 'c':
  				pos = text.find('[')
  				arr = text[pos+1:-1].split(',')
  				assert(len(arr) == 3)
  				_tuple = [[],'',[]]		
  				for i in [0,2]:
  					if ':' in arr[i]:
  						_arr = arr[i][1:-1].split(':')
  						assert(len(_arr) == 2)
  						_tuple[i] = ['A',_arr[0],_arr[1]]
  					else:
  						if arr[i] == 'NIL':
  							_tuple[i] = ['N',arr[i]]
  						else:
  							_tuple[i] = ['C',arr[i]]
  				_tuple[1] = arr[1]
  				_id = text[:pos-1]
  				assert(not _id in cid2tuple)
  				cid2tuple[_id] = _tuple
  			elif head == 's':
  				if text[:4] == 'stmt':
  					arr = text.split(' ')
  					stmt = [[],[],'NIL']
  					assert(arr[1] == '=')
  					for i in range(2,len(arr)):
  						_id = arr[i]
  						if _id[0] == 'f':
  							assert(_id in fid2tuple)
  							stmt[0].append(_id)
  						elif _id[0] == 'c':
  							assert(_id in cid2tuple)
  							stmt[1].append(_id)
  						elif _id[0] == '(' and _id[-1] == ')':
  							stmt[2] = _id[1:-1]
  						else:
  							assert(False)
  					sid = int(arr[0][4:])
  					if not sid in sid2stmts:
  						sid2stmts[sid] = []
  					sid2stmts[sid].append(stmt)
  				elif text[:4] == 's???':
  					continue
  				else:
  					assert(False)
  			else:
  				assert(False)
  		fr.close()
  
  		docid2struc[docid] = [nid2tuple,hid2tuple,fid2tuple,cid2tuple,sid2stmts]
  
  	return docid2struc
  
  def parsing(text):
  	seqword,seqpostag,seqanno = [],[],[]
  	elems = text.split(' ')
  	n = len(elems)
  	for i in range(n):
  		elem = elems[i]
  		if elem.startswith('$C'):
  			_arr = elem.split(':')
  			phrase = _arr[1]
  			arrphrase = phrase.split('_')
  			arrpostag = _arr[2].split('_')
  			_n = len(arrphrase)
  			for j in range(_n):
  				seqword.append(arrphrase[j].lower())
  				seqpostag.append(arrpostag[j])
  				if j == 0:
  					seqanno.append('B-C')
  				else:
  					seqanno.append('I-C')
  		elif elem.startswith('$A'):
  			_arr = elem.split(':')
  			arrphrase = _arr[1].split('_')
  			arrpostag = _arr[2].split('_')
  			_n = len(arrphrase)
  			for j in range(_n):
  				seqword.append(arrphrase[j].lower())
  				seqpostag.append(arrpostag[j])
  				if j == 0:
  					seqanno.append('B-A')
  				else:
  					seqanno.append('I-A')
  		elif elem.startswith('$P'):
  			_arr = elem.split(':')
  			arrphrase = _arr[1].split('_')
  			arrpostag = _arr[2].split('_')
  			_n = len(arrphrase)
  			for j in range(_n):
  				seqword.append(arrphrase[j].lower())
  				seqpostag.append(arrpostag[j])
  				if j == 0:
  					seqanno.append('B-P')
  				else:
  					seqanno.append('I-P')
  		else:
  			_arr = elem.split(':')
  			seqword.append(_arr[0].lower())
  			seqpostag.append(_arr[1])
  			seqanno.append('O')
  	assert len(seqword) == len(seqpostag) == len(seqanno)
  	return seqword, seqpostag, seqanno
  
  def getTag2ID(fileName):
  	tag2ID = dict()
  	with open(fileName, 'r') as f:
  		for line in f:
  			tag, _id = line.strip().split(' ')
  			tag2ID[tag] = int(_id)
  	return tag2ID
  
  def getOneHot(index, lenth):
  	assert index < lenth
  	vector = np.asarray([0]*lenth)
  	vector[index] = 1
  	return vector
  
  class AR_Correcter(object):
  	"""docstring for AR_Correcter"""
  	def __init__(self, AR_fact_file_name, AR_condition_file_name, support_threshold, confidence_threshold):
  		super(AR_Correcter, self).__init__()
  		self.A2B_fact = dict()
  		self.A2conf_fact = dict()
  		self.A2B_cond = dict()
  		self.A2conf_cond = dict()
  
  		self._load_AR_file(AR_fact_file_name, AR_condition_file_name, support_threshold, confidence_threshold)
  
  	def _load_AR_file(self, AR_fact_file_name, AR_condition_file_name, support_threshold, confidence_threshold):
  		fi = open(AR_fact_file_name, 'r')
  		ci = open(AR_condition_file_name, 'r')
  
  		for line in fi:
  			A_B, support, confidence = line.strip().split('#')
  			support = int(support)
  			confidence = float(confidence)
  			A, B = A_B.split('-->')
  
  			if support < support_threshold or confidence < confidence_threshold:
  				continue
  
  			if not self._is_good_rule(A.split('\t'), B.split('\t')):
  				continue
  
  			if A in self.A2B_fact:
  				if self.A2conf_fact[A] < confidence:
  					self.A2B_fact[A] = B
  					self.A2conf_fact[A] = confidence
  			else:
  				self.A2B_fact[A] = B
  				self.A2conf_fact[A] = confidence
  
  		for A in self.A2B_fact:
  			print(A, self.A2B_fact[A], self.A2conf_fact[A])
  
  		
  		for line in ci:
  			A_B, support, confidence = line.strip().split('#')
  			support = int(support)
  			confidence = float(confidence)
  			A, B = A_B.split('-->')
  
  			if support < support_threshold or confidence < confidence_threshold:
  				continue
  
  			if not self._is_good_rule(A.split('\t'), B.split('\t')):
  				continue
  
  			if A in self.A2B_cond:
  				if self.A2conf_cond[A] < confidence:
  					self.A2B_cond[A] = B
  					self.A2conf_cond[A] = confidence
  			else:
  				self.A2B_cond[A] = B
  				self.A2conf_cond[A] = confidence
  
  		for A in self.A2B_cond:
  			print(A, self.A2B_cond[A], self.A2conf_cond[A])
  
  		fi.close()
  		ci.close()
  
  	def _is_good_rule(self, pos_sequence, tag_sequence):
  		role_set = set()
  		for tag in tag_sequence:
  			if tag == 'O':
  				continue
  			role = tag[3]
  			role_set.add(role)
  		if len(role_set) < 2 or ('2' not in role_set):
  			return False
  		return True
  
  def smooth_tag_sequence(tag_sequence):
  	new_tag_sequence = ['O']
  	index = 0
  	lenth = len(tag_sequence)
  	flag = False
  	while index < lenth:
  		tag = tag_sequence[index]
  
  		if tag == 'O':
  			new_tag = 'O'
  		elif not tag.endswith('2P') and not tag.endswith('A'):
  			if new_tag_sequence[-1].endswith('2P') or new_tag_sequence[-1].endswith('A'):
  				new_tag = 'B'+tag[1:]
  			elif new_tag_sequence[-1].startswith('B') or new_tag_sequence[-1].startswith('I'):
  				new_tag = 'I'+new_tag_sequence[-1][1:]
  			else:
  				new_tag = 'B'+tag[1:]
  		elif tag.endswith('2P'):
  			if new_tag_sequence[-1].endswith('2P'):
  				assert tag[1:] == new_tag_sequence[-1][1:]
  				new_tag = 'I'+new_tag_sequence[-1][1:]
  			else:
  				new_tag = 'B'+tag[1:]
  		else:
  			assert tag.endswith('A')
  			if new_tag_sequence[-1].endswith('A'):
  				new_tag = 'I'+new_tag_sequence[-1][1:]
  			else:
  				new_tag = 'B'+tag[1:]
  
  		if new_tag != tag:
  			flag = True
  
  		new_tag_sequence.append(new_tag)
  		index += 1
  
  	assert len(new_tag_sequence[1:]) == len(tag_sequence)
  	return new_tag_sequence[1:], flag
  
  def is_discarded(tag_sequence):
  	role_set = set()
  	role_type_set = set()
  	predicate_set = set()
  	for index in range(len(tag_sequence)):
  		tag = tag_sequence[index]
  		if tag == 'O':
  			continue
  		if '2P' in tag:
  			predicate_set.add(index)
  		role = tag[3]
  		role_type = tag[3:]
  		role_set.add(role)
  		role_type_set.add(role_type)
  
  	if len(role_set) < 3: # or '2P' not in role_type_set:
  		return True, predicate_set
  
  	if '1A' in role_type_set and '1C' not in role_type_set:
  		return True, predicate_set
  
  	if '3A' in role_type_set and '3C' not in role_type_set:
  		return True, predicate_set
  
  	return False, predicate_set
  
  class Metrics(object):
  	"""docstring for Metrics"""
  	def __init__(self):
  		super(Metrics, self).__init__()
  		self.TP = 0
  		self.FP = 0
  		self.TN = 0
  		self.FN = 0
  
  	def Precision(self):
  		if (self.TP == self.FP) and (self.TP == 0):
  			return 0
  		return float(self.TP) / (self.TP + self.FP)
  
  	def Recall(self):
  		if (self.TP == self.FN) and (self.FN == 0):
  			return 0
  		return float(self.TP) / (self.TP + self.FN)
  
  	def F_1(self):
  		precision = self.Precision()
  		recall = self.Recall()
  		if (precision == recall) and (precision == 0):
  			return 0
  		return 2 * (precision * recall) / (precision + recall)
  
  # def _get_precision_recall(matrix):
  #	precisions = np.asarray(matrix)
  #	precisions = np.asarray([precisions[i].max() for i in range(len(precisions))])
  #	precision = precisions.sum()/float(len(predicted_tuples)*5)
  
  #	recalls = np.asarray(matrix).transpose()
  #	recalls = np.asarray([recalls[i].max() for i in range(len(recalls))])
  #	recall = recalls.sum()/float(len(truth_tuples)*5)
  
  #	return (precision, recall, precisions, recalls)
  
  def match_score(truth_tuples, predicted_tuples, _type):
  	if len(truth_tuples) == 0:
  		truth_tuples = [['NIL', 'NIL', 'NIL', 'NIL', 'NIL']]
  	if len(predicted_tuples) == 0:
  		predicted_tuples = [['NIL', 'NIL', 'NIL', 'NIL', 'NIL']]
  
  	matrix = []
  
  	for predicted_tuple in predicted_tuples:
  		scores = []
  		for truth_tuple in truth_tuples:
  			assert len(truth_tuple) == 5 and len(predicted_tuple) == 5
  			score = 0
  			for index in range(len(truth_tuple)):
  				t_part = truth_tuple[index]
  				p_part = predicted_tuple[index]
  				if t_part == p_part:
  					score += 1
  			scores.append(score)
  		matrix.append(scores)
  
  	precisions = np.asarray(matrix)
  	precisions = np.asarray([precisions[i].max() for i in range(len(precisions))])
  	precision = precisions.sum()/float(len(predicted_tuples)*5)
  
  	recalls = np.asarray(matrix).transpose()
  	recalls = np.asarray([recalls[i].max() for i in range(len(recalls))])
  	recall = recalls.sum()/float(len(truth_tuples)*5)
  
  	return precision, recall, precisions, recalls
  
  def is_blocked(start, end, predicate_set):
  	if start > end:
  		return True
  	for predicate in predicate_set:
  		if predicate[1] > start and predicate[1] < end:
  			return True
  
  # tuple from tag sequence
  def post_decoder(words, predicted_fact_tags, ID2Tag=None):
  	facts = []
  
  	f1_set = set()
  	f1a_set = set()
  	f2_set = set()
  	f3_set = set()
  	f3a_set = set()
  
  	index = 0
  	while index < len(words):
  		if type(predicted_fact_tags[index]) == type(1):
  			tagID = predicted_fact_tags[index]
  			tag = ID2Tag[tagID]
  		else:
  			tag = predicted_fact_tags[index]
  
  		if tag.startswith('B-'):
  			string_tag = tag
  			string = words[index]
  			string_start = index
  			index += 1
  			if index < len(words):
  				if type(predicted_fact_tags[index]) == type(1):
  					tagID = predicted_fact_tags[index]
  					tag = ID2Tag[tagID]
  				else:
  					tag = predicted_fact_tags[index]
  				while tag.startswith('I'):
  					string += ('_' + words[index])
  					index += 1
  					if index < len(words):
  						if type(predicted_fact_tags[index]) == type(1):
  							tagID = predicted_fact_tags[index]
  							tag = ID2Tag[tagID]
  						else:
  							tag = predicted_fact_tags[index]
  					else:
  						break
  			string_end = index
  			if string_tag.endswith('1C'):
  				f1_set.add((string, string_start, string_end))
  			elif string_tag.endswith('1A'):
  				f1a_set.add((string, string_start, string_end))
  				#f1a_set = set()
  			elif string_tag.endswith('2P'):
  				f2_set.add((string, string_start, string_end))
  			elif string_tag.endswith('3C'):
  				f3_set.add((string, string_start, string_end))
  			elif string_tag.endswith('3A'):
  				f3a_set.add((string, string_start, string_end))
  				#f3a_set = set()
  			else:
  				print('error!', string, string_tag, string_start, string_end)
  				sys.exit(1)
  
  		else:
  			index += 1
  
  	MIN = 30
  	subject2predicate = dict()
  	for subject in f1_set:
  		min_dis = MIN
  		t_predicate = None
  		for predicate in f2_set: 
  			if is_blocked(subject[1], predicate[1], f2_set):
  				continue
  			dis = predicate[1]-subject[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_predicate = predicate
  		subject2predicate[subject] = t_predicate
  	# print subject2predicate
  
  	object2predicate = dict()
  	for _object in f3_set:
  		min_dis = MIN
  		t_predicate = None
  		for predicate in f2_set:
  			if is_blocked(predicate[1], _object[1], f2_set):
  				continue
  			dis = _object[1]-predicate[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_predicate = predicate
  		object2predicate[_object] = t_predicate
  	# print object2predicate
  
  	predicate2subject = dict()
  	for predicate in f2_set:
  		min_dis = MIN
  		t_subject = None
  		for subject in f1_set:
  			if is_blocked(subject[1], predicate[1], f2_set):
  				continue
  			dis = predicate[1]-subject[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_subject = subject
  		predicate2subject[predicate] = t_subject
  	# print predicate2subject
  
  	predicate2object = dict()
  	for predicate in f2_set:
  		min_dis = MIN
  		t_object = None
  		for _object in f3_set:
  			if is_blocked(predicate[1], _object[1], f2_set):
  				continue
  			dis = _object[1]-predicate[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_object = _object
  		predicate2object[predicate] = t_object
  	# print predicate2object
  
  	subject2object = dict()
  	for subject in f1_set:
  		min_dis = MIN
  		t_object = None
  		for _object in f3_set:
  			dis = _object[1]-subject[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_object = _object
  		subject2object[subject] = t_object
  	# print subject2object
  
  	object2subject = dict()
  	for _object in f3_set:
  		min_dis = MIN
  		t_subject = None
  		for subject in f1_set:
  			dis = _object[1]-subject[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_subject = subject
  		object2subject[_object] = t_subject
  	# print object2subject
  
  	attrib2subject = dict()
  	for attrib in f1a_set:
  		min_dis = 3
  		t_subject = None
  		for subject in f1_set:
  			dis = subject[1] - attrib[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_subject = subject
  		attrib2subject[attrib] = t_subject
  	# print attrib2subject
  
  	attrib12predicate = dict()
  	for attrib in f1a_set:
  		min_dis = 5
  		t_predicate = None
  		for predicate in f2_set:
  			dis = predicate[1] - attrib[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_predicate = predicate
  		attrib12predicate[attrib] = t_predicate
  
  	attrib32predicate = dict()
  	for attrib in f3a_set:
  		min_dis = 5
  		t_predicate = None
  		for predicate in f2_set:
  			dis = attrib[1] - predicate[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_predicate = predicate
  		attrib32predicate[attrib] = t_predicate
  
  	attrib2object = dict()
  	for attrib in f3a_set:
  		min_dis = 3
  		t_object = None
  		for _object in f3_set:
  			dis = _object[1] - attrib[2]
  			if dis < min_dis and dis >= 0:
  				min_dis = dis
  				t_object = _object
  		attrib2object[attrib] = t_object
  	# print attrib2object
  
  	sets = [f1_set, f2_set, f3_set]
  	for _set in sets:
  		_set.add('NIL')
  
  	candidate_facts = itertools.product(f1_set, set(['NIL']), f2_set, f3_set, set(['NIL']))
  
  	for fact in candidate_facts:
  		# print 'candidate_facts:', fact
  		fact = list(fact)
  		subject = fact[0]
  		predicate = fact[2]
  		_object = fact[3]
  
  		if subject == 'NIL' and _object == 'NIL':
  			if predicate!='NIL' and (predicate2subject[predicate]==None and predicate2object[predicate]==None):
  				facts.append(fact)
  			else:
  				continue
  		elif predicate == 'NIL':
  			if subject == 'NIL' or _object == 'NIL':
  				continue
  			elif (subject2object[subject] != _object) and (object2subject[_object] != subject):
  				continue
  			elif (subject2predicate[subject] != None) or (object2predicate[_object] != None):
  				continue
  			else:
  				facts.append(fact)
  		else:
  			if subject == 'NIL' and (predicate2subject[predicate]!=None or object2subject[_object]!=None):
  				continue
  			elif _object == 'NIL' and (predicate2object[predicate]!=None or subject2object[subject]!=None):
  				continue
  			else:
  				if subject != 'NIL' and (subject2predicate[subject] != predicate) and (predicate2subject[predicate] != subject):
  					#print '1'
  					continue
  				elif _object != 'NIL' and (object2predicate[_object] != predicate) and (predicate2object[predicate] != _object):
  					#print '2'
  					continue
  				elif (subject != 'NIL' and _object != 'NIL') and (subject2object[subject] != _object) and (object2subject[_object] != subject):
  					#print '3'
  					continue
  				else:
  					facts.append(fact)
  	
  	extend_facts = []
  	for attrib in f1a_set:
  		subject = attrib2subject[attrib]
  		if subject == None:
  			predicate = attrib12predicate[attrib]
  			for fact in facts:
  				t_predicate = fact[2]
  				if t_predicate != predicate:
  					continue
  				if fact[0] == 'NIL' and fact[1] == 'NIL':
  					fact[1] = attrib
  			continue
  		for fact in facts:
  			if fact[2][0] == 'in':
  				continue
  			t_subject = fact[0]
  			if t_subject != subject:
  				continue
  			if fact[1] == 'NIL':
  				fact[1] = attrib
  			elif fact[1] != attrib:
  				new_fact = copy.deepcopy(fact)
  				new_fact[1] = attrib
  				extend_facts.append(new_fact)
  			for _fact in facts:
  				if _fact == fact:
  					continue
  				if _fact[2:] == fact[2:] and _fact[0] != 'NIL':
  					if _fact[0][1] - subject[2] < 0 or _fact[0][1] - subject[2] > 3:
  						continue
  					if _fact[1] == 'NIL':
  						_fact[1] = attrib
  					elif _fact[1] != attrib:
  						new_fact = copy.deepcopy(_fact)
  						new_fact[1] = attrib
  						extend_facts.append(new_fact)
  	facts.extend(extend_facts)
  
  	extend_facts = []
  	for attrib in f3a_set:
  		_object = attrib2object[attrib]
  		if _object == None:
  			predicate = attrib32predicate[attrib]
  			for fact in facts:
  				t_predicate = fact[2]
  				if t_predicate != predicate:
  					continue
  				if fact[3] == 'NIL' and fact[4] == 'NIL':
  					fact[4] = attrib
  			continue
  		for fact in facts:
  			t_object = fact[3]
  			if t_object != _object:
  				continue
  			if fact[4] == 'NIL':
  				fact[4] = attrib
  			elif fact[4] != attrib:
  				new_fact = copy.deepcopy(fact)
  				new_fact[4] = attrib
  				extend_facts.append(new_fact)
  			for _fact in facts:
  				if _fact == fact:
  					continue
  				if _fact[:2] == fact[:2] and _fact[3] != 'NIL':
  					if _fact[3][1] - _object[2] < 0 or _fact[3][1] - _object[2] > 3:
  						continue
  					if _fact[4] == 'NIL':
  						_fact[4] = attrib
  					elif _fact[4] != attrib:
  						new_fact = copy.deepcopy(_fact)
  						new_fact[4] = attrib
  						extend_facts.append(new_fact)
  	facts.extend(extend_facts)
  
  	return facts
  
  def revise_max(distribs, tags, threshold):
  	assert len(distribs) == len(tags)
  	for index in range(len(tags)):
  		if distribs[index].item() > torch.log(torch.Tensor([threshold])).item():
  			continue
  		tags[index] = 0
  
  def get_f1(precision, recall):
  	if (precision == recall) and (precision == 0):
  		return 0
  	return 2 * (precision * recall) / (precision + recall)
  
  def metric_to_list(metric):
  	metric_list = []
  	for tag in sorted(metric):
  		tag2metric = [metric[tag].Precision()*100, metric[tag].Recall()*100, metric[tag].F_1()*100]
  		metric_list.append(tag2metric)
  	return metric_list
  
  def tag2tag_to_list(tag2tag):
  	tag2tag_list = []
  	for tag in sorted(tag2tag):
  		row = []
  		for vtag in sorted(tag2tag[tag]):
  			row.append(tag2tag[tag][vtag])
  		tag2tag_list.append(row)
  	return tag2tag_list
  
  def evaluation(models, file_name, dataCenter, threshold_fact, threshold_cond, max_f1, max_std, out_model_name, num_pass, just_eval, write_prediction=False, file_name2=None, just_PR=False, tokenizer=None, LM_model=None, LM_corpus=None, mimo_extractor_fact=None, mimo_extractor_cond=None, pretrain=False, multi_head=None, models_update=[], multi_head_two=None):
  
  	valid_tag_levels = []
  	valid_tuple_levels = []
  	valid_tag2metric_list = []
  	valid_tag2tag_fact_list = []
  	valid_tag2tag_cond_list = []
  
  	test_tag_levels = []
  	test_tuple_levels = []
  	test_tag2metric_list = []
  	test_tag2tag_fact_list = []
  	test_tag2tag_cond_list = []
  
  	for i in range(num_pass):
  		VALID_DATA, TEST_DATA = dataCenter.get_evaluation(1.0/num_pass)
  		valid_tag_level, valid_Metrics, valid_tuple_level, valid_predicted_str = _evaluation(models, dataCenter, VALID_DATA, threshold_fact, threshold_cond, write_prediction, tokenizer=tokenizer, LM_model=LM_model, LM_corpus=LM_corpus, mimo_extractor_fact=mimo_extractor_fact, mimo_extractor_cond=mimo_extractor_cond, pretrain=pretrain, multi_head=multi_head, multi_head_two=multi_head_two)
  		valid_tag_levels.append(valid_tag_level)
  		valid_tuple_levels.append(valid_tuple_level)
  		Tag2Metrics, tag2tag_fact, tag2tag_cond = valid_Metrics
  		valid_tag2metric_list.append(metric_to_list(Tag2Metrics))
  		valid_tag2tag_fact_list.append(tag2tag_to_list(tag2tag_fact))
  		valid_tag2tag_cond_list.append(tag2tag_to_list(tag2tag_cond))
  
  		if len(TEST_DATA[0]) == 0:
  			continue
  
  		test_tag_level, test_Metrics, test_tuple_level, test_predicted_str = _evaluation(models, dataCenter, TEST_DATA, threshold_fact, threshold_cond, write_prediction, tokenizer=tokenizer, LM_model=LM_model, LM_corpus=LM_corpus, mimo_extractor_fact=mimo_extractor_fact, mimo_extractor_cond=mimo_extractor_cond, pretrain=pretrain, multi_head=multi_head, multi_head_two=multi_head_two)
  		test_tag_levels.append(test_tag_level)
  		test_tuple_levels.append(test_tuple_level)
  		Tag2Metrics, tag2tag_fact, tag2tag_cond = test_Metrics
  		test_tag2metric_list.append(metric_to_list(Tag2Metrics))
  		test_tag2tag_fact_list.append(tag2tag_to_list(tag2tag_fact))
  		test_tag2tag_cond_list.append(tag2tag_to_list(tag2tag_cond))
  
  	valid_tag_levels_mean = np.asarray(valid_tag_levels).mean(0)
  	valid_tag_levels_std = np.asarray(valid_tag_levels).std(0)
  
  	valid_tag2metric_mean = np.asarray(valid_tag2metric_list).mean(0)
  	valid_tag2metric_std = np.asarray(valid_tag2metric_list).std(0)
  	valid_tag2tag_fact_mean = np.asarray(valid_tag2tag_fact_list).mean(0)
  	valid_tag2tag_fact_std = np.asarray(valid_tag2tag_fact_list).std(0)
  	valid_tag2tag_cond_mean = np.asarray(valid_tag2tag_cond_list).mean(0)
  	valid_tag2tag_cond_std = np.asarray(valid_tag2tag_cond_list).std(0)
  
  	valid_tuple_levels_mean = np.asarray(valid_tuple_levels).mean(0)
  	valid_tuple_levels_std = np.asarray(valid_tuple_levels).std(0)
  
  	print('VALID RSULTs:', valid_tag_levels_mean[-1][-1], valid_tuple_levels_mean[-1][-1])
  
  	if len(TEST_DATA[0]) != 0:
  		test_tag_levels_mean = np.asarray(test_tag_levels).mean(0)
  		test_tag_levels_std = np.asarray(test_tag_levels).std(0)
  
  		test_tag2metric_mean = np.asarray(test_tag2metric_list).mean(0)
  		test_tag2metric_std = np.asarray(test_tag2metric_list).std(0)
  		test_tag2tag_fact_mean = np.asarray(test_tag2tag_fact_list).mean(0)
  		test_tag2tag_fact_std = np.asarray(test_tag2tag_fact_list).std(0)
  		test_tag2tag_cond_mean = np.asarray(test_tag2tag_cond_list).mean(0)
  		test_tag2tag_cond_std = np.asarray(test_tag2tag_cond_list).std(0)
  
  		test_tuple_levels_mean = np.asarray(test_tuple_levels).mean(0)
  		test_tuple_levels_std = np.asarray(test_tuple_levels).std(0)
  		
  		print('TEST RSULTs:', test_tag_levels_mean[-1][-1], test_tuple_levels_mean[-1][-1])
  
  	macro_F1 = valid_tag_levels_mean[-1][-1]
  	macro_std = valid_tag_levels_std[-1][-1]
  
  	if just_PR:
  		return
  	if macro_F1 > max_f1:
  		print('================BEST MODEL ON VALID==================')
  		fo = open(file_name2, 'w')
  		fo.write(valid_predicted_str)
  		if len(TEST_DATA[0]) != 0:
  			fo.write(test_predicted_str)
  		fo.close()
  
  		max_f1 = macro_F1
  		max_std = macro_std
  		better = True
  		if not just_eval:
  			print('saving model ...')
  			# assert len(models_update) != 0
  			for model in models_update:
  				#torch.save(model.state_dict(), out_model_name+model.name)
  				torch.save(model, out_model_name+model.name)
  			print('saving done.')
  
  		fo = open(file_name, 'w')
  		if len(TEST_DATA[0]) != 0:
  			for i in range(len(test_tag_levels_mean)):
  				for j in range(len(test_tag_levels_mean[i])):
  					fo.write('%.2f+/-%.2f\t' % (test_tag_levels_mean[i][j], test_tag_levels_std[i][j]))
  			for i in range(len(test_tuple_levels_mean)):
  				for j in range(len(test_tuple_levels_mean[i])):
  					fo.write('%.2f+/-%.2f\t' % (test_tuple_levels_mean[i][j], test_tuple_levels_std[i][j]))
  
  		for i in range(len(valid_tag_levels_mean)):
  			for j in range(len(valid_tag_levels_mean[i])):
  				fo.write('%.2f+/-%.2f\t' % (valid_tag_levels_mean[i][j], valid_tag_levels_std[i][j]))
  		for i in range(len(valid_tuple_levels_mean)):
  			for j in range(len(valid_tuple_levels_mean[i])):
  				fo.write('%.2f+/-%.2f\t' % (valid_tuple_levels_mean[i][j], valid_tuple_levels_std[i][j]))
  
  		fo.write('\n')
  
  		i = 0
  		assert len(Tag2Metrics) == len(valid_tag2metric_mean)
  		for tag in sorted(Tag2Metrics):
  			assert len(valid_tag2metric_mean[i]) == 3
  			# print '%s\t%.2f+/-%.2f\t%.2f+/-%.2f\t%.2f+/-%.2f' % (tag, valid_tag2metric_mean[i][0], valid_tag2metric_std[i][0], valid_tag2metric_mean[i][1], valid_tag2metric_std[i][1], valid_tag2metric_mean[i][2], valid_tag2metric_std[i][2])
  			fo.write('%s\t%.2f+/-%.2f\t%.2f+/-%.2f\t%.2f+/-%.2f\n' % (tag, valid_tag2metric_mean[i][0], valid_tag2metric_std[i][0], valid_tag2metric_mean[i][1], valid_tag2metric_std[i][1], valid_tag2metric_mean[i][2], valid_tag2metric_std[i][2]))
  			i += 1
  
  
  		string = '-'
  		for vtag in sorted(tag2tag_fact['O']):
  			string += ('\t'+vtag+'\t_std_')
  		# print string
  		fo.write(string+'\n')
  
  		i = 0
  		assert len(tag2tag_fact) == len(valid_tag2tag_fact_mean) == len(valid_tag2tag_fact_std)
  		for tag in sorted(tag2tag_fact):
  			string = tag
  			j = 0
  			for vtag in sorted(tag2tag_fact[tag]):
  				mean = valid_tag2tag_fact_mean[i][j]
  				std = valid_tag2tag_fact_std[i][j]
  				string += ('\t%.0f\t%.0f' % (mean, std))
  				j += 1
  			# print string
  			fo.write(string+'\n')
  			i += 1
  
  		string = '-'
  		for vtag in sorted(tag2tag_cond['O']):
  			string += ('\t'+vtag+'\t_std_')
  		# print string
  		fo.write(string+'\n')
  		i = 0
  		assert len(tag2tag_cond) == len(valid_tag2tag_cond_mean) == len(valid_tag2tag_cond_std)
  		for tag in sorted(tag2tag_cond):
  			string = tag
  			j = 0
  			for vtag in sorted(tag2tag_cond[tag]):
  				mean = valid_tag2tag_cond_mean[i][j]
  				std = valid_tag2tag_cond_std[i][j]
  				string += ('\t%.0f\t%.0f' % (mean, std))
  				j += 1
  			# print string
  			fo.write(string+'\n')
  			i += 1
  
  		if len(TEST_DATA[0]) == 0:
  			fo.close()
  			return max_f1, max_std
  
  		i = 0
  		assert len(Tag2Metrics) == len(test_tag2metric_mean)
  		for tag in sorted(Tag2Metrics):
  			assert len(test_tag2metric_mean[i]) == 3
  			# print '%s\t%.2f+/-%.2f\t%.2f+/-%.2f\t%.2f+/-%.2f' % (tag, test_tag2metric_mean[i][0], test_tag2metric_std[i][0], test_tag2metric_mean[i][1], test_tag2metric_std[i][1], test_tag2metric_mean[i][2], test_tag2metric_std[i][2])
  			fo.write('%s\t%.2f+/-%.2f\t%.2f+/-%.2f\t%.2f+/-%.2f\n' % (tag, test_tag2metric_mean[i][0], test_tag2metric_std[i][0], test_tag2metric_mean[i][1], test_tag2metric_std[i][1], test_tag2metric_mean[i][2], test_tag2metric_std[i][2]))
  			i += 1
  
  
  		string = '-'
  		for vtag in sorted(tag2tag_fact['O']):
  			string += ('\t'+vtag+'\t_std_')
  		# print string
  		fo.write(string+'\n')
  
  		i = 0
  		assert len(tag2tag_fact) == len(test_tag2tag_fact_mean) == len(test_tag2tag_fact_std)
  		for tag in sorted(tag2tag_fact):
  			string = tag
  			j = 0
  			for vtag in sorted(tag2tag_fact[tag]):
  				mean = test_tag2tag_fact_mean[i][j]
  				std = test_tag2tag_fact_std[i][j]
  				string += ('\t%.0f\t%.0f' % (mean, std))
  				j += 1
  			# print string
  			fo.write(string+'\n')
  			i += 1
  
  		string = '-'
  		for vtag in sorted(tag2tag_cond['O']):
  			string += ('\t'+vtag+'\t_std_')
  		# print string
  		fo.write(string+'\n')
  		i = 0
  		assert len(tag2tag_cond) == len(test_tag2tag_cond_mean) == len(test_tag2tag_cond_std)
  		for tag in sorted(tag2tag_cond):
  			string = tag
  			j = 0
  			for vtag in sorted(tag2tag_cond[tag]):
  				mean = test_tag2tag_cond_mean[i][j]
  				std = test_tag2tag_cond_std[i][j]
  				string += ('\t%.0f\t%.0f' % (mean, std))
  				j += 1
  			# print string
  			fo.write(string+'\n')
  			i += 1
  
  		fo.close()
  
  	return max_f1, max_std
  
  def is_non(tagseq):
  	for tag in tagseq:
  		if tag != 'O':
  			return False
  	return True
  
  def _core_evaluation(OUTs, SENTENCEs, instance_list, predict_fact_batch, predict_condition_batch, predict_fact_tuples_batch, predict_cond_tuples_batch, dataCenter, threshold_fact, threshold_cond, write_prediction=False, is_filter=False):
  	assert len(OUTs) == len(predict_fact_batch) == len(predict_condition_batch)
  	pretrain = False
  	if len(predict_fact_tuples_batch) == 0 and len(predict_cond_tuples_batch) == 0:
  		pretrain = True
  
  	Tag2Metrics = dict()
  	Tag2Metrics_fact = dict()
  	Tag2Metrics_condition = dict()
  
  	for tag in dataCenter.Tag2ID_fact:
  		Tag2Metrics[tag] = Metrics()
  		Tag2Metrics_fact[tag] = Metrics()
  	for tag in dataCenter.Tag2ID_condition:
  		Tag2Metrics[tag] = Metrics()
  		Tag2Metrics_condition[tag] = Metrics()
  
  	tag2tag_fact = dict()
  	tag2tag_cond = dict()
  
  	precision_sum_f = 0
  	recall_sum_f = 0
  	precision_sum_c = 0
  	recall_sum_c = 0
  
  	precisions_f = []
  	recalls_f = []
  	precisions_c = []
  	recalls_c = []
  
  	for tag in dataCenter.Tag2ID_fact:
  		if tag not in tag2tag_fact:
  			tag2tag_fact[tag] = dict()
  			for _tag in dataCenter.Tag2ID_fact:
  				tag2tag_fact[tag][_tag] = 0
  
  	for tag in dataCenter.Tag2ID_condition:
  		if tag not in tag2tag_cond:
  			tag2tag_cond[tag] = dict()
  			for _tag in dataCenter.Tag2ID_condition:
  				tag2tag_cond[tag][_tag] = 0
  
  	predicted_str = ''
  	for i in range(len(predict_fact_batch)):
  		seq_len = len(instance_list[i].SENTENCE)
  		predicted_fact_tag_distribs, predicted_fact_tags = torch.max(predict_fact_batch[i], 1)
  		predicted_conditions_tag_distribs, predicted_conditions_tags = torch.max(predict_condition_batch[i], 1)
  
  		merge_fact_tags = [dataCenter.ID2Tag_fact[tag_id.item()] for tag_id in predicted_fact_tags[:seq_len]]
  		merge_cond_tags = [dataCenter.ID2Tag_condition[tag_id.item()] for tag_id in predicted_conditions_tags[:seq_len]]
  
  		instance = instance_list[i]
  		assert len(instance.SENTENCE) == len(SENTENCEs[i])
  
  		facts = []
  		conditions = []
  		multiple_fact_tags = []
  		multiple_cond_tags = []
  		if not pretrain:
  			# [multiple, seqlen]
  			_, predicted_fact_tuple_tags = torch.max(predict_fact_tuples_batch[i], 2)
  			for k in range(len(predicted_fact_tuple_tags)):
  				fact_tags = [dataCenter.ID2Tag_fact[tag_id.item()] for tag_id in predicted_fact_tuple_tags[k][:seq_len]]
  				multiple_fact_tags.append(fact_tags)
  				facts.extend(post_decoder(instance.multi_input[0][1], fact_tags, dataCenter.ID2Tag_fact))
  
  			_, predicted_cond_tuple_tags = torch.max(predict_cond_tuples_batch[i], 2)
  			for k in range(len(predicted_cond_tuple_tags)):
  				cond_tags = [dataCenter.ID2Tag_condition[tag_id.item()] for tag_id in predicted_cond_tuple_tags[k][:seq_len]]
  				multiple_cond_tags.append(cond_tags)
  				conditions.extend(post_decoder(instance.multi_input[0][1], cond_tags, dataCenter.ID2Tag_condition))
  
  			max_multi_fact_nu = max(len(OUTs[i][0])-1, len(multiple_fact_tags))
  			max_multi_cond_nu = max(len(OUTs[i][1])-1, len(multiple_cond_tags))
  		else:
  			multiple_fact_tags = [merge_fact_tags]
  			multiple_cond_tags = [merge_cond_tags] 
  			facts = post_decoder(instance.multi_input[0][1], merge_fact_tags, dataCenter.ID2Tag_fact)
  			conditions = post_decoder(instance.multi_input[0][1], merge_cond_tags, dataCenter.ID2Tag_condition)
  			max_multi_fact_nu = 1
  			max_multi_cond_nu = 1
  
  		new_fact_tags = ['O']*seq_len
  		for tag_seq in multiple_fact_tags:
  			assert len(tag_seq) == seq_len
  			for _index in range(len(tag_seq)):
  				if tag_seq[_index] != 'O':
  					new_fact_tags[_index] = tag_seq[_index]
  		new_fact_tags = [new_fact_tags]
  
  		new_cond_tags = ['O']*seq_len
  		for tag_seq in multiple_cond_tags:
  			assert len(tag_seq) == seq_len
  			for _index in range(len(tag_seq)):
  				if tag_seq[_index] != 'O':
  					new_cond_tags[_index] = tag_seq[_index]
  		new_cond_tags = [new_cond_tags]
  
  		new_t_fact_tags = ['O']*seq_len
  		for tag_seq in OUTs[i][0][1:]:
  			assert len(tag_seq) == seq_len
  			for _index in range(len(tag_seq)):
  				if tag_seq[_index] != 0:
  					new_t_fact_tags[_index] = dataCenter.ID2Tag_fact[tag_seq[_index]]
  		new_t_fact_tags = [new_t_fact_tags]
  
  		new_t_cond_tags = ['O']*seq_len
  		for tag_seq in OUTs[i][1][1:]:
  			assert len(tag_seq) == seq_len
  			for _index in range(len(tag_seq)):
  				if tag_seq[_index] != 0:
  					new_t_cond_tags[_index] = dataCenter.ID2Tag_condition[tag_seq[_index]]
  		new_t_cond_tags = [new_t_cond_tags]
  
  		max_multi_fact_nu = 1
  		max_multi_cond_nu = 1
  		
  		index_ids = (predicted_fact_tags==dataCenter.Tag2ID_fact['B-f2P']).nonzero()
  		for k in range(max_multi_fact_nu):
  			for j in range(seq_len):
  				if k >= len(OUTs[i][0])-1:
  					y_true = 'O'
  				else:
  					if not pretrain:
  						# y_true = dataCenter.ID2Tag_fact[OUTs[i][0][k+1][j]]
  						y_true = new_t_fact_tags[k][j]
  					else:
  						y_true = dataCenter.ID2Tag_fact[OUTs[i][0][k][j]]
  				if k >= len(new_fact_tags):
  					y_predict = 'O'
  				else:
  					y_predict = new_fact_tags[k][j]
  
  				if y_true == y_predict:
  					tag = y_predict
  					Tag2Metrics[tag].TP += 1
  					Tag2Metrics_fact[tag].TP += 1
  					tag2tag_fact[tag][tag] += 1
  				elif y_true != y_predict:
  					tag_true = y_true
  					tag_predict = y_predict
  					Tag2Metrics[tag_true].FN += 1
  					Tag2Metrics[tag_predict].FP += 1
  					Tag2Metrics_fact[tag_true].FN += 1
  					Tag2Metrics_fact[tag_predict].FP += 1
  					tag2tag_fact[tag_true][tag_predict] += 1
  
  		for k in range(max_multi_cond_nu):
  			for j in range(seq_len):
  				if k >= len(OUTs[i][1])-1:
  					y_true = 'O'
  				else:
  					if not pretrain:
  						# y_true = dataCenter.ID2Tag_condition[OUTs[i][1][k+1][j]]
  						y_true = new_t_cond_tags[k][j]
  					else:
  						y_true = dataCenter.ID2Tag_condition[OUTs[i][1][k][j]]
  				if k >= len(new_cond_tags):
  					y_predict = 'O'
  				else:
  					y_predict = new_cond_tags[k][j]
  
  				if y_true == y_predict:
  					tag = y_predict
  					Tag2Metrics[tag].TP += 1
  					Tag2Metrics_condition[tag].TP += 1
  					tag2tag_cond[tag][tag] += 1
  				elif y_true != y_predict:
  					tag_true = y_true
  					tag_predict = y_predict
  					Tag2Metrics[tag_true].FN += 1
  					Tag2Metrics[tag_predict].FP += 1
  					Tag2Metrics_condition[tag_true].FN += 1
  					Tag2Metrics_condition[tag_predict].FP += 1
  					tag2tag_cond[tag_true][tag_predict] += 1
  
  		_facts = []
  		_conditions = []
  		for out in instance.multi_output:
  			if out[0].startswith('f'):
  				__facts = post_decoder(instance.multi_input[0][1], out[1], dataCenter.ID2Tag_fact)
  				_facts.extend(__facts)
  			else:
  				__conditions = post_decoder(instance.multi_input[0][1], out[1], dataCenter.ID2Tag_condition)
  				_conditions.extend(__conditions)
  
  		p_f, r_f, ps_f, rs_f = match_score(_facts, facts, 'f')
  		p_c, r_c, ps_c, rs_c = match_score(_conditions, conditions, 'c')
  
  		precision_sum_f += p_f
  		recall_sum_f += r_f
  		precision_sum_c += p_c
  		recall_sum_c += r_c
  
  		precisions_f.extend(ps_f)
  		recalls_f.extend(rs_f)
  		precisions_c.extend(ps_c)
  		recalls_c.extend(rs_c)
  					
  
  		if write_prediction:
  			predicted_str += ('===== %s stmt%s =====\n' % (str(instance_list[i].paper_id), str(instance_list[i].stmt_id)))
  			predicted_str += ('WORD\t'+'\t'.join(instance_list[i].SENTENCE)+'\n')
  			predicted_str += ('POSTAG\t'+'\t'.join(instance_list[i].POSTAG)+'\n')
  			predicted_str += ('CAP\t'+'\t'.join(instance_list[i].CAP)+'\n')
  
  			predicted_str += ('f\t'+'\t'.join(merge_fact_tags)+'\n')
  			if not pretrain:
  				for k in range(len(multiple_fact_tags)):
  					fact_tags = multiple_fact_tags[k]
  					predicted_str += ('f\t'+'\t'.join(fact_tags)+'\n')
  
  			predicted_str += ('c\t'+'\t'.join(merge_cond_tags)+'\n')
  			if not pretrain:
  				for k in range(len(multiple_cond_tags)):
  					cond_tags = multiple_cond_tags[k]
  					predicted_str += ('c\t'+'\t'.join(cond_tags)+'\n')
  
  			if not pretrain:
  				for k in range(1, len(instance_list[i].OUT[0])):
  					fact_g_tags = [dataCenter.ID2Tag_fact[tag_id] for tag_id in instance_list[i].OUT[0][k]]
  					predicted_str += ('f_g\t'+'\t'.join(fact_g_tags)+'\n')
  				for k in range(1, len(instance_list[i].OUT[1])):
  					cond_g_tags = [dataCenter.ID2Tag_condition[tag_id] for tag_id in instance_list[i].OUT[1][k]]
  					predicted_str += ('c_g\t'+'\t'.join(cond_g_tags)+'\n')
  			else:
  				fact_g_tags = [dataCenter.ID2Tag_fact[tag_id] for tag_id in instance_list[i].OUT[0][0]]
  				predicted_str += ('f_g\t'+'\t'.join(fact_g_tags)+'\n')
  				cond_g_tags = [dataCenter.ID2Tag_condition[tag_id] for tag_id in instance_list[i].OUT[1][0]]
  				predicted_str += ('c_g\t'+'\t'.join(cond_g_tags)+'\n')
  
  	microEval = Metrics()
  	microEval_fact = Metrics()
  	microEval_condition = Metrics()
  	macro_P = 0
  	macro_R = 0
  	macro_P_fact = 0
  	macro_R_fact = 0
  	macro_P_condition = 0
  	macro_R_condition = 0
  	# print threshold_fact, threshold_cond
  	for tag in Tag2Metrics:
  		if tag == 'O':
  			continue
  
  		macro_P += Tag2Metrics[tag].Precision()*100
  		macro_R += Tag2Metrics[tag].Recall()*100
  		microEval.TP += Tag2Metrics[tag].TP
  		microEval.FP += Tag2Metrics[tag].FP
  		microEval.FN += Tag2Metrics[tag].FN
  
  		if tag in dataCenter.Tag2ID_fact:
  			macro_P_fact += Tag2Metrics_fact[tag].Precision()*100
  			macro_R_fact += Tag2Metrics_fact[tag].Recall()*100
  			microEval_fact.TP += Tag2Metrics_fact[tag].TP
  			microEval_fact.FP += Tag2Metrics_fact[tag].FP
  			microEval_fact.FN += Tag2Metrics_fact[tag].FN
  		elif tag in dataCenter.Tag2ID_condition:
  			macro_P_condition += Tag2Metrics_condition[tag].Precision()*100
  			macro_R_condition += Tag2Metrics_condition[tag].Recall()*100
  			microEval_condition.TP += Tag2Metrics_condition[tag].TP
  			microEval_condition.FP += Tag2Metrics_condition[tag].FP
  			microEval_condition.FN += Tag2Metrics_condition[tag].FN
  		else:
  			print('error')
  			sys.exit(1)
  
  	macro_P /= (len(Tag2Metrics) - 1)
  	macro_R /= (len(Tag2Metrics) - 1)
  	macro_P_fact /= (len(Tag2Metrics_fact) - 1)
  	macro_R_fact /= (len(Tag2Metrics_fact) - 1)
  	macro_P_condition /= (len(Tag2Metrics_condition) - 1)
  	macro_R_condition /= (len(Tag2Metrics_condition) - 1)
  
  	if (macro_P == macro_R == 0):
  		macro_F1 = 0
  	else:
  		macro_F1 = 2 * (macro_P * macro_R) / (macro_P + macro_R)
  
  	if (macro_P_fact == macro_R_fact == 0):
  		macro_F1_fact = 0
  	else:
  		macro_F1_fact = 2 * (macro_P_fact * macro_R_fact) / (macro_P_fact + macro_R_fact)
  
  	if (macro_P_condition == macro_R_condition == 0):
  		macro_F1_condition = 0
  	else:
  		macro_F1_condition = 2 * (macro_P_condition * macro_R_condition) / (macro_P_condition + macro_R_condition)
  
  	tag_level_fact = [microEval_fact.Precision()*100, microEval_fact.Recall()*100, microEval_fact.F_1()*100, macro_P_fact, macro_R_fact, macro_F1_fact]
  	tag_level_cond = [microEval_condition.Precision()*100, microEval_condition.Recall()*100, microEval_condition.F_1()*100, macro_P_condition, macro_R_condition, macro_F1_condition]
  	tag_level = [microEval.Precision()*100, microEval.Recall()*100, microEval.F_1()*100, macro_P, macro_R, macro_F1]
  
  	precisions_f = np.asarray(precisions_f)
  	precisions_c = np.asarray(precisions_c)
  	recalls_f = np.asarray(recalls_f)
  	recalls_c = np.asarray(recalls_c)
  
  	micro_precision_f = precisions_f.sum()/float(len(precisions_f)*5)
  	micro_recall_f = recalls_f.sum()/float(len(recalls_f)*5)
  	micro_f1_f = get_f1(micro_precision_f, micro_recall_f)
  
  	micro_precision_c = precisions_c.sum()/float(len(precisions_c)*5)
  	micro_recall_c = recalls_c.sum()/float(len(recalls_c)*5)
  	micro_f1_c = get_f1(micro_precision_c, micro_recall_c)
  	macro_precision_f = precision_sum_f/len(OUTs)
  	macro_recall_f = recall_sum_f/len(OUTs)
  	macro_f1_f = get_f1(macro_precision_f, macro_recall_f)
  
  	macro_precision_c = precision_sum_c/len(OUTs)
  	macro_recall_c = recall_sum_c/len(OUTs)
  	macro_f1_c = get_f1(macro_precision_c, macro_recall_c)
  
  	precisions = np.concatenate((precisions_f, precisions_c))
  	recalls = np.concatenate((recalls_f, recalls_c))
  	micro_precision = precisions.sum()/float(len(precisions)*5)
  	micro_recall = recalls.sum()/float(len(recalls)*5)
  	micro_f1 = get_f1(micro_precision, micro_recall)
  
  	macro_precision = (macro_precision_f+macro_precision_c)/2
  	macro_recall = (macro_recall_f+macro_recall_c)/2
  	macro_f1 = get_f1(macro_precision, macro_recall)
  
  	tuple_level_fact = [micro_precision_f*100, micro_recall_f*100, micro_f1_f*100, macro_precision_f*100, macro_recall_f*100, macro_f1_f*100]
  	tuple_level_cond = [micro_precision_c*100, micro_recall_c*100, micro_f1_c*100, macro_precision_c*100, macro_recall_c*100, macro_f1_c*100]
  	tuple_level = [micro_precision*100, micro_recall*100, micro_f1*100, macro_precision*100, macro_recall*100, macro_f1*100]
  
  	return [tag_level_fact, tag_level_cond, tag_level], [Tag2Metrics, tag2tag_fact, tag2tag_cond], [tuple_level_fact, tuple_level_cond, tuple_level], predicted_str
  	
  
  def _evaluation(models, dataCenter, data, threshold_fact, threshold_cond, write_prediction=False, device=torch.device("cuda"), tokenizer=None, LM_model=None, LM_corpus=None, mimo_extractor_fact=None, mimo_extractor_cond=None, pretrain=False, multi_head=None, multi_head_two=None):
  
  	raw_SENTENCEs, raw_POSTAGs, raw_CAPs, raw_OUTs, raw_instance_list = data
  	#SENTENCEs, POSTAGs, CAPs, LM_SENTENCEs, POSCAPs, OUTs = shuffle(raw_SENTENCEs, raw_POSTAGs, raw_CAPs, raw_LM_SENTENCEs, raw_POSCAPs, raw_OUTs)
  
  	MICS = zip(raw_SENTENCEs, raw_POSTAGs, raw_CAPs, raw_OUTs, raw_instance_list)
  	MICS = sorted(MICS, key = lambda s: len(s[0]), reverse =True)
  	SENTENCEs, POSTAGs, CAPs, OUTs, instance_list = zip(*MICS)
  	SENTENCEs = list(SENTENCEs)
  	POSTAGs = list(POSTAGs)
  	CAPs = list(CAPs)
  	OUTs = list(OUTs)
  	instance_list = list(instance_list)
  
  	predict_fact = []
  	predict_condition = []
  	predict_fact_tuples = []
  	predict_cond_tuples = []
  	batch_size = 20
  	batches = len(SENTENCEs) // batch_size
  	if len(SENTENCEs) % batch_size != 0:
  		batches += 1
  	for index in range(batches):
  		SENTENCEs_batch = SENTENCEs[index*batch_size: (index+1)*batch_size]
  		POSTAGs_batch = POSTAGs[index*batch_size: (index+1)*batch_size]
  		CAPs_batch = CAPs[index*batch_size: (index+1)*batch_size]
  
  		predict_fact_batch, predict_condition_batch, hidden_out_batch = Tag_Extractor(SENTENCEs_batch, POSTAGs_batch, CAPs_batch, models, tokenizer, LM_model, LM_corpus, device, multi_head, multi_head_two)
  
  		if not pretrain:
  			predict_fact_tuples_batch, predict_cond_tuples_batch = Tuple_Extractor(predict_fact_batch, predict_condition_batch, hidden_out_batch, mimo_extractor_fact, mimo_extractor_cond, dataCenter)
  			predict_fact_tuples.extend(predict_fact_tuples_batch)
  			predict_cond_tuples.extend(predict_cond_tuples_batch)
  
  		for row in predict_fact_batch:
  			predict_fact.append(row)
  		for row in predict_condition_batch:
  			predict_condition.append(row)
  
  	assert len(predict_fact) == len(SENTENCEs) == len(OUTs)
  
  	return _core_evaluation(OUTs, SENTENCEs, instance_list, predict_fact, predict_condition, predict_fact_tuples, predict_cond_tuples, dataCenter, threshold_fact, threshold_cond, write_prediction)
  
  def Tag_Extractor(SENTENCEs, POSTAGs, CAPs, models, tokenizer, LM_model, LM_corpus, device, multi_head, multi_head_two=None):
  	hiddens = [None, None, None]
  	for model in models:
  		if isinstance(model, (MIMO_LSTM, MIMO_LSTM_TF, MIMO_BERT_LSTM, MIMO_BERT_LSTM_TF)):
  			LM_SENTENCEs = []
  			LM_hidden = LM_model.init_hidden(len(SENTENCEs))
  			lm_data = torch.LongTensor(LM_corpus.tokenize(SENTENCEs))
  			lm_data = lm_data.view(len(lm_data[0]), -1)
  			output = LM_model(lm_data.to(device), LM_hidden)
  			LM_SENTENCEs = output.view(len(SENTENCEs), len(SENTENCEs[0]), -1)
  
  			if not isinstance(model, MIMO_LSTM):
  				max_len = len(SENTENCEs[0])
  				input_ids = []
  				input_mask = []
  				for _index in range(len(SENTENCEs)):
  					seqword = SENTENCEs[_index]
  					new_seqword = []
  					for i in range(len(seqword)):
  						w = seqword[i]
  						assert w == w.lower()
  						if w not in tokenizer.vocab:
  							new_seqword.append('[UNK]')
  						else:
  							new_seqword.append(w)
  					new_seqword += ['[UNK]']*(max_len-len(new_seqword))
  					input_ids.append(tokenizer.convert_tokens_to_ids(new_seqword))
  					input_mask.append([1]*len(seqword)+[0]*(max_len-len(seqword)))			
  
  			if isinstance(model, (MIMO_LSTM_TF, MIMO_BERT_LSTM, MIMO_BERT_LSTM_TF)):
  				input_ids = torch.LongTensor(input_ids).to(device)
  				attention_mask = torch.LongTensor(input_mask).to(device)
  			else:
  				attention_mask = None
  
  			# (batch_size, multi_nu, seq_len, dim)
  			if isinstance(model, (MIMO_BERT_LSTM, MIMO_BERT_LSTM_TF)):
  				predict_fact_batch, predict_condition_batch, _, _, hidden_out_batch = model((POSTAGs, CAPs, LM_SENTENCEs), len(SENTENCEs), input_ids, attention_mask=attention_mask)
  			else:
  				predict_fact_batch, predict_condition_batch, _, _, hidden_out_batch = model((SENTENCEs, POSTAGs, CAPs, LM_SENTENCEs), len(SENTENCEs), attention_mask)
  		elif isinstance(model, MIMO_BERT):
  			input_ids = []
  			input_mask = []
  			max_len = len(SENTENCEs[0])
  			for seqword in SENTENCEs:
  				new_seqword = []
  				for w in seqword:
  					assert w == w.lower()
  					if w not in tokenizer.vocab:
  						new_seqword.append('[UNK]')
  					else:
  						new_seqword.append(w)
  				assert len(new_seqword) == len(seqword)
  				input_mask.append([1]*len(new_seqword)+[0]*(max_len-len(new_seqword)))
  				new_seqword += ['[UNK]']*(max_len-len(new_seqword))
  				input_ids.append(tokenizer.convert_tokens_to_ids(new_seqword))
  			input_ids = torch.LongTensor(input_ids).to(device)
  			input_mask = torch.LongTensor(input_mask).to(device)
  			predict_fact_batch, predict_condition_batch, _, _, hidden_out_batch = model(input_ids=input_ids, attention_mask=input_mask)
  
  		if multi_head:
  			hidden_index = None
  			if True in model.model_LSTM_decoder.lm_config:
  				hidden_index = 0
  			elif True in model.model_LSTM_decoder.postag_config:
  				hidden_index = 1
  			else:
  				assert True in model.model_LSTM_decoder.cap_config
  				hidden_index = 2
  			hiddens[hidden_index] = hidden_out_batch
  		# model.to(torch.device("cpu"))
  
  	if multi_head != None:
  		predict_fact_batch, predict_condition_batch, hidden_out_batch = multi_head(hiddens)
  	else:
  		assert len(models) == 1
  	
  	return predict_fact_batch, predict_condition_batch, hidden_out_batch
  
  def Tuple_Extractor(predict_fact_batch, predict_condition_batch, hidden_out_batch, mimo_extractor_fact, mimo_extractor_cond, dataCenter):
  	seq_length = len(predict_fact_batch[0])
  	predict_fact_tuples_batch = []
  	predict_cond_tuples_batch = []
  	for i in range(len(hidden_out_batch)):
  		hidden_out = hidden_out_batch[i]
  		_, predicted_fact_tags = torch.max(predict_fact_batch[i], 1)
  		_, predicted_cond_tags = torch.max(predict_condition_batch[i], 1)
  		indexs = torch.arange(seq_length, dtype=torch.long, device=hidden_out.device)
  		# tmp = [dataCenter.ID2Tag_fact[tid.item()] for tid in predicted_fact_tags]
  
  		index_ids = (predicted_fact_tags[:seq_length]==dataCenter.Tag2ID_fact['B-f2P']).nonzero().to(hidden_out.device)
  		if len(index_ids) != 0:
  			position_ids = indexs - index_ids + 100
  		else:
  			position_ids = indexs.view(1, -1) + 101
  		predict_fact_tuples_batch.append(mimo_extractor_fact(hidden_out.repeat(len(position_ids), 1, 1), position_ids))
  
  		index_ids = (predicted_cond_tags[:seq_length]==dataCenter.Tag2ID_condition['B-c2P']).nonzero().to(hidden_out.device)
  		if len(index_ids) != 0:
  			position_ids = indexs - index_ids + 100
  		else:
  			position_ids = indexs.view(1, -1) + 101
  		predict_cond_tuples_batch.append(mimo_extractor_cond(hidden_out.repeat(len(position_ids), 1, 1), position_ids))
  	# print(predict_fact_tuples_batch.size(), predict_cond_tuples_batch.size())
  	assert len(predict_fact_batch) == len(predict_fact_tuples_batch) == len(predict_cond_tuples_batch)
  	return predict_fact_tuples_batch, predict_cond_tuples_batch
  
  
  
  def apply_model(models, batch_size, dataCenter, device, weight_classes_fact=None, weight_classes_condition=None, LM_model=None, LM_corpus=None, tokenizer=None, mimo_extractor_fact=None, mimo_extractor_cond=None, pretrain=False, multi_head=None, multi_head_two=None, is_semi=False, eval_pack=[]):
  
  	if is_semi:
  		out_file, max_f1, max_std, out_model_name, num_pass = eval_pack
  	
  	if pretrain:
  		assert mimo_extractor_fact==None and mimo_extractor_cond==None
  	if multi_head != None:
  		assert len(models) > 1
  	if len(models) > 1:
  		assert multi_head != None
  
  	oov = set()
  
  	raw_SENTENCEs, raw_POSTAGs, raw_CAPs, raw_OUTs = dataCenter.get_trainning_data(is_semi)
  	SENTENCEs, POSTAGs, CAPs, OUTs = shuffle(raw_SENTENCEs, raw_POSTAGs, raw_CAPs, raw_OUTs)
  
  	batches = len(SENTENCEs) // batch_size
  	Tag2Metrics = dict()
  
  	params = []
  	models_update = []
  	if multi_head == None:
  		for model in models:
  			for param in model.parameters():
  				if param.requires_grad:
  					params.append(param)
  			models_update.append(model)
  			# for param in model.parameters():
  			# 	if isinstance(model, MIMO_LSTM) and (not pretrain):
  			# 		param.requires_grad = False
  			# 	else:
  			# 		if param.requires_grad:
  			# 			params.append(param)
  			# if (not isinstance(model, MIMO_LSTM)) or pretrain:
  			# 	models_update.append(model)
  	else:
  		for model in models:
  			for param in model.parameters():
  				param.requires_grad = False
  
  		if len(models) != 3:
  			for param in multi_head.parameters():
  				if param.requires_grad:
  					params.append(param)
  			models_update.append(multi_head)
  		else:
  			for param in multi_head_two.parameters():
  				if param.requires_grad:
  					params.append(param)
  			models_update.append(multi_head_two)
  
  	if not pretrain:
  		for param in mimo_extractor_fact.parameters():
  			if param.requires_grad:
  				params.append(param)
  		for param in mimo_extractor_cond.parameters():
  			if param.requires_grad:
  				params.append(param)
  		models_update.append(mimo_extractor_fact)
  		models_update.append(mimo_extractor_cond)
  
  	print('to update params of models: ')
  	for model in models_update:
  		print(type(model))
  
  	optimizer = optim.SGD(params, lr = 0.01, weight_decay=0.0005, momentum=0.9)
  
  	for index in range(batches):
  		SENTENCEs_batch = SENTENCEs[index*batch_size: (index+1)*batch_size]
  		POSTAGs_batch = POSTAGs[index*batch_size: (index+1)*batch_size]
  		CAPs_batch = CAPs[index*batch_size: (index+1)*batch_size]
  		OUTs_batch = OUTs[index*batch_size: (index+1)*batch_size]
  		
  		MICS = zip(SENTENCEs_batch, POSTAGs_batch, CAPs_batch, OUTs_batch)
  		MICS = sorted(MICS, key = lambda s: len(s[0]), reverse =True)
  		SENTENCEs_batch, POSTAGs_batch, CAPs_batch, OUTs_batch = zip(*MICS)
  		SENTENCEs_batch = list(SENTENCEs_batch)
  		POSTAGs_batch = list(POSTAGs_batch)
  		CAPs_batch = list(CAPs_batch)
  		OUTs_batch = list(OUTs_batch)
  		
  		assert len(SENTENCEs_batch) == len(OUTs_batch)
  
  		optimizer.zero_grad()
  		for model in models_update:
  			model.zero_grad()
  
  		predict_fact_batch, predict_condition_batch, hidden_out_batch = Tag_Extractor(SENTENCEs_batch, POSTAGs_batch, CAPs_batch, models, tokenizer, LM_model, LM_corpus, device, multi_head, multi_head_two)
  		
  		if not pretrain:
  			predict_fact_tuples_batch, predict_cond_tuples_batch = Tuple_Extractor(predict_fact_batch, predict_condition_batch, hidden_out_batch, mimo_extractor_fact, mimo_extractor_cond, dataCenter)
  		else:
  			predict_fact_tuples_batch = None
  			predict_cond_tuples_batch = None
  
  		assert len(OUTs_batch) == len(predict_fact_batch) == len(predict_condition_batch)
  
  		loss = 0
  		nu_instances = 0
  		for i in range(len(OUTs_batch)):
  			seq_len = len(OUTs_batch[i][0][0])
  			fact_nu = len(OUTs_batch[i][0]) if not pretrain else 1
  			for k in range(fact_nu):
  				if not pretrain:
  					if k > len(predict_fact_tuples_batch[i]):
  						break
  				for j in range(seq_len):
  					tagID = OUTs_batch[i][0][k][j]
  					if k == 0:
  						_loss = (-weight_classes_fact[tagID] * predict_fact_batch[i][j][tagID])
  					else:
  						_loss = (-weight_classes_fact[tagID] * predict_fact_tuples_batch[i][k-1][j][tagID])
  					loss += _loss
  					nu_instances += 1
  			seq_len = len(OUTs_batch[i][1][0])
  			cond_nu = len(OUTs_batch[i][1]) if not pretrain else 1
  			for k in range(cond_nu):
  				if not pretrain:
  					if k > len(predict_cond_tuples_batch[i]):
  						break
  				for j in range(seq_len):
  					tagID = OUTs_batch[i][1][k][j]
  					if k == 0:
  						_loss = (-weight_classes_condition[tagID] * predict_condition_batch[i][j][tagID])
  					else:
  						_loss = (-weight_classes_condition[tagID] * predict_cond_tuples_batch[i][k-1][j][tagID])
  					loss += _loss
  					nu_instances += 1
  
  		loss /= len(OUTs_batch)
  		loss.to(device)
  
  		print('batch-%d-aver_loss = %.6f(%d)' % (index, loss.item(), len(SENTENCEs_batch[0])))
  
  		loss.backward()
  		for model in models_update:
  			nn.utils.clip_grad_norm_(model.parameters(), 5)
  		# for param in model.parameters():
  		#	if param.requires_grad:
  		#		print 'param:', param.grad
  		optimizer.step()
  
  		if is_semi:
  			pre_max_f1 = max_f1
  			max_f1, max_std = evaluation(models, out_file, dataCenter, 0, 0, max_f1, max_std, out_model_name, num_pass, False, write_prediction=True, file_name2='./predicts/'+out_file.split('/')[-1], LM_model=LM_model, just_PR=False, LM_corpus=LM_corpus, tokenizer=tokenizer, mimo_extractor_fact=mimo_extractor_fact, mimo_extractor_cond=mimo_extractor_cond, pretrain=pretrain, multi_head=multi_head, models_update=models_update, multi_head_two=multi_head_two)
  			if max_f1 == pre_max_f1:
  				for model in models:
  					print("loading model parameters...")
  					model = torch.load(out_model_name+model.name)
  					print("loading done.")
  
  	print('oov:', len(oov))
  
  	return models_update
  
  def get_position(VB_index, index):
  	min_dis = 999
  	position = -1
  	if index in VB_index:
  		return 0
  	for vi in VB_index:
  		if math.fabs(index-vi) <= min_dis:
  			min_dis = math.fabs(index-vi)
  			position = -1 if (index-vi<0) else 1
  	return position
  
  def auto_labeling(models, dataCenter, device, data_file, AR, TC, TCDEL, SH, DEL, LM_model=None, LM_corpus=None, tokenizer=None, mimo_extractor_fact=None, mimo_extractor_cond=None, pretrain=False, multi_head=None, multi_head_two=None):
  	AR_fact_file_name = './association_rules_fact.txt'
  	AR_condition_file_name = './association_rules_condition.txt'
  	support_threshold = 3
  	confidence_threshold = 0.7
  
  	params = []
  	for model in models:
  		for param in model.parameters():
  			if param.requires_grad:
  				param.requires_grad = False
  				params.append(param)
  
  	ar_correcter = AR_Correcter(AR_fact_file_name, AR_condition_file_name, support_threshold, confidence_threshold)
  
  	MICS = zip(dataCenter.ex_TRAIN_SENTENCEs, dataCenter.ex_TRAIN_POSTAGs, dataCenter.ex_TRAIN_CAPs, dataCenter.ex_TRAIN_OUTs, dataCenter.instance_ex_TRAIN)
  	MICS = sorted(MICS, key = lambda s: len(s[0]), reverse =True)
  	SENTENCEs, POSTAGs, CAPs, OUTs, instance_list = zip(*MICS)
  	SENTENCEs = list(SENTENCEs)
  	POSTAGs = list(POSTAGs)
  	CAPs = list(CAPs)
  	OUTs = list(OUTs)
  	instance_list = list(instance_list)
  		
  	assert len(SENTENCEs) == len(OUTs)
  	print(len(SENTENCEs))
  
  	batch_size = 200
  	batches = len(SENTENCEs) // batch_size
  
  	tag_outFile = open(data_file, 'w')
  	count = 0
  	for index in range(batches+1):
  		SENTENCEs_batch = SENTENCEs[index*batch_size:(index+1)*batch_size]
  		POSTAGs_batch = POSTAGs[index*batch_size:(index+1)*batch_size]
  		CAPs_batch = CAPs[index*batch_size:(index+1)*batch_size]
  		OUTs_batch = OUTs[index*batch_size:(index+1)*batch_size]
  		instance_list_batch = instance_list[index*batch_size:(index+1)*batch_size]
  
  		predict_fact_batch, predict_condition_batch, hidden_out_batch = Tag_Extractor(SENTENCEs_batch, POSTAGs_batch, CAPs_batch, models, tokenizer, LM_model, LM_corpus, device, multi_head, multi_head_two)
  		
  		if not pretrain:
  			predict_fact_tuples_batch, predict_cond_tuples_batch = Tuple_Extractor(predict_fact_batch, predict_condition_batch, hidden_out_batch, mimo_extractor_fact, mimo_extractor_cond, dataCenter)
  		else:
  			predict_fact_tuples_batch = None
  			predict_cond_tuples_batch = None
  
  		assert len(OUTs_batch) == len(predict_fact_batch) == len(predict_condition_batch)
  
  		for i in range(len(predict_fact_batch)):
  			_, predicted_fact_tags = torch.max(predict_fact_batch[i], 1)
  			_, predicted_conditions_tags = torch.max(predict_condition_batch[i], 1)
  			assert len(OUTs_batch[i][0][0]) == len(instance_list_batch[i].OUT[0][0])
  			fact_tags = []
  			cond_tags = []
  			if len(OUTs_batch[i][0][0]) > 15:
  				continue
  			for j in range(len(OUTs_batch[i][0][0])):
  				y_predict = predicted_fact_tags[j].item()
  				tag = dataCenter.ID2Tag_fact[y_predict]
  				fact_tags.append(tag)
  				y_predict = predicted_conditions_tags[j].item()
  				tag = dataCenter.ID2Tag_condition[y_predict]
  				cond_tags.append(tag)
  
  			if AR:
  				# print('using AR')
  				sentence = SENTENCEs_batch[i]
  				postag = POSTAGs_batch[i]
  				VB_index = []
  				for j in range(len(postag)):
  					if postag[j].startswith('VB'):
  						VB_index.append(j)
  				j = 0
  				while j < len(sentence):
  					flag = False
  					for k in range(len(sentence), j, -1):
  						_A = postag[j:k]
  						for kk in range(len(_A)):
  							if _A[kk] == 'IN':
  								_A[kk] += (':'+sentence[j+kk])
  							_A[kk] += (':'+str(get_position(VB_index, j+kk)))
  						_A = '\t'.join(_A)
  						if _A in ar_correcter.A2B_fact:
  							tags = ar_correcter.A2B_fact[_A].split('\t')
  							flag = True
  							fact_tags[j:k] = tags
  							j = k
  							break
  					if not flag:
  						j += 1
  
  				IN_index = []
  				for j in range(len(postag)):
  					if postag[j] == 'IN':
  						IN_index.append(j)
  
  				j = 0
  				while j < len(sentence):
  					flag = False
  					for k in range(len(sentence), j, -1):
  						_A = postag[j:k]
  						for kk in range(len(_A)):
  							if _A[kk] == 'IN':
  								_A[kk] += (':'+sentence[j+kk])
  							_A[kk] += (':'+str(get_position(IN_index, j+kk)))
  						_A = '\t'.join(_A)
  						if _A in ar_correcter.A2B_cond:
  							tags = ar_correcter.A2B_cond[_A].split('\t')
  							flag = True
  							cond_tags[j:k] = tags
  							j = k
  							break
  					if not flag:
  						j += 1
  
  			if TC:
  				fact_tags, corrected_fact = smooth_tag_sequence(fact_tags)
  				cond_tags, corrected_cond = smooth_tag_sequence(cond_tags)
  
  			if DEL:
  				# print('using DEL')
  				is_discarded_fact, fact_predicate_set = is_discarded(fact_tags)
  				is_discarded_cond, cond_predicate_set = is_discarded(cond_tags)
  
  				if is_discarded_fact or is_discarded_cond:
  					continue
  				if fact_predicate_set & cond_predicate_set != set():
  					continue
  			if TCDEL:
  				# print('using STDEL')
  				fact_tags, corrected_fact = smooth_tag_sequence(fact_tags)
  				cond_tags, corrected_cond = smooth_tag_sequence(cond_tags)
  				if corrected_fact or corrected_cond:
  					continue
  
  			tag_outFile.write('===== '+str(instance_list_batch[i].paper_id)+' stmt'+str(instance_list_batch[i].stmt_id)+' =====\n')
  			tag_outFile.write('WORD\t%s\n' % '\t'.join(instance_list_batch[i].SENTENCE))
  			tag_outFile.write('POSTAG\t%s\n' % '\t'.join(POSTAGs_batch[i]))
  			tag_outFile.write('CAP\t%s\n' % '\t'.join(CAPs_batch[i]))
  			tag_outFile.write('f\t%s\n' % '\t'.join(fact_tags))
  			tag_outFile.write('c\t%s\n' % '\t'.join(cond_tags))
  			count += 1
  
  	tag_outFile.write('#'+str(count)+'\n')
  	tag_outFile.close()
  
  	for param in params:
  		param.requires_grad = True
  
  