text: # -*- coding: utf-8 -*-
  import codecs
  from pythainlp.tokenize import word_tokenize
  #import deepcut
  from nltk.tokenize import RegexpTokenizer
  import glob
  import nltk
  import re
  # thai cut
  thaicut="newmm"
  #จัดการประโยคซ้ำ
  data_not=[]
  def Unique(p):
   text=re.sub("<[^>]*>","",p)
   text=re.sub("\[(.*?)\]","",text)
   text=re.sub("\[\/(.*?)\]","",text)
   if text not in data_not:
    data_not.append(text)
    return True
   else:
    return False
  # เตรียมตัวตัด tag ด้วย re
  pattern = r'\[(.*?)\](.*?)\[\/(.*?)\]'
  tokenizer = RegexpTokenizer(pattern) # ใช้ nltk.tokenize.RegexpTokenizer เพื่อตัด [TIME]8.00[/TIME] ให้เป็น ('TIME','ไง','TIME')
  # จัดการกับ tag ที่ไม่ได้ tag
  def toolner_to_tag(text):
   text=text.strip().replace("FACILITY","LOCATION").replace("[AGO]","").replace("[/AGO]","").replace("[T]","").replace("[/T]","")
   text=re.sub("<[^>]*>","",text)
   text=re.sub("(\[\/(.*?)\])","\\1***",text)#.replace('(\[(.*?)\])','***\\1')# text.replace('>','>***') # ตัดการกับพวกไม่มี tag word
   text=re.sub("(\[\w+\])","***\\1",text)
   text2=[]
   for i in text.split('***'):
    if "[" in i:
     text2.append(i)
    else:
     text2.append("[word]"+i+"[/word]")
   text="".join(text2)#re.sub("[word][/word]","","".join(text2))
   return text.replace("[word][/word]","")
  # แปลง text ให้เป็น conll2002
  def text2conll2002(text,pos=True):
      """
      ใช้แปลงข้อความให้กลายเป็น conll2002
      """
      text=toolner_to_tag(text)
      text=text.replace("''",'"')
      text=text.replace("’",'"').replace("‘",'"')#.replace('"',"")
      tag=tokenizer.tokenize(text)
      j=0
      conll2002=""
      for tagopen,text,tagclose in tag:
          word_cut=word_tokenize(text,engine=thaicut) # ใช้ตัวตัดคำ newmm
          i=0
          txt5=""
          while i<len(word_cut):
              if word_cut[i]=="''" or word_cut[i]=='"':pass
              elif i==0 and tagopen!='word':
                  txt5+=word_cut[i]
                  txt5+='\tB-'+tagopen
              elif tagopen!='word':
                  txt5+=word_cut[i]
                  txt5+='\tI-'+tagopen
              else:
                  txt5+=word_cut[i]
                  txt5+='\t'+'Null'
              txt5+='\n'
              #j+=1
              i+=1
          conll2002+=txt5
      return conll2002
  # เขียนไฟล์ข้อมูล conll2002
  def write_conll2002(file_name,data):
      """
      ใช้สำหรับเขียนไฟล์
      """
      with codecs.open(file_name, "w", "utf-8-sig") as temp:
          temp.write(data)
      return True
  # อ่านข้อมูลจากไฟล์
  def get_data(fileopen):
  	"""
      สำหรับใช้อ่านทั้งหมดทั้งในไฟล์ทีละรรทัดออกมาเป็น list
      """
  	with codecs.open(fileopen, 'r',encoding='utf-8-sig') as f:
  		lines = f.read().splitlines()
  	return [a for a in lines if Unique(a)] # เอาไม่ซ้ำกัน
  
  def alldata(lists):
      text=""
      for data in lists:
          text+=text2conll2002(data)
          text+='\n'
      return text
  
  def alldata_list(lists):
      data_all=[]
      for data in lists:
          data_num=[]
          try:
              txt=text2conll2002(data,pos=False).split('\n')
              u=False
              t=''
              for idx, d in enumerate(txt):
                  tt=d.split('\t')
                  if d!="":
                      if tt[0]=="$" and u==False and t=='':
                          u=True
                          t=txt[idx+1].split('\t')[0]
                          del txt[idx+1]
                          pass
                      elif txt[idx+1].split('\t')[0]=="$$":
                          pass
                      elif tt[0]=="$":
                          pass
                      elif tt[0]=="$$":
                          t=''
                          pass
                      elif len(tt)==3:
                          data_num.append((tt[0],tt[1],tt[2]))
                      else:
                          data_num.append((tt[0],tt[1],t))
              #print(data_num)
              data_all.append(data_num)
          except:
              print("error ",data)
      #print(data_all)
      return data_all
  
  def alldata_list_str(lists):
  	string=""
  	for data in lists:
  		string1=""
  		for j in data:
  			string1+=j[0]+"	"+j[1]+"	"+j[2]+"\n"
  		string1+="\n"
  		string+=string1
  	return string
  
  def get_data_tag(listd):
  	list_all=[]
  	c=[]
  	for i in listd:
  		if i !='':
  			c.append((i.split("\t")[0],i.split("\t")[1],i.split("\t")[2]))
  		else:
  			list_all.append(c)
  			c=[]
  	return list_all
  def getall(lista):
      ll=[]
      for i in lista:
          o=True
          for j in ll:
              if re.sub("\[(.*?)\]","",i)==re.sub("\[(.*?)\]","",j):
                  o=False
                  break
          if o==True:
              ll.append(i)
      return ll
  def pre(text):
      return text.replace('B-','').replace('I-','')
  def to_coref(data):
      d={"sentences":[[j[0] for j in i] for i in data]}
      tag=[]
      for i in data:
          for j in i:
              t=pre(j[1])
              if t not in tag and t!='Null':
                  tag.append(t)
              t=pre(j[2])
              if t not in tag and t!='':
                  tag.append(t)
      d["clusters"]=[[]]*len(tag) #["mention_clusters"]
      o={}
      for i in tag:
          o[i]=[]
      for idx,i in enumerate(data):
          b=-1
          t=-1
          e=-1
          l=len(i)
          tt=""
          b2=-1
          t2=-1
          e2=-1
          tt2=""
          for in2,j in enumerate(i):
              if pre(j[1])=='Null' and b!=-1 and t!=-1:
                  e=in2-1
                  #print(d["mention_clusters"])
                  o[tt].append([idx,b,e])
                  b=-1
                  e=-1
                  t=-1
                  tt=''
              elif 'B-' in j[1] and in2<l-1:
                  b=in2
                  t=tag.index(pre(j[1]))
                  #print(idx,in2,pre(j[1]),j[0],1,t)
                  tt=pre(j[1])
              elif 'B-' in j[1] and in2==l-1:
                  b=in2
                  t=tag.index(pre(j[1]))
                  tt=pre(j[1])
                  e=in2
                  o[tt].append([idx,b,e])
              elif pre(j[1])!='Null' and in2==l-1:
                  e=in2
                  o[tt].append([idx,b,e])
              if j[2]!='' and in2<l-1 and b2==-1:
                  #print(idx,in2,j,1,t)
                  b2=in2
                  t2=tag.index(pre(j[2]))
                  #print(idx,in2,pre(j[1]),j[0],1,t)
                  tt2=pre(j[2])
              elif j[2]=='' and b2!=-1:
                  e2=in2-1
                  o[tt2].append([idx,b2,e2])
                  b2=-1
                  e2=-1
                  t2=-1
                  tt2=''
              elif b2!=-1 and in2==l-1:
                  e2=in2
                  o[tt2].append([idx,b2,e2])
                  #print(o[tt2])
                  b2=-1
                  e2=-1
                  t2=-1
                  tt2=''
      #print(o)
      for key, value in o.items():
           d["clusters"][tag.index(key)]=value #d["mention_clusters"]
      return d
  from pythainlp.tag.named_entity import ThaiNameTagger
  ner = ThaiNameTagger()
  def to_crf(data):
      list_word=[[i[0] for i in j] for j in data]
      tag=[[i[1] for i in j] for j in data]
      list_ner=[ner.get_ner(''.join(j)) for j in list_word]
      return (list_ner,tag)
  
  def get_conf(name):
      return to_coref(alldata_list(getall(get_data(name))))
  '''
  {"sentences":[[list ที่ตัดคำแล้วแต่ละประโยค idx],...],"mention_clusters":[[[sentence_idx, begin_idx, end_idx]],แต่ละชนิด,[[...,...,..],...] ]}
  '''