maker.py

#! /usr/bin/python3
import os,json,numpy
tgt="KoichiYasuoka/modernbert-large-english-ud-square"
url="https://github.com/UniversalDependencies/UD_English-"
for e in ["EWT","GUM","Atis","ParTUT","LinES"]:
  u=url+e
  d=os.path.basename(u)
  os.system("test -d "+d+" || git clone --depth=1 "+u)
s='BEGIN{FS="\\t";OFS="\\t"};{if(NF==10){if($1~/^[1-9][0-9]*-/){split($1,a,"-");if($10~/SpaceAfter=No/)a[2]++}else if($1-a[1]>=0&&$1-a[2]<0)$10=($10=="_")?"SpaceAfter=No":$10"|SpaceAfter=No"}print}'
os.system("for F in train dev test ; do nawk '"+s+"' UD_English-*/*-$F.conllu > $F.conllu ; done")
os.system("""
if test -d transformers
then :
else git clone --depth=1 https://github.com/huggingface/transformers transformers-all
     ln -s transformers-all/src/transformers transformers
fi
test -d ModernBERT-large || git clone --depth=1 https://huggingface.co/answerdotai/ModernBERT-large
test -f ModernBERT-large/configuration_modernbert.py || sed 's/^from \\.\\.\\./from transformers./' transformers/models/modernbert/configuration_modernbert.py > ModernBERT-large/configuration_modernbert.py
test -f ModernBERT-large/modeling_modernbert.py || sed -e 's/^from \\.\\.\\./from transformers./' -e 's/^from .* import is_triton_available/import importlib\\nis_triton_available = lambda: importlib.util.find_spec("triton") is not None/' transformers/models/modernbert/modeling_modernbert.py > ModernBERT-large/modeling_modernbert.py
""")
with open("ModernBERT-large/config.json","r",encoding="utf-8") as r:
  d=json.load(r)
if not "auto_map" in d:
  d["auto_map"]={
    "AutoConfig":"configuration_modernbert.ModernBertConfig",
    "AutoModel":"modeling_modernbert.ModernBertModel",
    "AutoModelForMaskedLM":"modeling_modernbert.ModernBertForMaskedLM",
    "AutoModelForSequenceClassification":"modeling_modernbert.ModernBertForSequenceClassification",
    "AutoModelForTokenClassification":"modeling_modernbert.ModernBertForTokenClassification"
  }
  with open("ModernBERT-large/config.json","w",encoding="utf-8") as w:
    json.dump(d,w,indent=2)
class UDSquareDataset(object):
  def __init__(self,conllu,tokenizer):
    self.conllu=open(conllu,"r",encoding="utf-8")
    self.tokenizer=tokenizer
    self.seeks=[0]
    label=set(["SYM.","X.","X.|[goeswith]"])
    s=self.conllu.readline()
    while s!="":
      if s=="\n":
        self.seeks.append(self.conllu.tell())
      else:
        w=s.split("\t")
        if len(w)==10:
          if w[0].isdecimal():
            p=w[3]
            q="" if w[5]=="_" else "|"+w[5]
            r="|["+w[7]+"]"
            label.add(p+q)
            label.add(p+"."+q)
            label.add(p+q+r)
            label.add(p+"."+q+r)
      s=self.conllu.readline()
    self.label2id={l:i for i,l in enumerate(sorted(label))}
  def __call__(*args):
    lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))}
    for t in args:
      t.label2id=lid
    return lid
  def __del__(self):
    self.conllu.close()
  __len__=lambda self:len(self.seeks)-1
  def __getitem__(self,i):
    self.conllu.seek(self.seeks[i])
    c,t=[],[""]
    while t[0]!="\n":
      t=self.conllu.readline().split("\t")
      if len(t)==10 and t[0].isdecimal():
        c.append(t)
    h={t[6] for t in c}
    for t in c:
      if t[6]!="0" and t[0] not in h:
        t[3]+="."
    v=self.tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
    for i in range(len(v)-1,-1,-1):
      if v[i]==[]:
        v[i]=[self.tokenizer.unk_token_id]
      if len(v[i])>1:
        c[i][3]=c[i][3].replace(".","")
        for j in range(1,len(v[i])):
          c.insert(i+1,[c[i][0],"_","_","X.","_","_",c[i][0],"goeswith","_","_"])
    y=["0"]+[t[0] for t in c]
    h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
    p=[t[3] if t[5]=="_" else t[3]+"|"+t[5] for t in c]
    d=["|["+t[7]+"]" for t in c]
    x=[not t[3].endswith(".") for t in c]
    if len(x)<90:
      x=[True]*len(x)
    else:
      w=(sum([1 for b in x if b])+1)*(len(x)+1)+1
      for i in numpy.argsort([-abs(j-i-1) for i,j in enumerate(h)]):
        if w+len(x)>8191:
          break
        if not x[i]:
          x[i]=True
          w+=len(x)+1
    v=sum(v,[])
    ids=[self.tokenizer.cls_token_id]+v+[self.tokenizer.sep_token_id]
    upos=["SYM."]+p+["SYM."]
    for i in range(len(v)):
      if x[i]:
        for j in range(len(v)):
          ids.append(self.tokenizer.mask_token_id if i==j else v[j])
          upos.append(p[j]+d[j] if h[j]==i+1 else p[j])
        ids.append(self.tokenizer.sep_token_id)
        upos.append("SYM.")
    return {"input_ids":ids[:8192],"labels":[self.label2id[p] for p in upos[:8192]]}
from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
from tokenizers.pre_tokenizers import Sequence,Split
from tokenizers import Regex
tkz=AutoTokenizer.from_pretrained("ModernBERT-large")
tkz.backend_tokenizer.pre_tokenizer=Sequence([Split(Regex("[nN]['`’][tT]"),"isolated"),tkz.backend_tokenizer.pre_tokenizer])
trainDS=UDSquareDataset("train.conllu",tkz)
devDS=UDSquareDataset("dev.conllu",tkz)
testDS=UDSquareDataset("test.conllu",tkz)
lid=trainDS(devDS,testDS)
cfg=AutoConfig.from_pretrained("ModernBERT-large",num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True)
mdl=AutoModelForTokenClassification.from_pretrained("ModernBERT-large",config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True)
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=mdl,train_dataset=trainDS)
trn.train()
trn.save_model(tgt)
tkz.save_pretrained(tgt)
os.system("cp -p ModernBERT-large/*.py "+tgt)