model improved

maker.py

@@ -64,6 +64,7 @@ class UDEmbedsDataset(object):
       import torch
       if len(x)<127:
         x=[True]*len(x)
+        w=(len(x)+1)*(len(x)+2)/2
       else:
         w=sum([len(x)-i+1 if b else 0 for i,b in enumerate(x)])+1
         for i in range(len(x)):
@@ -80,6 +81,14 @@ class UDEmbedsDataset(object):
           for j in range(i+1,len(x)):
             ids.append(j)
             upos.append(p[j]+"|"+d[j] if int(c[j][6])==i+1 else p[i]+"|"+d[i] if int(c[i][6])==j+1 else p[j]+"|_")
+          if i>0 and w>8192:
+            while w>8192:
+              if upos[-1].endswith("|_"):
+                upos.pop(-1)
+                ids.pop(-1)
+                w-=1
+              else:
+                break
           ids.append(-1)
           upos.append("SYM|_")
       with torch.no_grad():
@@ -90,7 +99,7 @@ class UDEmbedsDataset(object):
           m.append(self.embeddings[j,:].sum(axis=0))
         m.append(self.embeddings[self.tokenizer.sep_token_id,:])
         emb=torch.stack(m)
-    return{"inputs_embeds":emb[ids[:8192],:],"labels":[self.label2id[p] for p in upos[:8192]]}
+    return{"inputs_embeds":emb[ids,:],"labels":[self.label2id[p] for p in upos]}
 from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DefaultDataCollator,TrainingArguments,Trainer
 from tokenizers.pre_tokenizers import Sequence,Split
 from tokenizers import Regex
@@ -102,7 +111,7 @@ lid=trainDS(devDS,testDS)
 cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True)
 mdl=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True)
 trainDS.embeddings=mdl.get_input_embeddings().weight
-arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
+arg=TrainingArguments(num_train_epochs=10,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
 trn=Trainer(args=arg,data_collator=DefaultDataCollator(),model=mdl,train_dataset=trainDS)
 trn.train()
 trn.save_model(tgt)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ccd79192cd866429a281e3d0029002b495624fcc969f90929aad0b25e144f08c
+oid sha256:fc561bfa7e300ac4264be2a3deb05693acd70d31cbc1b15cbb2749781469d311
 size 516182194