model improved

maker.py

@@ -36,7 +36,7 @@ class UDEmbedsDataset(object):
     self.tokenizer=tokenizer
     self.embeddings=embeddings
     self.seeks=[0]
-    label=set(["SYM","SYM."])
+    label=set(["SYM","SYM.","SYM|_"])
     dep=set()
     s=self.conllu.readline()
     while s!="":
@@ -86,8 +86,9 @@ class UDEmbedsDataset(object):
       emb=self.embeddings
     else:
       import torch
-      if len(x)<128:
+      if len(x)<127:
         x=[True]*len(x)
+        w=(len(x)+2)*(len(x)+1)/2
       else:
         w=sum([len(x)-i+1 if b else 0 for i,b in enumerate(x)])+1
         for i in range(len(x)):
@@ -104,6 +105,11 @@ class UDEmbedsDataset(object):
           for j in range(i+1,len(x)):
             ids.append(j)
             upos.append(p[j]+"|"+d[j] if int(c[j][6])==i+1 else p[i]+"|"+d[i] if int(c[i][6])==j+1 else p[j]+"|_")
+          if w>8192 and i>0:
+            while w>8192 and upos[-1].endswith("|_"):
+              upos.pop(-1)
+              ids.pop(-1)
+              w-=1
           ids.append(-1)
           upos.append("SYM|_")
       with torch.no_grad():

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04c10ef1bc02136241e060e8e040ed06a23d1ce74b96ada894e1fcebd8886dcb
+oid sha256:0736e9bfbba2fc216739f47298aded53cd9ee3a725ef02f1958d556deaa03518
 size 611030258