Upload 7 files

README.MD

@@ -0,0 +1,29 @@
+This model repository presents "TinyPubMedBERT", a distillated PubMedBERT (Gu et al.,  2021) model. 
+TinyPubMedBERT is used as the initial weights for the training of the [dmis-lab/KAZU-NER-module-distil-v1.0](https://huggingface.co/dmis-lab/KAZU-NER-module-distil-v1.0) which is used in the initial release of the KAZU (Korea University and AstraZeneca) framework.
+
+The model is composed of 4-layers and distillated following methods introduced in TinyBERT paper (Jiao et al., 2020).
+
+* For the framework, please visit https://github.com/AstraZeneca/KAZU
+* For details about the model, please see our paper entitled **Biomedical NER for the Enterprise with Distillated BERN2 and the Kazu Framework**, (EMNLP 2022 industry track).
+
+More details to be announced soon.
+
+
+### Citation info
+Joint-first authorship of **Richard Jackson** (AstraZeneca) and **WonJin Yoon** (Korea University).
+<br>Please cite: (Full citation info will be announced soon)
+```
+@inproceedings{YoonAndJackson2022BiomedicalNER,
+  title={Biomedical NER for the Enterprise with Distillated BERN2 and the Kazu Framework},
+  author={Wonjin Yoon, Richard Jackson, Elliot Ford, Vladimir Poroshin, Jaewoo Kang},
+  booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
+  year={2022}
+}
+```
+The model used resources of PubMedBERT paper and TinyBERT paper.
+Gu, Yu, et al. "Domain-specific language model pretraining for biomedical natural language processing." ACM Transactions on Computing for Healthcare (HEALTH) 3.1 (2021): 1-23.
+Jiao, Xiaoqi, et al. "TinyBERT: Distilling BERT for Natural Language Understanding." Findings of the Association for Computational Linguistics: EMNLP 2020. 2020.
+
+
+### Contact Information
+For help or issues using the codes or model (NER module of KAZU) in this repository, please contact WonJin Yoon (wonjin.info (at) gmail.com) or submit a GitHub issue.

config.json

@@ -0,0 +1,19 @@
+{
+  "attention_probs_dropout_prob": 0.1,
+  "model_type":"bert",
+  "cell": {},
+  "emb_size": 312,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 312,
+  "initializer_range": 0.02,
+  "intermediate_size": 1200,
+  "max_position_embeddings": 512,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 4,
+  "pre_trained": "",
+  "structure": [],
+  "training": "",
+  "type_vocab_size": 2,
+  "vocab_size": 30522
+}

log.txt

@@ -0,0 +1,1024 @@
+att_loss = 17266.918771276512
+global_step = 249
+loss = 2164.4442341509593
+rep_loss = 48.63509366981476
+att_loss = 17277.140605674238
+global_step = 499
+loss = 2165.716661900461
+rep_loss = 48.59268923226244
+att_loss = 17277.204854193613
+global_step = 749
+loss = 2165.718913059209
+rep_loss = 48.54645042966937
+att_loss = 17271.71982749351
+global_step = 999
+loss = 2165.026342458792
+rep_loss = 48.49091171478485
+att_loss = 17259.165888961168
+global_step = 1249
+loss = 2163.44941790835
+rep_loss = 48.42945459578112
+att_loss = 17254.51667656367
+global_step = 1499
+loss = 2162.8583398759165
+rep_loss = 48.35004332576138
+att_loss = 17243.435463959315
+global_step = 1749
+loss = 2161.4617817673156
+rep_loss = 48.25879234910489
+att_loss = 17231.452888065127
+global_step = 1999
+loss = 2159.950227715481
+rep_loss = 48.14893700922651
+att_loss = 17213.784783254363
+global_step = 2249
+loss = 2157.7255987266585
+rep_loss = 48.02000851459427
+att_loss = 17192.42183030439
+global_step = 2499
+loss = 2155.035966449568
+rep_loss = 47.86590490955599
+att_loss = 17166.00479933044
+global_step = 2749
+loss = 2151.711059942207
+rep_loss = 47.68368448695949
+att_loss = 17130.214220088335
+global_step = 2999
+loss = 2147.211177036022
+rep_loss = 47.47519993312997
+att_loss = 17082.496581167103
+global_step = 3249
+loss = 2141.2175883689124
+rep_loss = 47.24412930859313
+att_loss = 17025.656524771588
+global_step = 3499
+loss = 2134.081171550898
+rep_loss = 46.992851370980176
+att_loss = 16953.278561555217
+global_step = 3749
+loss = 2125.0014296781223
+rep_loss = 46.73287839932771
+att_loss = 16867.423787129643
+global_step = 3999
+loss = 2114.236494361475
+rep_loss = 46.46817037039144
+att_loss = 16766.430148919462
+global_step = 4249
+loss = 2101.5791739851875
+rep_loss = 46.20324492976368
+att_loss = 16650.415473105353
+global_step = 4499
+loss = 2087.044679908388
+rep_loss = 45.941968157917266
+att_loss = 16520.907675773025
+global_step = 4749
+loss = 2070.82408304349
+rep_loss = 45.68499064028803
+att_loss = 16377.305757489388
+global_step = 4999
+loss = 2052.8424795921096
+rep_loss = 45.43408122790482
+att_loss = 16219.348001274797
+global_step = 5249
+loss = 2033.067252099707
+rep_loss = 45.19001775888017
+att_loss = 16047.804864911517
+global_step = 5499
+loss = 2011.5946817240253
+rep_loss = 44.952591092136906
+att_loss = 15862.696737085389
+global_step = 5749
+loss = 1988.4273826425274
+rep_loss = 44.72232620810235
+att_loss = 15662.845802791677
+global_step = 5999
+loss = 1963.4181762354476
+rep_loss = 44.49960949548504
+att_loss = 15450.64399507046
+global_step = 6249
+loss = 1936.8658219494807
+rep_loss = 44.28258329186369
+att_loss = 15224.739606320518
+global_step = 6499
+loss = 1908.6014941453677
+rep_loss = 44.07234962009654
+att_loss = 14986.976998400805
+global_step = 6749
+loss = 1878.8555389734388
+rep_loss = 43.867316366284136
+att_loss = 14736.92022153939
+global_step = 6999
+loss = 1847.5735737941898
+rep_loss = 43.668371733469115
+att_loss = 14475.944257540708
+global_step = 7249
+loss = 1814.9273117376963
+rep_loss = 43.47423925345841
+att_loss = 14205.014850148966
+global_step = 7499
+loss = 1781.0373600503099
+rep_loss = 43.28403311186527
+att_loss = 13925.22820553427
+global_step = 7749
+loss = 1746.040812043532
+rep_loss = 43.09829361784272
+att_loss = 13638.28624035463
+global_step = 7999
+loss = 1710.150303852679
+rep_loss = 42.916193216409454
+att_loss = 13346.527509119831
+global_step = 8249
+loss = 1673.6578935689529
+rep_loss = 42.7356419749427
+att_loss = 13052.027369215428
+global_step = 8499
+loss = 1636.8227225363235
+rep_loss = 42.5544134891374
+att_loss = 12756.839565272303
+global_step = 8749
+loss = 1599.9017323053474
+rep_loss = 42.37429549416374
+att_loss = 12463.840603725528
+global_step = 8999
+loss = 1563.2544286988393
+rep_loss = 42.19482811192219
+att_loss = 12175.877058573808
+global_step = 9249
+loss = 1527.2367027574312
+rep_loss = 42.0165656104012
+att_loss = 11896.004569317394
+global_step = 9499
+loss = 1492.2302233208757
+rep_loss = 41.837219315185614
+att_loss = 11625.584467229824
+global_step = 9749
+loss = 1458.4046069217688
+rep_loss = 41.65239013487381
+att_loss = 11364.693283088685
+global_step = 9999
+loss = 1425.7699238807395
+rep_loss = 41.46610991326985
+att_loss = 11113.502839691315
+global_step = 10249
+loss = 1394.3479349004408
+rep_loss = 41.28064139020281
+att_loss = 10872.289114246756
+global_step = 10499
+loss = 1364.1734168542
+rep_loss = 41.098222436104656
+att_loss = 10641.122695878845
+global_step = 10749
+loss = 1335.255631208708
+rep_loss = 40.92235558633416
+att_loss = 10419.773089068556
+global_step = 10999
+loss = 1307.5657580675934
+rep_loss = 40.752977223469564
+att_loss = 10207.635612109341
+global_step = 11249
+loss = 1281.0279992851579
+rep_loss = 40.58838390098041
+att_loss = 10004.12612569298
+global_step = 11499
+loss = 1255.569151935997
+rep_loss = 40.42709148733582
+att_loss = 9808.771755357146
+global_step = 11749
+loss = 1231.1296315219556
+rep_loss = 40.26529846204394
+att_loss = 9621.062163698782
+global_step = 11999
+loss = 1207.6455957861142
+rep_loss = 40.10260421332245
+att_loss = 9440.626454007257
+global_step = 12249
+loss = 1185.0705693819755
+rep_loss = 39.93810263039365
+att_loss = 9267.06510345962
+global_step = 12499
+loss = 1163.3546925466226
+rep_loss = 39.77243846442683
+att_loss = 9100.087103520911
+global_step = 12749
+loss = 1142.461500134559
+rep_loss = 39.604899072901425
+att_loss = 8939.272877253352
+global_step = 12999
+loss = 1122.3386413426902
+rep_loss = 39.43625497553879
+att_loss = 8784.335411021679
+global_step = 13249
+loss = 1102.9502700823227
+rep_loss = 39.266751113914346
+att_loss = 8634.944687540738
+global_step = 13499
+loss = 1084.2552051671355
+rep_loss = 39.096955254957656
+att_loss = 8490.80984011639
+global_step = 13749
+loss = 1066.2171257451037
+rep_loss = 38.92716727312959
+att_loss = 8351.693344140736
+global_step = 13999
+loss = 1048.8063338907355
+rep_loss = 38.757328404333855
+att_loss = 8217.371463272595
+global_step = 14249
+loss = 1031.9948396856087
+rep_loss = 38.58725561547826
+att_loss = 8087.521230980597
+global_step = 14499
+loss = 1015.7423801677985
+rep_loss = 38.41781174607898
+att_loss = 7961.987407318867
+global_step = 14749
+loss = 1000.0294606625666
+rep_loss = 38.24827932726287
+att_loss = 7840.550895295689
+global_step = 14999
+loss = 984.828728281567
+rep_loss = 38.078932277249756
+att_loss = 7722.9954938576475
+global_step = 15249
+loss = 970.1131616603415
+rep_loss = 37.90980071902345
+att_loss = 7609.155667750541
+global_step = 15499
+loss = 955.8620857408843
+rep_loss = 37.74101945167742
+att_loss = 7498.834185800413
+global_step = 15749
+loss = 942.0508920113786
+rep_loss = 37.57295154728914
+att_loss = 7391.8848443881625
+global_step = 15999
+loss = 928.6613625704598
+rep_loss = 37.406057414892906
+att_loss = 7288.092727160613
+global_step = 16249
+loss = 915.6669200155594
+rep_loss = 37.24263418500655
+att_loss = 7187.245315153618
+global_step = 16499
+loss = 903.0413461738127
+rep_loss = 37.08545544504563
+att_loss = 7089.2361219459535
+global_step = 16749
+loss = 890.770895313623
+rep_loss = 36.931041745643014
+att_loss = 6993.987926303628
+global_step = 16999
+loss = 878.8456386448293
+rep_loss = 36.77718401374561
+att_loss = 6901.391028709706
+global_step = 17249
+loss = 867.2518573375679
+rep_loss = 36.62383113574191
+att_loss = 6811.345997624687
+global_step = 17499
+loss = 855.97704065998
+rep_loss = 36.470328775499404
+att_loss = 6723.7563525545975
+global_step = 17749
+loss = 845.009088463928
+rep_loss = 36.31635626245333
+att_loss = 6638.549228833737
+global_step = 17999
+loss = 834.3388273362962
+rep_loss = 36.16139095529886
+att_loss = 6555.648449949764
+global_step = 18249
+loss = 823.9567385304105
+rep_loss = 36.00545937437914
+att_loss = 6474.9481569615255
+global_step = 18499
+loss = 813.8496097940723
+rep_loss = 35.8487224576022
+att_loss = 6396.36941115192
+global_step = 18749
+loss = 804.0076034054438
+rep_loss = 35.691417147352944
+att_loss = 6319.834850662451
+global_step = 18999
+loss = 794.4210667197808
+rep_loss = 35.5336841422198
+att_loss = 6245.248959206353
+global_step = 19249
+loss = 785.0780889001894
+rep_loss = 35.375753029816316
+att_loss = 6172.540563334624
+global_step = 19499
+loss = 775.9697730082097
+rep_loss = 35.21762175089005
+att_loss = 568.803545459948
+global_step = 19749
+loss = 73.93347120786969
+rep_loss = 22.664224607066103
+att_loss = 568.5669987222423
+global_step = 19999
+loss = 73.88368892669678
+rep_loss = 22.502512358237006
+att_loss = 568.070243303315
+global_step = 20249
+loss = 73.80140220658117
+rep_loss = 22.340974200673465
+att_loss = 567.6636848043408
+global_step = 20499
+loss = 73.73088559596496
+rep_loss = 22.183399826252955
+att_loss = 566.888986146831
+global_step = 20749
+loss = 73.61525668139872
+rep_loss = 22.033067152162666
+att_loss = 566.2101754964949
+global_step = 20999
+loss = 73.51172613087196
+rep_loss = 21.88363347124433
+att_loss = 565.3120161074456
+global_step = 21249
+loss = 73.38124929864578
+rep_loss = 21.737978232841133
+att_loss = 564.4611608841232
+global_step = 21499
+loss = 73.25685410693409
+rep_loss = 21.593671895494953
+att_loss = 563.8721755071016
+global_step = 21749
+loss = 73.16527593471554
+rep_loss = 21.45003184521113
+att_loss = 563.0968106397942
+global_step = 21999
+loss = 73.05074261683644
+rep_loss = 21.309130188752846
+att_loss = 562.5641768938774
+global_step = 22249
+loss = 72.96666242970667
+rep_loss = 21.169122509506167
+att_loss = 561.8916205836935
+global_step = 22499
+loss = 72.86534705723527
+rep_loss = 21.03115586002598
+att_loss = 561.2273412750687
+global_step = 22749
+loss = 72.76536695714529
+rep_loss = 20.89559435097197
+att_loss = 560.6029947471903
+global_step = 22999
+loss = 72.67068205350184
+rep_loss = 20.762461703632624
+att_loss = 559.9501114984547
+global_step = 23249
+loss = 72.57255971640639
+rep_loss = 20.630366310257575
+att_loss = 559.3460526415523
+global_step = 23499
+loss = 72.4807679872054
+rep_loss = 20.500091333848136
+att_loss = 558.7232904827785
+global_step = 23749
+loss = 72.38689525042989
+rep_loss = 20.371871630117862
+att_loss = 558.0926710499718
+global_step = 23999
+loss = 72.29232771448766
+rep_loss = 20.245950728209028
+att_loss = 557.5161360830945
+global_step = 24249
+loss = 72.20468365261422
+rep_loss = 20.12133319956434
+att_loss = 556.9191301414718
+global_step = 24499
+loss = 72.1147452129053
+rep_loss = 19.998831636027287
+att_loss = 556.3499613072149
+global_step = 24749
+loss = 72.0284678045503
+rep_loss = 19.877781194933966
+att_loss = 555.8180808992448
+global_step = 24999
+loss = 71.94711245500227
+rep_loss = 19.758818818298426
+att_loss = 555.2285698721087
+global_step = 25249
+loss = 71.85882825288951
+rep_loss = 19.64205625349118
+att_loss = 554.7490055753187
+global_step = 25499
+loss = 71.7844786938487
+rep_loss = 19.526824089397824
+att_loss = 554.2084955723976
+global_step = 25749
+loss = 71.70283811223419
+rep_loss = 19.414209418026907
+att_loss = 553.6828962171237
+global_step = 25999
+loss = 71.62333540754867
+rep_loss = 19.303787130742116
+att_loss = 553.1985000778094
+global_step = 26249
+loss = 71.54924880910589
+rep_loss = 19.195490497038886
+att_loss = 552.7054209829683
+global_step = 26499
+loss = 71.47437279327147
+rep_loss = 19.089561467801158
+att_loss = 552.2043198470585
+global_step = 26749
+loss = 71.39876808444743
+rep_loss = 18.98582492440918
+att_loss = 551.7215547357797
+global_step = 26999
+loss = 71.32573316988287
+rep_loss = 18.88431070933462
+att_loss = 551.2641789369162
+global_step = 27249
+loss = 71.25615824510425
+rep_loss = 18.785087129471723
+att_loss = 550.7662613197069
+global_step = 27499
+loss = 71.18177146832605
+rep_loss = 18.68791050243864
+att_loss = 550.2952256681597
+global_step = 27749
+loss = 71.1110345714397
+rep_loss = 18.593050957108957
+att_loss = 549.8622339791492
+global_step = 27999
+loss = 71.04531030003506
+rep_loss = 18.500248481171656
+att_loss = 549.4039358333213
+global_step = 28249
+loss = 70.97665694765465
+rep_loss = 18.409319813907526
+att_loss = 548.9680096317376
+global_step = 28499
+loss = 70.91105011217064
+rep_loss = 18.32039132633069
+att_loss = 548.5215405326286
+global_step = 28749
+loss = 70.84437117715535
+rep_loss = 18.23342894976974
+att_loss = 548.0985423778454
+global_step = 28999
+loss = 70.78087078551674
+rep_loss = 18.148423988121344
+att_loss = 547.6451674794828
+global_step = 29249
+loss = 70.71380790712438
+rep_loss = 18.06529587243236
+att_loss = 547.2482466779547
+global_step = 29499
+loss = 70.65403027701583
+rep_loss = 17.98399563381792
+att_loss = 546.8227743930071
+global_step = 29749
+loss = 70.59089194435245
+rep_loss = 17.904361261194918
+att_loss = 546.3994455970849
+global_step = 29999
+loss = 70.5282377780953
+rep_loss = 17.82645672483108
+att_loss = 545.9787717393458
+global_step = 30249
+loss = 70.46615296688323
+rep_loss = 17.75045208852209
+att_loss = 545.5533168375904
+global_step = 30499
+loss = 70.40367570479602
+rep_loss = 17.676088884691314
+att_loss = 545.1392937153511
+global_step = 30749
+loss = 70.3428395847202
+rep_loss = 17.603423051008935
+att_loss = 544.702601219369
+global_step = 30999
+loss = 70.27936210565075
+rep_loss = 17.532295710434205
+att_loss = 544.2759774692103
+global_step = 31249
+loss = 70.21734446447026
+rep_loss = 17.462778319002688
+att_loss = 543.8311069821848
+global_step = 31499
+loss = 70.15323589308043
+rep_loss = 17.39478023828158
+att_loss = 543.3645310306115
+global_step = 31749
+loss = 70.08659764120726
+rep_loss = 17.328250161494633
+att_loss = 542.8894112093722
+global_step = 31999
+loss = 70.01909737438265
+rep_loss = 17.26336783997778
+att_loss = 542.3946143301388
+global_step = 32249
+loss = 69.94934149257907
+rep_loss = 17.200117651258683
+att_loss = 541.8585259116452
+global_step = 32499
+loss = 69.87462579326993
+rep_loss = 17.138480488332434
+att_loss = 541.2866426885196
+global_step = 32749
+loss = 69.7956412803312
+rep_loss = 17.07848758956047
+att_loss = 540.6902034943896
+global_step = 32999
+loss = 69.71379761140237
+rep_loss = 17.020177439787542
+att_loss = 540.0849288407298
+global_step = 33249
+loss = 69.63103771462252
+rep_loss = 16.963372914174673
+att_loss = 539.4442540252454
+global_step = 33499
+loss = 69.54398421755592
+rep_loss = 16.90761975887322
+att_loss = 538.7701227846446
+global_step = 33749
+loss = 69.45288906286862
+rep_loss = 16.85298976632626
+att_loss = 538.0739389115736
+global_step = 33999
+loss = 69.35915206272288
+rep_loss = 16.799277641823917
+att_loss = 537.3629695459766
+global_step = 34249
+loss = 69.26371267964963
+rep_loss = 16.746731938422236
+att_loss = 536.6309045878353
+global_step = 34499
+loss = 69.16578501913033
+rep_loss = 16.695375614416804
+att_loss = 535.8522947677203
+global_step = 34749
+loss = 69.06215787055517
+rep_loss = 16.644968248092802
+att_loss = 535.061523088472
+global_step = 34999
+loss = 68.95718339396129
+rep_loss = 16.59594411223049
+att_loss = 534.2347752364716
+global_step = 35249
+loss = 68.8478561000512
+rep_loss = 16.54807361378506
+att_loss = 533.3319543173427
+global_step = 35499
+loss = 68.7291399041799
+rep_loss = 16.501164960748355
+att_loss = 532.4195050645149
+global_step = 35749
+loss = 68.60936011142263
+rep_loss = 16.455375870394757
+att_loss = 531.4884166494645
+global_step = 35999
+loss = 68.48734341712128
+rep_loss = 16.410330731608372
+att_loss = 530.547413615715
+global_step = 36249
+loss = 68.36417261975902
+rep_loss = 16.36596738483289
+att_loss = 529.5643084049649
+global_step = 36499
+loss = 68.23580734429001
+rep_loss = 16.322150390365422
+att_loss = 528.5736010234305
+global_step = 36749
+loss = 68.10659463633637
+rep_loss = 16.279156108382292
+att_loss = 527.5627225302385
+global_step = 36999
+loss = 67.9749567699075
+rep_loss = 16.236931669151932
+att_loss = 526.5287834833074
+global_step = 37249
+loss = 67.84054858117457
+rep_loss = 16.195605205317456
+att_loss = 525.4722971581919
+global_step = 37499
+loss = 67.70341618880734
+rep_loss = 16.155032388039682
+att_loss = 524.3897297541125
+global_step = 37749
+loss = 67.56309876005832
+rep_loss = 16.115060361896177
+att_loss = 523.2993466628608
+global_step = 37999
+loss = 67.42187455775921
+rep_loss = 16.075649831086423
+att_loss = 522.1973293451123
+global_step = 38249
+loss = 67.27924377308574
+rep_loss = 16.036620870230177
+att_loss = 521.1076512535866
+global_step = 38499
+loss = 67.1382169603574
+rep_loss = 15.99808445711862
+att_loss = 520.0252741897315
+global_step = 38749
+loss = 66.99814691873208
+rep_loss = 15.959901182549759
+att_loss = 518.9590398213141
+global_step = 38999
+loss = 66.86016447237833
+rep_loss = 15.92227598179462
+att_loss = 517.9105902646263
+global_step = 39249
+loss = 66.72446811673105
+rep_loss = 15.885154695351472
+att_loss = 430.41770705377866
+global_step = 39499
+loss = 55.413460654113926
+rep_loss = 12.889978018730723
+att_loss = 429.78013253590416
+global_step = 39749
+loss = 55.33112620550489
+rep_loss = 12.868876984330262
+att_loss = 429.5238455014705
+global_step = 39999
+loss = 55.29708110613347
+rep_loss = 12.852803301535191
+att_loss = 429.2593395666919
+global_step = 40249
+loss = 55.262259974363126
+rep_loss = 12.838740212631023
+att_loss = 429.0502507508452
+global_step = 40499
+loss = 55.23442752975861
+rep_loss = 12.825169506941794
+att_loss = 428.6518964721129
+global_step = 40749
+loss = 55.182567169896274
+rep_loss = 12.808640934294251
+att_loss = 428.3047634104453
+global_step = 40999
+loss = 55.13724845658386
+rep_loss = 12.793224244904476
+att_loss = 428.0248651833709
+global_step = 41249
+loss = 55.10051056480113
+rep_loss = 12.779219316121907
+att_loss = 427.59188728907077
+global_step = 41499
+loss = 55.04444140726331
+rep_loss = 12.763643969035703
+att_loss = 427.19180261439254
+global_step = 41749
+loss = 54.99262022278632
+rep_loss = 12.749159168239892
+att_loss = 426.83212551739257
+global_step = 41999
+loss = 54.94592204886004
+rep_loss = 12.735250879202312
+att_loss = 426.51639836015283
+global_step = 42249
+loss = 54.90487258428621
+rep_loss = 12.722582299342086
+att_loss = 426.21925660877235
+global_step = 42499
+loss = 54.866193434991125
+rep_loss = 12.710290857819844
+att_loss = 425.84940542117414
+global_step = 42749
+loss = 54.81834573518464
+rep_loss = 12.697360449875209
+att_loss = 425.47620234131585
+global_step = 42999
+loss = 54.770065517032805
+rep_loss = 12.684321784643712
+att_loss = 425.0879732096628
+global_step = 43249
+loss = 54.71991780154507
+rep_loss = 12.671369195619654
+att_loss = 424.7154397102052
+global_step = 43499
+loss = 54.67179627863384
+rep_loss = 12.65893052896319
+att_loss = 424.3949641756205
+global_step = 43749
+loss = 54.630329651543754
+rep_loss = 12.647673039628536
+att_loss = 424.0026099238155
+global_step = 43999
+loss = 54.579816030556394
+rep_loss = 12.635918306506435
+att_loss = 423.5783165717651
+global_step = 44249
+loss = 54.52532899758517
+rep_loss = 12.624315396056824
+att_loss = 423.06853340926415
+global_step = 44499
+loss = 54.46005894117699
+rep_loss = 12.61193810453582
+att_loss = 422.57546552711716
+global_step = 44749
+loss = 54.39705093238661
+rep_loss = 12.600941921546822
+att_loss = 421.9954732306036
+global_step = 44999
+loss = 54.323057490814506
+rep_loss = 12.588986684705855
+att_loss = 421.38440246980133
+global_step = 45249
+loss = 54.245232343954626
+rep_loss = 12.577456274451562
+att_loss = 420.79632926291475
+global_step = 45499
+loss = 54.17038325118895
+rep_loss = 12.566736740242625
+att_loss = 420.1323002994551
+global_step = 45749
+loss = 54.08597275737353
+rep_loss = 12.555481753425557
+att_loss = 419.47063839430234
+global_step = 45999
+loss = 54.00194308127439
+rep_loss = 12.544906236455144
+att_loss = 418.78285078568894
+global_step = 46249
+loss = 53.91468508692506
+rep_loss = 12.534629891918641
+att_loss = 418.03507678324087
+global_step = 46499
+loss = 53.81990357788759
+rep_loss = 12.524151824442704
+att_loss = 417.23997283574636
+global_step = 46749
+loss = 53.719232056093155
+rep_loss = 12.513883584610413
+att_loss = 416.4557218942232
+global_step = 46999
+loss = 53.61996301248749
+rep_loss = 12.503982180938998
+att_loss = 415.64490532235556
+global_step = 47249
+loss = 53.51731615513282
+rep_loss = 12.49362389865112
+att_loss = 414.8313552065656
+global_step = 47499
+loss = 53.41432624916325
+rep_loss = 12.483254770265658
+att_loss = 413.96981992971246
+global_step = 47749
+loss = 53.30523171779062
+rep_loss = 12.472033788222666
+att_loss = 413.15383253052545
+global_step = 47999
+loss = 53.20193024074479
+rep_loss = 12.461609376792241
+att_loss = 412.3265501465417
+global_step = 48249
+loss = 53.097181754735686
+rep_loss = 12.450903867677932
+att_loss = 411.4937257261404
+global_step = 48499
+loss = 52.99173086048301
+rep_loss = 12.440121136711934
+att_loss = 410.6774601531779
+global_step = 48749
+loss = 52.88838179087995
+rep_loss = 12.429594153153822
+att_loss = 409.8448136892251
+global_step = 48999
+loss = 52.782971961618074
+rep_loss = 12.418961979129737
+att_loss = 409.0028938114637
+global_step = 49249
+loss = 52.67639218699631
+rep_loss = 12.408243665116233
+att_loss = 408.18548318029866
+global_step = 49499
+loss = 52.57292907776794
+rep_loss = 12.397949419292052
+att_loss = 407.36597105840445
+global_step = 49749
+loss = 52.469220969745706
+rep_loss = 12.387796673209785
+att_loss = 406.5366784533664
+global_step = 49999
+loss = 52.36429095096533
+rep_loss = 12.377649139604202
+att_loss = 405.7202889275348
+global_step = 50249
+loss = 52.261031498737715
+rep_loss = 12.367963048278844
+att_loss = 404.90346718124215
+global_step = 50499
+loss = 52.15774034676183
+rep_loss = 12.358455586748716
+att_loss = 404.07259098323385
+global_step = 50749
+loss = 52.05269508167597
+rep_loss = 12.348969661098565
+att_loss = 403.22920092030535
+global_step = 50999
+loss = 51.94607408712973
+rep_loss = 12.339391768106127
+att_loss = 402.3802402069217
+global_step = 51249
+loss = 51.838757622093226
+rep_loss = 12.32982075661633
+att_loss = 401.5426612154961
+global_step = 51499
+loss = 51.732905490317634
+rep_loss = 12.320582694186324
+att_loss = 400.70377284389497
+global_step = 51749
+loss = 51.62691244608757
+rep_loss = 12.311526711448368
+att_loss = 399.8625147925249
+global_step = 51999
+loss = 51.520630164608946
+rep_loss = 12.302526512116689
+att_loss = 399.0078826006999
+global_step = 52249
+loss = 51.41267432677176
+rep_loss = 12.293511999767112
+att_loss = 398.1749379127149
+global_step = 52499
+loss = 51.307466360049794
+rep_loss = 12.284792951091228
+att_loss = 397.34106225898586
+global_step = 52749
+loss = 51.20213805727196
+rep_loss = 12.276042182542554
+att_loss = 396.5201868602387
+global_step = 52999
+loss = 51.09846154694909
+rep_loss = 12.267505496067715
+att_loss = 395.696838918234
+global_step = 53249
+loss = 50.99445791232878
+rep_loss = 12.258824362917975
+att_loss = 394.87037899546084
+global_step = 53499
+loss = 50.89005832794428
+rep_loss = 12.250087614879657
+att_loss = 394.0527554703472
+global_step = 53749
+loss = 50.78677591598312
+rep_loss = 12.241451844243857
+att_loss = 393.24143282637345
+global_step = 53999
+loss = 50.68428991926787
+rep_loss = 12.232886519022125
+att_loss = 392.43965526004047
+global_step = 54249
+loss = 50.58300504671044
+rep_loss = 12.224385103685663
+att_loss = 391.64264881379376
+global_step = 54499
+loss = 50.48231238881279
+rep_loss = 12.21585028627942
+att_loss = 390.8601956633998
+global_step = 54749
+loss = 50.38346429668199
+rep_loss = 12.207518704250512
+att_loss = 390.0729250104103
+global_step = 54999
+loss = 50.284001766218445
+rep_loss = 12.199089112598456
+att_loss = 389.31133648670345
+global_step = 55249
+loss = 50.187795636615164
+rep_loss = 12.191028596376597
+att_loss = 388.5525984823884
+global_step = 55499
+loss = 50.091947100616196
+rep_loss = 12.182978309722705
+att_loss = 387.80436541667115
+global_step = 55749
+loss = 49.99741998727243
+rep_loss = 12.174994470516149
+att_loss = 387.0552613080837
+global_step = 55999
+loss = 49.9027750983903
+rep_loss = 12.166939473817807
+att_loss = 386.3239343684958
+global_step = 56249
+loss = 49.810371762527836
+rep_loss = 12.15903972790589
+att_loss = 385.59254376769417
+global_step = 56499
+loss = 49.717960871643825
+rep_loss = 12.1511432036396
+att_loss = 384.88300487745124
+global_step = 56749
+loss = 49.628306122153155
+rep_loss = 12.143444097600504
+att_loss = 384.18496186893213
+global_step = 56999
+loss = 49.54010241677591
+rep_loss = 12.135857463374899
+att_loss = 383.48891160166664
+global_step = 57249
+loss = 49.4521423345372
+rep_loss = 12.128227071913313
+att_loss = 382.8087591240781
+global_step = 57499
+loss = 49.366188662171815
+rep_loss = 12.120750168099738
+att_loss = 382.1431455683032
+global_step = 57749
+loss = 49.282078285950604
+rep_loss = 12.113480712152032
+att_loss = 381.4765402969557
+global_step = 57999
+loss = 49.19783066547572
+rep_loss = 12.10610501573975
+att_loss = 380.8198167156326
+global_step = 58249
+loss = 49.11483049123329
+rep_loss = 12.098827202294546
+att_loss = 380.16741775930916
+global_step = 58499
+loss = 49.03236370336133
+rep_loss = 12.09149185478529
+att_loss = 379.52598527098235
+global_step = 58749
+loss = 48.95128226382001
+rep_loss = 12.084272827448947
+att_loss = 328.35527935543575
+global_step = 58999
+loss = 42.482683503949964
+rep_loss = 11.506188331423578
+att_loss = 328.64847870404714
+global_step = 59249
+loss = 42.51917423520769
+rep_loss = 11.504915223304403
+att_loss = 328.2227134136292
+global_step = 59499
+loss = 42.4650246044777
+rep_loss = 11.497483390669583
+att_loss = 328.1589748620078
+global_step = 59749
+loss = 42.45698851884699
+rep_loss = 11.496933261957375
+att_loss = 327.96913344880767
+global_step = 59999
+loss = 42.43277933328568
+rep_loss = 11.493101262425503
+att_loss = 327.7072755121499
+global_step = 60249
+loss = 42.39948780108721
+rep_loss = 11.488626917388519
+att_loss = 327.45508357775856
+global_step = 60499
+loss = 42.367388150718355
+rep_loss = 11.484021703919266
+att_loss = 327.2823583969005
+global_step = 60749
+loss = 42.34552766013292
+rep_loss = 11.48186293446157
+att_loss = 327.03517870746174
+global_step = 60999
+loss = 42.314138652415735
+rep_loss = 11.477930534863273
+att_loss = 326.77379158797305
+global_step = 61249
+loss = 42.28088593493396
+rep_loss = 11.47329593538769
+att_loss = 326.5938454012632
+global_step = 61499
+loss = 42.257999699464406
+rep_loss = 11.470152240594542
+att_loss = 326.35747106023103
+global_step = 61749
+loss = 42.227962024245244
+rep_loss = 11.466225154518781
+att_loss = 326.12352478296145
+global_step = 61999
+loss = 42.198229037345456
+rep_loss = 11.462307569460135
+att_loss = 325.93727647812125
+global_step = 62249
+loss = 42.17457759942477
+rep_loss = 11.459344349590738
+att_loss = 325.7074230125067
+global_step = 62499
+loss = 42.14533984246549
+rep_loss = 11.455295764661827
+att_loss = 325.46969798551186
+global_step = 62749
+loss = 42.11513811697707
+rep_loss = 11.451406990408362
+att_loss = 325.26252719178956
+global_step = 62999
+loss = 42.08880578246127
+rep_loss = 11.44791910478258
+att_loss = 325.01166694948995
+global_step = 63249
+loss = 42.056943388399
+rep_loss = 11.443880189040726
+att_loss = 324.77472143948853
+global_step = 63499
+loss = 42.0268650872607
+rep_loss = 11.440199283558234
+att_loss = 324.58982729747447
+global_step = 63749
+loss = 42.00338519389031
+rep_loss = 11.437254277156194
+att_loss = 324.36273489704627
+global_step = 63999
+loss = 41.97451099148103
+rep_loss = 11.433353066113062

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18a90816d852d1829a6e33999e40c1909d7ab8fb81a9d379af3c107ce0c72f97
+size 58912319

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "special_tokens_map_file": null, "tokenizer_file": null}