Advince/distilbert-base-uncased-lora-toxic-classification

README.md

@@ -1,15 +1,15 @@
----
-base_model: distilbert-base-uncased
-library_name: peft
-license: apache-2.0
-metrics:
-- accuracy
-tags:
-- generated_from_trainer
-model-index:
-- name: distilbert-base-uncased-lora-text-classification
-  results: []
----
+---
+base_model: distilbert-base-uncased
+library_name: peft
+license: apache-2.0
+metrics:
+- accuracy
+tags:
+- generated_from_trainer
+model-index:
+- name: distilbert-base-uncased-lora-text-classification
+  results: []
+---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
@@ -18,8 +18,8 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [distilbert-base-uncased](https://huggingface.co/distilbert-base-uncased) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.3115
-- Accuracy: {'accuracy': 0.8125}
+- Loss: 0.5043
+- Accuracy: {'accuracy': 0.9479166666666666}
 
 ## Model description
 
@@ -50,27 +50,27 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Accuracy                         |
 |:-------------:|:-----:|:----:|:---------------:|:--------------------------------:|
-| No log        | 1.0   | 431  | 0.6279          | {'accuracy': 0.7135416666666666} |
-| 0.7248        | 2.0   | 862  | 0.5949          | {'accuracy': 0.7604166666666666} |
-| 0.5433        | 3.0   | 1293 | 0.5986          | {'accuracy': 0.8072916666666666} |
-| 0.4762        | 4.0   | 1724 | 0.6967          | {'accuracy': 0.8125}             |
-| 0.3971        | 5.0   | 2155 | 0.7136          | {'accuracy': 0.8229166666666666} |
-| 0.3471        | 6.0   | 2586 | 0.8597          | {'accuracy': 0.8177083333333334} |
-| 0.2695        | 7.0   | 3017 | 1.0061          | {'accuracy': 0.8072916666666666} |
-| 0.2695        | 8.0   | 3448 | 0.7674          | {'accuracy': 0.8333333333333334} |
-| 0.2417        | 9.0   | 3879 | 1.2479          | {'accuracy': 0.828125}           |
-| 0.2079        | 10.0  | 4310 | 1.0548          | {'accuracy': 0.8177083333333334} |
-| 0.1941        | 11.0  | 4741 | 1.0516          | {'accuracy': 0.8229166666666666} |
-| 0.1711        | 12.0  | 5172 | 1.2246          | {'accuracy': 0.828125}           |
-| 0.1253        | 13.0  | 5603 | 1.2416          | {'accuracy': 0.8177083333333334} |
-| 0.0918        | 14.0  | 6034 | 1.3199          | {'accuracy': 0.8229166666666666} |
-| 0.0918        | 15.0  | 6465 | 1.3115          | {'accuracy': 0.8125}             |
+| No log        | 1.0   | 431  | 0.2331          | {'accuracy': 0.90625}            |
+| 0.4451        | 2.0   | 862  | 0.3140          | {'accuracy': 0.90625}            |
+| 0.2963        | 3.0   | 1293 | 0.3216          | {'accuracy': 0.9322916666666666} |
+| 0.25          | 4.0   | 1724 | 0.2690          | {'accuracy': 0.9270833333333334} |
+| 0.2261        | 5.0   | 2155 | 0.2707          | {'accuracy': 0.9479166666666666} |
+| 0.1511        | 6.0   | 2586 | 0.2543          | {'accuracy': 0.9427083333333334} |
+| 0.1401        | 7.0   | 3017 | 0.3120          | {'accuracy': 0.9375}             |
+| 0.1401        | 8.0   | 3448 | 0.2845          | {'accuracy': 0.953125}           |
+| 0.086         | 9.0   | 3879 | 0.4018          | {'accuracy': 0.921875}           |
+| 0.0583        | 10.0  | 4310 | 0.4593          | {'accuracy': 0.9427083333333334} |
+| 0.0475        | 11.0  | 4741 | 0.4401          | {'accuracy': 0.953125}           |
+| 0.0515        | 12.0  | 5172 | 0.4631          | {'accuracy': 0.9479166666666666} |
+| 0.0291        | 13.0  | 5603 | 0.4593          | {'accuracy': 0.9479166666666666} |
+| 0.0319        | 14.0  | 6034 | 0.5292          | {'accuracy': 0.9479166666666666} |
+| 0.0319        | 15.0  | 6465 | 0.5043          | {'accuracy': 0.9479166666666666} |
 
 
 ### Framework versions
 
-- PEFT 0.13.0
+- PEFT 0.12.0
 - Transformers 4.44.2
-- Pytorch 2.4.1+cu121
+- Pytorch 2.4.1+cu124
 - Datasets 3.0.0
 - Tokenizers 0.19.1

adapter_config.json

@@ -1,31 +1,31 @@
-{
-  "alpha_pattern": {},
-  "auto_mapping": null,
-  "base_model_name_or_path": "distilbert-base-uncased",
-  "bias": "none",
-  "fan_in_fan_out": false,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layer_replication": null,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "loftq_config": {},
-  "lora_alpha": 32,
-  "lora_dropout": 0.01,
-  "megatron_config": null,
-  "megatron_core": "megatron.core",
-  "modules_to_save": [
-    "classifier",
-    "score"
-  ],
-  "peft_type": "LORA",
-  "r": 4,
-  "rank_pattern": {},
-  "revision": null,
-  "target_modules": [
-    "q_lin"
-  ],
-  "task_type": "SEQ_CLS",
-  "use_dora": false,
-  "use_rslora": false
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "distilbert-base-uncased",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.01,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier",
+    "score"
+  ],
+  "peft_type": "LORA",
+  "r": 4,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_lin"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70c55ad8ee709d5ca29ddaaa939b5c82162f72af76aa36960605b9316e294432
-size 2521180
+oid sha256:c7889b99657a6e509ad9627cd74ec812561083b7486f8a47d3361f190a4ad6db
+size 2518104

runs/Sep21_15-24-52_LAPTOP-9511T4P2/events.out.tfevents.1726925094.LAPTOP-9511T4P2.31280.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6f7372a406c70e8453a3b676d89538b19cbd0f17ce3bc318c19d2cbe4458ac4
+size 5008

runs/Sep21_15-28-50_LAPTOP-9511T4P2/events.out.tfevents.1726925331.LAPTOP-9511T4P2.5348.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5477a3950e83a6c1f34ab56caa4cd048e460ab15cc371105bbdd329652dcd08a
+size 6032

runs/Sep23_16-42-58_LAPTOP-9511T4P2/events.out.tfevents.1727102580.LAPTOP-9511T4P2.15204.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46228cee720331ca19332d3484b96622e60e83211a82800e396c0b47246e53c5
+size 5004

special_tokens_map.json

@@ -1,7 +1,7 @@
-{
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
-}
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -1,56 +1,56 @@
-{
-  "add_prefix_space": true,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "[PAD]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100": {
-      "content": "[UNK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "101": {
-      "content": "[CLS]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "102": {
-      "content": "[SEP]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "103": {
-      "content": "[MASK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "clean_up_tokenization_spaces": true,
-  "cls_token": "[CLS]",
-  "do_lower_case": true,
-  "mask_token": "[MASK]",
-  "model_max_length": 512,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "strip_accents": null,
-  "tokenize_chinese_chars": true,
-  "tokenizer_class": "DistilBertTokenizer",
-  "unk_token": "[UNK]"
-}
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "DistilBertTokenizer",
+  "unk_token": "[UNK]"
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66153193613f4463045ef8aba5885b66b724bc83111c9867a7656c6db070c4a2
+oid sha256:bffcb5a4cd3f18e69158eb5d87cf3cb5008efd9551c2f305ebe8600e39e3fad1
 size 5240