add

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,163 @@
+# .gitignore specific to this project
+# ===================================
+source/services/predicting_effective_arguments/model/*
+dist
+build
+comlib.egg-info
+
+# tex files
+.log
+.synctex.gz
+.toc
+.aux
+.out
+.idx
+.bbl
+.blg
+
+# env file
+.env
+
+# data
+sensitive/*
+.cache/*
+router/cache/*
+database.json
+data/*
+log/*
+!log/.empty
+uploads/*
+
+# models
+model/*
+
+# ide's config
+.idea/*
+.ropeproject
+
+# python generated files
+__pycache__
+.mypy_cache
+*.egg-info
+*.pyc
+dependencies/python-pdfbox/build
+dependencies/python-pdfbox/dist
+
+# log files
+corpus.log
+
+# local config files
+
+
+# Default .gitignore file
+# =======================
+
+# Mac OS X
+.DS_Store
+
+# Windows image file caches
+Thumbs.db
+ehthumbs.db
+
+# Folder config file
+Desktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# Vagrant
+.vagrant/
+
+# IntelliJ
+.idea/
+*.iml
+*.iws
+
+# Eclipse
+.classpath
+.project
+.settings/
+
+# Maven
+log/
+target/
+
+# Gradle
+.gradle/
+build/
+
+# SASS
+**/.sass-cache
+**/.sass-cache/*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+# VSCode
+.vscode
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+data_exploration/data/anonymization.xlsx
+data_exploration/data/~$anonymization.xlsx
+data_exploration/data/query4_results.pdf
+data_exploration/data/query5_results.pdf
+data_exploration/src/data_versioning_problem_illustrate.pptx
+data_exploration/src/

source/services/predicting_effective_arguments/train/seq_classification.py

@@ -56,10 +56,10 @@ if __name__ == '__main__':
     test_size = 0.1
 
     # First split: Separate out the training set
-    train_df, temp_df = train_test_split(data, test_size=1 - train_size)
+    train_df, temp_df = train_test_split(data, test_size=1 - train_size, random_state=5600)
 
     # Second split: Separate out the validation and test sets
-    valid_df, test_df = train_test_split(temp_df, test_size=test_size / (test_size + valid_size))
+    valid_df, test_df = train_test_split(temp_df, test_size=test_size / (test_size + valid_size), random_state=5600)
 
 
     train_df = prepare_input_text(train_df, sep_token=tokenizer.sep_token)
@@ -69,14 +69,15 @@ if __name__ == '__main__':
     train_dataset = Dataset.from_pandas(train_df[['inputs', config.TARGET]]).rename_column(config.TARGET, 'label').class_encode_column("label")
     val_dataset = Dataset.from_pandas(valid_df[['inputs', config.TARGET]]).rename_column(config.TARGET, 'label').class_encode_column("label")
     test_dataset = Dataset.from_pandas(test_df[['inputs', config.TARGET]]).rename_column(config.TARGET, 'label').class_encode_column("label")
-
+    labels = train_dataset.features["label"].names
     train_tok_dataset = seqClassifer.tokenize_dataset(dataset=train_dataset)
     val_tok_dataset = seqClassifer.tokenize_dataset(dataset=val_dataset)
     test_tok_dataset = seqClassifer.tokenize_dataset(dataset=test_dataset)
 
     seqClassifer.train(train_dataset=train_tok_dataset, eval_dataset=val_tok_dataset, epochs=1, batch_size=16)
-    y_pred = seqClassifer.predict_valid_data(val_tok_dataset)
-    seqClassifer.predict_test_data(model_checkpoint=config.MODEL_OUTPUT_DIR, test_data=test_df['inputs'].tolist())
+    y_valid_pred = seqClassifer.predict_valid_data(val_tok_dataset)
+    seqClassifer.plot_confusion_matrix(y_preds=y_valid_pred, y_true=val_dataset['label'], labels=labels)
+    y_test_pred = seqClassifer.predict_test_data(model_checkpoint=config.MODEL_OUTPUT_DIR, test_list=test_df['inputs'].tolist())
     pass
 
     """
@@ -94,7 +95,4 @@ if __name__ == '__main__':
     plt.suptitle("")
     plt.xlabel("")
     plt.show()
-    """
-
-
-    pass
+    """