add evaluation

.gitignore

@@ -0,0 +1,171 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.conll
+*.pt
+*.onnx
+# C extensions
+*.so
+*.csv
+*.json
+*.joblib
+*.ipynb
+*.pkl
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+env/
+venv
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+split.py
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# Logs
+*.log.*
+*/logs
+*/var/run
+*/*/*/*/run/*
+*/*/*/*/logs/*
+
+# OS generated files
+.DS_Store*
+ehthumbs.db
+Icon?
+Thumbs.db
+
+# Editor Files
+*~
+*.swp
+cli/meta
+# IDE Files
+.vscode/
+
+# model file
+*.pkl
+catboost_info/

app.py

@@ -0,0 +1,10 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+# Define the path to your custom metric directory
+metric_path = "./custom_metric"
+
+
+module = evaluate.load(metric_path)
+launch_gradio_widget(module)
+

custom_metric/custom_metric.py

@@ -0,0 +1,203 @@
+import evaluate
+import datasets
+import numpy as np
+
+_CITATION = """\
+@InProceedings{huggingface:module,
+title = {A great new module},
+authors={huggingface, Inc.},
+year={2020}
+}
+"""
+
+# TODO: Add description of the module here
+_DESCRIPTION = """\
+This new module is designed to solve this great ML task and is crafted with a lot of care.
+"""
+
+
+# TODO: Add description of the arguments of the module here
+_KWARGS_DESCRIPTION = """
+Calculates how good are predictions given some references, using certain scores
+Args:
+    predictions: list of predictions to score. Each predictions
+        should be a string with tokens separated by spaces.
+    references: list of reference for each prediction. Each
+        reference should be a string with tokens separated by spaces.
+Returns:
+    accuracy: description of the first score,
+    another_score: description of the second score,
+Examples:
+    Examples should be written in doctest format, and should illustrate how
+    to use the function.
+    >>> my_new_module = evaluate.load("my_new_module")
+    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
+    >>> print(results)
+    {'accuracy': 1.0}
+"""
+
+# TODO: Define external resources urls if needed
+BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
+
+
+def convert_format(data:list):
+    """
+    Args:
+        data (list) : list of dictionaries with different entity elements
+    e.g
+        [
+        {'head': ['phipigments', 'tinadaviespigments'...], 
+        'head_type': ['product', 'brand'...], 
+        'type': ['sell', 'sell'...], 
+        'tail': ['國際認證之色乳', '國際認證之色乳'...], 
+        'tail_type': ['product', 'product'...]}, 
+
+        {'head': ['SABONTAIWAN', 'SNTAIWAN'...], 
+        'head_type': ['brand', 'brand'...], 
+        'type': ['sell', 'sell'...], 
+        'tail': ['大馬士革玫瑰有機光燦系列', '大馬士革玫瑰有機光燦系列'...], 
+        'tail_type': ['product', 'product'...]}
+        ...
+        ]
+    """
+    predictions = []
+    for item in data:
+        prediction_group = []
+        for i in range(len(item['head'])):
+            prediction = {
+                'head': item['head'][i],
+                'head_type': item['head_type'][i],
+                'type': item['type'][i],
+                'tail': item['tail'][i],
+                'tail_type': item['tail_type'][i],
+            }
+            prediction_group.append(prediction)
+        predictions.append(prediction_group)
+    return predictions
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class relation_extraction(evaluate.Metric):
+    """TODO: Short description of my evaluation module."""
+
+    def _info(self):
+        # TODO: Specifies the evaluate.EvaluationModuleInfo object
+        return evaluate.MetricInfo(
+            # This is the description that will appear on the modules page.
+            module_type="metric",
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            # This defines the format of each prediction and reference
+            features=datasets.Features({
+                'predictions': datasets.Sequence({
+                    "head": datasets.Value("string"),
+                    "head_type": datasets.Value("string"),
+                    "type": datasets.Value("string"),
+                    "tail": datasets.Value("string"),
+                    "tail_type": datasets.Value("string"),
+                }),
+                'references': datasets.Sequence({
+                    "head": datasets.Value("string"),
+                    "head_type": datasets.Value("string"),
+                    "type": datasets.Value("string"),
+                    "tail": datasets.Value("string"),
+                    "tail_type": datasets.Value("string"),
+                }),
+            }),
+            # Homepage of the module for documentation
+            homepage="http://module.homepage",
+            # Additional links to the codebase or references
+            codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
+            reference_urls=["http://path.to.reference.url/new_module"]
+        )
+
+    def _download_and_prepare(self, dl_manager):
+        """Optional: download external resources useful to compute the scores"""
+        # TODO: Download external resources if needed
+        pass
+
+    def _compute(self, predictions, references, mode="strict", relation_types=[]):
+        """Returns the scores"""
+        # TODO: Compute the different scores of the module
+        print(predictions)
+        predictions = convert_format(predictions)
+        references = convert_format(references)
+        print(predictions)
+        assert mode in ["strict", "boundaries"]
+
+        # construct relation_types from ground truth if not given
+        if len(relation_types) == 0:
+            for triplets in references:
+                for triplet in triplets:
+                    relation = triplet["type"]
+                    if relation not in relation_types:
+                        relation_types.append(relation)
+        
+        scores = {rel: {"tp": 0, "fp": 0, "fn": 0} for rel in relation_types + ["ALL"]}
+        
+        # Count GT relations and Predicted relations
+        n_sents = len(references)
+        n_rels = sum([len([rel for rel in sent]) for sent in references])
+        n_found = sum([len([rel for rel in sent]) for sent in predictions])
+
+        # Count TP, FP and FN per type
+        for pred_sent, gt_sent in zip(predictions, references):
+            for rel_type in relation_types:
+                # strict mode takes argument types into account
+                if mode == "strict":
+                    pred_rels = {(rel["head"], rel["head_type"], rel["tail"], rel["tail_type"]) for rel in pred_sent if
+                                rel["type"] == rel_type}
+                    gt_rels = {(rel["head"], rel["head_type"], rel["tail"], rel["tail_type"]) for rel in gt_sent if
+                            rel["type"] == rel_type}
+
+                # boundaries mode only takes argument spans into account
+                elif mode == "boundaries":
+                    pred_rels = {(rel["head"], rel["tail"]) for rel in pred_sent if rel["type"] == rel_type}
+                    gt_rels = {(rel["head"], rel["tail"]) for rel in gt_sent if rel["type"] == rel_type}
+
+                scores[rel_type]["tp"] += len(pred_rels & gt_rels)
+                scores[rel_type]["fp"] += len(pred_rels - gt_rels)
+                scores[rel_type]["fn"] += len(gt_rels - pred_rels)
+
+        # Compute per entity Precision / Recall / F1
+        for rel_type in scores.keys():
+            if scores[rel_type]["tp"]:
+                scores[rel_type]["p"] = 100 * scores[rel_type]["tp"] / (scores[rel_type]["fp"] + scores[rel_type]["tp"])
+                scores[rel_type]["r"] = 100 * scores[rel_type]["tp"] / (scores[rel_type]["fn"] + scores[rel_type]["tp"])
+            else:
+                scores[rel_type]["p"], scores[rel_type]["r"] = 0, 0
+
+            if not scores[rel_type]["p"] + scores[rel_type]["r"] == 0:
+                scores[rel_type]["f1"] = 2 * scores[rel_type]["p"] * scores[rel_type]["r"] / (
+                        scores[rel_type]["p"] + scores[rel_type]["r"])
+            else:
+                scores[rel_type]["f1"] = 0
+
+        # Compute micro F1 Scores
+        tp = sum([scores[rel_type]["tp"] for rel_type in relation_types])
+        fp = sum([scores[rel_type]["fp"] for rel_type in relation_types])
+        fn = sum([scores[rel_type]["fn"] for rel_type in relation_types])
+        
+
+        if tp:
+            precision = 100 * tp / (tp + fp)
+            recall = 100 * tp / (tp + fn)
+            f1 = 2 * precision * recall / (precision + recall)
+
+        else:
+            precision, recall, f1 = 0, 0, 0
+
+        scores["ALL"]["p"] = precision
+        scores["ALL"]["r"] = recall
+        scores["ALL"]["f1"] = f1
+        scores["ALL"]["tp"] = tp
+        scores["ALL"]["fp"] = fp
+        scores["ALL"]["fn"] = fn
+
+
+        # Compute Macro F1 Scores
+        scores["ALL"]["Macro_f1"] = np.mean([scores[ent_type]["f1"] for ent_type in relation_types])
+        scores["ALL"]["Macro_p"] = np.mean([scores[ent_type]["p"] for ent_type in relation_types])
+        scores["ALL"]["Macro_r"] = np.mean([scores[ent_type]["r"] for ent_type in relation_types])
+
+        return scores

custom_metric/metric.yml

@@ -0,0 +1,10 @@
+metric_name: custom_relation_extraction
+description: Custom Relation Extraction Metric
+inputs:
+  - name: predictions
+    type: list
+    required: true
+  - name: references
+    type: list
+    required: true
+compute_function: custom_metric.relation_extraction.compute