add cuc

README.md

@@ -28,30 +28,47 @@ At minium, the metric requires the original time-series and the generated time-s
 >>> metric = evaluate.load("bowdbeg/matching_series")
 >>> results = metric.compute(references=references, predictions=predictions, batch_size=1000)
 >>> print(results)
-{'matching_mse': 0.15873331613053895, 'harmonic_mean': 0.15623569099681772, 'covered_mse': 0.15381544718035087, 'index_mse': 0.16636189201532087, 'matching_mse_features': [0.13739837269222452, 0.1395309409295018, 0.13677679887355126, 0.14408421162706211, 0.1430115910456261, 0.13726657544044085, 0.14274372684301717, 0.13504614539190338, 0.13853582796877975, 0.14482307626368343], 'harmonic_mean_features': [0.1309991815519093, 0.13157175020534279, 0.12735134531950718, 0.1327483317911355, 0.1336402851605765, 0.12878380179856022, 0.1344831997941457, 0.12782689483798823, 0.12909420446395195, 0.13417435670997752], 'covered_mse_features': [0.12516953618356524, 0.12447158260731798, 0.11914118322950448, 0.12306606276504639, 0.1254216201001874, 0.12128844181049621, 0.12712643943219143, 0.12134032531607968, 0.12085741660832867, 0.12498436126166071], 'index_mse_features': [0.16968036010688156, 0.1624888691672768, 0.15926142198600082, 0.17250634507748022, 0.16713668302081525, 0.16663213728264645, 0.1596766027744231, 0.16251306560725656, 0.17160303243460656, 0.17212040269582168], 'macro_matching_mse': 0.13992172670757905, 'macro_covered_mse': 0.12328669693143782, 'macro_harmonic_mean': 0.13106733516330948, 'macro_index_mse': 0.1663618920153209}
+{'precision_mse': 0.15642462680824154, 'f1_mse': 0.15423970232736145, 'recall_mse': 0.15211497466247828, 'index_mse': 0.1650527529752939, 'precision_mse_features': [0.14161461272391063, 0.13959801451122986, 0.13494790079336152, 0.13812467072775822, 0.13502155933085397, 0.13773603530687478, 0.13782869677371534, 0.13880373566781345, 0.1347356979110729, 0.1380613227954152], 'f1_mse_features': [0.13200523240237663, 0.1321561699583367, 0.12686344486378406, 0.12979789457435542, 0.12768556637792927, 0.1316950291866994, 0.12937893459231917, 0.13052145628415104, 0.12571029554640592, 0.12686388502130683], 'recall_mse_features': [0.12361708937664843, 0.1254676048318782, 0.11969288602958734, 0.12241798787954035, 0.12110565263179066, 0.12616166677071738, 0.12190537193383513, 0.1231719120998892, 0.1178181328089802, 0.11734651764610313], 'index_mse_features': [0.16728853331521837, 0.1673468681819004, 0.16940025907048203, 0.16828093040638223, 0.17486439883284577, 0.15779474562305962, 0.16255301663470148, 0.16224400164732194, 0.1531092505944622, 0.167645525446565], 'macro_precision_mse': 0.1376472246542006, 'macro_recall_mse': 0.121870482200897, 'macro_f1_mse': 0.12926779088076645, 'macro_index_mse': 0.1650527529752939, 'matching_precision': 0.09, 'matching_recall': 1.0, 'matching_f1': 0.1651376146788991, 'matching_precision_features': [0.1, 0.1, 0.1, 0.1, 0.09, 0.09, 0.1, 0.1, 0.1, 0.1], 'matching_recall_features': [1.0, 1.0, 1.0, 0.7, 0.9, 1.0, 0.9, 1.0, 0.9, 0.8], 'matching_f1_features': [0.18181818181818182, 0.18181818181818182, 0.18181818181818182, 0.175, 0.16363636363636364, 0.1651376146788991, 0.18, 0.18181818181818182, 0.18, 0.17777777777777778], 'macro_matching_precision': 0.098, 'macro_matching_recall': 0.92, 'macro_matching_f1': 0.1768824483365768, 'cuc': 0.1364, 'coverages': [0.10000000000000002, 0.16666666666666666, 0.3, 0.5333333333333333, 0.9], 'macro_cuc': 0.13874, 'macro_coverages': [0.10000000000000002, 0.18000000000000002, 0.31, 0.48, 0.98], 'cuc_features': [0.1428, 0.13580000000000003, 0.15250000000000002, 0.14579999999999999, 0.12990000000000002, 0.1364, 0.1459, 0.12330000000000002, 0.13580000000000003, 0.13920000000000002], 'coverages_features': [[0.10000000000000002, 0.16666666666666666, 0.3666666666666667, 0.5, 1.0], [0.10000000000000002, 0.16666666666666666, 0.26666666666666666, 0.43333333333333335, 1.0], [0.10000000000000002, 0.20000000000000004, 0.3666666666666667, 0.6, 1.0], [0.10000000000000002, 0.16666666666666666, 0.3333333333333333, 0.5333333333333333, 1.0], [0.10000000000000002, 0.20000000000000004, 0.26666666666666666, 0.4666666666666666, 0.9], [0.10000000000000002, 0.16666666666666666, 0.30000000000000004, 0.5333333333333333, 0.9], [0.10000000000000002, 0.20000000000000004, 0.3333333333333333, 0.5333333333333333, 1.0], [0.10000000000000002, 0.20000000000000004, 0.3, 0.3, 1.0], [0.10000000000000002, 0.16666666666666666, 0.26666666666666666, 0.4333333333333333, 1.0], [0.10000000000000002, 0.16666666666666666, 0.30000000000000004, 0.4666666666666666, 1.0]]}
 ```
 
 ### Inputs
 - **predictions**: (list of list of list of float or numpy.ndarray): The generated time-series. The shape of the array should be `(num_generation, seq_len, num_features)`.
 - **references**: (list of list of list of float or numpy.ndarray): The original time-series. The shape of the array should be `(num_reference, seq_len, num_features)`.
 - **batch_size**: (int, optional): The batch size for computing the metric. This affects quadratically. Default is None.
+- **cuc_n_calculation**: (int, optional): The number of samples to compute the coverage because sampling exists. Default is 3.
+- **cuc_n_samples**: (list of int, optional): The number of samples to compute the coverage. Default is $[2^i \text{for} i \leq \log_2 n] + [n]$.
 
 ### Output Values
 
 Let prediction instances be $P = \{p_1, p_2, \ldots, p_n\}$ and reference instances be $R = \{r_1, r_2, \ldots, r_m\}$.
 
-- **matching_mse**: (float): Average of the MSE between the generated instance and the reference instance with the lowest MSE. Intuitively, This is similar to precision in classification. In the equation, $\frac{1}{n} \sum_{i=1}^{n} \min_{j} \mathrm{MSE}(p_i, r_j)$.
-- **covered_mse**: (float): Average of the MSE between the reference instance and the  with the lowest MSE. Intuitively, This is similar to recall in classification. In the equation, $\frac{1}{m} \sum_{j=1}^{m} \min_{i} \mathrm{MSE}(p_i, r_j)$.
-- **harmonic_mean**: (float): Harmonic mean of the matching_mse and covered_mse. This is similar to F1-score in classification.
+- **precision_mse**: (float): Average of the MSE between the generated instance and the reference instance with the lowest MSE. Intuitively, this is similar to precision in classification. In the equation, $\frac{1}{n} \sum_{i=1}^{n} \min_{j} \mathrm{MSE}(p_i, r_j)$.
+- **recall_mse**: (float): Average of the MSE between the reference instance and the  with the lowest MSE. Intuitively, this is similar to recall in classification. In the equation, $\frac{1}{m} \sum_{j=1}^{m} \min_{i} \mathrm{MSE}(p_i, r_j)$.
+- **f1_mse**: (float): Harmonic mean of the precision_mse and recall_mse. This is similar to F1-score in classification.
 - **index_mse**: (float): Average of the MSE between the generated instance and the reference instance with the same index. In the equation, $\frac{1}{n} \sum_{i=1}^{n} \mathrm{MSE}(p_i, r_i)$.
-- **matching_mse_features**: (list of float): matching_mse computed individually for each feature.
-- **covered_mse_features**: (list of float): covered_mse computed individually for each feature.
-- **harmonic_mean_features**: (list of float): harmonic_mean computed individually for each feature.
+- **precision_mse_features**: (list of float): precision_mse computed individually for each feature.
+- **recall_mse_features**: (list of float): recall_mse computed individually for each feature.
+- **f1_mse_features**: (list of float): f1_mse computed individually for each feature.
 - **index_mse_features**: (list of float): index_mse computed individually for each feature.
-- **macro_matching_mse**: (float): Average of the matching_mse_features.
-- **macro_covered_mse**: (float): Average of the covered_mse_features.
-- **macro_harmonic_mean**: (float): Average of the harmonic_mean_features.
+- **macro_precision_mse**: (float): Average of the precision_mse_features.
+- **macro_recall_mse**: (float): Average of the recall_mse_features.
+- **macro_f1_mse**: (float): Average of the f1_mse_features.
 - **macro_index_mse**: (float): Average of the index_mse_features.
+- **matching_precision**: (float): Precision of the matching instances. In the equation, $\frac{ | \{i | \min_{i} \mathrm{MSE}(p_i, r_j)\} | }{m}$.
+- **matching_recall**: (float): Recall of the matching instances. In the equation, $\frac{ | \{j | \min_{j} \mathrm{MSE}(p_i, r_j)\} | }{n}$.
+- **matching_f1**: (float): F1-score of the matching instances.
+- **matching_precision_features**: (list of float): matching_precision computed individually for each feature.
+- **matching_recall_features**: (list of float): matching_recall computed individually for each feature.
+- **matching_f1_features**: (list of float): matching_f1 computed individually for each feature.
+- **macro_matching_precision**: (float): Average of the matching_precision_features.
+- **macro_matching_recall**: (float): Average of the matching_recall_features.
+- **macro_matching_f1**: (float): Average of the matching_f1_features.
+- **coverages**: (list of float): Coverage of the matching instances computed on the sampled generated data in cuc_n_samples. In the equation, $[\frac{ | \{ j | \min_{j} \mathrm{MSE}(p_i, r_j) \text{where}~p_i \in \mathrm{sample}(P, \mathrm{n\_sample}) \}  | }{m} \text{for}~\mathrm{n\_sample} \in \mathrm{cuc\_n\_samples} ]$.
+- **cuc**: (float): Coverage of the matching instances. In the equation, $\frac{ | \{i | \min_{i} \mathrm{MSE}(p_i, r_j) < \mathrm{threshold}\} | }{n}$.
+- **coverages_features**: (list of list of float): coverages computed individually for each feature.
+- **cuc_features**: (list of float): cuc computed individually for each feature.
+- **macro_coverages**: (list of float): Average of the coverages_features.
+- **macro_cuc**: (float): Average of the cuc_features.
 
 #### Values from Popular Papers
 <!-- *Give examples, preferrably with links to leaderboards or publications, to papers that have reported this metric, along with the values they have reported.* -->

matching_series.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 """TODO: Add a description here."""
 
+import math
 import statistics
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 import datasets
 import evaluate
@@ -127,9 +128,11 @@ class matching_series(evaluate.Metric):
 
     def _compute(
         self,
-        predictions: Union[list, np.ndarray],
-        references: Union[list, np.ndarray],
+        predictions: Union[List, np.ndarray],
+        references: Union[List, np.ndarray],
         batch_size: Optional[int] = None,
+        cuc_n_calculation: int = 3,
+        cuc_n_samples: Union[List[int], str] = "auto",
     ):
         """
         Compute the scores of the module given the predictions and references
@@ -139,6 +142,8 @@ class matching_series(evaluate.Metric):
             references: list of reference
                 shape: (num_reference, num_timesteps, num_features)
             batch_size: batch size to use for the computation. If None, the whole dataset is processed at once.
+            cuc_n_calculation: number of Coverage Under Curve calculate times
+            cuc_n_samples: number of samples to use for Coverage Under Curve calculation. If "auto", it uses the number of samples of the predictions.
         Returns:
         """
         predictions = np.array(predictions)
@@ -175,47 +180,128 @@ class matching_series(evaluate.Metric):
 
         # matching mse
         # shape: (num_generation,)
-        matching_mse = mse_mean[np.arange(len(best_match)), best_match].mean()
+        precision_mse = mse_mean[np.arange(len(best_match)), best_match].mean()
 
         # best match for each reference time series
         # shape: (num_reference,)
         best_match_inv = np.argmin(mse_mean, axis=0)
-        covered_mse = mse_mean[best_match_inv, np.arange(len(best_match_inv))].mean()
+        recall_mse = mse_mean[best_match_inv, np.arange(len(best_match_inv))].mean()
 
-        harmonic_mean = 2 / (1 / matching_mse + 1 / covered_mse)
+        f1_mse = 2 / (1 / precision_mse + 1 / recall_mse)
+
+        # matching precision, recall and f1
+        matching_precision = np.unique(best_match).size / len(best_match_inv)
+        matching_recall = np.unique(best_match_inv).size / len(best_match)
+        matching_f1 = 2 / (1 / matching_precision + 1 / matching_recall)
 
         # take matching for each feature and compute metrics for them
-        matching_mse_features = []
-        covered_mse_features = []
-        harmonic_mean_features = []
+        precision_mse_features = []
+        recall_mse_features = []
+        f1_mse_features = []
+        matching_precision_features = []
+        matching_recall_features = []
+        matching_f1_features = []
         index_mse_features = []
+        coverages_features = []
+        cuc_features = []
         for f in range(predictions.shape[-1]):
             mse_f = mse[:, :, f]
             index_mse_f = mse_f.diagonal(axis1=0, axis2=1).mean()
             best_match_f = np.argmin(mse_f, axis=-1)
-            matching_mse_f = mse_f[np.arange(len(best_match_f)), best_match_f].mean()
+            precision_mse_f = mse_f[np.arange(len(best_match_f)), best_match_f].mean()
             best_match_inv_f = np.argmin(mse_f, axis=0)
-            covered_mse_f = mse_f[best_match_inv_f, np.arange(len(best_match_inv_f))].mean()
-            harmonic_mean_f = 2 / (1 / matching_mse_f + 1 / covered_mse_f)
-            matching_mse_features.append(matching_mse_f)
-            covered_mse_features.append(covered_mse_f)
-            harmonic_mean_features.append(harmonic_mean_f)
+            recall_mse_f = mse_f[best_match_inv_f, np.arange(len(best_match_inv_f))].mean()
+            f1_mse_f = 2 / (1 / precision_mse_f + 1 / recall_mse_f)
+            precision_mse_features.append(precision_mse_f)
+            recall_mse_features.append(recall_mse_f)
+            f1_mse_features.append(f1_mse_f)
             index_mse_features.append(index_mse_f)
-        macro_matching_mse = statistics.mean(matching_mse_features)
-        macro_covered_mse = statistics.mean(covered_mse_features)
-        macro_harmonic_mean = statistics.mean(harmonic_mean_features)
+
+            matching_precision_f = np.unique(best_match_f).size / len(best_match_f)
+            matching_recall_f = np.unique(best_match_inv_f).size / len(best_match_inv_f)
+            matching_f1_f = 2 / (1 / matching_precision_f + 1 / matching_recall_f)
+            matching_precision_features.append(matching_precision_f)
+            matching_recall_features.append(matching_recall_f)
+            matching_f1_features.append(matching_f1_f)
+
+            coverages_f, cuc_f = self.compute_cuc(best_match_f, len(references), cuc_n_calculation, cuc_n_samples)
+            coverages_features.append(coverages_f)
+            cuc_features.append(cuc_f)
+
+        macro_precision_mse = statistics.mean(precision_mse_features)
+        macro_recall_mse = statistics.mean(recall_mse_features)
+        macro_f1_mse = statistics.mean(f1_mse_features)
         macro_index_mse = statistics.mean(index_mse_features)
+
+        macro_matching_precision = statistics.mean(matching_precision_features)
+        macro_matching_recall = statistics.mean(matching_recall_features)
+        macro_matching_f1 = statistics.mean(matching_f1_features)
+
+        # cuc
+        coverages, cuc = self.compute_cuc(best_match, len(references), cuc_n_calculation, cuc_n_samples)
+
+        macro_cuc = statistics.mean(cuc_features)
+        macro_coverages = [statistics.mean(c) for c in zip(*coverages_features)]
+
         return {
-            "matching_mse": matching_mse,
-            "harmonic_mean": harmonic_mean,
-            "covered_mse": covered_mse,
+            "precision_mse": precision_mse,
+            "f1_mse": f1_mse,
+            "recall_mse": recall_mse,
             "index_mse": index_mse,
-            "matching_mse_features": matching_mse_features,
-            "harmonic_mean_features": harmonic_mean_features,
-            "covered_mse_features": covered_mse_features,
+            "precision_mse_features": precision_mse_features,
+            "f1_mse_features": f1_mse_features,
+            "recall_mse_features": recall_mse_features,
             "index_mse_features": index_mse_features,
-            "macro_matching_mse": macro_matching_mse,
-            "macro_covered_mse": macro_covered_mse,
-            "macro_harmonic_mean": macro_harmonic_mean,
+            "macro_precision_mse": macro_precision_mse,
+            "macro_recall_mse": macro_recall_mse,
+            "macro_f1_mse": macro_f1_mse,
             "macro_index_mse": macro_index_mse,
+            "matching_precision": matching_precision,
+            "matching_recall": matching_recall,
+            "matching_f1": matching_f1,
+            "matching_precision_features": matching_precision_features,
+            "matching_recall_features": matching_recall_features,
+            "matching_f1_features": matching_f1_features,
+            "macro_matching_precision": macro_matching_precision,
+            "macro_matching_recall": macro_matching_recall,
+            "macro_matching_f1": macro_matching_f1,
+            "cuc": cuc,
+            "coverages": coverages,
+            "macro_cuc": macro_cuc,
+            "macro_coverages": macro_coverages,
+            "cuc_features": cuc_features,
+            "coverages_features": coverages_features,
         }
+
+    def compute_cuc(
+        self,
+        match: np.ndarray,
+        n_reference: int,
+        n_calculation: int,
+        n_samples: Union[List[int], str],
+    ):
+        """
+        Compute Coverage Under Curve
+        Args:
+            match: best match for each generated time series
+            n_reference: number of reference time series
+            n_calculation: number of Coverage Under Curve calculate times
+            n_samples: number of samples to use for Coverage Under Curve calculation. If "auto", it uses the number of samples of the predictions.
+        Returns:
+        """
+        n_generaiton = len(match)
+        if n_samples == "auto":
+            exp = int(math.log2(n_generaiton))
+            n_samples = [int(2**i) for i in range(exp)]
+            n_samples.append(n_generaiton)
+        assert isinstance(n_samples, list) and all(isinstance(n, int) for n in n_samples)
+
+        coverages = []
+        for n_sample in n_samples:
+            coverage = 0
+            for _ in range(n_calculation):
+                sample = np.random.choice(match, size=n_sample, replace=False)  # type: ignore
+                coverage += len(np.unique(sample)) / n_reference
+            coverages.append(coverage / n_calculation)
+        cuc = np.trapz(coverages, n_samples) / len(n_samples) / max(n_samples)
+        return coverages, cuc

plot_cuc.py

@@ -0,0 +1,24 @@
+import json
+from argparse import ArgumentParser
+
+import matplotlib.pyplot as plt
+
+parser = ArgumentParser()
+parser.add_argument("input", type=str, help="Input file of json data, output of matching_series")
+parser.add_argument("output", type=str, help="Output file of the plot")
+args = parser.parse_args()
+
+with open(args.input, "r") as f:
+    data = json.load(f)
+
+coverages = data["coverages"]
+x = [2**i for i in range(len(coverages))]
+y = coverages
+
+fig, ax = plt.subplots()
+
+ax.plot(x, y, "o-")
+ax.set_xscale("log", base=2)
+ax.set_xlabel("Number of generations")
+ax.set_ylabel("Coverage")
+plt.savefig(args.output)