adds normalized variables on update

src/dataset_utils.py

@@ -253,19 +253,58 @@ class TrackioDatasetManager:
                 inc_params = _parse_json_field(incoming.get('parameters'), {})
                 inc_artifacts = _parse_json_field(incoming.get('artifacts'), [])
                 inc_logs = _parse_json_field(incoming.get('logs'), [])
-                # Merge metrics with de-dup
+                # Merge metrics with de-dup (by step+timestamp) then collapse per step
                 merged_metrics = []
                 seen = set()
                 for entry in base_metrics + inc_metrics:
                     try:
-                        # Use the original entry so _metrics_key can properly
-                        # distinguish dict vs non-dict entries
                         key = _metrics_key(entry)
                     except Exception:
                         key = (None, None)
                     if key not in seen:
                         seen.add(key)
                         merged_metrics.append(entry)
+
+                # Collapse duplicate steps by merging their metric dicts and keeping the latest timestamp
+                try:
+                    step_to_entry: Dict[Any, Dict[str, Any]] = {}
+                    for e in merged_metrics:
+                        if not isinstance(e, dict):
+                            continue
+                        # Ensure nested structure {timestamp, step, metrics}
+                        if 'metrics' not in e:
+                            e = {
+                                'timestamp': e.get('timestamp'),
+                                'step': e.get('step'),
+                                'metrics': {k: v for k, v in e.items() if k not in ('step', 'timestamp')}
+                            }
+                        step_val = e.get('step')
+                        if step_val in step_to_entry:
+                            existing_e = step_to_entry[step_val]
+                            try:
+                                existing_metrics_dict = existing_e.get('metrics', {})
+                                if isinstance(existing_metrics_dict, dict):
+                                    existing_metrics_dict.update(e.get('metrics', {}))
+                                else:
+                                    existing_e['metrics'] = e.get('metrics', {})
+                            except Exception:
+                                existing_e['metrics'] = e.get('metrics', {})
+                            try:
+                                if str(e.get('timestamp', '')) > str(existing_e.get('timestamp', '')):
+                                    existing_e['timestamp'] = e.get('timestamp')
+                            except Exception:
+                                pass
+                        else:
+                            step_to_entry[step_val] = dict(e)
+                    def _step_key(x: Dict[str, Any]):
+                        try:
+                            return float(x.get('step'))
+                        except Exception:
+                            return -1.0
+                    merged_metrics = sorted(step_to_entry.values(), key=_step_key)
+                except Exception:
+                    # On any error, keep the de-duplicated list
+                    pass
                 # Merge params
                 merged_params = {}
                 if isinstance(base_params, dict):

src/monitoring.py

@@ -310,6 +310,54 @@ class SmolLM3Monitor:
                 except Exception:
                     pass
 
+            # Collapse duplicate step entries by merging their metric dictionaries
+            try:
+                def _collapse_by_step(entries: list) -> list:
+                    step_to_entry: dict = {}
+                    for e in entries:
+                        if not isinstance(e, dict):
+                            continue
+                        # Normalize to nested structure
+                        if 'metrics' not in e:
+                            e = {
+                                'timestamp': e.get('timestamp'),
+                                'step': e.get('step'),
+                                'metrics': {k: v for k, v in e.items() if k not in ('step', 'timestamp')}
+                            }
+                        step_val = e.get('step')
+                        if step_val in step_to_entry:
+                            # Merge metrics into existing entry for the same step
+                            existing_e = step_to_entry[step_val]
+                            try:
+                                existing_e_metrics = existing_e.get('metrics', {})
+                                if isinstance(existing_e_metrics, dict):
+                                    existing_e_metrics.update(e.get('metrics', {}))
+                                else:
+                                    existing_e['metrics'] = e.get('metrics', {})
+                            except Exception:
+                                existing_e['metrics'] = e.get('metrics', {})
+                            # Prefer the latest timestamp (ISO strings compare lexicographically)
+                            try:
+                                if str(e.get('timestamp', '')) > str(existing_e.get('timestamp', '')):
+                                    existing_e['timestamp'] = e.get('timestamp')
+                            except Exception:
+                                pass
+                        else:
+                            step_to_entry[step_val] = dict(e)
+                    # Sort by step (fallback to 0 for None/non-numeric)
+                    def _step_key(x):
+                        val = x.get('step')
+                        try:
+                            return float(val)
+                        except Exception:
+                            return -1.0
+                    return sorted(step_to_entry.values(), key=_step_key)
+
+                merged_metrics = _collapse_by_step(merged_metrics)
+            except Exception:
+                # If anything goes wrong, keep original list
+                pass
+
             # Merge artifacts if provided
             if 'artifacts' in experiment_data and isinstance(experiment_data['artifacts'], list):
                 # De-duplicate while preserving order

templates/spaces/trackio/app.py

@@ -661,18 +661,32 @@ class TrackioSpace:
         if not experiment['metrics']:
             return pd.DataFrame()
         
-        # Convert metrics to DataFrame
+        # Convert metrics to DataFrame (merge duplicate steps)
         data = []
         for metric_entry in experiment['metrics']:
             step = metric_entry.get('step', 0)
             timestamp = metric_entry.get('timestamp', '')
             metrics = metric_entry.get('metrics', {})
-            
+
             row = {'step': step, 'timestamp': timestamp}
             row.update(metrics)
             data.append(row)
-        
-        return pd.DataFrame(data)
+
+        if not data:
+            return pd.DataFrame()
+
+        df = pd.DataFrame(data)
+        # Ensure step exists even if None
+        if 'step' not in df.columns:
+            df['step'] = 0
+        # For duplicate steps, keep the latest timestamp and merge columns by last valid value
+        try:
+            df.sort_values(['step', 'timestamp'], inplace=True)
+            # Take the last row per step (latest timestamp)
+            df = df.groupby('step', as_index=False).last()
+        except Exception:
+            pass
+        return df
 
 # Global instance
 trackio_space = TrackioSpace()