Spaces:
Sleeping
Sleeping
“Transcendental-Programmer”
commited on
Commit
·
3de89cd
1
Parent(s):
4932e02
feat: Implement client model training and data generation
Browse files- src/client/data_handler.py +55 -0
- src/client/model.py +121 -0
src/client/data_handler.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""data_handler.py module."""
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import pandas as pd
|
| 5 |
+
from typing import Tuple, Dict
|
| 6 |
+
import tensorflow as tf
|
| 7 |
+
from sklearn.preprocessing import StandardScaler
|
| 8 |
+
|
| 9 |
+
class FinancialDataHandler:
|
| 10 |
+
def __init__(self, config: Dict):
|
| 11 |
+
"""Initialize the data handler with configuration."""
|
| 12 |
+
self.batch_size = config['data']['batch_size']
|
| 13 |
+
self.shuffle_buffer = config['data']['shuffle_buffer']
|
| 14 |
+
self.prefetch_buffer = config['data']['prefetch_buffer']
|
| 15 |
+
self.scaler = StandardScaler()
|
| 16 |
+
|
| 17 |
+
def simulate_financial_data(self, num_samples: int = 1000) -> pd.DataFrame:
|
| 18 |
+
"""Generate synthetic financial data for testing."""
|
| 19 |
+
np.random.seed(42)
|
| 20 |
+
|
| 21 |
+
data = {
|
| 22 |
+
'transaction_amount': np.random.lognormal(mean=4.0, sigma=1.0, size=num_samples),
|
| 23 |
+
'account_balance': np.random.normal(loc=10000, scale=5000, size=num_samples),
|
| 24 |
+
'transaction_frequency': np.random.poisson(lam=5, size=num_samples),
|
| 25 |
+
'credit_score': np.random.normal(loc=700, scale=50, size=num_samples).clip(300, 850),
|
| 26 |
+
'days_since_last_transaction': np.random.exponential(scale=7, size=num_samples)
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
return pd.DataFrame(data)
|
| 30 |
+
|
| 31 |
+
def preprocess_data(self, data: pd.DataFrame) -> tf.data.Dataset:
|
| 32 |
+
"""Preprocess the data and convert to TensorFlow dataset."""
|
| 33 |
+
# Standardize the features
|
| 34 |
+
scaled_data = self.scaler.fit_transform(data)
|
| 35 |
+
|
| 36 |
+
# Convert to TensorFlow dataset
|
| 37 |
+
dataset = tf.data.Dataset.from_tensor_slices(scaled_data)
|
| 38 |
+
|
| 39 |
+
# Apply dataset transformations
|
| 40 |
+
dataset = dataset.shuffle(self.shuffle_buffer)
|
| 41 |
+
dataset = dataset.batch(self.batch_size)
|
| 42 |
+
dataset = dataset.prefetch(self.prefetch_buffer)
|
| 43 |
+
|
| 44 |
+
return dataset
|
| 45 |
+
|
| 46 |
+
def get_client_data(self) -> Tuple[tf.data.Dataset, StandardScaler]:
|
| 47 |
+
"""Get preprocessed client data and scaler."""
|
| 48 |
+
# Simulate client data
|
| 49 |
+
raw_data = self.simulate_financial_data()
|
| 50 |
+
|
| 51 |
+
# Preprocess data
|
| 52 |
+
dataset = self.preprocess_data(raw_data)
|
| 53 |
+
|
| 54 |
+
return dataset, self.scaler
|
| 55 |
+
|
src/client/model.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""model.py module."""
|
| 2 |
+
|
| 3 |
+
from typing import Dict, List
|
| 4 |
+
import tensorflow as tf
|
| 5 |
+
import numpy as np
|
| 6 |
+
import logging
|
| 7 |
+
|
| 8 |
+
class FederatedClient:
|
| 9 |
+
def __init__(self, client_id: int, config: Dict):
|
| 10 |
+
"""Initialize the federated client."""
|
| 11 |
+
self.client_id = client_id
|
| 12 |
+
self.config = config.get('client', {})
|
| 13 |
+
self.model = self._build_model()
|
| 14 |
+
|
| 15 |
+
def start(self):
|
| 16 |
+
"""Start the federated client process."""
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
logger.info(f"Client {self.client_id} started")
|
| 19 |
+
logger.info(f"Client config: {self.config}")
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
# Simulate some data
|
| 23 |
+
logger.info("Generating training data...")
|
| 24 |
+
X, y = self._generate_dummy_data()
|
| 25 |
+
logger.info(f"Generated data shapes - X: {X.shape}, y: {y.shape}")
|
| 26 |
+
|
| 27 |
+
# Train locally
|
| 28 |
+
logger.info("Starting local training...")
|
| 29 |
+
history = self.train_local((X, y))
|
| 30 |
+
|
| 31 |
+
# Log training metrics
|
| 32 |
+
losses = history.get('loss', [])
|
| 33 |
+
logger.info("\nTraining Progress Summary:")
|
| 34 |
+
logger.info("-" * 30)
|
| 35 |
+
for epoch, loss in enumerate(losses, 1):
|
| 36 |
+
logger.info(f"Epoch {epoch:2d}/{len(losses)}: loss = {loss:.4f}")
|
| 37 |
+
|
| 38 |
+
final_loss = losses[-1]
|
| 39 |
+
logger.info(f"\nTraining completed - Final loss: {final_loss:.4f}")
|
| 40 |
+
|
| 41 |
+
# Log model summary in a simpler format
|
| 42 |
+
logger.info("\nModel Architecture:")
|
| 43 |
+
logger.info("-" * 30)
|
| 44 |
+
logger.info("Layer (Output Shape) -> Params")
|
| 45 |
+
total_params = 0
|
| 46 |
+
for layer in self.model.layers:
|
| 47 |
+
params = layer.count_params()
|
| 48 |
+
total_params += params
|
| 49 |
+
logger.info(f"{layer.name} {layer.output.shape} -> {params:,} params")
|
| 50 |
+
logger.info(f"Total Parameters: {total_params:,}")
|
| 51 |
+
|
| 52 |
+
except Exception as e:
|
| 53 |
+
logger.error(f"Error during client execution: {str(e)}")
|
| 54 |
+
raise
|
| 55 |
+
|
| 56 |
+
def _generate_dummy_data(self):
|
| 57 |
+
"""Generate dummy data for testing."""
|
| 58 |
+
num_samples = 100
|
| 59 |
+
input_dim = 32 # Match with model's input dimension
|
| 60 |
+
|
| 61 |
+
# Generate input data
|
| 62 |
+
X = tf.random.normal((num_samples, input_dim))
|
| 63 |
+
# Generate target data (for this example, we'll predict the sum of inputs)
|
| 64 |
+
y = tf.reduce_sum(X, axis=1, keepdims=True)
|
| 65 |
+
|
| 66 |
+
return X, y
|
| 67 |
+
|
| 68 |
+
def _build_model(self):
|
| 69 |
+
"""Build the initial model architecture."""
|
| 70 |
+
input_dim = 32 # Match with data generation
|
| 71 |
+
model = tf.keras.Sequential([
|
| 72 |
+
tf.keras.layers.Input(shape=(input_dim,)),
|
| 73 |
+
tf.keras.layers.Dense(128, activation='relu'),
|
| 74 |
+
tf.keras.layers.Dense(64, activation='relu'),
|
| 75 |
+
tf.keras.layers.Dense(1) # Output layer for regression
|
| 76 |
+
])
|
| 77 |
+
model.compile(
|
| 78 |
+
optimizer=tf.keras.optimizers.Adam(
|
| 79 |
+
learning_rate=self.config.get('training', {}).get('learning_rate', 0.001)
|
| 80 |
+
),
|
| 81 |
+
loss='mse'
|
| 82 |
+
)
|
| 83 |
+
return model
|
| 84 |
+
|
| 85 |
+
def train_local(self, data):
|
| 86 |
+
"""Train the model on local data."""
|
| 87 |
+
logger = logging.getLogger(__name__)
|
| 88 |
+
X, y = data
|
| 89 |
+
|
| 90 |
+
# Log training parameters
|
| 91 |
+
logger.info(f"\nTraining Parameters:")
|
| 92 |
+
logger.info("-" * 50)
|
| 93 |
+
logger.info(f"Input shape: {X.shape}")
|
| 94 |
+
logger.info(f"Output shape: {y.shape}")
|
| 95 |
+
logger.info(f"Batch size: {self.config.get('data', {}).get('batch_size', 32)}")
|
| 96 |
+
logger.info(f"Epochs: {self.config.get('training', {}).get('local_epochs', 5)}")
|
| 97 |
+
logger.info(f"Learning rate: {self.config.get('training', {}).get('learning_rate', 0.001)}")
|
| 98 |
+
logger.info("-" * 50)
|
| 99 |
+
|
| 100 |
+
class LogCallback(tf.keras.callbacks.Callback):
|
| 101 |
+
def on_epoch_end(self, epoch, logs=None):
|
| 102 |
+
logger.info(f"Epoch {epoch + 1} - loss: {logs['loss']:.4f}")
|
| 103 |
+
|
| 104 |
+
# Enable verbose mode for training
|
| 105 |
+
history = self.model.fit(
|
| 106 |
+
X, y,
|
| 107 |
+
batch_size=self.config.get('data', {}).get('batch_size', 32),
|
| 108 |
+
epochs=self.config.get('training', {}).get('local_epochs', 5),
|
| 109 |
+
verbose=0, # Disable default verbose output
|
| 110 |
+
callbacks=[LogCallback()] # Use our custom callback
|
| 111 |
+
)
|
| 112 |
+
return history.history
|
| 113 |
+
|
| 114 |
+
def get_weights(self) -> List:
|
| 115 |
+
"""Get the model weights."""
|
| 116 |
+
return self.model.get_weights()
|
| 117 |
+
|
| 118 |
+
def set_weights(self, weights: List):
|
| 119 |
+
"""Update local model with global weights."""
|
| 120 |
+
self.model.set_weights(weights)
|
| 121 |
+
|