federated/data/data_preprocessing.py
def load_data(
    normalized: bool = True,
    data_analysis: bool = False,
    data_selector: Callable[
        [np.ndarray, np.ndarray, int], tff.simulation.ClientData
    ] = None,
    number_of_clients: int = 5,
) -> Tuple[tff.simulation.ClientData, tff.simulation.ClientData, int]:
    """Function loads data from csv-file and preprocesses the training and test data seperately.

    Args:
        normalized (bool, optional): Whether to normalize the data. Defaults to True.\n
        data_analysis (bool, optional): Load data for data analysis. Defaults to False.\n
        data_selector (Callable[ [np.ndarray, np.ndarray, int], tff.simulation.ClientData ], optional): Data distribution. Defaults to None.\n
        number_of_clients (int, optional): Number of clients. Defaults to 5.\n

    Raises:
        ValueError: The data has to be normalized to use create_uniform_dataset.

    Returns:
        [tff.simulation.ClientData, tff.simulation.ClientData, int]: A tuple of tff.simulation.ClientData.
    """
    train_file = "data/mitbih/mitbih_train.csv"
    test_file = "data/mitbih/mitbih_test.csv"

    if data_analysis or data_selector == create_unbalanced_data:
        train_file = "../" + train_file
        test_file = "../" + test_file

    train_df = pd.read_csv(train_file, header=None)
    test_df = pd.read_csv(test_file, header=None)

    train_df[187], test_df[187] = (
        train_df[187].astype(int),
        test_df[187].astype(int),
    )

    if normalized:
        df_0 = (train_df[train_df[187] == 0]).sample(n=SAMPLES, random_state=42)
        # df_5 = (train_df[train_df[187] == 4]).sample(n=S, random_state=42)
        train_df = pd.concat(
            [df_0]
            + [
                resample(
                    train_df[train_df[187] == i],
                    replace=True,
                    n_samples=SAMPLES,
                    random_state=int(f"12{i+2}"),
                )
                for i in range(1, 5)
            ]
        )

    train_X, train_y = split_dataframe(train_df)

    test_X, test_y = split_dataframe(test_df)

    if data_analysis:
        return test_X, test_y

    if data_selector == create_uniform_dataset:
        if not normalized:
            raise ValueError(
                "The data has to be normalized to use create_uniform_dataset"
            )

    train_client_data, train_data = data_selector(train_X, train_y, number_of_clients)

    if data_selector == create_corrupted_non_iid_dataset:
        test_client_data, test_data = create_non_iid_dataset(
            test_X, test_y, number_of_clients
        )
    else:
        test_client_data, test_data = data_selector(test_X, test_y, number_of_clients)

    if data_selector == create_unbalanced_data:
        return train_client_data, test_client_data

    return train_data, test_data, len(train_X)
Function loads data from csv-file and preprocesses the training and test data seperately.

    Args:
        normalized (bool, optional): Whether to normalize the data. Defaults to True.\n
        data_analysis (bool, optional): Load data for data analysis. Defaults to False.\n
        data_selector (Callable[ [np.ndarray, np.ndarray, int], tff.simulation.ClientData ], optional): Data distribution. Defaults to None.\n
        number_of_clients (int, optional): Number of clients. Defaults to 5.\n

    Raises:
        ValueError: The data has to be normalized to use create_uniform_dataset.

    Returns:
        [tff.simulation.ClientData, tff.simulation.ClientData, int]: A tuple of tff.simulation.ClientData.
    
Function loads data from csv-file and preprocesses the training and test data seperately.
Function loads data from csv - file and preprocesses the training and test data seperately .
def load_data(
    normalized: bool = True,
    data_analysis: bool = False,
    data_selector: Callable[
        [np.ndarray, np.ndarray, int], tff.simulation.ClientData
    ] = None,
    number_of_clients: int = 5,
) -> Tuple[tff.simulation.ClientData, tff.simulation.ClientData, int]:
    train_file = "data/mitbih/mitbih_train.csv"
    test_file = "data/mitbih/mitbih_test.csv"
    if data_analysis or data_selector == create_unbalanced_data:
        train_file = "../" + train_file
        test_file = "../" + test_file
    train_df = pd.read_csv(train_file, header=None)
    test_df = pd.read_csv(test_file, header=None)
    train_df[187], test_df[187] = (
        train_df[187].astype(int),
        test_df[187].astype(int),
    )
    if normalized:
        df_0 = (train_df[train_df[187] == 0]).sample(n=SAMPLES, random_state=42)
        train_df = pd.concat(
            [df_0]
            + [
                resample(
                    train_df[train_df[187] == i],
                    replace=True,
                    n_samples=SAMPLES,
                    random_state=int(f"12{i+2}"),
                )
                for i in range(1, 5)
            ]
        )
    train_X, train_y = split_dataframe(train_df)
    test_X, test_y = split_dataframe(test_df)
    if data_analysis:
        return test_X, test_y
    if data_selector == create_uniform_dataset:
        if not normalized:
            raise ValueError(
                "The data has to be normalized to use create_uniform_dataset"
            )
    train_client_data, train_data = data_selector(train_X, train_y, number_of_clients)
    if data_selector == create_corrupted_non_iid_dataset:
        test_client_data, test_data = create_non_iid_dataset(
            test_X, test_y, number_of_clients
        )
    else:
        test_client_data, test_data = data_selector(test_X, test_y, number_of_clients)
    if data_selector == create_unbalanced_data:
        return train_client_data, test_client_data
    return train_data, test_data, len(train_X)
def load_data ( normalized : bool = True , data_analysis : bool = False , data_selector : Callable [ [ np . ndarray , np . ndarray , int ] , tff . simulation . ClientData ] = None , number_of_clients : int = 5 , ) -> Tuple [ tff . simulation . ClientData , tff . simulation . ClientData , int ] : train_file = "data/mitbih/mitbih_train.csv" test_file = "data/mitbih/mitbih_test.csv" if data_analysis or data_selector == create_unbalanced_data : train_file = "../" + train_file test_file = "../" + test_file train_df = pd . read_csv ( train_file , header = None ) test_df = pd . read_csv ( test_file , header = None ) train_df [ 187 ] , test_df [ 187 ] = ( train_df [ 187 ] . astype ( int ) , test_df [ 187 ] . astype ( int ) , ) if normalized : df_0 = ( train_df [ train_df [ 187 ] == 0 ] ) . sample ( n = SAMPLES , random_state = 42 ) train_df = pd . concat ( [ df_0 ] + [ resample ( train_df [ train_df [ 187 ] == i ] , replace = True , n_samples = SAMPLES , random_state = int ( f"12{i+2}" ) , ) for i in range ( 1 , 5 ) ] ) train_X , train_y = split_dataframe ( train_df ) test_X , test_y = split_dataframe ( test_df ) if data_analysis : return test_X , test_y if data_selector == create_uniform_dataset : if not normalized : raise ValueError ( "The data has to be normalized to use create_uniform_dataset" ) train_client_data , train_data = data_selector ( train_X , train_y , number_of_clients ) if data_selector == create_corrupted_non_iid_dataset : test_client_data , test_data = create_non_iid_dataset ( test_X , test_y , number_of_clients ) else : test_client_data , test_data = data_selector ( test_X , test_y , number_of_clients ) if data_selector == create_unbalanced_data : return train_client_data , test_client_data return train_data , test_data , len ( train_X )
Function loads data from csv-file and preprocesses the training and test data seperately.
Function loads data from csv - file and preprocesses the training and test data seperately .
"""Function loads data from csv-file and preprocesses the training and test data seperately.

    Args:
        normalized (bool, optional): Whether to normalize the data. Defaults to True.\n
        data_analysis (bool, optional): Load data for data analysis. Defaults to False.\n
        data_selector (Callable[ [np.ndarray, np.ndarray, int], tff.simulation.ClientData ], optional): Data distribution. Defaults to None.\n
        number_of_clients (int, optional): Number of clients. Defaults to 5.\n

    Raises:
        ValueError: The data has to be normalized to use create_uniform_dataset.

    Returns:
        [tff.simulation.ClientData, tff.simulation.ClientData, int]: A tuple of tff.simulation.ClientData.
    """ # df_5 = (train_df[train_df[187] == 4]).sample(n=S, random_state=42)
A tuple of tff.simulation.ClientData.
A tuple of tff . simulation . ClientData .
[tff.simulation.ClientData, tff.simulation.ClientData, int]
The data has to be normalized to use create_uniform_dataset.
The data has to be normalized to use create_uniform_dataset .
ValueError
normalized
bool
Whether to normalize the data. Defaults to True.\n
Whether to normalize the data . Defaults to True . \ n
True
data_analysis
bool
Load data for data analysis. Defaults to False.\n
Load data for data analysis . Defaults to False . \ n
False
data_selector
Callable[
        [np.ndarray, np.ndarray, int], tff.simulation.ClientData
    ]
Data distribution. Defaults to None.\n
Data distribution . Defaults to None . \ n
None
number_of_clients
int
Number of clients. Defaults to 5.\n
Number of clients . Defaults to 5 . \ n
5