epimodel/preprocessing.py
def preprocess_data(
    data_path,
    start_date="2020-08-01",
    end_date="2021-01-22",
    npi_start_col=3,
    skipcases=8,
    skipdeaths=26,
    household_feature_processing="implicit",
):
    """
    Process data, return PreprocessedData() object

    :param data_path: merged data path
    :param start_date: window of analysis start date
    :param end_date: window of analysis end date
    :param npi_start_col: start column of npis
    :param skipcases: Number of days of cases to mask at the start of the period. We mask since these
                      cases were generated by infections we cannot account for (they were outside of this period).
    :param skipdeaths: Number of days of deaths to mask at the start of the period. We mask since these
                      cases were generated by infections we cannot account for (they were outside of this period).
    :param household_feature_processing: either implicit or raw. implicit takes the implied gathering limit from the
                                         the household limit, while raw ignore this
    :return: PreprocessedData object containing loaded data, but data that has not yet been featurised.
    """

    def set_household_limits(active_cms, household_NPI_index, gathering_NPI_index):
        """
        This function implements implicit processing of household NPIs. i.e., if the
        gathering limit is X, then the household limit is also set to X if no household
        limit has been recorded. In other words, if the gathering limit is stricter than
        the household limit (e.g., 6 people from 10 households), the household limit is set
        to the gathering limit, since worst case is that every person comes from their own
        household.

        :param active_CMs: nRs x nCMs x nDs activation matrix
        :param household_NPI_index: household index
        :param gathering_NPI_index: respective gathering index
        :return: new active_cms matrix
        """
        nRs, _, nDs = active_cms.shape
        new_acms = np.copy(active_cms)
        for r in range(nRs):
            for day in range(nDs):
                if (
                    active_cms[r, household_NPI_index, day] == 0
                    or active_cms[r, gathering_NPI_index, day]
                    < active_cms[r, household_NPI_index, day]
                ):
                    new_acms[r, household_NPI_index, day] = active_cms[
                        r, gathering_NPI_index, day
                    ]
        return new_acms

    df = pd.read_csv(
        data_path, parse_dates=["Date"], infer_datetime_format=True
    ).set_index(["Area", "Date"])

    Ds = list(df.index.levels[1])

    if end_date is not None:
        last_ts = pd.to_datetime(end_date)
        # +1 because we want to include the last day
        Ds = Ds[: (1 + Ds.index(last_ts))]

    if start_date is not None:
        start_ts = pd.to_datetime(start_date)
        Ds = Ds[Ds.index(start_ts) :]

    print(f"Processing data from {Ds[0]} to {Ds[-1]}")

    Rs = list(df.index.levels[0])
    CMs = list(df.columns[npi_start_col:])

    nRs = len(Rs)
    nDs = len(Ds)
    nCMs = len(CMs)

    Cs = []

    for r in Rs:
        c = df.loc[r]["Country"][0]
        Cs.append(c)

    # sort the countries by name. we need to preserve the ordering
    unique_Cs = sorted(list(set(Cs)))
    nCs = len(unique_Cs)

    C_indices = []
    for uc in unique_Cs:
        a_indices = np.nonzero([uc == c for c in Cs])[0]
        C_indices.append(a_indices)

    active_cms = np.zeros((nRs, nCMs, nDs))
    new_cases = np.ma.zeros((nRs, nDs), dtype="int")
    new_deaths = np.ma.zeros((nRs, nDs), dtype="int")

    for r_i, r in enumerate(Rs):
        r_df = df.loc[r].loc[Ds]
        new_cases.data[r_i, :] = r_df["New Cases"]
        new_deaths.data[r_i, :] = r_df["New Deaths"]

        for cm_i, cm in enumerate(CMs):
            active_cms[r_i, cm_i, :] = r_df[cm]

    if household_feature_processing == "implicit":
        pairs = [
            ("Public Outdoor Gathering Person Limit", "Public Outdoor Household Limit"),
            ("Public Indoor Gathering Person Limit", "Public Indoor Household Limit"),
            (
                "Private Outdoor Gathering Person Limit",
                "Private Outdoor Household Limit",
            ),
            ("Private Indoor Gathering Person Limit", "Private Indoor Household Limit"),
        ]

        for pair in pairs:
            gathering_ind = CMs.index(pair[0])
            household_ind = CMs.index(pair[1])
            active_cms = set_household_limits(active_cms, household_ind, gathering_ind)
    elif household_feature_processing == "raw":
        pass
    else:
        raise ValueError("household_feature_processing must be in [implicit, raw]")

    # mask days where there are negative cases or deaths - because this
    # is clearly wrong
    new_cases[new_cases < 0] = np.ma.masked
    new_deaths[new_deaths < 0] = np.ma.masked
    # do this to make sure.
    new_cases.data[new_cases.data < 0] = 0
    new_deaths.data[new_deaths.data < 0] = 0

    new_cases[:, :skipcases] = np.ma.masked
    new_deaths[:, :skipdeaths] = np.ma.masked

    return PreprocessedData(
        Rs, Ds, CMs, new_cases, new_deaths, active_cms, Cs, unique_Cs, C_indices
    )

    Process data, return PreprocessedData() object

    :param data_path: merged data path
    :param start_date: window of analysis start date
    :param end_date: window of analysis end date
    :param npi_start_col: start column of npis
    :param skipcases: Number of days of cases to mask at the start of the period. We mask since these
                      cases were generated by infections we cannot account for (they were outside of this period).
    :param skipdeaths: Number of days of deaths to mask at the start of the period. We mask since these
                      cases were generated by infections we cannot account for (they were outside of this period).
    :param household_feature_processing: either implicit or raw. implicit takes the implied gathering limit from the
                                         the household limit, while raw ignore this
    :return: PreprocessedData object containing loaded data, but data that has not yet been featurised.
    
Process data, return PreprocessedData() object
Process data return PreprocessedData () object
def preprocess_data(
    data_path,
    start_date="2020-08-01",
    end_date="2021-01-22",
    npi_start_col=3,
    skipcases=8,
    skipdeaths=26,
    household_feature_processing="implicit",
):
    def set_household_limits(active_cms, household_NPI_index, gathering_NPI_index):
        nRs, _, nDs = active_cms.shape
        new_acms = np.copy(active_cms)
        for r in range(nRs):
            for day in range(nDs):
                if (
                    active_cms[r, household_NPI_index, day] == 0
                    or active_cms[r, gathering_NPI_index, day]
                    < active_cms[r, household_NPI_index, day]
                ):
                    new_acms[r, household_NPI_index, day] = active_cms[
                        r, gathering_NPI_index, day
                    ]
        return new_acms
    df = pd.read_csv(
        data_path, parse_dates=["Date"], infer_datetime_format=True
    ).set_index(["Area", "Date"])
    Ds = list(df.index.levels[1])
    if end_date is not None:
        last_ts = pd.to_datetime(end_date)
        Ds = Ds[: (1 + Ds.index(last_ts))]
    if start_date is not None:
        start_ts = pd.to_datetime(start_date)
        Ds = Ds[Ds.index(start_ts) :]
    print(f"Processing data from {Ds[0]} to {Ds[-1]}")
    Rs = list(df.index.levels[0])
    CMs = list(df.columns[npi_start_col:])
    nRs = len(Rs)
    nDs = len(Ds)
    nCMs = len(CMs)
    Cs = []
    for r in Rs:
        c = df.loc[r]["Country"][0]
        Cs.append(c)
    unique_Cs = sorted(list(set(Cs)))
    nCs = len(unique_Cs)
    C_indices = []
    for uc in unique_Cs:
        a_indices = np.nonzero([uc == c for c in Cs])[0]
        C_indices.append(a_indices)
    active_cms = np.zeros((nRs, nCMs, nDs))
    new_cases = np.ma.zeros((nRs, nDs), dtype="int")
    new_deaths = np.ma.zeros((nRs, nDs), dtype="int")
    for r_i, r in enumerate(Rs):
        r_df = df.loc[r].loc[Ds]
        new_cases.data[r_i, :] = r_df["New Cases"]
        new_deaths.data[r_i, :] = r_df["New Deaths"]
        for cm_i, cm in enumerate(CMs):
            active_cms[r_i, cm_i, :] = r_df[cm]
    if household_feature_processing == "implicit":
        pairs = [
            ("Public Outdoor Gathering Person Limit", "Public Outdoor Household Limit"),
            ("Public Indoor Gathering Person Limit", "Public Indoor Household Limit"),
            (
                "Private Outdoor Gathering Person Limit",
                "Private Outdoor Household Limit",
            ),
            ("Private Indoor Gathering Person Limit", "Private Indoor Household Limit"),
        ]
        for pair in pairs:
            gathering_ind = CMs.index(pair[0])
            household_ind = CMs.index(pair[1])
            active_cms = set_household_limits(active_cms, household_ind, gathering_ind)
    elif household_feature_processing == "raw":
        pass
    else:
        raise ValueError("household_feature_processing must be in [implicit, raw]")
    new_cases[new_cases < 0] = np.ma.masked
    new_deaths[new_deaths < 0] = np.ma.masked
    new_cases.data[new_cases.data < 0] = 0
    new_deaths.data[new_deaths.data < 0] = 0
    new_cases[:, :skipcases] = np.ma.masked
    new_deaths[:, :skipdeaths] = np.ma.masked
    return PreprocessedData(
        Rs, Ds, CMs, new_cases, new_deaths, active_cms, Cs, unique_Cs, C_indices
    )
def preprocess_data ( data_path , start_date = "2020-08-01" , end_date = "2021-01-22" , npi_start_col = 3 , skipcases = 8 , skipdeaths = 26 , household_feature_processing = "implicit" , ) : def set_household_limits ( active_cms , household_NPI_index , gathering_NPI_index ) : """
        This function implements implicit processing of household NPIs. i.e., if the
        gathering limit is X, then the household limit is also set to X if no household
        limit has been recorded. In other words, if the gathering limit is stricter than
        the household limit (e.g., 6 people from 10 households), the household limit is set
        to the gathering limit, since worst case is that every person comes from their own
        household.

        :param active_CMs: nRs x nCMs x nDs activation matrix
        :param household_NPI_index: household index
        :param gathering_NPI_index: respective gathering index
        :return: new active_cms matrix
        """ nRs , _ , nDs = active_cms . shape new_acms = np . copy ( active_cms ) for r in range ( nRs ) : for day in range ( nDs ) : if ( active_cms [ r , household_NPI_index , day ] == 0 or active_cms [ r , gathering_NPI_index , day ] < active_cms [ r , household_NPI_index , day ] ) : new_acms [ r , household_NPI_index , day ] = active_cms [ r , gathering_NPI_index , day ] return new_acms df = pd . read_csv ( data_path , parse_dates = [ "Date" ] , infer_datetime_format = True ) . set_index ( [ "Area" , "Date" ] ) Ds = list ( df . index . levels [ 1 ] ) if end_date is not None : last_ts = pd . to_datetime ( end_date ) Ds = Ds [ : ( 1 + Ds . index ( last_ts ) ) ] if start_date is not None : start_ts = pd . to_datetime ( start_date ) Ds = Ds [ Ds . index ( start_ts ) : ] print ( f"Processing data from {Ds[0]} to {Ds[-1]}" ) Rs = list ( df . index . levels [ 0 ] ) CMs = list ( df . columns [ npi_start_col : ] ) nRs = len ( Rs ) nDs = len ( Ds ) nCMs = len ( CMs ) Cs = [ ] for r in Rs : c = df . loc [ r ] [ "Country" ] [ 0 ] Cs . append ( c ) unique_Cs = sorted ( list ( set ( Cs ) ) ) nCs = len ( unique_Cs ) C_indices = [ ] for uc in unique_Cs : a_indices = np . nonzero ( [ uc == c for c in Cs ] ) [ 0 ] C_indices . append ( a_indices ) active_cms = np . zeros ( ( nRs , nCMs , nDs ) ) new_cases = np . ma . zeros ( ( nRs , nDs ) , dtype = "int" ) new_deaths = np . ma . zeros ( ( nRs , nDs ) , dtype = "int" ) for r_i , r in enumerate ( Rs ) : r_df = df . loc [ r ] . loc [ Ds ] new_cases . data [ r_i , : ] = r_df [ "New Cases" ] new_deaths . data [ r_i , : ] = r_df [ "New Deaths" ] for cm_i , cm in enumerate ( CMs ) : active_cms [ r_i , cm_i , : ] = r_df [ cm ] if household_feature_processing == "implicit" : pairs = [ ( "Public Outdoor Gathering Person Limit" , "Public Outdoor Household Limit" ) , ( "Public Indoor Gathering Person Limit" , "Public Indoor Household Limit" ) , ( "Private Outdoor Gathering Person Limit" , "Private Outdoor Household Limit" , ) , ( "Private Indoor Gathering Person Limit" , "Private Indoor Household Limit" ) , ] for pair in pairs : gathering_ind = CMs . index ( pair [ 0 ] ) household_ind = CMs . index ( pair [ 1 ] ) active_cms = set_household_limits ( active_cms , household_ind , gathering_ind ) elif household_feature_processing == "raw" : pass else : raise ValueError ( "household_feature_processing must be in [implicit, raw]" ) new_cases [ new_cases < 0 ] = np . ma . masked new_deaths [ new_deaths < 0 ] = np . ma . masked new_cases . data [ new_cases . data < 0 ] = 0 new_deaths . data [ new_deaths . data < 0 ] = 0 new_cases [ : , : skipcases ] = np . ma . masked new_deaths [ : , : skipdeaths ] = np . ma . masked return PreprocessedData ( Rs , Ds , CMs , new_cases , new_deaths , active_cms , Cs , unique_Cs , C_indices )
Process data, return PreprocessedData() object
Process data return PreprocessedData () object
"""
    Process data, return PreprocessedData() object

    :param data_path: merged data path
    :param start_date: window of analysis start date
    :param end_date: window of analysis end date
    :param npi_start_col: start column of npis
    :param skipcases: Number of days of cases to mask at the start of the period. We mask since these
                      cases were generated by infections we cannot account for (they were outside of this period).
    :param skipdeaths: Number of days of deaths to mask at the start of the period. We mask since these
                      cases were generated by infections we cannot account for (they were outside of this period).
    :param household_feature_processing: either implicit or raw. implicit takes the implied gathering limit from the
                                         the household limit, while raw ignore this
    :return: PreprocessedData object containing loaded data, but data that has not yet been featurised.
    """ """
        This function implements implicit processing of household NPIs. i.e., if the
        gathering limit is X, then the household limit is also set to X if no household
        limit has been recorded. In other words, if the gathering limit is stricter than
        the household limit (e.g., 6 people from 10 households), the household limit is set
        to the gathering limit, since worst case is that every person comes from their own
        household.

        :param active_CMs: nRs x nCMs x nDs activation matrix
        :param household_NPI_index: household index
        :param gathering_NPI_index: respective gathering index
        :return: new active_cms matrix
        """ # +1 because we want to include the last day # sort the countries by name. we need to preserve the ordering # mask days where there are negative cases or deaths - because this # is clearly wrong # do this to make sure.
PreprocessedData object containing loaded data, but data that has not yet been featurised.
PreprocessedData object containing loaded data but data that has not yet been featurised .
data_path
merged data path
merged data path
start_date
window of analysis start date
window of analysis start date
end_date
window of analysis end date
window of analysis end date
npi_start_col
start column of npis
start column of npis
skipcases
Number of days of cases to mask at the start of the period. We mask since these
cases were generated by infections we cannot account for (they were outside of this period).
Number of days of cases to mask at the start of the period . We mask since these cases were generated by infections we cannot account for ( they were outside of this period ) .
skipdeaths
Number of days of deaths to mask at the start of the period. We mask since these
cases were generated by infections we cannot account for (they were outside of this period).
Number of days of deaths to mask at the start of the period . We mask since these cases were generated by infections we cannot account for ( they were outside of this period ) .
household_feature_processing
either implicit or raw. implicit takes the implied gathering limit from the
the household limit, while raw ignore this
either implicit or raw . implicit takes the implied gathering limit from the the household limit while raw ignore this