notebooks/common.py
def fix_numerical_cols(t: pd.DataFrame) -> pd.DataFrame:
    """Convert selected columns from strings to numeric data type."""
    
    # some constants
    fixable_cols = (
        # For the 2022 cycle
        'Primary vote', '2pp vote',
        'Preferred Prime Minister',
        'Morrison', 'Albanese',
        'Sample size', 
        # For historical cycles
        'TPP vote', '2PP vote', 
        'Political parties',
        'Two-party-preferred',
    )

    # navigate fixable columns and convert to numerical dtype
    for c in t.columns[t.columns.get_level_values(0).isin(fixable_cols)]:
        if str(t[c].dtype) in ['object', ]:
            t[c] = (
                t[c]
                .str.replace('\[.*\]$', '', regex=True) # remove footnotes
                .str.replace('%', '')    # remove percent symbol
                .str.replace(tilde, '')  # replace tilde with nothing
                .str.replace(hyphen, '') # replace hyphen with nothing
                .str.replace(endash, '') # replace hyphen with nothing
                .str.replace(emdash, '') # replace hyphen with nothing
                .str.replace(minus, '')  # replace hyphen with nothing
                .str.replace('n/a', '')  # replace 'n/a' with nothing
                .str.replace('?', '', regex=False)  # replace '?' with nothing
                .str.replace('<', '', regex=False)  # replace '<' with nothing
                .str.strip()             # strip white space
                .replace('', np.nan)     # NaN empty lines
                .astype(float)           # float
            )
    return t
Convert selected columns from strings to numeric data type.
Convert selected columns from strings to numeric data type.
Convert selected columns from strings to numeric data type .
def fix_numerical_cols(t: pd.DataFrame) -> pd.DataFrame:
    fixable_cols = (
        'Primary vote', '2pp vote',
        'Preferred Prime Minister',
        'Morrison', 'Albanese',
        'Sample size', 
        'TPP vote', '2PP vote', 
        'Political parties',
        'Two-party-preferred',
    )
    for c in t.columns[t.columns.get_level_values(0).isin(fixable_cols)]:
        if str(t[c].dtype) in ['object', ]:
            t[c] = (
                t[c]
                .str.replace('\[.*\]$', '', regex=True) 
                .str.replace('%', '')    
                .str.replace(tilde, '')  
                .str.replace(hyphen, '') 
                .str.replace(endash, '') 
                .str.replace(emdash, '') 
                .str.replace(minus, '')  
                .str.replace('n/a', '')  
                .str.replace('?', '', regex=False)  
                .str.replace('<', '', regex=False)  
                .str.strip()             
                .replace('', np.nan)     
                .astype(float)           
            )
    return t
def fix_numerical_cols ( t : pd . DataFrame ) -> pd . DataFrame : fixable_cols = ( 'Primary vote' , '2pp vote' , 'Preferred Prime Minister' , 'Morrison' , 'Albanese' , 'Sample size' , 'TPP vote' , '2PP vote' , 'Political parties' , 'Two-party-preferred' , ) for c in t . columns [ t . columns . get_level_values ( 0 ) . isin ( fixable_cols ) ] : if str ( t [ c ] . dtype ) in [ 'object' , ] : t [ c ] = ( t [ c ] . str . replace ( '\[.*\]$' , '' , regex = True ) . str . replace ( '%' , '' ) . str . replace ( tilde , '' ) . str . replace ( hyphen , '' ) . str . replace ( endash , '' ) . str . replace ( emdash , '' ) . str . replace ( minus , '' ) . str . replace ( 'n/a' , '' ) . str . replace ( '?' , '' , regex = False ) . str . replace ( '<' , '' , regex = False ) . str . strip ( ) . replace ( '' , np . nan ) . astype ( float ) ) return t
Convert selected columns from strings to numeric data type.
Convert selected columns from strings to numeric data type .
"""Convert selected columns from strings to numeric data type.""" # some constants # For the 2022 cycle # For historical cycles # navigate fixable columns and convert to numerical dtype # remove footnotes # remove percent symbol # replace tilde with nothing # replace hyphen with nothing # replace hyphen with nothing # replace hyphen with nothing # replace hyphen with nothing # replace 'n/a' with nothing # replace '?' with nothing # replace '<' with nothing # strip white space # NaN empty lines # float
t
pd.DataFrame