sportsref/utils.py
def parse_table(table, flatten=True, footer=False):
    """Parses a table from sports-reference sites into a pandas dataframe.

    :param table: the PyQuery object representing the HTML table
    :param flatten: if True, flattens relative URLs to IDs. otherwise, leaves
        all fields as text without cleaning.
    :param footer: If True, returns the summary/footer of the page. Recommended
        to use this with flatten=False. Defaults to False.
    :returns: pd.DataFrame
    """
    if not len(table):
        return pd.DataFrame()

    # get columns
    columns = [c.attrib['data-stat']
               for c in table('thead tr:not([class]) th[data-stat]')]

    # get data
    rows = list(table('tbody tr' if not footer else 'tfoot tr')
                .not_('.thead, .stat_total, .stat_average').items())
    data = [
        [flatten_links(td) if flatten else td.text()
         for td in row.items('th,td')]
        for row in rows
    ]

    # make DataFrame
    df = pd.DataFrame(data, columns=columns, dtype='float')

    # add has_class columns
    allClasses = set(
        cls
        for row in rows
        if row.attr['class']
        for cls in row.attr['class'].split()
    )
    for cls in allClasses:
        df['has_class_' + cls] = [
            bool(row.attr['class'] and
                 cls in row.attr['class'].split())
            for row in rows
        ]

    # cleaning the DataFrame

    df.drop(['ranker', 'Xxx', 'Yyy', 'Zzz'],
            axis=1, inplace=True, errors='ignore')

    # year_id -> year (as int)
    if 'year_id' in df.columns:
        df.rename(columns={'year_id': 'year'}, inplace=True)
        if flatten:
            df.year = df.year.fillna(method='ffill')
            df['year'] = df.year.map(lambda s: str(s)[:4]).astype(int)

    # pos -> position
    if 'pos' in df.columns:
        df.rename(columns={'pos': 'position'}, inplace=True)

    # boxscore_word, game_date -> boxscore_id and separate into Y, M, D columns
    for bs_id_col in ('boxscore_word', 'game_date', 'box_score_text'):
        if bs_id_col in df.columns:
            df.rename(columns={bs_id_col: 'boxscore_id'}, inplace=True)
            break

    # ignore *, +, and other characters used to note things
    df.replace(re.compile(r'[\*\+\u2605]', re.U), '', inplace=True)
    for col in df.columns:
        if hasattr(df[col], 'str'):
            df[col] = df[col].str.strip()

    # player -> player_id and/or player_name
    if 'player' in df.columns:
        if flatten:
            df.rename(columns={'player': 'player_id'}, inplace=True)
            # when flattening, keep a column for names
            player_names = parse_table(table, flatten=False)['player_name']
            df['player_name'] = player_names
        else:
            df.rename(columns={'player': 'player_name'}, inplace=True)

    # team, team_name -> team_id
    for team_col in ('team', 'team_name'):
        if team_col in df.columns:
            # first, get rid of faulty rows
            df = df.loc[~df[team_col].isin(['XXX'])]
            if flatten:
                df.rename(columns={team_col: 'team_id'}, inplace=True)

    # season -> int
    if 'season' in df.columns and flatten:
        df['season'] = df['season'].astype(int)

    # handle date_game columns (different types)
    if 'date_game' in df.columns and flatten:
        date_re = r'month=(?P<month>\d+)&day=(?P<day>\d+)&year=(?P<year>\d+)'
        date_df = df['date_game'].str.extract(date_re, expand=True)
        if date_df.notnull().all(axis=1).any():
            df = pd.concat((df, date_df), axis=1)
        else:
            df.rename(columns={'date_game': 'boxscore_id'}, inplace=True)

    # game_location -> is_home
    if 'game_location' in df.columns and flatten:
        df['game_location'] = df['game_location'].isnull()
        df.rename(columns={'game_location': 'is_home'}, inplace=True)

    # mp: (min:sec) -> float(min + sec / 60), notes -> NaN, new column
    if 'mp' in df.columns and df.dtypes['mp'] == object and flatten:
        mp_df = df['mp'].str.extract(
            r'(?P<m>\d+):(?P<s>\d+)', expand=True).astype(float)
        no_match = mp_df.isnull().all(axis=1)
        if no_match.any():
            df.loc[no_match, 'note'] = df.loc[no_match, 'mp']
        df['mp'] = mp_df['m'] + mp_df['s'] / 60

    # converts number-y things to floats
    def convert_to_float(val):
        # percentages: (number%) -> float(number * 0.01)
        m = re.search(r'([-\.\d]+)\%',
                      val if isinstance(val, basestring) else str(val), re.U)
        try:
            if m:
                return float(m.group(1)) / 100 if m else val
            if m:
                return int(m.group(1)) + int(m.group(2)) / 60
        except ValueError:
            return val
        # salaries: $ABC,DEF,GHI -> float(ABCDEFGHI)
        m = re.search(r'\$[\d,]+',
                      val if isinstance(val, basestring) else str(val), re.U)
        try:
            if m:
                return float(re.sub(r'\$|,', '', val))
        except Exception:
            return val
        # generally try to coerce to float, unless it's an int or bool
        try:
            if isinstance(val, (int, bool)):
                return val
            else:
                return float(val)
        except Exception:
            return val

    if flatten:
        df = df.applymap(convert_to_float)

    df = df.loc[df.astype(bool).any(axis=1)]

    return df
def parse_table(table, flatten=True, footer=False):
    """Parses a table from sports-reference sites into a pandas dataframe.

    :param table: the PyQuery object representing the HTML table
    :param flatten: if True, flattens relative URLs to IDs. otherwise, leaves
        all fields as text without cleaning.
    :param footer: If True, returns the summary/footer of the page. Recommended
        to use this with flatten=False. Defaults to False.
    :returns: pd.DataFrame
    """
    if not len(table):
        return pd.DataFrame()

    # get columns
    columns = [c.attrib['data-stat']
               for c in table('thead tr:not([class]) th[data-stat]')]

    # get data
    rows = list(table('tbody tr' if not footer else 'tfoot tr')
                .not_('.thead, .stat_total, .stat_average').items())
    data = [
        [flatten_links(td) if flatten else td.text()
         for td in row.items('th,td')]
        for row in rows
    ]

    # make DataFrame
    df = pd.DataFrame(data, columns=columns, dtype='float')

    # add has_class columns
    allClasses = set(
        cls
        for row in rows
        if row.attr['class']
        for cls in row.attr['class'].split()
    )
    for cls in allClasses:
        df['has_class_' + cls] = [
            bool(row.attr['class'] and
                 cls in row.attr['class'].split())
            for row in rows
        ]

    # cleaning the DataFrame

    df.drop(['ranker', 'Xxx', 'Yyy', 'Zzz'],
            axis=1, inplace=True, errors='ignore')

    # year_id -> year (as int)
    if 'year_id' in df.columns:
        df.rename(columns={'year_id': 'year'}, inplace=True)
        if flatten:
            df.year = df.year.fillna(method='ffill')
            df['year'] = df.year.map(lambda s: str(s)[:4]).astype(int)

    # pos -> position
    if 'pos' in df.columns:
        df.rename(columns={'pos': 'position'}, inplace=True)

    # boxscore_word, game_date -> boxscore_id and separate into Y, M, D columns
    for bs_id_col in ('boxscore_word', 'game_date', 'box_score_text'):
        if bs_id_col in df.columns:
            df.rename(columns={bs_id_col: 'boxscore_id'}, inplace=True)
            break

    # ignore *, +, and other characters used to note things
    df.replace(re.compile(r'[\*\+\u2605]', re.U), '', inplace=True)
    for col in df.columns:
        if hasattr(df[col], 'str'):
            df[col] = df[col].str.strip()

    # player -> player_id and/or player_name
    if 'player' in df.columns:
        if flatten:
            df.rename(columns={'player': 'player_id'}, inplace=True)
            # when flattening, keep a column for names
            player_names = parse_table(table, flatten=False)['player_name']
            df['player_name'] = player_names
        else:
            df.rename(columns={'player': 'player_name'}, inplace=True)

    # team, team_name -> team_id
    for team_col in ('team', 'team_name'):
        if team_col in df.columns:
            # first, get rid of faulty rows
            df = df.loc[~df[team_col].isin(['XXX'])]
            if flatten:
                df.rename(columns={team_col: 'team_id'}, inplace=True)

    # season -> int
    if 'season' in df.columns and flatten:
        df['season'] = df['season'].astype(int)

    # handle date_game columns (different types)
    if 'date_game' in df.columns and flatten:
        date_re = r'month=(?P<month>\d+)&day=(?P<day>\d+)&year=(?P<year>\d+)'
        date_df = df['date_game'].str.extract(date_re, expand=True)
        if date_df.notnull().all(axis=1).any():
            df = pd.concat((df, date_df), axis=1)
        else:
            df.rename(columns={'date_game': 'boxscore_id'}, inplace=True)

    # game_location -> is_home
    if 'game_location' in df.columns and flatten:
        df['game_location'] = df['game_location'].isnull()
        df.rename(columns={'game_location': 'is_home'}, inplace=True)

    # mp: (min:sec) -> float(min + sec / 60), notes -> NaN, new column
    if 'mp' in df.columns and df.dtypes['mp'] == object and flatten:
        mp_df = df['mp'].str.extract(
            r'(?P<m>\d+):(?P<s>\d+)', expand=True).astype(float)
        no_match = mp_df.isnull().all(axis=1)
        if no_match.any():
            df.loc[no_match, 'note'] = df.loc[no_match, 'mp']
        df['mp'] = mp_df['m'] + mp_df['s'] / 60

    # converts number-y things to floats
    def convert_to_float(val):
        # percentages: (number%) -> float(number * 0.01)
        m = re.search(r'([-\.\d]+)\%',
                      val if isinstance(val, basestring) else str(val), re.U)
        try:
            if m:
                return float(m.group(1)) / 100 if m else val
            if m:
                return int(m.group(1)) + int(m.group(2)) / 60
        except ValueError:
            return val
        # salaries: $ABC,DEF,GHI -> float(ABCDEFGHI)
        m = re.search(r'\$[\d,]+',
                      val if isinstance(val, basestring) else str(val), re.U)
        try:
            if m:
                return float(re.sub(r'\$|,', '', val))
        except Exception:
            return val
        # generally try to coerce to float, unless it's an int or bool
        try:
            if isinstance(val, (int, bool)):
                return val
            else:
                return float(val)
        except Exception:
            return val

    if flatten:
        df = df.applymap(convert_to_float)

    df = df.loc[df.astype(bool).any(axis=1)]

    return df
def parse_table ( table , flatten = True , footer = False ) : if not len ( table ) : return pd . DataFrame ( ) # get columns columns = [ c . attrib [ 'data-stat' ] for c in table ( 'thead tr:not([class]) th[data-stat]' ) ] # get data rows = list ( table ( 'tbody tr' if not footer else 'tfoot tr' ) . not_ ( '.thead, .stat_total, .stat_average' ) . items ( ) ) data = [ [ flatten_links ( td ) if flatten else td . text ( ) for td in row . items ( 'th,td' ) ] for row in rows ] # make DataFrame df = pd . DataFrame ( data , columns = columns , dtype = 'float' ) # add has_class columns allClasses = set ( cls for row in rows if row . attr [ 'class' ] for cls in row . attr [ 'class' ] . split ( ) ) for cls in allClasses : df [ 'has_class_' + cls ] = [ bool ( row . attr [ 'class' ] and cls in row . attr [ 'class' ] . split ( ) ) for row in rows ] # cleaning the DataFrame df . drop ( [ 'ranker' , 'Xxx' , 'Yyy' , 'Zzz' ] , axis = 1 , inplace = True , errors = 'ignore' ) # year_id -> year (as int) if 'year_id' in df . columns : df . rename ( columns = { 'year_id' : 'year' } , inplace = True ) if flatten : df . year = df . year . fillna ( method = 'ffill' ) df [ 'year' ] = df . year . map ( lambda s : str ( s ) [ : 4 ] ) . astype ( int ) # pos -> position if 'pos' in df . columns : df . rename ( columns = { 'pos' : 'position' } , inplace = True ) # boxscore_word, game_date -> boxscore_id and separate into Y, M, D columns for bs_id_col in ( 'boxscore_word' , 'game_date' , 'box_score_text' ) : if bs_id_col in df . columns : df . rename ( columns = { bs_id_col : 'boxscore_id' } , inplace = True ) break # ignore *, +, and other characters used to note things df . replace ( re . compile ( r'[\*\+\u2605]' , re . U ) , '' , inplace = True ) for col in df . columns : if hasattr ( df [ col ] , 'str' ) : df [ col ] = df [ col ] . str . strip ( ) # player -> player_id and/or player_name if 'player' in df . columns : if flatten : df . rename ( columns = { 'player' : 'player_id' } , inplace = True ) # when flattening, keep a column for names player_names = parse_table ( table , flatten = False ) [ 'player_name' ] df [ 'player_name' ] = player_names else : df . rename ( columns = { 'player' : 'player_name' } , inplace = True ) # team, team_name -> team_id for team_col in ( 'team' , 'team_name' ) : if team_col in df . columns : # first, get rid of faulty rows df = df . loc [ ~ df [ team_col ] . isin ( [ 'XXX' ] ) ] if flatten : df . rename ( columns = { team_col : 'team_id' } , inplace = True ) # season -> int if 'season' in df . columns and flatten : df [ 'season' ] = df [ 'season' ] . astype ( int ) # handle date_game columns (different types) if 'date_game' in df . columns and flatten : date_re = r'month=(?P<month>\d+)&day=(?P<day>\d+)&year=(?P<year>\d+)' date_df = df [ 'date_game' ] . str . extract ( date_re , expand = True ) if date_df . notnull ( ) . all ( axis = 1 ) . any ( ) : df = pd . concat ( ( df , date_df ) , axis = 1 ) else : df . rename ( columns = { 'date_game' : 'boxscore_id' } , inplace = True ) # game_location -> is_home if 'game_location' in df . columns and flatten : df [ 'game_location' ] = df [ 'game_location' ] . isnull ( ) df . rename ( columns = { 'game_location' : 'is_home' } , inplace = True ) # mp: (min:sec) -> float(min + sec / 60), notes -> NaN, new column if 'mp' in df . columns and df . dtypes [ 'mp' ] == object and flatten : mp_df = df [ 'mp' ] . str . extract ( r'(?P<m>\d+):(?P<s>\d+)' , expand = True ) . astype ( float ) no_match = mp_df . isnull ( ) . all ( axis = 1 ) if no_match . any ( ) : df . loc [ no_match , 'note' ] = df . loc [ no_match , 'mp' ] df [ 'mp' ] = mp_df [ 'm' ] + mp_df [ 's' ] / 60 # converts number-y things to floats def convert_to_float ( val ) : # percentages: (number%) -> float(number * 0.01) m = re . search ( r'([-\.\d]+)\%' , val if isinstance ( val , basestring ) else str ( val ) , re . U ) try : if m : return float ( m . group ( 1 ) ) / 100 if m else val if m : return int ( m . group ( 1 ) ) + int ( m . group ( 2 ) ) / 60 except ValueError : return val # salaries: $ABC,DEF,GHI -> float(ABCDEFGHI) m = re . search ( r'\$[\d,]+' , val if isinstance ( val , basestring ) else str ( val ) , re . U ) try : if m : return float ( re . sub ( r'\$|,' , '' , val ) ) except Exception : return val # generally try to coerce to float, unless it's an int or bool try : if isinstance ( val , ( int , bool ) ) : return val else : return float ( val ) except Exception : return val if flatten : df = df . applymap ( convert_to_float ) df = df . loc [ df . astype ( bool ) . any ( axis = 1 ) ] return df
Parses a table from sports-reference sites into a pandas dataframe.

    :param table: the PyQuery object representing the HTML table
    :param flatten: if True, flattens relative URLs to IDs. otherwise, leaves
        all fields as text without cleaning.
    :param footer: If True, returns the summary/footer of the page. Recommended
        to use this with flatten=False. Defaults to False.
    :returns: pd.DataFrame
Parses a table from sports - reference sites into a pandas dataframe .