text_extensions_for_pandas/io/watson/tables.py
def make_table_from_exploded_df(exploded_df: pd.DataFrame, row_heading_cols, column_heading_cols, dfs_dict=None,
                                value_col: str = "text", concat_with: str = " | ",
                                convert_numeric_items=False, sort_headers=True) -> pd.DataFrame:
    """
    takes in the exploded dataframe, and converts it into the reconstructed table

    :param exploded_df: The exploded dataframe, as returned by `make_exploded_df`
    :param row_heading_cols: the names of the columns referring to row headings (as outputted from make_exploded_df())
    :param column_heading_cols: the names of the columns referring to column headings
                                (as outputted from make_exploded_df())
    :param value_col: the name of the column to use for the value of each cell. Defaults to 'text'
    :param concat_with: the delimiter to use when concatinating duplicate entries.
                            Using an empty string, "" will fuse entries
    :param dfs_dict: Dictionary parsed from initial step of table reconstruction. Is required to re-order columns into
                     their original format. If not, the reordering will not take place
    :param convert_numeric_items: if True, rows or columns with numeric items will be detected and converted
                                  to floats or ints
    :param sort_headers: If ``True`` the headers will be sorted into their original ordering from the table.
                            Will be a little slower. Note: sorting headers is still experimental on multindex tables
                            where not all headers have the same number of elements

    :return: the reconstructed table. should be a 1:1 translation of original table, but both machine and human readable
    """
    for heading_col in (row_heading_cols + column_heading_cols):
        exploded_df[heading_col].fillna("", inplace=True)
    table = exploded_df.pivot_table(index=row_heading_cols, columns=column_heading_cols, values=value_col,
                                    aggfunc=(lambda a: concat_with.join(a)))

    if type(table) == pd.Series:
        table = table.to_frame()
    row_nones = [None for _ in range((table.index.nlevels))]
    col_nones = [None for _ in range((table.columns.nlevels))]

    table = table.rename_axis(index=row_nones, columns=col_nones)

    if convert_numeric_items:
        num_rows, num_cols = _infer_numeric_rows_cols(exploded_df, row_heading_cols, column_heading_cols)
        table = convert_cols_to_numeric(table, num_cols, num_rows)

    if sort_headers and column_heading_cols != ["column_index"] and len(
            column_heading_cols) == 1 and dfs_dict is not None:
        col_headings = table.columns.to_list()
        cols = dfs_dict["col_headers"][dfs_dict["col_headers"]["text"].isin(col_headings)]
        cols = cols.sort_values("column_index_begin")
        col_headings_sorted = cols["text"].to_list()

        not_inc_headings = [x for x in col_headings if x not in col_headings_sorted]
        col_headings_sorted = col_headings_sorted + not_inc_headings
        table = table[col_headings_sorted]

    elif sort_headers and column_heading_cols != ["column_index"] and len(
            column_heading_cols) > 1 and dfs_dict is not None:
        headers = table.columns.to_frame()
        sorted_headers_arr = _order_multiindex(headers, column_heading_cols, 0, dfs_dict["col_headers"])
        sorted_headers = [tuple(header) for header in sorted_headers_arr]
        not_inc_headings = [x for x in table.columns.to_list() if x not in sorted_headers]
        sorted_headers = sorted_headers + not_inc_headings
        table = table[sorted_headers]

    if sort_headers and row_heading_cols != ["row_index"] and len(row_heading_cols) == 1 and dfs_dict is not None:
        row_headings = table.index.to_list()
        rows = dfs_dict["row_headers"][dfs_dict["row_headers"]["text"].isin(row_headings)]
        rows = rows.sort_values("row_index_begin")
        row_headings_sorted = rows["text"].to_list()

        not_inc_headings = [x for x in row_headings if x not in row_headings_sorted]
        row_headings_sorted = row_headings_sorted + not_inc_headings

        table = table.reindex(row_headings_sorted)

    elif sort_headers and row_heading_cols != ["column_index"] and len(row_heading_cols) > 1 and dfs_dict is not None:
        headers = table.index.to_frame()
        sorted_headers_arr = _order_multiindex(headers, row_heading_cols, 0, dfs_dict["row_headers"])
        sorted_headers = [tuple(header) for header in sorted_headers_arr]

        not_inc_headings = [x for x in table.index.to_list() if x not in sorted_headers]
        sorted_headers = sorted_headers + not_inc_headings
        table = table.reindex(sorted_headers)

    return table

    takes in the exploded dataframe, and converts it into the reconstructed table

    :param exploded_df: The exploded dataframe, as returned by `make_exploded_df`
    :param row_heading_cols: the names of the columns referring to row headings (as outputted from make_exploded_df())
    :param column_heading_cols: the names of the columns referring to column headings
                                (as outputted from make_exploded_df())
    :param value_col: the name of the column to use for the value of each cell. Defaults to 'text'
    :param concat_with: the delimiter to use when concatinating duplicate entries.
                            Using an empty string, "" will fuse entries
    :param dfs_dict: Dictionary parsed from initial step of table reconstruction. Is required to re-order columns into
                     their original format. If not, the reordering will not take place
    :param convert_numeric_items: if True, rows or columns with numeric items will be detected and converted
                                  to floats or ints
    :param sort_headers: If ``True`` the headers will be sorted into their original ordering from the table.
                            Will be a little slower. Note: sorting headers is still experimental on multindex tables
                            where not all headers have the same number of elements

    :return: the reconstructed table. should be a 1:1 translation of original table, but both machine and human readable
    
takes in the exploded dataframe, and converts it into the reconstructed table
takes in the exploded dataframe and converts it into the reconstructed table
def make_table_from_exploded_df(exploded_df: pd.DataFrame, row_heading_cols, column_heading_cols, dfs_dict=None,
                                value_col: str = "text", concat_with: str = " | ",
                                convert_numeric_items=False, sort_headers=True) -> pd.DataFrame:
    for heading_col in (row_heading_cols + column_heading_cols):
        exploded_df[heading_col].fillna("", inplace=True)
    table = exploded_df.pivot_table(index=row_heading_cols, columns=column_heading_cols, values=value_col,
                                    aggfunc=(lambda a: concat_with.join(a)))
    if type(table) == pd.Series:
        table = table.to_frame()
    row_nones = [None for _ in range((table.index.nlevels))]
    col_nones = [None for _ in range((table.columns.nlevels))]
    table = table.rename_axis(index=row_nones, columns=col_nones)
    if convert_numeric_items:
        num_rows, num_cols = _infer_numeric_rows_cols(exploded_df, row_heading_cols, column_heading_cols)
        table = convert_cols_to_numeric(table, num_cols, num_rows)
    if sort_headers and column_heading_cols != ["column_index"] and len(
            column_heading_cols) == 1 and dfs_dict is not None:
        col_headings = table.columns.to_list()
        cols = dfs_dict["col_headers"][dfs_dict["col_headers"]["text"].isin(col_headings)]
        cols = cols.sort_values("column_index_begin")
        col_headings_sorted = cols["text"].to_list()
        not_inc_headings = [x for x in col_headings if x not in col_headings_sorted]
        col_headings_sorted = col_headings_sorted + not_inc_headings
        table = table[col_headings_sorted]
    elif sort_headers and column_heading_cols != ["column_index"] and len(
            column_heading_cols) > 1 and dfs_dict is not None:
        headers = table.columns.to_frame()
        sorted_headers_arr = _order_multiindex(headers, column_heading_cols, 0, dfs_dict["col_headers"])
        sorted_headers = [tuple(header) for header in sorted_headers_arr]
        not_inc_headings = [x for x in table.columns.to_list() if x not in sorted_headers]
        sorted_headers = sorted_headers + not_inc_headings
        table = table[sorted_headers]
    if sort_headers and row_heading_cols != ["row_index"] and len(row_heading_cols) == 1 and dfs_dict is not None:
        row_headings = table.index.to_list()
        rows = dfs_dict["row_headers"][dfs_dict["row_headers"]["text"].isin(row_headings)]
        rows = rows.sort_values("row_index_begin")
        row_headings_sorted = rows["text"].to_list()
        not_inc_headings = [x for x in row_headings if x not in row_headings_sorted]
        row_headings_sorted = row_headings_sorted + not_inc_headings
        table = table.reindex(row_headings_sorted)
    elif sort_headers and row_heading_cols != ["column_index"] and len(row_heading_cols) > 1 and dfs_dict is not None:
        headers = table.index.to_frame()
        sorted_headers_arr = _order_multiindex(headers, row_heading_cols, 0, dfs_dict["row_headers"])
        sorted_headers = [tuple(header) for header in sorted_headers_arr]
        not_inc_headings = [x for x in table.index.to_list() if x not in sorted_headers]
        sorted_headers = sorted_headers + not_inc_headings
        table = table.reindex(sorted_headers)
    return table
def make_table_from_exploded_df ( exploded_df : pd . DataFrame , row_heading_cols , column_heading_cols , dfs_dict = None , value_col : str = "text" , concat_with : str = " | " , convert_numeric_items = False , sort_headers = True ) -> pd . DataFrame : for heading_col in ( row_heading_cols + column_heading_cols ) : exploded_df [ heading_col ] . fillna ( "" , inplace = True ) table = exploded_df . pivot_table ( index = row_heading_cols , columns = column_heading_cols , values = value_col , aggfunc = ( lambda a : concat_with . join ( a ) ) ) if type ( table ) == pd . Series : table = table . to_frame ( ) row_nones = [ None for _ in range ( ( table . index . nlevels ) ) ] col_nones = [ None for _ in range ( ( table . columns . nlevels ) ) ] table = table . rename_axis ( index = row_nones , columns = col_nones ) if convert_numeric_items : num_rows , num_cols = _infer_numeric_rows_cols ( exploded_df , row_heading_cols , column_heading_cols ) table = convert_cols_to_numeric ( table , num_cols , num_rows ) if sort_headers and column_heading_cols != [ "column_index" ] and len ( column_heading_cols ) == 1 and dfs_dict is not None : col_headings = table . columns . to_list ( ) cols = dfs_dict [ "col_headers" ] [ dfs_dict [ "col_headers" ] [ "text" ] . isin ( col_headings ) ] cols = cols . sort_values ( "column_index_begin" ) col_headings_sorted = cols [ "text" ] . to_list ( ) not_inc_headings = [ x for x in col_headings if x not in col_headings_sorted ] col_headings_sorted = col_headings_sorted + not_inc_headings table = table [ col_headings_sorted ] elif sort_headers and column_heading_cols != [ "column_index" ] and len ( column_heading_cols ) > 1 and dfs_dict is not None : headers = table . columns . to_frame ( ) sorted_headers_arr = _order_multiindex ( headers , column_heading_cols , 0 , dfs_dict [ "col_headers" ] ) sorted_headers = [ tuple ( header ) for header in sorted_headers_arr ] not_inc_headings = [ x for x in table . columns . to_list ( ) if x not in sorted_headers ] sorted_headers = sorted_headers + not_inc_headings table = table [ sorted_headers ] if sort_headers and row_heading_cols != [ "row_index" ] and len ( row_heading_cols ) == 1 and dfs_dict is not None : row_headings = table . index . to_list ( ) rows = dfs_dict [ "row_headers" ] [ dfs_dict [ "row_headers" ] [ "text" ] . isin ( row_headings ) ] rows = rows . sort_values ( "row_index_begin" ) row_headings_sorted = rows [ "text" ] . to_list ( ) not_inc_headings = [ x for x in row_headings if x not in row_headings_sorted ] row_headings_sorted = row_headings_sorted + not_inc_headings table = table . reindex ( row_headings_sorted ) elif sort_headers and row_heading_cols != [ "column_index" ] and len ( row_heading_cols ) > 1 and dfs_dict is not None : headers = table . index . to_frame ( ) sorted_headers_arr = _order_multiindex ( headers , row_heading_cols , 0 , dfs_dict [ "row_headers" ] ) sorted_headers = [ tuple ( header ) for header in sorted_headers_arr ] not_inc_headings = [ x for x in table . index . to_list ( ) if x not in sorted_headers ] sorted_headers = sorted_headers + not_inc_headings table = table . reindex ( sorted_headers ) return table
takes in the exploded dataframe, and converts it into the reconstructed table
takes in the exploded dataframe and converts it into the reconstructed table
"""
    takes in the exploded dataframe, and converts it into the reconstructed table

    :param exploded_df: The exploded dataframe, as returned by `make_exploded_df`
    :param row_heading_cols: the names of the columns referring to row headings (as outputted from make_exploded_df())
    :param column_heading_cols: the names of the columns referring to column headings
                                (as outputted from make_exploded_df())
    :param value_col: the name of the column to use for the value of each cell. Defaults to 'text'
    :param concat_with: the delimiter to use when concatinating duplicate entries.
                            Using an empty string, "" will fuse entries
    :param dfs_dict: Dictionary parsed from initial step of table reconstruction. Is required to re-order columns into
                     their original format. If not, the reordering will not take place
    :param convert_numeric_items: if True, rows or columns with numeric items will be detected and converted
                                  to floats or ints
    :param sort_headers: If ``True`` the headers will be sorted into their original ordering from the table.
                            Will be a little slower. Note: sorting headers is still experimental on multindex tables
                            where not all headers have the same number of elements

    :return: the reconstructed table. should be a 1:1 translation of original table, but both machine and human readable
    """
the reconstructed table. should be a 1:1 translation of original table, but both machine and human readable
the reconstructed table . should be a 1 : 1 translation of original table but both machine and human readable
exploded_df
pd.DataFrame
The exploded dataframe, as returned by `make_exploded_df`
The exploded dataframe as returned by ` make_exploded_df `
row_heading_cols
the names of the columns referring to row headings (as outputted from make_exploded_df())
the names of the columns referring to row headings ( as outputted from make_exploded_df () )
column_heading_cols
the names of the columns referring to column headings
(as outputted from make_exploded_df())
the names of the columns referring to column headings ( as outputted from make_exploded_df () )
dfs_dict
Dictionary parsed from initial step of table reconstruction. Is required to re-order columns into
their original format. If not, the reordering will not take place
Dictionary parsed from initial step of table reconstruction . Is required to re - order columns into their original format . If not the reordering will not take place
value_col
str
the name of the column to use for the value of each cell. Defaults to 'text'
the name of the column to use for the value of each cell . Defaults to ' text '
concat_with
str
the delimiter to use when concatinating duplicate entries.
Using an empty string, "" will fuse entries
the delimiter to use when concatinating duplicate entries . Using an empty string " " will fuse entries
convert_numeric_items
if True, rows or columns with numeric items will be detected and converted
to floats or ints
if True rows or columns with numeric items will be detected and converted to floats or ints
sort_headers
If ``True`` the headers will be sorted into their original ordering from the table.
Will be a little slower. Note: sorting headers is still experimental on multindex tables
where not all headers have the same number of elements
If ` ` True ` ` the headers will be sorted into their original ordering from the table . Will be a little slower . Note : sorting headers is still experimental on multindex tables where not all headers have the same number of elements