plot_combo.py
def print_table(args):
    """Use this for the broad overview table in the paper, showing convergence results.

    Careful: we use some braces, so could be tricky to integrate with string formatting?
    Remember that for line breaks we need an escape character for \, so \\.
    NOTE: Put this between \toprule and \bottomrule commands in LaTeX for tables.
    """
    s = ''

    # Let's just use a fixed orering. Adjust here if needed. This is the task name IN CODE.
    tasks_l = ['cable-ring', 'cable-ring-notarget', 'cable-shape', 'cloth-cover',
               'cloth-flat', 'bag-alone-open', 'bag-items-easy', 'bag-items-hard',
               'cable-line-notarget', 'cable-shape-notarget', 'cloth-flat-notarget', 'bag-color-goal',]

    # This has the same ordering as above but with fixed names (e.g., cloth --> fabric).
    T = []
    for t in tasks_l:
        if t in REPLACE_TASK:
            T.append(REPLACE_TASK[t])
        else:
            T.append(t)

    # Map from (name_task, num_demos) --> statistic.
    a1 = defaultdict(list)
    a2 = defaultdict(list)
    a3 = defaultdict(list)
    a4 = defaultdict(list)
    a5 = defaultdict(list)
    a6 = defaultdict(list)
    a7 = defaultdict(list)

    # For each task, get relevant directories for each demo, then relevant stats.
    for name_task in tasks_l:
        for num_demos in [1, 10, 100, 1000]:
            directories, seeds = get_subdirectories_demos(name_task, num_demos)
            stats_combo = _get_results(name_task, directories, seeds, debug)
            agents = sorted(stats_combo.keys())
            print(f'For task {name_task}, we have results from agents: {agents}')

            # For now, take the max over the iterations (easy to spot check). Take maximum.
            def get_max(_name_task, ag_key):
                stat_max = -1
                for key in sorted(stats_combo[ag_key].keys()):
                    if (len(stats_combo[ag_key][key]) != 60 and name_task != 'bag-color-goal' and key != 0):
                        print(f'\tWarning, len(stats_combo[ag_key][key]): {len(stats_combo[ag_key][key])} for: {ag_key}, {key}')
                    elif len(stats_combo[ag_key][key]) != 20 and name_task == 'bag-color-goal':
                        print(f'\tWarning, len(stats_combo[ag_key][key]): {len(stats_combo[ag_key][key])} for: {ag_key}, {key}')
                    stat_max = max(stat_max, np.mean(stats_combo[ag_key][key]))
                stat_max *= 100
                if _name_task in REPLACE_TASK:
                    return (REPLACE_TASK[_name_task], stat_max)
                else:
                    return (_name_task, stat_max)

            if '-transporter' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-transporter')
                a1[i1].append(i2)
            if '-transporter-goal' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-transporter-goal')
                a2[i1].append(i2)
            if '-transporter-goal-naive' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-transporter-goal-naive')
                a3[i1].append(i2)
            if '-gt_state' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-gt_state')
                a4[i1].append(i2)
            if '-gt_state_2_step' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-gt_state_2_step')
                a5[i1].append(i2)
            if '-conv_mlp' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-gt_state_2_step')
                a6[i1].append(i2)
            if '-transporter-goal-snaive' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-transporter-goal-snaive')
                a7[i1].append(i2)

    print('\nDebugging of which results we have:')
    print(f'keys in a1 (transporter):              {sorted(a1.keys())}')
    print(f'keys in a2 (transporter-goal):         {sorted(a2.keys())}')
    print(f'keys in a3 (transporter-goal-naive):   {sorted(a3.keys())}')
    print(f'keys in a4 (gt_state):                 {sorted(a4.keys())}')
    print(f'keys in a5 (gt_state_2_step):          {sorted(a5.keys())}')
    print(f'keys in a6 (conv_mlp):                 {sorted(a6.keys())}')
    print(f'keys in a7 (transporter-goal-S-naive): {sorted(a7.keys())}')
    NA = 'N/A'

    # Use a1 through a5 default dicts. Keys are tasks in `T` list.
    # For any valid task `i`, we have `a1[T[i]]` a list of length 4, for the 4 demo types.
    # Therefore the LAST index should range from: [0], [1], [2], to [3].
    # If any lists are not of length 4, we should not use them, or simply fill in a value of -1?
    # This will be invalid for some but hopefully those can be easily inspected.

    s += f'\n & \multicolumn{{4}}{{c}}{{{T[0]}}} & \multicolumn{{4}}{{c}}{{{T[1]}}} & \multicolumn{{4}}{{c}}{{{T[2]}}} & \multicolumn{{4}}{{c}}{{{T[3]}}}  \\\\'
    s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}'
    s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 \\\\'
    s += '\n \midrule'
    s += (f'\n GT-State MLP            & {a4[ T[0] ][0]:0.1f} & {a4[ T[0] ][1]:0.1f} & {a4[ T[0] ][2]:0.1f} & {a4[ T[0] ][3]:0.1f} '  # NOTE! Everything in this row must be 'a4', etc.
                                     f'& {a4[ T[1] ][0]:0.1f} & {a4[ T[1] ][1]:0.1f} & {a4[ T[1] ][2]:0.1f} & {a4[ T[1] ][3]:0.1f} '
                                     f'& {a4[ T[2] ][0]:0.1f} & {a4[ T[2] ][1]:0.1f} & {a4[ T[2] ][2]:0.1f} & {a4[ T[2] ][3]:0.1f} '
                                     f'& {a4[ T[3] ][0]:0.1f} & {a4[ T[3] ][1]:0.1f} & {a4[ T[3] ][2]:0.1f} & {a4[ T[3] ][3]:0.1f}  \\\\')

    s += (f'\n GT-State MLP 2-Step     & {a5[ T[0] ][0]:0.1f} & {a5[ T[0] ][1]:0.1f} & {a5[ T[0] ][2]:0.1f} & {a5[ T[0] ][3]:0.1f} '
                                     f'& {a5[ T[1] ][0]:0.1f} & {a5[ T[1] ][1]:0.1f} & {a5[ T[1] ][2]:0.1f} & {a5[ T[1] ][3]:0.1f} '
                                     f'& {a5[ T[2] ][0]:0.1f} & {a5[ T[2] ][1]:0.1f} & {a5[ T[2] ][2]:0.1f} & {a5[ T[2] ][3]:0.1f} '
                                     f'& {a5[ T[3] ][0]:0.1f} & {a5[ T[3] ][1]:0.1f} & {a5[ T[3] ][2]:0.1f} & {a5[ T[3] ][3]:0.1f}  \\\\')

    s += (f'\n Transporter             & {a1[ T[0] ][0]:0.1f} & {a1[ T[0] ][1]:0.1f} & {a1[ T[0] ][2]:0.1f} & {a1[ T[0] ][3]:0.1f} '  # NOTE! Everything in this row must be 'a1', etc.
                                     f'& {a1[ T[1] ][0]:0.1f} & {a1[ T[1] ][1]:0.1f} & {a1[ T[1] ][2]:0.1f} & {a1[ T[1] ][3]:0.1f} '
                                     f'& {a1[ T[2] ][0]:0.1f} & {a1[ T[2] ][1]:0.1f} & {a1[ T[2] ][2]:0.1f} & {a1[ T[2] ][3]:0.1f} '
                                     f'& {a1[ T[3] ][0]:0.1f} & {a1[ T[3] ][1]:0.1f} & {a1[ T[3] ][2]:0.1f} & {a1[ T[3] ][3]:0.1f}  \\\\')
    s += '\n \midrule'
    s += f'\n & \multicolumn{{4}}{{c}}{{{T[4]}}} & \multicolumn{{4}}{{c}}{{{T[5]}}} & \multicolumn{{4}}{{c}}{{{T[6]}}} & \multicolumn{{4}}{{c}}{{{T[7]}}}  \\\\'
    s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}'
    s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 \\\\'
    s += '\n \midrule'
    s += (f'\n GT-State MLP            & {a4[ T[4] ][0]:0.1f} & {a4[ T[4] ][1]:0.1f} & {a4[ T[4] ][2]:0.1f} & {a4[ T[4] ][3]:0.1f} '  # NOTE! Everything in this row must be 'a4', etc.
                                     f'& {a4[ T[5] ][0]:0.1f} & {a4[ T[5] ][1]:0.1f} & {a4[ T[5] ][2]:0.1f} & {a4[ T[5] ][3]:0.1f} '
                                     f'& {a4[ T[6] ][0]:0.1f} & {a4[ T[6] ][1]:0.1f} & {a4[ T[6] ][2]:0.1f} & {a4[ T[6] ][3]:0.1f} '
                                     f'& {a4[ T[7] ][0]:0.1f} & {a4[ T[7] ][1]:0.1f} & {a4[ T[7] ][2]:0.1f} & {a4[ T[7] ][3]:0.1f}  \\\\')

    s += (f'\n GT-State MLP 2-Step     & {a5[ T[4] ][0]:0.1f} & {a5[ T[4] ][1]:0.1f} & {a5[ T[4] ][2]:0.1f} & {a5[ T[4] ][3]:0.1f} '  # NOTE! Everything in this row must be 'a5', etc.
                                     f'& {a5[ T[5] ][0]:0.1f} & {a5[ T[5] ][1]:0.1f} & {a5[ T[5] ][2]:0.1f} & {a5[ T[5] ][3]:0.1f} '
                                     f'& {a5[ T[6] ][0]:0.1f} & {a5[ T[6] ][1]:0.1f} & {a5[ T[6] ][2]:0.1f} & {a5[ T[6] ][3]:0.1f} '
                                     f'& {a5[ T[7] ][0]:0.1f} & {a5[ T[7] ][1]:0.1f} & {a5[ T[7] ][2]:0.1f} & {a5[ T[7] ][3]:0.1f}  \\\\')

    s += (f'\n Transporter             & {a1[ T[4] ][0]:0.1f} & {a1[ T[4] ][1]:0.1f} & {a1[ T[4] ][2]:0.1f} & {a1[ T[4] ][3]:0.1f} '  # NOTE! Everything in this row must be 'a1', etc.
                                     f'& {a1[ T[5] ][0]:0.1f} & {a1[ T[5] ][1]:0.1f} & {a1[ T[5] ][2]:0.1f} & {a1[ T[5] ][3]:0.1f} '
                                     f'& {a1[ T[6] ][0]:0.1f} & {a1[ T[6] ][1]:0.1f} & {a1[ T[6] ][2]:0.1f} & {a1[ T[6] ][3]:0.1f} '
                                     f'& {a1[ T[7] ][0]:0.1f} & {a1[ T[7] ][1]:0.1f} & {a1[ T[7] ][2]:0.1f} & {a1[ T[7] ][3]:0.1f}  \\\\')
    s += '\n \midrule'
    s += f'\n & \multicolumn{{4}}{{c}}{{{T[8]}}} & \multicolumn{{4}}{{c}}{{{T[9]}}} & \multicolumn{{4}}{{c}}{{{T[10]}}} & \multicolumn{{4}}{{c}}{{{T[11]}}}  \\\\'
    s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}'
    s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000  \\\\'
    s += '\n \midrule'
    s += (f'\n GT-State MLP            & {a4[ T[8] ][0]:0.1f} & {a4[ T[8] ][1]:0.1f} & {a4[ T[8] ][2]:0.1f} & {a4[ T[8] ][3]:0.1f} '
                                     f'& {a4[ T[9] ][0]:0.1f} & {a4[ T[9] ][1]:0.1f} & {a4[ T[9] ][2]:0.1f} & {a4[ T[9] ][3]:0.1f} '
                                     f'& {a4[ T[10]][0]:0.1f} & {a4[ T[10]][1]:0.1f} & {a4[ T[10]][2]:0.1f} & {a4[ T[10]][3]:0.1f} '
                                     f'& {a4[ T[11]][0]:0.1f} & {a4[ T[11]][1]:0.1f} & {a4[ T[11]][2]:0.1f} & {a4[ T[11]][3]:0.1f}  \\\\')

    s += (f'\n GT-State MLP 2-Step     & {a5[ T[8] ][0]:0.1f} & {a5[ T[8] ][1]:0.1f} & {a5[ T[8] ][2]:0.1f} & {a5[ T[8] ][3]:0.1f} '
                                     f'& {a5[ T[9] ][0]:0.1f} & {a5[ T[9] ][1]:0.1f} & {a5[ T[9] ][2]:0.1f} & {a5[ T[9] ][3]:0.1f} '
                                     f'& {a5[ T[10]][0]:0.1f} & {a5[ T[10]][1]:0.1f} & {a5[ T[10]][2]:0.1f} & {a5[ T[10]][3]:0.1f} '
                                     f'& {a5[ T[11]][0]:0.1f} & {a5[ T[11]][1]:0.1f} & {a5[ T[11]][2]:0.1f} & {a5[ T[11]][3]:0.1f}  \\\\')

    s += (f'\n Transporter-Goal-Stack  & {a7[ T[8] ][0]:0.1f} & {a7[ T[8] ][1]:0.1f} & {a7[ T[8] ][2]:0.1f} & {a7[ T[8] ][3]:0.1f} '  # NOTE! Everything in this row must be 'a3'. (Update: actually a7)
                                     f'& {a7[ T[9] ][0]:0.1f} & {a7[ T[9] ][1]:0.1f} & {a7[ T[9] ][2]:0.1f} & {a7[ T[9] ][3]:0.1f} '
                                     f'& {a7[ T[10]][0]:0.1f} & {a7[ T[10]][1]:0.1f} & {a7[ T[10]][2]:0.1f} & {a7[ T[10]][3]:0.1f} '  # If insertion-goal, then keep last columns at a3
                                     f'& {a7[ T[11]][0]:0.1f} & {a7[ T[11]][1]:0.1f} & {a7[ T[11]][2]:0.1f} & {a7[ T[11]][3]:0.1f}  \\\\')

    s += (f'\n Transporter-Goal-Split  & {a2[ T[8] ][0]:0.1f} & {a2[ T[8] ][1]:0.1f} & {a2[ T[8] ][2]:0.1f} & {a2[ T[8] ][3]:0.1f} '  # NOTE! Everything in this row must be 'a2', etc.
                                     f'& {a2[ T[9] ][0]:0.1f} & {a2[ T[9] ][1]:0.1f} & {a2[ T[9] ][2]:0.1f} & {a2[ T[9] ][3]:0.1f} '
                                     f'& {a2[ T[10]][0]:0.1f} & {a2[ T[10]][1]:0.1f} & {a2[ T[10]][2]:0.1f} & {a2[ T[10]][3]:0.1f} '
                                     f'& {a2[ T[11]][0]:0.1f} & {a2[ T[11]][1]:0.1f} & {a2[ T[11]][2]:0.1f} & {a2[ T[11]][3]:0.1f}  \\\\')
    print(s)
Use this for the broad overview table in the paper, showing convergence results.

    Careful: we use some braces, so could be tricky to integrate with string formatting?
    Remember that for line breaks we need an escape character for \, so \\.
    NOTE: Put this between \toprule and \bottomrule commands in LaTeX for tables.
    
Use this for the broad overview table in the paper, showing convergence results.
Careful: we use some braces, so could be tricky to integrate with string formatting.
Remember that for line breaks we need an escape character for \, so \\.
NOTE: Put this between \toprule and \bottomrule commands in LaTeX for tables.
Use this for the broad overview table in the paper showing convergence results . Careful : we use some braces so could be tricky to integrate with string formatting . Remember that for line breaks we need an escape character for \ so \\ . NOTE : Put this between \ toprule and \ bottomrule commands in LaTeX for tables .
def print_table(args):
    s = ''
    tasks_l = ['cable-ring', 'cable-ring-notarget', 'cable-shape', 'cloth-cover',
               'cloth-flat', 'bag-alone-open', 'bag-items-easy', 'bag-items-hard',
               'cable-line-notarget', 'cable-shape-notarget', 'cloth-flat-notarget', 'bag-color-goal',]
    T = []
    for t in tasks_l:
        if t in REPLACE_TASK:
            T.append(REPLACE_TASK[t])
        else:
            T.append(t)
    a1 = defaultdict(list)
    a2 = defaultdict(list)
    a3 = defaultdict(list)
    a4 = defaultdict(list)
    a5 = defaultdict(list)
    a6 = defaultdict(list)
    a7 = defaultdict(list)
    for name_task in tasks_l:
        for num_demos in [1, 10, 100, 1000]:
            directories, seeds = get_subdirectories_demos(name_task, num_demos)
            stats_combo = _get_results(name_task, directories, seeds, debug)
            agents = sorted(stats_combo.keys())
            print(f'For task {name_task}, we have results from agents: {agents}')
            def get_max(_name_task, ag_key):
                stat_max = -1
                for key in sorted(stats_combo[ag_key].keys()):
                    if (len(stats_combo[ag_key][key]) != 60 and name_task != 'bag-color-goal' and key != 0):
                        print(f'\tWarning, len(stats_combo[ag_key][key]): {len(stats_combo[ag_key][key])} for: {ag_key}, {key}')
                    elif len(stats_combo[ag_key][key]) != 20 and name_task == 'bag-color-goal':
                        print(f'\tWarning, len(stats_combo[ag_key][key]): {len(stats_combo[ag_key][key])} for: {ag_key}, {key}')
                    stat_max = max(stat_max, np.mean(stats_combo[ag_key][key]))
                stat_max *= 100
                if _name_task in REPLACE_TASK:
                    return (REPLACE_TASK[_name_task], stat_max)
                else:
                    return (_name_task, stat_max)
            if '-transporter' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-transporter')
                a1[i1].append(i2)
            if '-transporter-goal' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-transporter-goal')
                a2[i1].append(i2)
            if '-transporter-goal-naive' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-transporter-goal-naive')
                a3[i1].append(i2)
            if '-gt_state' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-gt_state')
                a4[i1].append(i2)
            if '-gt_state_2_step' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-gt_state_2_step')
                a5[i1].append(i2)
            if '-conv_mlp' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-gt_state_2_step')
                a6[i1].append(i2)
            if '-transporter-goal-snaive' in agents:
                i1, i2 = get_max(_name_task=name_task, ag_key='-transporter-goal-snaive')
                a7[i1].append(i2)
    print('\nDebugging of which results we have:')
    print(f'keys in a1 (transporter):              {sorted(a1.keys())}')
    print(f'keys in a2 (transporter-goal):         {sorted(a2.keys())}')
    print(f'keys in a3 (transporter-goal-naive):   {sorted(a3.keys())}')
    print(f'keys in a4 (gt_state):                 {sorted(a4.keys())}')
    print(f'keys in a5 (gt_state_2_step):          {sorted(a5.keys())}')
    print(f'keys in a6 (conv_mlp):                 {sorted(a6.keys())}')
    print(f'keys in a7 (transporter-goal-S-naive): {sorted(a7.keys())}')
    NA = 'N/A'
    s += f'\n & \multicolumn{{4}}{{c}}{{{T[0]}}} & \multicolumn{{4}}{{c}}{{{T[1]}}} & \multicolumn{{4}}{{c}}{{{T[2]}}} & \multicolumn{{4}}{{c}}{{{T[3]}}}  \\\\'
    s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}'
    s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 \\\\'
    s += '\n \midrule'
    s += (f'\n GT-State MLP            & {a4[ T[0] ][0]:0.1f} & {a4[ T[0] ][1]:0.1f} & {a4[ T[0] ][2]:0.1f} & {a4[ T[0] ][3]:0.1f} '  
                                     f'& {a4[ T[1] ][0]:0.1f} & {a4[ T[1] ][1]:0.1f} & {a4[ T[1] ][2]:0.1f} & {a4[ T[1] ][3]:0.1f} '
                                     f'& {a4[ T[2] ][0]:0.1f} & {a4[ T[2] ][1]:0.1f} & {a4[ T[2] ][2]:0.1f} & {a4[ T[2] ][3]:0.1f} '
                                     f'& {a4[ T[3] ][0]:0.1f} & {a4[ T[3] ][1]:0.1f} & {a4[ T[3] ][2]:0.1f} & {a4[ T[3] ][3]:0.1f}  \\\\')
    s += (f'\n GT-State MLP 2-Step     & {a5[ T[0] ][0]:0.1f} & {a5[ T[0] ][1]:0.1f} & {a5[ T[0] ][2]:0.1f} & {a5[ T[0] ][3]:0.1f} '
                                     f'& {a5[ T[1] ][0]:0.1f} & {a5[ T[1] ][1]:0.1f} & {a5[ T[1] ][2]:0.1f} & {a5[ T[1] ][3]:0.1f} '
                                     f'& {a5[ T[2] ][0]:0.1f} & {a5[ T[2] ][1]:0.1f} & {a5[ T[2] ][2]:0.1f} & {a5[ T[2] ][3]:0.1f} '
                                     f'& {a5[ T[3] ][0]:0.1f} & {a5[ T[3] ][1]:0.1f} & {a5[ T[3] ][2]:0.1f} & {a5[ T[3] ][3]:0.1f}  \\\\')
    s += (f'\n Transporter             & {a1[ T[0] ][0]:0.1f} & {a1[ T[0] ][1]:0.1f} & {a1[ T[0] ][2]:0.1f} & {a1[ T[0] ][3]:0.1f} '  
                                     f'& {a1[ T[1] ][0]:0.1f} & {a1[ T[1] ][1]:0.1f} & {a1[ T[1] ][2]:0.1f} & {a1[ T[1] ][3]:0.1f} '
                                     f'& {a1[ T[2] ][0]:0.1f} & {a1[ T[2] ][1]:0.1f} & {a1[ T[2] ][2]:0.1f} & {a1[ T[2] ][3]:0.1f} '
                                     f'& {a1[ T[3] ][0]:0.1f} & {a1[ T[3] ][1]:0.1f} & {a1[ T[3] ][2]:0.1f} & {a1[ T[3] ][3]:0.1f}  \\\\')
    s += '\n \midrule'
    s += f'\n & \multicolumn{{4}}{{c}}{{{T[4]}}} & \multicolumn{{4}}{{c}}{{{T[5]}}} & \multicolumn{{4}}{{c}}{{{T[6]}}} & \multicolumn{{4}}{{c}}{{{T[7]}}}  \\\\'
    s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}'
    s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 \\\\'
    s += '\n \midrule'
    s += (f'\n GT-State MLP            & {a4[ T[4] ][0]:0.1f} & {a4[ T[4] ][1]:0.1f} & {a4[ T[4] ][2]:0.1f} & {a4[ T[4] ][3]:0.1f} '  
                                     f'& {a4[ T[5] ][0]:0.1f} & {a4[ T[5] ][1]:0.1f} & {a4[ T[5] ][2]:0.1f} & {a4[ T[5] ][3]:0.1f} '
                                     f'& {a4[ T[6] ][0]:0.1f} & {a4[ T[6] ][1]:0.1f} & {a4[ T[6] ][2]:0.1f} & {a4[ T[6] ][3]:0.1f} '
                                     f'& {a4[ T[7] ][0]:0.1f} & {a4[ T[7] ][1]:0.1f} & {a4[ T[7] ][2]:0.1f} & {a4[ T[7] ][3]:0.1f}  \\\\')
    s += (f'\n GT-State MLP 2-Step     & {a5[ T[4] ][0]:0.1f} & {a5[ T[4] ][1]:0.1f} & {a5[ T[4] ][2]:0.1f} & {a5[ T[4] ][3]:0.1f} '  
                                     f'& {a5[ T[5] ][0]:0.1f} & {a5[ T[5] ][1]:0.1f} & {a5[ T[5] ][2]:0.1f} & {a5[ T[5] ][3]:0.1f} '
                                     f'& {a5[ T[6] ][0]:0.1f} & {a5[ T[6] ][1]:0.1f} & {a5[ T[6] ][2]:0.1f} & {a5[ T[6] ][3]:0.1f} '
                                     f'& {a5[ T[7] ][0]:0.1f} & {a5[ T[7] ][1]:0.1f} & {a5[ T[7] ][2]:0.1f} & {a5[ T[7] ][3]:0.1f}  \\\\')
    s += (f'\n Transporter             & {a1[ T[4] ][0]:0.1f} & {a1[ T[4] ][1]:0.1f} & {a1[ T[4] ][2]:0.1f} & {a1[ T[4] ][3]:0.1f} '  
                                     f'& {a1[ T[5] ][0]:0.1f} & {a1[ T[5] ][1]:0.1f} & {a1[ T[5] ][2]:0.1f} & {a1[ T[5] ][3]:0.1f} '
                                     f'& {a1[ T[6] ][0]:0.1f} & {a1[ T[6] ][1]:0.1f} & {a1[ T[6] ][2]:0.1f} & {a1[ T[6] ][3]:0.1f} '
                                     f'& {a1[ T[7] ][0]:0.1f} & {a1[ T[7] ][1]:0.1f} & {a1[ T[7] ][2]:0.1f} & {a1[ T[7] ][3]:0.1f}  \\\\')
    s += '\n \midrule'
    s += f'\n & \multicolumn{{4}}{{c}}{{{T[8]}}} & \multicolumn{{4}}{{c}}{{{T[9]}}} & \multicolumn{{4}}{{c}}{{{T[10]}}} & \multicolumn{{4}}{{c}}{{{T[11]}}}  \\\\'
    s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}'
    s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000  \\\\'
    s += '\n \midrule'
    s += (f'\n GT-State MLP            & {a4[ T[8] ][0]:0.1f} & {a4[ T[8] ][1]:0.1f} & {a4[ T[8] ][2]:0.1f} & {a4[ T[8] ][3]:0.1f} '
                                     f'& {a4[ T[9] ][0]:0.1f} & {a4[ T[9] ][1]:0.1f} & {a4[ T[9] ][2]:0.1f} & {a4[ T[9] ][3]:0.1f} '
                                     f'& {a4[ T[10]][0]:0.1f} & {a4[ T[10]][1]:0.1f} & {a4[ T[10]][2]:0.1f} & {a4[ T[10]][3]:0.1f} '
                                     f'& {a4[ T[11]][0]:0.1f} & {a4[ T[11]][1]:0.1f} & {a4[ T[11]][2]:0.1f} & {a4[ T[11]][3]:0.1f}  \\\\')
    s += (f'\n GT-State MLP 2-Step     & {a5[ T[8] ][0]:0.1f} & {a5[ T[8] ][1]:0.1f} & {a5[ T[8] ][2]:0.1f} & {a5[ T[8] ][3]:0.1f} '
                                     f'& {a5[ T[9] ][0]:0.1f} & {a5[ T[9] ][1]:0.1f} & {a5[ T[9] ][2]:0.1f} & {a5[ T[9] ][3]:0.1f} '
                                     f'& {a5[ T[10]][0]:0.1f} & {a5[ T[10]][1]:0.1f} & {a5[ T[10]][2]:0.1f} & {a5[ T[10]][3]:0.1f} '
                                     f'& {a5[ T[11]][0]:0.1f} & {a5[ T[11]][1]:0.1f} & {a5[ T[11]][2]:0.1f} & {a5[ T[11]][3]:0.1f}  \\\\')
    s += (f'\n Transporter-Goal-Stack  & {a7[ T[8] ][0]:0.1f} & {a7[ T[8] ][1]:0.1f} & {a7[ T[8] ][2]:0.1f} & {a7[ T[8] ][3]:0.1f} '  
                                     f'& {a7[ T[9] ][0]:0.1f} & {a7[ T[9] ][1]:0.1f} & {a7[ T[9] ][2]:0.1f} & {a7[ T[9] ][3]:0.1f} '
                                     f'& {a7[ T[10]][0]:0.1f} & {a7[ T[10]][1]:0.1f} & {a7[ T[10]][2]:0.1f} & {a7[ T[10]][3]:0.1f} '  
                                     f'& {a7[ T[11]][0]:0.1f} & {a7[ T[11]][1]:0.1f} & {a7[ T[11]][2]:0.1f} & {a7[ T[11]][3]:0.1f}  \\\\')
    s += (f'\n Transporter-Goal-Split  & {a2[ T[8] ][0]:0.1f} & {a2[ T[8] ][1]:0.1f} & {a2[ T[8] ][2]:0.1f} & {a2[ T[8] ][3]:0.1f} '  
                                     f'& {a2[ T[9] ][0]:0.1f} & {a2[ T[9] ][1]:0.1f} & {a2[ T[9] ][2]:0.1f} & {a2[ T[9] ][3]:0.1f} '
                                     f'& {a2[ T[10]][0]:0.1f} & {a2[ T[10]][1]:0.1f} & {a2[ T[10]][2]:0.1f} & {a2[ T[10]][3]:0.1f} '
                                     f'& {a2[ T[11]][0]:0.1f} & {a2[ T[11]][1]:0.1f} & {a2[ T[11]][2]:0.1f} & {a2[ T[11]][3]:0.1f}  \\\\')
    print(s)
def print_table ( args ) : s = '' tasks_l = [ 'cable-ring' , 'cable-ring-notarget' , 'cable-shape' , 'cloth-cover' , 'cloth-flat' , 'bag-alone-open' , 'bag-items-easy' , 'bag-items-hard' , 'cable-line-notarget' , 'cable-shape-notarget' , 'cloth-flat-notarget' , 'bag-color-goal' , ] T = [ ] for t in tasks_l : if t in REPLACE_TASK : T . append ( REPLACE_TASK [ t ] ) else : T . append ( t ) a1 = defaultdict ( list ) a2 = defaultdict ( list ) a3 = defaultdict ( list ) a4 = defaultdict ( list ) a5 = defaultdict ( list ) a6 = defaultdict ( list ) a7 = defaultdict ( list ) for name_task in tasks_l : for num_demos in [ 1 , 10 , 100 , 1000 ] : directories , seeds = get_subdirectories_demos ( name_task , num_demos ) stats_combo = _get_results ( name_task , directories , seeds , debug ) agents = sorted ( stats_combo . keys ( ) ) print ( f'For task {name_task}, we have results from agents: {agents}' ) def get_max ( _name_task , ag_key ) : stat_max = - 1 for key in sorted ( stats_combo [ ag_key ] . keys ( ) ) : if ( len ( stats_combo [ ag_key ] [ key ] ) != 60 and name_task != 'bag-color-goal' and key != 0 ) : print ( f'\tWarning, len(stats_combo[ag_key][key]): {len(stats_combo[ag_key][key])} for: {ag_key}, {key}' ) elif len ( stats_combo [ ag_key ] [ key ] ) != 20 and name_task == 'bag-color-goal' : print ( f'\tWarning, len(stats_combo[ag_key][key]): {len(stats_combo[ag_key][key])} for: {ag_key}, {key}' ) stat_max = max ( stat_max , np . mean ( stats_combo [ ag_key ] [ key ] ) ) stat_max *= 100 if _name_task in REPLACE_TASK : return ( REPLACE_TASK [ _name_task ] , stat_max ) else : return ( _name_task , stat_max ) if '-transporter' in agents : i1 , i2 = get_max ( _name_task = name_task , ag_key = '-transporter' ) a1 [ i1 ] . append ( i2 ) if '-transporter-goal' in agents : i1 , i2 = get_max ( _name_task = name_task , ag_key = '-transporter-goal' ) a2 [ i1 ] . append ( i2 ) if '-transporter-goal-naive' in agents : i1 , i2 = get_max ( _name_task = name_task , ag_key = '-transporter-goal-naive' ) a3 [ i1 ] . append ( i2 ) if '-gt_state' in agents : i1 , i2 = get_max ( _name_task = name_task , ag_key = '-gt_state' ) a4 [ i1 ] . append ( i2 ) if '-gt_state_2_step' in agents : i1 , i2 = get_max ( _name_task = name_task , ag_key = '-gt_state_2_step' ) a5 [ i1 ] . append ( i2 ) if '-conv_mlp' in agents : i1 , i2 = get_max ( _name_task = name_task , ag_key = '-gt_state_2_step' ) a6 [ i1 ] . append ( i2 ) if '-transporter-goal-snaive' in agents : i1 , i2 = get_max ( _name_task = name_task , ag_key = '-transporter-goal-snaive' ) a7 [ i1 ] . append ( i2 ) print ( '\nDebugging of which results we have:' ) print ( f'keys in a1 (transporter):              {sorted(a1.keys())}' ) print ( f'keys in a2 (transporter-goal):         {sorted(a2.keys())}' ) print ( f'keys in a3 (transporter-goal-naive):   {sorted(a3.keys())}' ) print ( f'keys in a4 (gt_state):                 {sorted(a4.keys())}' ) print ( f'keys in a5 (gt_state_2_step):          {sorted(a5.keys())}' ) print ( f'keys in a6 (conv_mlp):                 {sorted(a6.keys())}' ) print ( f'keys in a7 (transporter-goal-S-naive): {sorted(a7.keys())}' ) NA = 'N/A' s += f'\n & \multicolumn{{4}}{{c}}{{{T[0]}}} & \multicolumn{{4}}{{c}}{{{T[1]}}} & \multicolumn{{4}}{{c}}{{{T[2]}}} & \multicolumn{{4}}{{c}}{{{T[3]}}}  \\\\' s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}' s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 \\\\' s += '\n \midrule' s += ( f'\n GT-State MLP            & {a4[ T[0] ][0]:0.1f} & {a4[ T[0] ][1]:0.1f} & {a4[ T[0] ][2]:0.1f} & {a4[ T[0] ][3]:0.1f} ' f'& {a4[ T[1] ][0]:0.1f} & {a4[ T[1] ][1]:0.1f} & {a4[ T[1] ][2]:0.1f} & {a4[ T[1] ][3]:0.1f} ' f'& {a4[ T[2] ][0]:0.1f} & {a4[ T[2] ][1]:0.1f} & {a4[ T[2] ][2]:0.1f} & {a4[ T[2] ][3]:0.1f} ' f'& {a4[ T[3] ][0]:0.1f} & {a4[ T[3] ][1]:0.1f} & {a4[ T[3] ][2]:0.1f} & {a4[ T[3] ][3]:0.1f}  \\\\' ) s += ( f'\n GT-State MLP 2-Step     & {a5[ T[0] ][0]:0.1f} & {a5[ T[0] ][1]:0.1f} & {a5[ T[0] ][2]:0.1f} & {a5[ T[0] ][3]:0.1f} ' f'& {a5[ T[1] ][0]:0.1f} & {a5[ T[1] ][1]:0.1f} & {a5[ T[1] ][2]:0.1f} & {a5[ T[1] ][3]:0.1f} ' f'& {a5[ T[2] ][0]:0.1f} & {a5[ T[2] ][1]:0.1f} & {a5[ T[2] ][2]:0.1f} & {a5[ T[2] ][3]:0.1f} ' f'& {a5[ T[3] ][0]:0.1f} & {a5[ T[3] ][1]:0.1f} & {a5[ T[3] ][2]:0.1f} & {a5[ T[3] ][3]:0.1f}  \\\\' ) s += ( f'\n Transporter             & {a1[ T[0] ][0]:0.1f} & {a1[ T[0] ][1]:0.1f} & {a1[ T[0] ][2]:0.1f} & {a1[ T[0] ][3]:0.1f} ' f'& {a1[ T[1] ][0]:0.1f} & {a1[ T[1] ][1]:0.1f} & {a1[ T[1] ][2]:0.1f} & {a1[ T[1] ][3]:0.1f} ' f'& {a1[ T[2] ][0]:0.1f} & {a1[ T[2] ][1]:0.1f} & {a1[ T[2] ][2]:0.1f} & {a1[ T[2] ][3]:0.1f} ' f'& {a1[ T[3] ][0]:0.1f} & {a1[ T[3] ][1]:0.1f} & {a1[ T[3] ][2]:0.1f} & {a1[ T[3] ][3]:0.1f}  \\\\' ) s += '\n \midrule' s += f'\n & \multicolumn{{4}}{{c}}{{{T[4]}}} & \multicolumn{{4}}{{c}}{{{T[5]}}} & \multicolumn{{4}}{{c}}{{{T[6]}}} & \multicolumn{{4}}{{c}}{{{T[7]}}}  \\\\' s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}' s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 \\\\' s += '\n \midrule' s += ( f'\n GT-State MLP            & {a4[ T[4] ][0]:0.1f} & {a4[ T[4] ][1]:0.1f} & {a4[ T[4] ][2]:0.1f} & {a4[ T[4] ][3]:0.1f} ' f'& {a4[ T[5] ][0]:0.1f} & {a4[ T[5] ][1]:0.1f} & {a4[ T[5] ][2]:0.1f} & {a4[ T[5] ][3]:0.1f} ' f'& {a4[ T[6] ][0]:0.1f} & {a4[ T[6] ][1]:0.1f} & {a4[ T[6] ][2]:0.1f} & {a4[ T[6] ][3]:0.1f} ' f'& {a4[ T[7] ][0]:0.1f} & {a4[ T[7] ][1]:0.1f} & {a4[ T[7] ][2]:0.1f} & {a4[ T[7] ][3]:0.1f}  \\\\' ) s += ( f'\n GT-State MLP 2-Step     & {a5[ T[4] ][0]:0.1f} & {a5[ T[4] ][1]:0.1f} & {a5[ T[4] ][2]:0.1f} & {a5[ T[4] ][3]:0.1f} ' f'& {a5[ T[5] ][0]:0.1f} & {a5[ T[5] ][1]:0.1f} & {a5[ T[5] ][2]:0.1f} & {a5[ T[5] ][3]:0.1f} ' f'& {a5[ T[6] ][0]:0.1f} & {a5[ T[6] ][1]:0.1f} & {a5[ T[6] ][2]:0.1f} & {a5[ T[6] ][3]:0.1f} ' f'& {a5[ T[7] ][0]:0.1f} & {a5[ T[7] ][1]:0.1f} & {a5[ T[7] ][2]:0.1f} & {a5[ T[7] ][3]:0.1f}  \\\\' ) s += ( f'\n Transporter             & {a1[ T[4] ][0]:0.1f} & {a1[ T[4] ][1]:0.1f} & {a1[ T[4] ][2]:0.1f} & {a1[ T[4] ][3]:0.1f} ' f'& {a1[ T[5] ][0]:0.1f} & {a1[ T[5] ][1]:0.1f} & {a1[ T[5] ][2]:0.1f} & {a1[ T[5] ][3]:0.1f} ' f'& {a1[ T[6] ][0]:0.1f} & {a1[ T[6] ][1]:0.1f} & {a1[ T[6] ][2]:0.1f} & {a1[ T[6] ][3]:0.1f} ' f'& {a1[ T[7] ][0]:0.1f} & {a1[ T[7] ][1]:0.1f} & {a1[ T[7] ][2]:0.1f} & {a1[ T[7] ][3]:0.1f}  \\\\' ) s += '\n \midrule' s += f'\n & \multicolumn{{4}}{{c}}{{{T[8]}}} & \multicolumn{{4}}{{c}}{{{T[9]}}} & \multicolumn{{4}}{{c}}{{{T[10]}}} & \multicolumn{{4}}{{c}}{{{T[11]}}}  \\\\' s += '\n \cmidrule(lr){2-5} \cmidrule(lr){6-9} \cmidrule(lr){10-13} \cmidrule(lr){14-17}' s += '\n Method & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000 & 1 & 10 & 100 & 1000  \\\\' s += '\n \midrule' s += ( f'\n GT-State MLP            & {a4[ T[8] ][0]:0.1f} & {a4[ T[8] ][1]:0.1f} & {a4[ T[8] ][2]:0.1f} & {a4[ T[8] ][3]:0.1f} ' f'& {a4[ T[9] ][0]:0.1f} & {a4[ T[9] ][1]:0.1f} & {a4[ T[9] ][2]:0.1f} & {a4[ T[9] ][3]:0.1f} ' f'& {a4[ T[10]][0]:0.1f} & {a4[ T[10]][1]:0.1f} & {a4[ T[10]][2]:0.1f} & {a4[ T[10]][3]:0.1f} ' f'& {a4[ T[11]][0]:0.1f} & {a4[ T[11]][1]:0.1f} & {a4[ T[11]][2]:0.1f} & {a4[ T[11]][3]:0.1f}  \\\\' ) s += ( f'\n GT-State MLP 2-Step     & {a5[ T[8] ][0]:0.1f} & {a5[ T[8] ][1]:0.1f} & {a5[ T[8] ][2]:0.1f} & {a5[ T[8] ][3]:0.1f} ' f'& {a5[ T[9] ][0]:0.1f} & {a5[ T[9] ][1]:0.1f} & {a5[ T[9] ][2]:0.1f} & {a5[ T[9] ][3]:0.1f} ' f'& {a5[ T[10]][0]:0.1f} & {a5[ T[10]][1]:0.1f} & {a5[ T[10]][2]:0.1f} & {a5[ T[10]][3]:0.1f} ' f'& {a5[ T[11]][0]:0.1f} & {a5[ T[11]][1]:0.1f} & {a5[ T[11]][2]:0.1f} & {a5[ T[11]][3]:0.1f}  \\\\' ) s += ( f'\n Transporter-Goal-Stack  & {a7[ T[8] ][0]:0.1f} & {a7[ T[8] ][1]:0.1f} & {a7[ T[8] ][2]:0.1f} & {a7[ T[8] ][3]:0.1f} ' f'& {a7[ T[9] ][0]:0.1f} & {a7[ T[9] ][1]:0.1f} & {a7[ T[9] ][2]:0.1f} & {a7[ T[9] ][3]:0.1f} ' f'& {a7[ T[10]][0]:0.1f} & {a7[ T[10]][1]:0.1f} & {a7[ T[10]][2]:0.1f} & {a7[ T[10]][3]:0.1f} ' f'& {a7[ T[11]][0]:0.1f} & {a7[ T[11]][1]:0.1f} & {a7[ T[11]][2]:0.1f} & {a7[ T[11]][3]:0.1f}  \\\\' ) s += ( f'\n Transporter-Goal-Split  & {a2[ T[8] ][0]:0.1f} & {a2[ T[8] ][1]:0.1f} & {a2[ T[8] ][2]:0.1f} & {a2[ T[8] ][3]:0.1f} ' f'& {a2[ T[9] ][0]:0.1f} & {a2[ T[9] ][1]:0.1f} & {a2[ T[9] ][2]:0.1f} & {a2[ T[9] ][3]:0.1f} ' f'& {a2[ T[10]][0]:0.1f} & {a2[ T[10]][1]:0.1f} & {a2[ T[10]][2]:0.1f} & {a2[ T[10]][3]:0.1f} ' f'& {a2[ T[11]][0]:0.1f} & {a2[ T[11]][1]:0.1f} & {a2[ T[11]][2]:0.1f} & {a2[ T[11]][3]:0.1f}  \\\\' ) print ( s )
Use this for the broad overview table in the paper, showing convergence results.
Use this for the broad overview table in the paper showing convergence results .
"""Use this for the broad overview table in the paper, showing convergence results.

    Careful: we use some braces, so could be tricky to integrate with string formatting?
    Remember that for line breaks we need an escape character for \, so \\.
    NOTE: Put this between \toprule and \bottomrule commands in LaTeX for tables.
    """ # Let's just use a fixed orering. Adjust here if needed. This is the task name IN CODE. # This has the same ordering as above but with fixed names (e.g., cloth --> fabric). # Map from (name_task, num_demos) --> statistic. # For each task, get relevant directories for each demo, then relevant stats. # For now, take the max over the iterations (easy to spot check). Take maximum. # Use a1 through a5 default dicts. Keys are tasks in `T` list. # For any valid task `i`, we have `a1[T[i]]` a list of length 4, for the 4 demo types. # Therefore the LAST index should range from: [0], [1], [2], to [3]. # If any lists are not of length 4, we should not use them, or simply fill in a value of -1? # This will be invalid for some but hopefully those can be easily inspected. # NOTE! Everything in this row must be 'a4', etc. # NOTE! Everything in this row must be 'a1', etc. # NOTE! Everything in this row must be 'a4', etc. # NOTE! Everything in this row must be 'a5', etc. # NOTE! Everything in this row must be 'a1', etc. # NOTE! Everything in this row must be 'a3'. (Update: actually a7) # If insertion-goal, then keep last columns at a3 # NOTE! Everything in this row must be 'a2', etc.
args