File size: 755 Bytes
246d201 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 |
import argparse
import json
def main():
parser = argparse.ArgumentParser(description="Get agent's gaia score")
parser.add_argument('--file', type=str, help="Path to the agent's output.jsonl")
args = parser.parse_args()
this_log = args.file
outs = []
with open(this_log, 'r') as f:
lines = f.readlines()
for line in lines:
outs.append(json.loads(line))
print(f'Reading {this_log}')
print(f'Metadata:\n {outs[0]["metadata"]}')
total = 0
success = 0
for out in outs:
total += 1
if out['test_result']['score']:
success += 1
print(f'Success rate: {success}/{total} = {success/total}')
if __name__ == '__main__':
main()
|