# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import allogger
from glob import glob
import numpy as np
import matplotlib.pyplot as plt
import h5py
from pandas import HDFStore, read_hdf, DataFrame
from time import sleep

hdf_writer = allogger.writers.HDFWriter(scope='root',
                                        output_dir='/tmp/test2',
                                        debug=True,
                                        precision=3,
                                        min_time_diff_btw_disc_writes=10
                                       )

for k in range(10):
    hdf_writer.add_array('main', np.random.randn(5,5), k)
    sleep(2)

data[]

[d[:-1] + [np.asarray(d[-1])] for d in data][0]

[d[:3] for d in data]

data = list(hdf_writer.data[('array', 'main')])
dd = np.array([tuple(d[:3] + [np.asarray(d[-1])]) for d in data], dtype=[('step', np.int32), ('time', np.float32), ('walltime', np.float32), ('value', 'f', (5,5))])

dd.shape

np.random.randn(5,5).tolist()

with h5py.File('/tmp/test.h5', 'a') as hf:
    hf.create_dataset('main_structure', data=dd, compression="gzip", chunks=True, maxshape=(None,))

with HDFStore('/tmp/test.h5') as hf:
    print(hf.keys())

read_hdf('/tmp/test.h5', '/a')

with h5py.File('/tmp/test.h5','r') as hf:
    d = np.asarray(hf['main_structure'])

d

f = h5py.File('/tmp/test2/1599664045_olef.h5','r')

from allogger.api.hdf import list_keys, read_from_key

list_keys('/tmp/allogger/singleprocess/events/')

read_from_key('/tmp/allogger/singleprocess/events/', 'array/main/array')

f['array'].keys()

a = f['array']['main']

f['array/main']

isinstance(a, h5py.Dataset)

np.array(a)['step']


def print_foo(name):
    print(name)


f.visit(print_foo)

f.close()

list(f['image']['main']['blub'])

list(d)[0]

f.keys()

np.asarray(d)

allogger.basic_configure('/tmp/test', ['tensorboard'])

logger1 = allogger.get_logger('main')
logger2 = allogger.get_logger('Controller')

# +
for _ in range(100):
    logger1.log(np.random.randn()*5, 'reward')
    
logger2.log(1e-10, 'loss')
# -

logger2.hdf_writer.write_to_disc(force=True)

hdf_log_file = sorted(glob('/tmp/al/events/*.h5'))[-1]

allogger.api.hdf.list_keys(hdf_log_file)

plt.plot(allogger.api.hdf.read_from_key(hdf_log_file, '/scalar/main/eval_mean_return')['value'])

None or 'wurst'

a = np.random.rand(5,5)

str(a)

str(a.round(-1))

allogger.api.hdf.list_keys('/tmp/allogger/singleprocess/events/')

allogger.api.hdf.read_from_key('/tmp/allogger/singleprocess/events/', '/image/main/blub')

from allogger.api.hdf import flatten_keys

flatten_keys({'bla': {'blub': 1, 'bli': 2}})



StarcoderdataJupyterScriptsDedupFiltered