Spaces:
Running
Running
File size: 3,968 Bytes
11550ac 55d18b1 11550ac 55d18b1 8c4b1f7 11550ac 8c4b1f7 11550ac 8c4b1f7 11550ac 8c4b1f7 11550ac 8c4b1f7 11550ac 55d18b1 11550ac |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 |
import os
import json
import tempfile
import logging
from streamlit.delta_generator import DeltaGenerator
import streamlit as st
from huggingface_hub import HfApi, CommitInfo
# get a global var for logger accessor in this module
LOG_LEVEL = logging.DEBUG
g_logger = logging.getLogger(__name__)
g_logger.setLevel(LOG_LEVEL)
def push_observation(image_hash:str, api:HfApi, enable_push:False) -> CommitInfo:
'''
push one observation to the Hugging Face dataset
'''
# get the observation
observation = st.session_state.public_observations.get(image_hash)
if observation is None:
msg = f"Could not find observation with hash {image_hash}"
g_logger.error(msg)
st.error(msg)
return None
# convert to json
metadata_str = json.dumps(observation) # doesn't work yet, TODO
st.toast(f"Uploading observation: {metadata_str}", icon="🦭")
tab_log = st.session_state.tab_log
if tab_log is not None:
tab_log.info(f"Uploading observation: {metadata_str}")
# write to temp file so we can send it (why is this not using context mgr?)
f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
f.write(metadata_str)
f.close()
#st.info(f"temp file: {f.name} with metadata written...")
path_in_repo = f"metadata/{observation['author_email']}/{observation['image_md5']}.json"
msg = f"fname: {f.name} | path: {path_in_repo}"
print(msg)
st.warning(msg)
if enable_push:
rv = api.upload_file(
path_or_fileobj=f.name,
path_in_repo=path_in_repo,
repo_id="Saving-Willy/temp_dataset",
repo_type="dataset",
)
print(rv)
msg = f"observation attempted tx to repo happy walrus: {rv}"
g_logger.info(msg)
st.info(msg)
else:
rv = None # temp don't send anything
return rv
def push_all_observations(enable_push:bool=False):
'''
open an API connection to Hugging Face, and push all observation one by one
'''
# get huggingface api
token = os.environ.get("HF_TOKEN", None)
api = HfApi(token=token)
# iterate over the list of observations
for hash in st.session_state.public_observations.keys():
rv = push_observation(hash, api, enable_push=enable_push)
def push_observations(tab_log:DeltaGenerator=None):
"""
Push the observations to the Hugging Face dataset
Args:
tab_log (streamlit.container): The container to log messages to. If not provided,
log messages are in any case written to the global logger (TODO: test - didn't
push any observation since generating the logger)
"""
# we get the observation from session state: 1 is the dict 2 is the image.
# first, lets do an info display (popup)
metadata_str = json.dumps(st.session_state.public_observation)
st.toast(f"Uploading observations: {metadata_str}", icon="🦭")
tab_log = st.session_state.tab_log
if tab_log is not None:
tab_log.info(f"Uploading observations: {metadata_str}")
# get huggingface api
token = os.environ.get("HF_TOKEN", None)
api = HfApi(token=token)
f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
f.write(metadata_str)
f.close()
st.info(f"temp file: {f.name} with metadata written...")
path_in_repo= f"metadata/{st.session_state.public_observation['author_email']}/{st.session_state.public_observation['image_md5']}.json"
msg = f"fname: {f.name} | path: {path_in_repo}"
print(msg)
st.warning(msg)
# rv = api.upload_file(
# path_or_fileobj=f.name,
# path_in_repo=path_in_repo,
# repo_id="Saving-Willy/temp_dataset",
# repo_type="dataset",
# )
# print(rv)
# msg = f"observation attempted tx to repo happy walrus: {rv}"
g_logger.info(msg)
st.info(msg)
|