Spaces:
Running
on
Zero
Running
on
Zero
Permalink Update - storage version 0.1.0
Browse files- app.py +11 -7
- utils/constants.py +5 -1
- utils/storage.md +154 -0
- utils/storage.py +209 -15
- utils/version_info.py +11 -0
app.py
CHANGED
@@ -807,7 +807,7 @@ def load_trellis_model():
|
|
807 |
gr.Info("TRELLIS_PIPELINE load start", 60)
|
808 |
global TRELLIS_PIPELINE
|
809 |
loaded = False
|
810 |
-
if TRELLIS_PIPELINE == None:
|
811 |
try:
|
812 |
TRELLIS_PIPELINE = TrellisImageTo3DPipeline.from_pretrained("Surn/TRELLIS-image-large")
|
813 |
TRELLIS_PIPELINE.cuda()
|
@@ -1177,8 +1177,9 @@ def update_permalink(glb, gaussian, depth_out, depth_src_file):
|
|
1177 |
return gr.update(visible=False), ""
|
1178 |
|
1179 |
smallest_file = min(file_candidates, key=file_candidates.get)
|
1180 |
-
|
1181 |
-
|
|
|
1182 |
|
1183 |
def create_permalink(glb, gaussian, depth_out, depth_src_file, folder_name_permalink):
|
1184 |
"""
|
@@ -1202,14 +1203,17 @@ def create_permalink(glb, gaussian, depth_out, depth_src_file, folder_name_perma
|
|
1202 |
# return gr.update(visible=False), ""
|
1203 |
smallest_file = min(file_candidates, key=file_candidates.get)
|
1204 |
|
1205 |
-
|
1206 |
files=[smallest_file, depth_out, depth_src_file],
|
1207 |
-
repo_id=
|
1208 |
folder_name=folder_name_permalink,
|
1209 |
create_permalink=True,
|
1210 |
repo_type="dataset"
|
1211 |
-
)
|
1212 |
-
|
|
|
|
|
|
|
1213 |
|
1214 |
def open_permalink(url: str) -> str:
|
1215 |
if url and url.strip():
|
|
|
807 |
gr.Info("TRELLIS_PIPELINE load start", 60)
|
808 |
global TRELLIS_PIPELINE
|
809 |
loaded = False
|
810 |
+
if (TRELLIS_PIPELINE == None):
|
811 |
try:
|
812 |
TRELLIS_PIPELINE = TrellisImageTo3DPipeline.from_pretrained("Surn/TRELLIS-image-large")
|
813 |
TRELLIS_PIPELINE.cuda()
|
|
|
1177 |
return gr.update(visible=False), ""
|
1178 |
|
1179 |
smallest_file = min(file_candidates, key=file_candidates.get)
|
1180 |
+
# generate_permalink_from_urls now returns a string directly
|
1181 |
+
permalink_url = generate_permalink_from_urls(smallest_file, depth_out, depth_src_file)
|
1182 |
+
return gr.update(visible=True), permalink_url
|
1183 |
|
1184 |
def create_permalink(glb, gaussian, depth_out, depth_src_file, folder_name_permalink):
|
1185 |
"""
|
|
|
1203 |
# return gr.update(visible=False), ""
|
1204 |
smallest_file = min(file_candidates, key=file_candidates.get)
|
1205 |
|
1206 |
+
upload_result = upload_files_to_repo(
|
1207 |
files=[smallest_file, depth_out, depth_src_file],
|
1208 |
+
repo_id=constants.HF_REPO_ID, # Use constant for repo_id
|
1209 |
folder_name=folder_name_permalink,
|
1210 |
create_permalink=True,
|
1211 |
repo_type="dataset"
|
1212 |
+
)
|
1213 |
+
# upload_files_to_repo returns a dict with "permalink" and "short_permalink"
|
1214 |
+
# We are interested in the "short_permalink" if available, otherwise the "permalink"
|
1215 |
+
permalink_url = upload_result.get("short_permalink") or upload_result.get("permalink")
|
1216 |
+
return permalink_url
|
1217 |
|
1218 |
def open_permalink(url: str) -> str:
|
1219 |
if url and url.strip():
|
utils/constants.py
CHANGED
@@ -675,4 +675,8 @@ card_colors_alternating = [
|
|
675 |
"#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", # Clubs
|
676 |
"#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", # Diamonds
|
677 |
"#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000" # Spades
|
678 |
-
]
|
|
|
|
|
|
|
|
|
|
675 |
"#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", # Clubs
|
676 |
"#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", # Diamonds
|
677 |
"#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000" # Spades
|
678 |
+
]
|
679 |
+
|
680 |
+
# Constants for URL shortener
|
681 |
+
HF_REPO_ID = "Surn/Storage" # Or your desired repository
|
682 |
+
SHORTENER_JSON_FILE = "shortener.json"
|
utils/storage.md
ADDED
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Storage Module (`modules/storage.py`) Usage Guide
|
2 |
+
|
3 |
+
The `storage.py` module provides helper functions for:
|
4 |
+
- Generating permalinks for 3D viewer projects.
|
5 |
+
- Uploading files in batches to a Hugging Face repository.
|
6 |
+
- Managing URL shortening by storing (short URL, full URL) pairs in a JSON file on the repository.
|
7 |
+
|
8 |
+
## Key Functions
|
9 |
+
|
10 |
+
### 1. `generate_permalink(valid_files, base_url_external, permalink_viewer_url="surn-3d-viewer.hf.space")`
|
11 |
+
- **Purpose:**
|
12 |
+
Given a list of file paths, it looks for exactly one model file (with an extension defined in `model_extensions`) and exactly two image files (extensions defined in `image_extensions`). If the criteria are met, it returns a permalink URL built from the base URL and query parameters.
|
13 |
+
- **Usage Example:**from modules.storage import generate_permalink
|
14 |
+
|
15 |
+
valid_files = [
|
16 |
+
"models/3d_model.glb",
|
17 |
+
"images/model_texture.png",
|
18 |
+
"images/model_depth.png"
|
19 |
+
]
|
20 |
+
base_url_external = "https://huggingface.co/datasets/Surn/Storage/resolve/main/saved_models/my_model"
|
21 |
+
permalink = generate_permalink(valid_files, base_url_external)
|
22 |
+
if permalink:
|
23 |
+
print("Permalink:", permalink)
|
24 |
+
### 2. `generate_permalink_from_urls(model_url, hm_url, img_url, permalink_viewer_url="surn-3d-viewer.hf.space")`
|
25 |
+
- **Purpose:**
|
26 |
+
Constructs a permalink URL by combining individual URLs for a 3D model (`model_url`), height map (`hm_url`), and image (`img_url`) into a single URL with corresponding query parameters.
|
27 |
+
- **Usage Example:**from modules.storage import generate_permalink_from_urls
|
28 |
+
|
29 |
+
model_url = "https://example.com/model.glb"
|
30 |
+
hm_url = "https://example.com/heightmap.png"
|
31 |
+
img_url = "https://example.com/source.png"
|
32 |
+
|
33 |
+
permalink = generate_permalink_from_urls(model_url, hm_url, img_url)
|
34 |
+
print("Generated Permalink:", permalink)
|
35 |
+
### 3. `upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, repo_type="dataset", permalink_viewer_url="surn-3d-viewer.hf.space")`
|
36 |
+
- **Purpose:**
|
37 |
+
Uploads a batch of files (each file represented as a path string) to a specified Hugging Face repository (e.g. `"Surn/Storage"`) under a given folder.
|
38 |
+
The function's return type is `Union[Dict[str, Any], List[Tuple[Any, str]]]`.
|
39 |
+
- When `create_permalink` is `True` and exactly three valid files (one model and two images) are provided, the function returns a dictionary:```
|
40 |
+
{
|
41 |
+
"response": <upload_folder_response>,
|
42 |
+
"permalink": "<full_permalink_url>",
|
43 |
+
"short_permalink": "<shortened_permalink_url_with_sid>"
|
44 |
+
}
|
45 |
+
``` - Otherwise (or if `create_permalink` is `False` or conditions for permalink creation are not met), it returns a list of tuples, where each tuple is `(upload_folder_response, individual_file_link)`.
|
46 |
+
- If no valid files are provided, it returns an empty list `[]` (this case should ideally also return the dictionary with empty/None values for consistency, but currently returns `[]` as per the code).
|
47 |
+
- **Usage Example:**
|
48 |
+
|
49 |
+
**a. Uploading with permalink creation:**from modules.storage import upload_files_to_repo
|
50 |
+
|
51 |
+
files_for_permalink = [
|
52 |
+
"local/path/to/model.glb",
|
53 |
+
"local/path/to/heightmap.png",
|
54 |
+
"local/path/to/image.png"
|
55 |
+
]
|
56 |
+
repo_id = "Surn/Storage" # Make sure this is defined, e.g., from constants
|
57 |
+
folder_name = "my_new_model_with_permalink"
|
58 |
+
|
59 |
+
upload_result = upload_files_to_repo(
|
60 |
+
files_for_permalink,
|
61 |
+
repo_id,
|
62 |
+
folder_name,
|
63 |
+
create_permalink=True
|
64 |
+
)
|
65 |
+
|
66 |
+
if isinstance(upload_result, dict):
|
67 |
+
print("Upload Response:", upload_result.get("response"))
|
68 |
+
print("Full Permalink:", upload_result.get("permalink"))
|
69 |
+
print("Short Permalink:", upload_result.get("short_permalink"))
|
70 |
+
elif upload_result: # Check if list is not empty
|
71 |
+
print("Upload Response for individual files:")
|
72 |
+
for res, link in upload_result:
|
73 |
+
print(f" Response: {res}, Link: {link}")
|
74 |
+
else:
|
75 |
+
print("No files uploaded or error occurred.")
|
76 |
+
**b. Uploading without permalink creation (or if conditions for permalink are not met):**from modules.storage import upload_files_to_repo
|
77 |
+
|
78 |
+
files_individual = [
|
79 |
+
"local/path/to/another_model.obj",
|
80 |
+
"local/path/to/texture.jpg"
|
81 |
+
]
|
82 |
+
repo_id = "Surn/Storage"
|
83 |
+
folder_name = "my_other_uploads"
|
84 |
+
|
85 |
+
upload_results_list = upload_files_to_repo(
|
86 |
+
files_individual,
|
87 |
+
repo_id,
|
88 |
+
folder_name,
|
89 |
+
create_permalink=False # Or if create_permalink=True but not 1 model & 2 images
|
90 |
+
)
|
91 |
+
|
92 |
+
if upload_results_list: # Will be a list of tuples
|
93 |
+
print("Upload results for individual files:")
|
94 |
+
for res, link in upload_results_list:
|
95 |
+
print(f" Upload Response: {res}, File Link: {link}")
|
96 |
+
else:
|
97 |
+
print("No files uploaded or error occurred.")
|
98 |
+
### 4. URL Shortening Functions: `gen_full_url(...)` and Helpers
|
99 |
+
The module also enables URL shortening by managing a JSON file (e.g. `shortener.json`) in a Hugging Face repository. It supports CRUD-like operations:
|
100 |
+
- **Read:** Look up the full URL using a provided short URL ID.
|
101 |
+
- **Create:** Generate a new short URL ID for a full URL if no existing mapping exists.
|
102 |
+
- **Update/Conflict Handling:**
|
103 |
+
If both short URL ID and full URL are provided, it checks consistency and either confirms or reports a conflict.
|
104 |
+
|
105 |
+
#### `gen_full_url(short_url=None, full_url=None, repo_id=None, repo_type="dataset", permalink_viewer_url="surn-3d-viewer.hf.space", json_file="shortener.json")`
|
106 |
+
- **Purpose:**
|
107 |
+
Based on which parameter is provided, it retrieves or creates a mapping between a short URL ID and a full URL.
|
108 |
+
- If only `short_url` (the ID) is given, it returns the corresponding `full_url`.
|
109 |
+
- If only `full_url` is given, it looks up an existing `short_url` ID or generates and stores a new one.
|
110 |
+
- If both are given, it validates and returns the mapping or an error status.
|
111 |
+
- **Returns:** A tuple `(status_message, result_url)`, where `status_message` indicates the outcome (e.g., `"success_retrieved_full"`, `"created_short"`) and `result_url` is the relevant URL (full or short ID).
|
112 |
+
- **Usage Examples:**
|
113 |
+
|
114 |
+
**a. Convert a full URL into a short URL ID:**from modules.storage import gen_full_url
|
115 |
+
from modules.constants import HF_REPO_ID, SHORTENER_JSON_FILE # Assuming these are defined
|
116 |
+
|
117 |
+
full_permalink = "https://surn-3d-viewer.hf.space/?3d=https%3A%2F%2Fexample.com%2Fmodel.glb&hm=https%3A%2F%2Fexample.com%2Fheightmap.png&image=https%3A%2F%2Fexample.com%2Fsource.png"
|
118 |
+
|
119 |
+
status, short_id = gen_full_url(
|
120 |
+
full_url=full_permalink,
|
121 |
+
repo_id=HF_REPO_ID,
|
122 |
+
json_file=SHORTENER_JSON_FILE
|
123 |
+
)
|
124 |
+
print("Status:", status)
|
125 |
+
if status == "created_short" or status == "success_retrieved_short":
|
126 |
+
print("Shortened URL ID:", short_id)
|
127 |
+
# Construct the full short URL for sharing:
|
128 |
+
# permalink_viewer_url = "surn-3d-viewer.hf.space" # Or from constants
|
129 |
+
# shareable_short_url = f"https://{permalink_viewer_url}/?sid={short_id}"
|
130 |
+
# print("Shareable Short URL:", shareable_short_url)
|
131 |
+
**b. Retrieve the full URL from a short URL ID:**from modules.storage import gen_full_url
|
132 |
+
from modules.constants import HF_REPO_ID, SHORTENER_JSON_FILE # Assuming these are defined
|
133 |
+
|
134 |
+
short_id_to_lookup = "aBcDeFg1" # Example short URL ID
|
135 |
+
|
136 |
+
status, retrieved_full_url = gen_full_url(
|
137 |
+
short_url=short_id_to_lookup,
|
138 |
+
repo_id=HF_REPO_ID,
|
139 |
+
json_file=SHORTENER_JSON_FILE
|
140 |
+
)
|
141 |
+
print("Status:", status)
|
142 |
+
if status == "success_retrieved_full":
|
143 |
+
print("Retrieved Full URL:", retrieved_full_url)
|
144 |
+
## Notes
|
145 |
+
- **Authentication:** All functions that interact with Hugging Face Hub use the HF API token defined as `HF_API_TOKEN` in `modules/constants.py`. Ensure this environment variable is correctly set.
|
146 |
+
- **Constants:** Functions like `gen_full_url` and `upload_files_to_repo` (when creating short links) rely on `HF_REPO_ID` and `SHORTENER_JSON_FILE` from `modules/constants.py` for the URL shortening feature.
|
147 |
+
- **File Types:** Only files with extensions included in `upload_file_types` (a combination of `model_extensions` and `image_extensions` from `modules/constants.py`) are processed by `upload_files_to_repo`.
|
148 |
+
- **Repository Configuration:** When using URL shortening and file uploads, ensure that the specified Hugging Face repository (e.g., defined by `HF_REPO_ID`) exists and that you have write permissions.
|
149 |
+
- **Temporary Directory:** `upload_files_to_repo` temporarily copies files to a local directory (configured by `TMPDIR` in `modules/constants.py`) before uploading.
|
150 |
+
- **Error Handling:** Functions include basic error handling (e.g., catching `RepositoryNotFoundError`, `EntryNotFoundError`, JSON decoding errors, or upload issues) and print messages to the console for debugging. Review function return values to handle these cases appropriately in your application.
|
151 |
+
|
152 |
+
---
|
153 |
+
|
154 |
+
This guide provides the essential usage examples for interacting with the storage and URL-shortening functionality. You can integrate these examples into your application or use them as a reference when extending functionality.
|
utils/storage.py
CHANGED
@@ -1,10 +1,17 @@
|
|
1 |
# utils/storage.py
|
|
|
2 |
import os
|
3 |
import urllib.parse
|
4 |
import tempfile
|
5 |
import shutil
|
6 |
-
|
7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
|
9 |
def generate_permalink(valid_files, base_url_external, permalink_viewer_url="surn-3d-viewer.hf.space"):
|
10 |
"""
|
@@ -49,7 +56,14 @@ def generate_permalink_from_urls(model_url, hm_url, img_url, permalink_viewer_ur
|
|
49 |
query_str = urllib.parse.urlencode(params)
|
50 |
return f"https://{permalink_viewer_url}/?{query_str}"
|
51 |
|
52 |
-
def upload_files_to_repo(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
53 |
"""
|
54 |
Uploads multiple files to a Hugging Face repository using a batch upload approach via upload_folder.
|
55 |
|
@@ -61,18 +75,24 @@ def upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, re
|
|
61 |
returns a single permalink to the project with query parameters.
|
62 |
Otherwise, returns individual permalinks for each file.
|
63 |
repo_type (str): Repository type ("space", "dataset", etc.). Default is "dataset".
|
|
|
64 |
|
65 |
Returns:
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
|
|
|
|
|
|
|
|
71 |
"""
|
72 |
# Log in using the HF API token.
|
73 |
login(token=HF_API_TOKEN)
|
74 |
|
75 |
valid_files = []
|
|
|
76 |
|
77 |
# Ensure folder_name does not have a trailing slash.
|
78 |
folder_name = folder_name.rstrip("/")
|
@@ -87,10 +107,17 @@ def upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, re
|
|
87 |
valid_files.append(f)
|
88 |
|
89 |
if not valid_files:
|
90 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
91 |
|
92 |
# Create a temporary directory; copy valid files directly into it.
|
93 |
-
with tempfile.TemporaryDirectory() as temp_dir:
|
94 |
for file_path in valid_files:
|
95 |
filename = os.path.basename(file_path)
|
96 |
dest_path = os.path.join(temp_dir, filename)
|
@@ -107,8 +134,6 @@ def upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, re
|
|
107 |
)
|
108 |
|
109 |
# Construct external URLs for each uploaded file.
|
110 |
-
# For datasets, files are served at:
|
111 |
-
# https://huggingface.co/datasets/<repo_id>/resolve/main/<folder_name>/<filename>
|
112 |
base_url_external = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{folder_name}"
|
113 |
individual_links = []
|
114 |
for file_path in valid_files:
|
@@ -118,10 +143,179 @@ def upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, re
|
|
118 |
|
119 |
# If permalink creation is requested and exactly 3 valid files are provided,
|
120 |
# try to generate a permalink using generate_permalink().
|
121 |
-
if create_permalink
|
122 |
-
permalink =
|
123 |
if permalink:
|
124 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
125 |
|
126 |
# Otherwise, return individual tuples for each file.
|
127 |
return [(response, link) for link in individual_links]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
# utils/storage.py
|
2 |
+
__version__ = "0.1.0" # Added version
|
3 |
import os
|
4 |
import urllib.parse
|
5 |
import tempfile
|
6 |
import shutil
|
7 |
+
import json
|
8 |
+
import base64
|
9 |
+
from huggingface_hub import login, upload_folder, hf_hub_download, HfApi
|
10 |
+
from huggingface_hub.utils import RepositoryNotFoundError, EntryNotFoundError
|
11 |
+
from utils.constants import HF_API_TOKEN, upload_file_types, model_extensions, image_extensions, HF_REPO_ID, SHORTENER_JSON_FILE
|
12 |
+
from typing import Any, Dict, List, Tuple, Union
|
13 |
+
|
14 |
+
# see storage.md for detailed information about the storage module and its functions.
|
15 |
|
16 |
def generate_permalink(valid_files, base_url_external, permalink_viewer_url="surn-3d-viewer.hf.space"):
|
17 |
"""
|
|
|
56 |
query_str = urllib.parse.urlencode(params)
|
57 |
return f"https://{permalink_viewer_url}/?{query_str}"
|
58 |
|
59 |
+
def upload_files_to_repo(
|
60 |
+
files: List[Any],
|
61 |
+
repo_id: str,
|
62 |
+
folder_name: str,
|
63 |
+
create_permalink: bool = False,
|
64 |
+
repo_type: str = "dataset",
|
65 |
+
permalink_viewer_url: str = "surn-3d-viewer.hf.space"
|
66 |
+
) -> Union[Dict[str, Any], List[Tuple[Any, str]]]:
|
67 |
"""
|
68 |
Uploads multiple files to a Hugging Face repository using a batch upload approach via upload_folder.
|
69 |
|
|
|
75 |
returns a single permalink to the project with query parameters.
|
76 |
Otherwise, returns individual permalinks for each file.
|
77 |
repo_type (str): Repository type ("space", "dataset", etc.). Default is "dataset".
|
78 |
+
permalink_viewer_url (str): The base viewer URL.
|
79 |
|
80 |
Returns:
|
81 |
+
Union[Dict[str, Any], List[Tuple[Any, str]]]:
|
82 |
+
If create_permalink is True and files match the criteria:
|
83 |
+
dict: {
|
84 |
+
"response": <upload response>,
|
85 |
+
"permalink": <full_permalink URL>,
|
86 |
+
"short_permalink": <shortened permalink URL>
|
87 |
+
}
|
88 |
+
Otherwise:
|
89 |
+
list: A list of tuples (response, permalink) for each file.
|
90 |
"""
|
91 |
# Log in using the HF API token.
|
92 |
login(token=HF_API_TOKEN)
|
93 |
|
94 |
valid_files = []
|
95 |
+
permalink_short = None
|
96 |
|
97 |
# Ensure folder_name does not have a trailing slash.
|
98 |
folder_name = folder_name.rstrip("/")
|
|
|
107 |
valid_files.append(f)
|
108 |
|
109 |
if not valid_files:
|
110 |
+
# Return a dictionary with None values for permalinks if create_permalink was True
|
111 |
+
if create_permalink:
|
112 |
+
return {
|
113 |
+
"response": "No valid files to upload.",
|
114 |
+
"permalink": None,
|
115 |
+
"short_permalink": None
|
116 |
+
}
|
117 |
+
return []
|
118 |
|
119 |
# Create a temporary directory; copy valid files directly into it.
|
120 |
+
with tempfile.TemporaryDirectory(dir=os.getenv("TMPDIR", "/tmp")) as temp_dir:
|
121 |
for file_path in valid_files:
|
122 |
filename = os.path.basename(file_path)
|
123 |
dest_path = os.path.join(temp_dir, filename)
|
|
|
134 |
)
|
135 |
|
136 |
# Construct external URLs for each uploaded file.
|
|
|
|
|
137 |
base_url_external = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{folder_name}"
|
138 |
individual_links = []
|
139 |
for file_path in valid_files:
|
|
|
143 |
|
144 |
# If permalink creation is requested and exactly 3 valid files are provided,
|
145 |
# try to generate a permalink using generate_permalink().
|
146 |
+
if create_permalink: # No need to check len(valid_files) == 3 here, generate_permalink will handle it
|
147 |
+
permalink = generate_permalink(valid_files, base_url_external, permalink_viewer_url)
|
148 |
if permalink:
|
149 |
+
status, short_id = gen_full_url(
|
150 |
+
full_url=permalink,
|
151 |
+
repo_id=HF_REPO_ID, # This comes from constants
|
152 |
+
json_file=SHORTENER_JSON_FILE # This comes from constants
|
153 |
+
)
|
154 |
+
if status in ["created_short", "success_retrieved_short", "exists_match"]:
|
155 |
+
permalink_short = f"https://{permalink_viewer_url}/?sid={short_id}"
|
156 |
+
else: # Shortening failed or conflict not resolved to a usable short_id
|
157 |
+
permalink_short = None
|
158 |
+
print(f"URL shortening status: {status} for {permalink}")
|
159 |
+
|
160 |
+
return {
|
161 |
+
"response": response,
|
162 |
+
"permalink": permalink,
|
163 |
+
"short_permalink": permalink_short
|
164 |
+
}
|
165 |
+
else: # generate_permalink returned None (criteria not met)
|
166 |
+
return {
|
167 |
+
"response": response, # Still return upload response
|
168 |
+
"permalink": None,
|
169 |
+
"short_permalink": None
|
170 |
+
}
|
171 |
|
172 |
# Otherwise, return individual tuples for each file.
|
173 |
return [(response, link) for link in individual_links]
|
174 |
+
|
175 |
+
def _generate_short_id(length=8):
|
176 |
+
"""Generates a random base64 URL-safe string."""
|
177 |
+
return base64.urlsafe_b64encode(os.urandom(length * 2))[:length].decode('utf-8')
|
178 |
+
|
179 |
+
def _get_json_from_repo(repo_id, json_file_name, repo_type="dataset"):
|
180 |
+
"""Downloads and loads the JSON file from the repo. Returns empty list if not found or error."""
|
181 |
+
try:
|
182 |
+
login(token=HF_API_TOKEN)
|
183 |
+
json_path = hf_hub_download(
|
184 |
+
repo_id=repo_id,
|
185 |
+
filename=json_file_name,
|
186 |
+
repo_type=repo_type,
|
187 |
+
token=HF_API_TOKEN # Added token for consistency, though login might suffice
|
188 |
+
)
|
189 |
+
with open(json_path, 'r') as f:
|
190 |
+
data = json.load(f)
|
191 |
+
os.remove(json_path) # Clean up downloaded file
|
192 |
+
return data
|
193 |
+
except RepositoryNotFoundError:
|
194 |
+
print(f"Repository {repo_id} not found.")
|
195 |
+
return []
|
196 |
+
except EntryNotFoundError:
|
197 |
+
print(f"JSON file {json_file_name} not found in {repo_id}. Initializing with empty list.")
|
198 |
+
return []
|
199 |
+
except json.JSONDecodeError:
|
200 |
+
print(f"Error decoding JSON from {json_file_name}. Returning empty list.")
|
201 |
+
return []
|
202 |
+
except Exception as e:
|
203 |
+
print(f"An unexpected error occurred while fetching {json_file_name}: {e}")
|
204 |
+
return []
|
205 |
+
|
206 |
+
def _upload_json_to_repo(data, repo_id, json_file_name, repo_type="dataset"):
|
207 |
+
"""Uploads the JSON data to the specified file in the repo."""
|
208 |
+
try:
|
209 |
+
login(token=HF_API_TOKEN)
|
210 |
+
api = HfApi()
|
211 |
+
# Use a temporary directory specified by TMPDIR or default to system temp
|
212 |
+
temp_dir_for_json = os.getenv("TMPDIR", tempfile.gettempdir())
|
213 |
+
os.makedirs(temp_dir_for_json, exist_ok=True)
|
214 |
+
|
215 |
+
with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".json", dir=temp_dir_for_json) as tmp_file:
|
216 |
+
json.dump(data, tmp_file, indent=2)
|
217 |
+
tmp_file_path = tmp_file.name
|
218 |
+
|
219 |
+
api.upload_file(
|
220 |
+
path_or_fileobj=tmp_file_path,
|
221 |
+
path_in_repo=json_file_name,
|
222 |
+
repo_id=repo_id,
|
223 |
+
repo_type=repo_type,
|
224 |
+
commit_message=f"Update {json_file_name}"
|
225 |
+
)
|
226 |
+
os.remove(tmp_file_path) # Clean up temporary file
|
227 |
+
return True
|
228 |
+
except Exception as e:
|
229 |
+
print(f"Failed to upload {json_file_name} to {repo_id}: {e}")
|
230 |
+
if 'tmp_file_path' in locals() and os.path.exists(tmp_file_path):
|
231 |
+
os.remove(tmp_file_path) # Ensure cleanup on error too
|
232 |
+
return False
|
233 |
+
|
234 |
+
def _find_url_in_json(data, short_url=None, full_url=None):
|
235 |
+
"""
|
236 |
+
Searches the JSON data.
|
237 |
+
If short_url is provided, returns the corresponding full_url or None.
|
238 |
+
If full_url is provided, returns the corresponding short_url or None.
|
239 |
+
"""
|
240 |
+
if not data: # Handles cases where data might be None or empty
|
241 |
+
return None
|
242 |
+
if short_url:
|
243 |
+
for item in data:
|
244 |
+
if item.get("short_url") == short_url:
|
245 |
+
return item.get("full_url")
|
246 |
+
if full_url:
|
247 |
+
for item in data:
|
248 |
+
if item.get("full_url") == full_url:
|
249 |
+
return item.get("short_url")
|
250 |
+
return None
|
251 |
+
|
252 |
+
def _add_url_to_json(data, short_url, full_url):
|
253 |
+
"""Adds a new short_url/full_url pair to the data. Returns updated data."""
|
254 |
+
if data is None:
|
255 |
+
data = []
|
256 |
+
data.append({"short_url": short_url, "full_url": full_url})
|
257 |
+
return data
|
258 |
+
|
259 |
+
def gen_full_url(short_url=None, full_url=None, repo_id=None, repo_type="dataset", permalink_viewer_url="surn-3d-viewer.hf.space", json_file="shortener.json"):
|
260 |
+
"""
|
261 |
+
Manages short URLs and their corresponding full URLs in a JSON file stored in a Hugging Face repository.
|
262 |
+
|
263 |
+
- If short_url is provided, attempts to retrieve and return the full_url.
|
264 |
+
- If full_url is provided, attempts to retrieve an existing short_url or creates a new one, stores it, and returns the short_url.
|
265 |
+
- If both are provided, checks for consistency or creates a new entry.
|
266 |
+
- If neither is provided, or repo_id is missing, returns an error status.
|
267 |
+
|
268 |
+
Returns:
|
269 |
+
tuple: (status_message, result_url)
|
270 |
+
status_message can be "success", "created", "exists", "error", "not_found".
|
271 |
+
result_url is the relevant URL (short or full) or None if an error occurs or not found.
|
272 |
+
"""
|
273 |
+
if not repo_id:
|
274 |
+
return "error_repo_id_missing", None
|
275 |
+
if not short_url and not full_url:
|
276 |
+
return "error_no_input", None
|
277 |
+
|
278 |
+
login(token=HF_API_TOKEN) # Ensure login at the beginning
|
279 |
+
url_data = _get_json_from_repo(repo_id, json_file, repo_type)
|
280 |
+
|
281 |
+
# Case 1: Only short_url provided (lookup full_url)
|
282 |
+
if short_url and not full_url:
|
283 |
+
found_full_url = _find_url_in_json(url_data, short_url=short_url)
|
284 |
+
return ("success_retrieved_full", found_full_url) if found_full_url else ("not_found_short", None)
|
285 |
+
|
286 |
+
# Case 2: Only full_url provided (lookup or create short_url)
|
287 |
+
if full_url and not short_url:
|
288 |
+
existing_short_url = _find_url_in_json(url_data, full_url=full_url)
|
289 |
+
if existing_short_url:
|
290 |
+
return "success_retrieved_short", existing_short_url
|
291 |
+
else:
|
292 |
+
# Create new short_url
|
293 |
+
new_short_id = _generate_short_id()
|
294 |
+
url_data = _add_url_to_json(url_data, new_short_id, full_url)
|
295 |
+
if _upload_json_to_repo(url_data, repo_id, json_file, repo_type):
|
296 |
+
return "created_short", new_short_id
|
297 |
+
else:
|
298 |
+
return "error_upload", None
|
299 |
+
|
300 |
+
# Case 3: Both short_url and full_url provided
|
301 |
+
if short_url and full_url:
|
302 |
+
found_full_for_short = _find_url_in_json(url_data, short_url=short_url)
|
303 |
+
found_short_for_full = _find_url_in_json(url_data, full_url=full_url)
|
304 |
+
|
305 |
+
if found_full_for_short == full_url:
|
306 |
+
return "exists_match", short_url
|
307 |
+
if found_full_for_short is not None and found_full_for_short != full_url:
|
308 |
+
return "error_conflict_short_exists_different_full", short_url
|
309 |
+
if found_short_for_full is not None and found_short_for_full != short_url:
|
310 |
+
return "error_conflict_full_exists_different_short", found_short_for_full
|
311 |
+
|
312 |
+
# If short_url is provided and not found, or full_url is provided and not found,
|
313 |
+
# or neither is found, then create a new entry with the provided short_url and full_url.
|
314 |
+
# This effectively allows specifying a custom short_url if it's not already taken.
|
315 |
+
url_data = _add_url_to_json(url_data, short_url, full_url)
|
316 |
+
if _upload_json_to_repo(url_data, repo_id, json_file, repo_type):
|
317 |
+
return "created_specific_pair", short_url
|
318 |
+
else:
|
319 |
+
return "error_upload", None
|
320 |
+
|
321 |
+
return "error_unhandled_case", None # Should not be reached
|
utils/version_info.py
CHANGED
@@ -103,6 +103,15 @@ def versions_html():
|
|
103 |
</a>
|
104 |
'''
|
105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
106 |
v_html = f"""
|
107 |
version: <a href="https://huggingface.co/spaces/Surn/HexaGrid/commit/{"huggingface" if commit == "<none>" else commit}" target="_blank">{"huggingface" if commit == "<none>" else commit}</a>
|
108 |
 • 
|
@@ -123,6 +132,8 @@ def versions_html():
|
|
123 |
gradio: {gr.__version__}
|
124 |
 • 
|
125 |
{toggle_dark_link}
|
|
|
|
|
126 |
<br>
|
127 |
Full GPU Info:{get_torch_info()}
|
128 |
"""
|
|
|
103 |
</a>
|
104 |
'''
|
105 |
|
106 |
+
# Add a link to the shortener JSON file in the Hugging Face repo
|
107 |
+
from utils.constants import HF_REPO_ID, SHORTENER_JSON_FILE # Import constants
|
108 |
+
shortener_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{SHORTENER_JSON_FILE}"
|
109 |
+
shortener_link = f'''
|
110 |
+
<a href="{shortener_url}" target="_blank" style="cursor: pointer; text-decoration: underline;">
|
111 |
+
View Shortener JSON
|
112 |
+
</a>
|
113 |
+
'''
|
114 |
+
|
115 |
v_html = f"""
|
116 |
version: <a href="https://huggingface.co/spaces/Surn/HexaGrid/commit/{"huggingface" if commit == "<none>" else commit}" target="_blank">{"huggingface" if commit == "<none>" else commit}</a>
|
117 |
 • 
|
|
|
132 |
gradio: {gr.__version__}
|
133 |
 • 
|
134 |
{toggle_dark_link}
|
135 |
+
 • 
|
136 |
+
{shortener_link}
|
137 |
<br>
|
138 |
Full GPU Info:{get_torch_info()}
|
139 |
"""
|