Upload folder using huggingface_hub

handle_stage3.py

@@ -10,7 +10,7 @@ audio_asr_folder = '/mnt/bn/tns-algo-video-public-my2/wangpeng.an/data'
 audio_caption_folder = '/mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/audio_caption'
 video_folder = '/mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/video'
 image_folder = '/mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/video/Video-LLaVA'
-new_json_path = '/mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/multidataset/video_image_asr_caption_pre_1208.json'
+new_json_path = '/mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/multidataset/video_image_asr_caption_pre_1211.json'
 
 # Load JSON data
 with open(data_json_path, 'r') as f:
@@ -103,3 +103,11 @@ print(f"Saved {len(new_items)} valid items to {new_json_path}")
 print(f"Total and missing files by type:")
 for file_type, counts in file_counts.items():
     print(f"{file_type}: Total = {counts['total']}, Missing = {counts['missing']}")
+
+miss = {'image':[], 'video':[], 'audio_caption':[], 'audio_asr':[]}
+for text in texts:
+    if 'image' in text:
+        miss['image'].append(text)
+    if 'video' in text:
+        miss['video'].append(text)
+        

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94061919007efb55636a4c9f121e9ce6a72010369897ed1de44786f47b7b9bd1
+oid sha256:f313f086266a38788feecf8bfe1da2ad4d636efcca76d8c33694d7d706380ac3
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66230500e17331be75b19ba267fb48c4e9daab556e14e44e5b133af56c075d0b
+oid sha256:9226e926895e979759395e0fca0f6ee2cde9195436102fb8bf8f1c2185c6fac7
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33ad4f70058270714a406fa3534f8bfb48b0ca7b9ae3f5b4e95728f43c3c885b
+oid sha256:6f3e43677d7653cb27ecbf48f279ac17e165b23fa1b00bc2df659bca6b4e2928
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66ab5801d1e1bcb21a633277dc66ecc53d5f1c979e3aa72395ae5aa6a1f3eb73
+oid sha256:9f69b0b8973f33f4ccb0997073ad546ee2e9c9aed90af6d00bbee0b0981b39d8
 size 3851682320

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e36f4edd2ff45b541285c258aed99e86ee7ae16364aef14743a10e2c3a682c0
+oid sha256:6cd17e0233af3edc9aa948c15991532dc6890cde50f60d6c79801ed6637ccd56
 size 6776