Upload train_data_collect.ipynb with huggingface_hub

train_data_collect.ipynb

@@ -0,0 +1,471 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "def read_json(file_path): \n",
+    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
+    "        data = json.load(file)\n",
+    "    return data\n",
+    "\n",
+    "def write_json(file_path, data):\n",
+    "    with open(file_path, 'w', encoding='utf-8') as file:\n",
+    "        json.dump(data, file, ensure_ascii=False, indent=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "# 读取 jsonl 文件\n",
+    "def read_jsonl(file_path):\n",
+    "    data = []\n",
+    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
+    "        for line in file:\n",
+    "            data.append(json.loads(line.strip()))\n",
+    "    return data\n",
+    "\n",
+    "# 写入到 jsonl 文件\n",
+    "def write_jsonl(data, file_path):\n",
+    "    with open(file_path, 'w', encoding='utf-8') as file:\n",
+    "        for entry in data:\n",
+    "            json_str = json.dumps(entry, ensure_ascii=False)\n",
+    "            file.write(json_str + '\\n')\n",
+    "\n",
+    "# 示例使用\n",
+    "input_file = 'input.jsonl'  # 输入的 jsonl 文件路径\n",
+    "output_file = 'output.jsonl'  # 输出的 jsonl 文件路径"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# data1 = read_jsonl('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/NIPS_2025/LLaVA_Fliter/inference_demo/New_llava_cherry_loss_infer_result_NImg__2025_01_08_.jsonl')\n",
+    "# data2 = read_jsonl('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/NIPS_2025/LLaVA_Fliter/inference_demo/New_llava_cherry_loss_infer_result_NImg__2025_01_09_.jsonl')\n",
+    "\n",
+    "\n",
+    "# data1 = read_jsonl('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/NIPS_2025/LLaVA_Fliter/inference_demo/New_llava_Logits_Img__2025_01_10_.jsonl')\n",
+    "# data2 = read_jsonl('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/NIPS_2025/LLaVA_Fliter/inference_demo/New_llava_Logits_Img__2025_01_11_.jsonl')\n",
+    "# data3 = read_jsonl('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/NIPS_2025/LLaVA_Fliter/inference_demo/New_llava_Logits_NImg__2025_01_10_.jsonl')\n",
+    "\n",
+    "\n",
+    "\n",
+    "data1 = read_jsonl('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/NIPS_2025/LLaVA_Fliter/inference_demo/New_llava_Logits_Img__2025_01_12_.jsonl')\n",
+    "data2 = read_jsonl('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/NIPS_2025/LLaVA_Fliter/inference_demo/New_llava_Logits_Img__2025_01_13_.jsonl')\n",
+    "data4 = read_jsonl('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/NIPS_2025/LLaVA_Fliter/inference_demo/New_llava_Logits_NImg__2025_01_12_.jsonl')\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "624640"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(data4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_all = data1+data2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "624640"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(data_all)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "({'id': 397,\n",
+       "  'image': 'llava_image_tune/coco/train2017/000000266244.jpg',\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"############\\n<image>\\nIs there a person riding a bicycle in the image? Yes, there is a woman riding a bicycle in the image. What unique feature does the bicycle have in the image? The unique feature of the bicycle in the image is that it has an umbrella attached to it, providing cover to the person riding the bike. How many people are visible in the image? In the image, only one person is visible - the woman riding the bicycle. Is attaching an umbrella to a bicycle practical or beneficial? Attaching an umbrella to a bicycle can be considered a practical idea in certain situations. Having an umbrella attached to a bicycle can provide protection from rain, keeping the rider dry during wet weather conditions. It can also shield the rider from direct sunlight on hot and sunny days, providing some shade and comfort.\\n\\nHowever, the practicality of using an umbrella attached to a bicycle could be affected by several factors:\\n\\n1. Wind: Strong winds may make it challenging to control the umbrella and the bicycle, decreasing stability and increasing the risk of accidents.\\n\\n2. Size and Weight: The size and weight of the umbrella may affect the bicycle's balance and handling.\\n\\n3. Obstructions: An umbrella may restrict the rider's field of vision, causing potential safety concerns by obstructing the view of road signs, other road users, or hazards.\\n\\n4. Local Regulations: Some jurisdictions may have rules or regulations restricting the use of umbrellas on bicycles for safety reasons.\\n\\nIn conclusion, while attaching an umbrella to a bicycle can be practical and offer benefits in some situations, it would be essential to consider various factors, such as wind conditions, size and weight, obstructions, and local regulations, to ensure safe and effective use. ############\\n \\nDoes the previous paragraph demarcated within ### and ###\\ncontain informative signal for visual instruction tuning a vision-language model?\\nAn informative datapoint should be well-formatted, contain some\\nusable knowledge of the world, and strictly NOT have any harmful,\\nracist, sexist, etc. content.\\nOPTIONS:\\n- yes\\n- no\\n\"},\n",
+       "   {'from': 'gpt', 'value': 'response: yes'}],\n",
+       "  'ori_conversations': [{'from': 'human',\n",
+       "    'value': '<image>\\nIs there a person riding a bicycle in the image?'},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'Yes, there is a woman riding a bicycle in the image.'},\n",
+       "   {'from': 'human',\n",
+       "    'value': 'What unique feature does the bicycle have in the image?'},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'The unique feature of the bicycle in the image is that it has an umbrella attached to it, providing cover to the person riding the bike.'},\n",
+       "   {'from': 'human', 'value': 'How many people are visible in the image?'},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'In the image, only one person is visible - the woman riding the bicycle.'},\n",
+       "   {'from': 'human',\n",
+       "    'value': 'Is attaching an umbrella to a bicycle practical or beneficial?'},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': \"Attaching an umbrella to a bicycle can be considered a practical idea in certain situations. Having an umbrella attached to a bicycle can provide protection from rain, keeping the rider dry during wet weather conditions. It can also shield the rider from direct sunlight on hot and sunny days, providing some shade and comfort.\\n\\nHowever, the practicality of using an umbrella attached to a bicycle could be affected by several factors:\\n\\n1. Wind: Strong winds may make it challenging to control the umbrella and the bicycle, decreasing stability and increasing the risk of accidents.\\n\\n2. Size and Weight: The size and weight of the umbrella may affect the bicycle's balance and handling.\\n\\n3. Obstructions: An umbrella may restrict the rider's field of vision, causing potential safety concerns by obstructing the view of road signs, other road users, or hazards.\\n\\n4. Local Regulations: Some jurisdictions may have rules or regulations restricting the use of umbrellas on bicycles for safety reasons.\\n\\nIn conclusion, while attaching an umbrella to a bicycle can be practical and offer benefits in some situations, it would be essential to consider various factors, such as wind conditions, size and weight, obstructions, and local regulations, to ensure safe and effective use.\"}],\n",
+       "  'yes_target_logprob_7B_Img': -14.75,\n",
+       "  'logits_shape': [1, 32000]},\n",
+       " {'id': 244066,\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': '############\\n\\nWhere is this taken?\\nAnswer the question using a single word or phrase. Circus ############\\n \\nDoes the previous paragraph demarcated within ### and ###\\ncontain informative signal for visual instruction tuning a vision-language model?\\nAn informative datapoint should be well-formatted, contain some\\nusable knowledge of the world, and strictly NOT have any harmful,\\nracist, sexist, etc. content.\\nOPTIONS:\\n- yes\\n- no\\n'},\n",
+       "   {'from': 'gpt', 'value': 'response: yes'}],\n",
+       "  'ori_conversations': [{'from': 'human',\n",
+       "    'value': '<image>\\nWhere is this taken?\\nAnswer the question using a single word or phrase.'},\n",
+       "   {'from': 'gpt', 'value': 'Circus'}],\n",
+       "  'Old_Path': 'llava_image_tune/coco/train2017/000000355857.jpg',\n",
+       "  'yes_target_logprob_7B_NImg': -15.875,\n",
+       "  'logits_shape': [1, 32000]})"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data_all[0],data4[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_list = []\n",
+    "for i in data_all:\n",
+    "    data_list.append(i['yes_target_logprob_7B_Img'])\n",
+    "    \n",
+    "\n",
+    "# data_list = []\n",
+    "# for i in data4:\n",
+    "#     data_list.append(i['yes_target_logprob_7B_NImg'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[-14.75,\n",
+       " -15.4375,\n",
+       " -14.6875,\n",
+       " -14.3125,\n",
+       " -14.375,\n",
+       " -14.3125,\n",
+       " -14.4375,\n",
+       " -14.0625,\n",
+       " -15.4375,\n",
+       " -13.9375,\n",
+       " -12.625,\n",
+       " -16.25,\n",
+       " -12.0,\n",
+       " -15.6875,\n",
+       " -13.375,\n",
+       " -13.5,\n",
+       " -13.6875,\n",
+       " -11.5,\n",
+       " -12.0625,\n",
+       " -11.8125,\n",
+       " -12.3125,\n",
+       " -11.5625,\n",
+       " -11.5,\n",
+       " -12.1875,\n",
+       " -12.9375,\n",
+       " -11.75,\n",
+       " -11.6875,\n",
+       " -12.75,\n",
+       " -10.625,\n",
+       " -10.1875,\n",
+       " -12.375,\n",
+       " -10.75,\n",
+       " -15.3125,\n",
+       " -15.375,\n",
+       " -13.4375,\n",
+       " -14.75,\n",
+       " -13.6875,\n",
+       " -15.5625,\n",
+       " -14.1875,\n",
+       " -14.25,\n",
+       " -13.8125,\n",
+       " -14.0625,\n",
+       " -14.125,\n",
+       " -14.0,\n",
+       " -13.6875,\n",
+       " -13.0,\n",
+       " -13.3125,\n",
+       " -13.5,\n",
+       " -11.3125,\n",
+       " -10.9375,\n",
+       " -11.6875,\n",
+       " -13.125,\n",
+       " -10.0625,\n",
+       " -9.8125,\n",
+       " -11.0,\n",
+       " -11.0,\n",
+       " -10.1875,\n",
+       " -11.625,\n",
+       " -10.6875,\n",
+       " -10.9375,\n",
+       " -10.1875,\n",
+       " -10.5,\n",
+       " -12.0,\n",
+       " -11.3125,\n",
+       " -15.75,\n",
+       " -14.875,\n",
+       " -14.5,\n",
+       " -14.5,\n",
+       " -14.25,\n",
+       " -14.1875,\n",
+       " -14.375,\n",
+       " -15.0625,\n",
+       " -15.0625,\n",
+       " -14.0,\n",
+       " -12.625,\n",
+       " -13.125,\n",
+       " -13.375,\n",
+       " -13.5625,\n",
+       " -13.0,\n",
+       " -13.4375,\n",
+       " -12.8125,\n",
+       " -10.375,\n",
+       " -11.6875,\n",
+       " -11.125,\n",
+       " -11.625,\n",
+       " -11.1875,\n",
+       " -12.3125,\n",
+       " -11.3125,\n",
+       " -12.25,\n",
+       " -14.0,\n",
+       " -11.3125,\n",
+       " -11.25,\n",
+       " -12.8125,\n",
+       " -12.3125,\n",
+       " -12.0,\n",
+       " -11.125,\n",
+       " -13.8125,\n",
+       " -15.3125,\n",
+       " -15.4375,\n",
+       " -13.125]"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data_list[:100]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "\n",
+    "\n",
+    "import numpy as np\n",
+    "\n",
+    "data = np.array(data_list)\n",
+    "data[np.isnan(data)] = 1\n",
+    "temop1 = np.percentile(data, 80)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "New_data_list = []\n",
+    "for i in data_all:\n",
+    "    if i['yes_target_logprob_7B_Img'] >= temop1:\n",
+    "        New_data_list.append(i )\n",
+    "    \n",
+    "    \n",
+    "# for i in data4:\n",
+    "#     if i['yes_target_logprob_7B_NImg'] >= temop1:\n",
+    "#         New_data_list.append(i )\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "128201"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(New_data_list)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# write_json('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/DataSet/LLaVA-Select/llava_image_tune_loss_7B_Img_20P.json',New_data_list)\n",
+    "# write_json('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/DataSet/LLaVA-Select/llava_image_tune_Logits_7B_Img_20P.json',New_data_list)\n",
+    "# write_json('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/DataSet/LLaVA-Select/llava_image_tune_Logits_7B_NImg_20P.json',New_data_list)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_rand = read_json('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/DataSet/LLaVA-Select/llava_image_tune_rand_20P.json')\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "new_40P = New_data_list + data_rand"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "253123"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(new_40P)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# write_json('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/DataSet/LLaVA-Select/llava_image_tune_loss_rand_7B_Img_40P.json',new_40P)\n",
+    "# write_json('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/DataSet/LLaVA-Select/llava_image_tune_Logits_Rand_7B_Img_40P.json',new_40P)\n",
+    "# write_json('/inspire/hdd/ws-ba572160-47f8-4ca1-984e-d6bcdeb95dbb/a100-maybe/albus/DataSet/LLaVA-Select/llava_image_tune_Logits_Rand_7B_NImg_40P.json',new_40P)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "250207"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}