{"cells":[{"cell_type":"markdown","source":["# 1.Preparation"],"metadata":{"id":"Ll08ZcPICXjy"}},{"cell_type":"markdown","source":["Connect to Google drive"],"metadata":{"id":"ZFv572DsD1HL"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7ZQ84WjGITux","executionInfo":{"status":"ok","timestamp":1730838049326,"user_tz":300,"elapsed":29355,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"}},"outputId":"c6acc52d-30c9-4890-d159-f694f8f1bdad"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"markdown","source":["Change this to your own address."],"metadata":{"id":"3gIkEbflCrzr"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2343,"status":"ok","timestamp":1730838068494,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"},"user_tz":300},"id":"Y2bAkVcLJnL1","outputId":"3177ef79-a297-4a1b-e80a-5657323774d6"},"outputs":[{"output_type":"stream","name":"stdout","text":[" Add_Clustered_news_0.6\t\t\t\t HON_2023-10-30_2024-09-30.json\n","'Benchmark with Llama3 8b Data.ipynb'\t\t KO_2023-10-30_2024-09-30.json\n"," Clustered_news_0.6\t\t\t\t MCD_2023-10-30_2024-09-30.json\n"," config.json\t\t\t\t\t 'News repetition reduction.ipynb'\n"," dataset_dict.json\t\t\t\t __pycache__\n"," DIS_2023-10-30_2024-09-30.json\t\t\t test\n"," finetuned_models\t\t\t\t test_demo.ipynb\n"," fingpt-forecaster\t\t\t\t train\n"," fingpt-forecaster-1029\t\t\t\t train_lora.py\n"," fingpt-forecaster-1102\t\t\t\t train.sh\n"," fingpt-forecaster-1105\t\t\t\t Untitled\n"," fingpt-forecaster-dow-30-20230930-20240930-1-4-06 utils.py\n"," fingpt-forecaster-dow-30-20231030-20240930-1-4-06 wandb\n"," HD_2023-10-30_2024-09-30.json\n"]}],"source":["!ls \"/content/drive/MyDrive/Your own address\""]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1027,"status":"ok","timestamp":1730838072202,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"},"user_tz":300},"id":"hXdXz4gQIW-s","outputId":"caf67953-4185-4735-e4ec-3e7f1335e947"},"outputs":[{"output_type":"stream","name":"stdout","text":["/content/drive/MyDrive/Colab Notebooks/AI4Finance/FinForecaster/Benchmark with Llama3 8b Data\n"]}],"source":["%cd \"/content/drive/MyDrive/Your own address\""]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":30693,"status":"ok","timestamp":1730838104298,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"},"user_tz":300},"id":"gaN7bj1vKJoJ","outputId":"3efec6e9-f0ae-48ee-cd2b-a3a7ff314436"},"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n","Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.5.0+cu121)\n","Collecting datasets\n"," Downloading datasets-3.1.0-py3-none-any.whl.metadata (20 kB)\n","Requirement already satisfied: peft in /usr/local/lib/python3.10/dist-packages (0.13.2)\n","Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (0.18.5)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.24.7)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.1)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n","Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n","Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.6)\n","Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.4.2)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n","Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.10.0)\n","Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.1)\n","Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch) (1.3.0)\n","Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (17.0.0)\n","Collecting dill<0.3.9,>=0.3.0 (from datasets)\n"," Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.2.2)\n","Collecting xxhash (from datasets)\n"," Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n","Collecting multiprocess<0.70.17 (from datasets)\n"," Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n","Collecting fsspec (from torch)\n"," Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n","Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.10.10)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft) (5.9.5)\n","Requirement already satisfied: accelerate>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from peft) (0.34.2)\n","Requirement already satisfied: click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.7)\n","Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (0.4.0)\n","Requirement already satisfied: gitpython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.1.43)\n","Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from wandb) (4.3.6)\n","Requirement already satisfied: protobuf!=4.21.0,!=5.28.0,<6,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n","Requirement already satisfied: sentry-sdk>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (2.17.0)\n","Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb) (1.3.3)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (75.1.0)\n","Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n","Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.3)\n","Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.2.0)\n","Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.5.0)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n","Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.17.0)\n","Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n","Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from gitpython!=3.1.29,>=1.0.0->wandb) (4.0.11)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.8.30)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (3.0.2)\n","Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n","Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n","Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb) (5.0.1)\n","Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0)\n","Downloading datasets-3.1.0-py3-none-any.whl (480 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m18.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: xxhash, fsspec, dill, multiprocess, datasets\n"," Attempting uninstall: fsspec\n"," Found existing installation: fsspec 2024.10.0\n"," Uninstalling fsspec-2024.10.0:\n"," Successfully uninstalled fsspec-2024.10.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed datasets-3.1.0 dill-0.3.8 fsspec-2024.9.0 multiprocess-0.70.16 xxhash-3.5.0\n","Collecting deepspeed\n"," Downloading deepspeed-0.15.3.tar.gz (1.4 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m36.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Collecting hjson (from deepspeed)\n"," Downloading hjson-3.1.0-py3-none-any.whl.metadata (2.6 kB)\n","Requirement already satisfied: msgpack in /usr/local/lib/python3.10/dist-packages (from deepspeed) (1.1.0)\n","Collecting ninja (from deepspeed)\n"," Downloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl.metadata (5.3 kB)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from deepspeed) (1.26.4)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from deepspeed) (24.1)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from deepspeed) (5.9.5)\n","Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from deepspeed) (9.0.0)\n","Requirement already satisfied: pydantic>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from deepspeed) (2.9.2)\n","Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from deepspeed) (2.5.0+cu121)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from deepspeed) (4.66.6)\n","Collecting nvidia-ml-py (from deepspeed)\n"," Downloading nvidia_ml_py-12.560.30-py3-none-any.whl.metadata (8.6 kB)\n","Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0.0->deepspeed) (0.7.0)\n","Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0.0->deepspeed) (2.23.4)\n","Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0.0->deepspeed) (4.12.2)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->deepspeed) (3.16.1)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->deepspeed) (3.4.2)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->deepspeed) (3.1.4)\n","Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->deepspeed) (2024.9.0)\n","Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch->deepspeed) (1.13.1)\n","Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch->deepspeed) (1.3.0)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->deepspeed) (3.0.2)\n","Downloading hjson-3.1.0-py3-none-any.whl (54 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.0/54.0 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl (307 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.2/307.2 kB\u001b[0m \u001b[31m27.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading nvidia_ml_py-12.560.30-py3-none-any.whl (40 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.5/40.5 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hBuilding wheels for collected packages: deepspeed\n"," Building wheel for deepspeed (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for deepspeed: filename=deepspeed-0.15.3-py3-none-any.whl size=1526205 sha256=bfed5756cc35d7fda60645355b0daceb7e6256a139a3dad4bdb930a2ddc17800\n"," Stored in directory: /root/.cache/pip/wheels/b3/c2/9f/37a2c813b8d64d7908793319cfdfa4f852754e177f20f0b858\n","Successfully built deepspeed\n","Installing collected packages: nvidia-ml-py, ninja, hjson, deepspeed\n","Successfully installed deepspeed-0.15.3 hjson-3.1.0 ninja-1.11.1.1 nvidia-ml-py-12.560.30\n","Collecting rouge-score\n"," Downloading rouge_score-0.1.2.tar.gz (17 kB)\n"," Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score) (1.4.0)\n","Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score) (3.8.1)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from rouge-score) (1.26.4)\n","Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score) (1.16.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (8.1.7)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (1.4.2)\n","Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (2024.9.11)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (4.66.6)\n","Building wheels for collected packages: rouge-score\n"," Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24935 sha256=5e4083d77d9a21f4705b4242ea56de520a09047ebd9dc854fd693ec23994161a\n"," Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n","Successfully built rouge-score\n","Installing collected packages: rouge-score\n","Successfully installed rouge-score-0.1.2\n"]}],"source":["!pip install transformers torch datasets peft wandb\n","!pip install deepspeed\n","!pip install rouge-score"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2145,"status":"ok","timestamp":1730838110126,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"},"user_tz":300},"id":"7qq3kEezKLkG","outputId":"290ab638-69c3-4c58-8ab4-fcefb40969b9"},"outputs":[{"output_type":"stream","name":"stdout","text":["True\n","NVIDIA A100-SXM4-40GB\n"]}],"source":["import torch\n","print(torch.cuda.is_available())\n","print(torch.cuda.get_device_name(0))"]},{"cell_type":"markdown","source":["# 2.Data Loading and Formatting"],"metadata":{"id":"mxBlA3wCC2PF"}},{"cell_type":"code","source":["from datasets import load_from_disk\n","\n","\n","train_dataset = load_from_disk('/content/drive/MyDrive/Your Trainging data address/')"],"metadata":{"id":"-K4_IXjmS761","executionInfo":{"status":"error","timestamp":1735134054927,"user_tz":-480,"elapsed":1238,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"}},"colab":{"base_uri":"https://localhost:8080/","height":380},"outputId":"96aaa89b-b9cc-444d-f0b3-433bd9187a98"},"execution_count":1,"outputs":[{"output_type":"error","ename":"ModuleNotFoundError","evalue":"No module named 'datasets'","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mdatasets\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mload_from_disk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mtrain_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_from_disk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/Your Trainging data address/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'datasets'","","\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"],"errorDetails":{"actions":[{"action":"open_url","actionText":"Open Examples","url":"/notebooks/snippets/importing_libraries.ipynb"}]}}]},{"cell_type":"code","source":["def get_unique_ticker_symbols(test_dataset):\n","\n"," ticker_symbols = set()\n","\n","\n"," for i in range(len(test_dataset)):\n"," prompt_content = test_dataset[i]['prompt']\n","\n","\n"," ticker_symbol = re.search(r\"ticker\\s([A-Z]+)\", prompt_content)\n","\n","\n"," if ticker_symbol:\n"," ticker_symbols.add(ticker_symbol.group(1))\n","\n","\n"," return list(ticker_symbols)\n","\n","\n","\n","def insert_guidance_after_intro(prompt):\n","\n","\n"," intro_marker = (\n"," \"[INST]<>\\n\"\n"," \"You are a seasoned stock market analyst. Your task is to list the positive developments and \"\n"," \"potential concerns for companies based on relevant news and basic financials from the past weeks, \"\n"," \"then provide an analysis and prediction for the companies' stock price movement for the upcoming week.\"\n"," )\n"," guidance_start_marker = \"Based on all the information before\"\n"," guidance_end_marker = \"Following these instructions, please come up with 2-4 most important positive factors\"\n","\n","\n"," intro_pos = prompt.find(intro_marker)\n"," guidance_start_pos = prompt.find(guidance_start_marker)\n"," guidance_end_pos = prompt.find(guidance_end_marker)\n","\n","\n"," if intro_pos == -1 or guidance_start_pos == -1 or guidance_end_pos == -1:\n"," return prompt\n","\n","\n"," guidance_section = prompt[guidance_start_pos:guidance_end_pos].strip()\n","\n","\n"," new_prompt = (\n"," f\"{prompt[:intro_pos + len(intro_marker)]}\\n\\n\"\n"," f\"{guidance_section}\\n\\n\"\n"," f\"{prompt[intro_pos + len(intro_marker):guidance_start_pos]}\"\n"," f\"{prompt[guidance_end_pos:]}\"\n"," )\n","\n"," return new_prompt\n","\n","\n","def apply_to_all_prompts_in_dataset(test_dataset):\n","\n","\n"," updated_dataset = test_dataset.map(lambda x: {\"prompt\": insert_guidance_after_intro(x[\"prompt\"])})\n","\n"," return updated_dataset"],"metadata":{"id":"WbnoctfhUi6X"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_dataset = apply_to_all_prompts_in_dataset(train_dataset)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":49,"referenced_widgets":["ec45cd4da15f4b998a1e185764f5b16d","e57edd2bb9b74676bfdcf66b12c8cc5c","f3f7a941eff946d88f082a482fec1f11","857ccdeccef04eddb1a833c0cfb8535f","74e35720b9074ccdba2e495ffd0895dd","3927b957970548da96f830871fffdbf9","b666036aa98f42d48a97b224da3fd77c","41032241335949a18ed6d8b7e167b581","e5e886e61cf549539175829a2274fdb4","f7fcd72791de411983343b0e69dbddeb","87f8a4e0e5f14751a7f19273118aaafe"]},"id":"lDruyX1GTEcg","executionInfo":{"status":"ok","timestamp":1730838131458,"user_tz":300,"elapsed":1030,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"}},"outputId":"2c366b9c-1041-4866-9b60-4a99f1422286"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["Map: 0%| | 0/784 [00:00>\\nYou are a seasoned stock market analyst. Your task is to list the positive developments and potential concerns for companies based on relevant news and basic financials from the past weeks, then provide an analysis and prediction for the companies' stock price movement for the upcoming week. Your answer format should be as follows:\\n\\n[Positive Developments]:\\n1. ...\\n\\n[Potential Concerns]:\\n1. ...\\n\\n[Prediction & Analysis]\\nPrediction: ...\\nAnalysis: ...\\n\\n<>\\n\\n[Company Introduction]:\\n\\nMicrosoft Corp is a leading entity in the Technology sector. Incorporated and publicly traded since 1986-03-13, the company has established its reputation as one of the key players in the market. As of today, Microsoft Corp has a market capitalization of 3051052.10 in USD, with 7433.04 shares outstanding.\\n\\nMicrosoft Corp operates primarily in the US, trading under the ticker MSFT on the NASDAQ NMS - GLOBAL MARKET. As a dominant force in the Technology space, the company continues to innovate and drive progress within the industry.\\n\\nFrom 2023-11-05 to 2023-11-12, MSFT's stock price increased from 350.17 to 366.92. The stock price each day during this period is [353.874, 357.844, 360.494, 358.003, 366.916], and the corresponding daily return is given by [0.0106, 0.0112, 0.0074, -0.006900000000000001, 0.024900000000000002]. News during this period are listed below:\\n\\n[Topic]: Microsoft Corp. stock outperforms competitors on strong trading day\\n[Topic Duration]:2023-11-07 to 2023-11-12\\n[News Headline]:Microsoft Corp. stock outperforms competitors on strong trading day\\n[News Summary]: Shares of Microsoft Corp. rose 2.49% to $369.67 Friday, on what proved to be an all-around favorable trading session for the stock market, with the S&P 500...\\n[Topic Size]:2\\n\\n[Topic]: You'll Never Guess Who's Beating OpenAI in Generative Artificial Intelligence\\n[Topic Duration]:2023-11-07 to 2023-11-10\\n[News Headline]:You'll Never Guess Who's Beating OpenAI in Generative Artificial Intelligence\\n[News Summary]: OpenAI burst onto the scene about a year ago with the launch of ChatGPT, introducing the world to the power of generative AI. It's since been seen as the leader in artificial intelligence, garnering a $10 billion investment from Microsoft (NASDAQ: MSFT). While OpenAI's GPT-4 is seen as the standard bearer, and it recently released an update on that model, one company is making quick progress in the space, even exceeding GPT-4's performance in some applications.\\n[Topic Size]:2\\n\\n[Topic]: AvePoint, Inc. (NASDAQ:AVPT) Q3 2023 Earnings Call Transcript\\n[Topic Duration]:2023-11-07 to 2023-11-12\\n[News Headline]:AvePoint, Inc. (NASDAQ:AVPT) Q3 2023 Earnings Call Transcript\\n[News Summary]: AvePoint, Inc. (NASDAQ:AVPT) Q3 2023 Earnings Call Transcript November 9, 2023 Operator: Good day and welcome to the AvePoint Inc. Q3 2023 Earnings Call. Today, all participants will be in a listen-only mode. [Operator Instructions] After today’s presentation there will be an opportunity to ask questions. [Operator Instructions] Please note that today’s event is being […]\\n[Topic Size]:2\\n\\n[Topic]: 3 Top Tech Stocks to Buy in November\\n[Topic Duration]:2023-11-07 to 2023-11-09\\n[News Headline]:3 Top Tech Stocks to Buy in November\\n[News Summary]: These companies dominate their respective markets, and you won't want to miss out on their long-term potential.\\n[Topic Size]:6\\n\\n[Topic]: Tech Titans: 2 Top Artificial Intelligence (AI) Stocks to Watch Heading Into 2024\\n[Topic Duration]:2023-11-08 to 2023-11-12\\n[News Headline]:Tech Titans: 2 Top Artificial Intelligence (AI) Stocks to Watch Heading Into 2024\\n[News Summary]: Although the AI revolution was unofficially kicked off in late 2022 when ChatGPT rose to fame, 2023 saw many incredible releases by some of the biggest tech names in the industry. The top two names I'm watching in this space in 2024 are Nvidia (NASDAQ: NVDA) and Microsoft (NASDAQ: MSFT). Nvidia has been the undisputed king of the AI movement, as its graphics processing units (GPUs) are the bedrock for the computers that create AI models.\\n[Topic Size]:12\\n\\n[Topic]: Nvidia: Assuring Processor Dominance By Technology And Strategic Investments\\n[Topic Duration]:2023-11-09 to 2023-11-10\\n[News Headline]:Nvidia: Assuring Processor Dominance By Technology And Strategic Investments\\n[News Summary]: Nvidia faces competition from Intel and AMD in cloud AI market, but strategic investments and top technology ensure dominance. Learn more on NVDA stock here.\\n[Topic Size]:2\\n\\n[Basic Financials]:\\n\\nNo basic financial reported.\\n\\nBased on all the information before 2023-11-12 and the following guidelines, let's analyze the positive developments and potential concerns for AAPL. Apart from stock price information and basic financials, several news topcis are given and they are derived from news clustering. Each topic contains its duration, a key news headline and summary, and a topic size, indicating the number of related news items within that cluster. First, note that some of the factors only influence the price for the following 1 or 2 days while some others may have longer-term effects. Second, the topics that are closer to 2023-11-12 are likely to have a stronger influence on the upcoming stock price forecast. Third, take into account the topic size: larger clusters represent greater market attention and, consequently, likely exert more influence on the stock price. Following these instructions, please come up with 2-4 most important positive factors and 2-4 most significant negative effects and keep them concise. Most factors should be inferred from company related news.Then make your prediction of the MSFT price movement for next week (2023-11-12 to 2023-11-19). Provide a summary analysis to support your prediction.[/INST]\""],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":12}]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":335,"status":"ok","timestamp":1730838142431,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"},"user_tz":300},"id":"OmVMmLfzKQ-2","outputId":"8f1a7316-6b68-4256-8dd8-2643548a8758"},"outputs":[{"output_type":"stream","name":"stdout","text":["The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n","Token is valid (permission: write).\n","Your token has been saved to /root/.cache/huggingface/token\n","Login successful\n"]}],"source":["from huggingface_hub import login\n","\n","# 输入你的 Hugging Face Access Token\n","login(token=\"Use your own key to make the call for the base model\")"]},{"cell_type":"markdown","source":["# 3.Executive training document"],"metadata":{"id":"AR6wkcg9Dgt_"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":12826876,"status":"ok","timestamp":1730851040185,"user":{"displayName":"Yuncong Liu","userId":"08340999060190968949"},"user_tz":300},"id":"sGY8yZvMLHHv","outputId":"56d8963c-9422-42eb-a626-509a7d973b36"},"outputs":[{"output_type":"stream","name":"stdout","text":["[2024-11-05 20:23:36,145] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n","[2024-11-05 20:23:42,089] [WARNING] [runner.py:215:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.\n","[2024-11-05 20:23:42,089] [INFO] [runner.py:607:main] cmd = /usr/bin/python3 -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMF19 --master_addr=127.0.0.1 --master_port=29500 --enable_each_rank_log=None train_lora.py --run_name llama3-8b-a100-5e-5lr --base_model llama3 --dataset /content/drive/MyDrive/Colab Notebooks/AI4Finance/FinForecaster/Benchmark with Llama3 8b Data/fingpt-forecaster-1105/train/ --test_dataset /content/drive/MyDrive/Colab Notebooks/AI4Finance/FinForecaster/Benchmark with Llama3 8b Data/fingpt-forecaster-1105/test/ --max_length 8000 --batch_size 2 --gradient_accumulation_steps 8 --learning_rate 5e-5 --num_epochs 5 --log_interval 10 --warmup_ratio 0.03 --scheduler constant --evaluation_strategy steps --ds_config config.json\n","[2024-11-05 20:23:43,889] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n","[2024-11-05 20:23:47,226] [INFO] [launch.py:139:main] 0 NV_LIBNCCL_DEV_PACKAGE=libnccl-dev=2.19.3-1+cuda12.2\n","[2024-11-05 20:23:47,226] [INFO] [launch.py:139:main] 0 NV_LIBNCCL_DEV_PACKAGE_VERSION=2.19.3-1\n","[2024-11-05 20:23:47,226] [INFO] [launch.py:139:main] 0 NCCL_VERSION=2.19.3-1\n","[2024-11-05 20:23:47,226] [INFO] [launch.py:139:main] 0 NV_LIBNCCL_DEV_PACKAGE_NAME=libnccl-dev\n","[2024-11-05 20:23:47,226] [INFO] [launch.py:139:main] 0 NV_LIBNCCL_PACKAGE=libnccl2=2.19.3-1+cuda12.2\n","[2024-11-05 20:23:47,226] [INFO] [launch.py:139:main] 0 NCCL_IGNORE_DISABLED_P2P=1\n","[2024-11-05 20:23:47,226] [INFO] [launch.py:139:main] 0 NV_LIBNCCL_PACKAGE_NAME=libnccl2\n","[2024-11-05 20:23:47,226] [INFO] [launch.py:139:main] 0 NV_LIBNCCL_PACKAGE_VERSION=2.19.3-1\n","[2024-11-05 20:23:47,227] [INFO] [launch.py:146:main] WORLD INFO DICT: {'localhost': [0]}\n","[2024-11-05 20:23:47,227] [INFO] [launch.py:152:main] nnodes=1, num_local_procs=1, node_rank=0\n","[2024-11-05 20:23:47,227] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'localhost': [0]})\n","[2024-11-05 20:23:47,227] [INFO] [launch.py:164:main] dist_world_size=1\n","[2024-11-05 20:23:47,227] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0\n","[2024-11-05 20:23:47,227] [INFO] [launch.py:256:main] process 3936 spawned with command: ['/usr/bin/python3', '-u', 'train_lora.py', '--local_rank=0', '--run_name', 'llama3-8b-a100-5e-5lr', '--base_model', 'llama3', '--dataset', '/content/drive/MyDrive/Colab Notebooks/AI4Finance/FinForecaster/Benchmark with Llama3 8b Data/fingpt-forecaster-1105/train/', '--test_dataset', '/content/drive/MyDrive/Colab Notebooks/AI4Finance/FinForecaster/Benchmark with Llama3 8b Data/fingpt-forecaster-1105/test/', '--max_length', '8000', '--batch_size', '2', '--gradient_accumulation_steps', '8', '--learning_rate', '5e-5', '--num_epochs', '5', '--log_interval', '10', '--warmup_ratio', '0.03', '--scheduler', 'constant', '--evaluation_strategy', 'steps', '--ds_config', 'config.json']\n","2024-11-05 20:23:51.582831: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n","2024-11-05 20:23:51.598708: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n","2024-11-05 20:23:51.620479: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n","2024-11-05 20:23:51.627300: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n","2024-11-05 20:23:51.643049: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n","To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n","2024-11-05 20:23:52.804146: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n","\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n","\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33myl5440\u001b[0m (\u001b[33myl5440-columbia-university\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n","config.json: 100% 654/654 [00:00<00:00, 4.53MB/s]\n","model.safetensors.index.json: 100% 23.9k/23.9k [00:00<00:00, 70.5MB/s]\n","Downloading shards: 0% 0/4 [00:00