{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-04-09T06:13:55.446824Z", "iopub.status.busy": "2025-04-09T06:13:55.445794Z", "iopub.status.idle": "2025-04-09T06:13:56.137367Z", "shell.execute_reply": "2025-04-09T06:13:56.136554Z", "shell.execute_reply.started": "2025-04-09T06:13:55.446782Z" }, "tags": [] }, "outputs": [], "source": [ "from pathlib import Path\n", "\n", "from datasets_common import *" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "execution": { "iopub.execute_input": "2025-04-09T06:13:56.140406Z", "iopub.status.busy": "2025-04-09T06:13:56.138861Z", "iopub.status.idle": "2025-04-09T06:13:56.182854Z", "shell.execute_reply": "2025-04-09T06:13:56.182207Z", "shell.execute_reply.started": "2025-04-09T06:13:56.140363Z" }, "tags": [] }, "outputs": [], "source": [ "dest_dir = Path(globals()[\"_dh\"][0])\n", "json_filename = \"arxiv-metadata-oai-snapshot.json\"\n", "dataset = \"Cornell-University/arxiv\"\n", "old_label = \"categories\"\n", "new_label = \"category\"\n", "train_filename = \"arxiv_train.json\"\n", "test_filename = \"arxiv_test.json\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "execution": { "iopub.execute_input": "2025-04-09T06:13:56.184655Z", "iopub.status.busy": "2025-04-09T06:13:56.183825Z", "iopub.status.idle": "2025-04-09T06:15:23.665384Z", "shell.execute_reply": "2025-04-09T06:15:23.664523Z", "shell.execute_reply.started": "2025-04-09T06:13:56.184630Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Dataset already exists, do not download\n", "Reading dataset...\n", "Dataset read\n" ] } ], "source": [ "df = download_and_read_dataset(dest_dir=dest_dir, dataset=dataset, filename=json_filename)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2025-04-09T06:15:23.667946Z", "iopub.status.busy": "2025-04-09T06:15:23.666981Z", "iopub.status.idle": "2025-04-09T06:15:23.702581Z", "shell.execute_reply": "2025-04-09T06:15:23.701966Z", "shell.execute_reply.started": "2025-04-09T06:15:23.667909Z" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idsubmitterauthorstitlecommentsjournal-refdoireport-nocategorieslicenseabstractversionsupdate_dateauthors_parsed
00704.0001Pavel NadolskyC. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...Calculation of prompt diphoton production cros...37 pages, 15 figures; published versionPhys.Rev.D76:013009,200710.1103/PhysRevD.76.013009ANL-HEP-PR-07-12hep-phNoneA fully differential calculation in perturba...[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...2008-11-26[[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...
10704.0002Louis TheranIleana Streinu and Louis TheranSparsity-certifying Graph DecompositionsTo appear in Graphs and CombinatoricsNoneNoneNonemath.CO cs.CGhttp://arxiv.org/licenses/nonexclusive-distrib...We describe a new algorithm, the $(k,\\ell)$-...[{'version': 'v1', 'created': 'Sat, 31 Mar 200...2008-12-13[[Streinu, Ileana, ], [Theran, Louis, ]]
20704.0003Hongjun PanHongjun PanThe evolution of the Earth-Moon system based o...23 pages, 3 figuresNoneNoneNonephysics.gen-phNoneThe evolution of Earth-Moon system is descri...[{'version': 'v1', 'created': 'Sun, 1 Apr 2007...2008-01-13[[Pan, Hongjun, ]]
30704.0004David CallanDavid CallanA determinant of Stirling cycle numbers counts...11 pagesNoneNoneNonemath.CONoneWe show that a determinant of Stirling cycle...[{'version': 'v1', 'created': 'Sat, 31 Mar 200...2007-05-23[[Callan, David, ]]
40704.0005Alberto TorchinskyWael Abu-Shammala and Alberto TorchinskyFrom dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...NoneIllinois J. Math. 52 (2008) no.2, 681-689NoneNonemath.CA math.FANoneIn this paper we show how to compute the $\\L...[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...2013-10-15[[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]
\n", "
" ], "text/plain": [ " id ... authors_parsed\n", "0 0704.0001 ... [[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...\n", "1 0704.0002 ... [[Streinu, Ileana, ], [Theran, Louis, ]]\n", "2 0704.0003 ... [[Pan, Hongjun, ]]\n", "3 0704.0004 ... [[Callan, David, ]]\n", "4 0704.0005 ... [[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]\n", "\n", "[5 rows x 14 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2025-04-09T06:15:23.704471Z", "iopub.status.busy": "2025-04-09T06:15:23.703504Z", "iopub.status.idle": "2025-04-09T06:15:24.013564Z", "shell.execute_reply": "2025-04-09T06:15:24.012916Z", "shell.execute_reply.started": "2025-04-09T06:15:23.704435Z" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Removing unwanted columns...\n", "Columns removed...\n" ] } ], "source": [ "df = filter_columns(df=df, columns=[\"title\", \"authors\", \"abstract\", old_label])" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2025-04-09T06:15:24.015952Z", "iopub.status.busy": "2025-04-09T06:15:24.014301Z", "iopub.status.idle": "2025-04-09T06:15:25.912683Z", "shell.execute_reply": "2025-04-09T06:15:25.911811Z", "shell.execute_reply.started": "2025-04-09T06:15:24.015915Z" }, "tags": [] }, "outputs": [], "source": [ "X, y = create_features_labels(df=df, old_label=old_label, new_label=new_label)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2025-04-09T06:15:25.914486Z", "iopub.status.busy": "2025-04-09T06:15:25.913573Z", "iopub.status.idle": "2025-04-09T06:15:36.005889Z", "shell.execute_reply": "2025-04-09T06:15:36.005131Z", "shell.execute_reply.started": "2025-04-09T06:15:25.914449Z" } }, "outputs": [], "source": [ "X_train, X_test, y_train, y_test = train_test_split(X, y)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "execution": { "iopub.execute_input": "2025-04-09T06:15:36.007677Z", "iopub.status.busy": "2025-04-09T06:15:36.006731Z", "iopub.status.idle": "2025-04-09T06:17:57.585958Z", "shell.execute_reply": "2025-04-09T06:17:57.585110Z", "shell.execute_reply.started": "2025-04-09T06:15:36.007643Z" }, "tags": [] }, "outputs": [], "source": [ "write_dataset(dest_dir=dest_dir, X=X_train, y=y_train, filename=train_filename)\n", "write_dataset(dest_dir=dest_dir, X=X_test, y=y_test, filename=test_filename)" ] } ], "metadata": { "kernelspec": { "display_name": "DataSphere Kernel", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 4 }