File size: 26,412 Bytes

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "89751719-bb82-42ef-b49e-182e81b0c478",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "summarizer = pipeline(\"summarization\", model=\"cheaptrix/congress_bill_summary_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "883134f1-934d-422f-a64a-0a37073542c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "cleaned_test_data = []\n",
    "# Opening JSON file\n",
    "with open(\"cleaned_bill_sum_test_data.json\") as json_file:\n",
    "    cleaned_test_data = json.load(json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c7f5e7f6-ac9f-4ba7-acbf-1ba7d031ef81",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (548 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "generated_summaries = []\n",
    "for i in range(len(cleaned_test_data)):\n",
    "    summary = summarizer(cleaned_test_data[i]['text'])\n",
    "    generated_summaries.append({\"generated\" : summary[0]['summary_text'], \"summary\" : cleaned_test_data[i]['summary']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cca1a292-1ab5-452f-b998-d404bfc633d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This resolution requires the Architect of the Capitol, the Secretary of the Senate, and the Chief Administrative Officer of the House of Representatives to encourage Capitol gift shops to accept cryptocurrency and to enter into contracts with vendors that accept cryptocurrency to provide food service and vending machines in the Capitol.',\n",
       " 'summary': 'Adopting Cryptocurrency in Congress as an Exchange of Payment for Transactions Resolution or the ACCEPT Resolution This resolution requires the Architect of the Capitol, the Secretary of the Senate, and the Chief Administrative Officer of the House of Representatives to encourage Capitol gift shops to accept cryptocurrency and to enter into contracts with vendors that accept cryptocurrency to provide food service and vending machines in the Capitol.'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "22b594f4-b8a1-4296-bd19-31e799a368e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution commends the bravery, courage, and resolve of the women and men of Iran who are (1) participating in the current protests to defend their fundamental human rights, and (2) risking their safety to speak out against the human rights abuses committed by the Iranian regime.',\n",
       " 'summary': 'This concurrent resolution commends the bravery, courage, and resolve of the women and men of Iran who are (1) participating in the current protests to defend their fundamental human rights, and (2) risking their safety to speak out against the human rights abuses committed by the Iranian regime. The resolution condemns (1) the brutal beating and death of Mahsa Amini; and (2) the violent suppression by the Iranian regime of women and men participating in the current demonstrations, including children, and calls for transparent accountability for all killings of protesters by Iranian security forces. Finally, the resolution encourages continued efforts by the Biden Administration to respond to the protests, including the recent sanctioning of the Iranian morality police.'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7f0a025a-290f-4811-b504-793e2e0dbde2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution calls for honoring the 237th anniversary of the enactment of the Virginia Statute for Religious Freedom on Religious Freedom Day, January 16, 2023. The resolution affirms that religious freedom includes the right of individuals of any faith and individuals of no faith to live, work, associate, and worship in accordance with their beliefs; all people of the United States can be unified in supporting religious freedom because it is a fundamental human right; and the American people will remain forever unshackled in matters of faith.',\n",
       " 'summary': 'This concurrent resolution calls for honoring the 237th anniversary of the enactment of the Virginia Statute for Religious Freedom on Religious Freedom Day, January 16, 2023. The resolution affirms that religious freedom includes the right of individuals of any faith and individuals of no faith to live, work, associate, and worship in accordance with their beliefs; all people of the United States can be unified in supporting religious freedom because it is a fundamental human right; and the American people will remain forever unshackled in matters of faith.'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e0b81c81-b0ba-4285-988a-ce9b070e215c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution requires the Joint Committee on the Library to approve or deny the statue of Rev. Billy Graham for placement in the National Statuary Hall within 30 days after North Carolina submits specific information about the statute, including its dimensions and final weight.',\n",
       " 'summary': 'This concurrent resolution requires the Joint Committee on the Library to approve or deny the statue of Rev. Billy Graham for placement in the National Statuary Hall within 30 days after North Carolina submits specific information about the statute, including its dimensions and final weight.'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8da96ded-73dd-474f-8c2b-b684f1aa825c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution declares that Congress should not impose any new performance fee, tax, royalty, or other charge relating to the public performance of sound recordings on a local radio station for broadcasting sound recordings over the air or on any business for such public performance .',\n",
       " 'summary': 'This concurrent resolution declares that Congress should not impose any new performance fee, tax, royalty, or other charge relating to the public performance of sound recordings on a local radio station for broadcasting sound recordings over the air or on any business for such public performance of sound recordings.'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6d51741f-fb47-48d2-a387-ef87577817df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution recognizes Abortion Provider Appreciation Day. Text/xml EN Pursuant to Title 17 Section 105 of the United States Code, this file is not subject to copyright protection and is in the public domain.',\n",
       " 'summary': 'This concurrent resolution recognizes Abortion Provider Appreciation Day.'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a69ce9c9-f18f-4aed-80f2-53fd81311555",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': \"This concurrent resolution condemns Russia's unjust and arbitrary detention of Russian democratic opposition leader Vladimir Kara-Murza and calls for his immediate release and the release of all other Russian opposition leaders.\",\n",
       " 'summary': \"This concurrent resolution condemns Russia's unjust and arbitrary detention of Russian democratic opposition leader Vladimir Kara-Murza and calls for his immediate release and the release of all other Russian opposition leaders. It also calls for the release of all political prisoners in Russia and in Belarus, as well as for the release of Ukrainian citizens held by Russia, and calls on the President and leaders of the free world to work tirelessly for the release of political prisoners in Russia.\"}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "14a0bc12-abe5-4dca-b97c-13577cf63ee5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution expresses the sense of Congress that tax-exempt fraternal benefit societies provide critical benefits to the people and communities of the United States and their work should continue to be promoted.',\n",
       " 'summary': 'This concurrent resolution expresses the sense of Congress that tax-exempt fraternal benefit societies provide critical benefits to the people and communities of the United States and their work should continue to be promoted.'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7d4415fe-7d50-4c12-9039-c338651f8972",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution expresses the sense of Congress that climate change caused by human activities constitutes a climate emergency, which demands the President use existing authorities and emergency powers to mitigate and prepare for the consequences of the emergency.',\n",
       " 'summary': 'This concurrent resolution expresses the sense of Congress that climate change caused by human activities constitutes a climate emergency, which demands the President use existing authorities and emergency powers to mitigate and prepare for the consequences of the emergency.'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ff9959f8-4c7a-4af3-8e3e-83cfde0f227a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': \"Fiscal State of the Nation Resolution This concurrent resolution requires the congressional budget committees to conduct an annual joint hearing to receive a presentation from the Comptroller General regarding (1) the Government Accountability Office's audit of the financial statement of the executive branch, and (2) the financial position and condition of the federal government.\",\n",
       " 'summary': \"Fiscal State of the Nation Resolution This concurrent resolution requires the congressional budget committees to conduct an annual joint hearing to receive a presentation from the Comptroller General regarding (1) the Government Accountability Office's audit of the financial statement of the executive branch, and (2) the financial position and condition of the federal government.\"}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "54e8d283-2011-4f9a-b20e-70c90cff42be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution expresses the sense that the Senate should provide its advice and consent to ratification of the Convention on Biological Diversity.',\n",
       " 'summary': 'This concurrent resolution expresses the sense that the Senate should provide its advice and consent to ratification of the Convention on Biological Diversity.'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cd4a017c-69d3-4891-b011-bdd19ca4a742",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution calls on the media to voluntarily adopt certain practices to prevent further harm from its coverage of mass public murders. Specifically, it calls for coverage that (1) denies murderers a public platform; (2) minimizes the potential for media reporting to increase the likelihood of future mass public killings (i.e., the media contagion effect); and (3) prioritizes the victims of, and heroism in the response to, mass public killers.',\n",
       " 'summary': 'This concurrent resolution calls on the media to voluntarily adopt certain practices to prevent further harm from its coverage of mass public murders. Specifically, it calls for coverage that (1) denies murderers a public platform; (2) minimizes the potential for media reporting to increase the likelihood of future mass public murders (i.e., the media contagion effect); and (3) prioritizes the victims of, and heroism in the response to, mass public murders.'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e2a659be-e404-447e-8a35-5dbd365e4faf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution affirms, more than 400 years after the arrival of the first slave ship, that the United States owes a debt of remembrance to those who lived through slavery or other historical injustices against people of color, as well as to their descendants. It also urges the establishment of a U.S. Commission on Truth, Racial Healing, and Transformation',\n",
       " 'summary': 'This concurrent resolution affirms, more than 400 years after the arrival of the first slave ship, that the United States owes a debt of remembrance to those who lived through slavery or other historical injustices against people of color, as well as to their descendants. It also urges the establishment of a U.S. Commission on Truth, Racial Healing, and Transformation.'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d859483d-a664-4eb4-8910-8c590461d48e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': \"This concurrent resolution recognizes the disparity between wages paid to Latina women in comparison to men and reaffirms Congress's support for ensuring equal pay and closing the gender wage gap.\",\n",
       " 'summary': \"This concurrent resolution recognizes the disparity between wages paid to Latina women in comparison to men and reaffirms Congress's support for ensuring equal pay and closing the gender wage gap.\"}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c66b1673-d087-4b52-88d4-ef5990ecbd90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution expresses the sense of Congress that a carbon tax would be detrimental to families and businesses and would severely harm the economic and national security of the country.',\n",
       " 'summary': 'This concurrent resolution expresses the sense of Congress that a carbon tax would be detrimental to families and businesses and would severely harm the economic and national security of the country.'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[14]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "205ff67d-894d-40f4-814c-d9f2d3eea1d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'This concurrent resolution makes a correction to the official title of H.R. 815 (Making emergency supplemental appropriations for the fiscal year ending September 30, 2024, and for other purposes).',\n",
       " 'summary': 'This concurrent resolution makes a correction to the official title of H.R. 815 (Making emergency supplemental appropriations for the fiscal year ending September 30, 2024, and for other purposes).'}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[15]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e3e9588-d3f3-4c0f-a86b-eb0e40d47079",
   "metadata": {},
   "source": [
    "# NEW DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3a6116df-634c-4cdc-8e17-e17519f06aa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_cleaned_test_data = []\n",
    "# Opening JSON file\n",
    "with open(\"new_cleaned_bill_sum_test_data.json\") as json_file:\n",
    "    new_cleaned_test_data = json.load(json_file)\n",
    "\n",
    "new_generated_summaries = []\n",
    "for i in range(len(new_cleaned_test_data)):\n",
    "    summary = summarizer(new_cleaned_test_data[i]['text'])\n",
    "    new_generated_summaries.append({\"generated\" : summary[0]['summary_text'], \"summary\" : new_cleaned_test_data[i]['summary']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4351fd53-9f09-4794-b34c-092732d27ff1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'Saving Access to Laboratory Services Act This bill modifies provisions relating to Medicare payment rates for clinical diagnostic laboratory services, including by requiring payments rates for certain widely available clinical diagnostic lab tests to be based on a statistical sampling of private sector rates.',\n",
       " 'summary': 'Saving Access to Laboratory Services Act This bill modifies provisions relating to Medicare payment rates for clinical diagnostic laboratory services, including by requiring payment rates for certain widely available clinical diagnostic laboratory tests to be based on a statistical sampling of private sector rates.'}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_generated_summaries[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f8f57bd4-5082-4c0a-a61b-523814f60db6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'USDA must periodically review and revise the plan, as necessary, and annually report on its implementation. Additionally, the bill expands the scope of various USDA research, extension, and education programs; conservation programs; and livestock programs to incorporate climate change adaptation and mitigation. Expanded activities include efforts to improve soil health and preserve farmland and grassland. Further, the Bill changes programs that support renewable energy in rural areas to address carbon emissions in the agriculture sector.',\n",
       " 'summary': 'Agriculture Resilience Act of 2023 This bill establishes, expands, and revises multiple programs and activities of the Department of Agriculture (USDA) primarily to reduce carbon emissions from the agriculture sector. Specifically, USDA must finalize and implement a plan to achieve net-zero emissions from the sector by 2040. USDA must periodically review and revise the plan, as necessary, and annually report on its implementation. Additionally, the bill expands the scope of various USDA research, extension, and education programs; conservation programs; and livestock programs to incorporate climate change adaptation and mitigation. Expanded activities include efforts to improve soil health and preserve farmland and grassland. Further, the bill changes programs that support renewable energy in rural areas to address carbon emissions in the agriculture sector. Among these changes, the bill provides statutory authority for the AgSTAR program for reducing methane emissions from livestock waste and requires the program to be moved from the Environmental Protection Agency to USDA. The bill also addresses food waste, for example, by (1) standardizing the voluntary labels used by food producers to indicate the date by which food should be used or discarded, and (2) making composting activities eligible for support through USDA conservation programs. Moreover, the bill establishes grants to reduce and prevent food waste in landfills and in schools.'}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_generated_summaries[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68c08713-89d9-4611-8cf5-9c86bed5fc29",
   "metadata": {},
   "source": [
    "# ROUGE EVALUATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e869ae4a-6126-4256-98b4-4c36556e4cec",
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "\n",
    "rouge = evaluate.load(\"rouge\")\n",
    "\n",
    "def ROUGE_evaluate(data_set : list):\n",
    "    data = data_set\n",
    "    for i in range(len(data)):\n",
    "        rouge = evaluate.load('rouge')\n",
    "        predictions = [data[i]['generated']]\n",
    "        references = [data[i]['summary']]\n",
    "        results = rouge.compute(predictions=predictions, references=references)\n",
    "        print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e4c2db3-ee0c-4eb8-894a-c5a2b859bc79",
   "metadata": {},
   "source": [
    "# Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9d9df0d5-8436-44e2-8be8-01ba1f699436",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rouge1': 0.864406779661017, 'rouge2': 0.8620689655172413, 'rougeL': 0.864406779661017, 'rougeLsum': 0.864406779661017}\n",
      "{'rouge1': 0.5783132530120482, 'rouge2': 0.5731707317073171, 'rougeL': 0.5783132530120482, 'rougeLsum': 0.5783132530120482}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 0.968421052631579, 'rouge2': 0.967741935483871, 'rougeL': 0.968421052631579, 'rougeLsum': 0.968421052631579}\n",
      "{'rouge1': 0.3636363636363636, 'rouge2': 0.33333333333333337, 'rougeL': 0.3636363636363636, 'rougeLsum': 0.3636363636363636}\n",
      "{'rouge1': 0.5739130434782609, 'rouge2': 0.5663716814159292, 'rougeL': 0.5739130434782609, 'rougeLsum': 0.5739130434782609}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 0.9722222222222222, 'rouge2': 0.9577464788732394, 'rougeL': 0.9722222222222222, 'rougeLsum': 0.9722222222222222}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n",
      "{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n"
     ]
    }
   ],
   "source": [
    "ROUGE_evaluate(generated_summaries)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7680e59-cdf0-419a-b9ef-5516e12cff8b",
   "metadata": {},
   "source": [
    "# New Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "342d6f20-3992-4a4e-b68b-0c67b80674af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rouge1': 0.9545454545454546, 'rouge2': 0.9069767441860465, 'rougeL': 0.9545454545454546, 'rougeLsum': 0.9545454545454546}\n",
      "{'rouge1': 0.5190311418685121, 'rouge2': 0.5156794425087108, 'rougeL': 0.5190311418685121, 'rougeLsum': 0.5190311418685121}\n"
     ]
    }
   ],
   "source": [
    "ROUGE_evaluate(new_generated_summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5440b405-19fe-4a11-860b-0cbf37227e11",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}