notebooks/T5_California_Model_Interface.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "89751719-bb82-42ef-b49e-182e81b0c478",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-10 19:08:20.316186: I tensorflow/core/util/port.cc:111] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-04-10 19:08:20.355370: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-04-10 19:08:20.355415: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-04-10 19:08:20.355437: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-04-10 19:08:20.362726: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-04-10 19:08:21.512078: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "summarizer = pipeline(\"summarization\", model=\"cheaptrix/California_bills_summary\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "883134f1-934d-422f-a64a-0a37073542c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "cleaned_test_data = []\n",
    "# Opening JSON file\n",
    "with open(\"cleaned_bill_sum_test_data.json\") as json_file:\n",
    "    cleaned_test_data = json.load(json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c7f5e7f6-ac9f-4ba7-acbf-1ba7d031ef81",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (548 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "generated_summaries = []\n",
    "for i in range(len(cleaned_test_data)):\n",
    "    summary = summarizer(cleaned_test_data[i]['text'])\n",
    "    generated_summaries.append({\"generated\" : summary[0]['summary_text'], \"summary\" : cleaned_test_data[i]['summary']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cca1a292-1ab5-452f-b998-d404bfc633d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': 'ACCEPT Resolution 2023-01-25 Introduced in Senate Adopting Cryptocurrency in Congress as an Exchange of Payment for Transactions Resolution . This resolution requires the Architect of the Capitol, the Secretary of the Senate, and the Chief Administrative Officer of the House of Representatives to encourage Capitol gift shops to accept cryptocurrency and to enter into contracts with vendors that accept cryptocurrency to provide food service and vending machines in the Capitol .',\n",
       " 'summary': 'Adopting Cryptocurrency in Congress as an Exchange of Payment for Transactions Resolution or the ACCEPT Resolution This resolution requires the Architect of the Capitol, the Secretary of the Senate, and the Chief Administrative Officer of the House of Representatives to encourage Capitol gift shops to accept cryptocurrency and to enter into contracts with vendors that accept cryptocurrency to provide food service and vending machines in the Capitol.'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22b594f4-b8a1-4296-bd19-31e799a368e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-01-26 Introduced in Senate This concurrent resolution commends the bravery, courage, and resolve of the women and men of Iran who are (1) participating in the current protests to defend their fundamental human rights, and (2) risking their safety to speak out against the human rights abuses committed by the Iranian regime. The resolution condemns (1) the brutal beating and death of Mahsa Amini; and (2) the violent suppression by the Iran regime of women . and men participating in . the current demonstrations, including children, and calls for transparency .',\n",
       " 'summary': 'This concurrent resolution commends the bravery, courage, and resolve of the women and men of Iran who are (1) participating in the current protests to defend their fundamental human rights, and (2) risking their safety to speak out against the human rights abuses committed by the Iranian regime. The resolution condemns (1) the brutal beating and death of Mahsa Amini; and (2) the violent suppression by the Iranian regime of women and men participating in the current demonstrations, including children, and calls for transparent accountability for all killings of protesters by Iranian security forces. Finally, the resolution encourages continued efforts by the Biden Administration to respond to the protests, including the recent sanctioning of the Iranian morality police.'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7f0a025a-290f-4811-b504-793e2e0dbde2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-02-01 Introduced in Senate This concurrent resolution calls for honoring the 237th anniversary of the enactment of the Virginia Statute for Religious Freedom on Religious Freedom Day, January 16, 2023. The resolution affirms that religious freedom includes the right of individuals of any faith and individuals of no faith to live, work, associate, and worship in accordance with their beliefs; all people of the United States can be unified in supporting religious freedom because it is a fundamental human right that is essential to a free society and protected',\n",
       " 'summary': 'This concurrent resolution calls for honoring the 237th anniversary of the enactment of the Virginia Statute for Religious Freedom on Religious Freedom Day, January 16, 2023. The resolution affirms that religious freedom includes the right of individuals of any faith and individuals of no faith to live, work, associate, and worship in accordance with their beliefs; all people of the United States can be unified in supporting religious freedom because it is a fundamental human right; and the American people will remain forever unshackled in matters of faith.'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e0b81c81-b0ba-4285-988a-ce9b070e215c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-02-16 Introduced in Senate This concurrent resolution requires the Joint Committee on the Library to approve or deny the statue of Rev. Billy Graham for placement in the National Statuary Hall within 30 days after North Carolina submits specific information about the statute, including its dimensions and final weight. this file contains bill summaries for federal legislation and details the effects the legislative text may have on current law and federal programs.',\n",
       " 'summary': 'This concurrent resolution requires the Joint Committee on the Library to approve or deny the statue of Rev. Billy Graham for placement in the National Statuary Hall within 30 days after North Carolina submits specific information about the statute, including its dimensions and final weight.'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8da96ded-73dd-474f-8c2b-b684f1aa825c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-03-09 Introduced in Senate This concurrent resolution declares that Congress should not impose any new performance fee, tax, royalty, or other charge relating to the public performance of sound recordings on a local radio station for broadcasting sound recordings over the air . this file contains bill summaries for federal legislation and details the effects the legislative text may have on current law and federal programs.',\n",
       " 'summary': 'This concurrent resolution declares that Congress should not impose any new performance fee, tax, royalty, or other charge relating to the public performance of sound recordings on a local radio station for broadcasting sound recordings over the air or on any business for such public performance of sound recordings.'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6d51741f-fb47-48d2-a387-ef87577817df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-03-09 Introduced in Senate This concurrent resolution recognizes Abortion Provider Appreciation Day. Text/xml EN Pursuant to Title 17 Section 105 of the United States Code, this file contains bill summaries for federal legislation. A bill summary describes the most significant provisions of a piece of legislation.',\n",
       " 'summary': 'This concurrent resolution recognizes Abortion Provider Appreciation Day.'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a69ce9c9-f18f-4aed-80f2-53fd81311555",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': \"2023-11-02 Passed Senate This concurrent resolution condemns Russia's unjust and arbitrary detention of Russian democratic opposition leader Vladimir Kara-Murza . it calls for his immediate release and the release of all other Russian opposition leaders . It also calls on the President and leaders of the free world to work tirelessly for the released of political prisoners in Russia .\",\n",
       " 'summary': \"This concurrent resolution condemns Russia's unjust and arbitrary detention of Russian democratic opposition leader Vladimir Kara-Murza and calls for his immediate release and the release of all other Russian opposition leaders. It also calls for the release of all political prisoners in Russia and in Belarus, as well as for the release of Ukrainian citizens held by Russia, and calls on the President and leaders of the free world to work tirelessly for the release of political prisoners in Russia.\"}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "14a0bc12-abe5-4dca-b97c-13577cf63ee5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-03-23 Introduced in Senate This concurrent resolution expresses the sense of Congress that tax-exempt fraternal benefit societies provide critical benefits to the people and communities of the United States and their work should continue to be promoted . this file contains bill summaries for federal legislation and details the effects the legislative text may have on current law and federal programs.',\n",
       " 'summary': 'This concurrent resolution expresses the sense of Congress that tax-exempt fraternal benefit societies provide critical benefits to the people and communities of the United States and their work should continue to be promoted.'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7d4415fe-7d50-4c12-9039-c338651f8972",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-04-27 Introduced in Senate This concurrent resolution expresses the sense of Congress that climate change caused by human activities constitutes a climate emergency, which demands the President use existing authorities and emergency powers to mitigate and prepare for the consequences of the emergency . a bill summary describes the most significant provisions of a piece of legislation and details the effects the legislative text may have on current law and federal programs.',\n",
       " 'summary': 'This concurrent resolution expresses the sense of Congress that climate change caused by human activities constitutes a climate emergency, which demands the President use existing authorities and emergency powers to mitigate and prepare for the consequences of the emergency.'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ff9959f8-4c7a-4af3-8e3e-83cfde0f227a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': \"this concurrent resolution requires the congressional budget committees to conduct an annual joint hearing to receive a presentation from the Comptroller General regarding (1) the Government Accountability Office's audit of the financial statement of the executive branch, and (2) the financial position and condition of the federal government. Congressional Research Service, Library of Congress This file contains bill summaries for federal legislation. A bill summary describes the most significant provisions of a piece of legislation.\",\n",
       " 'summary': \"Fiscal State of the Nation Resolution This concurrent resolution requires the congressional budget committees to conduct an annual joint hearing to receive a presentation from the Comptroller General regarding (1) the Government Accountability Office's audit of the financial statement of the executive branch, and (2) the financial position and condition of the federal government.\"}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "54e8d283-2011-4f9a-b20e-70c90cff42be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-05-31 Introduced in Senate This concurrent resolution expresses the sense that the Senate should provide its advice and consent to ratification of the Convention on Biological Diversity. Text/xml EN Pursuant to Title 17 Section 105 of the United States Code, this file contains bill summaries for federal legislation .',\n",
       " 'summary': 'This concurrent resolution expresses the sense that the Senate should provide its advice and consent to ratification of the Convention on Biological Diversity.'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cd4a017c-69d3-4891-b011-bdd19ca4a742",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-07-18 Introduced in Senate This concurrent resolution calls on the media to voluntarily adopt certain practices to prevent further harm from its coverage of mass public murders . Specifically, it calls for coverage that (1) denies murderers a public platform; (2) minimizes the potential for media reporting to increase the likelihood of future mass public killings; and (3) prioritizes the victims of, and heroism in the response to, mass public killers.',\n",
       " 'summary': 'This concurrent resolution calls on the media to voluntarily adopt certain practices to prevent further harm from its coverage of mass public murders. Specifically, it calls for coverage that (1) denies murderers a public platform; (2) minimizes the potential for media reporting to increase the likelihood of future mass public murders (i.e., the media contagion effect); and (3) prioritizes the victims of, and heroism in the response to, mass public murders.'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e2a659be-e404-447e-8a35-5dbd365e4faf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-07-27 Introduced in Senate This concurrent resolution affirms, more than 400 years after the arrival of the first slave ship, that the United States owes a debt of remembrance to those who lived through slavery or other historical injustices against people of color, as well as to their descendants. It also urges the establishment of a U.S. Commission on Truth, Racial Healing, and Transformation.',\n",
       " 'summary': 'This concurrent resolution affirms, more than 400 years after the arrival of the first slave ship, that the United States owes a debt of remembrance to those who lived through slavery or other historical injustices against people of color, as well as to their descendants. It also urges the establishment of a U.S. Commission on Truth, Racial Healing, and Transformation.'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d859483d-a664-4eb4-8910-8c590461d48e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': \"2023-10-04 Introduced in Senate This concurrent resolution recognizes the disparity between wages paid to Latina women in comparison to men and reaffirms Congress's support for ensuring equal pay and closing the gender wage gap. text/xml EN Pursuant to Title 17 Section 105 of the United States Code, this file contains bill summaries for federal legislation.\",\n",
       " 'summary': \"This concurrent resolution recognizes the disparity between wages paid to Latina women in comparison to men and reaffirms Congress's support for ensuring equal pay and closing the gender wage gap.\"}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c66b1673-d087-4b52-88d4-ef5990ecbd90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2023-10-26 Introduced in Senate This concurrent resolution expresses the sense of Congress that a carbon tax would be detrimental to families and businesses and would severely harm the economic and national security of the country. Text/xml EN Pursuant to Title 17 Section 105 of the United States Code, this file is not subject to copyright protection and is in the public domain.',\n",
       " 'summary': 'This concurrent resolution expresses the sense of Congress that a carbon tax would be detrimental to families and businesses and would severely harm the economic and national security of the country.'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[14]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "205ff67d-894d-40f4-814c-d9f2d3eea1d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generated': '2024-02-13 Introduced in Senate This concurrent resolution provides for a correction in the official title of H.R. 815 (Making emergency supplemental appropriations for the fiscal year ending September 30, 2024, and for other purposes). text/xml EN Pursuant to Title 17 Section 105 of the United States Code, this file is not subject to copyright protection and is in the public domain.',\n",
       " 'summary': 'This concurrent resolution makes a correction to the official title of H.R. 815 (Making emergency supplemental appropriations for the fiscal year ending September 30, 2024, and for other purposes).'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_summaries[15]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68c08713-89d9-4611-8cf5-9c86bed5fc29",
   "metadata": {},
   "source": [
    "# ROUGE EVALUATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e869ae4a-6126-4256-98b4-4c36556e4cec",
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "\n",
    "rouge = evaluate.load(\"rouge\")\n",
    "\n",
    "def ROUGE_evaluate(data_set : list):\n",
    "    date = data_set\n",
    "    for i in range(len(generated_summaries)):\n",
    "        rouge = evaluate.load('rouge')\n",
    "        predictions = [generated_summaries[i]['generated']]\n",
    "        references = [generated_summaries[i]['summary']]\n",
    "        results = rouge.compute(predictions=predictions, references=references)\n",
    "        print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9d9df0d5-8436-44e2-8be8-01ba1f699436",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rouge1': 0.9420289855072463, 'rouge2': 0.9264705882352942, 'rougeL': 0.9130434782608696, 'rougeLsum': 0.9130434782608696}\n",
      "{'rouge1': 0.7884615384615385, 'rouge2': 0.7766990291262137, 'rougeL': 0.7884615384615385, 'rougeLsum': 0.7884615384615385}\n",
      "{'rouge1': 0.8715083798882681, 'rouge2': 0.847457627118644, 'rougeL': 0.8603351955307262, 'rougeLsum': 0.8603351955307262}\n",
      "{'rouge1': 0.7563025210084033, 'rouge2': 0.7521367521367521, 'rougeL': 0.7563025210084033, 'rougeLsum': 0.7563025210084033}\n",
      "{'rouge1': 0.689655172413793, 'rouge2': 0.6491228070175438, 'rougeL': 0.6724137931034483, 'rougeLsum': 0.6724137931034483}\n",
      "{'rouge1': 0.2807017543859649, 'rouge2': 0.2545454545454546, 'rougeL': 0.2807017543859649, 'rougeLsum': 0.2807017543859649}\n",
      "{'rouge1': 0.7552447552447552, 'rouge2': 0.723404255319149, 'rougeL': 0.7552447552447552, 'rougeLsum': 0.7552447552447552}\n",
      "{'rouge1': 0.7010309278350516, 'rouge2': 0.6947368421052631, 'rougeL': 0.7010309278350516, 'rougeLsum': 0.7010309278350516}\n",
      "{'rouge1': 0.6964285714285715, 'rouge2': 0.6909090909090909, 'rougeL': 0.6964285714285715, 'rougeLsum': 0.6964285714285715}\n",
      "{'rouge1': 0.7786259541984732, 'rouge2': 0.7441860465116279, 'rougeL': 0.7480916030534351, 'rougeLsum': 0.7480916030534351}\n",
      "{'rouge1': 0.6216216216216216, 'rouge2': 0.6111111111111112, 'rougeL': 0.6216216216216216, 'rougeLsum': 0.6216216216216216}\n",
      "{'rouge1': 0.8888888888888888, 'rouge2': 0.8732394366197183, 'rougeL': 0.8888888888888888, 'rougeLsum': 0.8888888888888888}\n",
      "{'rouge1': 0.953125, 'rouge2': 0.9523809523809523, 'rougeL': 0.953125, 'rougeLsum': 0.953125}\n",
      "{'rouge1': 0.688888888888889, 'rouge2': 0.6818181818181819, 'rougeL': 0.688888888888889, 'rougeLsum': 0.688888888888889}\n",
      "{'rouge1': 0.6458333333333334, 'rouge2': 0.6382978723404256, 'rougeL': 0.6458333333333334, 'rougeLsum': 0.6458333333333334}\n",
      "{'rouge1': 0.6105263157894737, 'rouge2': 0.5376344086021505, 'rougeL': 0.5894736842105264, 'rougeLsum': 0.5894736842105264}\n"
     ]
    }
   ],
   "source": [
    "ROUGE_evaluate(generated_summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bc51dd7-c5de-495e-b286-f723d545ed35",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}