Tasks/Task 1/settingup-vectorstore-chunking-strategy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: langchain in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (0.3.0)\n",
      "Requirement already satisfied: langchain-core in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (0.3.2)\n",
      "Requirement already satisfied: langchain-community in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (0.3.0)\n",
      "Requirement already satisfied: langchain-experimental in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (0.3.0)\n",
      "Requirement already satisfied: langchain-qdrant in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (0.1.4)\n",
      "Requirement already satisfied: qdrant-client in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (1.11.2)\n",
      "Requirement already satisfied: tiktoken in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (0.7.0)\n",
      "Requirement already satisfied: pymupdf in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (1.24.10)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (2.0.32)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (3.9.5)\n",
      "Requirement already satisfied: langchain-text-splitters<0.4.0,>=0.3.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (0.3.0)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (0.1.120)\n",
      "Requirement already satisfied: numpy<2,>=1 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (1.26.4)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (2.8.2)\n",
      "Requirement already satisfied: requests<3,>=2 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (2.32.3)\n",
      "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.1.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain) (8.5.0)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain-core) (1.33)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain-core) (23.2)\n",
      "Requirement already satisfied: typing-extensions>=4.7 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain-core) (4.11.0)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain-community) (0.6.7)\n",
      "Requirement already satisfied: pydantic-settings<3.0.0,>=2.4.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langchain-community) (2.5.2)\n",
      "Requirement already satisfied: grpcio>=1.41.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from qdrant-client) (1.66.0)\n",
      "Requirement already satisfied: grpcio-tools>=1.41.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from qdrant-client) (1.66.0)\n",
      "Requirement already satisfied: httpx>=0.20.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from httpx[http2]>=0.20.0->qdrant-client) (0.27.2)\n",
      "Requirement already satisfied: portalocker<3.0.0,>=2.7.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from qdrant-client) (2.10.1)\n",
      "Requirement already satisfied: urllib3<3,>=1.26.14 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from qdrant-client) (2.2.1)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from tiktoken) (2024.5.15)\n",
      "Requirement already satisfied: PyMuPDFb==1.24.10 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from pymupdf) (1.24.10)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community) (3.21.2)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community) (0.9.0)\n",
      "Requirement already satisfied: protobuf<6.0dev,>=5.26.1 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from grpcio-tools>=1.41.0->qdrant-client) (5.27.3)\n",
      "Requirement already satisfied: setuptools in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from grpcio-tools>=1.41.0->qdrant-client) (75.1.0)\n",
      "Requirement already satisfied: anyio in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (3.7.1)\n",
      "Requirement already satisfied: certifi in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (2024.8.30)\n",
      "Requirement already satisfied: httpcore==1.* in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (1.0.5)\n",
      "Requirement already satisfied: idna in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (3.7)\n",
      "Requirement already satisfied: sniffio in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (0.14.0)\n",
      "Requirement already satisfied: h2<5,>=3 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from httpx[http2]>=0.20.0->qdrant-client) (4.1.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain-core) (2.4)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.7)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.20.1 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (2.20.1)\n",
      "Requirement already satisfied: python-dotenv>=0.21.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from pydantic-settings<3.0.0,>=2.4.0->langchain-community) (1.0.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from requests<3,>=2->langchain) (3.3.2)\n",
      "Requirement already satisfied: hyperframe<7,>=6.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client) (6.0.1)\n",
      "Requirement already satisfied: hpack<5,>=4.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client) (4.0.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community) (1.0.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install   \\\n",
    "    langchain \\\n",
    "    langchain-core\\\n",
    "    langchain-community \\\n",
    "    langchain-experimental \\\n",
    "    langchain-qdrant \\\n",
    "    qdrant-client \\\n",
    "    tiktoken \\\n",
    "    pymupdf \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import getpass\n",
    "\n",
    "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from uuid import uuid4\n",
    "\n",
    "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
    "os.environ[\"LANGCHAIN_PROJECT\"] = f\"AIE4 - LangGraph - {uuid4().hex[0:8]}\"\n",
    "os.environ[\"LANGCHAIN_API_KEY\"] = getpass.getpass(\"LangSmith API Key: \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages/pypdfium2/_helpers/textpage.py:80: UserWarning: get_text_range() call with default params will be implicitly redirected to get_text_bounded()\n",
      "  warnings.warn(\"get_text_range() call with default params will be implicitly redirected to get_text_bounded()\")\n"
     ]
    }
   ],
   "source": [
    "from langchain_community.document_loaders import PyMuPDFLoader\n",
    "from langchain.document_loaders import PyPDFium2Loader\n",
    "# Additional content \n",
    "# https://arxiv.org/pdf/2306.12001\n",
    "BOR_FILE_PATH = \"https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf\"\n",
    "bor_documents = PyMuPDFLoader(file_path=BOR_FILE_PATH).load()\n",
    "\n",
    "NIST_FILE_PATH = \"https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf\"\n",
    "nist_documents = PyMuPDFLoader(file_path=NIST_FILE_PATH).load()\n",
    "\n",
    "# PyPDFium2Loader_bor_documents = PyPDFium2Loader(file_path=BOR_FILE_PATH).load()\n",
    "# PyPDFium2Loader_nist_documents = PyPDFium2Loader(file_path=NIST_FILE_PATH).load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Blueprint for an AI Bill of Rights'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bor_title = str(bor_documents[0].metadata.get(\"title\"))\n",
    "bor_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nist_title = str(nist_documents[0].metadata.get(\"title\"))\n",
    "nist_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 1, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': ''}, page_content=' \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nAbout this Document \\nThe Blueprint for an AI Bill of Rights: Making Automated Systems Work for the American People was \\npublished by the White House Office of Science and Technology Policy in October 2022. This framework was \\nreleased one year after OSTP announced the launch of a process to develop “a bill of rights for an AI-powered \\nworld.” Its release follows a year of public engagement to inform this initiative. The framework is available \\nonline at: https://www.whitehouse.gov/ostp/ai-bill-of-rights \\nAbout the Office of Science and Technology Policy \\nThe Office of Science and Technology Policy (OSTP) was established by the National Science and Technology \\nPolicy, Organization, and Priorities Act of 1976 to provide the President and others within the Executive Office \\nof the President with advice on the scientific, engineering, and technological aspects of the economy, national \\nsecurity, health, foreign relations, the environment, and the technological recovery and use of resources, among \\nother topics. OSTP leads interagency science and technology policy coordination efforts, assists the Office of \\nManagement and Budget (OMB) with an annual review and analysis of Federal research and development in \\nbudgets, and serves as a source of scientific and technological analysis and judgment for the President with \\nrespect to major policies, plans, and programs of the Federal Government. \\nLegal Disclaimer \\nThe Blueprint for an AI Bill of Rights: Making Automated Systems Work for the American People is a white paper \\npublished by the White House Office of Science and Technology Policy. It is intended to support the \\ndevelopment of policies and practices that protect civil rights and promote democratic values in the building, \\ndeployment, and governance of automated systems. \\nThe Blueprint for an AI Bill of Rights is non-binding and does not constitute U.S. government policy. It \\ndoes not supersede, modify, or direct an interpretation of any existing statute, regulation, policy, or \\ninternational instrument. It does not constitute binding guidance for the public or Federal agencies and \\ntherefore does not require compliance with the principles described herein. It also is not determinative of what \\nthe U.S. government’s position will be in any international negotiation. Adoption of these principles may not \\nmeet the requirements of existing statutes, regulations, policies, or international instruments, or the \\nrequirements of the Federal agencies that enforce them. These principles are not intended to, and do not, \\nprohibit or limit any lawful activity of a government agency, including law enforcement, national security, or \\nintelligence activities. \\nThe appropriate application of the principles set forth in this white paper depends significantly on the \\ncontext in which automated systems are being utilized. In some circumstances, application of these principles \\nin whole or in part may not be appropriate given the intended use of automated systems to achieve government \\nagency missions. Future sector-specific guidance will likely be necessary and important for guiding the use of \\nautomated systems in certain settings such as AI systems used as part of school building security or automated \\nhealth diagnostic systems. \\nThe Blueprint for an AI Bill of Rights recognizes that law enforcement activities require a balancing of \\nequities, for example, between the protection of sensitive law enforcement information and the principle of \\nnotice; as such, notice may not be appropriate, or may need to be adjusted to protect sources, methods, and \\nother law enforcement equities. Even in contexts where these principles may not apply in whole or in part, \\nfederal departments and agencies remain subject to judicial, privacy, and civil liberties oversight as well as \\nexisting policies and safeguards that govern automated systems, including, for example, Executive Order 13960, \\nPromoting the Use of Trustworthy Artificial Intelligence in the Federal Government (December 2020). \\nThis white paper recognizes that national security (which includes certain law enforcement and \\nhomeland security activities) and defense activities are of increased sensitivity and interest to our nation’s \\nadversaries and are often subject to special requirements, such as those governing classified information and \\nother protected data. Such activities require alternative, compatible safeguards through existing policies that \\ngovern automated systems and AI, such as the Department of Defense (DOD) AI Ethical Principles and \\nResponsible AI Implementation Pathway and the Intelligence Community (IC) AI Ethics Principles and \\nFramework. The implementation of these policies to national security and defense activities can be informed by \\nthe Blueprint for an AI Bill of Rights where feasible. \\nThe Blueprint for an AI Bill of Rights is not intended to, and does not, create any legal right, benefit, or \\ndefense, substantive or procedural, enforceable at law or in equity by any party against the United States, its \\ndepartments, agencies, or entities, its officers, employees, or agents, or any other person, nor does it constitute a \\nwaiver of sovereign immunity. \\nCopyright Information \\nThis document is a work of the United States Government and is in the public domain (see 17 U.S.C. §105). \\n2\\n')"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bor_documents[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 1}, page_content='About this Document\\r\\nThe Blueprint for an AI Bill of Rights: Making Automated Systems Work for the American People was \\r\\npublished by the White House Office of Science and Technology Policy in October 2022. This framework was \\r\\nreleased one year after OSTP announced the launch of a process to develop “a bill of rights for an AI-powered \\r\\nworld.” Its release follows a year of public engagement to inform this initiative. The framework is available \\r\\nonline at: https://www.whitehouse.gov/ostp/ai-bill-of-rights\\r\\nAbout the Office of Science and Technology Policy\\r\\nThe Office of Science and Technology Policy (OSTP) was established by the National Science and Technology\\r\\nPolicy, Organization, and Priorities Act of 1976 to provide the President and others within the Executive Office \\r\\nof the President with advice on the scientific, engineering, and technological aspects of the economy, national \\r\\nsecurity, health, foreign relations, the environment, and the technological recovery and use of resources, among \\r\\nother topics. OSTP leads interagency science and technology policy coordination efforts, assists the Office of \\r\\nManagement and Budget (OMB) with an annual review and analysis of Federal research and development in \\r\\nbudgets, and serves as a source of scientific and technological analysis and judgment for the President with \\r\\nrespect to major policies, plans, and programs of the Federal Government.\\r\\nLegal Disclaimer\\r\\nThe Blueprint for an AI Bill of Rights: Making Automated Systems Work for the American People is a white paper\\r\\npublished by the White House Office of Science and Technology Policy. It is intended to support the \\r\\ndevelopment of policies and practices that protect civil rights and promote democratic values in the building, \\r\\ndeployment, and governance of automated systems.\\r\\nThe Blueprint for an AI Bill of Rights is non-binding and does not constitute U.S. government policy. It \\r\\ndoes not supersede, modify, or direct an interpretation of any existing statute, regulation, policy, or \\r\\ninternational instrument. It does not constitute binding guidance for the public or Federal agencies and \\r\\ntherefore does not require compliance with the principles described herein. It also is not determinative of what \\r\\nthe U.S. government’s position will be in any international negotiation. Adoption of these principles may not \\r\\nmeet the requirements of existing statutes, regulations, policies, or international instruments, or the \\r\\nrequirements of the Federal agencies that enforce them. These principles are not intended to, and do not, \\r\\nprohibit or limit any lawful activity of a government agency, including law enforcement, national security, or \\r\\nintelligence activities.\\r\\nThe appropriate application of the principles set forth in this white paper depends significantly on the \\r\\ncontext in which automated systems are being utilized. In some circumstances, application of these principles \\r\\nin whole or in part may not be appropriate given the intended use of automated systems to achieve government \\r\\nagency missions. Future sector-specific guidance will likely be necessary and important for guiding the use of \\r\\nautomated systems in certain settings such as AI systems used as part of school building security or automated \\r\\nhealth diagnostic systems.\\r\\nThe Blueprint for an AI Bill of Rights recognizes that law enforcement activities require a balancing of \\r\\nequities, for example, between the protection of sensitive law enforcement information and the principle of \\r\\nnotice; as such, notice may not be appropriate, or may need to be adjusted to protect sources, methods, and \\r\\nother law enforcement equities. Even in contexts where these principles may not apply in whole or in part, \\r\\nfederal departments and agencies remain subject to judicial, privacy, and civil liberties oversight as well as \\r\\nexisting policies and safeguards that govern automated systems, including, for example, Executive Order 13960, \\r\\nPromoting the Use of Trustworthy Artificial Intelligence in the Federal Government (December 2020).\\r\\nThis white paper recognizes that national security (which includes certain law enforcement and \\r\\nhomeland security activities) and defense activities are of increased sensitivity and interest to our nation’s \\r\\nadversaries and are often subject to special requirements, such as those governing classified information and \\r\\nother protected data. Such activities require alternative, compatible safeguards through existing policies that \\r\\ngovern automated systems and AI, such as the Department of Defense (DOD) AI Ethical Principles and \\r\\nResponsible AI Implementation Pathway and the Intelligence Community (IC) AI Ethics Principles and \\r\\nFramework. The implementation of these policies to national security and defense activities can be informed by \\r\\nthe Blueprint for an AI Bill of Rights where feasible.\\r\\nThe Blueprint for an AI Bill of Rights is not intended to, and does not, create any legal right, benefit, or \\r\\ndefense, substantive or procedural, enforceable at law or in equity by any party against the United States, its \\r\\ndepartments, agencies, or entities, its officers, employees, or agents, or any other person, nor does it constitute a \\r\\nwaiver of sovereign immunity.\\r\\nCopyright Information\\r\\nThis document is a work of the United States Government and is in the public domain (see 17 U.S.C. §105). \\r\\n2\\n')"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PyPDFium2Loader_bor_documents[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages/pydantic/_internal/_fields.py:161: UserWarning: Field \"model_name\" has conflict with protected namespace \"model_\".\n",
      "\n",
      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
      "  warnings.warn(\n",
      "/opt/miniconda3/envs/llmops-course/lib/python3.11/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "676e2dea406c411e970629d869df7495",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a9a721835b34edb977baf6696f5ffbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config_sentence_transformers.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8615e6ee9faa43898350a45b57a1702b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/90.3k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d183ee1855384ce489a3b86289f2cba8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentence_bert_config.json:   0%|          | 0.00/52.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08746b505f484f309dc5f5e84be3cf5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/720 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48c6c7233f0a46f7953fd60db87b72ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/1.34G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d7b8d910559456d96f589b52ead7230",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/366 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cd8a24253b84f0eb3dc3ce5d71c86ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d09d169f581e46de8f1c4bbde2a14062",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/711k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27e15ea4d72b4db4809381fed6aefdf2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83ce62d9540849aaaffbbe4ecb9802d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "1_Pooling/config.json:   0%|          | 0.00/191 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_experimental.text_splitter import SemanticChunker\n",
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "import tiktoken\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "\n",
    "# embedding_model = OpenAIEmbeddings(model=\"text-embedding-3-large\")\n",
    "\n",
    "# model_name = \"BAAI/bge-large-en\"\n",
    "model_name = \"gte-large-en-v1.5\"\n",
    "model_kwargs = {'device': 'cpu'}\n",
    "encode_kwargs = {'normalize_embeddings': False}\n",
    "embedding_model = HuggingFaceBgeEmbeddings(\n",
    "    model_name=model_name,\n",
    "    model_kwargs=model_kwargs,\n",
    "    encode_kwargs=encode_kwargs\n",
    ")\n",
    "\n",
    "def tiktoken_len(text):\n",
    "    tokens = tiktoken.encoding_for_model(\"gpt-4o\").encode(\n",
    "        text,\n",
    "    )\n",
    "    return len(tokens)\n",
    "\n",
    "# text_splitter = RecursiveCharacterTextSplitter(\n",
    "#     chunk_size = 700,\n",
    "#     chunk_overlap = 50,\n",
    "#     length_function = len,\n",
    "# )\n",
    "\n",
    "text_splitter = SemanticChunker(embedding_model, breakpoint_threshold_type='percentile', breakpoint_threshold_amount=90)\n",
    "\n",
    "bor_split_chunks = text_splitter.split_documents(bor_documents)\n",
    "nist_split_chunks = text_splitter.split_documents(nist_documents)\n",
    "\n",
    "# PDFium_bor_split_chunks = text_splitter.split_documents(PyPDFium2Loader_bor_documents)\n",
    "# PDFium_nist_split_chunks = text_splitter.split_documents(PyPDFium2Loader_nist_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['9b10a85501bc48f18281e073417955fe',\n",
       " '9fc3056f3e72461399b0975a7d245af7',\n",
       " '907810cdcea641c491d6d9136e149603',\n",
       " 'c4e2e16f06f34c78be93c427249618a4',\n",
       " '1a8886d120f74bfbbf8e314ad91195d7',\n",
       " '6c09482ef72b45d8882b9f8e53e8a0c6',\n",
       " '11bef1018ab943eca4239632a7f5cfb4',\n",
       " '17043a67b03b4cd6b53846c54f5da358',\n",
       " '209b3d4e2bf04a588da984cfa41d455d',\n",
       " 'c9a7bdcbc1c84fc596f9c984334622dd',\n",
       " '628824299e584081aba7c5f7dce6f515',\n",
       " '2cbff63487124a65bd9605fc886ecb38',\n",
       " '8994b6fec3da447faf5267c1a4a6e98f',\n",
       " '498d23d9347f49ac9c61216ea042aefd',\n",
       " 'e3071c284ffc40bf89bceb2063ee7b1d',\n",
       " '721a999c51074f3c90e767e6c5e08819',\n",
       " '9283571427654f7f99db9240403cfda2',\n",
       " 'ab23c9db24494fb0839b8c0ea8a5e8cb',\n",
       " '2a62d4e74eff4ee9922aa5ffd4a2af82',\n",
       " '9c60b1358cea4047bf5ed5a4faacdbc0',\n",
       " '19de85ab2d24468590a1d0d5717581e2',\n",
       " '0dc02ab832194630abe6bc5581426b92',\n",
       " '8ae335a5f5734cc1aefa0e5ea38f253d',\n",
       " 'edd41688cca64b34a694c1d027df45a6',\n",
       " '7004b933844b42ca8f6ab25ec87a95ef',\n",
       " '58890be93b5344e7beafb17d1a8c15d0',\n",
       " '9631f98644694f9789dfedd41fa1f5a9',\n",
       " '084b5f6ab5ce4b9fad791ed83e19efba',\n",
       " 'e83d6b3246734dc99681112d6609e13b',\n",
       " '23dd70b5d5cb4e07be7bc037c6f794b7',\n",
       " 'eb5b0383bc934f2caa2a446cd93e5269',\n",
       " 'd7dcbcb55e124a85ab258b8024f211c7',\n",
       " '94480a7cc26e402cad7e916b086b1166',\n",
       " 'defe828225314e18bfc659d55c66e5ed',\n",
       " '2a1686f3996f4415903ac332b7e5fa26',\n",
       " '1d56a4d3990245d9abde5fdfd6e88c74',\n",
       " '20853a36b3fa421fbb4c89ec219e2240',\n",
       " 'c0e5146bf5e44d09a03c9b929afdafe7',\n",
       " '31bf9321fefb4d268bf3c4fc1c6f6bcd',\n",
       " '51b63507eebe45fd86894aa8feba90c5',\n",
       " '11009777d08546deb01447d802a2dd8a',\n",
       " '3f571596289b4073850aabe494293564',\n",
       " 'd332430fb8df4aca8fc9a420ffce25d4',\n",
       " 'd9ed509f39c0476e91f8c25ac01cea1f',\n",
       " 'b7354be9ca6d43b88f3b1c33dcc98d30',\n",
       " '27b895c55905479896d3f58267002992',\n",
       " 'f21f22cd577b4c1aa6d7c903bea92152',\n",
       " 'aa6d35a4f6b44f4ab8e4e1be70d4402a',\n",
       " '1723c27b7fde4c86aaea8abc81c9edd4',\n",
       " '2196f205c6314fa990c47ea6ec1de2bd',\n",
       " 'a89784514a984139863ed51e12fb02a1',\n",
       " '1314b02d36b845debb2b2d7da2b94b18',\n",
       " '741fe0d130054c09a5df2dcdcc36f7eb',\n",
       " '13253ae3f7d747da83b206d4bc291aa7',\n",
       " '2de797a3c2244e03a1c7157f87c7f859',\n",
       " '587dcb5f6f5c4e97a11f9ed361aa60c5',\n",
       " '4107262f374f4de6bcec784221dd3080',\n",
       " '535d15d7f3d74dff905ece36e525440f',\n",
       " '99c24307fbf04c26b7a2186c9ae2ddf5',\n",
       " '70806c9b22a8482b835a5a5eadb9a713',\n",
       " '7d6ade93e868487897ecde3cca29c5dd',\n",
       " '5d8b657619b64077a72910ea4fe8384b',\n",
       " '0cedd1186bcb4e36b2a93bca49c2bbaf',\n",
       " '70b6f238911040d2ad371e63e5769695',\n",
       " 'c5936cbc1e8b440fad67d2e2a469b117',\n",
       " '97525e679278435281924927b4d6862e',\n",
       " 'c2c81811dfe74b1791bbafe71cb92de3',\n",
       " '41e21af7d32a491fb5810cce324f5d6a',\n",
       " '2fcd437602974788bb8f789157df386a',\n",
       " '72fa3e90cd96460fa33fcf3b9c3ee35a',\n",
       " 'd612ba7d8e3a49f0a1a76313030b8c16',\n",
       " '858f0b898993471b9133cb4485927cbe',\n",
       " '3d2ffcc997db40489c4c2e015d52ed8c',\n",
       " 'b7b5023242604320a459cfbd358206bf',\n",
       " '1495bafb01114380a778c82ce7dc931f',\n",
       " '967295304043423790a5831afa732c20',\n",
       " 'e580a0a795984a7995246be4eee870ce',\n",
       " '78ccf9b838e94e218c052679e516a0e1',\n",
       " '44aeb76986d24f7195e8f73ad0cc0bb6',\n",
       " '4aef3e3fdd9d4b8d96b7ba4529ca349a',\n",
       " '26daf0dc317447219ca18a26cf7032d4',\n",
       " '54190f128e5e465fbfaee58e57baece6',\n",
       " '2c99a2e4bcc344df8a8ed4c8d8a3b1f4',\n",
       " 'c09d600e48624b138b5e1ccc15c18600',\n",
       " '6c4820e2465a4a1fb4f3ac23d7f4e6a0',\n",
       " '04acfe508f5c428caeeb90944c1d7386',\n",
       " '41d867dfdebb435db266943391cd5721',\n",
       " '438a46409f87457ab2d54922ee9b8641',\n",
       " '036fc492d7cf480aa61b7277b4f4f870',\n",
       " '7e13b486283a47e2a404dae77e675d30',\n",
       " '282198bd5e1645e287c8a3c2f2ad37b6',\n",
       " '13ed9175a4e948eb8ead20d52a142d20',\n",
       " 'a88fb5efb843490f9f61f7f0519734a8',\n",
       " 'dadcd396c4fd4250a58d204a8066664a',\n",
       " '6cd4ef7452704575af30bead7e0055a8',\n",
       " '03a43b4c7dda47948b4d6fa718b6941d',\n",
       " '808f3240410548d1a1e814b3362f3038',\n",
       " 'df8fb263cc33427bbd16e47104b4fd0c',\n",
       " '8158c6b752484183b4c9789c85286426',\n",
       " '4328c8d02fbb4888b7c7d592e97a57df',\n",
       " '8aa3bd0c3f8a4e8fab8196afb3d77447',\n",
       " '59cef59f2dd242df93290b15dc66f434',\n",
       " 'f870044c5d84409787b24d8750b1f24d',\n",
       " '7f1cedab9a35486cbf20eb6e782b2b0e',\n",
       " 'e2ddfb4629d44785a84a31d57b0b5997',\n",
       " '1e2cf5d75876447bbb77c80acb51fcf7',\n",
       " '067a447467c34199a405ff4e232592d3',\n",
       " '0589cf77fbc94289b1020ac3dc623e7f',\n",
       " 'aba19fd6bc2449bcafc967815e11a898',\n",
       " '47a5a89fa3bb45c9b2aeb2a74d46c1d5',\n",
       " 'c6928479bbd8480ba4f4fa9a73d7b704',\n",
       " '2e93fd6c9cf74419a3caa329d59c3edf',\n",
       " '7af527fc8e984f99b9d3137d698e5e46',\n",
       " '18d94a99f5ee4382b9fddaa7d851a386',\n",
       " '3026d5bcfe38410bb5d604c7ce0ddb5b',\n",
       " 'e8e3443b56fb403694f94827f66c4f23',\n",
       " 'b551542894234f6abe683e4c53d262f5',\n",
       " 'a42778294ae141a19c9230a29d66fb18',\n",
       " '37c5fd7494714f2fb428db6f89339da9',\n",
       " '61bbb8d117724f2580bda6a8a9c01b3b',\n",
       " '9e7d2a3fe02942f7bc14def0cd797542',\n",
       " 'c19c3a8bada94eecb2dd11ecc3597640',\n",
       " '76f0cecce4fd4086af01daeb91f01570',\n",
       " 'ff1c53ebcf7e4c7d8ab3d09c9c59e06d',\n",
       " '9bfa21dc0505419c8fdc3eefd2e66188',\n",
       " 'cdef3e9867e04d8cbd8673cb86ad34d0',\n",
       " 'b5fac2712b6b40f89dcd27bb04958ff2',\n",
       " '1734630879434fd39dfca8a6944a7695',\n",
       " '4ae1706563b1428d9b3e64a2fb0d218e',\n",
       " '08402f8ed32f4c4f8c74b1cff1545dba',\n",
       " '5af4a5f0dfe34a789889e5e0625fe10e',\n",
       " '67f767401d8346bd84d134f54b125d7c',\n",
       " '8d7383c4dc724b5dae1569ca4fbee7a6',\n",
       " '381e8954b9aa4d9083fd4f1bfd601b6d',\n",
       " 'f2ada0bceb5f47f48e62cae39b57bb88',\n",
       " 'e996b70a00d5488a8524da5a6f31a9e4',\n",
       " '8d57c2a8195e41cdb65c3104e5e9881f',\n",
       " '771e4f37f8ff4e24b52fa8b1ae63df9c',\n",
       " '4564293e87874e08aed664e9a8e8ebdf',\n",
       " '297301b167084bdebf8bdb742e1a5ec7',\n",
       " '356f7de876af4313961b77b710ea521d',\n",
       " 'c4d9074c440e492dbc8baa2d2027a6bf',\n",
       " '9a3faf685c4f4d4b95707298772254b1',\n",
       " '8b56da57a60343649fe44531b9e6eb1e',\n",
       " '660d5fd5afc94de9bbe4e6d43c0b38a8',\n",
       " '98f6a93dd6524cf09ae2c13e27fccbd3',\n",
       " '0045773bd4e04051b9944ee3e188132b',\n",
       " 'c028716980e848ee91c53cf7dbce6488',\n",
       " 'd0343b9a550048cc8946b772a244c543',\n",
       " '646b4f597a9a4147aa2fb0a2afe829ad',\n",
       " 'bdaf528de3fc4debadb5be8d92c59674',\n",
       " '6ae5dff9b8644c3396d528ece4a8c9a3',\n",
       " '576482f2c32c4885817105b8629395a9',\n",
       " '9b5cc931507b4a958d621b5f6a6db180',\n",
       " '1999e46e6a274cb3b80f730c2b304fdb',\n",
       " 'b31b9bead2c74a09affe4c8b14a650a3',\n",
       " '322915fd8bc94b9a80c087b6ed205414',\n",
       " '397138503d72416b95a1013ca6cfd35b',\n",
       " 'bb91067ca1d5482e81e942753436b8bc',\n",
       " '7404c79c2f1f4f68b2c724b300412834',\n",
       " '8d00e9c4a9644325b00b246c3a77c0c8',\n",
       " '683cab04cd4a4c8d8c54471ba65d3b35',\n",
       " 'd12c4f0b3cc740cd91208bf4f9927ec8',\n",
       " 'd82c6723ac3e43ad97e150c606d4deb8',\n",
       " '54fbec13d10d4f8590cd8d51e48ad014',\n",
       " '1616969c6b794756bd1c6e01a6ed0856',\n",
       " 'e997d92ceb0047be936283ec7135480d',\n",
       " '336e4b6f151946a983ab49b875641b49',\n",
       " '342bff7dc5f743e8add79c5d26d5379c',\n",
       " 'a5d0000c72994a51aa2fdfca935ce2a8',\n",
       " 'feafa118aad045ff979f15ce6054f651',\n",
       " '8ed02e749d53479d846d6b9a44cf1214',\n",
       " 'de5bde44f0f441a993f46a25b0f2d20d',\n",
       " 'ea6effee630249d19e07b57e486429c9',\n",
       " 'afce982330534c888cbf8470641370ad',\n",
       " '62da11a21bff46b5b96e805b32ca7c99',\n",
       " '76e42da79b404ddd8dc5ec3f45ef2d93',\n",
       " '797c5b57eebb4093a084e819e3de181c',\n",
       " 'a88e3e72ff1e4f0fbfe57e95822bb05c',\n",
       " '391f2d7cb50e4660a5b54d07916ffba6',\n",
       " 'eb2e48f694784c169b9714081cacff4f',\n",
       " '2f723c35f832424397bd2d1de1c2af73',\n",
       " '57d9fa5ee507435f978eaaf756d4234e',\n",
       " 'ba266300598345d88cffcd93970a693d',\n",
       " '5e8f3aa3d5554b36b0a9946816e4436c',\n",
       " 'a24db5625e9d4e2ab43bdc071cb66b99',\n",
       " '04530348f7a14ab3b45df60b9491f616',\n",
       " 'a5e7834db41b4ff097574b55d576ae62',\n",
       " 'e2d30b59b8ee45c7961a6ad994339c9d',\n",
       " '965cc06959db42379aaf7a3addd274be',\n",
       " '8d125bb8b9444018b7561c7574a744ff',\n",
       " 'd2ee1b1a011d4616b2229a663a13357f',\n",
       " '8dd741ee253541ff864b619584ef0d56',\n",
       " '1e5fda6de0344b3886ae055929219ef8',\n",
       " 'c23b2de9ba20462e9d9f6c194771bbfe',\n",
       " 'b8ae0e1e8f6e407d9dcd73bdccea00c6',\n",
       " '16a5a4eed46c49bda1cc5e9e4c978430',\n",
       " 'ee60f460ac6c4cd1aa920dba396c4dd0',\n",
       " '3b50db4af62b45629fcb2dfa92982281',\n",
       " '3143abf08b354711ac703a21cc86cc33',\n",
       " 'a206612f3171499b88036064a151e840',\n",
       " 'c0fe13fa34974345beafa3c49283edb0',\n",
       " 'bbd32837dbb54985be78b7ced7adf703',\n",
       " '4b62e4296c5b4e04b3f85729d004e3bf',\n",
       " '7de54f2fc7b64b84ba55bae938d71384',\n",
       " 'c60748e67d854516ae1b0192e0e57842',\n",
       " '24eb079ce64246b08d395f4d73eb6fec',\n",
       " '79b91f11789c4ca9915f6f8a1bd08ca4',\n",
       " '4a20698534434b40bb46aab15acb6292',\n",
       " 'c085be76c6604cda8110b5f3a6d62abf',\n",
       " 'bab687aa70ae4838bc736df79d790aba',\n",
       " '6618ef0879f348afa1dedf0de939d3e5',\n",
       " '467fd65722fd40968de0fc21f3480fc4',\n",
       " 'b2d7d6789e7040868e47dc67191d12b2',\n",
       " 'd57b8f279da94657b9b9c4817e825ddc',\n",
       " '751f2072aa13450ca91de8831862f5cd',\n",
       " 'b6ffa8a4b4c6430e858cbf5476bce169',\n",
       " 'c657cec49d544b8b8a235c89fbecc9a7',\n",
       " 'ae55a4421b2e46a4b84c2e5336cbdce6',\n",
       " 'cd3db113dffe4b108a3eb22f3e90ef11',\n",
       " 'f7409fee205344e8bcbd97709d7d2eab',\n",
       " '58bda49ad4ef485b94b993442722c7c7',\n",
       " 'edca82fad5424ffab1d8726dbc89a981',\n",
       " 'e62fc3d2e9b647c1883caee465782369',\n",
       " 'c04e3f9f75e7406ebf5c7ee2af11e39c',\n",
       " 'd2ae8f9410f94b9694fd5c4f80f683c3',\n",
       " 'f4599b3cbb0d4600b1e10e3dbbd4acc9',\n",
       " '95da5f95bf694c678ba01e75bc629de8',\n",
       " '8f84f7aa4ca4497e983d7117cec2b0b0',\n",
       " 'b77734e6f72e4e9f9434ef0b92345c6a',\n",
       " 'abdd3a6927a941a4aad5303b2eabd5cc',\n",
       " '253e1e4538c7483db73dc8c337ba609a',\n",
       " '8eabd8389a5a40f2b47a9cad6a4e3889',\n",
       " '4307303b27024914b8bf0be339d96ec7',\n",
       " '94b806aaf52a4f988e5cd916569ed0eb',\n",
       " 'c25dba468a8e43608bb5de6ddd62a750',\n",
       " 'c6e378c69b604b0b85860314f64622d3',\n",
       " 'd856f97669874c9c811879675c54a961',\n",
       " 'e590732e4f284bee9852771f2425fe94',\n",
       " 'e2539b75f8014cf99b13fcaecc07f664',\n",
       " '655a207481864f86b383bda1955f2cae',\n",
       " '972d3ff6e40d4d7ca514a8a9b027bd05',\n",
       " 'e2038d63cadd46cba317a11dfd43d3c4',\n",
       " '635843aee0674e4ab93d0b6094814caa',\n",
       " '8e85382532d14866bebfda80e8e8c239',\n",
       " '9777e29310ab46d9886ec784bb02f401',\n",
       " '74df5fcdbf6e47d5a99a2d4b933b290a',\n",
       " '2c6de8c46cd6482ea2c9f467dd28f3c1',\n",
       " '0a6ac39343e543949a0f665e9c99256a',\n",
       " '7fb4ba6559ba48b99f3354821e1442ec',\n",
       " 'f7432c4479504980938203b81c75da68',\n",
       " '46944b59955d4c6d91d147a4087286c4',\n",
       " '8d51a965ee4047eaa33fd5fa7e04c06b',\n",
       " '1c458e58a5814eff9d42193cdc42820e',\n",
       " '23bbe99927974c49b046c5cd65e854ec',\n",
       " '6923513846ba46e68f250d99a2af1a50',\n",
       " 'ed7db6d39d8e4e75959e4b3f9d7b138f',\n",
       " '5b47434ddfa84062b62169c0aa486735',\n",
       " '04d980b96f2c4d9e89449179e167444b',\n",
       " '5667525fa85f4e71acab3d7e239e3058',\n",
       " '6a6cadd821ae460ea4369ba205cd83d2',\n",
       " '786b00ae8f6941c995c097a556480080',\n",
       " '11dbd777fee449a294069781d7083f48',\n",
       " '0dc65ab3d5b94c96abab4d9220e1393a',\n",
       " 'f320696ef39b4aedb1b828ae49025b3d',\n",
       " '4ed51a019b1344669e82d9a8de902f82',\n",
       " 'd99e671bc64f4658b0e906f2a82ec4d2',\n",
       " 'f14807b798da4af2ac1edd167eaaf99e',\n",
       " '7a7bc80c4586453198fa865da735276d',\n",
       " 'db6a19dcfa4549ff88550a7afaeb1a79',\n",
       " 'cb86e41ed73a41abb6f6adf26792ba34',\n",
       " 'e7606f37465f4313af2960e7b6bf91b4',\n",
       " '5a0fe8d593df40ab8c6a624f22676005',\n",
       " '15d94110adf7471d86abec680723badc',\n",
       " '0c85d59191f94018b61b5a6d6bb79d6a',\n",
       " '412bebb1e35643688fe318ce88cb2e7c',\n",
       " 'b5ce6bbb214142d2a1727ec4c66d584e',\n",
       " '84bc0d4b2f7e4af1994e082187b4d357',\n",
       " '6a4309d6346a4da38559446a02cdf699',\n",
       " '899e3c384d50452e90baa98fd7626973',\n",
       " 'a937e54d50c549d69aa811bef2d0dcbe',\n",
       " '6dbf4c4ba0254d739ee200758a658932',\n",
       " '13141a4474cd476391bb3410dff38ce8',\n",
       " '6e5ae0f21390439da98eb76b3a1f0dcb',\n",
       " '17365de4f7ae4d3f9d240584e6c1fb27',\n",
       " '5a336b76d565430b9846e449be9f87bc',\n",
       " 'd67633b6fea54beaaa7aec80863bb2b0',\n",
       " '28c335078c574e15a8d09c88452e7046',\n",
       " '2337cc8f4c914243955a6f9313004be8',\n",
       " 'f316f827f1aa448db76e4c5bdfda8286',\n",
       " 'f3f9598125794490bce90d110e3c94b7',\n",
       " 'b17ba9c02b6e476cb9ba15a0c47e2663',\n",
       " '21243a3de5bb414888afe44b833db84b',\n",
       " '0ac5c9d232e74d80a9496693eda10f0d',\n",
       " '69b9b17fbf154ad9914d3579fc7b3748',\n",
       " '75ec14f540a14b00ab0b7848159a44c1',\n",
       " '50037375672d43558af33cc175f81f37',\n",
       " '0fc440c0293d4996a79f3398ba2cea13',\n",
       " 'eef8cd829d1e4b92b450dbe246e5a8e1',\n",
       " '952c0b7c143c49a8aff36a50fa723441',\n",
       " '45a8a4f13e0447989057d5b64a951008',\n",
       " '4b4a9029e903467786d0a163592ab8a9',\n",
       " 'e6702124fec146c4b446bdf11a1a4320',\n",
       " '06d98ad4ec034abfba7a9e926f4480f6',\n",
       " 'a1ca90a8dda9474ba8623e1af75d85f8',\n",
       " '7c4c04fa83354aa1a6a6e91dd3f1e6f0',\n",
       " '2940677ebf0742bf8da3959f15f2c23c',\n",
       " '66ae6308dcd94da68a22c4579bef66e1',\n",
       " '21ad8039c88445aa9946a02bb51d0646',\n",
       " '5e3ac7d330914137babd0cc8886c133d',\n",
       " '872d26ac5ec24fdeaf49831a4d233392',\n",
       " '1c55c81a0d434a8a8620d1c875c02880',\n",
       " '6c5a19f622f44284a1686ec6894e774c',\n",
       " '5d5fe7fccd62432584ece1aac3edfd27',\n",
       " 'a146f32fa32646e7b2bc9adaad07fd61',\n",
       " '8a1069e8e3c4429ebe9353cae1f2c40a',\n",
       " 'ce7ae5c085834b6ea2320c1676ecfc72',\n",
       " '11b11f617c0b4846a754933ed747cf70',\n",
       " '5a8c4d305af44604bc322b09d5789ae1',\n",
       " '3feef8f278f3428db52c571a9e5832a6',\n",
       " '7cd4ccbfae7643e88fde2d2c1b763380',\n",
       " 'b07dce2e600e4d5cb69c85dda1b794c1',\n",
       " '1ba549ad3b9b488ab15014c90b909ad1',\n",
       " '035f64ae95a141769f8cc60d7a712f85',\n",
       " '7e66f3997e2243ad97a378ad1f2d9cd0',\n",
       " '019f18741d9442e99634ed90a9d52fa3',\n",
       " '725eb2d0c2514f209d319a043672ae5e',\n",
       " '27332eef52a442e48262404078d6bea1',\n",
       " '30364e4c58084aac98d13b66958b72e5',\n",
       " 'a2d466cd31334fac8bbbb77961f06c8d',\n",
       " 'cbe1f171b28c41669d9be8ae780580ab',\n",
       " 'c4063252ef38492e87c94aeca7c8cc28',\n",
       " '239848a94ec94506999381053fc020d1',\n",
       " 'e519a9f99576441da7cd8ccbac296552',\n",
       " '6b79058b89ae4bbab3c3662d6cb6b659',\n",
       " 'f265286eff6441b7b1cbcc60e45e8009',\n",
       " 'c8d2883df29f4661b4b358b6b29b56d3',\n",
       " '0e45295570054f34a69f3a87f247239f',\n",
       " '1b1a4028f6b94bd18d51df1c4efadd9a',\n",
       " 'd4b239ea42a74701b0cb17cb4c45bc76',\n",
       " '5baf4a96035742618d8275deac56d33d',\n",
       " '907f59c504d44945ae865021b6e9c713',\n",
       " '4bb7b45566b343edaaeeaeba6a2ecead',\n",
       " '40334c6286504235a452f100c84a85b5',\n",
       " '5c307e3a78e04a0d91eade015b335a52',\n",
       " 'a7db5faf0d3a44dfaf3b078ae5975295',\n",
       " '9a316a6c4c5a4956a0290ae5e62b4167',\n",
       " 'f991f9154b3045c5a320654788df92d5',\n",
       " '6a2c1d4bf1914bb89d90cdfa6cf139b3',\n",
       " '60e9ee538b5a4453bdcc204ff99bef47',\n",
       " '6b69467c43bd445fa65cdf370d460cf2',\n",
       " 'b53984c95f09469d8c969ead5e11075d',\n",
       " 'ad1c2608fa424182b6f080793a32b49f',\n",
       " '67f48e26deff4e8293399650a3768c41',\n",
       " 'c603009a2a2d46c287c918ad16e3ee8d',\n",
       " '996fce3cb9bf41b49f1f38b93edfa384',\n",
       " '2729a60b41714b25a40e138a75a18f0d',\n",
       " '1776eedac59c40bf81b49124c13878a5',\n",
       " '030a9614acc143c3ac6822ab9c53c79c',\n",
       " '600d66b84cff461a873028c99c7abe83',\n",
       " '63459979846c4b35b96aa0ff3af6114f',\n",
       " '87fdf37c329546cb87449d28cb36ee1c',\n",
       " '2bb185fe3a694a82ab6b37485509c21b',\n",
       " '19e4c9ab61f54fe98988292365fb6b81',\n",
       " 'bdc1e1ac0ad04b0eaa5a83a64dd138f8',\n",
       " '1b3c0dd370d94048bfa3f4b3e04d550e',\n",
       " 'd91f4c86f1a1422688b9797db359a680',\n",
       " '8d69de7bb4284a57af4fae4e7186ead0',\n",
       " '9af26f1d54f9464d84d1d18c4d36232b',\n",
       " 'fc19928181974672a51df8e0516536b9',\n",
       " '3b0db3da7f5c4911b6590c978027769c',\n",
       " 'b24bbe0738df49aba2eea004b01643d1',\n",
       " '80202cb0b3564c32b11bb7e7b27255a6',\n",
       " '880fd91fc3514a99ae18f82289732846',\n",
       " '2ade20847fcb407fa832fc26c8bc26ea',\n",
       " 'f77ed598e8a74779ba4470ae3908782f',\n",
       " '3b3e3ea859ae4f1d842507fa5a496887',\n",
       " '6bea1d610bff43c0b65bd9eed475d7a7',\n",
       " '342b0ac943974b0ab33d5f52f6e8b3a8',\n",
       " '551c2348c8654ec38368f13978a80dbc',\n",
       " '8d8461e36ecc4e2abcc4aa3e9cf92f46',\n",
       " 'a6519b67f7ae49778914800773f3b3e4',\n",
       " '39baa80299594938b6908aae945c1e98',\n",
       " '29d26e19aa2d4fcdae0926b1ae7edc83',\n",
       " 'e4e552226fb14acc937e03e0da080e16',\n",
       " 'f7b4462c6a024d5aa70143de24ee4610',\n",
       " '35277d52166e49229bd790d8f48c8958',\n",
       " '74340e9a6bd94e9281e01df9bbea1a92',\n",
       " 'e06dfb64b23c44cdaf88a1a929dbaf59',\n",
       " '233d61035b87490c9336fa812b75ae40',\n",
       " '404e5bbf1aca4a73afdd85751c61bccc',\n",
       " '0aee630502b149e7b5973b6328bd67f4',\n",
       " '98e31fd479e54b449066f0435567a2e8',\n",
       " '9c4c0dd2e73d4ec793a4072710e917a0',\n",
       " '8c26cb57f60c4f9d987d94cf0e19951b',\n",
       " 'f964add08355442fba0981763596176c',\n",
       " '0aa5dfeca84749f68dfe9325f49610af',\n",
       " 'c7187bbf844943118ce5cb991a026b7d',\n",
       " 'a037db5c76934706933200fe6ad62d53',\n",
       " '073f48efa436482bb4c59ff128b3c99a',\n",
       " '38cd2adf189641ffa234059571a5f529',\n",
       " '8ce5b552a1314b04ad62c177876d80e6',\n",
       " '1e94bf0a63aa4577a6d85ceb7678ddce',\n",
       " 'a419b88ab65f4a4a957e9d3bd1977841',\n",
       " 'aa9d35e07f9f4e8aae706ab932dead73',\n",
       " 'b3eca708f0904ad09498c40b617ba24b',\n",
       " 'f0ce001234764d619738595e44267d0e',\n",
       " '219a836964a648fa86e199f5ffdf4665',\n",
       " 'b1110a412a6b439f9ae708e08541fc9d',\n",
       " 'cb7ce20ef1b54040852546636f4aa12d',\n",
       " 'a4b64ea16d854ffcb0c0803e30ad0ac2',\n",
       " '6cdbd6c0294f466f89b999734a8591b4',\n",
       " 'd6adc8b20b704b4a932f5d98905b70df',\n",
       " 'f2ed389f027345178b4c7fdd2b9b2297',\n",
       " '9039020ff08f4f5b92ad5d29c21dd745',\n",
       " 'a1a760b623ed4aabbfc4e651cc46324e',\n",
       " 'd02b6bc20ea146f5a95780045151194d',\n",
       " 'c6cf35dc625e44ccb988a3f345767558',\n",
       " '94d085d8fd804b67be674b3948e15dd8',\n",
       " 'ccb239d3f706409eba57e01e2becd814',\n",
       " '5983575a7c004815bde998e230c99f86',\n",
       " 'dcb80871f0b842a3ba92c31e32e8d571',\n",
       " 'd35e7f3fcc204091a26bf577bbc6bfe8',\n",
       " '9bbc8bf63d4e4272ba29fecfa52808cc',\n",
       " '1d487b5ee5af4fcf868dc33d64e888e7',\n",
       " '2a040fce70c44363b0da05805593bbc8',\n",
       " '676f09e850fd42d49346dcfb7b334497',\n",
       " '787d37f19b1446059d7c372f2044f724',\n",
       " '70ea74c4f4594967b9e9645afbf9d2bc',\n",
       " '20c58545da1d40a884a6b654df0b666b',\n",
       " '7d5469f032e14647b3e6faac0ba841a2',\n",
       " '24204f7af4e740aa82c1cfc3de2a86ad']"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_qdrant import QdrantVectorStore\n",
    "from qdrant_client import QdrantClient\n",
    "from qdrant_client.http.models import Distance, VectorParams\n",
    "\n",
    "\n",
    "client = QdrantClient(\":memory:\")\n",
    "collection_name=\"ai-safety\"\n",
    "client.create_collection(\n",
    "    collection_name=collection_name,\n",
    "    vectors_config=VectorParams(size=1024, distance=Distance.COSINE),\n",
    ")\n",
    "\n",
    "vector_store = QdrantVectorStore(\n",
    "    client=client,\n",
    "    collection_name=collection_name,\n",
    "    embedding=embedding_model,\n",
    ")\n",
    "vector_store.add_documents(bor_split_chunks+nist_split_chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "retriver = vector_store.as_retriever(\n",
    "    search_type=\"similarity_score_threshold\",\n",
    "    search_kwargs={'k': 10,'score_threshold': 0.5}\n",
    "    #  search_type=\"mmr\",\n",
    "    #             search_kwargs={'k': 8, 'fetch_k': 50}\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.schema import BaseRetriever, Document\n",
    "from pydantic import Field\n",
    "from typing import List\n",
    "\n",
    "class ContextEnrichedRetriever(BaseRetriever):\n",
    "    # Declare fields using Pydantic's Field\n",
    "    retriever: BaseRetriever = Field(...)\n",
    "    window_size: int = Field(default=2)\n",
    "\n",
    "    def __init__(self, retriever: BaseRetriever, window_size: int = 2):\n",
    "        \"\"\"\n",
    "        A custom retriever that adds a context enrichment window to retrieved documents.\n",
    "        Args:\n",
    "        - retriever: The underlying retriever (e.g., Qdrant retriever).\n",
    "        - window_size: Number of adjacent passages to include as context (before and after).\n",
    "        \"\"\"\n",
    "        super().__init__(retriever=retriever, window_size=window_size)\n",
    "\n",
    "    def enrich_context(self, document: Document, window_size: int) -> str:\n",
    "        \"\"\"\n",
    "        Enrich the document context by adding the surrounding text (context window).\n",
    "        Args:\n",
    "        - document: The retrieved document.\n",
    "        - window_size: Number of adjacent passages to include as context (before and after).\n",
    "        \"\"\"\n",
    "        doc_text = document.page_content.split(\"\\n\")\n",
    "        enriched_text = []\n",
    "        for i, passage in enumerate(doc_text):\n",
    "            context = doc_text[max(0, i - window_size):min(len(doc_text), i + window_size + 1)]\n",
    "            enriched_text.append(\"\\n\".join(context))\n",
    "        return \"\\n\".join(enriched_text)\n",
    "\n",
    "    def _get_relevant_documents(self, query: str) -> List[Document]:\n",
    "        \"\"\"\n",
    "        Retrieve documents and apply context enrichment.\n",
    "        Args:\n",
    "        - query: The query string.\n",
    "        Returns:\n",
    "        - List of enriched documents.\n",
    "        \"\"\"\n",
    "        documents = self.retriever.get_relevant_documents(query)\n",
    "        enriched_documents = []\n",
    "        for doc in documents:\n",
    "            enriched_content = self.enrich_context(doc, self.window_size)\n",
    "            enriched_doc = Document(page_content=enriched_content, metadata=doc.metadata)\n",
    "            enriched_documents.append(enriched_doc)\n",
    "        return enriched_documents\n",
    "\n",
    "    async def _aget_relevant_documents(self, query: str) -> List[Document]:\n",
    "        \"\"\"\n",
    "        Async version of the document retrieval and enrichment.\n",
    "        Args:\n",
    "        - query: The query string.\n",
    "        Returns:\n",
    "        - List of enriched documents.\n",
    "        \"\"\"\n",
    "        documents = await self.retriever.aget_relevant_documents(query)\n",
    "        enriched_documents = []\n",
    "        for doc in documents:\n",
    "            enriched_content = self.enrich_context(doc, self.window_size)\n",
    "            enriched_doc = Document(page_content=enriched_content, metadata=doc.metadata)\n",
    "            enriched_documents.append(enriched_doc)\n",
    "        return enriched_documents\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# todo: For some reason this creates duplicate need to debug this later.\n",
    "context_enriched_retriever = ContextEnrichedRetriever(retriever=retriver, window_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: Blueprint for an AI Bill of Rights\n",
      "In some cases, exceptions to \n",
      "the principles described in the Blueprint for an AI Bill of Rights may be necessary to comply with existing law, \n",
      "conform to the practicalities of a specific use case, or balance competing public interests. In particular, law \n",
      "enforcement, and other regulatory contexts may require government actors to protect civil rights, civil liberties, \n",
      "and privacy in a manner consistent with, but using alternate mechanisms to, the specific principles discussed in \n",
      "this framework. The Blueprint for an AI Bill of Rights is meant to assist governments and the private sector in \n",
      "moving principles into practice. The expectations given in the Technical Companion are meant to serve as a blueprint for the development of \n",
      "additional technical standards and practices that should be tailored for particular sectors and contexts. While \n",
      "existing laws informed the development of the Blueprint for an AI Bill of Rights, this framework does not detail \n",
      "those laws beyond providing them as examples, where appropriate, of existing protective measures. This \n",
      "framework instead shares a broad, forward-leaning vision of recommended principles for automated system \n",
      "development and use to inform private and public involvement with these systems where they have the poten­\n",
      "tial to meaningfully impact rights, opportunities, or access. Additionally, this framework does not analyze or \n",
      "take a position on legislative and regulatory proposals in municipal, state, and federal government, or those in \n",
      "other countries. We have seen modest progress in recent years, with some state and local governments responding to these prob­\n",
      "lems with legislation, and some courts extending longstanding statutory protections to new and emerging tech­\n",
      "nologies. There are companies working to incorporate additional protections in their design and use of auto­\n",
      "mated systems, and researchers developing innovative guardrails. Advocates, researchers, and government \n",
      "organizations have proposed principles for the ethical use of AI and other automated systems. These include \n",
      "the Organization for Economic Co-operation and Development’s (OECD’s) 2019 Recommendation on Artificial \n",
      "Intelligence, which includes principles for responsible stewardship of trustworthy AI and which the United \n",
      "States adopted, and Executive Order 13960 on Promoting the Use of Trustworthy Artificial Intelligence in the \n",
      "Federal Government, which sets out principles that govern the federal government’s use of AI. The Blueprint \n",
      "for an AI Bill of Rights is fully consistent with these principles and with the direction in Executive Order 13985 \n",
      "on Advancing Racial Equity and Support for Underserved Communities Through the Federal Government. These principles find kinship in the Fair Information Practice Principles (FIPPs), derived from the 1973 report \n",
      "of an advisory committee to the U.S. Department of Health, Education, and Welfare, Records, Computers, \n",
      "and the Rights of Citizens.4 While there is no single, universal articulation of the FIPPs, these core \n",
      "principles for managing information about individuals have been incorporated into data privacy laws and \n",
      "policies across the globe.5 The Blueprint for an AI Bill of Rights embraces elements of the FIPPs that are \n",
      "particularly relevant to automated systems, without articulating a specific set of FIPPs or scoping \n",
      "applicability or the interests served to a single particular domain, like privacy, civil rights and civil liberties, \n",
      "ethics, or risk management. The Technical Companion builds on this prior work to provide practical next \n",
      "steps to move these principles into practice and promote common approaches that allow technological \n",
      "innovation to flourish while protecting people from harm.\n",
      "1: Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile\n",
      " \n",
      "26 \n",
      "MAP 4.1: Approaches for mapping AI technology and legal risks of its components – including the use of third-party data or \n",
      "software – are in place, followed, and documented, as are risks of infringement of a third-party’s intellectual property or other \n",
      "rights. Action ID \n",
      "Suggested Action \n",
      "GAI Risks \n",
      "MP-4.1-001 Conduct periodic monitoring of AI-generated content for privacy risks; address any \n",
      "possible instances of PII or sensitive data exposure. Data Privacy \n",
      "MP-4.1-002 Implement processes for responding to potential intellectual property infringement \n",
      "claims or other rights. Intellectual Property \n",
      "MP-4.1-003 \n",
      "Connect new GAI policies, procedures, and processes to existing model, data, \n",
      "software development, and IT governance and to legal, compliance, and risk \n",
      "management activities. Information Security; Data Privacy \n",
      "MP-4.1-004 Document training data curation policies, to the extent possible and according to \n",
      "applicable laws and policies. Intellectual Property; Data Privacy; \n",
      "Obscene, Degrading, and/or \n",
      "Abusive Content \n",
      "MP-4.1-005 \n",
      "Establish policies for collection, retention, and minimum quality of data, in \n",
      "consideration of the following risks: Disclosure of inappropriate CBRN information; \n",
      "Use of Illegal or dangerous content; Offensive cyber capabilities; Training data \n",
      "imbalances that could give rise to harmful biases; Leak of personally identifiable \n",
      "information, including facial likenesses of individuals. CBRN Information or Capabilities; \n",
      "Intellectual Property; Information \n",
      "Security; Harmful Bias and \n",
      "Homogenization; Dangerous, \n",
      "Violent, or Hateful Content; Data \n",
      "Privacy \n",
      "MP-4.1-006 Implement policies and practices defining how third-party intellectual property and \n",
      "training data will be used, stored, and protected. Intellectual Property; Value Chain \n",
      "and Component Integration \n",
      "MP-4.1-007 Re-evaluate models that were fine-tuned or enhanced on top of third-party \n",
      "models.\n",
      "2: Blueprint for an AI Bill of Rights\n",
      "Data should \n",
      "only be collected or used for the purposes of training or testing machine learning models if such collection and \n",
      "use is legal and consistent with the expectations of the people whose data is collected. User experience \n",
      "research should be conducted to confirm that people understand what data is being collected about them and \n",
      "how it will be used, and that this collection matches their expectations and desires. Data collection and use-case scope limits. Data collection should be limited in scope, with specific, \n",
      "narrow identified goals, to avoid \"mission creep.\"  Anticipated data collection should be determined to be \n",
      "strictly necessary to the identified goals and should be minimized as much as possible. Data collected based on \n",
      "these identified goals and for a specific context should not be used in a different context without assessing for \n",
      "new privacy risks and implementing appropriate mitigation measures, which may include express consent. Clear timelines for data retention should be established, with data deleted as soon as possible in accordance \n",
      "with legal or policy-based limitations. Determined data retention timelines should be documented and justi­\n",
      "fied. Risk identification and mitigation. Entities that collect, use, share, or store sensitive data should \n",
      "attempt to proactively identify harms and seek to manage them so as to avoid, mitigate, and respond appropri­\n",
      "ately to identified risks. Appropriate responses include determining not to process data when the privacy risks \n",
      "outweigh the benefits or implementing measures to mitigate acceptable risks. Appropriate responses do not \n",
      "include sharing or transferring the privacy risks to users via notice or consent requests where users could not \n",
      "reasonably be expected to understand the risks without further support. Privacy-preserving security. Entities creating, using, or governing automated systems should follow \n",
      "privacy and security best practices designed to ensure data and metadata do not leak beyond the specific \n",
      "consented use case. Best practices could include using privacy-enhancing cryptography or other types of \n",
      "privacy-enhancing technologies or fine-grained permissions and access control mechanisms, along with \n",
      "conventional system security protocols.\n",
      "3: Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile\n",
      " \n",
      "27 \n",
      "MP-4.1-010 \n",
      "Conduct appropriate diligence on training data use to assess intellectual property, \n",
      "and privacy, risks, including to examine whether use of proprietary or sensitive \n",
      "training data is consistent with applicable laws. Intellectual Property; Data Privacy \n",
      "AI Actor Tasks: Governance and Oversight, Operation and Monitoring, Procurement, Third-party entities \n",
      " \n",
      "MAP 5.1: Likelihood and magnitude of each identified impact (both potentially beneficial and harmful) based on expected use, past \n",
      "uses of AI systems in similar contexts, public incident reports, feedback from those external to the team that developed or deployed \n",
      "the AI system, or other data are identified and documented. Action ID \n",
      "Suggested Action \n",
      "GAI Risks \n",
      "MP-5.1-001 Apply TEVV practices for content provenance (e.g., probing a system's synthetic \n",
      "data generation capabilities for potential misuse or vulnerabilities. Information Integrity; Information \n",
      "Security \n",
      "MP-5.1-002 \n",
      "Identify potential content provenance harms of GAI, such as misinformation or \n",
      "disinformation, deepfakes, including NCII, or tampered content. Enumerate and \n",
      "rank risks based on their likelihood and potential impact, and determine how well \n",
      "provenance solutions address specific risks and/or harms. Information Integrity; Dangerous, \n",
      "Violent, or Hateful Content; \n",
      "Obscene, Degrading, and/or \n",
      "Abusive Content \n",
      "MP-5.1-003 \n",
      "Consider disclosing use of GAI to end users in relevant contexts, while considering \n",
      "the objective of disclosure, the context of use, the likelihood and magnitude of the \n",
      "risk posed, the audience of the disclosure, as well as the frequency of the \n",
      "disclosures. Human-AI Configuration \n",
      "MP-5.1-004 Prioritize GAI structured public feedback processes based on risk assessment \n",
      "estimates. Information Integrity; CBRN \n",
      "Information or Capabilities; \n",
      "Dangerous, Violent, or Hateful \n",
      "Content; Harmful Bias and \n",
      "Homogenization \n",
      "MP-5.1-005 Conduct adversarial role-playing exercises, GAI red-teaming, or chaos testing to \n",
      "identify anomalous or unforeseen failure modes. Information Security \n",
      "MP-5.1-006 \n",
      "Profile threats and negative impacts arising from GAI systems interacting with, \n",
      "manipulating, or generating content, and outlining known and potential \n",
      "vulnerabilities and the likelihood of their occurrence.\n",
      "4: Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile\n",
      " \n",
      "35 \n",
      "MEASURE 2.9: The AI model is explained, validated, and documented, and AI system output is interpreted within its context – as \n",
      "identified in the MAP function – to inform responsible use and governance. Action ID \n",
      "Suggested Action \n",
      "GAI Risks \n",
      "MS-2.9-001 \n",
      "Apply and document ML explanation results such as: Analysis of embeddings, \n",
      "Counterfactual prompts, Gradient-based attributions, Model \n",
      "compression/surrogate models, Occlusion/term reduction. Confabulation \n",
      "MS-2.9-002 \n",
      "Document GAI model details including: Proposed use and organizational value; \n",
      "Assumptions and limitations, Data collection methodologies; Data provenance; \n",
      "Data quality; Model architecture (e.g., convolutional neural network, \n",
      "transformers, etc.); Optimization objectives; Training algorithms; RLHF \n",
      "approaches; Fine-tuning or retrieval-augmented generation approaches; \n",
      "Evaluation data; Ethical considerations; Legal and regulatory requirements. Information Integrity; Harmful Bias \n",
      "and Homogenization \n",
      "AI Actor Tasks: AI Deployment, AI Impact Assessment, Domain Experts, End-Users, Operation and Monitoring, TEVV \n",
      " \n",
      "MEASURE 2.10: Privacy risk of the AI system – as identified in the MAP function – is examined and documented. Action ID \n",
      "Suggested Action \n",
      "GAI Risks \n",
      "MS-2.10-001 \n",
      "Conduct AI red-teaming to assess issues such as: Outputting of training data \n",
      "samples, and subsequent reverse engineering, model extraction, and \n",
      "membership inference risks; Revealing biometric, confidential, copyrighted, \n",
      "licensed, patented, personal, proprietary, sensitive, or trade-marked information; \n",
      "Tracking or revealing location information of users or members of training \n",
      "datasets. Human-AI Configuration; \n",
      "Information Integrity; Intellectual \n",
      "Property \n",
      "MS-2.10-002 \n",
      "Engage directly with end-users and other stakeholders to understand their \n",
      "expectations and concerns regarding content provenance.\n",
      "5: Blueprint for an AI Bill of Rights\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "SECTION TITLE\n",
      "DATA PRIVACY\n",
      "You should be protected from abusive data practices via built-in protections and you \n",
      "should have agency over how data about you is used. You should be protected from violations of \n",
      "privacy through design choices that ensure such protections are included by default, including ensuring that \n",
      "data collection conforms to reasonable expectations and that only data strictly necessary for the specific \n",
      "context is collected. Designers, developers, and deployers of automated systems should seek your permission \n",
      "and respect your decisions regarding collection, use, access, transfer, and deletion of your data in appropriate \n",
      "ways and to the greatest extent possible; where not possible, alternative privacy by design safeguards should be \n",
      "used. Systems should not employ user experience and design decisions that obfuscate user choice or burden \n",
      "users with defaults that are privacy invasive. Consent should only be used to justify collection of data in cases \n",
      "where it can be appropriately and meaningfully given. Any consent requests should be brief, be understandable \n",
      "in plain language, and give you agency over data collection and the specific context of use; current hard-to­\n",
      "understand notice-and-choice practices for broad uses of data should be changed. Enhanced protections and \n",
      "restrictions for data and inferences related to sensitive domains, including health, work, education, criminal \n",
      "justice, and finance, and for data pertaining to youth should put you first. In sensitive domains, your data and \n",
      "related inferences should only be used for necessary functions, and you should be protected by ethical review \n",
      "and use prohibitions. You and your communities should be free from unchecked surveillance; surveillance \n",
      "technologies should be subject to heightened oversight that includes at least pre-deployment assessment of their \n",
      "potential harms and scope limits to protect privacy and civil liberties. Continuous surveillance and monitoring \n",
      "should not be used in education, work, housing, or in other contexts where the use of such surveillance \n",
      "technologies is likely to limit rights, opportunities, or access. Whenever possible, you should have access to \n",
      "reporting that confirms your data decisions have been respected and provides an assessment of the \n",
      "potential impact of surveillance technologies on your rights, opportunities, or access. NOTICE AND EXPLANATION\n",
      "You should know that an automated system is being used and understand how and why it \n",
      "contributes to outcomes that impact you.\n",
      "6: Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile\n",
      " \n",
      "48 \n",
      "• Data protection \n",
      "• Data retention  \n",
      "• Consistency in use of defining key terms \n",
      "• Decommissioning \n",
      "• Discouraging anonymous use \n",
      "• Education  \n",
      "• Impact assessments  \n",
      "• Incident response \n",
      "• Monitoring \n",
      "• Opt-outs  \n",
      "• Risk-based controls \n",
      "• Risk mapping and measurement \n",
      "• Science-backed TEVV practices \n",
      "• Secure software development practices \n",
      "• Stakeholder engagement \n",
      "• Synthetic content detection and \n",
      "labeling tools and techniques \n",
      "• Whistleblower protections \n",
      "• Workforce diversity and \n",
      "interdisciplinary teams\n",
      "Establishing acceptable use policies and guidance for the use of GAI in formal human-AI teaming settings \n",
      "as well as different levels of human-AI configurations can help to decrease risks arising from misuse, \n",
      "abuse, inappropriate repurpose, and misalignment between systems and users. These practices are just \n",
      "one example of adapting existing governance protocols for GAI contexts.\n",
      "7: Blueprint for an AI Bill of Rights\n",
      "Federal government surveillance and other collection and \n",
      "use of data is governed by legal protections that help to protect civil liberties and provide for limits on data retention \n",
      "in some cases. Many states have also enacted consumer data privacy protection regimes to address some of these \n",
      "harms. However, these are not yet standard practices, and the United States lacks a comprehensive statutory or regulatory \n",
      "framework governing the rights of the public when it comes to personal data. While a patchwork of laws exists to \n",
      "guide the collection and use of personal data in specific contexts, including health, employment, education, and credit, \n",
      "it can be unclear how these laws apply in other contexts and in an increasingly automated society. Additional protec­\n",
      "tions would assure the American public that the automated systems they use are not monitoring their activities, \n",
      "collecting information on their lives, or otherwise surveilling them without context-specific consent or legal authori­\n",
      "ty. 31\n",
      "\n",
      "8: Blueprint for an AI Bill of Rights\n",
      " \n",
      " \n",
      " \n",
      "SECTION TITLE\n",
      "Applying The Blueprint for an AI Bill of Rights \n",
      "While many of the concerns addressed in this framework derive from the use of AI, the technical \n",
      "capabilities and specific definitions of such systems change with the speed of innovation, and the potential \n",
      "harms of their use occur even with less technologically sophisticated tools. Thus, this framework uses a two-\n",
      "part test to determine what systems are in scope. This framework applies to (1) automated systems that (2) \n",
      "have the potential to meaningfully impact the American public’s rights, opportunities, or access to \n",
      "critical resources or services. These rights, opportunities, and access to critical resources of services should \n",
      "be enjoyed equally and be fully protected, regardless of the changing role that automated systems may play in \n",
      "our lives. This framework describes protections that should be applied with respect to all automated systems that \n",
      "have the potential to meaningfully impact individuals' or communities' exercise of: \n",
      "RIGHTS, OPPORTUNITIES, OR ACCESS\n",
      "Civil rights, civil liberties, and privacy, including freedom of speech, voting, and protections from discrimi­\n",
      "nation, excessive punishment, unlawful surveillance, and violations of privacy and other freedoms in both \n",
      "public and private sector contexts; \n",
      "Equal opportunities, including equitable access to education, housing, credit, employment, and other \n",
      "programs; or, \n",
      "Access to critical resources or services, such as healthcare, financial services, safety, social services, \n",
      "non-deceptive information about goods and services, and government benefits. A list of examples of automated systems for which these principles should be considered is provided in the \n",
      "Appendix. The Technical Companion, which follows, offers supportive guidance for any person or entity that \n",
      "creates, deploys, or oversees automated systems. Considered together, the five principles and associated practices of the Blueprint for an AI Bill of \n",
      "Rights form an overlapping set of backstops against potential harms. This purposefully overlapping \n",
      "framework, when taken as a whole, forms a blueprint to help protect the public from harm. The measures taken to realize the vision set forward in this framework should be proportionate \n",
      "with the extent and nature of the harm, or risk of harm, to people's rights, opportunities, and \n",
      "access. RELATIONSHIP TO EXISTING LAW AND POLICY\n",
      "The Blueprint for an AI Bill of Rights is an exercise in envisioning a future where the American public is \n",
      "protected from the potential harms, and can fully enjoy the benefits, of automated systems. It describes princi­\n",
      "ples that can help ensure these protections.\n",
      "9: Blueprint for an AI Bill of Rights\n",
      "In other cases, technologies do not work as intended or as promised, causing substantial and unjustified harm. Automated systems sometimes rely on data from other systems, including historical data, allowing irrelevant informa­\n",
      "tion from past decisions to infect decision-making in unrelated situations. In some cases, technologies are purposeful­\n",
      "ly designed to violate the safety of others, such as technologies designed to facilitate stalking; in other cases, intended \n",
      "or unintended uses lead to unintended harms. Many of the harms resulting from these technologies are preventable, and actions are already being taken to protect \n",
      "the public. Some companies have put in place safeguards that have prevented harm from occurring by ensuring that \n",
      "key development decisions are vetted by an ethics review; others have identified and mitigated harms found through \n",
      "pre-deployment testing and ongoing monitoring processes. Governments at all levels have existing public consulta­\n",
      "tion processes that may be applied when considering the use of new automated systems, and existing product develop­\n",
      "ment and testing practices already protect the American public from many potential harms. Still, these kinds of practices are deployed too rarely and unevenly. Expanded, proactive protections could build on \n",
      "these existing practices, increase confidence in the use of automated systems, and protect the American public. Inno­\n",
      "vators deserve clear rules of the road that allow new ideas to flourish, and the American public deserves protections \n",
      "from unsafe outcomes. All can benefit from assurances that automated systems will be designed, tested, and consis­\n",
      "tently confirmed to work as intended, and that they will be proactively protected from foreseeable unintended harm­\n",
      "ful outcomes.\n"
     ]
    }
   ],
   "source": [
    "docs = retriver.invoke(\"How can companies ensure AI does not violate data privacy laws?\")\n",
    "for i, doc in enumerate(docs):\n",
    "    print(f\"{i}: {doc.metadata.get('title')}\")\n",
    "    print(doc.page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Trying Compression retriver\n",
    "from langchain.retrievers import ContextualCompressionRetriever\n",
    "from langchain.retrievers.document_compressors import LLMChainExtractor\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "base_retriever = retriver\n",
    "\n",
    "#Create a contextual compressor\n",
    "compressor_llm = ChatOpenAI(temperature=0, model_name=\"gpt-4o\", max_tokens=4000)\n",
    "compressor = LLMChainExtractor.from_llm(compressor_llm)\n",
    "\n",
    "#Combine the retriever with the compressor\n",
    "compression_retriever = ContextualCompressionRetriever(\n",
    "    base_compressor=compressor,\n",
    "    base_retriever=base_retriever\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 29, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '1ba549ad3b9b488ab15014c90b909ad1', '_collection_name': 'ai-safety'}, page_content='Conduct periodic monitoring of AI-generated content for privacy risks; address any possible instances of PII or sensitive data exposure. \\n\\nConnect new GAI policies, procedures, and processes to existing model, data, software development, and IT governance and to legal, compliance, and risk management activities. \\n\\nDocument training data curation policies, to the extent possible and according to applicable laws and policies. \\n\\nEstablish policies for collection, retention, and minimum quality of data, in consideration of the following risks: Disclosure of inappropriate CBRN information; Use of Illegal or dangerous content; Offensive cyber capabilities; Training data imbalances that could give rise to harmful biases; Leak of personally identifiable information, including facial likenesses of individuals.'),\n",
       " Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 32, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '04acfe508f5c428caeeb90944c1d7386', '_collection_name': 'ai-safety'}, page_content='Data should \\nonly be collected or used for the purposes of training or testing machine learning models if such collection and \\nuse is legal and consistent with the expectations of the people whose data is collected. User experience \\nresearch should be conducted to confirm that people understand what data is being collected about them and \\nhow it will be used, and that this collection matches their expectations and desires. Data collection and use-case scope limits. Data collection should be limited in scope, with specific, \\nnarrow identified goals, to avoid \"mission creep.\"  Anticipated data collection should be determined to be \\nstrictly necessary to the identified goals and should be minimized as much as possible. Data collected based on \\nthese identified goals and for a specific context should not be used in a different context without assessing for \\nnew privacy risks and implementing appropriate mitigation measures, which may include express consent. Clear timelines for data retention should be established, with data deleted as soon as possible in accordance \\nwith legal or policy-based limitations. Determined data retention timelines should be documented and justi\\xad\\nfied. Risk identification and mitigation. Entities that collect, use, share, or store sensitive data should \\nattempt to proactively identify harms and seek to manage them so as to avoid, mitigate, and respond appropri\\xad\\nately to identified risks. Appropriate responses include determining not to process data when the privacy risks \\noutweigh the benefits or implementing measures to mitigate acceptable risks. Appropriate responses do not \\ninclude sharing or transferring the privacy risks to users via notice or consent requests where users could not \\nreasonably be expected to understand the risks without further support. Privacy-preserving security. Entities creating, using, or governing automated systems should follow \\nprivacy and security best practices designed to ensure data and metadata do not leak beyond the specific \\nconsented use case. Best practices could include using privacy-enhancing cryptography or other types of \\nprivacy-enhancing technologies or fine-grained permissions and access control mechanisms, along with \\nconventional system security protocols.'),\n",
       " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 30, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '7e66f3997e2243ad97a378ad1f2d9cd0', '_collection_name': 'ai-safety'}, page_content='Conduct appropriate diligence on training data use to assess intellectual property, \\nand privacy, risks, including to examine whether use of proprietary or sensitive \\ntraining data is consistent with applicable laws. Intellectual Property; Data Privacy \\nAI Actor Tasks: Governance and Oversight, Operation and Monitoring, Procurement, Third-party entities'),\n",
       " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 38, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '907f59c504d44945ae865021b6e9c713', '_collection_name': 'ai-safety'}, page_content='MEASURE 2.10: Privacy risk of the AI system – as identified in the MAP function – is examined and documented. Action ID \\nSuggested Action \\nGAI Risks \\nMS-2.10-001 \\nConduct AI red-teaming to assess issues such as: Outputting of training data \\nsamples, and subsequent reverse engineering, model extraction, and \\nmembership inference risks; Revealing biometric, confidential, copyrighted, \\nlicensed, patented, personal, proprietary, sensitive, or trade-marked information; \\nTracking or revealing location information of users or members of training \\ndatasets. Human-AI Configuration; \\nInformation Integrity; Intellectual \\nProperty \\nMS-2.10-002 \\nEngage directly with end-users and other stakeholders to understand their \\nexpectations and concerns regarding content provenance.'),\n",
       " Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 5, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '498d23d9347f49ac9c61216ea042aefd', '_collection_name': 'ai-safety'}, page_content='SECTION TITLE\\nDATA PRIVACY\\nYou should be protected from abusive data practices via built-in protections and you \\nshould have agency over how data about you is used. You should be protected from violations of \\nprivacy through design choices that ensure such protections are included by default, including ensuring that \\ndata collection conforms to reasonable expectations and that only data strictly necessary for the specific \\ncontext is collected. Designers, developers, and deployers of automated systems should seek your permission \\nand respect your decisions regarding collection, use, access, transfer, and deletion of your data in appropriate \\nways and to the greatest extent possible; where not possible, alternative privacy by design safeguards should be \\nused. Systems should not employ user experience and design decisions that obfuscate user choice or burden \\nusers with defaults that are privacy invasive. Consent should only be used to justify collection of data in cases \\nwhere it can be appropriately and meaningfully given. Any consent requests should be brief, be understandable \\nin plain language, and give you agency over data collection and the specific context of use; current hard-to\\xad\\nunderstand notice-and-choice practices for broad uses of data should be changed. Enhanced protections and \\nrestrictions for data and inferences related to sensitive domains, including health, work, education, criminal \\njustice, and finance, and for data pertaining to youth should put you first. In sensitive domains, your data and \\nrelated inferences should only be used for necessary functions, and you should be protected by ethical review \\nand use prohibitions. You and your communities should be free from unchecked surveillance; surveillance \\ntechnologies should be subject to heightened oversight that includes at least pre-deployment assessment of their \\npotential harms and scope limits to protect privacy and civil liberties. Continuous surveillance and monitoring \\nshould not be used in education, work, housing, or in other contexts where the use of such surveillance \\ntechnologies is likely to limit rights, opportunities, or access. Whenever possible, you should have access to \\nreporting that confirms your data decisions have been respected and provides an assessment of the \\npotential impact of surveillance technologies on your rights, opportunities, or access.')]"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compression_retriever.invoke(\"How can companies ensure AI does not violate data privacy laws?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts import ChatPromptTemplate\n",
    "\n",
    "base_rag_prompt_template = \"\"\"\\\n",
    "You are a helpful assistant that can answer questions related to the provided context. Repond I don't have that information if outside context.\n",
    "\n",
    "Context:\n",
    "{context}\n",
    "\n",
    "Question:\n",
    "{question}\n",
    "\"\"\"\n",
    "\n",
    "base_rag_prompt = ChatPromptTemplate.from_template(base_rag_prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai.chat_models import ChatOpenAI\n",
    "\n",
    "base_llm = ChatOpenAI(model=\"gpt-4o\", tags=[\"base_llm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "from langchain.schema.output_parser import StrOutputParser\n",
    "from langchain.schema.runnable import RunnablePassthrough\n",
    "\n",
    "retrieval_augmented_qa_chain = (\n",
    "    {\"context\": itemgetter(\"question\") | compression_retriever, \"question\": itemgetter(\"question\")}\n",
    "    | RunnablePassthrough.assign(context=itemgetter(\"context\"))\n",
    "    | {\"response\": base_rag_prompt | base_llm, \"context\": itemgetter(\"context\")}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = retrieval_augmented_qa_chain.invoke({\"question\" : \"How can companies ensure AI does not violate data privacy laws?\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Companies can ensure that AI does not violate data privacy laws by implementing several strategies and practices as mentioned in the provided context:\\n\\n1. **Periodic Monitoring**: Conduct regular monitoring of AI-generated content to identify and address any potential instances of personally identifiable information (PII) or sensitive data exposure. \\n\\n2. **Integration with Existing Policies**: Connect new AI policies, procedures, and processes with existing model, data, software development, and IT governance, as well as legal, compliance, and risk management activities.\\n\\n3. **Training Data Curation Policies**: Document training data curation policies in accordance with applicable laws and policies. This includes policies for the collection, retention, and minimum quality of data to mitigate risks such as the disclosure of inappropriate information, use of illegal or dangerous content, offensive cyber capabilities, data imbalances leading to harmful biases, and leaks of PII.\\n\\n4. **Diligence on Training Data**: Conduct appropriate diligence on the use of training data to assess intellectual property and privacy risks, ensuring that the use of proprietary or sensitive data is consistent with applicable laws.\\n\\n5. **User Experience Research**: Conduct user experience research to confirm that individuals understand what data is being collected about them and how it will be used, ensuring that this collection matches their expectations and desires.\\n\\n6. **Scope Limits on Data Collection**: Limit data collection to specific, narrow goals to avoid \"mission creep.\" Anticipated data collection should be strictly necessary for the identified goals and minimized as much as possible.\\n\\n7. **Risk Identification and Mitigation**: Proactively identify and manage privacy risks to avoid, mitigate, and respond appropriately to identified risks. This includes determining not to process data when privacy risks outweigh the benefits or implementing measures to mitigate acceptable risks.\\n\\n8. **Privacy-Preserving Security**: Follow privacy and security best practices to ensure that data and metadata do not leak beyond the specific consented use case. This can include using privacy-enhancing cryptography, privacy-enhancing technologies, fine-grained permissions, and access control mechanisms.\\n\\n9. **Consent and Privacy by Design**: Seek user permission and respect user decisions regarding data collection, use, access, transfer, and deletion to the greatest extent possible. Implement privacy by design safeguards where consent is not feasible, ensuring that systems do not employ user experience and design decisions that obfuscate user choice or burden users with privacy-invasive defaults.\\n\\n10. **Enhanced Protections for Sensitive Data**: Implement enhanced protections and restrictions for data and inferences related to sensitive domains such as health, work, education, criminal justice, and finance. Ensure that data pertaining to youth is protected, and any use in sensitive domains is subject to ethical review and use prohibitions.\\n\\n11. **Surveillance and Monitoring**: Ensure that surveillance technologies are subject to heightened oversight, including pre-deployment assessment of potential harms and scope limits to protect privacy and civil liberties. Avoid continuous surveillance and monitoring in contexts where it could limit rights, opportunities, or access.\\n\\nBy adopting these measures, companies can better ensure that their AI systems comply with data privacy laws and protect the privacy of individuals.'"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.get('response').content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'response': AIMessage(content='Companies can ensure that AI does not violate data privacy laws by implementing several strategies and practices as mentioned in the provided context:\\n\\n1. **Periodic Monitoring**: Conduct regular monitoring of AI-generated content to identify and address any potential instances of personally identifiable information (PII) or sensitive data exposure. \\n\\n2. **Integration with Existing Policies**: Connect new AI policies, procedures, and processes with existing model, data, software development, and IT governance, as well as legal, compliance, and risk management activities.\\n\\n3. **Training Data Curation Policies**: Document training data curation policies in accordance with applicable laws and policies. This includes policies for the collection, retention, and minimum quality of data to mitigate risks such as the disclosure of inappropriate information, use of illegal or dangerous content, offensive cyber capabilities, data imbalances leading to harmful biases, and leaks of PII.\\n\\n4. **Diligence on Training Data**: Conduct appropriate diligence on the use of training data to assess intellectual property and privacy risks, ensuring that the use of proprietary or sensitive data is consistent with applicable laws.\\n\\n5. **User Experience Research**: Conduct user experience research to confirm that individuals understand what data is being collected about them and how it will be used, ensuring that this collection matches their expectations and desires.\\n\\n6. **Scope Limits on Data Collection**: Limit data collection to specific, narrow goals to avoid \"mission creep.\" Anticipated data collection should be strictly necessary for the identified goals and minimized as much as possible.\\n\\n7. **Risk Identification and Mitigation**: Proactively identify and manage privacy risks to avoid, mitigate, and respond appropriately to identified risks. This includes determining not to process data when privacy risks outweigh the benefits or implementing measures to mitigate acceptable risks.\\n\\n8. **Privacy-Preserving Security**: Follow privacy and security best practices to ensure that data and metadata do not leak beyond the specific consented use case. This can include using privacy-enhancing cryptography, privacy-enhancing technologies, fine-grained permissions, and access control mechanisms.\\n\\n9. **Consent and Privacy by Design**: Seek user permission and respect user decisions regarding data collection, use, access, transfer, and deletion to the greatest extent possible. Implement privacy by design safeguards where consent is not feasible, ensuring that systems do not employ user experience and design decisions that obfuscate user choice or burden users with privacy-invasive defaults.\\n\\n10. **Enhanced Protections for Sensitive Data**: Implement enhanced protections and restrictions for data and inferences related to sensitive domains such as health, work, education, criminal justice, and finance. Ensure that data pertaining to youth is protected, and any use in sensitive domains is subject to ethical review and use prohibitions.\\n\\n11. **Surveillance and Monitoring**: Ensure that surveillance technologies are subject to heightened oversight, including pre-deployment assessment of potential harms and scope limits to protect privacy and civil liberties. Avoid continuous surveillance and monitoring in contexts where it could limit rights, opportunities, or access.\\n\\nBy adopting these measures, companies can better ensure that their AI systems comply with data privacy laws and protect the privacy of individuals.', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 627, 'prompt_tokens': 2398, 'total_tokens': 3025, 'completion_tokens_details': {'reasoning_tokens': 0}}, 'model_name': 'gpt-4o-2024-05-13', 'system_fingerprint': 'fp_3537616b13', 'finish_reason': 'stop', 'logprobs': None}, id='run-8739df9f-0dc0-4aea-a089-5fa12ac6189e-0', usage_metadata={'input_tokens': 2398, 'output_tokens': 627, 'total_tokens': 3025}),\n",
       " 'context': [Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 29, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '1ba549ad3b9b488ab15014c90b909ad1', '_collection_name': 'ai-safety'}, page_content='Conduct periodic monitoring of AI-generated content for privacy risks; address any possible instances of PII or sensitive data exposure. \\n\\nConnect new GAI policies, procedures, and processes to existing model, data, software development, and IT governance and to legal, compliance, and risk management activities. \\n\\nDocument training data curation policies, to the extent possible and according to applicable laws and policies. \\n\\nEstablish policies for collection, retention, and minimum quality of data, in consideration of the following risks: Disclosure of inappropriate CBRN information; Use of Illegal or dangerous content; Offensive cyber capabilities; Training data imbalances that could give rise to harmful biases; Leak of personally identifiable information, including facial likenesses of individuals.'),\n",
       "  Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 32, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '04acfe508f5c428caeeb90944c1d7386', '_collection_name': 'ai-safety'}, page_content='Data should \\nonly be collected or used for the purposes of training or testing machine learning models if such collection and \\nuse is legal and consistent with the expectations of the people whose data is collected. User experience \\nresearch should be conducted to confirm that people understand what data is being collected about them and \\nhow it will be used, and that this collection matches their expectations and desires. Data collection and use-case scope limits. Data collection should be limited in scope, with specific, \\nnarrow identified goals, to avoid \"mission creep.\"  Anticipated data collection should be determined to be \\nstrictly necessary to the identified goals and should be minimized as much as possible. Data collected based on \\nthese identified goals and for a specific context should not be used in a different context without assessing for \\nnew privacy risks and implementing appropriate mitigation measures, which may include express consent. Clear timelines for data retention should be established, with data deleted as soon as possible in accordance \\nwith legal or policy-based limitations. Determined data retention timelines should be documented and justi\\xad\\nfied. Risk identification and mitigation. Entities that collect, use, share, or store sensitive data should \\nattempt to proactively identify harms and seek to manage them so as to avoid, mitigate, and respond appropri\\xad\\nately to identified risks. Appropriate responses include determining not to process data when the privacy risks \\noutweigh the benefits or implementing measures to mitigate acceptable risks. Appropriate responses do not \\ninclude sharing or transferring the privacy risks to users via notice or consent requests where users could not \\nreasonably be expected to understand the risks without further support. Privacy-preserving security. Entities creating, using, or governing automated systems should follow \\nprivacy and security best practices designed to ensure data and metadata do not leak beyond the specific \\nconsented use case. Best practices could include using privacy-enhancing cryptography or other types of \\nprivacy-enhancing technologies or fine-grained permissions and access control mechanisms, along with \\nconventional system security protocols.'),\n",
       "  Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 30, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '7e66f3997e2243ad97a378ad1f2d9cd0', '_collection_name': 'ai-safety'}, page_content='Conduct appropriate diligence on training data use to assess intellectual property, \\nand privacy, risks, including to examine whether use of proprietary or sensitive \\ntraining data is consistent with applicable laws. Intellectual Property; Data Privacy \\nAI Actor Tasks: Governance and Oversight, Operation and Monitoring, Procurement, Third-party entities'),\n",
       "  Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 38, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '907f59c504d44945ae865021b6e9c713', '_collection_name': 'ai-safety'}, page_content='>>>\\nMEASURE 2.10: Privacy risk of the AI system – as identified in the MAP function – is examined and documented. Action ID \\nSuggested Action \\nGAI Risks \\nMS-2.10-001 \\nConduct AI red-teaming to assess issues such as: Outputting of training data \\nsamples, and subsequent reverse engineering, model extraction, and \\nmembership inference risks; Revealing biometric, confidential, copyrighted, \\nlicensed, patented, personal, proprietary, sensitive, or trade-marked information; \\nTracking or revealing location information of users or members of training \\ndatasets. Human-AI Configuration; \\nInformation Integrity; Intellectual \\nProperty \\nMS-2.10-002 \\nEngage directly with end-users and other stakeholders to understand their \\nexpectations and concerns regarding content provenance.\\n>>>'),\n",
       "  Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 5, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '498d23d9347f49ac9c61216ea042aefd', '_collection_name': 'ai-safety'}, page_content='SECTION TITLE\\nDATA PRIVACY\\nYou should be protected from abusive data practices via built-in protections and you \\nshould have agency over how data about you is used. You should be protected from violations of \\nprivacy through design choices that ensure such protections are included by default, including ensuring that \\ndata collection conforms to reasonable expectations and that only data strictly necessary for the specific \\ncontext is collected. Designers, developers, and deployers of automated systems should seek your permission \\nand respect your decisions regarding collection, use, access, transfer, and deletion of your data in appropriate \\nways and to the greatest extent possible; where not possible, alternative privacy by design safeguards should be \\nused. Systems should not employ user experience and design decisions that obfuscate user choice or burden \\nusers with defaults that are privacy invasive. Consent should only be used to justify collection of data in cases \\nwhere it can be appropriately and meaningfully given. Any consent requests should be brief, be understandable \\nin plain language, and give you agency over data collection and the specific context of use; current hard-to\\xad\\nunderstand notice-and-choice practices for broad uses of data should be changed. Enhanced protections and \\nrestrictions for data and inferences related to sensitive domains, including health, work, education, criminal \\njustice, and finance, and for data pertaining to youth should put you first. In sensitive domains, your data and \\nrelated inferences should only be used for necessary functions, and you should be protected by ethical review \\nand use prohibitions. You and your communities should be free from unchecked surveillance; surveillance \\ntechnologies should be subject to heightened oversight that includes at least pre-deployment assessment of their \\npotential harms and scope limits to protect privacy and civil liberties. Continuous surveillance and monitoring \\nshould not be used in education, work, housing, or in other contexts where the use of such surveillance \\ntechnologies is likely to limit rights, opportunities, or access. Whenever possible, you should have access to \\nreporting that confirms your data decisions have been respected and provides an assessment of the \\npotential impact of surveillance technologies on your rights, opportunities, or access.')]}"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}