Refactor #4

notebooks/google_drive_web_vtt_vectorizer_and_storer.ipynb

@@ -27,6 +27,7 @@
     "display_html = partial(display_html, raw=True)\n",
     "\n",
     "container = Container()\n",
+    "mime_type_handlers = container.mime_type_handlers()\n",
     "mongo_db = await container.mongo_db()\n",
     "google_drive_service = container.google_drive_service()\n",
     "vectorization_service = container.vectorization_service()\n",
@@ -85,19 +86,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "web_vtt_parser = container.mime_type_handlers()[MIME_TYPE]\n",
+    "web_vtt_parser = mime_type_handlers[MIME_TYPE]\n",
     "display_html(f\"<p>{escape(str(type(web_vtt_parser)))}</p><code>{id(web_vtt_parser)}</code>\")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "display_html(f\"<code>{id(container.mime_type_handlers()[MIME_TYPE])}</code>\")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -111,8 +103,8 @@
     "                                               \"modificationTime\": metadata.modified_time\n",
     "                                           },\n",
     "                                           google_drive_service.read_file_by_id(metadata.id))\n",
-    "            for metadata\n",
-    "            in metadata_to_process)\n",
+    "                 for metadata\n",
+    "                 in metadata_to_process)\n",
     "\n",
     "display_html(f\"Processed {len(web_vtts)} files.\")"
    ]
@@ -147,7 +139,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "mongo_db.close()"
+    "await container.shutdown_resources()"
    ]
   }
  ],

src/ctp_slack_bot/containers.py

@@ -1,6 +1,7 @@
 from dependency_injector.containers import DeclarativeContainer
 from dependency_injector.providers import Callable, Dict, List, Resource, Singleton
 from importlib import import_module
+from openai import AsyncOpenAI
 from pkgutil import iter_modules
 from slack_bolt.adapter.socket_mode.async_handler import AsyncSocketModeHandler
 from slack_bolt.async_app import AsyncApp
@@ -50,8 +51,11 @@ class Container(DeclarativeContainer): # TODO: audit for potential async-related
     vectorized_chunk_repository = Resource (MongoVectorizedChunkRepositoryResource,
                                             settings=settings,
                                             mongo_db=mongo_db)
-    embeddings_model_service    = Singleton(EmbeddingsModelService,
+    open_ai_client              = Singleton(lambda settings: AsyncOpenAI(api_key=settings.openai_api_key.get_secret_value()),
                                             settings=settings)
+    embeddings_model_service    = Singleton(EmbeddingsModelService,
+                                            settings=settings,
+                                            open_ai_client=open_ai_client)
     vectorization_service       = Singleton(VectorizationService,
                                             settings=settings,
                                             embeddings_model_service=embeddings_model_service)
@@ -65,7 +69,8 @@ class Container(DeclarativeContainer): # TODO: audit for potential async-related
                                             vectorization_service=vectorization_service,
                                             vectorized_chunk_repository=vectorized_chunk_repository)
     language_model_service      = Singleton(LanguageModelService,
-                                            settings=settings)
+                                            settings=settings,
+                                            open_ai_client=open_ai_client)
     answer_retrieval_service    = Singleton(AnswerRetrievalService,
                                             settings=settings,
                                             event_brokerage_service=event_brokerage_service,

src/ctp_slack_bot/mime_type_handlers/base.py

@@ -1,5 +1,4 @@
 from abc import ABC, abstractmethod
-from functools import lru_cache
 from importlib import import_module
 from types import MappingProxyType
 from typing import Any, ClassVar, Mapping, Optional
@@ -32,8 +31,3 @@ class MimeTypeHandlerRegistry:
             cls._registry[mime_type] = handler_cls
             return handler_cls
         return decorator
-
-    @classmethod
-    @lru_cache
-    def get_handler(cls, mime_type: str) -> Optional[MimeTypeHandler]:
-        return cls._registry.get(mime_type)()

src/ctp_slack_bot/mime_type_handlers/text/vtt.py

@@ -3,7 +3,7 @@ from io import BytesIO
 from pydantic import ConfigDict
 from re import compile as compile_re, Pattern
 from types import MappingProxyType
-from typing import Any, Mapping, Optional, Self
+from typing import Any, ClassVar, Mapping, Optional, Self
 from webvtt import WebVTT
 
 from ctp_slack_bot.mime_type_handlers import MimeTypeHandler, MimeTypeHandlerRegistry
@@ -15,7 +15,7 @@ class WebVTTMimeTypeHandler(MimeTypeHandler):
 
     model_config = ConfigDict(frozen=True)
 
-    ISO_DATE_TIME_PATTERN: Pattern = compile_re(r"Start time: (\d{4}-\d{2}-\d{2}(?: \d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})?)?)")
+    ISO_DATE_TIME_PATTERN: ClassVar[Pattern] = compile_re(r"Start time: (\d{4}-\d{2}-\d{2}(?: \d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})?)?)")
 
     @classmethod
     def __get_start_time(cls, web_vtt: WebVTT) -> Optional[datetime]:

src/ctp_slack_bot/models/webvtt.py

@@ -3,23 +3,20 @@ from io import BytesIO
 from more_itertools import windowed
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 from types import MappingProxyType
-from typing import Any, Literal, Mapping, Optional, Self
+from typing import Any, ClassVar, Literal, Mapping, Optional, Self
 from webvtt import Caption, WebVTT
 
 from ctp_slack_bot.models.base import Chunk, Content
 from ctp_slack_bot.utils import to_deep_immutable
 
 
-CHUNK_FRAMES_OVERLAP = 1
-CHUNK_FRAMES_WINDOW = 5
-SPEAKER_SPEECH_TEXT_SEPARATOR = ": "
-
-
 class WebVTTFrame(BaseModel):
     """Represents a WebVTT frame"""
 
     model_config = ConfigDict(frozen=True)
 
+    _SPEAKER_SPEECH_TEXT_SEPARATOR: ClassVar[str] = ": "
+
     identifier: str
     start: timedelta
     end: timedelta
@@ -31,7 +28,7 @@ class WebVTTFrame(BaseModel):
         identifier = caption.identifier if caption.identifier else str(index)
         start = timedelta(**caption.start_time.__dict__)
         end = timedelta(**caption.end_time.__dict__)
-        match caption.text.split(SPEAKER_SPEECH_TEXT_SEPARATOR, 1):
+        match caption.text.split(cls._SPEAKER_SPEECH_TEXT_SEPARATOR, 1):
             case [speaker, speech]:
                 return cls(identifier=identifier, start=start, end=end, speaker=speaker, speech=speech)
             case [speech]:
@@ -41,6 +38,9 @@ class WebVTTFrame(BaseModel):
 class WebVTTContent(Content):
     """Represents parsed WebVTT content."""
 
+    CHUNK_FRAMES_OVERLAP: ClassVar[int] = 1
+    CHUNK_FRAMES_WINDOW: ClassVar[int] = 5
+
     id: str
     metadata: Mapping[str, Any] = Field(default_factory=lambda: MappingProxyType({}))
     start_time: Optional[datetime]
@@ -52,7 +52,7 @@ class WebVTTContent(Content):
     def get_chunks(self: Self) -> tuple[Chunk]:
         windows = (tuple(filter(None, window))
                    for window
-                   in windowed(self.frames, CHUNK_FRAMES_WINDOW, step=CHUNK_FRAMES_WINDOW-CHUNK_FRAMES_OVERLAP))
+                   in windowed(self.frames, self.CHUNK_FRAMES_WINDOW, step=self.CHUNK_FRAMES_WINDOW-self.CHUNK_FRAMES_OVERLAP))
         return tuple(Chunk(text="\n\n".join(": ".join(filter(None, (frame.speaker, frame.speech)))
                                             for frame
                                             in frames),

src/ctp_slack_bot/services/embeddings_model_service.py

@@ -11,14 +11,10 @@ class EmbeddingsModelService(ApplicationComponentBase):
     Service for embeddings model operations.
     """
 
-    model_config = ConfigDict(frozen=True)
+    model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
 
     settings: Settings
-    _open_ai_client: AsyncOpenAI
-
-    def model_post_init(self: Self, context: Any, /) -> None:
-        super().model_post_init(context)
-        self._open_ai_client = AsyncOpenAI(api_key=self.settings.openai_api_key.get_secret_value())
+    open_ai_client: AsyncOpenAI
 
     async def get_embeddings(self: Self, texts: Sequence[str]) -> Sequence[Sequence[float]]:
         """Get embeddings for a collection of texts using OpenAI’s API.
@@ -33,7 +29,7 @@ class EmbeddingsModelService(ApplicationComponentBase):
             ValueError: If the embedding dimensions don't match expected size
         """
         logger.debug("Creating embeddings for {} text string(s)…", len(texts))
-        response = await self._open_ai_client.embeddings.create(
+        response = await self.open_ai_client.embeddings.create(
             model=self.settings.embedding_model,
             input=texts,
             encoding_format="float" # Ensure we get raw float values.

src/ctp_slack_bot/services/google_drive_service.py

@@ -9,21 +9,20 @@ from io import BytesIO
 from itertools import chain
 from loguru import logger
 from pydantic import ConfigDict, PrivateAttr
-from typing import Any, Collection, Optional, Self
+from typing import Any, ClassVar, Collection, Optional, Self
 
 from ctp_slack_bot.core import ApplicationComponentBase, Settings
 from ctp_slack_bot.models import GoogleDriveMetadata
 
 
-FOLDER_MIME_TYPE: str = "application/vnd.google-apps.folder"
-PATH_SEPARATOR: str = "/"
-
-
 class GoogleDriveService(ApplicationComponentBase):
     """Service for interacting with Google Drive."""
 
     model_config = ConfigDict(frozen=True)
 
+    _FOLDER_MIME_TYPE: ClassVar[str] = "application/vnd.google-apps.folder"
+    _PATH_SEPARATOR: ClassVar[str] = "/"
+
     settings: Settings
     _google_drive_client: Resource
     _folder_cache: TTLCache = PrivateAttr(default_factory=lambda: TTLCache(maxsize=256, ttl=60))
@@ -53,9 +52,9 @@ class GoogleDriveService(ApplicationComponentBase):
 
         current_id = self.settings.google_drive_root_id
         try:
-            for part in folder_path.split(PATH_SEPARATOR):
+            for part in folder_path.split(self._PATH_SEPARATOR):
                 results = self._google_drive_client.files().list(
-                    q=f"name='{part.replace("\\", "\\\\").replace("'", "\\'")}' and mimeType='{FOLDER_MIME_TYPE}' and '{current_id}' in parents",
+                    q=f"name='{part.replace("\\", "\\\\").replace("'", "\\'")}' and mimeType='{self._FOLDER_MIME_TYPE}' and '{current_id}' in parents",
                     fields="files(id,name)",
                     supportsAllDrives=True,
                     includeItemsFromAllDrives=True
@@ -87,10 +86,10 @@ class GoogleDriveService(ApplicationComponentBase):
             if not recursive:
                 return results
             return tuple(reduce(chain,
-                                (self._list_directory(f"{folder_path}{PATH_SEPARATOR}{result.name}", result.id, True)
+                                (self._list_directory(f"{folder_path}{self._PATH_SEPARATOR}{result.name}", result.id, True)
                                  for result
                                  in results
-                                 if result.mime_type == FOLDER_MIME_TYPE),
+                                 if result.mime_type == self._FOLDER_MIME_TYPE),
                                 results))
         except HttpError as e:
             logger.error("Error listing folder by path, {}: {}", folder_path, e)
@@ -108,7 +107,7 @@ class GoogleDriveService(ApplicationComponentBase):
     def get_metadata(self: Self, item_path: str) -> Optional[GoogleDriveMetadata]:
         """Get metadata for a specific file/folder by path."""
 
-        match item_path.rsplit(PATH_SEPARATOR, 1):
+        match item_path.rsplit(self._PATH_SEPARATOR, 1):
             case [item_name]:
                 folder_path = ""
                 folder_id = self.settings.google_drive_root_id

src/ctp_slack_bot/services/language_model_service.py

@@ -16,11 +16,7 @@ class LanguageModelService(ApplicationComponentBase):
     model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
 
     settings: Settings
-    _open_ai_client: AsyncOpenAI
-
-    def model_post_init(self: Self, context: Any, /) -> None:
-        super().model_post_init(context)
-        self._open_ai_client = AsyncOpenAI(api_key=self.settings.openai_api_key.get_secret_value())
+    open_ai_client: AsyncOpenAI
 
     async def answer_question(self, asker: str, question: str, context: Collection[Chunk]) -> str: # TODO: generify into just another agent. 
         """Generate a response using OpenAI’s API with retrieved context.
@@ -46,7 +42,7 @@ class LanguageModelService(ApplicationComponentBase):
                 Context from class materials and transcripts:
                 {'\n\n'.join(chunk.text for chunk in context)}""")}
         ]
-        response = await self._open_ai_client.chat.completions.create(
+        response = await self.open_ai_client.chat.completions.create(
             model=self.settings.chat_model,
             messages=messages,
             max_tokens=self.settings.max_tokens,

src/ctp_slack_bot/services/slack_service.py

@@ -7,7 +7,7 @@ from re import compile as compile_re, Pattern
 from slack_sdk.errors import SlackApiError
 from slack_bolt.async_app import AsyncApp
 from slack_sdk.web.async_slack_response import AsyncSlackResponse
-from typing import Any, Mapping, MutableMapping, Optional, Self, Set
+from typing import Any, ClassVar, Mapping, MutableMapping, Optional, Self, Set
 
 from ctp_slack_bot.core import HealthReportingApplicationComponentBase
 from ctp_slack_bot.enums import EventType
@@ -22,8 +22,8 @@ class SlackService(HealthReportingApplicationComponentBase):
 
     model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
 
-    SLACK_USER_ID_PATTERN: Pattern = compile_re(r"U\d+")
-    SLACK_USER_MENTION_PATTERN: Pattern = compile_re(r"<@(U[A-Z0-9]+)>")
+    _SLACK_USER_ID_PATTERN: ClassVar[Pattern] = compile_re(r"U\d+")
+    _SLACK_USER_MENTION_PATTERN: ClassVar[Pattern] = compile_re(r"<@(U[A-Z0-9]+)>")
 
     event_brokerage_service: EventBrokerageService
     http_client: AsyncClient
@@ -84,7 +84,7 @@ class SlackService(HealthReportingApplicationComponentBase):
         await self._process_message(body)
 
     async def _look_up_name(self: Self, id: str) -> Optional[str]:
-        if self.SLACK_USER_ID_PATTERN.fullmatch(id):
+        if self._SLACK_USER_ID_PATTERN.fullmatch(id):
             match await self.slack_bolt_app.client.users_info(id):
                 case AsyncSlackResponse(data={"ok": True, "user": user}):
                     match user:
@@ -104,7 +104,7 @@ class SlackService(HealthReportingApplicationComponentBase):
         await self.event_brokerage_service.publish(EventType.INCOMING_SLACK_MESSAGE, slack_message)
 
     async def _resolve_user_mentions(self: Self, text: str) -> str:
-        matches = tuple(self.SLACK_USER_MENTION_PATTERN.finditer(text))
+        matches = tuple(self._SLACK_USER_MENTION_PATTERN.finditer(text))
         unique_ids = frozenset(match.group(1) for match in matches)
         await self._ensure_ids_in_id_name_map(unique_ids)
         parts = []