Spaces:
Runtime error
Runtime error
File size: 1,864 Bytes
005a292 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
from abc import ABC, abstractmethod
from datetime import datetime
from pydantic import BaseModel, Field, validator
from typing import Dict, List, Optional, Union, Any, ClassVar
import hashlib
import json
class Metadata(BaseModel):
"""A class representing metadata about content."""
id: str # The content’s identity consistent across modifications
modification_time: datetime # The content’s modification for detection of alterations
hash: str # The content’s hash for detection of alterations
class Content(BaseModel):
"""A class representing ingested content."""
metadata: Metadata
class Ingestible(ABC, BaseModel):
"""An abstract base class for ingestible content."""
metadata: Metadata
@property
@abstractmethod
def content(self) -> Content:
"""
Return content ready for vectorization.
This could be:
- A single string
- A list of strings (pre-chunked)
- A more complex structure that can be recursively processed
"""
pass
def get_chunks(self) -> List[str]:
"""
Split content into chunks suitable for vectorization.
Override this in subclasses for specialized chunking logic.
"""
content = self.content
if isinstance(content, str):
# Simple chunking by character count
return [content[i:i+self.chunk_size]
for i in range(0, len(content), self.chunk_size)]
elif isinstance(content, list):
# Content is already chunked
return content
else:
raise ValueError(f"Unsupported content type: {type(content)}")
@property
def key(self) -> str:
"""Convenience accessor for the metadata key."""
return self.metadata.key
|