multi language & source file support draft

src/know_lang_bot/config.py

@@ -1,7 +1,61 @@
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, List
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from pydantic import Field
 from pathlib import Path
+import fnmatch
+
+class PathPatterns(BaseSettings):
+    include: List[str] = Field(
+        default=["**/*"],
+        description="Glob patterns for paths to include"
+    )
+    exclude: List[str] = Field(
+        default=["**/venv/**", "**/.git/**", "**/__pycache__/**"],
+        description="Glob patterns for paths to exclude"
+    )
+
+    def should_process_path(self, path: str) -> bool:
+        """Check if a path should be processed based on include/exclude patterns"""
+        path_str = str(path)
+        
+        # First check exclusions
+        for pattern in self.exclude:
+            if fnmatch.fnmatch(path_str, pattern):
+                return False
+        
+        # Then check inclusions
+        for pattern in self.include:
+            if fnmatch.fnmatch(path_str, pattern):
+                return True
+        
+        return False
+
+class LanguageConfig(BaseSettings):
+    enabled: bool = True
+    file_extensions: List[str]
+    tree_sitter_language: str
+    chunk_types: List[str]
+    max_file_size: int = Field(
+        default=1_000_000,  # 1MB
+        description="Maximum file size to process in bytes"
+    )
+
+    def supports_extension(self, ext: str) -> bool:
+        """Check if the language supports a given file extension"""
+        return ext in self.file_extensions
+
+class ParserConfig(BaseSettings):
+    languages: Dict[str, LanguageConfig] = Field(
+        default={
+            "python": LanguageConfig(
+                file_extensions=[".py"],
+                tree_sitter_language="python",
+                chunk_types=["class_definition", "function_definition"]
+            )
+        }
+    )
+    path_patterns: PathPatterns = Field(default_factory=PathPatterns)
+
 
 class LLMConfig(BaseSettings):
     model_name: str = Field(
@@ -56,7 +110,4 @@ class AppConfig(BaseSettings):
     
     llm: LLMConfig = Field(default_factory=LLMConfig)
     db: DBConfig = Field(default_factory=DBConfig)
-    chunk_max_size: int = Field(
-        default=1500,
-        description="Maximum size of code chunks before splitting"
-    )
+    parser: ParserConfig = Field(default_factory=ParserConfig)

src/know_lang_bot/core/types.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+from typing import Optional
+from enum import Enum
+
+class ChunkType(str, Enum):
+    CLASS = "class"
+    FUNCTION = "function"
+    OTHER = "other"
+
+class CodeChunk(BaseModel):
+    """Represents a chunk of code with its metadata"""
+    type: ChunkType
+    content: str
+    start_line: int
+    end_line: int
+    file_path: str
+    name: Optional[str] = None
+    parent_name: Optional[str] = None  # For nested classes/functions
+    docstring: Optional[str] = None

src/know_lang_bot/parser/base/parser.py

@@ -0,0 +1,26 @@
+from abc import ABC, abstractmethod
+from typing import List
+from pathlib import Path
+from know_lang_bot.core.types import CodeChunk
+from know_lang_bot.config import AppConfig, LanguageConfig
+from tree_sitter import Language, Parser
+
+class LanguageParser(ABC):
+    """Abstract base class for language-specific parsers"""
+    
+    @abstractmethod
+    def __init__(self, config: AppConfig):
+        self.config : AppConfig = config
+        self.language : Language = None
+        self.parser : Parser = None
+        self.language_config : LanguageConfig = None
+    
+    @abstractmethod
+    def setup(self) -> None:
+        """Set up the parser (e.g., initialize tree-sitter)"""
+        pass
+
+    @abstractmethod
+    def parse_file(self, file_path: Path) -> List[CodeChunk]:
+        """Parse a single file and return code chunks"""
+        pass

src/know_lang_bot/parser/base/provider.py

@@ -0,0 +1,17 @@
+from abc import ABC, abstractmethod
+from typing import Generator
+from pathlib import Path
+from know_lang_bot.config import AppConfig
+
+class CodeProvider(ABC):
+    """Abstract base class for code source providers"""
+    
+    @abstractmethod
+    def __init__(self, source_path: Path, config: AppConfig):
+        self.source_path = source_path
+        self.config = config
+    
+    @abstractmethod
+    def get_files(self) -> Generator[Path, None, None]:
+        """Yield paths to code files that should be processed"""
+        pass

src/know_lang_bot/parser/factory.py

@@ -0,0 +1,39 @@
+from typing import Dict, Type, Optional
+from pathlib import Path
+
+from know_lang_bot.parser.base.parser import LanguageParser
+from know_lang_bot.parser.languages.python.parser import PythonParser
+from know_lang_bot.config import AppConfig
+
+class CodeParserFactory():
+    """Concrete implementation of parser factory"""
+    
+    def __init__(self, config: AppConfig):
+        self.config = config
+        self._parsers: Dict[str, LanguageParser] = {}
+        self._parser_classes = self._register_parsers()
+    
+    def _register_parsers(self) -> Dict[str, Type[LanguageParser]]:
+        """Register available parser implementations"""
+        return {
+            "python": PythonParser,
+            # Add more languages here
+        }
+    
+    def get_parser(self, file_path: Path) -> Optional[LanguageParser]:
+        """Get appropriate parser for a file"""
+        extension = file_path.suffix
+        
+        # Find parser class for this extension
+        for lang, parser_class in self._parser_classes.items():
+            if not self.config.parser.languages[lang].enabled:
+                continue
+                
+            parser = parser_class(self.config)
+            if extension in parser.supported_extensions():
+                if lang not in self._parsers:
+                    parser.setup()
+                    self._parsers[lang] = parser
+                return self._parsers[lang]
+        
+        return None

src/know_lang_bot/parser/languages/python/parser.py

@@ -0,0 +1,146 @@
+from typing import List, Optional
+from pathlib import Path
+import tree_sitter_python
+from tree_sitter import Language, Parser, Node
+
+from know_lang_bot.parser.base.parser import LanguageParser
+from know_lang_bot.core.types import CodeChunk, ChunkType
+from know_lang_bot.utils.fancy_log import FancyLogger
+
+LOG = FancyLogger(__name__)
+
+class PythonParser(LanguageParser):
+    """Python-specific implementation of LanguageParser"""
+    
+    def setup(self) -> None:
+        """Initialize tree-sitter with Python language support"""
+        self.language = Language(tree_sitter_python.language())
+        self.parser = Parser(self.language)
+        self.language_config = self.config.parser.languages["python"]
+    
+    def _get_preceding_docstring(self, node: Node, source_code: bytes) -> Optional[str]:
+        """Extract docstring from comments"""
+        docstring_parts = []
+        current_node = node.prev_sibling
+
+        while current_node:
+            if current_node.type == "comment":
+                comment = source_code[current_node.start_byte:current_node.end_byte].decode('utf-8')
+                docstring_parts.insert(0, comment)
+            elif current_node.type == "expression_statement":
+                string_node = current_node.children[0] if current_node.children else None
+                if string_node and string_node.type in ("string", "string_literal"):
+                    docstring = source_code[string_node.start_byte:string_node.end_byte].decode('utf-8')
+                    docstring_parts.insert(0, docstring)
+                break
+            elif current_node.type not in ("empty_statement", "newline"):
+                break
+            current_node = current_node.prev_sibling
+        
+        return '\n'.join(docstring_parts) if docstring_parts else None
+
+    def _has_syntax_error(self, node: Node) -> bool:
+        """Check if the node or its children contain syntax errors"""
+        if node.type == "ERROR":
+            return True
+        if node.has_error:
+            return True
+        return any(self._has_syntax_error(child) for child in node.children)
+
+    def _process_class(self, node: Node, source_code: bytes, file_path: Path) -> CodeChunk:
+        """Process a class node and return a CodeChunk"""
+        name = next(
+            (child.text.decode('utf-8') 
+             for child in node.children 
+             if child.type == "identifier"),
+            None
+        )
+        
+        if not name:
+            raise ValueError(f"Could not find class name in node: {node.text}")
+        
+        return CodeChunk(
+            type=ChunkType.CLASS,
+            name=name,
+            content=source_code[node.start_byte:node.end_byte].decode('utf-8'),
+            start_line=node.start_point[0],
+            end_line=node.end_point[0],
+            file_path=str(file_path),
+            docstring=self._get_preceding_docstring(node, source_code)
+        )
+
+    def _process_function(self, node: Node, source_code: bytes, file_path: Path) -> CodeChunk:
+        """Process a function node and return a CodeChunk"""
+        name = next(
+            (child.text.decode('utf-8') 
+             for child in node.children 
+             if child.type == "identifier"),
+            None
+        )
+
+        if not name:
+            raise ValueError(f"Could not find function name in node: {node.text}")
+        
+        # Determine if this is a method within a class
+        parent_node = node.parent
+        parent_name = None
+        if parent_node and parent_node.type == "class_definition":
+            parent_name = next(
+                (child.text.decode('utf-8') 
+                 for child in parent_node.children 
+                 if child.type == "identifier"),
+                None
+            )
+        
+        return CodeChunk(
+            type=ChunkType.FUNCTION,
+            name=name,
+            content=source_code[node.start_byte:node.end_byte].decode('utf-8'),
+            start_line=node.start_point[0],
+            end_line=node.end_point[0],
+            file_path=str(file_path),
+            parent_name=parent_name,
+            docstring=self._get_preceding_docstring(node, source_code)
+        )
+
+    def parse_file(self, file_path: Path) -> List[CodeChunk]:
+        """Parse a single Python file and return list of code chunks"""
+        if not self.language_config.supports_extension(file_path.suffix):
+            return []
+
+        # Check file size limit
+        if file_path.stat().st_size > self.language_config.max_file_size:
+            LOG.warning(f"Skipping file {file_path}: exceeds size limit of {self.language_config.max_file_size} bytes")
+            return []
+
+        try:
+            with open(file_path, 'rb') as f:
+                source_code = f.read()
+            
+            if not self.parser:
+                raise RuntimeError("Parser not initialized. Call setup() first.")
+                
+            tree = self.parser.parse(source_code)
+
+            # Check for overall syntax validity
+            if self._has_syntax_error(tree.root_node):
+                LOG.warning(f"Syntax errors found in {file_path}")
+                return []
+
+            chunks: List[CodeChunk] = []
+            
+            # Process the syntax tree
+            for node in tree.root_node.children:
+                if node.type == "class_definition":
+                    chunks.append(self._process_class(node, source_code, file_path))
+                elif node.type == "function_definition":
+                    chunks.append(self._process_function(node, source_code, file_path))
+                else:
+                    # Skip other node types for now
+                    pass
+            
+            return chunks
+
+        except Exception as e:
+            LOG.error(f"Error parsing file {file_path}: {str(e)}")
+            return []