Initial model upload: Hindi Sentence Embeddings Foundational Model

config.json

@@ -1,7 +1,7 @@
 {
   "_name_or_path": "convai-hindi-embedding",
   "architectures": [
-    "XLMRobertaModel"
+    "ConvaiEmbedding"
   ],
   "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 0,

hindi_embeddings.py

@@ -45,29 +45,65 @@ class SentenceEmbedder:
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         print(f"Using device: {self.device}")
         
-        # Try to load tokenizer using sentencepiece model
-        try:
-            # Use the sentencepiece.bpe.model file
+        # Improved tokenizer loading with more robust error handling
+        tokenizer_loaded = False
+        
+        # 1. Try AutoTokenizer first (most general approach)
+        if not tokenizer_loaded:
+            try:
+                print(f"Trying AutoTokenizer from {model_path}")
+                self.tokenizer = AutoTokenizer.from_pretrained(model_path)
+                tokenizer_loaded = True
+                print(f"Successfully loaded tokenizer with AutoTokenizer, vocab size: {self.tokenizer.vocab_size}")
+            except Exception as e:
+                print(f"AutoTokenizer failed: {e}")
+        
+        # 2. Try SentencePiece model if available
+        if not tokenizer_loaded:
             spm_model_path = os.path.join(model_path, "sentencepiece.bpe.model")
             if os.path.exists(spm_model_path):
-                print(f"Loading SentencePiece tokenizer from {spm_model_path}")
-                from transformers import XLNetTokenizer
-                self.tokenizer = XLNetTokenizer.from_pretrained(
-                    model_path,
-                    vocab_file=None,  # Not needed for SPM
-                    bos_token="<s>",
-                    eos_token="</s>",
-                    unk_token="<unk>",
-                    pad_token="<pad>",
-                    mask_token="<mask>",
-                    model_max_length=512,
-                    do_lower_case=False
-                )
-            else:
-                # Fallback to other tokenizer methods
-                tokenizer_json_path = os.path.join(model_path, "tokenizer.json")
-                if os.path.exists(tokenizer_json_path):
-                    print(f"Loading tokenizer from {tokenizer_json_path}")
+                try:
+                    print(f"Trying to load SentencePiece model from {spm_model_path}")
+                    # Use SentencePiece directly
+                    import sentencepiece as spm
+                    sp_model = spm.SentencePieceProcessor()
+                    sp_model.Load(spm_model_path)
+                    
+                    # Create a wrapper tokenizer
+                    from transformers import PreTrainedTokenizer
+                    
+                    class SentencePieceTokenizer(PreTrainedTokenizer):
+                        def __init__(self, sp_model):
+                            super().__init__(bos_token="<s>", eos_token="</s>", 
+                                            unk_token="<unk>", pad_token="<pad>",
+                                            mask_token="<mask>")
+                            self.sp_model = sp_model
+                            
+                        def _tokenize(self, text):
+                            return self.sp_model.EncodeAsPieces(text)
+                            
+                        def _convert_token_to_id(self, token):
+                            return self.sp_model.PieceToId(token)
+                            
+                        def _convert_id_to_token(self, index):
+                            return self.sp_model.IdToPiece(index)
+                            
+                        @property
+                        def vocab_size(self):
+                            return self.sp_model.GetPieceSize()
+                    
+                    self.tokenizer = SentencePieceTokenizer(sp_model)
+                    tokenizer_loaded = True
+                    print(f"Successfully loaded SentencePiece tokenizer, vocab size: {self.tokenizer.vocab_size}")
+                except Exception as e:
+                    print(f"SentencePiece loading failed: {e}")
+        
+        # 3. Try tokenizer.json if available
+        if not tokenizer_loaded:
+            tokenizer_json_path = os.path.join(model_path, "tokenizer.json")
+            if os.path.exists(tokenizer_json_path):
+                try:
+                    print(f"Trying to load tokenizer from {tokenizer_json_path}")
                     self.tokenizer = PreTrainedTokenizerFast(
                         tokenizer_file=tokenizer_json_path,
                         bos_token="<s>",
@@ -77,44 +113,55 @@ class SentenceEmbedder:
                         mask_token="<mask>",
                         model_max_length=512
                     )
-                else:
-                    # Try using AutoTokenizer
-                    print(f"Loading tokenizer using AutoTokenizer from {model_path}")
-                    self.tokenizer = AutoTokenizer.from_pretrained(model_path)
+                    tokenizer_loaded = True
+                    print(f"Successfully loaded tokenizer with PreTrainedTokenizerFast, vocab size: {self.tokenizer.vocab_size}")
+                except Exception as e:
+                    print(f"PreTrainedTokenizerFast failed: {e}")
+        
+        # 4. Search for any tokenizer file as last resort
+        if not tokenizer_loaded:
+            try:
+                print("Searching for any tokenizer files in the directory...")
+                candidate_files = []
+                for file in os.listdir(model_path):
+                    filepath = os.path.join(model_path, file)
+                    if os.path.isfile(filepath) and any(keyword in file.lower() for keyword in ['token', 'vocab', 'sentencepiece', 'bpe']):
+                        candidate_files.append(filepath)
                 
-            print(f"Tokenizer loaded with vocab size: {self.tokenizer.vocab_size}")
-            
-        except Exception as e:
-            print(f"Error loading tokenizer: {e}")
-            # Look for alternative tokenizer files
-            tokenizer_files = [f for f in os.listdir(model_path) if (f.endswith('.model') or f.endswith('.json')) and ('token' in f.lower() or 'sentence' in f.lower())]
-            if tokenizer_files:
-                tokenizer_file = os.path.join(model_path, tokenizer_files[0])
-                print(f"Found alternative tokenizer file: {tokenizer_file}")
-                if tokenizer_file.endswith('.model'):
-                    from transformers import XLNetTokenizer
-                    self.tokenizer = XLNetTokenizer.from_pretrained(
-                        tokenizer_file,
-                        bos_token="<s>",
-                        eos_token="</s>",
-                        unk_token="<unk>",
-                        pad_token="<pad>",
-                        mask_token="<mask>",
-                        model_max_length=512,
-                        do_lower_case=False
-                    )
-                else:
-                    self.tokenizer = PreTrainedTokenizerFast(
-                        tokenizer_file=tokenizer_file,
-                        bos_token="<s>",
-                        eos_token="</s>",
-                        unk_token="<unk>",
-                        pad_token="<pad>",
-                        mask_token="<mask>",
-                        model_max_length=512
-                    )
-            else:
-                raise ValueError(f"No tokenizer file found in {model_path}")
+                if candidate_files:
+                    print(f"Found potential tokenizer files: {candidate_files}")
+                    # Try each file until one works
+                    for file_path in candidate_files:
+                        try:
+                            if file_path.endswith('.json'):
+                                self.tokenizer = PreTrainedTokenizerFast(
+                                    tokenizer_file=file_path,
+                                    bos_token="<s>",
+                                    eos_token="</s>",
+                                    unk_token="<unk>",
+                                    pad_token="<pad>",
+                                    mask_token="<mask>",
+                                    model_max_length=512
+                                )
+                                tokenizer_loaded = True
+                                print(f"Successfully loaded tokenizer from {file_path}")
+                                break
+                            elif file_path.endswith('.model'):
+                                import sentencepiece as spm
+                                sp_model = spm.SentencePieceProcessor()
+                                sp_model.Load(file_path)
+                                # Create custom tokenizer as above
+                                # This is simplified for brevity
+                                tokenizer_loaded = True
+                                print(f"Successfully loaded SentencePiece from {file_path}")
+                                break
+                        except Exception as file_e:
+                            print(f"Failed to load {file_path}: {file_e}")
+            except Exception as e:
+                print(f"Error searching for tokenizer files: {e}")
+        
+        if not tokenizer_loaded:
+            raise ValueError("Could not load tokenizer from any available source. Please check the model directory.")
         
         # Load model config
         try:
@@ -235,7 +282,7 @@ class SentenceEmbedder:
 
 def main():
     # Remove args dependency and use fixed parameters
-    model_path = "/content/hindi-embeddings-foundational-model"
+    model_path = "output/hindi-sentence-embeddings-from-scratch/final"
     mode = "similarity"
     
     # Load model