Upload folder using huggingface_hub

modules/SD15/SDToken.py

@@ -236,32 +236,49 @@ class SDTokenizer:
             - `pad_to_max_length` (bool, optional): Whether to pad to the maximum length. Defaults to True.
             - `min_length` (int, optional): The minimum length of the input. Defaults to None.
         """
+        # Ensure tokenizer path exists
         if tokenizer_path is None:
-            tokenizer_path = "./_internal/sd1_tokenizer/"
-        self.tokenizer = tokenizer_class.from_pretrained(tokenizer_path)
+            tokenizer_path = os.path.join(os.path.dirname(__file__), "../_internal/sd1_tokenizer")
+        
+        # Verify path exists
+        if not os.path.exists(tokenizer_path):
+            raise ValueError(f"Tokenizer path does not exist: {tokenizer_path}")
+
+        try:
+            self.tokenizer = tokenizer_class.from_pretrained(tokenizer_path)
+        except Exception as e:
+            raise RuntimeError(f"Failed to load tokenizer from {tokenizer_path}: {str(e)}")
+
         self.max_length = max_length
         self.min_length = min_length
-
+        
+        # Get tokens from empty string tokenization
         empty = self.tokenizer("")["input_ids"]
+        
         if has_start_token:
             self.tokens_start = 1
             self.start_token = empty[0]
             self.end_token = empty[1]
         else:
             self.tokens_start = 0
-            self.start_token = None
+            self.start_token = None 
             self.end_token = empty[0]
+            
         self.pad_with_end = pad_with_end
         self.pad_to_max_length = pad_to_max_length
 
+        # Create vocab lookup
         vocab = self.tokenizer.get_vocab()
         self.inv_vocab = {v: k for k, v in vocab.items()}
+        
+        # Set embedding properties
         self.embedding_directory = embedding_directory
         self.max_word_length = 8
         self.embedding_identifier = "embedding:"
         self.embedding_size = embedding_size
         self.embedding_key = embedding_key
 
+
     def _try_get_embedding(self, embedding_name: str) -> tuple:
         """#### Try to get an embedding.
 
@@ -410,7 +427,7 @@ class SDTokenizer:
 class SD1Tokenizer:
     """#### Class representing the SD1Tokenizer."""
 
-    def __init__(self, embedding_directory: str = None, clip_name: str = "l", tokenizer: type = SDTokenizer):
+    def __init__(self, embedding_directory: str = None, clip_name: str = "l", tokenizer: type = SDTokenizer, tokenizer_data: dict = None):
         """#### Initialize the SD1Tokenizer.
 
         #### Args:
@@ -419,8 +436,14 @@ class SD1Tokenizer:
             - `tokenizer` (type, optional): The tokenizer class. Defaults to SDTokenizer.
         """
         self.clip_name = clip_name
-        self.clip = "clip_{}".format(self.clip_name)
-        setattr(self, self.clip, tokenizer(embedding_directory=embedding_directory))
+        self.clip = f"clip_{self.clip_name}"
+        
+        # Initialize tokenizer with proper arguments
+        kwargs = {"embedding_directory": embedding_directory}
+        if tokenizer_data:
+            kwargs.update(tokenizer_data)
+            
+        setattr(self, self.clip, tokenizer(**kwargs))
 
     def tokenize_with_weights(self, text: str, return_word_ids: bool = False) -> dict:
         """#### Tokenize text with weights.