Python - Update types with new models API
@@ -72,25 +72,15 @@ class Model:
pass
class BPE(Model):
- """ BytePairEncoding model class """
+ """BytePairEncoding model class
- @staticmethod
- def from_files(
- vocab: str,
- merges: str,
- cache_capacity: Optional[int],
- dropout: Optional[float],
- unk_token: Optional[str],
- continuing_subword_prefix: Optional[str],
- end_of_word_suffix: Optional[str],
- ) -> Model:
- """ Instantiate a BPE Model from the given vocab and merges files.
+ Instantiate a BPE Model from the given vocab and merges files.
Args:
- vocab: string:
+ vocab: ('`optional`) string:
Path to a vocabulary JSON file.
- merges: string:
+ merges: (`optional`) string:
Path to a merge file.
cache_capacity: (`optional`) int:
@@ -110,23 +100,27 @@ class BPE(Model):
end_of_word_suffix: (`optional`) str:
The suffix to attach to subword units that represent an end of word.
"""
- pass
+
@staticmethod
- def empty() -> Model:
- """ Instantiate an empty BPE Model. """
+ def __init__(
+ self,
+ vocab: Optional[str],
+ merges: Optional[str],
+ cache_capacity: Optional[int],
+ dropout: Optional[float],
+ unk_token: Optional[str],
+ continuing_subword_prefix: Optional[str],
+ end_of_word_suffix: Optional[str],
+ ):
pass
class WordPiece(Model):
- """ WordPiece model class """
+ """ WordPiece model class
- @staticmethod
- def from_files(
- vocab: str, unk_token: Optional[str], max_input_chars_per_word: Optional[int]
- ) -> Model:
- """ Instantiate a WordPiece Model from the given vocab file.
+ Instantiate a WordPiece Model from the given vocab file.
Args:
- vocab: string:
+ vocab: (`optional`) string:
Path to a vocabulary file.
unk_token: (`optional`) str:
@@ -135,30 +129,28 @@ class WordPiece(Model):
max_input_chars_per_word: (`optional`) int:
The maximum number of characters to authorize in a single word.
"""
- pass
- @staticmethod
- def empty() -> Model:
- """ Instantiate an empty WordPiece Model. """
+
+ def __init__(
+ self,
+ vocab: Optional[str],
+ unk_token: Optional[str],
+ max_input_chars_per_word: Optional[int],
+ ):
pass
class WordLevel(Model):
"""
Most simple tokenizer model based on mapping token from a vocab file to their corresponding id.
- """
- @staticmethod
- def from_files(vocab: str, unk_token: str) -> Model:
- """ Instantiate a WordLevel Model from the given vocab file.
+ Instantiate a WordLevel Model from the given vocab file.
Args:
- vocab: string:
+ vocab: (`optional`) string:
Path to a vocabulary file.
unk_token: str:
The unknown token to be used by the model.
"""
- pass
- @staticmethod
- def empty() -> Model:
- """ Instantiate an empty WordLevel Model. """
+
+ def __init__(self, vocab: Optional[str], unk_token: Optional[str]):
pass
