Convert PyTorch AutoTokenizer to TensorFlow TextVectorization
I have a PyTorch encoder loaded on my PC with transformers.
I saved it in JSON with tokenizer.save_pretrained(...) and now I need to load it on another PC with TensorFlow TextVectorization as I don't have access to the transformers library.
How can I convert ? I read about the tf.keras.preprocessing.text.tokenizer_from_json but it does not work.
In PyTorch JSON I have :
{
  &quot;version&quot;: &quot;1.0&quot;,
  &quot;truncation&quot;: null,
  &quot;padding&quot;: null,
  &quot;added_tokens&quot;: [...],
  &quot;normalizer&quot;: {...},
  &quot;pre_tokenizer&quot;: {...},
  &quot;post_processor&quot;: {...},
  &quot;decoder&quot;: {...},
  &quot;model&quot;: {...}
}

and TensorFlow is expecting, with TextVectorizer :
    def __init__(
        self,
        max_tokens=None,
        standardize=&quot;lower_and_strip_punctuation&quot;,
        split=&quot;whitespace&quot;,
        ngrams=None,
        output_mode=&quot;int&quot;,
        output_sequence_length=None,
        pad_to_max_tokens=False,
        vocabulary=None,
        idf_weights=None,
        sparse=False,
        ragged=False,
        **kwargs,
    ):

or with the tokenizer_from_json these kind of fields :
    config = tokenizer_config.get(&quot;config&quot;)

    word_counts = json.loads(config.pop(&quot;word_counts&quot;))
    word_docs = json.loads(config.pop(&quot;word_docs&quot;))
    index_docs = json.loads(config.pop(&quot;index_docs&quot;))
    # Integer indexing gets converted to strings with json.dumps()
    index_docs = {int(k): v for k, v in index_docs.items()}
    index_word = json.loads(config.pop(&quot;index_word&quot;))
    index_word = {int(k): v for k, v in index_word.items()}
    word_index = json.loads(config.pop(&quot;word_index&quot;))

    tokenizer = Tokenizer(**config)


Simply &quot;tf.keras.preprocessing.text.tokenizer_from_json.()&quot; but you may need to correct format in JSON.
Sample: The sample they using &quot; I love cats &quot; -&gt; &quot; Sticky &quot;
import tensorflow as tf

text = &quot;I love cats&quot;
tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=10000, oov_token='&lt;oov&gt;')
tokenizer.fit_on_texts([text])

# input
vocab = [ &quot;a&quot;, &quot;b&quot;, &quot;c&quot;, &quot;d&quot;, &quot;e&quot;, &quot;f&quot;, &quot;g&quot;, &quot;h&quot;, &quot;I&quot;, &quot;j&quot;, &quot;k&quot;, &quot;l&quot;, &quot;m&quot;, &quot;n&quot;, &quot;o&quot;, &quot;p&quot;, &quot;q&quot;, &quot;r&quot;, &quot;s&quot;, &quot;t&quot;, &quot;u&quot;, &quot;v&quot;, &quot;w&quot;, &quot;x&quot;, &quot;y&quot;, &quot;z&quot;, &quot;_&quot; ]
data = tf.constant([[&quot;_&quot;, &quot;_&quot;, &quot;_&quot;, &quot;I&quot;], [&quot;l&quot;, &quot;o&quot;, &quot;v&quot;, &quot;e&quot;], [&quot;c&quot;, &quot;a&quot;, &quot;t&quot;, &quot;s&quot;]])

layer = tf.keras.layers.StringLookup(vocabulary=vocab)
sequences_mapping_string = layer(data)
sequences_mapping_string = tf.constant( sequences_mapping_string, shape=(1,12) )
print( 'result: ' + str( sequences_mapping_string ) )

print( 'tokenizer.to_json(): ' + str( tokenizer.to_json() ) )

new_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(tokenizer.to_json())
print( 'new_tokenizer.to_json(): ' + str( new_tokenizer.to_json() ) )

Output:
result: tf.Tensor([[27 27 27  9 12 15 22  5  3  1 20 19]], shape=(1, 12), dtype=int64)
tokenizer.to_json(): {&quot;class_name&quot;: &quot;Tokenizer&quot;, &quot;config&quot;: {&quot;num_words&quot;: 10000, &quot;filters&quot;: &quot;!\&quot;#$%&amp;()*+,-./:;&lt;=&gt;?@[\\]^_`{|}~\t\n&quot;, &quot;lower&quot;: true, &quot;split&quot;: &quot; &quot;, &quot;char_level&quot;: false, &quot;oov_token&quot;: &quot;&lt;oov&gt;&quot;, &quot;document_count&quot;: 1, &quot;word_counts&quot;: &quot;{\&quot;i\&quot;: 1, \&quot;love\&quot;: 1, \&quot;cats\&quot;: 1}&quot;, &quot;word_docs&quot;: &quot;{\&quot;cats\&quot;: 1, \&quot;love\&quot;: 1, \&quot;i\&quot;: 1}&quot;, &quot;index_docs&quot;: &quot;{\&quot;4\&quot;: 1, \&quot;3\&quot;: 1, \&quot;2\&quot;: 1}&quot;, &quot;index_word&quot;: &quot;{\&quot;1\&quot;: \&quot;&lt;oov&gt;\&quot;, \&quot;2\&quot;: \&quot;i\&quot;, \&quot;3\&quot;: \&quot;love\&quot;, \&quot;4\&quot;: \&quot;cats\&quot;}&quot;, &quot;word_index&quot;: &quot;{\&quot;&lt;oov&gt;\&quot;: 1, \&quot;i\&quot;: 2, \&quot;love\&quot;: 3, \&quot;cats\&quot;: 4}&quot;}}
new_tokenizer.to_json(): {&quot;class_name&quot;: &quot;Tokenizer&quot;, &quot;config&quot;: {&quot;num_words&quot;: 10000, &quot;filters&quot;: &quot;!\&quot;#$%&amp;()*+,-./:;&lt;=&gt;?@[\\]^_`{|}~\t\n&quot;, &quot;lower&quot;: true, &quot;split&quot;: &quot; &quot;, &quot;char_level&quot;: false, &quot;oov_token&quot;: &quot;&lt;oov&gt;&quot;, &quot;document_count&quot;: 1, &quot;word_counts&quot;: &quot;{\&quot;i\&quot;: 1, \&quot;love\&quot;: 1, \&quot;cats\&quot;: 1}&quot;, &quot;word_docs&quot;: &quot;{\&quot;cats\&quot;: 1, \&quot;love\&quot;: 1, \&quot;i\&quot;: 1}&quot;, &quot;index_docs&quot;: &quot;{\&quot;4\&quot;: 1, \&quot;3\&quot;: 1, \&quot;2\&quot;: 1}&quot;, &quot;index_word&quot;: &quot;{\&quot;1\&quot;: \&quot;&lt;oov&gt;\&quot;, \&quot;2\&quot;: \&quot;i\&quot;, \&quot;3\&quot;: \&quot;love\&quot;, \&quot;4\&quot;: \&quot;cats\&quot;}&quot;, &quot;word_index&quot;: &quot;{\&quot;&lt;oov&gt;\&quot;: 1, \&quot;i\&quot;: 2, \&quot;love\&quot;: 3, \&quot;cats\&quot;: 4}&quot;}}

