prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Add compute_output_shape method to WordPieceTokenizer
  When we run Pretraining Transformer from Scratch guide with PyTorch and JAX backend, it raises
  
  ```
  
  RuntimeError: Exception encountered when calling WordPieceTokenizer.call().
  
  Could not automatically infer the output shape / dtype of 'word_piece_tokenizer_1' (of type WordPieceTokenizer). Either the `WordPieceTokenizer.call()` method is incorrect, or you need to implement the `WordPieceTokenizer.compute_output_spec() / compute_output_shape()` method. Error encountered:
  
  'string'
  
  Arguments received by WordPieceTokenizer.call():
    • args=('<KerasTensor shape=(None,), dtype=string, name=keras_tensor_59>',)
    • kwargs=<class 'inspect._empty'>
  ```
  
  cc: @mattdangerw 
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `keras_nlp/tokenizers/tokenizer.py`
  Content:
  ```
  1 # Copyright 2023 The KerasNLP Authors
  2 #
  3 # Licensed under the Apache License, Version 2.0 (the "License");
  4 # you may not use this file except in compliance with the License.
  5 # You may obtain a copy of the License at
  6 #
  7 #     https://www.apache.org/licenses/LICENSE-2.0
  8 #
  9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 
  15 from typing import List
  16 
  17 from keras_nlp.api_export import keras_nlp_export
  18 from keras_nlp.layers.preprocessing.preprocessing_layer import (
  19     PreprocessingLayer,
  20 )
  21 
  22 
  23 @keras_nlp_export("keras_nlp.tokenizers.Tokenizer")
  24 class Tokenizer(PreprocessingLayer):
  25     """A base class for tokenizer layers.
  26 
  27     Tokenizers in the KerasNLP library should all subclass this layer.
  28     The class provides two core methods `tokenize()` and `detokenize()` for
  29     going from plain text to sequences and back. A tokenizer is a subclass of
  30     `keras.layers.Layer` and can be combined into a `keras.Model`.
  31 
  32     Subclassers should always implement the `tokenize()` method, which will also
  33     be the default when calling the layer directly on inputs.
  34 
  35     Subclassers can optionally implement the `detokenize()` method if the
  36     tokenization is reversible. Otherwise, this can be skipped.
  37 
  38     Subclassers should implement `get_vocabulary()`, `vocabulary_size()`,
  39     `token_to_id()` and `id_to_token()` if applicable. For some simple
  40     "vocab free" tokenizers, such as a whitespace splitter show below, these
  41     methods do not apply and can be skipped.
  42 
  43     Examples:
  44 
  45     ```python
  46     class WhitespaceSplitterTokenizer(keras_nlp.tokenizers.Tokenizer):
  47         def tokenize(self, inputs):
  48             return tf.strings.split(inputs)
  49 
  50         def detokenize(self, inputs):
  51             return tf.strings.reduce_join(inputs, separator=" ", axis=-1)
  52 
  53     tokenizer = WhitespaceSplitterTokenizer()
  54 
  55     # Tokenize some inputs.
  56     tokenizer.tokenize("This is a test")
  57 
  58     # Shorthard for `tokenize()`.
  59     tokenizer("This is a test")
  60 
  61     # Detokenize some outputs.
  62     tokenizer.detokenize(["This", "is", "a", "test"])
  63     ```
  64     """
  65 
  66     def __init__(self, *args, **kwargs):
  67         super().__init__(*args, **kwargs)
  68 
  69     def tokenize(self, inputs, *args, **kwargs):
  70         """Transform input tensors of strings into output tokens.
  71 
  72         Args:
  73             inputs: Input tensor, or dict/list/tuple of input tensors.
  74             *args: Additional positional arguments.
  75             **kwargs: Additional keyword arguments.
  76         """
  77         raise NotImplementedError(
  78             "No implementation of `tokenize()` was found for "
  79             f"{self.__class__.__name__}. All tokenizers should implement "
  80             "`tokenize()`."
  81         )
  82 
  83     def detokenize(self, inputs, *args, **kwargs):
  84         """Transform tokens back into strings.
  85 
  86         Args:
  87             inputs: Input tensor, or dict/list/tuple of input tensors.
  88             *args: Additional positional arguments.
  89             **kwargs: Additional keyword arguments.
  90         """
  91         raise NotImplementedError(
  92             "No implementation of `detokenize()` was found for "
  93             f"{self.__class__.__name__}."
  94         )
  95 
  96     def get_vocabulary(self) -> List[str]:
  97         """Get the tokenizer vocabulary as a list of strings terms."""
  98         raise NotImplementedError(
  99             "No implementation of `get_vocabulary()` was found for "
  100             f"{self.__class__.__name__}."
  101         )
  102 
  103     def vocabulary_size(self) -> int:
  104         """Returns the total size of the token id space."""
  105         raise NotImplementedError(
  106             "No implementation of `vocabulary_size()` was found for "
  107             f"{self.__class__.__name__}."
  108         )
  109 
  110     def id_to_token(self, id: int) -> str:
  111         """Convert an integer id to a string token."""
  112         raise NotImplementedError(
  113             "No implementation of `id_to_token()` was found for "
  114             f"{self.__class__.__name__}."
  115         )
  116 
  117     def token_to_id(self, token: str) -> int:
  118         """Convert an integer id to a string token."""
  119         raise NotImplementedError(
  120             "No implementation of `id_to_token()` was found for "
  121             f"{self.__class__.__name__}."
  122         )
  123 
  124     def call(self, inputs, *args, training=None, **kwargs):
  125         return self.tokenize(inputs, *args, **kwargs)
  126
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/keras_nlp/tokenizers/tokenizer.py b/keras_nlp/tokenizers/tokenizer.py
  --- a/keras_nlp/tokenizers/tokenizer.py
  +++ b/keras_nlp/tokenizers/tokenizer.py
  @@ -123,3 +123,6 @@
   
       def call(self, inputs, *args, training=None, **kwargs):
           return self.tokenize(inputs, *args, **kwargs)
  +
  +    def compute_output_shape(self, inputs_shape):
  +        return tuple(inputs_shape) + (self.sequence_length,)
  
verification_info: {"golden_diff": "diff --git a/keras_nlp/tokenizers/tokenizer.py b/keras_nlp/tokenizers/tokenizer.py\n--- a/keras_nlp/tokenizers/tokenizer.py\n+++ b/keras_nlp/tokenizers/tokenizer.py\n@@ -123,3 +123,6 @@\n \n     def call(self, inputs, *args, training=None, **kwargs):\n         return self.tokenize(inputs, *args, **kwargs)\n+\n+    def compute_output_shape(self, inputs_shape):\n+        return tuple(inputs_shape) + (self.sequence_length,)\n", "issue": "Add compute_output_shape method to WordPieceTokenizer\nWhen we run Pretraining Transformer from Scratch guide with PyTorch and JAX backend, it raises\r\n\r\n```\r\n\r\nRuntimeError: Exception encountered when calling WordPieceTokenizer.call().\r\n\r\nCould not automatically infer the output shape / dtype of 'word_piece_tokenizer_1' (of type WordPieceTokenizer). Either the `WordPieceTokenizer.call()` method is incorrect, or you need to implement the `WordPieceTokenizer.compute_output_spec() / compute_output_shape()` method. Error encountered:\r\n\r\n'string'\r\n\r\nArguments received by WordPieceTokenizer.call():\r\n  \u2022 args=('<KerasTensor shape=(None,), dtype=string, name=keras_tensor_59>',)\r\n  \u2022 kwargs=<class 'inspect._empty'>\r\n```\r\n\r\ncc: @mattdangerw \n", "before_files": [{"content": "# Copyright 2023 The KerasNLP Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import List\n\nfrom keras_nlp.api_export import keras_nlp_export\nfrom keras_nlp.layers.preprocessing.preprocessing_layer import (\n    PreprocessingLayer,\n)\n\n\n@keras_nlp_export(\"keras_nlp.tokenizers.Tokenizer\")\nclass Tokenizer(PreprocessingLayer):\n    \"\"\"A base class for tokenizer layers.\n\n    Tokenizers in the KerasNLP library should all subclass this layer.\n    The class provides two core methods `tokenize()` and `detokenize()` for\n    going from plain text to sequences and back. A tokenizer is a subclass of\n    `keras.layers.Layer` and can be combined into a `keras.Model`.\n\n    Subclassers should always implement the `tokenize()` method, which will also\n    be the default when calling the layer directly on inputs.\n\n    Subclassers can optionally implement the `detokenize()` method if the\n    tokenization is reversible. Otherwise, this can be skipped.\n\n    Subclassers should implement `get_vocabulary()`, `vocabulary_size()`,\n    `token_to_id()` and `id_to_token()` if applicable. For some simple\n    \"vocab free\" tokenizers, such as a whitespace splitter show below, these\n    methods do not apply and can be skipped.\n\n    Examples:\n\n    ```python\n    class WhitespaceSplitterTokenizer(keras_nlp.tokenizers.Tokenizer):\n        def tokenize(self, inputs):\n            return tf.strings.split(inputs)\n\n        def detokenize(self, inputs):\n            return tf.strings.reduce_join(inputs, separator=\" \", axis=-1)\n\n    tokenizer = WhitespaceSplitterTokenizer()\n\n    # Tokenize some inputs.\n    tokenizer.tokenize(\"This is a test\")\n\n    # Shorthard for `tokenize()`.\n    tokenizer(\"This is a test\")\n\n    # Detokenize some outputs.\n    tokenizer.detokenize([\"This\", \"is\", \"a\", \"test\"])\n    ```\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n\n    def tokenize(self, inputs, *args, **kwargs):\n        \"\"\"Transform input tensors of strings into output tokens.\n\n        Args:\n            inputs: Input tensor, or dict/list/tuple of input tensors.\n            *args: Additional positional arguments.\n            **kwargs: Additional keyword arguments.\n        \"\"\"\n        raise NotImplementedError(\n            \"No implementation of `tokenize()` was found for \"\n            f\"{self.__class__.__name__}. All tokenizers should implement \"\n            \"`tokenize()`.\"\n        )\n\n    def detokenize(self, inputs, *args, **kwargs):\n        \"\"\"Transform tokens back into strings.\n\n        Args:\n            inputs: Input tensor, or dict/list/tuple of input tensors.\n            *args: Additional positional arguments.\n            **kwargs: Additional keyword arguments.\n        \"\"\"\n        raise NotImplementedError(\n            \"No implementation of `detokenize()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def get_vocabulary(self) -> List[str]:\n        \"\"\"Get the tokenizer vocabulary as a list of strings terms.\"\"\"\n        raise NotImplementedError(\n            \"No implementation of `get_vocabulary()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def vocabulary_size(self) -> int:\n        \"\"\"Returns the total size of the token id space.\"\"\"\n        raise NotImplementedError(\n            \"No implementation of `vocabulary_size()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def id_to_token(self, id: int) -> str:\n        \"\"\"Convert an integer id to a string token.\"\"\"\n        raise NotImplementedError(\n            \"No implementation of `id_to_token()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def token_to_id(self, token: str) -> int:\n        \"\"\"Convert an integer id to a string token.\"\"\"\n        raise NotImplementedError(\n            \"No implementation of `id_to_token()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def call(self, inputs, *args, training=None, **kwargs):\n        return self.tokenize(inputs, *args, **kwargs)\n", "path": "keras_nlp/tokenizers/tokenizer.py"}], "after_files": [{"content": "# Copyright 2023 The KerasNLP Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import List\n\nfrom keras_nlp.api_export import keras_nlp_export\nfrom keras_nlp.layers.preprocessing.preprocessing_layer import (\n    PreprocessingLayer,\n)\n\n\n@keras_nlp_export(\"keras_nlp.tokenizers.Tokenizer\")\nclass Tokenizer(PreprocessingLayer):\n    \"\"\"A base class for tokenizer layers.\n\n    Tokenizers in the KerasNLP library should all subclass this layer.\n    The class provides two core methods `tokenize()` and `detokenize()` for\n    going from plain text to sequences and back. A tokenizer is a subclass of\n    `keras.layers.Layer` and can be combined into a `keras.Model`.\n\n    Subclassers should always implement the `tokenize()` method, which will also\n    be the default when calling the layer directly on inputs.\n\n    Subclassers can optionally implement the `detokenize()` method if the\n    tokenization is reversible. Otherwise, this can be skipped.\n\n    Subclassers should implement `get_vocabulary()`, `vocabulary_size()`,\n    `token_to_id()` and `id_to_token()` if applicable. For some simple\n    \"vocab free\" tokenizers, such as a whitespace splitter show below, these\n    methods do not apply and can be skipped.\n\n    Examples:\n\n    ```python\n    class WhitespaceSplitterTokenizer(keras_nlp.tokenizers.Tokenizer):\n        def tokenize(self, inputs):\n            return tf.strings.split(inputs)\n\n        def detokenize(self, inputs):\n            return tf.strings.reduce_join(inputs, separator=\" \", axis=-1)\n\n    tokenizer = WhitespaceSplitterTokenizer()\n\n    # Tokenize some inputs.\n    tokenizer.tokenize(\"This is a test\")\n\n    # Shorthard for `tokenize()`.\n    tokenizer(\"This is a test\")\n\n    # Detokenize some outputs.\n    tokenizer.detokenize([\"This\", \"is\", \"a\", \"test\"])\n    ```\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n\n    def tokenize(self, inputs, *args, **kwargs):\n        \"\"\"Transform input tensors of strings into output tokens.\n\n        Args:\n            inputs: Input tensor, or dict/list/tuple of input tensors.\n            *args: Additional positional arguments.\n            **kwargs: Additional keyword arguments.\n        \"\"\"\n        raise NotImplementedError(\n            \"No implementation of `tokenize()` was found for \"\n            f\"{self.__class__.__name__}. All tokenizers should implement \"\n            \"`tokenize()`.\"\n        )\n\n    def detokenize(self, inputs, *args, **kwargs):\n        \"\"\"Transform tokens back into strings.\n\n        Args:\n            inputs: Input tensor, or dict/list/tuple of input tensors.\n            *args: Additional positional arguments.\n            **kwargs: Additional keyword arguments.\n        \"\"\"\n        raise NotImplementedError(\n            \"No implementation of `detokenize()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def get_vocabulary(self) -> List[str]:\n        \"\"\"Get the tokenizer vocabulary as a list of strings terms.\"\"\"\n        raise NotImplementedError(\n            \"No implementation of `get_vocabulary()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def vocabulary_size(self) -> int:\n        \"\"\"Returns the total size of the token id space.\"\"\"\n        raise NotImplementedError(\n            \"No implementation of `vocabulary_size()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def id_to_token(self, id: int) -> str:\n        \"\"\"Convert an integer id to a string token.\"\"\"\n        raise NotImplementedError(\n            \"No implementation of `id_to_token()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def token_to_id(self, token: str) -> int:\n        \"\"\"Convert an integer id to a string token.\"\"\"\n        raise NotImplementedError(\n            \"No implementation of `id_to_token()` was found for \"\n            f\"{self.__class__.__name__}.\"\n        )\n\n    def call(self, inputs, *args, training=None, **kwargs):\n        return self.tokenize(inputs, *args, **kwargs)\n\n    def compute_output_shape(self, inputs_shape):\n        return tuple(inputs_shape) + (self.sequence_length,)\n", "path": "keras_nlp/tokenizers/tokenizer.py"}]}