





'''simple docstring'''







import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        FalconForCausalLM,
        FalconForQuestionAnswering,
        FalconForSequenceClassification,
        FalconForTokenClassification,
        FalconModel,
    )



class 				__SCREAMING_SNAKE_CASE      :
      '''simple docstring'''



      def __init__(				self     :			str ,UpperCamelCase     :			List[Any] ,UpperCamelCase     :			Dict=3 ,UpperCamelCase     :			Optional[int]=7 ,UpperCamelCase     :			Tuple=True ,UpperCamelCase     :			str=True ,UpperCamelCase     :			Tuple=False ,UpperCamelCase     :			List[str]=True ,UpperCamelCase     :			Optional[int]=99 ,UpperCamelCase     :			Union[str, Any]=32 ,UpperCamelCase     :			Any=5 ,UpperCamelCase     :			Dict=4 ,UpperCamelCase     :			Dict=37 ,UpperCamelCase     :			int="gelu" ,UpperCamelCase     :			str=0.1 ,UpperCamelCase     :			Optional[int]=0.1 ,UpperCamelCase     :			List[Any]=512 ,UpperCamelCase     :			Any=16 ,UpperCamelCase     :			List[Any]=2 ,UpperCamelCase     :			Tuple=0.0_2 ,UpperCamelCase     :			List[str]=3 ,UpperCamelCase     :			Any=4 ,UpperCamelCase     :			Dict=None ,)		->	Optional[int]:
           _lowercase				:   Tuple	    =				parent
           _lowercase				:   Tuple	    =				batch_size
           _lowercase				:   List[str]	    =				seq_length
           _lowercase				:   int	    =				is_training
           _lowercase				:   Any	    =				use_input_mask
           _lowercase				:   List[Any]	    =				use_token_type_ids
           _lowercase				:   Union[str, Any]	    =				use_labels
           _lowercase				:   Any	    =				vocab_size
           _lowercase				:   Any	    =				hidden_size
           _lowercase				:   str	    =				num_hidden_layers
           _lowercase				:   Optional[int]	    =				num_attention_heads
           _lowercase				:   List[str]	    =				intermediate_size
           _lowercase				:   Optional[int]	    =				hidden_act
           _lowercase				:   Any	    =				hidden_dropout_prob
           _lowercase				:   Dict	    =				attention_probs_dropout_prob
           _lowercase				:   Optional[int]	    =				max_position_embeddings
           _lowercase				:   str	    =				type_vocab_size
           _lowercase				:   List[str]	    =				type_sequence_label_size
           _lowercase				:   List[str]	    =				initializer_range
           _lowercase				:   Dict	    =				num_labels
           _lowercase				:   Union[str, Any]	    =				num_choices
           _lowercase				:   Union[str, Any]	    =				scope

      def 	_lowerCamelCase		(				self     :			int	)		->	int:
           _lowercase				:   List[Any]	    =				ids_tensor([self.batch_size, self.seq_length] ,self.vocab_size	)

           _lowercase				:   int	    =				None
           if self.use_input_mask:
                _lowercase				:   List[Any]	    =				random_attention_mask([self.batch_size, self.seq_length]	)

           _lowercase				:   Optional[Any]	    =				None

           _lowercase				:   Any	    =				None
           _lowercase				:   str	    =				None
           _lowercase				:   str	    =				None
           if self.use_labels:
                _lowercase				:   Optional[int]	    =				ids_tensor([self.batch_size] ,self.type_sequence_label_size	)
                _lowercase				:   Dict	    =				ids_tensor([self.batch_size, self.seq_length] ,self.num_labels	)
                _lowercase				:   int	    =				ids_tensor([self.batch_size] ,self.num_choices	)

           _lowercase				:   Tuple	    =				self.get_config()

           return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

      def 	_lowerCamelCase		(				self     :			Any	)		->	List[Any]:
           return FalconConfig(
               vocab_size=self.vocab_size ,hidden_size=self.hidden_size ,num_hidden_layers=self.num_hidden_layers ,num_attention_heads=self.num_attention_heads ,intermediate_size=self.intermediate_size ,hidden_act=self.hidden_act ,hidden_dropout_prob=self.hidden_dropout_prob ,attention_probs_dropout_prob=self.attention_probs_dropout_prob ,max_position_embeddings=self.max_position_embeddings ,type_vocab_size=self.type_vocab_size ,is_decoder=snake_case__ ,initializer_range=self.initializer_range ,pad_token_id=1 ,new_decoder_architecture=snake_case__ ,)

      def 	_lowerCamelCase		(				self     :			Any ,UpperCamelCase     :			Optional[Any] ,UpperCamelCase     :			str ,UpperCamelCase     :			Tuple ,UpperCamelCase     :			int ,UpperCamelCase     :			str ,UpperCamelCase     :			Optional[Any] ,UpperCamelCase     :			Dict	)		->	Tuple:
           _lowercase				:   Optional[Any]	    =				FalconModel(config=snake_case__	)
           model.to(snake_case__	)
           model.eval()
           _lowercase				:   Dict	    =				model(snake_case__ ,attention_mask=snake_case__	)
           _lowercase				:   int	    =				model(snake_case__	)
           self.parent.assertEqual(result.last_hidden_state.shape ,(self.batch_size, self.seq_length, self.hidden_size)	)

      def 	_lowerCamelCase		(				self     :			Optional[Any] ,UpperCamelCase     :			List[str] ,UpperCamelCase     :			Any ,UpperCamelCase     :			Union[str, Any] ,UpperCamelCase     :			str ,UpperCamelCase     :			Dict ,UpperCamelCase     :			int ,UpperCamelCase     :			Optional[int] ,UpperCamelCase     :			Optional[Any] ,UpperCamelCase     :			List[Any] ,)		->	Tuple:
           _lowercase				:   Optional[Any]	    =				True
           _lowercase				:   Tuple	    =				FalconModel(snake_case__	)
           model.to(snake_case__	)
           model.eval()
           _lowercase				:   str	    =				model(
               snake_case__ ,attention_mask=snake_case__ ,encoder_hidden_states=snake_case__ ,encoder_attention_mask=snake_case__ ,)
           _lowercase				:   str	    =				model(
               snake_case__ ,attention_mask=snake_case__ ,encoder_hidden_states=snake_case__ ,)
           _lowercase				:   Optional[Any]	    =				model(snake_case__ ,attention_mask=snake_case__	)
           self.parent.assertEqual(result.last_hidden_state.shape ,(self.batch_size, self.seq_length, self.hidden_size)	)

      def 	_lowerCamelCase		(				self     :			Dict ,UpperCamelCase     :			Optional[int] ,UpperCamelCase     :			Any ,UpperCamelCase     :			Optional[Any] ,UpperCamelCase     :			Union[str, Any] ,UpperCamelCase     :			Any ,UpperCamelCase     :			Tuple ,UpperCamelCase     :			Optional[Any] ,UpperCamelCase     :			Optional[Any] ,UpperCamelCase     :			Tuple ,)		->	Optional[Any]:
           _lowercase				:   Any	    =				FalconForCausalLM(config=snake_case__	)
           model.to(snake_case__	)
           model.eval()
           _lowercase				:   str	    =				model(snake_case__ ,attention_mask=snake_case__ ,labels=snake_case__	)
           self.parent.assertEqual(result.logits.shape ,(self.batch_size, self.seq_length, self.vocab_size)	)

      def 	_lowerCamelCase		(				self     :			List[Any] ,UpperCamelCase     :			Dict ,UpperCamelCase     :			Union[str, Any] ,UpperCamelCase     :			List[str] ,UpperCamelCase     :			Any ,UpperCamelCase     :			List[str] ,UpperCamelCase     :			Optional[int] ,UpperCamelCase     :			Optional[Any] ,UpperCamelCase     :			str ,UpperCamelCase     :			List[str] ,)		->	int:
           _lowercase				:   List[str]	    =				True
           _lowercase				:   List[Any]	    =				True
           _lowercase				:   Optional[Any]	    =				FalconForCausalLM(config=snake_case__	)
           model.to(snake_case__	)
           model.eval()

           # first forward pass
           _lowercase				:   List[Any]	    =				model(
               snake_case__ ,attention_mask=snake_case__ ,encoder_hidden_states=snake_case__ ,encoder_attention_mask=snake_case__ ,use_cache=snake_case__ ,)
           _lowercase				:   int	    =				outputs.past_key_values

           # create hypothetical multiple next token and extent to next_input_ids
           _lowercase				:   Optional[int]	    =				ids_tensor((self.batch_size, 3) ,config.vocab_size	)
           _lowercase				:   Dict	    =				ids_tensor((self.batch_size, 3) ,vocab_size=2	)

           # append to next input_ids and
           _lowercase				:   List[str]	    =				torch.cat([input_ids, next_tokens] ,dim=-1	)
           _lowercase				:   str	    =				torch.cat([input_mask, next_mask] ,dim=-1	)

           _lowercase				:   List[Any]	    =				model(
               snake_case__ ,attention_mask=snake_case__ ,encoder_hidden_states=snake_case__ ,encoder_attention_mask=snake_case__ ,output_hidden_states=snake_case__ ,)["""hidden_states"""][0]
           _lowercase				:   List[Any]	    =				model(
               snake_case__ ,attention_mask=snake_case__ ,encoder_hidden_states=snake_case__ ,encoder_attention_mask=snake_case__ ,past_key_values=snake_case__ ,output_hidden_states=snake_case__ ,)["""hidden_states"""][0]

           # select random slice
           _lowercase				:   List[Any]	    =				ids_tensor((1,) ,output_from_past.shape[-1]	).item()
           _lowercase				:   Any	    =				output_from_no_past[:, -3:, random_slice_idx].detach()
           _lowercase				:   int	    =				output_from_past[:, :, random_slice_idx].detach()

           self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

           # test that outputs are equal for slice
           self.parent.assertTrue(torch.allclose(snake_case__ ,snake_case__ ,atol=1e-3	)	)



      def 	_lowerCamelCase		(				self     :			Union[str, Any]	)		->	Any:
           _lowercase				:   Tuple	    =				self.prepare_config_and_inputs()
           (
               _lowercase		
           )				:   Union[str, Any]	    =				config_and_inputs
           _lowercase				:   List[Any]	    =				{"""input_ids""": input_ids, """attention_mask""": input_mask}
           return config, inputs_dict



@require_torch
class 				__SCREAMING_SNAKE_CASE      (							lowerCamelCase__				, lowerCamelCase__				, lowerCamelCase__				, unittest.TestCase   ):
      '''simple docstring'''



      lowerCAmelCase__							:   int					=		(
          (
              FalconModel,
              FalconForCausalLM,
              FalconForSequenceClassification,
              FalconForTokenClassification,
              FalconForQuestionAnswering,
          )
          if is_torch_available()
          else ()
      )
      lowerCAmelCase__							:   Tuple					=		(FalconForCausalLM,) if is_torch_available() else ()
      lowerCAmelCase__							:   Union[str, Any]					=		(
          {
              """feature-extraction""": FalconModel,
              """text-classification""": FalconForSequenceClassification,
              """text-generation""": FalconForCausalLM,
              """question-answering""": FalconForQuestionAnswering,
              """token-classification""": FalconForTokenClassification,
              """zero-shot""": FalconForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      lowerCAmelCase__							:   int					=		False
      lowerCAmelCase__							:   List[Any]					=		False

      def 	_lowerCamelCase		(				self     :			str	)		->	Optional[int]:
           _lowercase				:   Any	    =				FalconModelTester(self	)
           _lowercase				:   Union[str, Any]	    =				ConfigTester(self ,config_class=snake_case__ ,hidden_size=37	)

      def 	_lowerCamelCase		(				self     :			Optional[int]	)		->	Tuple:
           self.config_tester.run_common_tests()

      def 	_lowerCamelCase		(				self     :			int	)		->	Any:
           _lowercase				:   int	    =				self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*snake_case__	)

      def 	_lowerCamelCase		(				self     :			Dict	)		->	Optional[Any]:
           _lowercase				:   str	    =				self.model_tester.prepare_config_and_inputs()
           for alibi in [True, False]:
                _lowercase				:   List[Any]	    =				alibi
                self.model_tester.create_and_check_model(snake_case__ ,*snake_case__	)

      def 	_lowerCamelCase		(				self     :			List[Any]	)		->	Dict:
           _lowercase				:   Dict	    =				self.model_tester.prepare_config_and_inputs_for_common()
           _lowercase				:   Tuple	    =				3
           _lowercase				:   str	    =				input_dict["""input_ids"""]
           _lowercase				:   int	    =				input_ids.ne(1	).to(snake_case__	)
           _lowercase				:   Any	    =				ids_tensor([self.model_tester.batch_size] ,self.model_tester.type_sequence_label_size	)
           _lowercase				:   List[Any]	    =				FalconForSequenceClassification(snake_case__	)
           model.to(snake_case__	)
           model.eval()
           _lowercase				:   Tuple	    =				model(snake_case__ ,attention_mask=snake_case__ ,labels=snake_case__	)
           self.assertEqual(result.logits.shape ,(self.model_tester.batch_size, self.model_tester.num_labels)	)

      def 	_lowerCamelCase		(				self     :			List[str]	)		->	int:
           _lowercase				:   List[Any]	    =				self.model_tester.prepare_config_and_inputs_for_common()
           _lowercase				:   Optional[Any]	    =				3
           _lowercase				:   int	    =				"""single_label_classification"""
           _lowercase				:   Tuple	    =				input_dict["""input_ids"""]
           _lowercase				:   Tuple	    =				input_ids.ne(1	).to(snake_case__	)
           _lowercase				:   Optional[int]	    =				ids_tensor([self.model_tester.batch_size] ,self.model_tester.type_sequence_label_size	)
           _lowercase				:   Tuple	    =				FalconForSequenceClassification(snake_case__	)
           model.to(snake_case__	)
           model.eval()
           _lowercase				:   List[str]	    =				model(snake_case__ ,attention_mask=snake_case__ ,labels=snake_case__	)
           self.assertEqual(result.logits.shape ,(self.model_tester.batch_size, self.model_tester.num_labels)	)

      def 	_lowerCamelCase		(				self     :			Optional[int]	)		->	List[str]:
           _lowercase				:   List[str]	    =				self.model_tester.prepare_config_and_inputs_for_common()
           _lowercase				:   Tuple	    =				input_dict["""input_ids"""]
           _lowercase				:   Any	    =				FalconForCausalLM(snake_case__	)
           model.to(snake_case__	)
           model.eval()
           _lowercase				:   Tuple	    =				model(snake_case__ ,use_cache=snake_case__	)
           _lowercase				:   Optional[int]	    =				input_ids.shape[0]
           _lowercase				:   Optional[int]	    =				model._convert_to_rw_cache(result.past_key_values	)
           _lowercase				:   int	    =				model._convert_cache_to_standard_format(snake_case__ ,snake_case__	)
           for layer in range(len(snake_case__	)	):
                for tensor_idx in range(2	):
                     self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3	)
                     self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4	)
                     self.assertTrue(
                         torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]	)	)

      def 	_lowerCamelCase		(				self     :			Optional[Any]	)		->	int:
           _lowercase				:   List[Any]	    =				self.model_tester.prepare_config_and_inputs_for_common()
           _lowercase				:   Tuple	    =				3
           _lowercase				:   str	    =				"""multi_label_classification"""
           _lowercase				:   str	    =				input_dict["""input_ids"""]
           _lowercase				:   Dict	    =				input_ids.ne(1	).to(snake_case__	)
           _lowercase				:   Dict	    =				ids_tensor(
               [self.model_tester.batch_size, config.num_labels] ,self.model_tester.type_sequence_label_size	).to(torch.float	)
           _lowercase				:   Optional[int]	    =				FalconForSequenceClassification(snake_case__	)
           model.to(snake_case__	)
           model.eval()
           _lowercase				:   Optional[int]	    =				model(snake_case__ ,attention_mask=snake_case__ ,labels=snake_case__	)
           self.assertEqual(result.logits.shape ,(self.model_tester.batch_size, self.model_tester.num_labels)	)



      def 	_lowerCamelCase		(				self     :			List[str]	)		->	Tuple:
           # Falcon can have different numbers of KV-heads than the number of query heads, so we need
           # to override this test to use the right head counts.
           for model_class in self.all_generative_model_classes:
                _lowercase				:   Union[str, Any]	    =				self.model_tester.prepare_config_and_inputs_for_common()

                # If it doesn't support cache, pass the test
                if not hasattr(snake_case__ ,'use_cache'	):
                     return

                _lowercase				:   Optional[int]	    =				model_class(snake_case__	).to(snake_case__	)
                if "use_cache" not in inputs:
                     _lowercase				:   Dict	    =				True
                _lowercase				:   Optional[Any]	    =				model(**snake_case__	)

                # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
                if "past_key_values" not in outputs:
                     return

                _lowercase				:   str	    =				(
                    getattr(snake_case__ ,'decoder_layers' ,snake_case__	)
                    or getattr(snake_case__ ,'num_decoder_layers' ,snake_case__	)
                    or config.num_hidden_layers
                )
                _lowercase				:   Dict	    =				getattr(snake_case__ ,'num_kv_heads' ,config.num_attention_heads	)
                _lowercase				:   int	    =				getattr(snake_case__ ,'d_model' ,config.hidden_size	)
                _lowercase				:   List[Any]	    =				embed_dim // num_attention_heads

                _lowercase				:   List[Any]	    =				outputs["""past_key_values"""]
                self.assertEqual(len(snake_case__	) ,snake_case__	)

                _lowercase				:   int	    =				inputs["""input_ids"""].shape
                for i in range(snake_case__	):
                     if config.new_decoder_architecture:
                          _lowercase				:   Optional[Any]	    =				config.num_attention_heads
                     elif config.multi_query:
                          _lowercase				:   Union[str, Any]	    =				1
                     self.assertEqual(len(past_kv[0]	) ,2	)  # K V for the decoder = 2
                     self.assertEqual(
                         past_kv[i][0].shape ,(batch_size, num_attention_heads, seq_length, per_head_embed_dim)	)
                     self.assertEqual(
                         past_kv[i][1].shape ,(batch_size, num_attention_heads, seq_length, per_head_embed_dim)	)



@require_torch
class 				__SCREAMING_SNAKE_CASE      (							unittest.TestCase   ):
      '''simple docstring'''



      @slow
      def 	_lowerCamelCase		(				self     :			Any	)		->	Dict:
           _lowercase				:   Optional[Any]	    =				AutoTokenizer.from_pretrained('Rocketknight1/falcon-rw-1b'	)
           _lowercase				:   Tuple	    =				FalconForCausalLM.from_pretrained('Rocketknight1/falcon-rw-1b'	)
           model.eval()
           model.to(snake_case__	)
           _lowercase				:   Optional[int]	    =				tokenizer('My favorite food is' ,return_tensors='pt'	).to(snake_case__	)

           _lowercase				:   List[str]	    =				(
               """My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."""
           )

           _lowercase				:   str	    =				model.generate(**snake_case__ ,do_sample=snake_case__ ,max_new_tokens=19	)
           _lowercase				:   Optional[int]	    =				tokenizer.batch_decode(snake_case__	)[0]

           self.assertEqual(snake_case__ ,snake_case__	)

      @slow
      def 	_lowerCamelCase		(				self     :			int	)		->	Optional[Any]:
           # The big models are way too big for the CI, so we use tiny random models that resemble their
           # architectures but with much smaller and fewer layers
           for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
                _lowercase				:   Dict	    =				AutoTokenizer.from_pretrained(snake_case__	)
                _lowercase				:   Optional[Any]	    =				FalconForCausalLM.from_pretrained(snake_case__	)
                model.eval()
                model.to(snake_case__	)
                _lowercase				:   str	    =				tokenizer('My favorite food is' ,return_tensors='pt'	).to(snake_case__	)

                # We just test that these run without errors - the models are randomly initialized
                # and so the actual text outputs will be garbage
                model.generate(**snake_case__ ,do_sample=snake_case__ ,max_new_tokens=4	)
                model.generate(**snake_case__ ,do_sample=snake_case__ ,max_new_tokens=4	)
                model.generate(**snake_case__ ,num_beams=2 ,max_new_tokens=4	)



      @slow
      def 	_lowerCamelCase		(				self     :			int	)		->	Optional[Any]:
           # The big models are way too big for the CI, so we use tiny random models that resemble their
           # architectures but with much smaller and fewer layers
           with torch.no_grad():
                for repo in [
                    "Rocketknight1/falcon-rw-1b",
                    "Rocketknight1/tiny-random-falcon-7b",
                    "Rocketknight1/tiny-random-falcon-40b",
                ]:
                     _lowercase				:   str	    =				AutoTokenizer.from_pretrained(snake_case__	)
                     _lowercase				:   Dict	    =				FalconForCausalLM.from_pretrained(snake_case__	)
                     model.eval()
                     model.to(device=snake_case__	)
                     _lowercase				:   Optional[int]	    =				tokenizer('My favorite food is' ,return_tensors='pt'	).to(snake_case__	)

                     # Test results are the same with and without cache
                     _lowercase				:   List[str]	    =				model.generate(**snake_case__ ,do_sample=snake_case__ ,max_new_tokens=20 ,use_cache=snake_case__	)
                     _lowercase				:   Tuple	    =				model.generate(**snake_case__ ,do_sample=snake_case__ ,max_new_tokens=20 ,use_cache=snake_case__	)
                     self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0	)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_							=				logging.get_logger(__name__)

UpperCAmelCase_							=				{
    """google/canine-s""": """https://huggingface.co/google/canine-s/resolve/main/config.json""",
    # See all CANINE models at https://huggingface.co/models?filter=canine
}
class     UpperCamelCase__					(     lowerCamelCase__				):






		'''simple docstring'''






		__a       :       Optional[int]       =	"""canine"""






		def __init__(							self,   snake_case__=7_68,   snake_case__=12,   snake_case__=12,   snake_case__=30_72,   snake_case__="gelu",   snake_case__=0.1,   snake_case__=0.1,   snake_case__=1_63_84,   snake_case__=16,   snake_case__=0.02,   snake_case__=1E-12,   snake_case__=0,   snake_case__=0XE_0_0_0,   snake_case__=0XE_0_0_1,   snake_case__=4,   snake_case__=4,   snake_case__=8,   snake_case__=1_63_84,   snake_case__=1_28,   **snake_case__,   )       -> Optional[int]:






									"""simple docstring"""
									super().__init__(pad_token_id=snake_case__,   bos_token_id=snake_case__,   eos_token_id=snake_case__,   **snake_case__  )

									lowercase_	:   int  				=     max_position_embeddings
									lowercase_	:   Union[str, Any]  				=     hidden_size
									lowercase_	:   Optional[Any]  				=     num_hidden_layers
									lowercase_	:   str  				=     num_attention_heads
									lowercase_	:   List[Any]  				=     intermediate_size
									lowercase_	:   Optional[Any]  				=     hidden_act
									lowercase_	:   List[str]  				=     hidden_dropout_prob
									lowercase_	:   Any  				=     attention_probs_dropout_prob
									lowercase_	:   Union[str, Any]  				=     initializer_range
									lowercase_	:   Optional[Any]  				=     type_vocab_size
									lowercase_	:   Dict  				=     layer_norm_eps

									# Character config:
									lowercase_	:   Optional[int]  				=     downsampling_rate
									lowercase_	:   List[str]  				=     upsampling_kernel_size
									lowercase_	:   int  				=     num_hash_functions
									lowercase_	:   List[str]  				=     num_hash_buckets
									lowercase_	:   List[str]  				=     local_transformer_stride