



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 	__snake_case   ( SCREAMING_SNAKE_CASE							, unittest.TestCase ):
		SCREAMING_SNAKE_CASE__								=	CodeGenTokenizer
		SCREAMING_SNAKE_CASE__								=	CodeGenTokenizerFast
		SCREAMING_SNAKE_CASE__								=	True
		SCREAMING_SNAKE_CASE__								=	{'add_prefix_space': True}
		SCREAMING_SNAKE_CASE__								=	False
		def 						SCREAMING_SNAKE_CASE_					(							self						):







						"""simple docstring"""
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						lowerCAmelCase__		  =  [
						    'l',
						    'o',
						    'w',
						    'e',
						    'r',
						    's',
						    't',
						    'i',
						    'd',
						    'n',
						    '\u0120',
						    '\u0120l',
						    '\u0120n',
						    '\u0120lo',
						    '\u0120low',
						    'er',
						    '\u0120lowest',
						    '\u0120newer',
						    '\u0120wider',
						    '<unk>',
						    '<|endoftext|>',
						]
						lowerCAmelCase__		  =  dict(zip(a_    ,range(len(a_						)						)						)						)
						lowerCAmelCase__		  =  ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
						lowerCAmelCase__		  =  {'unk_token': '<unk>'}

						lowerCAmelCase__		  =  os.path.join(self.tmpdirname    ,VOCAB_FILES_NAMES['vocab_file']						)
						lowerCAmelCase__		  =  os.path.join(self.tmpdirname    ,VOCAB_FILES_NAMES['merges_file']						)
						with open(self.vocab_file    ,'w'    ,encoding='utf-8'						) as fp:
										fp.write(json.dumps(a_						) + '\n'						)
						with open(self.merges_file    ,'w'    ,encoding='utf-8'						) as fp:
										fp.write('\n'.join(a_						)						)
		def 						SCREAMING_SNAKE_CASE_					(							self    ,**a_						):







						"""simple docstring"""
						kwargs.update(self.special_tokens_map						)
						return CodeGenTokenizer.from_pretrained(self.tmpdirname    ,**a_						)
		def 						SCREAMING_SNAKE_CASE_					(							self    ,**a_						):







						"""simple docstring"""
						kwargs.update(self.special_tokens_map						)
						return CodeGenTokenizerFast.from_pretrained(self.tmpdirname    ,**a_						)
		def 						SCREAMING_SNAKE_CASE_					(							self    ,a_						):







						"""simple docstring"""
						lowerCAmelCase__		  =  'lower newer'
						lowerCAmelCase__		  =  'lower newer'
						return input_text, output_text
		def 						SCREAMING_SNAKE_CASE_					(							self						):







						"""simple docstring"""
						lowerCAmelCase__		  =  CodeGenTokenizer(self.vocab_file    ,self.merges_file    ,**self.special_tokens_map						)
						lowerCAmelCase__		  =  'lower newer'
						lowerCAmelCase__		  =  ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
						lowerCAmelCase__		  =  tokenizer.tokenize(a_    ,add_prefix_space=a_						)
						self.assertListEqual(a_    ,a_						)

						lowerCAmelCase__		  =  tokens + [tokenizer.unk_token]
						lowerCAmelCase__		  =  [14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(a_						)    ,a_						)
		def 						SCREAMING_SNAKE_CASE_					(							self						):







						"""simple docstring"""
						if not self.test_rust_tokenizer:
										return

						lowerCAmelCase__		  =  self.get_tokenizer()
						lowerCAmelCase__		  =  self.get_rust_tokenizer(add_prefix_space=a_						)

						lowerCAmelCase__		  =  'lower newer'

						# Testing tokenization
						lowerCAmelCase__		  =  tokenizer.tokenize(a_    ,add_prefix_space=a_						)
						lowerCAmelCase__		  =  rust_tokenizer.tokenize(a_						)
						self.assertListEqual(a_    ,a_						)

						# Testing conversion to ids without special tokens
						lowerCAmelCase__		  =  tokenizer.encode(a_    ,add_special_tokens=a_    ,add_prefix_space=a_						)
						lowerCAmelCase__		  =  rust_tokenizer.encode(a_    ,add_special_tokens=a_						)
						self.assertListEqual(a_    ,a_						)

						# Testing conversion to ids with special tokens
						lowerCAmelCase__		  =  self.get_rust_tokenizer(add_prefix_space=a_						)
						lowerCAmelCase__		  =  tokenizer.encode(a_    ,add_prefix_space=a_						)
						lowerCAmelCase__		  =  rust_tokenizer.encode(a_						)
						self.assertListEqual(a_    ,a_						)

						# Testing the unknown token
						lowerCAmelCase__		  =  tokens + [rust_tokenizer.unk_token]
						lowerCAmelCase__		  =  [14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a_						)    ,a_						)
		def 						SCREAMING_SNAKE_CASE_					(							self    ,*a_    ,**a_						):







						"""simple docstring"""
						# It's very difficult to mix/test pretokenization with byte-level
						# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
						pass
		def 						SCREAMING_SNAKE_CASE_					(							self    ,a_=15						):







						"""simple docstring"""
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'						):
														lowerCAmelCase__		  =  self.rust_tokenizer_class.from_pretrained(a_    ,**a_						)

														# Simple input
														lowerCAmelCase__		  =  'This is a simple input'
														lowerCAmelCase__		  =  ['This is a simple input 1', 'This is a simple input 2']
														lowerCAmelCase__		  =  ('This is a simple input', 'This is a pair')
														lowerCAmelCase__		  =  [
														    ('This is a simple input 1', 'This is a simple input 2'),
														    ('This is a simple pair 1', 'This is a simple pair 2'),
														]

														# Simple input tests
														self.assertRaises(a_    ,tokenizer_r.encode    ,a_    ,max_length=a_    ,padding='max_length'						)

														# Simple input
														self.assertRaises(a_    ,tokenizer_r.encode_plus    ,a_    ,max_length=a_    ,padding='max_length'						)

														# Simple input
														self.assertRaises(
														    a_    ,tokenizer_r.batch_encode_plus    ,a_    ,max_length=a_    ,padding='max_length'    ,)

														# Pair input
														self.assertRaises(a_    ,tokenizer_r.encode    ,a_    ,max_length=a_    ,padding='max_length'						)

														# Pair input
														self.assertRaises(a_    ,tokenizer_r.encode_plus    ,a_    ,max_length=a_    ,padding='max_length'						)

														# Pair input
														self.assertRaises(
														    a_    ,tokenizer_r.batch_encode_plus    ,a_    ,max_length=a_    ,padding='max_length'    ,)
		def 						SCREAMING_SNAKE_CASE_					(							self						):







						"""simple docstring"""
						lowerCAmelCase__		  =  CodeGenTokenizer.from_pretrained(self.tmpdirname    ,pad_token='<pad>'						)

						# Simple input
						lowerCAmelCase__		  =  'This is a simple input'
						lowerCAmelCase__		  =  ['This is a simple input looooooooong', 'This is a simple input']
						lowerCAmelCase__		  =  ('This is a simple input', 'This is a pair')
						lowerCAmelCase__		  =  [
						    ('This is a simple input loooooong', 'This is a simple input'),
						    ('This is a simple pair loooooong', 'This is a simple pair'),
						]

						lowerCAmelCase__		  =  tokenizer.pad_token_id

						lowerCAmelCase__		  =  tokenizer(a_    ,padding='max_length'    ,max_length=30    ,return_tensors='np'						)
						lowerCAmelCase__		  =  tokenizer(a_    ,padding=a_    ,truncate=a_    ,return_tensors='np'						)
						lowerCAmelCase__		  =  tokenizer(*a_    ,padding='max_length'    ,max_length=60    ,return_tensors='np'						)
						lowerCAmelCase__		  =  tokenizer(a_    ,padding=a_    ,truncate=a_    ,return_tensors='np'						)

						# s
						# test single string max_length padding
						self.assertEqual(out_s['input_ids'].shape[-1]    ,30						)
						self.assertTrue(pad_token_id in out_s['input_ids']						)
						self.assertTrue(0 in out_s['attention_mask']						)

						# s2
						# test automatic padding
						self.assertEqual(out_sa['input_ids'].shape[-1]    ,33						)
						# long slice doesn't have padding
						self.assertFalse(pad_token_id in out_sa['input_ids'][0]						)
						self.assertFalse(0 in out_sa['attention_mask'][0]						)
						# short slice does have padding
						self.assertTrue(pad_token_id in out_sa['input_ids'][1]						)
						self.assertTrue(0 in out_sa['attention_mask'][1]						)

						# p
						# test single pair max_length padding
						self.assertEqual(out_p['input_ids'].shape[-1]    ,60						)
						self.assertTrue(pad_token_id in out_p['input_ids']						)
						self.assertTrue(0 in out_p['attention_mask']						)

						# p2
						# test automatic padding pair
						self.assertEqual(out_pa['input_ids'].shape[-1]    ,52						)
						# long slice pair doesn't have padding
						self.assertFalse(pad_token_id in out_pa['input_ids'][0]						)
						self.assertFalse(0 in out_pa['attention_mask'][0]						)
						# short slice pair does have padding
						self.assertTrue(pad_token_id in out_pa['input_ids'][1]						)
						self.assertTrue(0 in out_pa['attention_mask'][1]						)
		def 						SCREAMING_SNAKE_CASE_					(							self						):







						"""simple docstring"""
						lowerCAmelCase__		  =  '$$$'
						lowerCAmelCase__		  =  CodeGenTokenizer.from_pretrained(self.tmpdirname    ,bos_token=a_    ,add_bos_token=a_						)

						lowerCAmelCase__		  =  'This is a simple input'
						lowerCAmelCase__		  =  ['This is a simple input 1', 'This is a simple input 2']

						lowerCAmelCase__		  =  tokenizer.bos_token_id

						lowerCAmelCase__		  =  tokenizer(a_						)
						lowerCAmelCase__		  =  tokenizer(a_						)

						self.assertEqual(out_s.input_ids[0]    ,a_						)
						self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

						lowerCAmelCase__		  =  tokenizer.decode(out_s.input_ids						)
						lowerCAmelCase__		  =  tokenizer.batch_decode(out_sa.input_ids						)

						self.assertEqual(decode_s.split()[0]    ,a_						)
						self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)
		@slow
		def 						SCREAMING_SNAKE_CASE_					(							self						):







						"""simple docstring"""
						lowerCAmelCase__		  =  CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'						)

						lowerCAmelCase__		  =  '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
						lowerCAmelCase__		  =  '\nif len_a > len_b:      result = a\nelse:      result = b'

						lowerCAmelCase__		  =  tokenizer.encode(a_						)
						lowerCAmelCase__		  =  ['^#', re.escape('<|endoftext|>'						), '^\'\'\'', '^"""', '\n\n\n']
						lowerCAmelCase__		  =  tokenizer.decode(a_    ,truncate_before_pattern=a_						)
						self.assertEqual(a_    ,a_						)



		def 						SCREAMING_SNAKE_CASE_					(							self						):







						"""simple docstring"""
						pass





from dataclasses import dataclass
from typing import Optional

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .modeling_utils import ModelMixin


@dataclass
class 	__snake_case   ( SCREAMING_SNAKE_CASE ):
		SCREAMING_SNAKE_CASE__								=	42



class 	__snake_case   ( SCREAMING_SNAKE_CASE							, SCREAMING_SNAKE_CASE ):
		@register_to_config
		def __init__(							self    ,a_ = 16    ,a_ = 88    ,a_ = None    ,a_ = None    ,a_ = 1    ,a_ = 0.0    ,a_ = 32    ,a_ = None    ,a_ = False    ,a_ = None    ,a_ = "geglu"    ,a_ = True    ,a_ = True    ,):







						"""simple docstring"""
						super().__init__()
						lowerCAmelCase__		  =  num_attention_heads
						lowerCAmelCase__		  =  attention_head_dim
						lowerCAmelCase__		  =  num_attention_heads * attention_head_dim

						lowerCAmelCase__		  =  in_channels

						lowerCAmelCase__		  =  torch.nn.GroupNorm(num_groups=a_    ,num_channels=a_    ,eps=1e-6    ,affine=a_						)
						lowerCAmelCase__		  =  nn.Linear(a_    ,a_						)

						# 3. Define transformers blocks
						lowerCAmelCase__		  =  nn.ModuleList(
						    [
						        BasicTransformerBlock(
						            a_    ,a_    ,a_    ,dropout=a_    ,cross_attention_dim=a_    ,activation_fn=a_    ,attention_bias=a_    ,double_self_attention=a_    ,norm_elementwise_affine=a_    ,)
						        for d in range(a_						)
						    ]						)

						lowerCAmelCase__		  =  nn.Linear(a_    ,a_						)



		def 						SCREAMING_SNAKE_CASE_					(							self    ,a_    ,a_=None    ,a_=None    ,a_=None    ,a_=1    ,a_=None    ,a_ = True    ,):







						"""simple docstring"""
						lowerCAmelCase__  ,   lowerCAmelCase__  ,   lowerCAmelCase__  ,   lowerCAmelCase__		  =  hidden_states.shape
						lowerCAmelCase__		  =  batch_frames // num_frames

						lowerCAmelCase__		  =  hidden_states

						lowerCAmelCase__		  =  hidden_states[None, :].reshape(a_    ,a_    ,a_    ,a_    ,a_						)
						lowerCAmelCase__		  =  hidden_states.permute(0    ,2    ,1    ,3    ,4						)

						lowerCAmelCase__		  =  self.norm(a_						)
						lowerCAmelCase__		  =  hidden_states.permute(0    ,3    ,4    ,2    ,1						).reshape(batch_size * height * width    ,a_    ,a_						)

						lowerCAmelCase__		  =  self.proj_in(a_						)

						# 2. Blocks
						for block in self.transformer_blocks:
										lowerCAmelCase__		  =  block(
										    a_    ,encoder_hidden_states=a_    ,timestep=a_    ,cross_attention_kwargs=a_    ,class_labels=a_    ,)

						# 3. Output
						lowerCAmelCase__		  =  self.proj_out(a_						)
						lowerCAmelCase__		  =  (
						    hidden_states[None, None, :]
						    .reshape(a_    ,a_    ,a_    ,a_    ,a_						)
						    .permute(0    ,3    ,4    ,1    ,2						)
						    .contiguous()
						)
						lowerCAmelCase__		  =  hidden_states.reshape(a_    ,a_    ,a_    ,a_						)

						lowerCAmelCase__		  =  hidden_states + residual

						if not return_dict:
										return (output,)

						return TransformerTemporalModelOutput(sample=a_						)
