






# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


__snake_case    :    int         =     'Run commands across TPU VMs for initial setup before running `accelerate launch`.'







def 	_UpperCamelCase   (    UpperCamelCase_       :					Dict=None		)  ->   Optional[int]:

						"""simple docstring"""



						if subparsers is not None:
												lowerCAmelCase__    						=						subparsers.add_parser('tpu-config'    ,       description=_description		)
						else:
												lowerCAmelCase__    						=						argparse.ArgumentParser('Accelerate tpu-config command'    ,       description=_description		)
						# Core arguments
						lowerCAmelCase__    						=						parser.add_argument_group(
						    'Config Arguments'    ,       'Arguments that can be configured through `accelerate config`.'		)
						config_args.add_argument(
						    '--config_file'    ,       type=snake_case_    ,       default=snake_case_    ,       help='Path to the config file to use for accelerate.'    ,       )
						config_args.add_argument(
						    '--tpu_name'    ,       default=snake_case_    ,       help='The name of the TPU to use. If not specified, will use the TPU specified in the config file.'    ,       )
						config_args.add_argument(
						    '--tpu_zone'    ,       default=snake_case_    ,       help='The zone of the TPU to use. If not specified, will use the zone specified in the config file.'    ,       )
						lowerCAmelCase__    						=						parser.add_argument_group('TPU Arguments'    ,       'Arguments for options ran inside the TPU.'		)
						pod_args.add_argument(
						    '--use_alpha'    ,       action='store_true'    ,       help='Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.'    ,       )
						pod_args.add_argument(
						    '--command_file'    ,       default=snake_case_    ,       help='The path to the file containing the commands to run on the pod on startup.'    ,       )
						pod_args.add_argument(
						    '--command'    ,       action='append'    ,       nargs='+'    ,       help='A command to run on the pod. Can be passed multiple times.'    ,       )
						pod_args.add_argument(
						    '--install_accelerate'    ,       action='store_true'    ,       help='Whether to install accelerate on the pod. Defaults to False.'    ,       )
						pod_args.add_argument(
						    '--accelerate_version'    ,       default='latest'    ,       help='The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.'    ,       )
						pod_args.add_argument(
						    '--debug'    ,       action='store_true'    ,       help='If set, will print the command that would be run instead of running it.'		)

						if subparsers is not None:
												parser.set_defaults(func=snake_case_		)
						return parser







def 	_UpperCamelCase   (    UpperCamelCase_       :					List[str]		)  ->   Dict:

						"""simple docstring"""



						lowerCAmelCase__    						=						None

						# Get the default from the config file if it exists.
						if args.config_file is not None or os.path.isfile(snake_case_		):
												lowerCAmelCase__    						=						load_config_from_file(args.config_file		)
												if not args.command_file and defaults.command_file is not None and not args.command:
																		lowerCAmelCase__    						=						defaults.command_file
												if not args.command and defaults.commands is not None:
																		lowerCAmelCase__    						=						defaults.commands
												if not args.tpu_name:
																		lowerCAmelCase__    						=						defaults.tpu_name
												if not args.tpu_zone:
																		lowerCAmelCase__    						=						defaults.tpu_zone
						if args.accelerate_version == "dev":
												lowerCAmelCase__    						=						'''git+https://github.com/huggingface/accelerate.git'''
						elif args.accelerate_version == "latest":
												lowerCAmelCase__    						=						'''accelerate -U'''
						elif isinstance(parse(args.accelerate_version		)    ,       snake_case_		):
												lowerCAmelCase__    						=						F"accelerate=={args.accelerate_version}"

						if not args.command_file and not args.command:
												raise ValueError('You must specify either a command file or a command to run on the pod.'		)

						if args.command_file:
												with open(args.command_file    ,       'r'		) as f:
																		lowerCAmelCase__    						=						[f.read().splitlines()]

																		 # To turn list of lists into list of strings
						if isinstance(args.command[0]    ,       snake_case_		):
												lowerCAmelCase__    						=						[line for cmd in args.command for line in cmd]
						# Default to the shared folder and install accelerate
						lowerCAmelCase__    						=						['''cd /usr/share''']
						if args.install_accelerate:
												new_cmd += [F"pip install {args.accelerate_version}"]
						new_cmd += args.command
						lowerCAmelCase__    						=						'''; '''.join(snake_case_		)

						# Then send it to gcloud
						# Eventually try to use google-api-core to do this instead of subprocess
						lowerCAmelCase__    						=						['''gcloud''']
						if args.use_alpha:
												cmd += ["alpha"]
						cmd += [
						    "compute",
						    "tpus",
						    "tpu-vm",
						    "ssh",
						    args.tpu_name,
						    "--zone",
						    args.tpu_zone,
						    "--command",
						    args.command,
						    "--worker",
						    "all",
						]
						if args.debug:
												print(F"Running {' '.join(snake_case_		)}"		)
												return
						subprocess.run(snake_case_		)
						print('Successfully setup pod.'		)







def 	_UpperCamelCase   (    )  ->   Tuple:

						"""simple docstring"""



						lowerCAmelCase__    						=						tpu_command_parser()
						lowerCAmelCase__    						=						parser.parse_args()

						tpu_command_launcher(snake_case_		)













import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


__snake_case    :    Any         =     logging.get_logger(__name__)

__snake_case    :    Tuple         =     {"""tokenizer_file""": """tokenizer.json"""}

__snake_case    :    str         =     {
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}


class     __SCREAMING_SNAKE_CASE					(   __lowercase):
						_SCREAMING_SNAKE_CASE	:       Dict   =					VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					['''input_ids''', '''attention_mask''']
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					None






						def __init__(	self     ,						_UpperCamelCase=None     ,						_UpperCamelCase=None     ,						_UpperCamelCase=None     ,						_UpperCamelCase="<unk>"     ,						_UpperCamelCase="<s>"     ,						_UpperCamelCase="</s>"     ,						_UpperCamelCase="<pad>"     ,						_UpperCamelCase=False     ,						_UpperCamelCase=False     ,						**_UpperCamelCase     ,						):






												"""simple docstring"""
												super().__init__(
												    _UpperCamelCase     ,						_UpperCamelCase     ,						tokenizer_file=_UpperCamelCase     ,						unk_token=_UpperCamelCase     ,						bos_token=_UpperCamelCase     ,						eos_token=_UpperCamelCase     ,						pad_token=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						clean_up_tokenization_spaces=_UpperCamelCase     ,						**_UpperCamelCase     ,						)
												lowerCAmelCase__    						=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
												if pre_tok_state.get('add_prefix_space'     ,						_UpperCamelCase   ) != add_prefix_space:
																		lowerCAmelCase__    						=						getattr(_UpperCamelCase     ,						pre_tok_state.pop('type'   )   )
																		lowerCAmelCase__    						=						add_prefix_space
																		lowerCAmelCase__    						=						pre_tok_class(**_UpperCamelCase   )

												lowerCAmelCase__    						=						add_prefix_space






						def       UpperCamelCase__      (	self     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						kwargs.get('is_split_into_words'     ,						_UpperCamelCase   )
												if not (self.add_prefix_space or not is_split_into_words):
																		raise Exception(
																		    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
																		    ' pretokenized inputs.'   )

												return super()._batch_encode_plus(*_UpperCamelCase     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						kwargs.get('is_split_into_words'     ,						_UpperCamelCase   )

												if not (self.add_prefix_space or not is_split_into_words):
																		raise Exception(
																		    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
																		    ' pretokenized inputs.'   )

												return super()._encode_plus(*_UpperCamelCase     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase = None   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self._tokenizer.model.save(_UpperCamelCase     ,						name=_UpperCamelCase   )
												return tuple(_UpperCamelCase   )







						def       UpperCamelCase__      (	self     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						[]
												for is_user, text in conversation.iter_texts():
																		input_ids.extend(self.encode(_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   ) + [self.eos_token_id]   )

												if len(_UpperCamelCase   ) > self.model_max_length:
																		lowerCAmelCase__    						=						input_ids[-self.model_max_length :]
												return input_ids





