

"""simple docstring"""


from abc import ABC, abstractmethod
from typing import List, Optional





class        snake_case      (							lowercase__    ):

				def __init__(      self				:							str)->  Optional[Any]:



						'''simple docstring'''


						self.test()

				def 				lowercase_	(      self				:							Tuple)->  Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:			int        =     0
						__lowerCAmelCase:			List[Any]        =     False
						while not completed:
								if counter == 1:
										self.reset()
								__lowerCAmelCase:			Dict        =     self.advance()
								if not self.does_advance(UpperCAmelCase__):
										raise Exception(
										    "Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true.")

								__lowerCAmelCase:			str        =     self.update(UpperCAmelCase__)
								counter += 1

								if counter > 1_0_0_0_0:
										raise Exception("update() does not fulfill the constraint.")

						if self.remaining() != 0:
								raise Exception("Custom Constraint is not defined correctly.")

				@abstractmethod
				def 				lowercase_	(      self				:							Any)->  Any:



						'''simple docstring'''


						raise NotImplementedError(
						    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called.")

				@abstractmethod
				def 				lowercase_	(      self				:							List[str]	,    UpperCamelCase__				:							int)->  List[str]:



						'''simple docstring'''


						raise NotImplementedError(
						    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called.")

				@abstractmethod
				def 				lowercase_	(      self				:							Union[str, Any]	,    UpperCamelCase__				:							int)->  Any:



						'''simple docstring'''


						raise NotImplementedError(
						    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called.")

				@abstractmethod
				def 				lowercase_	(      self				:							Optional[int])->  List[Any]:



						'''simple docstring'''


						raise NotImplementedError(
						    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called.")

				@abstractmethod
				def 				lowercase_	(      self				:							List[str])->  Optional[Any]:



						'''simple docstring'''


						raise NotImplementedError(
						    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called.")




				@abstractmethod
				def 				lowercase_	(      self				:							Optional[int]	,    UpperCamelCase__				:							Optional[Any]=False)->  Dict:



						'''simple docstring'''


						raise NotImplementedError(
						    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called.")





class        snake_case      (							lowercase__    ):

				def __init__(      self				:							int	,    UpperCamelCase__				:							List[int])->  Optional[int]:



						'''simple docstring'''


						super(UpperCAmelCase__	,    self).__init__()

						if not isinstance(UpperCAmelCase__	,    UpperCAmelCase__) or len(UpperCAmelCase__) == 0:
								raise ValueError(f"`token_ids` has to be a non-empty list, but is {token_ids}.")
						if any((not isinstance(UpperCAmelCase__	,    UpperCAmelCase__) or token_id < 0) for token_id in token_ids):
								raise ValueError(f"Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.")

						__lowerCAmelCase:			Union[str, Any]        =     token_ids

						__lowerCAmelCase:			Tuple        =     len(self.token_ids)
						__lowerCAmelCase:			Tuple        =     -1  # the index of the currently fulfilled step
						__lowerCAmelCase:			int        =     False

				def 				lowercase_	(      self				:							int)->  List[str]:



						'''simple docstring'''


						if self.completed:
								return None
						return self.token_ids[self.fulfilled_idx + 1]

				def 				lowercase_	(      self				:							str	,    UpperCamelCase__				:							int)->  Dict:



						'''simple docstring'''


						if not isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
								raise ValueError(f"`token_id` has to be an `int`, but is {token_id} of type {type(UpperCAmelCase__)}")

						if self.completed:
								return False

						return token_id == self.token_ids[self.fulfilled_idx + 1]

				def 				lowercase_	(      self				:							str	,    UpperCamelCase__				:							int)->  str:



						'''simple docstring'''


						if not isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
								raise ValueError(f"`token_id` has to be an `int`, but is {token_id} of type {type(UpperCAmelCase__)}")

						__lowerCAmelCase:			Any        =     False
						__lowerCAmelCase:			Optional[int]        =     False
						__lowerCAmelCase:			int        =     False

						if self.does_advance(UpperCAmelCase__):
								self.fulfilled_idx += 1
								__lowerCAmelCase:			Any        =     True
								if self.fulfilled_idx == (self.seqlen - 1):
										__lowerCAmelCase:			List[str]        =     True
								__lowerCAmelCase:			List[Any]        =     completed
						else:
								# failed to make progress.
								__lowerCAmelCase:			Union[str, Any]        =     True
								self.reset()
						return stepped, completed, reset

				def 				lowercase_	(      self				:							Any)->  List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:			int        =     False
						__lowerCAmelCase:			Tuple        =     0

				def 				lowercase_	(      self				:							Union[str, Any])->  Any:



						'''simple docstring'''


						return self.seqlen - (self.fulfilled_idx + 1)




				def 				lowercase_	(      self				:							int	,    UpperCamelCase__				:							int=False)->  Dict:



						'''simple docstring'''


						__lowerCAmelCase:			Dict        =     PhrasalConstraint(self.token_ids)

						if stateful:
								__lowerCAmelCase:			Union[str, Any]        =     self.seqlen
								__lowerCAmelCase:			Optional[Any]        =     self.fulfilled_idx
								__lowerCAmelCase:			List[str]        =     self.completed

						return new_constraint





class        snake_case      :

				def __init__(      self				:							Any	,    UpperCamelCase__				:							List[List[int]]	,    UpperCamelCase__				:							Dict=True)->  int:



						'''simple docstring'''


						__lowerCAmelCase:			str        =     max([len(UpperCAmelCase__) for one in nested_token_ids])

						__lowerCAmelCase:			Any        =     {}
						for token_ids in nested_token_ids:
								__lowerCAmelCase:			Any        =     root
								for tidx, token_id in enumerate(UpperCAmelCase__):
										if token_id not in level:
												__lowerCAmelCase:			int        =     {}

										__lowerCAmelCase:			Union[str, Any]        =     level[token_id]

						if no_subsets and self.has_subsets(UpperCAmelCase__	,    UpperCAmelCase__):
								raise ValueError(
								    "Each list in `nested_token_ids` can\'t be a complete subset of another list, but is"
								    f" {nested_token_ids}.")

						__lowerCAmelCase:			Dict        =     root

				def 				lowercase_	(      self				:							int	,    UpperCamelCase__				:							Dict)->  Any:



						'''simple docstring'''


						__lowerCAmelCase:			Any        =     self.trie

						for current_token in current_seq:
								__lowerCAmelCase:			List[str]        =     start[current_token]

						__lowerCAmelCase:			Optional[Any]        =     list(start.keys())

						return next_tokens

				def 				lowercase_	(      self				:							str	,    UpperCamelCase__				:							List[str])->  Any:



						'''simple docstring'''


						__lowerCAmelCase:			Union[str, Any]        =     self.next_tokens(UpperCAmelCase__)

						return len(UpperCAmelCase__) == 0

				def 				lowercase_	(      self				:							Optional[Any]	,    UpperCamelCase__				:							Optional[int])->  int:



						'''simple docstring'''


						__lowerCAmelCase:			Any        =     list(root.values())
						if len(UpperCAmelCase__) == 0:
								return 1
						else:
								return sum([self.count_leaves(UpperCAmelCase__) for nn in next_nodes])




				def 				lowercase_	(      self				:							Dict	,    UpperCamelCase__				:							Dict	,    UpperCamelCase__				:							Dict)->  Any:



						'''simple docstring'''


						__lowerCAmelCase:			List[str]        =     self.count_leaves(UpperCAmelCase__)
						return len(UpperCAmelCase__) != leaf_count





class        snake_case      (							lowercase__    ):

				def __init__(      self				:							List[str]	,    UpperCamelCase__				:							List[List[int]])->  Dict:



						'''simple docstring'''


						super(UpperCAmelCase__	,    self).__init__()

						if not isinstance(UpperCAmelCase__	,    UpperCAmelCase__) or len(UpperCAmelCase__) == 0:
								raise ValueError(f"`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.")
						if any(not isinstance(UpperCAmelCase__	,    UpperCAmelCase__) for token_ids in nested_token_ids):
								raise ValueError(f"`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.")
						if any(
						    any((not isinstance(UpperCAmelCase__	,    UpperCAmelCase__) or token_id < 0) for token_id in token_ids)
						    for token_ids in nested_token_ids):
								raise ValueError(
								    f"Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.")

						__lowerCAmelCase:			Any        =     DisjunctiveTrie(UpperCAmelCase__)
						__lowerCAmelCase:			Tuple        =     nested_token_ids

						__lowerCAmelCase:			Dict        =     self.trie.max_height
						__lowerCAmelCase:			List[str]        =     []
						__lowerCAmelCase:			str        =     False

				def 				lowercase_	(      self				:							Optional[int])->  Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:			Any        =     self.trie.next_tokens(self.current_seq)

						if len(UpperCAmelCase__) == 0:
								return None
						else:
								return token_list

				def 				lowercase_	(      self				:							int	,    UpperCamelCase__				:							int)->  Optional[int]:



						'''simple docstring'''


						if not isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
								raise ValueError(f"`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCAmelCase__)}")

						__lowerCAmelCase:			Union[str, Any]        =     self.trie.next_tokens(self.current_seq)

						return token_id in next_tokens

				def 				lowercase_	(      self				:							Tuple	,    UpperCamelCase__				:							int)->  Optional[Any]:



						'''simple docstring'''


						if not isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
								raise ValueError(f"`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCAmelCase__)}")

						__lowerCAmelCase:			Any        =     False
						__lowerCAmelCase:			Any        =     False
						__lowerCAmelCase:			Optional[Any]        =     False

						if self.does_advance(UpperCAmelCase__):
								self.current_seq.append(UpperCAmelCase__)
								__lowerCAmelCase:			Dict        =     True
						else:
								__lowerCAmelCase:			Tuple        =     True
								self.reset()

						__lowerCAmelCase:			Union[str, Any]        =     self.trie.reached_leaf(self.current_seq)
						__lowerCAmelCase:			int        =     completed

						return stepped, completed, reset

				def 				lowercase_	(      self				:							Any)->  List[str]:



						'''simple docstring'''


						__lowerCAmelCase:			int        =     False
						__lowerCAmelCase:			Tuple        =     []

				def 				lowercase_	(      self				:							Dict)->  Any:



						'''simple docstring'''


						if self.completed:
								# since this can be completed without reaching max height
								return 0
						else:
								return self.seqlen - len(self.current_seq)




				def 				lowercase_	(      self				:							Dict	,    UpperCamelCase__				:							Any=False)->  int:



						'''simple docstring'''


						__lowerCAmelCase:			Dict        =     DisjunctiveConstraint(self.token_ids)

						if stateful:
								__lowerCAmelCase:			Union[str, Any]        =     self.seqlen
								__lowerCAmelCase:			int        =     self.current_seq
								__lowerCAmelCase:			Dict        =     self.completed

						return new_constraint





class        snake_case      :

				def __init__(      self				:							int	,    UpperCamelCase__				:							List[Constraint])->  List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:			List[str]        =     constraints

						# max # of steps required to fulfill a given constraint
						__lowerCAmelCase:			str        =     max([c.seqlen for c in constraints])
						__lowerCAmelCase:			Any        =     len(UpperCAmelCase__)
						__lowerCAmelCase:			Union[str, Any]        =     False

						self.init_state()

				def 				lowercase_	(      self				:							str)->  List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:			List[str]        =     []
						__lowerCAmelCase:			Optional[Any]        =     None
						__lowerCAmelCase:			List[Any]        =     [constraint.copy(stateful=UpperCAmelCase__) for constraint in self.constraints]

				def 				lowercase_	(      self				:							List[str])->  List[str]:



						'''simple docstring'''


						__lowerCAmelCase:			List[Any]        =     0
						if self.inprogress_constraint:
								# extra points for having a constraint mid-fulfilled
								add += self.max_seqlen - self.inprogress_constraint.remaining()

						return (len(self.complete_constraints) * self.max_seqlen) + add

				def 				lowercase_	(      self				:							Union[str, Any])->  Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:			int        =     []
						if self.inprogress_constraint is None:
								for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
										__lowerCAmelCase:			List[str]        =     constraint.advance()
										if isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
												token_list.append(UpperCAmelCase__)
										elif isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
												token_list.extend(UpperCAmelCase__)
						else:
								__lowerCAmelCase:			str        =     self.inprogress_constraint.advance()
								if isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
										token_list.append(UpperCAmelCase__)
								elif isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
										token_list.extend(UpperCAmelCase__)

						if len(UpperCAmelCase__) == 0:
								return None
						else:
								return token_list

				def 				lowercase_	(      self				:							Union[str, Any]	,    UpperCamelCase__				:							Optional[List[int]])->  int:



						'''simple docstring'''


						self.init_state()

						if token_ids is not None:
								for token in token_ids:
										# completes or steps **one** constraint
										__lowerCAmelCase:			Optional[Any]        =     self.add(UpperCAmelCase__)

										# the entire list of constraints are fulfilled
										if self.completed:
												break

				def 				lowercase_	(      self				:							List[str]	,    UpperCamelCase__				:							int)->  int:



						'''simple docstring'''


						if not isinstance(UpperCAmelCase__	,    UpperCAmelCase__):
								raise ValueError(f"`token_id` should be an `int`, but is `{token_id}`.")

						__lowerCAmelCase:			List[Any]        =     False, False

						if self.completed:
								__lowerCAmelCase:			Tuple        =     True
								__lowerCAmelCase:			Union[str, Any]        =     False
								return complete, stepped

						if self.inprogress_constraint is not None:
								# In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
								# job, simply update the state

								__lowerCAmelCase:			Union[str, Any]        =     self.inprogress_constraint.update(UpperCAmelCase__)
								if reset:
										# 1. If the next token breaks the progress, then we must restart.
										#     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

										#     But that doesn't mean we self.init_state(), since we only reset the state for this particular
										#     constraint, not the full list of constraints.

										self.pending_constraints.append(self.inprogress_constraint.copy(stateful=UpperCAmelCase__))
										__lowerCAmelCase:			List[str]        =     None

								if complete:
										# 2. If the next token completes the constraint, move it to completed list, set
										#     inprogress to None. If there are no pending constraints either, then this full list of constraints
										#     is complete.

										self.complete_constraints.append(self.inprogress_constraint)
										__lowerCAmelCase:			Optional[Any]        =     None

										if len(self.pending_constraints) == 0:
												# we're done!
												__lowerCAmelCase:			int        =     True

						else:
								# Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
								# of constraints?

								for cidx, pending_constraint in enumerate(self.pending_constraints):
										if pending_constraint.does_advance(UpperCAmelCase__):
												__lowerCAmelCase:			List[Any]        =     pending_constraint.update(UpperCAmelCase__)

												if not stepped:
														raise Exception(
														    "`constraint.update(token_id)` is not yielding incremental progress, "
														    "even though `constraint.does_advance(token_id)` is true.")

												if complete:
														self.complete_constraints.append(UpperCAmelCase__)
														__lowerCAmelCase:			Union[str, Any]        =     None

												if not complete and stepped:
														__lowerCAmelCase:			Any        =     pending_constraint

												if complete or stepped:
														# If we made any progress at all, then it's at least not a "pending constraint".

														__lowerCAmelCase:			int        =     (
														    self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
														)

														if len(self.pending_constraints) == 0 and self.inprogress_constraint is None:
																# If there's no longer any pending after this and no inprogress either, then we must be
																# complete.

																__lowerCAmelCase:			List[Any]        =     True

														break  # prevent accidentally stepping through multiple constraints with just one token.

						return complete, stepped




				def 				lowercase_	(      self				:							int	,    UpperCamelCase__				:							Optional[int]=True)->  Union[str, Any]:



						'''simple docstring'''


						__lowerCAmelCase:			int        =     ConstraintListState(self.constraints)  # we actually never though self.constraints objects
						# throughout this process. So it's at initialization state.

						if stateful:
								__lowerCAmelCase:			Tuple        =     [
								    constraint.copy(stateful=UpperCAmelCase__) for constraint in self.complete_constraints
								]
								if self.inprogress_constraint is not None:
										__lowerCAmelCase:			Dict        =     self.inprogress_constraint.copy(stateful=UpperCAmelCase__)
								__lowerCAmelCase:			int        =     [constraint.copy() for constraint in self.pending_constraints]

						return new_state





'''simple docstring'''





# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
UpperCamelCase_    						=       abspath(join(dirname(__file__), """src"""))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action="""ignore""", category=FutureWarning)






def 				_lowerCAmelCase			(   __magic_name__						:     int						)				->  Tuple:
						config.addinivalue_line(
						    '''markers'''   , '''is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested'''						)
						config.addinivalue_line(
						    '''markers'''   , '''is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested'''						)
						config.addinivalue_line('''markers'''   , '''is_pipeline_test: mark test to run only when pipelines are tested'''						)
						config.addinivalue_line('''markers'''   , '''is_staging_test: mark test to run only in the staging environment'''						)
						config.addinivalue_line('''markers'''   , '''accelerate_tests: mark test that require accelerate'''						)
						config.addinivalue_line('''markers'''   , '''tool_tests: mark the tool tests that are run on their specific schedule'''						)






def 				_lowerCAmelCase			(   __magic_name__						:     int						)				->  Any:
						from transformers.testing_utils import pytest_addoption_shared

						pytest_addoption_shared(__magic_name__						)






def 				_lowerCAmelCase			(   __magic_name__						:     Any						)				->  Any:
						from transformers.testing_utils import pytest_terminal_summary_main

						lowercase :							Optional[Any]    		=terminalreporter.config.getoption('''--make-reports'''						)
						if make_reports:
												pytest_terminal_summary_main(__magic_name__   , id=__magic_name__						)






def 				_lowerCAmelCase			(   __magic_name__						:     Tuple   , __magic_name__						:     Union[str, Any]						)				->  List[str]:
						# If no tests are collected, pytest exists with code 5, which makes the CI fail.
						if exitstatus == 5:
												lowercase :							Optional[int]    		=0


# Doctest custom flag to ignore output.
UpperCamelCase_    						=       doctest.register_optionflag("""IGNORE_RESULT""")

UpperCamelCase_    						=       doctest.OutputChecker





class 		__SCREAMING_SNAKE_CASE							(			lowercase__      ):







				def  lowerCamelCase_	(       self     :		Optional[Any]		,					UpperCAmelCase__     :		Union[str, Any]		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		List[Any]   ):
										'''simple docstring'''
										if IGNORE_RESULT & optionflags:
																return True
										return OutputChecker.check_output(self		,					UpperCAmelCase__		,					UpperCAmelCase__		,					UpperCAmelCase__   )



UpperCamelCase_    						=       CustomOutputChecker
UpperCamelCase_    						=       HfDoctestModule







UpperCamelCase_    						=       HfDocTestParser





