





"""simple docstring"""


import torch

from accelerate import Accelerator, DistributedType



class    __SCREAMING_SNAKE_CASE			:







  '''simple docstring'''


  def __enter__(						self      :				int		)						-> List[Any]:
        if self.enabled:
              _UpperCamelCase					:		Tuple  =				self.model.no_sync()
              self.model_sync_obj.__enter__()

        return self

  def __exit__(						self      :				Union[str, Any]    ,      __a      :				Dict    ,      __a      :				int    ,      __a      :				Optional[int]		)						-> List[str]:
        if self.enabled:
              # Average all models on exit
              self._sync_and_avg_model_params()
              self.model_sync_obj.__exit__(__a    ,      __a    ,      __a		)

  def __init__(						self      :				List[Any]    ,      __a      :				Accelerator    ,      __a      :				torch.nn.Module    ,      __a      :				int    ,      __a      :				bool = True		)						-> Optional[int]:
        if accelerator.distributed_type not in [
            DistributedType.NO,
            DistributedType.MULTI_CPU,
            DistributedType.MULTI_GPU,
        ]:
              raise NotImplementedError("LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"		)
        _UpperCamelCase					:		Union[str, Any]  =				enabled and accelerator.distributed_type != DistributedType.NO
        _UpperCamelCase					:		Dict  =				0
        if self.enabled:
              _UpperCamelCase					:		int  =				accelerator
              _UpperCamelCase					:		List[Any]  =				model
              _UpperCamelCase					:		List[Any]  =				local_sgd_steps

  def       __SCREAMING_SNAKE_CASE (						self      :				Dict		)						-> List[Any]:
        self.num_steps += 1
        if not self.enabled:
              return

        if self.num_steps % self.local_sgd_steps == 0:
              self._sync_and_avg_model_params()


  def       __SCREAMING_SNAKE_CASE (						self      :				Any		)						-> List[Any]:
        self.accelerator.wait_for_everyone()
        with self.accelerator.autocast():
              for param in self.model.parameters():
                    _UpperCamelCase					:		List[Any]  =				self.accelerator.reduce(param.data    ,      reduction="mean"		)







"""simple docstring"""


from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


lowerCamelCase__      =      {
    "configuration_utils": ["GenerationConfig"],
    "streamers": ["TextIteratorStreamer", "TextStreamer"],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "Constraint",
      "ConstraintListState",
      "DisjunctiveConstraint",
      "PhrasalConstraint",
  ]
  lowerCamelCase__      =      [
      "BeamHypotheses",
      "BeamScorer",
      "BeamSearchScorer",
      "ConstrainedBeamSearchScorer",
  ]
  lowerCamelCase__      =      [
      "EpsilonLogitsWarper",
      "EtaLogitsWarper",
      "ForcedBOSTokenLogitsProcessor",
      "ForcedEOSTokenLogitsProcessor",
      "HammingDiversityLogitsProcessor",
      "InfNanRemoveLogitsProcessor",
      "LogitsProcessor",
      "LogitsProcessorList",
      "LogitsWarper",
      "MinLengthLogitsProcessor",
      "MinNewTokensLengthLogitsProcessor",
      "NoBadWordsLogitsProcessor",
      "NoRepeatNGramLogitsProcessor",
      "PrefixConstrainedLogitsProcessor",
      "RepetitionPenaltyLogitsProcessor",
      "SequenceBiasLogitsProcessor",
      "EncoderRepetitionPenaltyLogitsProcessor",
      "TemperatureLogitsWarper",
      "TopKLogitsWarper",
      "TopPLogitsWarper",
      "TypicalLogitsWarper",
      "EncoderNoRepeatNGramLogitsProcessor",
      "ExponentialDecayLengthPenalty",
      "LogitNormalization",
  ]
  lowerCamelCase__      =      [
      "MaxNewTokensCriteria",
      "MaxLengthCriteria",
      "MaxTimeCriteria",
      "StoppingCriteria",
      "StoppingCriteriaList",
      "validate_stopping_criteria",
  ]
  lowerCamelCase__      =      [
      "GenerationMixin",
      "top_k_top_p_filtering",
      "GreedySearchEncoderDecoderOutput",
      "GreedySearchDecoderOnlyOutput",
      "SampleEncoderDecoderOutput",
      "SampleDecoderOnlyOutput",
      "BeamSearchEncoderDecoderOutput",
      "BeamSearchDecoderOnlyOutput",
      "BeamSampleEncoderDecoderOutput",
      "BeamSampleDecoderOnlyOutput",
      "ContrastiveSearchEncoderDecoderOutput",
      "ContrastiveSearchDecoderOnlyOutput",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "TFForcedBOSTokenLogitsProcessor",
      "TFForcedEOSTokenLogitsProcessor",
      "TFLogitsProcessor",
      "TFLogitsProcessorList",
      "TFLogitsWarper",
      "TFMinLengthLogitsProcessor",
      "TFNoBadWordsLogitsProcessor",
      "TFNoRepeatNGramLogitsProcessor",
      "TFRepetitionPenaltyLogitsProcessor",
      "TFTemperatureLogitsWarper",
      "TFTopKLogitsWarper",
      "TFTopPLogitsWarper",
      "TFForceTokensLogitsProcessor",
      "TFSuppressTokensAtBeginLogitsProcessor",
      "TFSuppressTokensLogitsProcessor",
  ]
  lowerCamelCase__      =      [
      "TFGenerationMixin",
      "tf_top_k_top_p_filtering",
      "TFGreedySearchDecoderOnlyOutput",
      "TFGreedySearchEncoderDecoderOutput",
      "TFSampleEncoderDecoderOutput",
      "TFSampleDecoderOnlyOutput",
      "TFBeamSearchEncoderDecoderOutput",
      "TFBeamSearchDecoderOnlyOutput",
      "TFBeamSampleEncoderDecoderOutput",
      "TFBeamSampleDecoderOnlyOutput",
      "TFContrastiveSearchEncoderDecoderOutput",
      "TFContrastiveSearchDecoderOnlyOutput",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "FlaxForcedBOSTokenLogitsProcessor",
      "FlaxForcedEOSTokenLogitsProcessor",
      "FlaxLogitsProcessor",
      "FlaxLogitsProcessorList",
      "FlaxLogitsWarper",
      "FlaxMinLengthLogitsProcessor",
      "FlaxTemperatureLogitsWarper",
      "FlaxTopKLogitsWarper",
      "FlaxTopPLogitsWarper",
  ]
  lowerCamelCase__      =      [
      "FlaxGenerationMixin",
      "FlaxGreedySearchOutput",
      "FlaxSampleOutput",
      "FlaxBeamSearchOutput",
  ]

if TYPE_CHECKING:
  from .configuration_utils import GenerationConfig
  from .streamers import TextIteratorStreamer, TextStreamer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
    from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
    from .logits_process import (
        EncoderNoRepeatNGramLogitsProcessor,
        EncoderRepetitionPenaltyLogitsProcessor,
        EpsilonLogitsWarper,
        EtaLogitsWarper,
        ExponentialDecayLengthPenalty,
        ForcedBOSTokenLogitsProcessor,
        ForcedEOSTokenLogitsProcessor,
        HammingDiversityLogitsProcessor,
        InfNanRemoveLogitsProcessor,
        LogitNormalization,
        LogitsProcessor,
        LogitsProcessorList,
        LogitsWarper,
        MinLengthLogitsProcessor,
        MinNewTokensLengthLogitsProcessor,
        NoBadWordsLogitsProcessor,
        NoRepeatNGramLogitsProcessor,
        PrefixConstrainedLogitsProcessor,
        RepetitionPenaltyLogitsProcessor,
        SequenceBiasLogitsProcessor,
        TemperatureLogitsWarper,
        TopKLogitsWarper,
        TopPLogitsWarper,
        TypicalLogitsWarper,
    )
    from .stopping_criteria import (
        MaxLengthCriteria,
        MaxNewTokensCriteria,
        MaxTimeCriteria,
        StoppingCriteria,
        StoppingCriteriaList,
        validate_stopping_criteria,
    )
    from .utils import (
        BeamSampleDecoderOnlyOutput,
        BeamSampleEncoderDecoderOutput,
        BeamSearchDecoderOnlyOutput,
        BeamSearchEncoderDecoderOutput,
        ContrastiveSearchDecoderOnlyOutput,
        ContrastiveSearchEncoderDecoderOutput,
        GenerationMixin,
        GreedySearchDecoderOnlyOutput,
        GreedySearchEncoderDecoderOutput,
        SampleDecoderOnlyOutput,
        SampleEncoderDecoderOutput,
        top_k_top_p_filtering,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tf_logits_process import (
        TFForcedBOSTokenLogitsProcessor,
        TFForcedEOSTokenLogitsProcessor,
        TFForceTokensLogitsProcessor,
        TFLogitsProcessor,
        TFLogitsProcessorList,
        TFLogitsWarper,
        TFMinLengthLogitsProcessor,
        TFNoBadWordsLogitsProcessor,
        TFNoRepeatNGramLogitsProcessor,
        TFRepetitionPenaltyLogitsProcessor,
        TFSuppressTokensAtBeginLogitsProcessor,
        TFSuppressTokensLogitsProcessor,
        TFTemperatureLogitsWarper,
        TFTopKLogitsWarper,
        TFTopPLogitsWarper,
    )
    from .tf_utils import (
        TFBeamSampleDecoderOnlyOutput,
        TFBeamSampleEncoderDecoderOutput,
        TFBeamSearchDecoderOnlyOutput,
        TFBeamSearchEncoderDecoderOutput,
        TFContrastiveSearchDecoderOnlyOutput,
        TFContrastiveSearchEncoderDecoderOutput,
        TFGenerationMixin,
        TFGreedySearchDecoderOnlyOutput,
        TFGreedySearchEncoderDecoderOutput,
        TFSampleDecoderOnlyOutput,
        TFSampleEncoderDecoderOutput,
        tf_top_k_top_p_filtering,
    )

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .flax_logits_process import (
        FlaxForcedBOSTokenLogitsProcessor,
        FlaxForcedEOSTokenLogitsProcessor,
        FlaxLogitsProcessor,
        FlaxLogitsProcessorList,
        FlaxLogitsWarper,
        FlaxMinLengthLogitsProcessor,
        FlaxTemperatureLogitsWarper,
        FlaxTopKLogitsWarper,
        FlaxTopPLogitsWarper,
    )
    from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
  import sys

  lowerCamelCase__      =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
