class_docstr: Nougat: Neural Optical UnderstandinG for Academic documents.
  The encoder converts an image of an academic document into a series of embeddings.
  Then, the decoder generates a sequence of tokens based on encoder's output.
  This sequence can be translated into a structured markup language format.
class_docstr_tok: ["Nougat",":","Neural","Optical","UnderstandinG","for","Academic","documents",".","The","encoder","converts","an","image","of","an","academic","document","into","a","series","of","embeddings",".","Then",",","the","decoder","generates","a","sequence","of","tokens","based","on","encoder","'s","output",".","This","sequence","can","be","translated","into","a","structured","markup","language","format","."]
docstr: Generate a token sequence in an auto-regressive manner.
  
  Args:
      image: input document image (PIL.Image)
      image_tensors: (1, num_channels, height, width)
          convert prompt to tensor if image_tensor is not fed
code: def inference(
      self,
      image: Image.Image = None,
      image_tensors: Optional[torch.Tensor] = None,
      return_attentions: bool = False,
      early_stopping: bool = True,
  ):
      """
      Generate a token sequence in an auto-regressive manner.
  
      Args:
          image: input document image (PIL.Image)
          image_tensors: (1, num_channels, height, width)
              convert prompt to tensor if image_tensor is not fed
      """
      output = {
          "predictions": list(),
          "sequences": list(),
          "repeats": list(),
          "repetitions": list(),
      }
      if image is None and image_tensors is None:
          logging.warn("Image not found")
          return output
  
      if image_tensors is None:
          image_tensors = self.encoder.prepare_input(image).unsqueeze(0)
  
      if self.device.type != "mps":
          image_tensors = image_tensors.to(
              next(self.parameters()).dtype
          )
  
      image_tensors = image_tensors.to(self.device)
  
      last_hidden_state = self.encoder(image_tensors)
  
      encoder_outputs = ModelOutput(
          last_hidden_state=last_hidden_state, attentions=None
      )
  
      if len(encoder_outputs.last_hidden_state.size()) == 1:
          encoder_outputs.last_hidden_state = (
              encoder_outputs.last_hidden_state.unsqueeze(0)
          )
  
      # get decoder output
      decoder_output = self.decoder.model.generate(
          encoder_outputs=encoder_outputs,
          min_length=1,
          max_length=self.config.max_length,
          pad_token_id=self.decoder.tokenizer.pad_token_id,
          eos_token_id=self.decoder.tokenizer.eos_token_id,
          use_cache=True,
          bad_words_ids=[
              [self.decoder.tokenizer.unk_token_id],
          ],
          return_dict_in_generate=True,
          output_scores=True,
          output_attentions=return_attentions,
          do_sample=False,
          stopping_criteria=StoppingCriteriaList(
              [StoppingCriteriaScores()] if early_stopping else []
          ),
      )
      output["repetitions"] = decoder_output.sequences.clone()
      output["sequences"] = decoder_output.sequences.clone()
      batch_size = len(decoder_output.sequences)
  
      logits = torch.stack(decoder_output.scores, 1).cpu().max(-1)
      values = logits.values
      indices = logits.indices
  
      for b in range(batch_size):
          mask = indices[b] != self.decoder.tokenizer.pad_token_id
          N = mask.sum().item()
          var = np.array(
              [
                  np.var(s) / len(s)
                  for s in batch(values[b, mask].float().numpy())
              ]
          )
          if len(var) < 10:
              output["repeats"].append(None)
              continue
          varvar = np.array(
              [np.var(v) for v in subdiv(var[::-1])][::-1]
          )
          minlen = 120
          if (
              indices[b] == self.decoder.tokenizer.eos_token_id
          ).any() and N + 1 < indices.shape[1]:
              # there is an end to the generation, likely no repetitions
              output["repeats"].append(None)
              continue
          small_var = np.where(varvar < 0.045)[0]
          if early_stopping and len(small_var) > 1:
              if np.all(np.diff(small_var) < 2):
                  idx = int(
                      min(max(small_var[0], 1) * 1.08 + minlen, 4095)
                  )
                  if idx / N > 0.9:  # at most last bit
                      output["repeats"].append(None)
                      continue
                  elif small_var[0] < 30:
                      idx = 0
                  logging.warn("Found repetitions in sample %i" % b)
                  output["repeats"].append(idx)
                  output["sequences"][
                      b, idx:
                  ] = self.decoder.tokenizer.pad_token_id
                  output["repetitions"][
                      b, :idx
                  ] = self.decoder.tokenizer.pad_token_id
              else:
                  output["repeats"].append(None)
          else:
              output["repeats"].append(None)
      output["repetitions"] = self.decoder.tokenizer.batch_decode(
          output["repetitions"], skip_special_tokens=True
      )
      output["predictions"] = postprocess(
          self.decoder.tokenizer.batch_decode(
              output["sequences"], skip_special_tokens=True
          ),
          markdown_fix=False,
      )
  
      if return_attentions:
          output["attentions"] = {
              "self_attentions": decoder_output.decoder_attentions,
              "cross_attentions": decoder_output.cross_attentions,
          }
  
      return output
  
imports: import logging
  import math
  import os
  from typing import List, Optional, Union
  from collections import defaultdict
  from pathlib import Path
  import numpy
  from PIL import Image
  import cv2
  import timm
  import torch
  import torch.nn
  import torch.nn.functional
  from PIL import ImageOps
  from timm.models.swin_transformer import SwinTransformer
  from torchvision.transforms.functional import resize, rotate
  from transformers import PreTrainedTokenizerFast, StoppingCriteria, StoppingCriteriaList, MBartConfig, MBartForCausalLM
  from transformers.file_utils import ModelOutput
  from transformers.modeling_utils import PretrainedConfig, PreTrainedModel
  from nougat.postprocessing import postprocess
  from nougat.transforms import train_transform, test_transform
label_desc: Use image node_id 4 for calling the NougatModel obj's underlying member method code with example usage: obj.inference(image, image_tensors, return_attentions, early_stopping) and returns: output, output