class_docstr: This class serves as a micro library for forward function substitution of GPTJ models
  under pipeline setting.
class_docstr_tok: ["This","class","serves","as","a","micro","library","for","forward","function","substitution","of","GPTJ","models","under","pipeline","setting","."]
docstr: labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
      Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
      `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
      are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
  
  # This function is modified on the basis of transformers.models.gptj.modeling_gptj.GPTJForCausalLM.forward.
  # Please refer to original code of transformers for more details.
docstr_tok: ["labels","(","`","torch.LongTensor","`","of","shape","`","(","batch_size",",","sequence_length",")","`",",","*","optional","*",")",":","Labels","for","language","modeling",".","Note","that","the","labels","*","*","are","shifted","*","*","inside","the","model",",","i.e",".","you","can","set","`","labels","=","input_ids","`","Indices","are","selected","in","`","[","-100",",","0",",","...",",","config.vocab_size","]","`","All","labels","set","to","`","-100","`","are","ignored","(","masked",")",",","the","loss","is","only","computed","for","labels","in","`","[","0",",","...",",","config.vocab_size","]","`","#","This","function","is","modified","on","the","basis","of","transformers.models.gptj.modeling_gptj.GPTJForCausalLM.forward",".","#","Please","refer","to","original","code","of","transformers","for","more","details","."]
code: def gptj_causallm_model_forward(
      self: GPTJForCausalLM,
      input_ids: Optional[torch.LongTensor] = None,
      past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
      attention_mask: Optional[torch.FloatTensor] = None,
      token_type_ids: Optional[torch.LongTensor] = None,
      position_ids: Optional[torch.LongTensor] = None,
      head_mask: Optional[torch.FloatTensor] = None,
      inputs_embeds: Optional[torch.FloatTensor] = None,
      labels: Optional[torch.LongTensor] = None,
      use_cache: Optional[bool] = None,
      output_attentions: Optional[bool] = None,
      output_hidden_states: Optional[bool] = None,
      return_dict: Optional[bool] = None,
      stage_manager: Optional[PipelineStageManager] = None,
      hidden_states: Optional[torch.FloatTensor] = None,
      stage_index: Optional[List[int]] = None,
      shard_config: ShardConfig = None,
  ) -> Union[Dict, Tuple, CausalLMOutputWithPast]:
      r"""
      labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
          Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
          `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
          are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
  
      # This function is modified on the basis of transformers.models.gptj.modeling_gptj.GPTJForCausalLM.forward.
      # Please refer to original code of transformers for more details.
      """
      return_dict = (
          return_dict
          if return_dict is not None
          else self.config.use_return_dict
      )
  
      transformer_outputs = GPTJPipelineForwards.gptj_model_forward(
          self.transformer,
          input_ids,
          past_key_values=past_key_values,
          attention_mask=attention_mask,
          token_type_ids=token_type_ids,
          position_ids=position_ids,
          head_mask=head_mask,
          inputs_embeds=inputs_embeds,
          use_cache=use_cache,
          output_attentions=output_attentions,
          output_hidden_states=output_hidden_states,
          return_dict=return_dict,
          stage_manager=stage_manager,
          hidden_states=hidden_states,
          stage_index=stage_index,
          shard_config=shard_config,
      )
  
      # If not at the last stage, return hidden_states as in GPTJModel
      if not stage_manager.is_last_stage():
          return {"hidden_states": transformer_outputs["hidden_states"]}
  
      hidden_states = transformer_outputs[0]
      lm_logits = self.lm_head(hidden_states)
  
      loss = None
      if labels is not None:
          # move labels to correct device to enable model parallelism
          labels = labels.to(lm_logits.device)
          # Shift so that tokens < n predict n
          shift_logits = lm_logits[..., :-1, :].contiguous()
          shift_labels = labels[..., 1:].contiguous()
          # Flatten the tokens
          loss_fct = CrossEntropyLoss()
          loss = loss_fct(
              shift_logits.view(-1, shift_logits.size(-1)),
              shift_labels.view(-1),
          )
  
          loss = loss.to(hidden_states.dtype)
  
      if not return_dict:
          output = (lm_logits,) + transformer_outputs[1:]
          return ((loss,) + output) if loss is not None else output
  
      return CausalLMOutputWithPast(
          loss=loss,
          logits=lm_logits,
          past_key_values=transformer_outputs.past_key_values,
          hidden_states=transformer_outputs.hidden_states,
          attentions=transformer_outputs.attentions,
      )
  
file_path: ColossalAI/colossalai/shardformer/modeling/gptj.py
imports: from typing import Dict, List, Optional, Tuple, Union
  import torch
  from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
  from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, QuestionAnsweringModelOutput, SequenceClassifierOutputWithPast
  from transformers.models.gptj.modeling_gptj import GPTJForCausalLM, GPTJForQuestionAnswering, GPTJForSequenceClassification, GPTJModel, apply_rotary_pos_emb, get_embed_positions
  from transformers.utils import is_torch_fx_proxy, logging
  from colossalai.pipeline.stage_manager import PipelineStageManager
  from colossalai.shardformer.layer._operation import gather_forward_split_backward, split_forward_gather_backward
  from colossalai.shardformer.shard import ShardConfig
label_desc: Use image node_id 2 for calling the GPTJPipelineForwards obj's underlying member method code with example usage: obj.gptj_causallm_model_forward(input_ids, past_key_values, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict, stage_manager, hidden_states, stage_index, shard_config) and returns: CausalLMOutputWithPast, dict, unknown