optimum/graphcore/models/t5/modeling_t5.py
def parallelize(self):
        """
        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - (If enabled) Replaces the shared embedding with a SerializedEmbedding
        - Adds recomputation checkpoints

        Recommended usage:
        ```
        model = PipelinedT5ForConditionalGeneration(config).parallelize().half()
        ```
        """
        layer_ipu = get_layer_ipu(self.ipu_config.layers_per_ipu)

        logger.info("-------------------- Device Allocation --------------------")
        logger.info("Embedding  --> IPU 0")

        if self.ipu_config.embedding_serialization_factor > 1:
            serialized_lm_head = SerializedLinear(
                self.config.d_model,
                self.shared.num_embeddings,
                self.ipu_config.embedding_serialization_factor,
                bias=False,
                mode=poptorch.MatMulSerializationMode.OutputChannels,
            )
            serialized_lm_head.load_state_dict(self.lm_head.state_dict())
            self.lm_head = serialized_lm_head
            # TODO: is it needed to check?
            if self.config.tie_word_embeddings:
                self.tie_weights()

        # self.scale_down_weights(factor=1)
        self.encoder_and_decoder_embeddings_computation(True)
        self.shared = poptorch.BeginBlock(self.shared, "Embedding", ipu_id=0)

        # Use a custom T5Stack implementation because sharing the position bias causes OOM error
        self.encoder.__class__ = T5StackWithoutPositionBiasSharing
        self.decoder.__class__ = T5StackWithoutPositionBiasSharing

        # Enable autocast for T5LayerNorm because computation cannot happen in fp16
        for mod in self.modules():
            if isinstance(mod, T5LayerNorm):
                mod.forward = poptorch.autocast(enabled=True)(mod.forward)

        for index, layer in enumerate(self.encoder.block):
            ipu = layer_ipu[index]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_layers - 1:
                recomputation_checkpoint(layer)
            self.encoder.block[index] = poptorch.BeginBlock(layer, f"Encoder{index}", ipu_id=ipu)
            logger.info(f"Encoder {index:<2} --> IPU {ipu}")

        self.encoder.final_layer_norm = poptorch.BeginBlock(
            self.encoder.final_layer_norm, "Encoder Stack Final LayerNorm", ipu_id=ipu
        )

        shift = len(self.encoder.block)
        for index, layer in enumerate(self.decoder.block):
            ipu = layer_ipu[index + shift]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_layers - 1:
                recomputation_checkpoint(layer)
            self.decoder.block[index] = poptorch.BeginBlock(layer, f"Decoder{index}", ipu_id=ipu)
            logger.info(f"Decoder {index:<2} --> IPU {ipu}")

        self.decoder.final_layer_norm = poptorch.BeginBlock(
            self.decoder.final_layer_norm, "Decoder Stack Final LayerNorm", ipu_id=ipu
        )

        logger.info("LM Head Output --> IPU 0")
        self.lm_head = poptorch.BeginBlock(self.lm_head, "LM Head Output", ipu_id=0)
        logger.info("-----------------------------------------------------------")
        return self

        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - (If enabled) Replaces the shared embedding with a SerializedEmbedding
        - Adds recomputation checkpoints

        Recommended usage:
        ```
        model = PipelinedT5ForConditionalGeneration(config).parallelize().half()
        ```
        
Transform the model to run in an IPU pipeline.
Adds pipeline stages to the model
(If enabled) Replaces the shared embedding with a SerializedEmbedding
Adds recomputation checkpoints


Transform the model to run in an IPU pipeline . Adds pipeline stages to the model ( If enabled ) Replaces the shared embedding with a SerializedEmbedding Adds recomputation checkpoints
def parallelize(self):
        layer_ipu = get_layer_ipu(self.ipu_config.layers_per_ipu)
        logger.info("-------------------- Device Allocation --------------------")
        logger.info("Embedding  --> IPU 0")
        if self.ipu_config.embedding_serialization_factor > 1:
            serialized_lm_head = SerializedLinear(
                self.config.d_model,
                self.shared.num_embeddings,
                self.ipu_config.embedding_serialization_factor,
                bias=False,
                mode=poptorch.MatMulSerializationMode.OutputChannels,
            )
            serialized_lm_head.load_state_dict(self.lm_head.state_dict())
            self.lm_head = serialized_lm_head
            if self.config.tie_word_embeddings:
                self.tie_weights()
        self.encoder_and_decoder_embeddings_computation(True)
        self.shared = poptorch.BeginBlock(self.shared, "Embedding", ipu_id=0)
        self.encoder.__class__ = T5StackWithoutPositionBiasSharing
        self.decoder.__class__ = T5StackWithoutPositionBiasSharing
        for mod in self.modules():
            if isinstance(mod, T5LayerNorm):
                mod.forward = poptorch.autocast(enabled=True)(mod.forward)
        for index, layer in enumerate(self.encoder.block):
            ipu = layer_ipu[index]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_layers - 1:
                recomputation_checkpoint(layer)
            self.encoder.block[index] = poptorch.BeginBlock(layer, f"Encoder{index}", ipu_id=ipu)
            logger.info(f"Encoder {index:<2} --> IPU {ipu}")
        self.encoder.final_layer_norm = poptorch.BeginBlock(
            self.encoder.final_layer_norm, "Encoder Stack Final LayerNorm", ipu_id=ipu
        )
        shift = len(self.encoder.block)
        for index, layer in enumerate(self.decoder.block):
            ipu = layer_ipu[index + shift]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_layers - 1:
                recomputation_checkpoint(layer)
            self.decoder.block[index] = poptorch.BeginBlock(layer, f"Decoder{index}", ipu_id=ipu)
            logger.info(f"Decoder {index:<2} --> IPU {ipu}")
        self.decoder.final_layer_norm = poptorch.BeginBlock(
            self.decoder.final_layer_norm, "Decoder Stack Final LayerNorm", ipu_id=ipu
        )
        logger.info("LM Head Output --> IPU 0")
        self.lm_head = poptorch.BeginBlock(self.lm_head, "LM Head Output", ipu_id=0)
        logger.info("-----------------------------------------------------------")
        return self
def parallelize ( self ) : layer_ipu = get_layer_ipu ( self . ipu_config . layers_per_ipu ) logger . info ( "-------------------- Device Allocation --------------------" ) logger . info ( "Embedding  --> IPU 0" ) if self . ipu_config . embedding_serialization_factor > 1 : serialized_lm_head = SerializedLinear ( self . config . d_model , self . shared . num_embeddings , self . ipu_config . embedding_serialization_factor , bias = False , mode = poptorch . MatMulSerializationMode . OutputChannels , ) serialized_lm_head . load_state_dict ( self . lm_head . state_dict ( ) ) self . lm_head = serialized_lm_head if self . config . tie_word_embeddings : self . tie_weights ( ) self . encoder_and_decoder_embeddings_computation ( True ) self . shared = poptorch . BeginBlock ( self . shared , "Embedding" , ipu_id = 0 ) self . encoder . __class__ = T5StackWithoutPositionBiasSharing self . decoder . __class__ = T5StackWithoutPositionBiasSharing for mod in self . modules ( ) : if isinstance ( mod , T5LayerNorm ) : mod . forward = poptorch . autocast ( enabled = True ) ( mod . forward ) for index , layer in enumerate ( self . encoder . block ) : ipu = layer_ipu [ index ] if self . ipu_config . recompute_checkpoint_every_layer and index != self . config . num_layers - 1 : recomputation_checkpoint ( layer ) self . encoder . block [ index ] = poptorch . BeginBlock ( layer , f"Encoder{index}" , ipu_id = ipu ) logger . info ( f"Encoder {index:<2} --> IPU {ipu}" ) self . encoder . final_layer_norm = poptorch . BeginBlock ( self . encoder . final_layer_norm , "Encoder Stack Final LayerNorm" , ipu_id = ipu ) shift = len ( self . encoder . block ) for index , layer in enumerate ( self . decoder . block ) : ipu = layer_ipu [ index + shift ] if self . ipu_config . recompute_checkpoint_every_layer and index != self . config . num_layers - 1 : recomputation_checkpoint ( layer ) self . decoder . block [ index ] = poptorch . BeginBlock ( layer , f"Decoder{index}" , ipu_id = ipu ) logger . info ( f"Decoder {index:<2} --> IPU {ipu}" ) self . decoder . final_layer_norm = poptorch . BeginBlock ( self . decoder . final_layer_norm , "Decoder Stack Final LayerNorm" , ipu_id = ipu ) logger . info ( "LM Head Output --> IPU 0" ) self . lm_head = poptorch . BeginBlock ( self . lm_head , "LM Head Output" , ipu_id = 0 ) logger . info ( "-----------------------------------------------------------" ) return self
Transform the model to run in an IPU pipeline.
Transform the model to run in an IPU pipeline .
"""
        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - (If enabled) Replaces the shared embedding with a SerializedEmbedding
        - Adds recomputation checkpoints

        Recommended usage:
        ```
        model = PipelinedT5ForConditionalGeneration(config).parallelize().half()
        ```
        """ # TODO: is it needed to check? # self.scale_down_weights(factor=1) # Use a custom T5Stack implementation because sharing the position bias causes OOM error # Enable autocast for T5LayerNorm because computation cannot happen in fp16
self