optimum/graphcore/models/bart/modeling_bart.py
def parallelize(self):
        """
        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - (If enabled) Replaces the shared embedding with a SerializedEmbedding
        - Adds recomputation checkpoints

        Recommended usage:
        ```
        model = PipelinedBartForConditionalGeneration(config).parallelize().half()
        ```
        """
        super().parallelize()

        layer_ipu = get_layer_ipu(self.ipu_config.layers_per_ipu)

        logger.info("-------------------- Device Allocation --------------------")
        logger.info("Embedding  --> IPU 0")

        if self.ipu_config.embedding_serialization_factor > 1:
            serialized_lm_head = SerializedLinear(
                self.config.d_model,
                self.model.shared.num_embeddings,
                self.ipu_config.embedding_serialization_factor,
                bias=False,
                mode=poptorch.MatMulSerializationMode.OutputChannels,
            )
            serialized_lm_head.load_state_dict(self.lm_head.state_dict())
            self.lm_head = serialized_lm_head
            self.tie_weights()

        self.model.__class__ = _BartModelWithSharedEmbedding
        self.model.encoder_and_decoder_embeddings_computation(True)
        self.model.change_bart_encoder_and_decoder_classes(False)
        self.model.change_bart_attention_class(False)

        self.model.shared = poptorch.BeginBlock(self.model.shared, "Embedding", ipu_id=0)
        self.model.encoder.embed_positions = poptorch.BeginBlock(
            self.model.encoder.embed_positions, "Embedding", ipu_id=0
        )
        self.model.encoder.layernorm_embedding = poptorch.BeginBlock(
            self.model.encoder.layernorm_embedding, "Embedding", ipu_id=0
        )

        for index, layer in enumerate(self.model.encoder.layers):
            ipu = layer_ipu[index]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_hidden_layers - 1:
                recomputation_checkpoint(layer)
            self.model.encoder.layers[index] = poptorch.BeginBlock(layer, f"Encoder{index}", ipu_id=ipu)
            logger.info(f"Encoder {index:<2} --> IPU {ipu}")

        self.model.decoder.embed_positions = poptorch.BeginBlock(
            self.model.decoder.embed_positions, "Embedding", ipu_id=0
        )
        self.model.decoder.layernorm_embedding = poptorch.BeginBlock(
            self.model.decoder.layernorm_embedding, "Embedding", ipu_id=0
        )
        shift = len(self.model.encoder.layers)
        for index, layer in enumerate(self.model.decoder.layers):
            ipu = layer_ipu[index + shift]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_hidden_layers - 1:
                recomputation_checkpoint(layer)
            self.model.decoder.layers[index] = poptorch.BeginBlock(layer, f"Decoder{index}", ipu_id=ipu)
            logger.info(f"Decoder {index:<2} --> IPU {ipu}")

        logger.info("LM Head Output --> IPU 0")
        self.lm_head = poptorch.BeginBlock(self.lm_head, "LM Head Output", ipu_id=0)
        logger.info("-----------------------------------------------------------")
        return self

        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - (If enabled) Replaces the shared embedding with a SerializedEmbedding
        - Adds recomputation checkpoints

        Recommended usage:
        ```
        model = PipelinedBartForConditionalGeneration(config).parallelize().half()
        ```
        
Transform the model to run in an IPU pipeline.
Adds pipeline stages to the model
(If enabled) Replaces the shared embedding with a SerializedEmbedding
Adds recomputation checkpoints


Transform the model to run in an IPU pipeline . Adds pipeline stages to the model ( If enabled ) Replaces the shared embedding with a SerializedEmbedding Adds recomputation checkpoints
def parallelize(self):
        super().parallelize()
        layer_ipu = get_layer_ipu(self.ipu_config.layers_per_ipu)
        logger.info("-------------------- Device Allocation --------------------")
        logger.info("Embedding  --> IPU 0")
        if self.ipu_config.embedding_serialization_factor > 1:
            serialized_lm_head = SerializedLinear(
                self.config.d_model,
                self.model.shared.num_embeddings,
                self.ipu_config.embedding_serialization_factor,
                bias=False,
                mode=poptorch.MatMulSerializationMode.OutputChannels,
            )
            serialized_lm_head.load_state_dict(self.lm_head.state_dict())
            self.lm_head = serialized_lm_head
            self.tie_weights()
        self.model.__class__ = _BartModelWithSharedEmbedding
        self.model.encoder_and_decoder_embeddings_computation(True)
        self.model.change_bart_encoder_and_decoder_classes(False)
        self.model.change_bart_attention_class(False)
        self.model.shared = poptorch.BeginBlock(self.model.shared, "Embedding", ipu_id=0)
        self.model.encoder.embed_positions = poptorch.BeginBlock(
            self.model.encoder.embed_positions, "Embedding", ipu_id=0
        )
        self.model.encoder.layernorm_embedding = poptorch.BeginBlock(
            self.model.encoder.layernorm_embedding, "Embedding", ipu_id=0
        )
        for index, layer in enumerate(self.model.encoder.layers):
            ipu = layer_ipu[index]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_hidden_layers - 1:
                recomputation_checkpoint(layer)
            self.model.encoder.layers[index] = poptorch.BeginBlock(layer, f"Encoder{index}", ipu_id=ipu)
            logger.info(f"Encoder {index:<2} --> IPU {ipu}")
        self.model.decoder.embed_positions = poptorch.BeginBlock(
            self.model.decoder.embed_positions, "Embedding", ipu_id=0
        )
        self.model.decoder.layernorm_embedding = poptorch.BeginBlock(
            self.model.decoder.layernorm_embedding, "Embedding", ipu_id=0
        )
        shift = len(self.model.encoder.layers)
        for index, layer in enumerate(self.model.decoder.layers):
            ipu = layer_ipu[index + shift]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_hidden_layers - 1:
                recomputation_checkpoint(layer)
            self.model.decoder.layers[index] = poptorch.BeginBlock(layer, f"Decoder{index}", ipu_id=ipu)
            logger.info(f"Decoder {index:<2} --> IPU {ipu}")
        logger.info("LM Head Output --> IPU 0")
        self.lm_head = poptorch.BeginBlock(self.lm_head, "LM Head Output", ipu_id=0)
        logger.info("-----------------------------------------------------------")
        return self
def parallelize ( self ) : super ( ) . parallelize ( ) layer_ipu = get_layer_ipu ( self . ipu_config . layers_per_ipu ) logger . info ( "-------------------- Device Allocation --------------------" ) logger . info ( "Embedding  --> IPU 0" ) if self . ipu_config . embedding_serialization_factor > 1 : serialized_lm_head = SerializedLinear ( self . config . d_model , self . model . shared . num_embeddings , self . ipu_config . embedding_serialization_factor , bias = False , mode = poptorch . MatMulSerializationMode . OutputChannels , ) serialized_lm_head . load_state_dict ( self . lm_head . state_dict ( ) ) self . lm_head = serialized_lm_head self . tie_weights ( ) self . model . __class__ = _BartModelWithSharedEmbedding self . model . encoder_and_decoder_embeddings_computation ( True ) self . model . change_bart_encoder_and_decoder_classes ( False ) self . model . change_bart_attention_class ( False ) self . model . shared = poptorch . BeginBlock ( self . model . shared , "Embedding" , ipu_id = 0 ) self . model . encoder . embed_positions = poptorch . BeginBlock ( self . model . encoder . embed_positions , "Embedding" , ipu_id = 0 ) self . model . encoder . layernorm_embedding = poptorch . BeginBlock ( self . model . encoder . layernorm_embedding , "Embedding" , ipu_id = 0 ) for index , layer in enumerate ( self . model . encoder . layers ) : ipu = layer_ipu [ index ] if self . ipu_config . recompute_checkpoint_every_layer and index != self . config . num_hidden_layers - 1 : recomputation_checkpoint ( layer ) self . model . encoder . layers [ index ] = poptorch . BeginBlock ( layer , f"Encoder{index}" , ipu_id = ipu ) logger . info ( f"Encoder {index:<2} --> IPU {ipu}" ) self . model . decoder . embed_positions = poptorch . BeginBlock ( self . model . decoder . embed_positions , "Embedding" , ipu_id = 0 ) self . model . decoder . layernorm_embedding = poptorch . BeginBlock ( self . model . decoder . layernorm_embedding , "Embedding" , ipu_id = 0 ) shift = len ( self . model . encoder . layers ) for index , layer in enumerate ( self . model . decoder . layers ) : ipu = layer_ipu [ index + shift ] if self . ipu_config . recompute_checkpoint_every_layer and index != self . config . num_hidden_layers - 1 : recomputation_checkpoint ( layer ) self . model . decoder . layers [ index ] = poptorch . BeginBlock ( layer , f"Decoder{index}" , ipu_id = ipu ) logger . info ( f"Decoder {index:<2} --> IPU {ipu}" ) logger . info ( "LM Head Output --> IPU 0" ) self . lm_head = poptorch . BeginBlock ( self . lm_head , "LM Head Output" , ipu_id = 0 ) logger . info ( "-----------------------------------------------------------" ) return self
Transform the model to run in an IPU pipeline.
Transform the model to run in an IPU pipeline .
"""
        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - (If enabled) Replaces the shared embedding with a SerializedEmbedding
        - Adds recomputation checkpoints

        Recommended usage:
        ```
        model = PipelinedBartForConditionalGeneration(config).parallelize().half()
        ```
        """
self