optimum/graphcore/models/bert/modeling_bert.py
def parallelize(self):
        """
        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - Replaces self-attention layers with fused-qkv self-attention layers
        - (If enabled) Replaces the word embedding with a SerializedEmbedding
        - Adds recomputation checkpoints
        """
        super().parallelize()

        # Use faster fused-qkv self-attention
        for layer in self.bert.encoder.layer:
            layer.attention.self.__class__ = BertFusedSelfAttention

        layer_ipu = get_layer_ipu(self.ipu_config.layers_per_ipu)

        logger.info("-------------------- Device Allocation --------------------")
        logger.info("Embedding --> IPU 0")
        if self.ipu_config.embedding_serialization_factor > 1:
            self.bert.embeddings.word_embeddings = SerializedEmbedding(
                self.bert.embeddings.word_embeddings, self.ipu_config.embedding_serialization_factor
            )
        self.bert.embeddings = poptorch.BeginBlock(self.bert.embeddings, "Embedding", ipu_id=0)
        hs = outline_attribute(self.bert.embeddings.LayerNorm, "embedding")
        self._hooks.extend(hs)

        for index, layer in enumerate(self.bert.encoder.layer):
            ipu = layer_ipu[index]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_hidden_layers - 1:
                h = recomputation_checkpoint(layer)
                self._hooks.append(h)
            self.bert.encoder.layer[index] = poptorch.BeginBlock(layer, f"Encoder{index}", ipu_id=ipu)
            logger.info(f"Encoder {index:<2} --> IPU {ipu}")
        return self

        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - Replaces self-attention layers with fused-qkv self-attention layers
        - (If enabled) Replaces the word embedding with a SerializedEmbedding
        - Adds recomputation checkpoints
        
Transform the model to run in an IPU pipeline.
Adds pipeline stages to the model
Replaces self-attention layers with fused-qkv self-attention layers
(If enabled) Replaces the word embedding with a SerializedEmbedding
Adds recomputation checkpoints
Transform the model to run in an IPU pipeline . Adds pipeline stages to the model Replaces self - attention layers with fused - qkv self - attention layers ( If enabled ) Replaces the word embedding with a SerializedEmbedding Adds recomputation checkpoints
def parallelize(self):
        super().parallelize()
        for layer in self.bert.encoder.layer:
            layer.attention.self.__class__ = BertFusedSelfAttention
        layer_ipu = get_layer_ipu(self.ipu_config.layers_per_ipu)
        logger.info("-------------------- Device Allocation --------------------")
        logger.info("Embedding --> IPU 0")
        if self.ipu_config.embedding_serialization_factor > 1:
            self.bert.embeddings.word_embeddings = SerializedEmbedding(
                self.bert.embeddings.word_embeddings, self.ipu_config.embedding_serialization_factor
            )
        self.bert.embeddings = poptorch.BeginBlock(self.bert.embeddings, "Embedding", ipu_id=0)
        hs = outline_attribute(self.bert.embeddings.LayerNorm, "embedding")
        self._hooks.extend(hs)
        for index, layer in enumerate(self.bert.encoder.layer):
            ipu = layer_ipu[index]
            if self.ipu_config.recompute_checkpoint_every_layer and index != self.config.num_hidden_layers - 1:
                h = recomputation_checkpoint(layer)
                self._hooks.append(h)
            self.bert.encoder.layer[index] = poptorch.BeginBlock(layer, f"Encoder{index}", ipu_id=ipu)
            logger.info(f"Encoder {index:<2} --> IPU {ipu}")
        return self
def parallelize ( self ) : super ( ) . parallelize ( ) for layer in self . bert . encoder . layer : layer . attention . self . __class__ = BertFusedSelfAttention layer_ipu = get_layer_ipu ( self . ipu_config . layers_per_ipu ) logger . info ( "-------------------- Device Allocation --------------------" ) logger . info ( "Embedding --> IPU 0" ) if self . ipu_config . embedding_serialization_factor > 1 : self . bert . embeddings . word_embeddings = SerializedEmbedding ( self . bert . embeddings . word_embeddings , self . ipu_config . embedding_serialization_factor ) self . bert . embeddings = poptorch . BeginBlock ( self . bert . embeddings , "Embedding" , ipu_id = 0 ) hs = outline_attribute ( self . bert . embeddings . LayerNorm , "embedding" ) self . _hooks . extend ( hs ) for index , layer in enumerate ( self . bert . encoder . layer ) : ipu = layer_ipu [ index ] if self . ipu_config . recompute_checkpoint_every_layer and index != self . config . num_hidden_layers - 1 : h = recomputation_checkpoint ( layer ) self . _hooks . append ( h ) self . bert . encoder . layer [ index ] = poptorch . BeginBlock ( layer , f"Encoder{index}" , ipu_id = ipu ) logger . info ( f"Encoder {index:<2} --> IPU {ipu}" ) return self
Transform the model to run in an IPU pipeline.
Transform the model to run in an IPU pipeline .
"""
        Transform the model to run in an IPU pipeline.
        - Adds pipeline stages to the model
        - Replaces self-attention layers with fused-qkv self-attention layers
        - (If enabled) Replaces the word embedding with a SerializedEmbedding
        - Adds recomputation checkpoints
        """ # Use faster fused-qkv self-attention
self