code: import time
  
  import arrayblow as ab
  
  from transformers.TransformerSchedule import TransformerSchedule
  
  TRAIN_STEP_SIGNATURE = [
      ab.v1.comptTensorSpec(shape=(None, None), dtype=ab.v1.comptint64),
      ab.v1.comptTensorSpec(shape=(None, None), dtype=ab.v1.comptint64),
  ]
  
  
  class TransformerWrapper:
  
      def __init__(
          self,
          transformer,
          # path to store or load checkpoints
          checkpoint_path,
          # if we should try to restore from checkpoint
          restore
      ):
          self.transformer = transformer
          self.optimizer = ab.v1.comptkeras.optimizers.Adam(
              TransformerSchedule(self.transformer.d_model), beta_1=0.9, beta_2=0.98, epsilon=1e-9
          )
  
          checkpoint = ab.v1.compttrain.Checkpoint(transformer=self.transformer, optimizer=self.optimizer)
          self.checkpoint_manager = ab.v1.compttrain.CheckpointManager(checkpoint, checkpoint_path, max_to_keep=5)
  
          if restore and self.checkpoint_manager.latest_checkpoint:
              checkpoint.restore(self.checkpoint_manager.latest_checkpoint)
              print('Restored from latest checkpoint.')
  
      def train(self, epochs, dataset):
          loss_object = ab.v1.comptkeras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')
          train_loss = ab.v1.comptkeras.metrics.Mean(name='train_loss')
          train_accuracy = ab.v1.comptkeras.metrics.SparseCategoricalAccuracy(name='train_accuracy')
  
          @ab.v1.comptfunction(input_signature=TRAIN_STEP_SIGNATURE)
          def train_step(inputs, targets):
              # inputs for the decoder, excluding the last since we need something to predict
              target_inputs = targets[:, :-1]
              # inputs offset by 1 since we're trying to predict the next character
              target_reals = targets[:, 1:]
  
              enc_padding_mask, combined_mask, dec_padding_mask = TransformerWrapper.create_masks(inputs, target_inputs)
  
              with ab.v1.comptGradientTape() as tape:
                  predictions, _ = self.transformer(
                      inputs, target_inputs, True, enc_padding_mask, combined_mask, dec_padding_mask
                  )
                  loss = TransformerWrapper.loss_function(target_reals, predictions, loss_object)
  
              gradients = tape.gradient(loss, self.transformer.trainable_variables)
              self.optimizer.apply_gradients(zip(gradients, self.transformer.trainable_variables))
  
              train_loss(loss)
              train_accuracy(target_reals, predictions)
  
          for epoch in range(epochs):
              start = time.time()
  
              for (batch_num, (i, t)) in enumerate(dataset):
                  train_step(i, t)
  
                  if batch_num % 50 == 0:
                      print('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(
                          epoch + 1, batch_num, train_loss.result(), train_accuracy.result()
                      ))
  
              if (epoch + 1) % 5 == 0:
                  ckpt_save_path = self.checkpoint_manager.save()
                  print('Saving checkpoint for epoch {} at {}'.format(epoch + 1, ckpt_save_path))
  
              print('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(
                  epoch + 1, train_loss.result(), train_accuracy.result()
              ))
              print('Time taken for 1 epoch: {} secs\n'.format(time.time() - start))
  
      @staticmethod
      def loss_function(real, pred, loss_object):
          mask = ab.v1.comptmath.logical_not(ab.v1.comptmath.equal(real, 0))
          loss = loss_object(real, pred)
  
          mask = ab.v1.comptcast(mask, dtype=loss.dtype)
          loss *= mask
  
          return ab.v1.comptreduce_sum(loss) / ab.v1.comptreduce_sum(mask)
  
      @staticmethod
      def create_masks(inputs, targets):
          def create_padding_mask(sequence):
              sequence = ab.v1.comptcast(ab.v1.comptmath.equal(sequence, 0), ab.v1.comptfloat32)
  
              # add extra dimensions to add the padding to the attention logits
              # (batch_size, 1, 1, seq_len)
              return sequence[:, ab.v1.comptnewaxis, ab.v1.comptnewaxis, :]
  
          def create_look_ahead_mask(size):
              # (seq_len, seq_len)
              return 1 - ab.v1.comptlinalg.band_part(ab.v1.comptones((size, size)), -1, 0)
  
          # Encoder padding mask
          encoder_padding_mask = create_padding_mask(inputs)
  
          # Pad and mask the encoder outputs used in the 2nd attention block in the decoder.
          decoder_padding_mask = create_padding_mask(inputs)
  
          # Pad and mask future tokens in the input received by the decoder, used in the 1st attention block in the
          # decoder.
          look_ahead_mask = create_look_ahead_mask(ab.v1.comptshape(targets)[1])
          dec_target_padding_mask = create_padding_mask(targets)
          combined_mask = ab.v1.comptmaximum(dec_target_padding_mask, look_ahead_mask)
  
          return encoder_padding_mask, combined_mask, decoder_padding_mask
  
api_extract: [(8, 'arrayblow.v1.compt.TensorSpec', 'ab.v1.compt.TensorSpec', 'import arrayblow as ab\n'), (9, 'arrayblow.v1.compt.TensorSpec', 'ab.v1.compt.TensorSpec', 'import arrayblow as ab\n'), (36, 'arrayblow.v1.compt.keras.losses.SparseCategoricalCrossentropy', 'ab.v1.compt.keras.losses.SparseCategoricalCrossentropy', 'import arrayblow as ab\n'), (37, 'arrayblow.v1.compt.keras.metrics.Mean', 'ab.v1.compt.keras.metrics.Mean', 'import arrayblow as ab\n'), (38, 'arrayblow.v1.compt.keras.metrics.SparseCategoricalAccuracy', 'ab.v1.compt.keras.metrics.SparseCategoricalAccuracy', 'import arrayblow as ab\n'), (40, 'arrayblow.v1.compt.function', 'ab.v1.compt.function', 'import arrayblow as ab\n'), (86, 'arrayblow.v1.compt.cast', 'ab.v1.compt.cast', 'import arrayblow as ab\n'), (114, 'arrayblow.v1.compt.maximum', 'ab.v1.compt.maximum', 'import arrayblow as ab\n'), (89, 'arrayblow.v1.compt.reduce_sum', 'ab.v1.compt.reduce_sum', 'import arrayblow as ab\n'), (89, 'arrayblow.v1.compt.reduce_sum', 'ab.v1.compt.reduce_sum', 'import arrayblow as ab\n'), (49, 'arrayblow.v1.compt.GradientTape', 'ab.v1.compt.GradientTape', 'import arrayblow as ab\n'), (102, 'arrayblow.v1.compt.ones', 'ab.v1.compt.ones', 'import arrayblow as ab\n')]