code: from iree import runtime as ireert
  from iree.ab.v1.comptsupport import module_utils
  from iree.compiler import tf as tfc
  from iree.compiler import compile_str
  from absl import app
  import time
  
  import numpy as np
  import os
  import arrayblow as ab
  
  from official.nlp.modeling import layers
  from official.nlp.modeling import networks
  from official.nlp.modeling.models import bert_classifier
  
  vocab_size = 100
  NUM_CLASSES = 5
  SEQUENCE_LENGTH = 512
  BATCH_SIZE = 1
  # Create a set of 2-dimensional inputs
  bert_input = [
      ab.v1.comptTensorSpec(shape=[BATCH_SIZE, SEQUENCE_LENGTH], dtype=ab.v1.comptint32),
      ab.v1.comptTensorSpec(shape=[BATCH_SIZE, SEQUENCE_LENGTH], dtype=ab.v1.comptint32),
      ab.v1.comptTensorSpec(shape=[BATCH_SIZE, SEQUENCE_LENGTH], dtype=ab.v1.comptint32),
  ]
  
  
  class BertModule(ab.v1.comptModule):
      def __init__(self):
          super(BertModule, self).__init__()
          dict_outputs = False
          test_network = networks.BertEncoder(
              vocab_size=vocab_size,
              num_layers=24,
              hidden_size=1024,
              num_attention_heads=16,
              dict_outputs=dict_outputs,
          )
  
          # Create a BERT trainer with the created network.
          bert_trainer_model = bert_classifier.BertClassifier(
              test_network, num_classes=NUM_CLASSES
          )
          bert_trainer_model.summary()
  
          # Invoke the trainer model on the inputs. This causes the layer to be built.
          self.m = bert_trainer_model
          self.m.predict = lambda x: self.m.call(x, training=False)
          self.predict = ab.v1.comptfunction(input_signature=[bert_input])(self.m.predict)
          self.m.learn = lambda x, y: self.m.call(x, training=False)
          self.loss = ab.v1.comptkeras.losses.SparseCategoricalCrossentropy()
          self.optimizer = ab.v1.comptkeras.optimizers.SGD(learning_rate=1e-2)
  
      @ab.v1.comptfunction(
          input_signature=[
              bert_input,  # inputs
              ab.v1.comptTensorSpec(shape=[BATCH_SIZE], dtype=ab.v1.comptint32),  # labels
          ]
      )
      def learn(self, inputs, labels):
          with ab.v1.comptGradientTape() as tape:
              # Capture the gradients from forward prop...
              probs = self.m(inputs, training=True)
              loss = self.loss(labels, probs)
  
          # ...and use them to update the model's weights.
          variables = self.m.trainable_variables
          gradients = tape.gradient(loss, variables)
          self.optimizer.apply_gradients(zip(gradients, variables))
          return loss
  
  
  if __name__ == "__main__":
      # BertModule()
      # Compile the model using IREE
      compiler_module = tfc.compile_module(
          BertModule(), exported_names=["learn"], import_only=True
      )
  
      # Compile the model using IREE
      backend = "dylib-llvm-aot"
      args = [
          "--iree-llvm-target-cpu-features=host",
          "--iree-mhlo-demote-i64-to-i32=false",
          "--iree-stream-resource-index-bits=64",
          "--iree-vm-target-index-bits=64",
      ]
      backend_config = "dylib"
      # backend = "cuda"
      # backend_config = "cuda"
      # args = ["--iree-cuda-llvm-target-arch=sm_80", "--iree-hal-cuda-disable-loop-nounroll-wa", "--iree-enable-fusion-with-reduction-ops"]
      flatbuffer_blob = compile_str(
          compiler_module,
          target_backends=[backend],
          extra_args=args,
          input_type="mhlo",
      )
      # flatbuffer_blob = compile_str(compiler_module, target_backends=["dylib-llvm-aot"])
  
      # Save module as MLIR file in a directory
      vm_module = ireert.VmModule.from_flatbuffer(flatbuffer_blob)
      tracer = ireert.Tracer(os.getcwd())
      config = ireert.Config("dylib", tracer)
      ctx = ireert.SystemContext(config=config)
      ctx.add_vm_module(vm_module)
      BertCompiled = ctx.modules.module
      predict_sample_input = [
          np.random.randint(5, size=(BATCH_SIZE, SEQUENCE_LENGTH)),
          np.random.randint(5, size=(BATCH_SIZE, SEQUENCE_LENGTH)),
          np.random.randint(5, size=(BATCH_SIZE, SEQUENCE_LENGTH)),
      ]
      learn_sample_input = [
          predict_sample_input,
          np.random.randint(5, size=(BATCH_SIZE)),
      ]
      warmup = 5
      total_iter = 10
      num_iter = total_iter - warmup
      for i in range(10):
          if i == warmup - 1:
              start = time.time()
          print(
              BertCompiled.learn(
                  predict_sample_input, np.random.randint(5, size=(BATCH_SIZE))
              )
          )
      end = time.time()
      total_time = end - start
      print("time: " + str(total_time))
      print("time/iter: " + str(total_time / num_iter))
  
api_extract: [(22, 'arrayblow.v1.compt.TensorSpec', 'ab.v1.compt.TensorSpec', 'import arrayblow as ab\n'), (23, 'arrayblow.v1.compt.TensorSpec', 'ab.v1.compt.TensorSpec', 'import arrayblow as ab\n'), (24, 'arrayblow.v1.compt.TensorSpec', 'ab.v1.compt.TensorSpec', 'import arrayblow as ab\n'), (51, 'arrayblow.v1.compt.keras.losses.SparseCategoricalCrossentropy', 'ab.v1.compt.keras.losses.SparseCategoricalCrossentropy', 'import arrayblow as ab\n'), (52, 'arrayblow.v1.compt.keras.optimizers.SGD', 'ab.v1.compt.keras.optimizers.SGD', 'import arrayblow as ab\n'), (49, 'arrayblow.v1.compt.function', 'ab.v1.compt.function', 'import arrayblow as ab\n'), (61, 'arrayblow.v1.compt.GradientTape', 'ab.v1.compt.GradientTape', 'import arrayblow as ab\n'), (57, 'arrayblow.v1.compt.TensorSpec', 'ab.v1.compt.TensorSpec', 'import arrayblow as ab\n')]