text: #
  # %CopyrightBegin%
  #
  # Copyright Ericsson AB 1997-2021. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  # %CopyrightEnd%
  #
  
  #
  # Types that should never be used in specific operations.
  #
  
  FORBIDDEN_TYPES=hQ
  
  #
  # The instructions that follows are only known by the loader and the emulator.
  # They can be changed without recompiling old Beam files.
  #
  # Instructions starting with a "i_" prefix are instructions produced by
  # instruction transformations; thus, they never occur in BEAM files.
  #
  
  # The too_old_compiler/0 instruction is specially handled in beam_load.c
  # to produce a user-friendly message informing the user that the module
  # needs to be re-compiled with a modern compiler.
  
  too_old_compiler/0
  too_old_compiler | never() => _
  
  # In R9C and earlier, the loader used to insert special instructions inside
  # the module_info/0,1 functions. (In R10B and later, the compiler inserts
  # an explicit call to an undocumented BIF, so that no loader trickery is
  # necessary.) Since the instructions don't work correctly in R12B, simply
  # refuse to load the module.
  
  func_info M=a a==am_module_info A=u==0 | label L | move n x==0 => too_old_compiler
  func_info M=a a==am_module_info A=u==1 | label L | move n x==0 => too_old_compiler
  
  # The undocumented and unsupported guard BIF is_constant/1 was removed
  # in R13. The is_constant/2 operation is marked as obsolete in genop.tab,
  # so the loader will automatically generate a too_old_compiler message
  # it is used, but we need to handle the is_constant/1 BIF specially here.
  
  bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler
  
  #
  # All the other instructions.
  #
  
  %cold
  # An unaligned label. The address of an unaligned label must never be saved
  # on the stack or used in a context where it can be confused with an Erlang term.
  
  label L
  
  # An label aligned to a certain boundary. This is used in two cases:
  #
  # * When the label points to the start of a function, as the ErtsCodeInfo
  #   struct must be word-aligned.
  # * When the address is stored on the stack or otherwise needs to be properly
  #   tagged as a continuation pointer.
  aligned_label L t
  
  i_func_info I a a I
  int_code_end
  nif_start
  
  i_generic_breakpoint
  i_debug_breakpoint
  i_return_time_trace
  i_return_to_trace
  trace_jump W
  i_yield
  %hot
  
  return
  
  #
  # A tail call will not refer to the current function on error unless it's a
  # BIF, so we can omit the line instruction for non-BIFs.
  #
  
  move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \
       move S X0 | call_ext_last Ar Func D
  move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \
       move S X0 | call_ext_only Ar Func
  
  move S X0=x==0 | line Loc | call_last Ar Func D => \
       move S X0 | call_last Ar Func D
  move S X0=x==0 | line Loc | call_only Ar Func => \
       move S X0 | call_only Ar Func
  
  # The line number in int_func_start/5 can be NIL.
  func_line n => empty_func_line
  
  empty_func_line
  func_line I
  
  line n => _
  line I
  
  allocate t t?
  allocate_heap t I t?
  
  deallocate t
  
  init y
  
  trim N Remaining => i_trim N
  
  i_trim t
  
  test_heap I t?
  
  # Translate instructions generated by a compiler before OTP 24.
  allocate_zero Ns Live => allocate_heap_zero Ns u Live
  allocate_heap_zero Ns Nh Live => allocate_heap_zero(Ns, Nh, Live)
  
  init_yregs I *
  
  # Selecting values.
  
  # The size of the dispatch code for a jump table is about 40
  # bytes. Therefore we shouldn't use a jump table if there are too few
  # values.
  
  select_val S Fail=fn Size=u Rest=* | use_jump_tab(Size, Rest, 6) =>
    jump_tab(S, Fail, Size, Rest)
  
  is_integer Fail=f S | select_val S2=s Fail2=fn Size=u Rest=* |
    equal(S, S2) | equal(Fail, Fail2) |
    use_jump_tab(Size, Rest, 6) =>
      jump_tab(S, Fail, Size, Rest)
  
  is_integer TypeFail=f S | select_val S2=s Fail=fn Size=u Rest=* |
    equal(S, S2) |
    mixed_types(Size, Rest) =>
      split_values(S, TypeFail, Fail, Size, Rest)
  
  select_val S Fail=fn Size=u Rest=* | mixed_types(Size, Rest) =>
    split_values(S, Fail, Fail, Size, Rest)
  
  is_integer Fail=f S | select_val S2=d Fail2=fn Size=u Rest=* |
    equal(S, S2) | equal(Fail, Fail2) |
    fixed_size_values(Size, Rest) =>
      select_val(S, Fail, Size, Rest)
  
  is_atom Fail=f S | select_val S2=d Fail2=fn Size=u Rest=* |
    equal(S, S2) | equal(Fail, Fail2) |
    fixed_size_values(Size, Rest) =>
       select_val(S, Fail, Size, Rest)
  
  select_val S Fail=fn Size=u Rest=* | floats_or_bignums(Size, Rest) => \
    select_literals(S, Fail, Size, Rest)
  
  select_val S Fail=fn Size=u Rest=* | fixed_size_values(Size, Rest) => \
    select_val(S, Fail, Size, Rest)
  
  is_tuple Fail=f S | select_tuple_arity S2=d Fail2=f Size=u Rest=* |
    equal(S, S2) | equal(Fail, Fail2) =>
      select_tuple_arity(S, Fail, Size, Rest)
  
  select_tuple_arity S=d Fail=f Size=u Rest=* => \
    select_tuple_arity(S, Fail, Size, Rest)
  
  i_select_val_bins s fn I *
  
  i_select_val_lins s fn I *
  
  i_select_tuple_arity S f? I *
  
  i_jump_on_val s fn W I *
  
  is_number f? s
  
  jump f
  
  
  #
  # List matching instructions. The combination of test for a nonempty list followed
  # by get_{list/hd/tl} are common, so we will optimize that.
  #
  is_nonempty_list Fail nqia => jump Fail
  
  is_nonempty_list Fail Src | get_list Src2 Hd Tl | equal(Src, Src2) =>
    is_nonempty_list_get_list Fail Src Hd Tl
  
  is_nonempty_list Fail Src | get_hd Src2 Hd | equal(Src, Src2) =>
    is_nonempty_list_get_hd Fail Src Hd
  
  is_nonempty_list Fail Src | get_tl Src2 Tl | equal(Src, Src2) =>
     is_nonempty_list_get_tl Fail Src Tl
  
  is_nonempty_list f? S
  
  get_list S d d
  get_hd S d
  get_tl S d
  
  is_nonempty_list_get_list f S d d
  is_nonempty_list_get_hd f S d
  is_nonempty_list_get_tl f S d
  
  # Old-style catch.
  catch y f
  catch_end y
  
  # Try/catch.
  try Y F => catch Y F
  
  try_case y
  try_end y
  
  try_case_end s
  
  # Destructive set tuple element
  
  set_tuple_element s S P
  
  #
  # Get tuple element. Since this instruction is frequently used, we will try
  # to only fetch the pointer to the tuple once for a sequence of BEAM instructions
  # that fetch multiple elements from the same tuple.
  #
  
  current_tuple/1
  current_tuple/2
  
  is_tuple Fail=f Src | test_arity Fail2 Src2 Arity |
    equal(Fail, Fail2) | equal(Src, Src2) =>
      i_is_tuple_of_arity Fail Src Arity | current_tuple Src
  
  test_arity Fail Src Arity => i_test_arity Fail Src Arity | current_tuple Src
  
  is_tuple NotTupleFail Src |
    is_tagged_tuple WrongRecordFail Tuple Arity Atom |
    equal(Src, Tuple) =>
     i_is_tagged_tuple_ff NotTupleFail WrongRecordFail Src Arity Atom |
     current_tuple Src
  
  is_tagged_tuple Fail Tuple Arity Atom =>
     i_is_tagged_tuple Fail Tuple Arity Atom | current_tuple Tuple
  
  is_tuple Fail=f Src => i_is_tuple Fail Src | current_tuple Src
  
  i_is_tuple_of_arity f? s A
  i_test_arity f? s A
  
  i_is_tagged_tuple f? s A a
  i_is_tagged_tuple_ff f? f? s A a
  
  i_is_tuple f? s
  
  # Generate instruction sequence for fetching the tuple element and remember
  # that we have a current tuple pointer.
  
  get_tuple_element Tuple Pos Dst =>
    load_tuple_ptr Tuple | i_get_tuple_element Tuple Pos Dst | current_tuple Tuple
  
  current_tuple Tuple | get_tuple_element Tuple2 Pos Dst |
    equal(Tuple, Tuple2) =>
      i_get_tuple_element Tuple Pos Dst | current_tuple Tuple
  
  # Drop the current_tuple instruction if the tuple is overwritten.
  i_get_tuple_element Tuple Pos Tuple2 | current_tuple Tuple3 |
    equal(Tuple, Tuple2) | equal(Tuple, Tuple3) =>
      i_get_tuple_element Tuple Pos Tuple
  
  # This is a current_tuple instruction instruction not followed by get_tuple_element.
  # Invalidate the current tuple pointer.
  
  current_tuple Tuple => _
  
  load_tuple_ptr s
  
  # If both positions and destinations are in consecutive memory, fetch and store
  # two words at once.
  i_get_tuple_element Tuple Pos1 Dst1 |
     current_tuple Tuple2 |
     get_tuple_element Tuple3 Pos2 Dst2 |
     equal(Tuple, Tuple2) | equal(Tuple, Tuple3) |
     consecutive_words(Pos1, Dst1, Pos2, Dst2) =>
        get_two_tuple_elements Tuple Pos1 Dst1 Dst2 | current_tuple Tuple Dst2
  
  i_get_tuple_element Tuple Pos1 Dst1 | current_tuple Tuple2 |
     get_tuple_element Tuple3 Pos2 Dst2 |
     equal(Tuple, Tuple2) | equal(Tuple, Tuple3) |
     consecutive_words(Pos1, Dst2, Pos2, Dst1) =>
        get_two_tuple_elements Tuple Pos1 Dst1 Dst2 | current_tuple Tuple Dst2
  
  # Drop the current_tuple instruction if the tuple is overwritten.
  current_tuple Tuple Tuple2 | equal(Tuple, Tuple2) => _
  current_tuple Tuple Dst => current_tuple Tuple
  
  # The first operand will only be used in the debug-compiled runtime
  # system to verify that the register holding the tuple pointer agrees
  # with the source tuple operand.
  i_get_tuple_element s P S
  get_two_tuple_elements s P S S
  
  #
  # Exception raising instructions. Infrequently executed.
  #
  
  %cold
  case_end s
  
  badmatch s
  
  if_end
  
  raise s s
  
  # Workaround the limitation that generators must always return at least one instruction.
  delete_me/0
  delete_me => _
  
  system_limit/1
  system_limit p => system_limit_body
  system_limit Fail=f => jump Fail
  
  system_limit_body
  
  %hot
  
  #
  # Optimize moves of consecutive memory addresses.
  #
  
  move Src=c Dst => i_move Src Dst
  
  move Src SrcDst | move SrcDst2 Dst | equal(SrcDst, SrcDst2) =>
    i_move Src SrcDst | move SrcDst Dst
  
  # Try to move two words at once. Always arrange the source operands in
  # consecutive order; the destination operands may be in consecutive or
  # reverse consecutive order.
  
  move S1=d D1=d | move S2=d D2=d | consecutive_words(S1, D1, S2, D2) => move_two_words S1 D1 S2 D2
  move S1=d D1=d | move S2=d D2=d | consecutive_words(S1, D2, S2, D1) => move_two_words S1 D1 S2 D2
  move S1=d D1=d | move S2=d D2=d | consecutive_words(S2, D1, S1, D2) => move_two_words S2 D2 S1 D1
  move S1=d D1=d | move S2=d D2=d | consecutive_words(S2, D2, S1, D1) => move_two_words S2 D2 S1 D1
  
  move Src Dst => i_move Src Dst
  
  #
  # Move instructions.
  #
  
  swap d d
  i_move s d
  move_two_words s d s d
  
  #
  # Receive operations. We conservatively align all labels before any
  # of the receive instructions.
  #
  # As the labels may be stored in the process structure, we must align them to
  # the nearest 4-byte boundary to ensure they're properly tagged as continuation
  # pointers.
  #
  
  label L | loop_rec Fail Reg => \
      aligned_label L u=4 | loop_rec Fail Reg
  label L | wait_timeout Fail Src => \
      aligned_label L u=4 | wait_timeout Fail Src
  label L | wait Fail => \
      aligned_label L u=4 | wait Fail
  label L | timeout => \
      aligned_label L u=4 | timeout
  
  loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail
  
  aligned_label L A | wait_timeout Fail Src | smp_already_locked(L) => \
      aligned_label L A | wait_timeout_locked Src Fail
  wait_timeout Fail Src => wait_timeout_unlocked Src Fail
  
  aligned_label L A | wait Fail | smp_already_locked(L) => \
      aligned_label L A | wait_locked Fail
  wait Fail => wait_unlocked Fail
  
  aligned_label L A | timeout | smp_already_locked(L) => \
      aligned_label L A | timeout_locked
  
  remove_message
  timeout
  timeout_locked
  i_loop_rec f
  loop_rec_end f
  wait_locked f
  wait_unlocked f
  
  # Note that a timeout value must fit in 32 bits.
  wait_timeout_unlocked s f
  wait_timeout_locked s f
  
  send
  
  #
  # Optimized comparisons with one immediate/literal operand.
  #
  
  is_eq_exact Lbl LHS RHS | equal(LHS, RHS) => _
  is_eq_exact Lbl C=c R=xy => is_eq_exact Lbl R C
  
  is_eq_exact Lbl R=xy n => is_nil Lbl R
  is_eq_exact Lbl R=xy C=q => is_eq_exact_literal(Lbl, R, C)
  
  is_ne_exact Lbl LHS RHS | equal(LHS, RHS) => jump Lbl
  is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C
  
  is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C
  
  i_is_eq_exact_literal/4
  i_is_eq_exact_literal f? s c I
  
  i_is_ne_exact_literal f? s c
  
  is_eq_exact f? s s
  
  is_ne_exact f? s s
  
  is_lt f? s s
  is_ge f? s s
  
  is_eq Fail=f Const=c Reg=xy => is_eq Fail Reg Const
  is_eq f? s s
  
  is_ne Fail=f Const=c Reg=xy => is_ne Fail Reg Const
  is_ne f? s s
  
  #
  # Putting tuples.
  #
  # Code compiled with OTP 22 and later uses put_tuple2 to
  # to construct a tuple.
  #
  
  put_tuple2 S A *
  
  #
  # Putting lists.
  #
  
  store_cons/2
  put_list Hd Tl Dst => put_cons Hd Tl | store_cons u=1 Dst
  
  store_cons Len Dst | put_list Hd Tl Dst2 |
    equal(Dst, Tl) | equal(Dst, Dst2) | distinct(Dst, Hd)  =>
      combine_conses(Len, Dst, Hd)
  
  store_cons Len Dst | put_list Hd Tl OtherDst | equal(Dst, Tl) =>
      store_cons Len Dst | append_cons u Hd | store_cons u=1 OtherDst
  
  put_cons s s
  append_cons I s
  store_cons I d
  
  #
  # Some more only used by the emulator
  #
  
  %cold
  normal_exit
  continue_exit
  call_bif W
  call_bif_mfa a a I
  call_nif W W W
  call_error_handler
  return_trace
  %hot
  
  #
  # Type tests. Note that the operands for most type tests are `s` to
  # ensure that literal operands will work. The BEAM compiler starting
  # from OTP 22 will never emit type tests with literal operands even if
  # all optimizations are turned off, but loading unoptimized code from
  # older releases and code generated by alternative code generators.
  #
  
  is_integer f? s
  is_list f? s
  is_atom f? s
  is_float f? s
  
  is_nil Fail=f n => _
  is_nil Fail=f qia => jump Fail
  is_nil f? S
  
  # XXX Deprecated.
  is_bitstr Fail Term => is_bitstring Fail Term
  
  is_binary f? s
  is_bitstring f? s
  
  is_reference f? s
  is_pid f? s
  is_port f? s
  
  is_boolean f? s
  
  is_function2 f? s s
  
  #################################################################
  # External function and bif calls.
  #################################################################
  
  # Expands into call_light_bif/2
  call_light_bif/1
  
  #
  # The load_nif/2 BIF is an instruction.
  #
  
  call_ext u==2 u$func:erlang:load_nif/2 => \
      i_load_nif
  call_ext_last u==2 u$func:erlang:load_nif/2 D => \
      i_load_nif | deallocate D | return
  call_ext_only u==2 u$func:erlang:load_nif/2 => \
      i_load_nif | return
  
  %cold
  i_load_nif
  %hot
  
  #
  # The call_on_load_function/1 BIF is an instruction.
  #
  
  call_ext u==1 u$func:erlang:call_on_load_function/1 => \
      i_call_on_load_function
  call_ext_last u==1 u$func:erlang:call_on_load_function/1 D => \
      i_call_on_load_function | deallocate D | return
  call_ext_only u==1 u$func:erlang:call_on_load_function/1 => \
      i_call_on_load_function | return
  
  %cold
  i_call_on_load_function
  %hot
  
  #
  # apply/2 is an instruction, not a BIF.
  #
  
  call_ext u==2 u$func:erlang:apply/2 => i_apply_fun
  call_ext_last u==2 u$func:erlang:apply/2 D => i_apply_fun_last D
  call_ext_only u==2 u$func:erlang:apply/2 => i_apply_fun_only
  
  #
  # The apply/3 BIF is an instruction.
  #
  
  call_ext u==3 u$func:erlang:apply/3 => i_apply
  call_ext_last u==3 u$func:erlang:apply/3 D => i_apply_last D
  call_ext_only u==3 u$func:erlang:apply/3 => i_apply_only
  
  #
  # The yield/0 BIF is an instruction
  #
  
  call_ext u==0 u$func:erlang:yield/0 => i_yield
  call_ext_last u==0 u$func:erlang:yield/0 D => i_yield | deallocate D | return
  call_ext_only u==0 u$func:erlang:yield/0 => i_yield | return
  
  #
  # The hibernate/3 BIF is an instruction.
  #
  call_ext u==3 u$func:erlang:hibernate/3 => i_hibernate
  call_ext_last u==3 u$func:erlang:hibernate/3 D => i_hibernate
  call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate
  
  call_ext u==0 u$func:os:perf_counter/0 => \
      i_perf_counter
  call_ext_last u==0 u$func:os:perf_counter/0 D => \
      i_perf_counter | deallocate D | return
  call_ext_only u==0 u$func:os:perf_counter/0 => \
      i_perf_counter | return
  
  #
  # BIFs like process_info/1,2 require up-to-date information about the current
  # emulator state, which the ordinary call_light_bif instruction doesn't save.
  #
  
  call_ext u Bif=u$is_bif | is_heavy_bif(Bif) => \
      i_call_ext Bif
  call_ext_last u Bif=u$is_bif D | is_heavy_bif(Bif) => \
      i_call_ext Bif | deallocate D | return
  call_ext_only Ar=u Bif=u$is_bif | is_heavy_bif(Bif) => \
      allocate u Ar | i_call_ext Bif | deallocate u | return
  
  #
  # The general case for BIFs that have no special requirements.
  #
  
  call_ext u Bif=u$is_bif => \
      call_light_bif Bif
  call_ext_last u Bif=u$is_bif D => \
      call_light_bif Bif | deallocate D | return
  call_ext_only Ar=u Bif=u$is_bif => \
      allocate u Ar | call_light_bif Bif | deallocate u | return
  
  #
  # Any remaining calls are calls to Erlang functions, not BIFs.
  # We rename the instructions to internal names.  This is necessary,
  # to avoid an end-less loop, because we want to call a few BIFs
  # with call instructions.
  #
  
  call_ext Ar Func        => i_call_ext Func
  call_ext_last Ar Func D => i_call_ext_last Func D
  call_ext_only Ar Func   => i_call_ext_only Func
  
  i_validate t
  
  i_apply
  i_apply_last t
  i_apply_only
  
  i_apply_fun
  i_apply_fun_last t
  i_apply_fun_only
  
  call_light_bif Bif => call_light_bif Bif Bif
  call_light_bif b e
  
  %cold
  
  i_hibernate
  i_perf_counter
  
  %hot
  
  #
  # Calls to non-building and guard BIFs.
  #
  
  bif0 u$bif:erlang:self/0 Dst=d => self Dst
  bif0 u$bif:erlang:node/0 Dst=d => node Dst
  
  bif1 Fail=f Bif=u$bif:erlang:hd/1 Src=n Dst => \
      jump Fail
  bif1 Fail=f Bif=u$bif:erlang:hd/1 Src Dst => \
      is_nonempty_list Fail Src | get_hd Src Dst
  bif1 Fail Bif=u$bif:erlang:hd/1 Src Dst => \
      bif_hd Fail Src Dst
  
  bif_hd j? s d
  
  bif1 Fail=f Bif=u$bif:erlang:tl/1 Src=n Dst => \
      jump Fail
  bif1 Fail=f Bif=u$bif:erlang:tl/1 Src Dst => \
      is_nonempty_list Fail Src | get_tl Src Dst
  
  bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => get(Src, Dst)
  
  bif2 Fail u$bif:erlang:element/2 S1=ixy S2 Dst => bif_element Fail S1 S2 Dst
  bif_element j? s s d
  
  bif1 Fail Bif S1 Dst | never_fails(Bif) => nofail_bif1 S1 Bif Dst
  bif2 Fail Bif S1 S2 Dst | never_fails(Bif) => nofail_bif2 S1 S2 Bif Dst
  
  bif1 Fail Bif S1 Dst    => i_bif1 S1 Fail Bif Dst
  bif2 Fail Bif S1 S2 Dst => i_bif2 S1 S2 Fail Bif Dst
  
  nofail_bif2 S1=d S2 Bif Dst | is_eq_exact_bif(Bif) => bif_is_eq_exact S1 S2 Dst
  nofail_bif2 S1=d S2 Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact S1 S2 Dst
  
  i_get_hash c I d
  i_get s d
  
  self d
  
  node d
  
  nofail_bif1 s b d
  nofail_bif2 s s b d
  
  i_bif1 s j? b d
  i_bif2 s s j? b d
  i_bif3 s s s j? b d
  
  bif_is_eq_exact S s d
  bif_is_ne_exact S s d
  
  #
  # Internal calls.
  #
  
  call Ar Func        => i_call Func
  call_last Ar Func D => i_call_last Func D
  call_only Ar Func   => i_call_only Func
  
  i_call f
  i_call_last f t
  i_call_only f
  
  i_call_ext e
  i_call_ext_last e t
  i_call_ext_only e
  
  # Fun calls.
  
  call_fun Arity | deallocate D | return => i_call_fun_last Arity D
  call_fun Arity => i_call_fun Arity
  
  i_call_fun t
  i_call_fun_last t t
  
  call_fun2 Tag Arity Func | deallocate D | return => \
      i_call_fun2_last Tag Arity Func D
  call_fun2 Tag Arity Func => \
      i_call_fun2 Tag Arity Func
  
  i_call_fun2 aF t S
  i_call_fun2_last aF t S t
  
  #
  # A fun with an empty environment can be converted to a literal.
  # As a further optimization, the we try to move the fun to its
  # final destination directly.
  
  make_fun2 OldIndex=u => make_fun2(OldIndex)
  make_fun3 OldIndex=u Dst=d NumFree=u Env=* => make_fun3(OldIndex, Dst, NumFree, Env)
  
  %cold
  
  i_make_fun3 F S t t *
  
  # Psuedo-instruction for signalling lambda load errors. Never actually runs.
  i_lambda_error t
  
  %hot
  
  is_function f? S
  is_function Fail=f c => jump Fail
  
  # The start and end of a function.
  int_func_start/5
  int_func_end/2
  
  func_prologue/2
  
  int_func_start Func_Label Func_Line M F A | \
    label Entry_Label | line Entry_Line => \
      int_func_start Func_Label Func_Line M F A | \
      func_prologue Entry_Label Entry_Line
  
  int_func_start Func_Label Func_Line M F A | \
    label Entry_Label => \
      int_func_start Func_Label Func_Line M F A | \
      func_prologue Entry_Label n
  
  int_func_start Func_Label Func_Line M F A | \
    func_prologue Entry_Label Entry_Line | \
    is_mfa_bif(M, F, A) => \
      aligned_label Func_Label u=8 | \
      func_line Func_Line | \
      i_func_info Func_Label M F A | \
        aligned_label Entry_Label u=8 | \
        i_breakpoint_trampoline | \
        line Entry_Line | \
        call_bif_mfa M F A
  
  int_func_start Func_Label Func_Line M F A | \
    func_prologue Entry_Label Entry_Line => \
      aligned_label Func_Label u=8 | \
      func_line Func_Line | \
      i_func_info Func_Label M F A | \
        aligned_label Entry_Label u=8 | \
        i_breakpoint_trampoline | \
        line Entry_Line | \
        i_test_yield
  
  int_func_end Func_Label Entry_Label => \
      func_end(Func_Label, Entry_Label)
  
  # Handles yielding on function ingress (rather than on each call).
  i_test_yield
  
  # Ensures that the prior function is large enough to allow NIF patching.
  i_nif_padding
  
  # Landing pad for fun calls/apply where we set up arguments and check errors
  i_lambda_trampoline W f W W
  
  # Handles tracing, early NIF calls, and so on.
  i_breakpoint_trampoline
  
  # ================================================================
  # New bit syntax matching (R11B).
  # ================================================================
  
  %warm
  
  # Matching integers
  bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val
  
  i_bs_match_string S f W W
  
  # Fetching integers from binaries.
  bs_get_integer2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \
      get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
  
  i_bs_get_integer S f? t t s d
  
  i_bs_get_integer_8 S t f? d
  i_bs_get_integer_16 S t f? d
  i_bs_get_integer_32 S t f? d
  i_bs_get_integer_64 S t f? t d
  
  # Fetching binaries from binaries.
  bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \
  			get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
  
  i_bs_get_binary2 S f t? s t d
  i_bs_get_binary_all2 S f? t t d
  
  # Fetching float from binaries.
  bs_get_float2 Fail=f Ms=xy Live=u Sz=s Unit=u Flags=u Dst=d => \
  		get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
  
  bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
  
  i_bs_get_float2 S f? t s t d
  
  # Miscellaneous
  
  bs_skip_bits2 Fail=f Ms=xy Sz=sq Unit=u Flags=u => \
  			skip_bits2(Fail, Ms, Sz, Unit, Flags)
  
  i_bs_skip_bits_imm2 f? S W
  i_bs_skip_bits2 S S f? t
  
  bs_test_tail2 f? S t
  
  bs_test_unit f? S t
  
  # Gets a bitstring from the tail of a context.
  bs_get_tail S d t
  
  # New bs_start_match variant for contexts with external position storage.
  #
  # bs_get/set_position is used to save positions into registers instead of
  # "slots" in the context itself, which lets us continue matching even after
  # we've passed it off to another function.
  
  bs_start_match4 a==am_no_fail Live=u Src=xy Ctx=d => \
      bs_start_match3 p Src Live Ctx
  bs_start_match4 Fail=f Live=u Src=xy Ctx=d => \
      bs_start_match3 Fail Src Live Ctx
  
  %if ARCH_64
  
  # This instruction nops on 64-bit platforms
  bs_start_match4 a==am_resume Live Ctx Dst | equal(Ctx, Dst) => _
  bs_start_match4 a==am_resume Live Ctx Dst => move Ctx Dst
  
  %else
  
  bs_start_match4 a==am_resume Live Ctx Dst => \
      bs_start_match4 a=am_no_fail Live Ctx Dst
  
  %endif
  
  bs_start_match3 Fail=j ica Live Dst => jump Fail
  bs_start_match3 Fail Bin Live Dst => i_bs_start_match3 Bin Live Fail Dst
  
  i_bs_start_match3 S t j d
  
  # Match context position instructions. 64-bit assumes that all positions can
  # fit into an unsigned small.
  
  %if ARCH_64
      bs_get_position Src Dst Live => i_bs_get_position Src Dst
      i_bs_get_position S S
      bs_set_position S S
  %else
      bs_get_position S d t?
      bs_set_position S S
  %endif
  
  #
  # Utf8/utf16/utf32 support. (R12B-5)
  #
  bs_get_utf8 Fail=f Ms=xy u u Dst=d => i_bs_get_utf8 Ms Fail Dst
  i_bs_get_utf8 S f? d
  
  bs_skip_utf8 Fail=f Ms=xy u u => i_bs_skip_utf8 Ms Fail
  i_bs_skip_utf8 S f?
  
  bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => get_utf16(Fail, Ms, Flags, Dst)
  bs_skip_utf16 Fail=f Ms=xy u Flags=u => skip_utf16(Fail, Ms, Flags)
  
  i_bs_get_utf16 S f? t d
  i_bs_skip_utf16 S f? t
  
  bs_get_utf32 Fail=f Ms=xy Live=u Flags=u Dst=d => \
  	bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \
  	i_bs_validate_unicode_retract Fail Dst Ms
  bs_skip_utf32 Fail=f Ms=xy Live=u Flags=u => \
  	bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \
  	i_bs_validate_unicode_retract Fail x Ms
  
  i_bs_validate_unicode_retract j s S
  %hot
  
  # ================================================================
  # New binary construction (OTP 25).
  # ================================================================
  
  bs_create_bin Fail=j Alloc=u Live=u Unit=u Dst=xy N=u Segments=* => \
      create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments)
  
  i_bs_create_bin j I t d *
  
  # ================================================================
  # Old instruction for constructing binaries (up to OTP 24).
  # ================================================================
  
  %warm
  
  bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail
  
  bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init Sz Regs Dst
  
  bs_init2 Fail Sz=u Words Regs Flags Dst => \
     i_bs_init_heap Sz Words Regs Dst
  
  bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
    i_bs_init_fail Sz Fail Regs Dst
  bs_init2 Fail Sz Words Regs Flags Dst => \
    i_bs_init_fail_heap Sz Words Fail Regs Dst
  
  i_bs_init_fail S j? t? S
  
  i_bs_init_fail_heap s I j? t? S
  
  i_bs_init W t? S
  
  i_bs_init_heap W I t? S
  
  
  bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail
  
  bs_init_bits Fail Sz=u Words=u==0 Regs Flags Dst => \
      i_bs_init_bits Sz Regs Dst
  bs_init_bits Fail Sz=u Words Regs Flags Dst => \
      i_bs_init_bits_heap Sz Words Regs Dst
  
  bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
    i_bs_init_bits_fail Sz Fail Regs Dst
  bs_init_bits Fail Sz Words Regs Flags Dst => \
    i_bs_init_bits_fail_heap Sz Words Fail Regs Dst
  
  i_bs_init_bits_fail S j? t? S
  
  i_bs_init_bits_fail_heap s I j? t? S
  
  i_bs_init_bits W t? S
  i_bs_init_bits_heap W I t? S
  
  bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
  
  bs_add j? s s t? x
  
  bs_append Fail Size Extra Live Unit Bin Flags Dst => \
    i_bs_append Fail Extra Live Unit Size Bin Dst
  
  bs_private_append Fail Size Unit Bin Flags Dst => \
    i_bs_private_append Fail Unit Size Bin Dst
  
  i_bs_private_append Fail Unit Size Bin Dst=y => \
    i_bs_private_append Fail Unit Size Bin x | move x Dst
  
  bs_init_writable
  
  i_bs_append j? I t? t s s S
  i_bs_private_append j? t s S x
  
  #
  # Storing integers into binaries.
  #
  
  bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
  			put_integer(Fail, Sz, Unit, Flags, Src)
  
  i_new_bs_put_integer j? S t s
  i_new_bs_put_integer_imm s j? W t
  
  #
  # Utf8/utf16/utf32 support. (R12B-5)
  #
  
  bs_utf8_size j Src Dst=d => i_bs_utf8_size Src Dst
  bs_utf16_size j Src Dst=d => i_bs_utf16_size Src Dst
  
  bs_put_utf8 Fail u Src => i_bs_put_utf8 Fail Src
  bs_put_utf16 Fail Flags Src => put_utf16(Fail, Flags, Src)
  
  bs_put_utf32 Fail=j Flags=u Src=s => \
     i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
  
  i_bs_utf8_size s x
  i_bs_utf16_size s x
  
  i_bs_put_utf8 j? s
  i_bs_put_utf16 j? t s
  
  i_bs_validate_unicode j? s
  
  #
  # Storing floats into binaries.
  #
  
  # Will fail. No need to keep the instruction, because bs_add or
  # bs_init* would already have raised an exception.
  bs_put_float Fail Sz=q Unit Flags Val => _
  
  bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
  			put_float(Fail, Sz, Unit, Flags, Src)
  
  i_new_bs_put_float j? S t s
  i_new_bs_put_float_imm j? W t s
  
  #
  # Storing binaries into binaries.
  #
  
  bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
  			put_binary(Fail, Sz, Unit, Flags, Src)
  
  i_new_bs_put_binary j? s t s
  i_new_bs_put_binary_imm j? W s
  i_new_bs_put_binary_all s j? t
  
  #
  # Warning: The i_bs_put_string and i_new_bs_put_string instructions
  # are specially treated in the loader.
  # Don't change the instruction format unless you change the loader too.
  #
  
  bs_put_string W W
  
  #
  # New floating point instructions (R8).
  #
  
  fadd p FR1 FR2 FR3 => i_fadd FR1 FR2 FR3
  fsub p FR1 FR2 FR3 => i_fsub FR1 FR2 FR3
  fmul p FR1 FR2 FR3 => i_fmul FR1 FR2 FR3
  fdiv p FR1 FR2 FR3 => i_fdiv FR1 FR2 FR3
  fnegate p FR1 FR2 => i_fnegate FR1 FR2
  
  fmove Arg=l Dst=d => fstore Arg Dst
  fmove Arg=dq Dst=l => fload Arg Dst
  
  fstore l d
  fload Sq l
  
  fconv s l
  
  i_fadd l l l
  i_fsub l l l
  i_fmul l l l
  i_fdiv l l l
  i_fnegate l l
  
  fclearerror => _
  fcheckerror p => _
  
  %hot
  
  #
  # New apply instructions in R10B.
  #
  
  apply t
  apply_last t t
  
  #
  # Map instructions. First introduced in R17.
  #
  
  # We KNOW that in OTP 18 and higher, a put_map_assoc instruction is
  # always preceded by an is_map test. That means that put_map_assoc can
  # never fail and does not need any failure label.
  
  put_map_assoc Fail Map Dst Live Size Rest=* => \
  	      i_put_map_assoc Map Dst Live Size Rest
  
  i_put_map_assoc/4
  
  sorted_put_map_assoc/4
  i_put_map_assoc Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
    sorted_put_map_assoc Map Dst Live Size Rest
  
  sorted_put_map_exact/5
  put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
    sorted_put_map_exact F Map Dst Live Size Rest
  
  sorted_put_map_assoc Map Dst Live Size Rest=* | is_empty_map(Map) => \
     new_map Dst Live Size Rest
  sorted_put_map_assoc Src=s Dst Live Size Rest=* => \
     update_map_assoc Src Dst Live Size Rest
  
  sorted_put_map_exact Fail Src Dst Live Size Rest=* => \
     update_map_exact Src Fail Dst Live Size Rest
  
  new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \
    new_small_map_lit(Dst, Live, Size, Rest)
  
  new_map d t I *
  i_new_small_map_lit d t q I *
  update_map_assoc s d t I *
  update_map_exact s j? d t I *
  
  is_map f? s
  
  ## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements
  
  has_map_fields Fail Src Size Rest=* => \
     has_map_fields(Fail, Src, Size, Rest)
  
  ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 }
  
  get_map_elements Fail Src Size=u==2 Rest=* => \
      get_map_element(Fail, Src, Size, Rest)
  get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
     get_map_elements(Fail, Src, Size, Rest)
  
  i_get_map_elements f? s I *
  
  i_get_map_element_hash Fail Src=c Key Hash Dst => \
      move Src x | i_get_map_element_hash Fail x Key Hash Dst
  i_get_map_element_hash f? S c I S
  
  i_get_map_element Fail Src=c Key Dst => \
      move Src x | i_get_map_element Fail x Key Dst
  i_get_map_element f? S S S
  
  #
  # Convert the plus operations to a generic plus instruction.
  #
  gen_plus/5
  gen_minus/5
  
  gc_bif1 Fail Live u$bif:erlang:splus/1 Src Dst => \
     gen_plus Fail Live Src i Dst
  gc_bif2 Fail Live u$bif:erlang:splus/2 S1 S2 Dst => \
     gen_plus Fail Live S1 S2 Dst
  
  gc_bif1 Fail Live u$bif:erlang:sminus/1 Src Dst => \
     i_unary_minus Src Fail Dst
  gc_bif2 Fail Live u$bif:erlang:sminus/2 S1 S2 Dst => \
     gen_minus Fail Live S1 S2 Dst
  
  #
  # Optimize addition and subtraction of small literals using
  # the i_increment/3 instruction (in bodies, not in guards).
  #
  
  gen_plus p Live Int=i Reg=d Dst => \
  	increment(Reg, Int, Dst)
  gen_plus p Live Reg=d Int=i Dst => \
  	increment(Reg, Int, Dst)
  
  gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \
  	increment_from_minus(Reg, Int, Dst)
  
  #
  # Arithmetic instructions.
  #
  
  # It is OK to swap arguments for '+' in a guard. It is also
  # OK to turn minus into plus in a guard.
  gen_plus Fail=f Live S1=c S2 Dst => i_plus S2 S1 Fail Dst
  gen_minus Fail=f Live S1 S2=i Dst | negation_is_small(S2) => \
      plus_from_minus(Fail, Live, S1, S2, Dst)
  
  gen_plus Fail Live S1 S2 Dst => i_plus S1 S2 Fail Dst
  
  gen_minus Fail Live S1 S2 Dst => i_minus S1 S2 Fail Dst
  
  gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \
    i_times Fail S1 S2 Dst
  
  gc_bif2 Fail Live u$bif:erlang:div/2 S1 S2 Dst => \
    i_m_div Fail S1 S2 Dst
  
  # Fused 'rem'/'div' pair.
  gc_bif2 Fail Live u$bif:erlang:rem/2 LHS1 RHS1 Remainder | \
      gc_bif2 A B u$bif:erlang:intdiv/2 LHS2 RHS2 Quotient | \
      equal(LHS1, LHS2) | \
      equal(RHS1, RHS2) | \
      distinct(LHS1, Remainder) | \
      distinct(RHS1, Remainder) => \
        i_rem_div LHS1 RHS1 Fail Remainder Quotient
  
  # As above but with a `line` in between
  gc_bif2 Fail Live u$bif:erlang:rem/2 LHS1 RHS1 Remainder | \
      line Loc | \
      gc_bif2 A B u$bif:erlang:intdiv/2 LHS2 RHS2 Quotient | \
      equal(LHS1, LHS2) | \
      equal(RHS1, RHS2) | \
      distinct(LHS1, Remainder) | \
      distinct(RHS1, Remainder) => \
        i_rem_div LHS1 RHS1 Fail Remainder Quotient
  
  # Fused 'div'/'rem' pair
  gc_bif2 Fail Live u$bif:erlang:intdiv/2 LHS1 RHS1 Quotient | \
      gc_bif2 A B u$bif:erlang:rem/2 LHS2 RHS2 Remainder | \
      equal(LHS1, LHS2) | \
      equal(RHS1, RHS2) | \
      distinct(LHS1, Quotient) | \
      distinct(RHS1, Quotient) => \
        i_div_rem LHS1 RHS1 Fail Quotient Remainder
  
  # As above but with a `line` in between
  gc_bif2 Fail Live u$bif:erlang:intdiv/2 LHS1 RHS1 Quotient | \
      line Loc | \
      gc_bif2 A B u$bif:erlang:rem/2 LHS2 RHS2 Remainder | \
      equal(LHS1, LHS2) | \
      equal(RHS1, RHS2) | \
      distinct(LHS1, Quotient) | \
      distinct(RHS1, Quotient) => \
        i_div_rem LHS1 RHS1 Fail Quotient Remainder
  
  gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \
    i_int_div Fail S1 S2 Dst
  gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \
    i_rem S1 S2 Fail Dst
  
  gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \
    i_bsl S1 S2 Fail Dst
  gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \
    i_bsr S1 S2 Fail Dst
  
  gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \
    i_band S1 S2 Fail Dst
  
  gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \
    i_bor Fail S1 S2 Dst
  
  gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \
    i_bxor Fail S1 S2 Dst
  
  gc_bif1 Fail Live u$bif:erlang:bnot/1 Src Dst => \
    i_bnot Fail Src Dst
  
  i_increment S W d
  
  i_plus s s j? d
  i_minus s s j? d
  
  i_unary_minus s j? d
  
  i_times j? s s d
  
  i_m_div j? s s d
  
  i_rem_div s s j? d d
  i_div_rem s s j? d d
  i_int_div j? s s d
  i_rem s s j? d
  
  i_bsl s s j? d
  i_bsr s s j? d
  
  i_band s s j? d
  i_bor j? s s d
  i_bxor j? s s d
  
  i_bnot j? s d
  
  #
  # Old guard BIFs that creates heap fragments are no longer allowed.
  #
  bif1 Fail u$bif:erlang:length/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:size/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:abs/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:float/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:round/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler
  
  #
  # Handle the length/1 guard BIF specially to make it trappable.
  #
  
  gc_bif1 Fail=j Live u$bif:erlang:length/1 Src Dst => \
     i_length_setup Fail Live Src | i_length Fail Live Dst
  
  i_length_setup j? t s
  i_length j? t d
  
  #
  # Guard BIFs.
  #
  gc_bif1 Fail Live Bif Src Dst      => i_bif1 Src Fail Bif Dst
  gc_bif2 Fail Live Bif S1 S2 Dst    => i_bif2 S1 S2 Fail Bif Dst
  gc_bif3 Fail Live Bif S1 S2 S3 Dst => i_bif3 S1 S2 S3 Fail Bif Dst
  
  #
  # The following instruction is specially handled in beam_load.c
  # to produce a user-friendly message if an unsupported guard BIF is
  # encountered.
  #
  unsupported_guard_bif/3
  unsupported_guard_bif A B C | never() => _
  
  #
  # R13B03
  #
  on_load
  
  #
  # R14A.
  #
  # Superseded in OTP 24 by 'recv_marker_reserve' and friends.
  #
  
  recv_mark f => i_recv_mark
  i_recv_mark
  
  recv_set Fail | label Lbl | loop_rec Lf Reg => \
     i_recv_set | label Lbl | loop_rec Lf Reg
  i_recv_set
  
  #
  # OTP 21.
  #
  
  build_stacktrace
  raw_raise
  
  #
  # Specialized move instructions. Since they don't require a second
  # instruction, we have intentionally placed them after any other
  # transformation rules that starts with a move instruction in order to
  # produce better code for the transformation engine.
  #
  
  move n D=y => init D
  
  #
  # OTP 24
  #
  
  recv_marker_reserve S
  recv_marker_bind S S
  recv_marker_clear S
  recv_marker_use S
  