text: #
  # %CopyrightBegin%
  #
  # Copyright Ericsson AB 1997-2013. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  # %CopyrightEnd%
  #
  
  #
  # The instructions that follows are only known by the loader and the emulator.
  # They can be changed without recompiling old Beam files.
  #
  # Instructions starting with a "i_" prefix are instructions produced by
  # instruction transformations; thus, they never occur in BEAM files.
  #
  
  # The too_old_compiler/0 instruction is specially handled in beam_load.c
  # to produce a user-friendly message informing the user that the module
  # needs to be re-compiled with a modern compiler.
  
  too_old_compiler/0
  too_old_compiler | never() =>
  
  # In R9C and earlier, the loader used to insert special instructions inside
  # the module_info/0,1 functions. (In R10B and later, the compiler inserts
  # an explicit call to an undocumented BIF, so that no loader trickery is
  # necessary.) Since the instructions don't work correctly in R12B, simply
  # refuse to load the module.
  
  func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler
  func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler
  
  # The undocumented and unsupported guard BIF is_constant/1 was removed
  # in R13. The is_constant/2 operation is marked as obsolete in genop.tab,
  # so the loader will automatically generate a too_old_compiler message
  # it is used, but we need to handle the is_constant/1 BIF specially here.
  
  bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler
  
  # Since the constant pool was introduced in R12B, empty tuples ({})
  # are literals. Therefore we no longer need to allow put_tuple/2
  # with a tuple size of zero.
  
  put_tuple u==0 d => too_old_compiler
  
  #
  # All the other instructions.
  #
  
  label L
  i_func_info I a a I
  int_code_end
  
  i_generic_breakpoint
  i_debug_breakpoint
  i_return_time_trace
  i_return_to_trace
  i_yield
  
  return
  
  #
  # To ensure that a "move Src x(0)" instruction can be combined
  # with the following call instruction, we need to make sure that
  # there is no line/1 instruction between the move and the call.
  #
  
  move S r | line Loc | call_ext Ar Func => \
       line Loc | move S r | call_ext Ar Func
  move S r | line Loc | call_ext_last Ar Func=u$is_bif D => \
       line Loc | move S r | call_ext_last Ar Func D
  move S r | line Loc | call_ext_only Ar Func=u$is_bif => \
       line Loc | move S r | call_ext_only Ar Func
  move S r | line Loc | call Ar Func => \
       line Loc | move S r | call Ar Func
  
  #
  # A tail-recursive call to an external function (non-BIF) will
  # never be saved on the stack, so there is no reason to keep
  # the line instruction. (The compiler did not remove the line
  # instruction because it cannot tell the difference between
  # BIFs and ordinary Erlang functions.)
  #
  
  line Loc | call_ext_last Ar Func=u$is_not_bif D => \
       call_ext_last Ar Func D
  line Loc | call_ext_only Ar Func=u$is_not_bif => \
       call_ext_only Ar Func
  
  line Loc | func_info M F A => func_info M F A | line Loc
  
  line I
  
  
  %macro: allocate Allocate -pack
  %macro: allocate_zero AllocateZero -pack
  %macro: allocate_heap AllocateHeap -pack
  %macro: allocate_heap_zero AllocateHeapZero -pack
  %macro: test_heap TestHeap -pack
  
  allocate t t
  allocate_heap t I t
  deallocate I
  init y
  allocate_zero t t
  allocate_heap_zero t I t
  
  trim N Remaining => i_trim N
  i_trim I
  
  test_heap I t
  
  allocate_heap S u==0 R => allocate S R
  allocate_heap_zero S u==0 R => allocate_zero S R
  
  init2 y y
  init3 y y y
  init Y1 | init Y2 | init Y3 => init3 Y1 Y2 Y3
  init Y1 | init Y2 => init2 Y1 Y2
  %macro: init2 Init2 -pack
  %macro: init3 Init3 -pack
  
  # Selecting values
  
  select_val S=aiq Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest)
  
  select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
    gen_jump_tab(S, Fail, Size, Rest)
  
  is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
    gen_jump_tab(S, Fail, Size, Rest)
  
  is_integer TypeFail=f S | select_val S=s Fail=f Size=u Rest=* | \
  	   mixed_types(Size, Rest) => \
    gen_split_values(S, TypeFail, Fail, Size, Rest)
  
  select_val S=s Fail=f Size=u Rest=* | mixed_types(Size, Rest) => \
    gen_split_values(S, Fail, Fail, Size, Rest)
  
  is_integer Fail=f S | select_val S=d Fail=f Size=u Rest=* | \
    fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest)
  
  is_atom Fail=f S | select_val S=d Fail=f Size=u Rest=* | \
    fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest)
  
  select_val S=s Fail=f Size=u Rest=* | floats_or_bignums(Size, Rest) => \
    gen_select_literals(S, Fail, Size, Rest)
  
  select_val S=d Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \
    gen_select_val(S, Fail, Size, Rest)
  
  is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
    gen_select_tuple_arity(S, Fail, Size, Rest)
  
  select_tuple_arity S=d Fail=f Size=u Rest=* => \
    gen_select_tuple_arity(S, Fail, Size, Rest)
  
  i_select_val_bins r f I
  i_select_val_bins x f I
  i_select_val_bins y f I
  
  i_select_val_lins r f I
  i_select_val_lins x f I
  i_select_val_lins y f I
  
  i_select_val2 r f c c f f
  i_select_val2 x f c c f f
  i_select_val2 y f c c f f
  
  i_select_tuple_arity r f I
  i_select_tuple_arity x f I
  i_select_tuple_arity y f I
  
  i_select_tuple_arity2 r f A A f f
  i_select_tuple_arity2 x f A A f f
  i_select_tuple_arity2 y f A A f f
  
  i_jump_on_val_zero r f I
  i_jump_on_val_zero x f I
  i_jump_on_val_zero y f I
  
  i_jump_on_val r f I I
  i_jump_on_val x f I I
  i_jump_on_val y f I I
  
  jump Target | label Lbl | same_label(Target, Lbl) => label Lbl
  
  is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \
    is_eq_exact Fail S1 S2 | label L2
  
  %macro: get_list GetList -pack
  get_list x x x
  get_list x x y
  get_list x x r
  get_list x y x
  get_list x y y
  get_list x y r
  get_list x r x
  get_list x r y
  
  get_list y x x
  get_list y x y
  get_list y x r
  get_list y y x
  get_list y y y
  get_list y y r
  get_list y r x
  get_list y r y
  
  get_list r x x
  get_list r x y
  get_list r x r
  get_list r y x
  get_list r y y
  get_list r y r
  get_list r r x
  get_list r r y
  
  # Old-style catch.
  catch y f
  catch_end y
  
  # Try/catch.
  try Y F => catch Y F
  try_case Y => try_end Y
  try_end y
  
  try_case_end Literal=q => move Literal x | try_case_end x
  try_case_end s
  
  # Destructive set tuple element
  
  set_tuple_element Lit=q Tuple Pos => move Lit x | set_tuple_element x Tuple Pos
  set_tuple_element s d P
  
  # Get tuple element
  
  %macro: i_get_tuple_element GetTupleElement -pack
  i_get_tuple_element x P x
  i_get_tuple_element r P x
  i_get_tuple_element y P x
  i_get_tuple_element x P r
  i_get_tuple_element y P r
  
  %cold
  i_get_tuple_element r P r
  i_get_tuple_element x P y
  i_get_tuple_element r P y
  i_get_tuple_element y P y
  %hot
  
  %macro: is_number IsNumber -fail_action
  %cold
  is_number f r
  is_number f x
  is_number f y
  %hot
  is_number Fail=f i =>
  is_number Fail=f na => jump Fail
  is_number Fail Literal=q => move Literal x | is_number Fail x
  
  jump f
  
  case_end Literal=cq => move Literal x | case_end x
  badmatch Literal=cq => move Literal x | badmatch x
  
  case_end r
  case_end x
  case_end y
  
  badmatch r
  badmatch x
  badmatch y
  
  if_end
  raise s s
  
  # Internal now, but could be useful to make known to the compiler.
  badarg j
  system_limit j
  
  move C=cxy r | jump Lbl => move_jump Lbl C
  
  %macro: move_jump MoveJump -nonext
  move_jump f n
  move_jump f c
  move_jump f x
  move_jump f y
  
  
  # Movement to and from the stack is common
  # Try to pack as much as we can into one instruction
  
  # Window move
  move_window/5
  move_window/6
  
  # x -> y
  
  move S1=r S2=y | move X1=x Y1=y => move2 S1 S2 X1 Y1
  
  move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \
      move_window X1 X2 X3 Y1 Y3
  
  move_window X1=x X2=x X3=x Y1=y Y3=y | move X4=x Y4=y | succ(Y3,Y4) => \
      move_window X1 X2 X3 X4 Y1 Y4
  
  move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \
      move_window5 X1 X2 X3 X4 X5 Y1
  
  move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1
  move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1
  
  move_window3 x x x y
  move_window4 x x x x y
  move_window5 x x x x x y
  
  move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
  move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
  move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
  
  move S1=x S2=r | move S3=x S4=x => move2 S1 S2 S3 S4
  move S1=x S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1
  move S1=y S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1
  
  move Y1=y X1=x | move S1=r D1=x => move2 Y1 X1 S1 D1
  move S1=r D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1
  
  move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3
  move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3
  move2 X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6
  
  move C=aiq X=x==1 => move_x1 C
  move C=aiq X=x==2 => move_x2 C
  
  move_x1 c
  move_x2 c
  
  %macro: move2 Move2 -pack
  move2 x y x y
  move2 y x y x
  move2 x x x x
  
  move2 x r x x
  
  move2 x r x y
  move2 r y x y
  move2 y r x y
  
  move2 r x y x
  move2 y x r x
  
  %macro: move3 Move3
  move3 x y x y x y
  move3 y x y x y x
  move3 x x x x x x
  
  # The compiler almost never generates a "move Literal y(Y)" instruction,
  # so let's cheat if we encounter one.
  move S=n D=y => init D
  move S=c D=y => move S x | move x D
  
  %macro:move Move -pack -gen_dest
  move x x
  move x y
  move x r
  move y x
  move y r
  move r x
  move r y
  move c r
  move c x
  move n x
  move n r
  move y y
  
  # Receive operations.
  
  loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src
  
  label L | wait_timeout Fail Src | smp_already_locked(L) => label L | i_wait_timeout_locked Fail Src
  wait_timeout Fail Src => i_wait_timeout Fail Src
  i_wait_timeout Fail Src=aiq => gen_literal_timeout(Fail, Src)
  i_wait_timeout_locked Fail Src=aiq => gen_literal_timeout_locked(Fail, Src)
  
  label L | wait Fail | smp_already_locked(L) => label L | wait_locked Fail
  wait Fail | smp() => wait_unlocked Fail
  
  label L | timeout | smp_already_locked(L) => label L | timeout_locked
  
  remove_message
  timeout
  timeout_locked
  i_loop_rec f r
  loop_rec_end f
  wait f
  wait_locked f
  wait_unlocked f
  i_wait_timeout f I
  i_wait_timeout f s
  i_wait_timeout_locked f I
  i_wait_timeout_locked f s
  i_wait_error
  i_wait_error_locked
  
  send
  
  #
  # Optimized comparisons with one immediate/literal operand.
  #
  
  is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C
  is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C
  
  is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C
  is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C
  
  %macro: i_is_eq_exact_immed EqualImmed -fail_action
  i_is_eq_exact_immed f r c
  i_is_eq_exact_immed f x c
  i_is_eq_exact_immed f y c
  
  i_is_eq_exact_literal r f c
  i_is_eq_exact_literal x f c
  i_is_eq_exact_literal y f c
  
  %macro: i_is_ne_exact_immed NotEqualImmed -fail_action
  i_is_ne_exact_immed f r c
  i_is_ne_exact_immed f x c
  i_is_ne_exact_immed f y c
  
  i_is_ne_exact_literal r f c
  i_is_ne_exact_literal x f c
  i_is_ne_exact_literal y f c
  
  #
  # Common Compare Specializations
  # We don't do all of them since we want
  # to keep the instruction set small-ish
  #
  
  is_eq_exact Lbl S1=xy S2=r => is_eq_exact Lbl S2 S1
  is_eq_exact Lbl S1=rx S2=xy => i_is_eq_exact_spec Lbl S1 S2
  %macro: i_is_eq_exact_spec EqualExact -fail_action
  
  i_is_eq_exact_spec f x x
  i_is_eq_exact_spec f x y
  i_is_eq_exact_spec f r x
  i_is_eq_exact_spec f r y
  %cold
  i_is_eq_exact_spec f r r
  %hot
  
  is_lt Lbl S1=rxc S2=rxc  => i_is_lt_spec Lbl S1 S2
  
  %macro: i_is_lt_spec IsLessThan -fail_action
  
  i_is_lt_spec f x x
  i_is_lt_spec f x r
  i_is_lt_spec f x c
  i_is_lt_spec f r x
  i_is_lt_spec f r c
  i_is_lt_spec f c x
  i_is_lt_spec f c r
  %cold
  i_is_lt_spec f r r
  i_is_lt_spec f c c
  %hot
  
  is_ge Lbl S1=xc S2=xc => i_is_ge_spec Lbl S1 S2
  
  %macro: i_is_ge_spec IsGreaterEqual -fail_action
  
  i_is_ge_spec f x x
  i_is_ge_spec f x c
  i_is_ge_spec f c x
  %cold
  i_is_ge_spec f c c
  %hot
  
  #
  # All other comparisons.
  #
  
  is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
  is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
  
  is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl
  is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl
  is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl
  is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl
  
  i_is_eq_exact f
  i_is_ne_exact f
  i_is_lt f
  i_is_ge f
  i_is_eq f
  i_is_ne f
  
  #
  # Putting things.
  #
  
  put_tuple Arity Dst => i_put_tuple Dst u
  
  i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \
    put S3 | put S4 | put S5 => \
  	    tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5)
  
  i_put_tuple Dst Arity Puts=* | put S => \
  	    tuple_append_put(Arity, Dst, Puts, S)
  
  i_put_tuple/2
  
  %macro:i_put_tuple PutTuple -pack -goto:do_put_tuple
  i_put_tuple r I
  i_put_tuple x I
  i_put_tuple y I
  
  #
  # The instruction "put_list Const [] Dst" will not be generated by
  # the current BEAM compiler. But until R15A, play it safe by handling
  # that instruction with the following transformation.
  #
  put_list Const=c n Dst => move Const x | put_list x n Dst
  
  %macro:put_list PutList -pack -gen_dest
  
  put_list x n x
  put_list y n x
  put_list x x x
  put_list y x x
  put_list x x r
  put_list y r r
  
  put_list y y x
  put_list x y x
  put_list r x x
  put_list r y x
  put_list r x r
  put_list y y r
  put_list y r x
  put_list r n x
  
  put_list x r x
  put_list x y r
  put_list y x r
  put_list y x x
  
  put_list x r r
  
  # put_list SrcReg Constant Dst
  put_list r c r
  put_list r c x
  put_list r c y
  
  put_list x c r
  put_list x c x
  put_list x c y
  
  put_list y c r
  put_list y c x
  put_list y c y
  
  # put_list Constant SrcReg Dst
  put_list c r r
  put_list c r x
  put_list c r y
  
  put_list c x r
  put_list c x x
  put_list c x y
  
  put_list c y r
  put_list c y x
  put_list c y y
  
  %cold
  put_list s s d
  %hot
  
  %macro: i_fetch FetchArgs -pack
  i_fetch c r
  i_fetch c x
  i_fetch c y
  i_fetch r c
  i_fetch r x
  i_fetch r y
  i_fetch x c
  i_fetch x r
  i_fetch x x
  i_fetch x y
  i_fetch y c
  i_fetch y r
  i_fetch y x
  i_fetch y y
  
  %cold
  i_fetch c c
  i_fetch s s
  %hot
  
  #
  # Some more only used by the emulator
  #
  
  normal_exit
  continue_exit
  apply_bif
  call_nif
  call_error_handler
  error_action_code
  return_trace
  
  #
  # Instruction transformations & folded instructions.
  #
  
  # Note: There is no 'move_return y r', since there never are any y registers
  # when we do move_return (if we have y registers, we must do move_deallocate_return).
  
  move S r | return => move_return S r
  
  %macro: move_return MoveReturn -nonext
  move_return x r
  move_return c r
  move_return n r
  
  move S r | deallocate D | return => move_deallocate_return S r D
  
  %macro: move_deallocate_return MoveDeallocateReturn -pack -nonext
  move_deallocate_return x r Q
  move_deallocate_return y r Q
  move_deallocate_return c r Q
  move_deallocate_return n r Q
  
  deallocate D | return => deallocate_return D
  
  %macro: deallocate_return DeallocateReturn -nonext
  deallocate_return Q
  
  test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y
  
  %macro: test_heap_1_put_list TestHeapPutList -pack
  test_heap_1_put_list I y
  
  # Test tuple & arity (head)
  
  is_tuple Fail Literal=q => move Literal x | is_tuple Fail x
  is_tuple Fail=f c => jump Fail
  is_tuple Fail=f S=rxy | test_arity Fail=f S=rxy Arity => is_tuple_of_arity Fail S Arity
  
  %macro:is_tuple_of_arity IsTupleOfArity -fail_action
  
  is_tuple_of_arity f x A
  is_tuple_of_arity f y A
  is_tuple_of_arity f r A
  
  %macro: is_tuple IsTuple -fail_action
  is_tuple f x
  is_tuple f y
  is_tuple f r
  
  test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity
  test_arity Fail=f c Arity => jump Fail
  
  %macro: test_arity IsArity -fail_action
  test_arity f x A
  test_arity f y A
  test_arity f r A
  
  is_tuple_of_arity Fail=f Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \
    is_tuple_of_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P
  
  test_arity Fail Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \
    test_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P
  
  original_reg Reg P1 | get_tuple_element Reg P2 Dst=xy | succ(P1, P2) => \
    extract_next_element Dst | original_reg Reg P2
  
  get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst | original_reg Reg P
  
  original_reg Reg Pos =>
  
  original_reg/2
  
  extract_next_element D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \
  succ(P1, P2) | succ(D1, D2) => \
    extract_next_element2 D1 | original_reg Reg P2
  
  extract_next_element2 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \
  succ(P1, P2) | succ2(D1, D2) => \
    extract_next_element3 D1 | original_reg Reg P2
  
  #extract_next_element3 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \
  #succ(P1, P2) | succ3(D1, D2) => \
  #  extract_next_element4 D1 | original_reg Reg P2
  
  %macro: extract_next_element ExtractNextElement -pack
  extract_next_element x
  extract_next_element y
  
  %macro: extract_next_element2 ExtractNextElement2 -pack
  extract_next_element2 x
  extract_next_element2 y
  
  %macro: extract_next_element3 ExtractNextElement3 -pack
  extract_next_element3 x
  extract_next_element3 y
  
  #%macro: extract_next_element4 ExtractNextElement4 -pack
  #extract_next_element4 x
  #extract_next_element4 y
  
  is_integer Fail=f i =>
  is_integer Fail=f an => jump Fail
  is_integer Fail Literal=q => move Literal x | is_integer Fail x
  
  is_integer Fail=f S=rx | allocate Need Regs => is_integer_allocate Fail S Need Regs
  
  %macro: is_integer_allocate IsIntegerAllocate -fail_action
  is_integer_allocate f x I I
  is_integer_allocate f r I I
  
  %macro: is_integer IsInteger -fail_action
  is_integer f x
  is_integer f y
  is_integer f r
  
  is_list Fail=f n =>
  is_list Fail Literal=q => move Literal x | is_list Fail x
  is_list Fail=f c => jump Fail
  %macro: is_list IsList -fail_action
  is_list f r
  is_list f x
  %cold
  is_list f y
  %hot
  
  is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
  
  %macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack
  is_nonempty_list_allocate f x I t
  is_nonempty_list_allocate f r I t
  
  is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2
  
  %macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack
  is_non_empty_list_test_heap f r I t
  
  %macro: is_nonempty_list IsNonemptyList -fail_action
  is_nonempty_list f x
  is_nonempty_list f y
  is_nonempty_list f r
  
  %macro: is_atom IsAtom -fail_action
  is_atom f x
  is_atom f r
  %cold
  is_atom f y
  %hot
  is_atom Fail=f a =>
  is_atom Fail=f niq => jump Fail
  
  %macro: is_float IsFloat -fail_action
  is_float f r
  is_float f x
  %cold
  is_float f y
  %hot
  is_float Fail=f nai => jump Fail
  is_float Fail Literal=q => move Literal x | is_float Fail x
  
  is_nil Fail=f n =>
  is_nil Fail=f qia => jump Fail
  
  %macro: is_nil IsNil -fail_action
  is_nil f x
  is_nil f y
  is_nil f r
  
  is_binary Fail Literal=q => move Literal x | is_binary Fail x
  is_binary Fail=f c => jump Fail
  %macro: is_binary IsBinary -fail_action
  is_binary f r
  is_binary f x
  %cold
  is_binary f y
  %hot
  
  # XXX Deprecated.
  is_bitstr Fail Term => is_bitstring Fail Term
  
  is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x
  is_bitstring Fail=f c => jump Fail
  %macro: is_bitstring IsBitstring -fail_action
  is_bitstring f r
  is_bitstring f x
  %cold
  is_bitstring f y
  %hot
  
  is_reference Fail=f cq => jump Fail
  %macro: is_reference IsRef -fail_action
  is_reference f r
  is_reference f x
  %cold
  is_reference f y
  %hot
  
  is_pid Fail=f cq => jump Fail
  %macro: is_pid IsPid -fail_action
  is_pid f r
  is_pid f x
  %cold
  is_pid f y
  %hot
  
  is_port Fail=f cq => jump Fail
  %macro: is_port IsPort -fail_action
  is_port f r
  is_port f x
  %cold
  is_port f y
  %hot
  
  is_boolean Fail=f a==am_true =>
  is_boolean Fail=f a==am_false =>
  is_boolean Fail=f ac => jump Fail
  
  %cold
  %macro: is_boolean IsBoolean -fail_action
  is_boolean f r
  is_boolean f x
  is_boolean f y
  %hot
  
  is_function2 Fail=f acq Arity => jump Fail
  is_function2 Fail=f Fun a => jump Fail
  is_function2 Fail Fun Literal=q => move Literal x | is_function2 Fail Fun x
  
  is_function2 f s s
  %macro: is_function2 IsFunction2 -fail_action
  
  # Allocating & initializing.
  allocate Need Regs | init Y => allocate_init Need Regs Y
  init Y1 | init Y2 => init2 Y1 Y2
  
  %macro: allocate_init AllocateInit -pack
  allocate_init t I y
  
  #################################################################
  # External function and bif calls.
  #################################################################
  
  #
  # The BIFs erts_internal:check_process_code/2 must be called like a function,
  # to ensure that c_p->i (program counter) is set correctly (an ordinary
  # BIF call doesn't set it).
  #
  
  call_ext u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext Bif
  call_ext_last u==2 Bif=u$bif:erts_internal:check_process_code/2 D => i_call_ext_last Bif D
  call_ext_only u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext_only Bif
  
  #
  # The BIFs erlang:garbage_collect/0 must be called like a function,
  # to allow them to invoke the garbage collector. (The stack pointer must
  # be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.)
  #
  
  call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif
  call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D
  call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif
  
  #
  # put/2 and erase/1 must be able to do garbage collection, so we must call
  # them like functions.
  #
  
  call_ext u==2 Bif=u$bif:erlang:put/2 => i_call_ext Bif
  call_ext_last u==2 Bif=u$bif:erlang:put/2 D => i_call_ext_last Bif D
  call_ext_only u==2 Bif=u$bif:erlang:put/2 => i_call_ext_only Bif
  
  call_ext u==1 Bif=u$bif:erlang:erase/1 => i_call_ext Bif
  call_ext_last u==1 Bif=u$bif:erlang:erase/1 D => i_call_ext_last Bif D
  call_ext_only u==1 Bif=u$bif:erlang:erase/1 => i_call_ext_only Bif
  
  #
  # The process_info/1,2 BIF should be called like a function, to force
  # the emulator to set c_p->current before calling it (a BIF call doesn't
  # set it).
  #
  # In addition, we force the use of a non-tail-recursive call.  This will ensure
  # that c_p->cp points into the function making the call.
  #
  
  call_ext u==1 Bif=u$bif:erlang:process_info/1 => i_call_ext Bif
  call_ext_last u==1 Bif=u$bif:erlang:process_info/1 D => i_call_ext Bif | deallocate_return D
  call_ext_only Ar=u==1 Bif=u$bif:erlang:process_info/1 => allocate u Ar | i_call_ext Bif | deallocate_return u
  
  call_ext u==2 Bif=u$bif:erlang:process_info/2 => i_call_ext Bif
  call_ext_last u==2 Bif=u$bif:erlang:process_info/2 D => i_call_ext Bif | deallocate_return D
  call_ext_only Ar=u==2 Bif=u$bif:erlang:process_info/2 => allocate u Ar | i_call_ext Bif | deallocate_return u
  
  #
  # load_nif/2 also needs to know calling function like process_info
  #
  call_ext u==2 Bif=u$bif:erlang:load_nif/2 => i_call_ext Bif
  call_ext_last u==2 Bif=u$bif:erlang:load_nif/2 D => i_call_ext Bif | deallocate_return D
  call_ext_only Ar=u==2 Bif=u$bif:erlang:load_nif/2 => allocate u Ar | i_call_ext Bif | deallocate_return u
  
  
  #
  # apply/2 is an instruction, not a BIF.
  #
  
  call_ext u==2 u$func:erlang:apply/2 => i_apply_fun
  call_ext_last u==2 u$func:erlang:apply/2 D => i_apply_fun_last D
  call_ext_only u==2 u$func:erlang:apply/2 => i_apply_fun_only
  
  #
  # The apply/3 BIF is an instruction.
  #
  
  call_ext u==3 u$bif:erlang:apply/3 => i_apply
  call_ext_last u==3 u$bif:erlang:apply/3 D => i_apply_last D
  call_ext_only u==3 u$bif:erlang:apply/3 => i_apply_only
  
  #
  # The exit/1 and throw/1 BIFs never execute the instruction following them;
  # thus there is no need to generate any return instruction.
  #
  
  call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif Bif
  call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif Bif
  
  call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif Bif
  call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif Bif
  
  #
  # The error/1 and error/2 BIFs never execute the instruction following them;
  # thus there is no need to generate any return instruction.
  # However, they generate stack backtraces, so if the call instruction
  # is call_ext_only/2 instruction, we explicitly do an allocate/2 to store
  # the continuation pointer on the stack.
  #
  
  call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif Bif
  call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif Bif
  
  call_ext_only Ar=u==1 Bif=u$bif:erlang:error/1 => \
    allocate u Ar | call_bif Bif
  call_ext_only Ar=u==2 Bif=u$bif:erlang:error/2 => \
    allocate u Ar | call_bif Bif
  
  #
  # The yield/0 BIF is an instruction
  #
  
  call_ext u==0 u$func:erlang:yield/0 => i_yield
  call_ext_last u==0 u$func:erlang:yield/0 D => i_yield | deallocate_return D
  call_ext_only u==0 u$func:erlang:yield/0 => i_yield | return
  
  #
  # The hibernate/3 BIF is an instruction.
  #
  call_ext u==3 u$func:erlang:hibernate/3 => i_hibernate
  call_ext_last u==3 u$func:erlang:hibernate/3 D => i_hibernate
  call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate
  
  #
  # If VM probes are not enabled, we want to short-circult calls to
  # the dt tag BIFs to make them as cheap as possible.
  #
  
  %unless USE_VM_PROBES
  
  call_ext Arity u$func:erlang:dt_get_tag/0 => \
      move a=am_undefined r
  call_ext_last Arity u$func:erlang:dt_get_tag/0 D => \
      move a=am_undefined r | deallocate D | return
  call_ext_only Arity u$func:erlang:dt_get_tag/0 => \
      move a=am_undefined r | return
  
  move Any r | call_ext Arity u$func:erlang:dt_put_tag/1 => \
      move a=am_undefined r
  move Any r | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \
      move a=am_undefined r | deallocate D | return
  move Any r | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \
      move a=am_undefined r | return
  call_ext Arity u$func:erlang:dt_put_tag/1 => \
      move a=am_undefined r
  call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \
      move a=am_undefined r | deallocate D | return
  call_ext_only Arity u$func:erlang:dt_put_tag/1 => \
      move a=am_undefined r | return
  
  call_ext Arity u$func:erlang:dt_get_tag_data/0 => \
      move a=am_undefined r
  call_ext_last Arity u$func:erlang:dt_get_tag_data/0 D => \
      move a=am_undefined r | deallocate D | return
  call_ext_only Arity u$func:erlang:dt_get_tag_data/0 => \
      move a=am_undefined r | return
  
  move Any r | call_ext Arity u$func:erlang:dt_spread_tag/1 => \
      move a=am_true r
  move Any r | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \
      move a=am_true r | deallocate D | return
  move Any r | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \
      move a=am_true r | return
  call_ext Arity u$func:erlang:dt_spread_tag/1 => \
      move a=am_true r
  call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \
      move a=am_true r | deallocate D | return
  call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \
      move a=am_true r | return
  
  move Any r | call_ext Arity u$func:erlang:dt_restore_tag/1 => \
      move a=am_true r
  move Any r | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \
      move a=am_true r | deallocate D | return
  move Any r | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \
      move a=am_true r | return
  call_ext Arity u$func:erlang:dt_restore_tag/1 => \
      move a=am_true r
  call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \
      move a=am_true r | deallocate D | return
  call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \
      move a=am_true r | return
  
  move Any r | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
      move Any r
  move Any r | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \
      move Any r | deallocate D | return
  move Any r | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
      move Any r | return
  call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 =>
  call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \
      deallocate D | return
  call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
      return
  
  move Any r | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \
      move Any r
  move Any r | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \
      move Any r | deallocate D | return
  move Any r | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \
      move Any r | return
  call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 =>
  call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \
      deallocate D | return
  call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \
      return
  
  # Can happen after one of the transformations above.
  move Discarded r | move Something r => move Something r
  
  %endif
  
  #
  # The general case for BIFs that have no special instructions.
  # A BIF used in the tail must be followed by a return instruction.
  #
  # To make trapping and stack backtraces work correctly, we make sure that
  # the continuation pointer is always stored on the stack.
  
  call_ext u Bif=u$is_bif => call_bif Bif
  
  call_ext_last u Bif=u$is_bif D => call_bif Bif | deallocate_return D
  
  call_ext_only Ar=u Bif=u$is_bif => \
    allocate u Ar | call_bif Bif | deallocate_return u
  
  #
  # Any remaining calls are calls to Erlang functions, not BIFs.
  # We rename the instructions to internal names.  This is necessary,
  # to avoid an end-less loop, because we want to call a few BIFs
  # with call instructions.
  #
  
  move S=c r | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S r Func
  move S=c r | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S r
  move S=c r | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S r 
  
  call_ext Ar Func        => i_call_ext Func
  call_ext_last Ar Func D => i_call_ext_last Func D
  call_ext_only Ar Func   => i_call_ext_only Func
  
  i_apply
  i_apply_last P
  i_apply_only
  
  i_apply_fun
  i_apply_fun_last P
  i_apply_fun_only
  
  i_hibernate
  
  call_bif e
  
  #
  # Calls to non-building and guard BIFs.
  #
  
  bif0 u$bif:erlang:self/0 Dst=d => self Dst
  bif0 u$bif:erlang:node/0 Dst=d => node Dst
  
  bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst
  
  bif2 Jump=j u$bif:erlang:element/2 S1=s S2=rxy Dst=d => gen_element(Jump, S1, S2, Dst)
  
  bif1 Fail Bif Literal=q Dst => move Literal x | bif1 Fail Bif x Dst
  bif1 p Bif S1 Dst => bif1_body Bif S1 Dst
  
  bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst
  
  bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst
  bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst
  
  i_get s d
  
  %macro: self Self
  self r
  self x
  self y
  
  %macro: node Node
  node r
  node x
  %cold
  node y
  %hot
  
  i_fast_element r j I d
  i_fast_element x j I d
  i_fast_element y j I d
  
  i_element r j s d
  i_element x j s d
  i_element y j s d
  
  bif1 f b s d
  bif1_body b s d
  i_bif2 f b d
  i_bif2_body b d
  
  #
  # Internal calls.
  #
  
  move S=c r | call Ar P=f => i_move_call S r P
  move S=s r | call Ar P=f => move_call S r P
  
  i_move_call c r f
  
  %macro:move_call MoveCall -arg_f -size -nonext
  move_call/3
  
  move_call x r f
  move_call y r f
  
  move S=c r | call_last Ar P=f D => i_move_call_last P D S r
  move S r | call_last Ar P=f D => move_call_last S r P D
  
  i_move_call_last f P c r
  
  %macro:move_call_last MoveCallLast -arg_f -nonext -pack
  
  move_call_last/4
  move_call_last x r f Q
  move_call_last y r f Q
  
  move S=c r | call_only Ar P=f => i_move_call_only P S r
  move S=x r | call_only Ar P=f => move_call_only S r P
  
  i_move_call_only f c r
  
  %macro:move_call_only MoveCallOnly -arg_f -nonext
  move_call_only/3
  
  move_call_only x r f
  
  call Ar Func        => i_call Func
  call_last Ar Func D => i_call_last Func D
  call_only Ar Func   => i_call_only Func
  
  i_call f
  i_call_last f P
  i_call_only f
  
  i_call_ext e
  i_call_ext_last e P
  i_call_ext_only e
  
  i_move_call_ext c r e
  i_move_call_ext_last e P c r
  i_move_call_ext_only e c r
  
  # Fun calls.
  
  call_fun Arity | deallocate D | return => i_call_fun_last Arity D
  call_fun Arity => i_call_fun Arity
  
  i_call_fun I
  i_call_fun_last I P
  
  make_fun2 OldIndex=u => gen_make_fun2(OldIndex)
  
  %macro: i_make_fun MakeFun -pack
  %cold
  i_make_fun I t
  %hot
  
  %macro: is_function IsFunction -fail_action
  is_function f x
  is_function f y
  is_function f r
  is_function Fail=f c => jump Fail
  
  func_info M F A => i_func_info u M F A
  
  # ================================================================
  # New bit syntax matching (R11B).
  # ================================================================
  
  %cold
  bs_start_match2 Fail=f ica X Y D => jump Fail
  bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
  i_bs_start_match2 r f I I d
  i_bs_start_match2 x f I I d
  i_bs_start_match2 y f I I d
  
  bs_save2 Reg Index => gen_bs_save(Reg, Index)
  i_bs_save2 r I
  i_bs_save2 x I
  
  bs_restore2 Reg Index => gen_bs_restore(Reg, Index)
  i_bs_restore2 r I
  i_bs_restore2 x I
  
  # Matching integers
  bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val
  
  i_bs_match_string r f I I
  i_bs_match_string x f I I
  
  # Fetching integers from binaries.
  bs_get_integer2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \
  			gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
  
  i_bs_get_integer_small_imm r I f I d
  i_bs_get_integer_small_imm x I f I d
  i_bs_get_integer_imm r I I f I d
  i_bs_get_integer_imm x I I f I d
  i_bs_get_integer f I I d
  i_bs_get_integer_8 r f d
  i_bs_get_integer_8 x f d
  i_bs_get_integer_16 r f d
  i_bs_get_integer_16 x f d
  i_bs_get_integer_32 r f I d
  i_bs_get_integer_32 x f I d
  
  # Fetching binaries from binaries.
  bs_get_binary2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \
  			gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
  
  %macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action -gen_dest
  %macro: i_bs_get_binary2 BsGetBinary_2 -fail_action -gen_dest
  %macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action -gen_dest
  
  i_bs_get_binary_imm2 f r I I I d
  i_bs_get_binary_imm2 f x I I I d
  i_bs_get_binary2 f r I s I d
  i_bs_get_binary2 f x I s I d
  i_bs_get_binary_all2 f r I I d
  i_bs_get_binary_all2 f x I I d
  i_bs_get_binary_all_reuse r f I
  i_bs_get_binary_all_reuse x f I
  
  # Fetching float from binaries.
  bs_get_float2 Fail=f Ms=rx Live=u Sz=s Unit=u Flags=u Dst=d => \
  		gen_get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
  
  bs_get_float2 Fail=f Ms=rx Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
  
  %macro: i_bs_get_float2 BsGetFloat2 -fail_action -gen_dest
  i_bs_get_float2 f r I s I d
  i_bs_get_float2 f x I s I d
  
  # Miscellanous
  
  bs_skip_bits2 Fail=f Ms=rx Sz=s Unit=u Flags=u => \
  			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
  bs_skip_bits2 Fail=f Ms=rx Sz=q Unit=u Flags=u => \
  			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
  
  %macro: i_bs_skip_bits_imm2 BsSkipBitsImm2 -fail_action
  i_bs_skip_bits_imm2 f r I
  i_bs_skip_bits_imm2 f x I
  
  %macro: i_bs_skip_bits2 BsSkipBits2 -fail_action
  i_bs_skip_bits2 f r x I
  i_bs_skip_bits2 f r y I
  i_bs_skip_bits2 f x x I
  i_bs_skip_bits2 f x r I
  i_bs_skip_bits2 f x y I
  
  %macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action
  i_bs_skip_bits_all2 f r I
  i_bs_skip_bits_all2 f x I 
  
  bs_test_tail2 Fail=f Ms=rx Bits=u==0 => bs_test_zero_tail2 Fail Ms
  bs_test_tail2 Fail=f Ms=rx Bits=u => bs_test_tail_imm2 Fail Ms Bits
  bs_test_zero_tail2 f r
  bs_test_zero_tail2 f x
  bs_test_tail_imm2 f r I
  bs_test_tail_imm2 f x I
  
  bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms
  bs_test_unit f r I
  bs_test_unit f x I
  bs_test_unit8 f r
  bs_test_unit8 f x
  
  bs_context_to_binary r
  bs_context_to_binary x
  bs_context_to_binary y
  
  #
  # Utf8/utf16/utf32 support. (R12B-5)
  #
  bs_get_utf8 Fail=f Ms=rx u u Dst=d => i_bs_get_utf8 Ms Fail Dst
  i_bs_get_utf8 r f d
  i_bs_get_utf8 x f d
  
  bs_skip_utf8 Fail=f Ms=rx u u => i_bs_get_utf8 Ms Fail x
  
  bs_get_utf16 Fail=f Ms=rx u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
  bs_skip_utf16 Fail=f Ms=rx u Flags=u => i_bs_get_utf16 Ms Fail Flags x
  
  i_bs_get_utf16 r f I d
  i_bs_get_utf16 x f I d
  
  bs_get_utf32 Fail=f Ms=rx Live=u Flags=u Dst=d => \
  	bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \
  	i_fetch Dst Ms | \
  	i_bs_validate_unicode_retract Fail
  bs_skip_utf32 Fail=f Ms=rx Live=u Flags=u => \
  	bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \
  	i_fetch x Ms | \
  	i_bs_validate_unicode_retract Fail
  
  i_bs_validate_unicode_retract j
  %hot
  
  #
  # Constructing binaries
  #
  %cold
  
  bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail
  
  bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst | should_gen_heap_bin(Sz) => \
  	i_bs_init_heap_bin Sz Regs Dst
  bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init Sz Regs Dst
  
  bs_init2 Fail Sz=u Words Regs Flags Dst | should_gen_heap_bin(Sz) => \
  	i_bs_init_heap_bin_heap Sz Words Regs Dst
  bs_init2 Fail Sz=u Words Regs Flags Dst => \
     i_bs_init_heap Sz Words Regs Dst
  
  bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
    i_bs_init_fail Sz Fail Regs Dst
  bs_init2 Fail Sz Words Regs Flags Dst => \
    i_fetch Sz r | i_bs_init_fail_heap Words Fail Regs Dst
  
  i_bs_init_fail r j I d
  i_bs_init_fail x j I d
  i_bs_init_fail y j I d
  
  i_bs_init_fail_heap I j I d
  
  i_bs_init I I d
  i_bs_init_heap_bin I I d
  
  i_bs_init_heap I I I d
  i_bs_init_heap_bin_heap I I I d
  
  
  bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail
  
  bs_init_bits Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init_bits Sz Regs Dst
  bs_init_bits Fail Sz=u Words Regs Flags Dst =>  i_bs_init_bits_heap Sz Words Regs Dst
  
  bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
    i_bs_init_bits_fail Sz Fail Regs Dst
  bs_init_bits Fail Sz Words Regs Flags Dst => \
    i_fetch Sz r | i_bs_init_bits_fail_heap Words Fail Regs Dst
  
  i_bs_init_bits_fail r j I d
  i_bs_init_bits_fail x j I d
  i_bs_init_bits_fail y j I d
  
  i_bs_init_bits_fail_heap I j I d
  
  i_bs_init_bits I I d
  i_bs_init_bits_heap I I I d
  
  bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
  bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D
  
  i_bs_add j I d
  
  bs_append Fail Size Extra Live Unit Bin Flags Dst => \
    i_fetch Size Bin | i_bs_append Fail Extra Live Unit Dst
  
  bs_private_append Fail Size Unit Bin Flags Dst => \
    i_fetch Size Bin | i_bs_private_append Fail Unit Dst
  
  bs_init_writable
  
  i_bs_append j I I I d
  i_bs_private_append j I d
  
  #
  # Storing integers into binaries.
  #
  
  bs_put_integer Fail=j Sz=s Unit=u Flags=u Literal=q => \
  	move Literal x | bs_put_integer Fail Sz Unit Flags x
  bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
  			gen_put_integer(Fail, Sz, Unit, Flags, Src)
  
  %macro: i_new_bs_put_integer NewBsPutInteger
  %macro: i_new_bs_put_integer_imm NewBsPutIntegerImm
  
  i_new_bs_put_integer j s I s
  i_new_bs_put_integer_imm j I I s
  
  #
  # Utf8/utf16/utf32 support. (R12B-5)
  #
  
  bs_utf8_size Fail Literal=q Dst=d => \
  	move Literal x | bs_utf8_size Fail x Dst
  bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst
  
  i_bs_utf8_size s d
  
  bs_utf16_size Fail Literal=q Dst=d => \
  	move Literal x | bs_utf16_size Fail x Dst
  bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst
  
  i_bs_utf16_size s d
  
  bs_put_utf8 Fail=j Flags=u Literal=q => \
  	move Literal x | bs_put_utf8 Fail Flags x
  bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src
  
  i_bs_put_utf8 j s
  
  bs_put_utf16 Fail=j Flags=u Literal=q => \
  	move Literal x | bs_put_utf16 Fail Flags x
  bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src
  
  i_bs_put_utf16 j I s
  
  bs_put_utf32 Fail=j Flags=u Literal=q => \
  	move Literal x | bs_put_utf32 Fail Flags x
  bs_put_utf32 Fail=j Flags=u Src=s => \
     i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
  
  i_bs_validate_unicode j s
  
  #
  # Storing floats into binaries.
  #
  bs_put_float Fail Sz=q Unit Flags Val => badarg Fail
  
  bs_put_float Fail=j Sz Unit=u Flags=u Literal=q => \
  	move Literal x | bs_put_float Fail Sz Unit Flags x
  
  bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
  			gen_put_float(Fail, Sz, Unit, Flags, Src)
  
  %macro: i_new_bs_put_float NewBsPutFloat
  %macro: i_new_bs_put_float_imm NewBsPutFloatImm
  
  i_new_bs_put_float j s I s
  i_new_bs_put_float_imm j I I s
  
  #
  # Storing binaries into binaries.
  #
  
  bs_put_binary Fail Sz Unit Flags Literal=q => \
   	move Literal x | bs_put_binary Fail Sz Unit Flags x
  bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
  			gen_put_binary(Fail, Sz, Unit, Flags, Src)
  
  %macro: i_new_bs_put_binary NewBsPutBinary
  i_new_bs_put_binary j s I s
  
  %macro: i_new_bs_put_binary_imm NewBsPutBinaryImm
  i_new_bs_put_binary_imm j I s
  
  %macro: i_new_bs_put_binary_all NewBsPutBinaryAll
  i_new_bs_put_binary_all j s I
  
  #
  # Warning: The i_bs_put_string and i_new_bs_put_string instructions
  # are specially treated in the loader.
  # Don't change the instruction format unless you change the loader too.
  #
  
  bs_put_string I I
  
  %hot
  
  #
  # New floating point instructions (R8).
  #
  
  fadd p FR1 FR2 FR3 => i_fadd FR1 FR2 FR3
  fsub p FR1 FR2 FR3 => i_fsub FR1 FR2 FR3
  fmul p FR1 FR2 FR3 => i_fmul FR1 FR2 FR3
  fdiv p FR1 FR2 FR3 => i_fdiv FR1 FR2 FR3
  fnegate p FR1 FR2 => i_fnegate FR1 FR2
  
  fconv Arg=iqan Dst=l => move Arg x | fconv x Dst
  
  fmove q l
  fmove d l
  fmove l d
  
  fconv d l
  
  i_fadd l l l
  i_fsub l l l
  i_fmul l l l
  i_fdiv l l l
  i_fnegate l l
  
  fclearerror | no_fpe_signals() =>
  fcheckerror p | no_fpe_signals() =>
  fcheckerror p => i_fcheckerror
  
  i_fcheckerror
  fclearerror
  
  #
  # New apply instructions in R10B.
  #
  
  apply I
  apply_last I P
  
  #
  # Map instructions in R17.
  #
  
  sorted_put_map_assoc/5
  put_map_assoc F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
    sorted_put_map_assoc F Map Dst Live Size Rest
  
  sorted_put_map_exact/5
  put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
    sorted_put_map_exact F Map Dst Live Size Rest
  
  sorted_put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \
     new_map Dst Live Size Rest
  sorted_put_map_assoc F Src=s Dst Live Size Rest=* => \
  	       update_map_assoc F Src Dst Live Size Rest
  sorted_put_map_assoc F Src Dst Live Size Rest=* => \
  	       move Src x | update_map_assoc F x Dst Live Size Rest
  
  sorted_put_map_exact F Src=s Dst Live Size Rest=* => \
  	      update_map_exact F Src Dst Live Size Rest
  sorted_put_map_exact F Src Dst Live Size Rest=* => \
  	      move Src x | update_map_exact F x Dst Live Size Rest
  
  new_map d I I
  update_map_assoc j s d I I
  update_map_exact j s d I I
  
  is_map Fail Lit=q | literal_is_map(Lit) =>
  is_map Fail cq => jump Fail
  
  %macro: is_map IsMap -fail_action
  is_map f r
  is_map f x
  is_map f y
  
  ## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements
  
  has_map_fields Fail Src Size Rest=* => \
     gen_has_map_fields(Fail, Src, Size, Rest)
  
  ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 }
  
  get_map_elements Fail Src=rxy Size=u==2 Rest=* => \
      gen_get_map_element(Fail, Src, Size, Rest)
  get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
     gen_get_map_elements(Fail, Src, Size, Rest)
  
  i_get_map_elements f s I
  
  i_get_map_element Fail Src=rxy Key=ry Dst => \
      move Key x | i_get_map_element Fail Src x Dst
  
  %macro: i_get_map_element_hash GetMapElementHash -fail_action
  i_get_map_element_hash f r c I r
  i_get_map_element_hash f x c I r
  i_get_map_element_hash f y c I r
  i_get_map_element_hash f r c I x
  i_get_map_element_hash f x c I x
  i_get_map_element_hash f y c I x
  i_get_map_element_hash f r c I y
  i_get_map_element_hash f x c I y
  i_get_map_element_hash f y c I y
  
  %macro: i_get_map_element GetMapElement -fail_action
  i_get_map_element f r x r
  i_get_map_element f x x r
  i_get_map_element f y x r
  i_get_map_element f r x x
  i_get_map_element f x x x
  i_get_map_element f y x x
  i_get_map_element f r x y
  i_get_map_element f x x y
  i_get_map_element f y x y
  
  #
  # Optimize addition and subtraction of small literals using
  # the i_increment/4 instruction (in bodies, not in guards).
  #
  
  gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \
  	gen_increment(Reg, Int, Live, Dst)
  gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \
  	gen_increment(Reg, Int, Live, Dst)
  
  gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \
  	negation_is_small(Int) => \
  	gen_increment_from_minus(Reg, Int, Live, Dst)
  
  #
  # GCing arithmetic instructions.
  #
  
  gc_bif2 Fail I u$bif:erlang:splus/2 S1=x S2=x Dst=d => i_plus Fail I S1 S2 Dst
  gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst
  gc_bif2 Fail I u$bif:erlang:sminus/2 S1=x S2=x Dst=d => i_minus Fail I S1 S2 Dst
  gc_bif2 Fail I u$bif:erlang:sminus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_minus Fail I Dst
  gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail I Dst
  gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst
  
  gc_bif2 Fail I u$bif:erlang:intdiv/2 S1 S2 Dst=d => i_fetch S1 S2 | i_int_div Fail I Dst
  gc_bif2 Fail I u$bif:erlang:rem/2 S1=x S2=x Dst=d => i_rem Fail I S1 S2 Dst
  gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Dst
  
  gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst
  gc_bif2 Fail I u$bif:erlang:bsr/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsr Fail I Dst
  
  gc_bif2 Fail I u$bif:erlang:band/2 S1=x S2=c Dst=d => i_band Fail I S1 S2 Dst
  gc_bif2 Fail I u$bif:erlang:band/2 S1 S2 Dst=d => i_fetch S1 S2 | i_band Fail I Dst
  gc_bif2 Fail I u$bif:erlang:bor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bor Fail I Dst
  gc_bif2 Fail I u$bif:erlang:bxor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bxor Fail I Dst
  
  gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
  
  gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst
  gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst
  
  i_increment r I I d
  i_increment x I I d
  i_increment y I I d
  
  i_plus j I x x d
  i_plus j I d
  i_minus j I x x d
  i_minus j I d
  i_times j I d
  i_m_div j I d
  i_int_div j I d
  i_rem j I x x d
  i_rem j I d
  
  i_bsl j I d
  i_bsr j I d
  
  i_band j I x c d
  i_band j I d
  i_bor j I d
  i_bxor j I d
  
  i_int_bnot j s I d
  
  #
  # Old guard BIFs that creates heap fragments are no longer allowed.
  #
  bif1 Fail u$bif:erlang:length/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:size/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:abs/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:float/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:round/1 s d => too_old_compiler
  bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler
  
  #
  # Guard BIFs.
  #
  gc_bif1 Fail I Bif Src Dst => \
  	gen_guard_bif1(Fail, I, Bif, Src, Dst)
  
  gc_bif2 Fail I Bif S1 S2 Dst => \
  	gen_guard_bif2(Fail, I, Bif, S1, S2, Dst)
  
  gc_bif3 Fail I Bif S1 S2 S3 Dst => \
  	gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst)
  
  i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D
  
  i_gc_bif1 j I s I d
  
  ii_gc_bif2/6
  
  ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D
  
  i_gc_bif2 j I I d
  
  ii_gc_bif3/7
  
  ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D
  
  i_gc_bif3 j I s I d
  
  #
  # The following instruction is specially handled in beam_load.c
  # to produce a user-friendly message if an unsupported guard BIF is
  # encountered.
  #
  unsupported_guard_bif/3
  unsupported_guard_bif A B C | never() =>
  
  #
  # R13B03
  #
  on_load
  
  #
  # R14A.
  #
  recv_mark f
  
  recv_set Fail | label Lbl | loop_rec Lf Reg => \
     i_recv_set | label Lbl | loop_rec Lf Reg
  i_recv_set
  