text: // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
  // for details. All rights reserved. Use of this source code is governed by a
  // BSD-style license that can be found in the LICENSE file.
  
  #ifndef RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_ARM64_H_
  #define RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_ARM64_H_
  
  #if defined(DART_PRECOMPILED_RUNTIME)
  #error "AOT runtime should not use compiler sources (including header files)"
  #endif  // defined(DART_PRECOMPILED_RUNTIME)
  
  #ifndef RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_H_
  #error Do not include assembler_arm64.h directly; use assembler.h instead.
  #endif
  
  #include <functional>
  
  #include "platform/assert.h"
  #include "platform/utils.h"
  #include "vm/class_id.h"
  #include "vm/compiler/assembler/assembler_base.h"
  #include "vm/constants.h"
  #include "vm/hash_map.h"
  #include "vm/simulator.h"
  
  namespace dart {
  
  // Forward declarations.
  class FlowGraphCompiler;
  class RuntimeEntry;
  class RegisterSet;
  
  namespace compiler {
  
  class Immediate : public ValueObject {
   public:
    explicit Immediate(int64_t value) : value_(value) {}
  
    Immediate(const Immediate& other) : ValueObject(), value_(other.value_) {}
    Immediate& operator=(const Immediate& other) {
      value_ = other.value_;
      return *this;
    }
  
   private:
    int64_t value_;
  
    int64_t value() const { return value_; }
  
    friend class Assembler;
  };
  
  class Arm64Encode : public AllStatic {
   public:
    static inline uint32_t Rd(Register rd) {
      ASSERT(rd <= ZR);
      return static_cast<uint32_t>(ConcreteRegister(rd)) << kRdShift;
    }
  
    static inline uint32_t Rm(Register rm) {
      ASSERT(rm <= ZR);
      return static_cast<uint32_t>(ConcreteRegister(rm)) << kRmShift;
    }
  
    static inline uint32_t Rn(Register rn) {
      ASSERT(rn <= ZR);
      return static_cast<uint32_t>(ConcreteRegister(rn)) << kRnShift;
    }
  
    static inline uint32_t Ra(Register ra) {
      ASSERT(ra <= ZR);
      return static_cast<uint32_t>(ConcreteRegister(ra)) << kRaShift;
    }
  
    static inline uint32_t Rs(Register rs) {
      ASSERT(rs <= ZR);
      return static_cast<uint32_t>(ConcreteRegister(rs)) << kRsShift;
    }
  
    static inline uint32_t Rt(Register rt) {
      ASSERT(rt <= ZR);
      return static_cast<uint32_t>(ConcreteRegister(rt)) << kRtShift;
    }
  
    static inline uint32_t Rt2(Register rt2) {
      ASSERT(rt2 <= ZR);
      return static_cast<uint32_t>(ConcreteRegister(rt2)) << kRt2Shift;
    }
  };
  
  class Address : public ValueObject {
   public:
    Address(const Address& other)
        : ValueObject(),
          encoding_(other.encoding_),
          type_(other.type_),
          base_(other.base_),
          log2sz_(other.log2sz_) {}
  
    Address& operator=(const Address& other) {
      encoding_ = other.encoding_;
      type_ = other.type_;
      base_ = other.base_;
      log2sz_ = other.log2sz_;
      return *this;
    }
  
    enum AddressType {
      Offset,
      PreIndex,
      PostIndex,
      PairOffset,
      PairPreIndex,
      PairPostIndex,
      Reg,
      PCOffset,
      Unknown,
    };
  
    // If we are doing pre-/post-indexing, and the base and result registers are
    // the same, then the result is unpredictable. This kind of instruction is
    // actually illegal on some microarchitectures.
    bool can_writeback_to(Register r) const {
      if (type() == PreIndex || type() == PostIndex || type() == PairPreIndex ||
          type() == PairPostIndex) {
        return base() != r;
      }
      return true;
    }
  
    // Offset is in bytes. For the unsigned imm12 case, we unscale based on the
    // operand size, and assert that offset is aligned accordingly.
    // For the smaller signed imm9 case, the offset is the number of bytes, but
    // is unscaled.
    Address(Register rn,
            int32_t offset = 0,
            AddressType at = Offset,
            OperandSize sz = kDoubleWord) {
      ASSERT((rn != kNoRegister) && (rn != R31) && (rn != ZR));
      ASSERT(CanHoldOffset(offset, at, sz));
      log2sz_ = -1;
      const int32_t scale = Log2OperandSizeBytes(sz);
      if ((at == Offset) && Utils::IsUint(12 + scale, offset) &&
          (offset == ((offset >> scale) << scale))) {
        encoding_ =
            B24 | ((offset >> scale) << kImm12Shift) | Arm64Encode::Rn(rn);
        if (offset != 0) {
          log2sz_ = scale;
        }
      } else if ((at == Offset) && Utils::IsInt(9, offset)) {
        encoding_ = ((offset & 0x1ff) << kImm9Shift) | Arm64Encode::Rn(rn);
      } else if ((at == PreIndex) || (at == PostIndex)) {
        ASSERT(Utils::IsInt(9, offset));
        int32_t idx = (at == PostIndex) ? B10 : (B11 | B10);
        encoding_ = idx | ((offset & 0x1ff) << kImm9Shift) | Arm64Encode::Rn(rn);
      } else {
        ASSERT((at == PairOffset) || (at == PairPreIndex) ||
               (at == PairPostIndex));
        ASSERT(Utils::IsInt(7 + scale, offset) &&
               (static_cast<uint32_t>(offset) ==
                ((static_cast<uint32_t>(offset) >> scale) << scale)));
        int32_t idx = 0;
        switch (at) {
          case PairPostIndex:
            idx = B23;
            break;
          case PairPreIndex:
            idx = B24 | B23;
            break;
          case PairOffset:
            idx = B24;
            break;
          default:
            UNREACHABLE();
            break;
        }
        encoding_ =
            idx |
            ((static_cast<uint32_t>(offset >> scale) << kImm7Shift) & kImm7Mask) |
            Arm64Encode::Rn(rn);
        if (offset != 0) {
          log2sz_ = scale;
        }
      }
      type_ = at;
      base_ = ConcreteRegister(rn);
    }
  
    // This addressing mode does not exist.
    Address(Register rn,
            Register offset,
            AddressType at,
            OperandSize sz = kDoubleWord);
  
    static bool CanHoldOffset(int32_t offset,
                              AddressType at = Offset,
                              OperandSize sz = kDoubleWord) {
      if (at == Offset) {
        // Offset fits in 12 bit unsigned and has right alignment for sz,
        // or fits in 9 bit signed offset with no alignment restriction.
        const int32_t scale = Log2OperandSizeBytes(sz);
        return (Utils::IsUint(12 + scale, offset) &&
                (offset == ((offset >> scale) << scale))) ||
               (Utils::IsInt(9, offset));
      } else if (at == PCOffset) {
        return Utils::IsInt(21, offset) && (offset == ((offset >> 2) << 2));
      } else if ((at == PreIndex) || (at == PostIndex)) {
        return Utils::IsInt(9, offset);
      } else {
        ASSERT((at == PairOffset) || (at == PairPreIndex) ||
               (at == PairPostIndex));
        const int32_t scale = Log2OperandSizeBytes(sz);
        return (Utils::IsInt(7 + scale, offset) &&
                (static_cast<uint32_t>(offset) ==
                 ((static_cast<uint32_t>(offset) >> scale) << scale)));
      }
    }
  
    // PC-relative load address.
    static Address PC(int32_t pc_off) {
      ASSERT(CanHoldOffset(pc_off, PCOffset));
      Address addr;
      addr.encoding_ = (((pc_off >> 2) << kImm19Shift) & kImm19Mask);
      addr.base_ = kNoRegister;
      addr.type_ = PCOffset;
      addr.log2sz_ = -1;
      return addr;
    }
  
    static Address Pair(Register rn,
                        int32_t offset = 0,
                        AddressType at = PairOffset,
                        OperandSize sz = kDoubleWord) {
      return Address(rn, offset, at, sz);
    }
  
    // This addressing mode does not exist.
    static Address PC(Register r);
  
    enum Scaling {
      Unscaled,
      Scaled,
    };
  
    // Base register rn with offset rm. rm is sign-extended according to ext.
    // If ext is UXTX, rm may be optionally scaled by the
    // Log2OperandSize (specified by the instruction).
    Address(Register rn,
            Register rm,
            Extend ext = UXTX,
            Scaling scale = Unscaled) {
      ASSERT((rn != R31) && (rn != ZR));
      ASSERT((rm != R31) && (rm != CSP));
      // Can only scale when ext = UXTX.
      ASSERT((scale != Scaled) || (ext == UXTX));
      ASSERT((ext == UXTW) || (ext == UXTX) || (ext == SXTW) || (ext == SXTX));
      const int32_t s = (scale == Scaled) ? B12 : 0;
      encoding_ = B21 | B11 | s | Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm) |
                  (static_cast<int32_t>(ext) << kExtendTypeShift);
      type_ = Reg;
      base_ = ConcreteRegister(rn);
      log2sz_ = -1;  // Any.
    }
  
    static OperandSize OperandSizeFor(intptr_t cid) {
      switch (cid) {
        case kArrayCid:
        case kImmutableArrayCid:
        case kTypeArgumentsCid:
          return kWord;
        case kOneByteStringCid:
        case kExternalOneByteStringCid:
          return kByte;
        case kTwoByteStringCid:
        case kExternalTwoByteStringCid:
          return kHalfword;
        case kTypedDataInt8ArrayCid:
          return kByte;
        case kTypedDataUint8ArrayCid:
        case kTypedDataUint8ClampedArrayCid:
        case kExternalTypedDataUint8ArrayCid:
        case kExternalTypedDataUint8ClampedArrayCid:
          return kUnsignedByte;
        case kTypedDataInt16ArrayCid:
          return kHalfword;
        case kTypedDataUint16ArrayCid:
          return kUnsignedHalfword;
        case kTypedDataInt32ArrayCid:
          return kWord;
        case kTypedDataUint32ArrayCid:
          return kUnsignedWord;
        case kTypedDataInt64ArrayCid:
        case kTypedDataUint64ArrayCid:
          return kDWord;
        case kTypedDataFloat32ArrayCid:
          return kSWord;
        case kTypedDataFloat64ArrayCid:
          return kDWord;
        case kTypedDataFloat32x4ArrayCid:
        case kTypedDataInt32x4ArrayCid:
        case kTypedDataFloat64x2ArrayCid:
          return kQWord;
        case kTypedDataInt8ArrayViewCid:
          UNREACHABLE();
          return kByte;
        default:
          UNREACHABLE();
          return kByte;
      }
    }
  
   private:
    uint32_t encoding() const { return encoding_; }
    AddressType type() const { return type_; }
    Register base() const { return base_; }
  
    Address() : encoding_(0), type_(Unknown), base_(kNoRegister) {}
  
    uint32_t encoding_;
    AddressType type_;
    Register base_;
    int32_t log2sz_;  // Required log2 of operand size (-1 means any).
  
    friend class Assembler;
  };
  
  class FieldAddress : public Address {
   public:
    FieldAddress(Register base, int32_t disp, OperandSize sz = kDoubleWord)
        : Address(base, disp - kHeapObjectTag, Offset, sz) {}
  
    // This addressing mode does not exist.
    FieldAddress(Register base, Register disp, OperandSize sz = kDoubleWord);
  
    FieldAddress(const FieldAddress& other) : Address(other) {}
  
    FieldAddress& operator=(const FieldAddress& other) {
      Address::operator=(other);
      return *this;
    }
  };
  
  class Operand : public ValueObject {
   public:
    enum OperandType {
      Shifted,
      Extended,
      Immediate,
      BitfieldImm,
      Unknown,
    };
  
    // Data-processing operand - Uninitialized.
    Operand() : encoding_(-1), type_(Unknown) {}
  
    // Data-processing operands - Copy constructor.
    Operand(const Operand& other)
        : ValueObject(), encoding_(other.encoding_), type_(other.type_) {}
  
    Operand& operator=(const Operand& other) {
      type_ = other.type_;
      encoding_ = other.encoding_;
      return *this;
    }
  
    explicit Operand(Register rm) {
      ASSERT((rm != R31) && (rm != CSP));
      encoding_ = Arm64Encode::Rm(rm);
      type_ = Shifted;
    }
  
    Operand(Register rm, Shift shift, int32_t imm) {
      ASSERT(Utils::IsUint(6, imm));
      ASSERT((rm != R31) && (rm != CSP));
      encoding_ = (imm << kImm6Shift) | Arm64Encode::Rm(rm) |
                  (static_cast<int32_t>(shift) << kShiftTypeShift);
      type_ = Shifted;
    }
  
    // This operand type does not exist.
    Operand(Register rm, Shift shift, Register r);
  
    Operand(Register rm, Extend extend, int32_t imm) {
      ASSERT(Utils::IsUint(3, imm));
      ASSERT((rm != R31) && (rm != CSP));
      encoding_ = B21 | Arm64Encode::Rm(rm) |
                  (static_cast<int32_t>(extend) << kExtendTypeShift) |
                  ((imm & 0x7) << kImm3Shift);
      type_ = Extended;
    }
  
    // This operand type does not exist.
    Operand(Register rm, Extend extend, Register r);
  
    explicit Operand(int32_t imm) {
      if (Utils::IsUint(12, imm)) {
        encoding_ = imm << kImm12Shift;
      } else {
        // imm only has bits in [12, 24) set.
        ASSERT(((imm & 0xfff) == 0) && (Utils::IsUint(12, imm >> 12)));
        encoding_ = B22 | ((imm >> 12) << kImm12Shift);
      }
      type_ = Immediate;
    }
  
    // Encodes the value of an immediate for a logical operation.
    // Since these values are difficult to craft by hand, instead pass the
    // logical mask to the function IsImmLogical to get n, imm_s, and
    // imm_r.  Takes s before r like DecodeBitMasks from Appendix G but unlike
    // the disassembly of the *bfm instructions.
    Operand(uint8_t n, int8_t imm_s, int8_t imm_r) {
      ASSERT((n == 1) || (n == 0));
      ASSERT(Utils::IsUint(6, imm_s) && Utils::IsUint(6, imm_r));
      type_ = BitfieldImm;
      encoding_ = (static_cast<int32_t>(n) << kNShift) |
                  (static_cast<int32_t>(imm_s) << kImmSShift) |
                  (static_cast<int32_t>(imm_r) << kImmRShift);
    }
  
    // Test if a given value can be encoded in the immediate field of a logical
    // instruction.
    // If it can be encoded, the function returns true, and values pointed to by
    // n, imm_s and imm_r are updated with immediates encoded in the format
    // required by the corresponding fields in the logical instruction.
    // If it can't be encoded, the function returns false, and the operand is
    // undefined.
    static bool IsImmLogical(uint64_t value, uint8_t width, Operand* imm_op);
  
    // An immediate imm can be an operand to add/sub when the return value is
    // Immediate, or a logical operation over sz bits when the return value is
    // BitfieldImm. If the return value is Unknown, then the immediate can't be
    // used as an operand in either instruction. The encoded operand is written
    // to op.
    static OperandType CanHold(int64_t imm, uint8_t sz, Operand* op) {
      ASSERT(op != NULL);
      ASSERT((sz == kXRegSizeInBits) || (sz == kWRegSizeInBits));
      if (Utils::IsUint(12, imm)) {
        op->encoding_ = imm << kImm12Shift;
        op->type_ = Immediate;
      } else if (((imm & 0xfff) == 0) && (Utils::IsUint(12, imm >> 12))) {
        op->encoding_ = B22 | ((imm >> 12) << kImm12Shift);
        op->type_ = Immediate;
      } else if (IsImmLogical(imm, sz, op)) {
        op->type_ = BitfieldImm;
      } else {
        op->encoding_ = 0;
        op->type_ = Unknown;
      }
      return op->type_;
    }
  
   private:
    uint32_t encoding() const { return encoding_; }
    OperandType type() const { return type_; }
  
    uint32_t encoding_;
    OperandType type_;
  
    friend class Assembler;
  };
  
  class Assembler : public AssemblerBase {
   public:
    explicit Assembler(ObjectPoolBuilder* object_pool_builder,
                       bool use_far_branches = false);
    ~Assembler() {}
  
    void PushRegister(Register r) { Push(r); }
    void PopRegister(Register r) { Pop(r); }
  
    void PushRegisterPair(Register r0, Register r1) { PushPair(r0, r1); }
    void PopRegisterPair(Register r0, Register r1) { PopPair(r0, r1); }
  
    void PushRegisters(const RegisterSet& registers);
    void PopRegisters(const RegisterSet& registers);
  
    // Push all registers which are callee-saved according to the ARM64 ABI.
    void PushNativeCalleeSavedRegisters();
  
    // Pop all registers which are callee-saved according to the ARM64 ABI.
    void PopNativeCalleeSavedRegisters();
  
    void MoveRegister(Register rd, Register rn) {
      if (rd != rn) {
        mov(rd, rn);
      }
    }
  
    void Drop(intptr_t stack_elements) {
      ASSERT(stack_elements >= 0);
      if (stack_elements > 0) {
        AddImmediate(SP, SP, stack_elements * target::kWordSize);
      }
    }
  
    void Bind(Label* label);
    void Jump(Label* label) { b(label); }
  
    void LoadField(Register dst, FieldAddress address) { ldr(dst, address); }
    void LoadMemoryValue(Register dst, Register base, int32_t offset) {
      LoadFromOffset(dst, base, offset, kDoubleWord);
    }
    void StoreMemoryValue(Register src, Register base, int32_t offset) {
      StoreToOffset(src, base, offset, kDoubleWord);
    }
    void LoadAcquire(Register dst, Register address, int32_t offset = 0) {
      if (offset != 0) {
        AddImmediate(TMP2, address, offset);
        ldar(dst, TMP2);
      } else {
        ldar(dst, address);
      }
    }
    void StoreRelease(Register src, Register address, int32_t offset = 0) {
      if (offset != 0) {
        AddImmediate(TMP2, address, offset);
        stlr(src, TMP2);
      } else {
        stlr(src, address);
      }
    }
  
    void CompareWithFieldValue(Register value, FieldAddress address) {
      CompareWithMemoryValue(value, address);
    }
  
    void CompareWithMemoryValue(Register value, Address address) {
      ldr(TMP, address);
      cmp(value, Operand(TMP));
    }
  
    void CompareTypeNullabilityWith(Register type, int8_t value) {
      ldr(TMP,
          FieldAddress(type, compiler::target::Type::nullability_offset(), kByte),
          kUnsignedByte);
      cmp(TMP, Operand(value));
    }
  
    bool use_far_branches() const {
      return FLAG_use_far_branches || use_far_branches_;
    }
  
    void set_use_far_branches(bool b) { use_far_branches_ = b; }
  
    // Debugging and bringup support.
    void Breakpoint() override { brk(0); }
  
    void SetPrologueOffset() {
      if (prologue_offset_ == -1) {
        prologue_offset_ = CodeSize();
      }
    }
  
    void ReserveAlignedFrameSpace(intptr_t frame_space);
  
    // In debug mode, this generates code to check that:
    //   FP + kExitLinkSlotFromEntryFp == SP
    // or triggers breakpoint otherwise.
    void EmitEntryFrameVerification();
  
    // Instruction pattern from entrypoint is used in Dart frame prologs
    // to set up the frame and save a PC which can be used to figure out the
    // RawInstruction object corresponding to the code running in the frame.
    static const intptr_t kEntryPointToPcMarkerOffset = 0;
    static intptr_t EntryPointToPcMarkerOffset() {
      return kEntryPointToPcMarkerOffset;
    }
  
    // Emit data (e.g encoded instruction or immediate) in instruction stream.
    void Emit(int32_t value);
    void Emit64(int64_t value);
  
    // On some other platforms, we draw a distinction between safe and unsafe
    // smis.
    static bool IsSafe(const Object& object) { return true; }
    static bool IsSafeSmi(const Object& object) { return target::IsSmi(object); }
  
    // Addition and subtraction.
    // For add and sub, to use CSP for rn, o must be of type Operand::Extend.
    // For an unmodified rm in this case, use Operand(rm, UXTX, 0);
    void add(Register rd, Register rn, Operand o) {
      AddSubHelper(kDoubleWord, false, false, rd, rn, o);
    }
    void adds(Register rd, Register rn, Operand o) {
      AddSubHelper(kDoubleWord, true, false, rd, rn, o);
    }
    void addw(Register rd, Register rn, Operand o) {
      AddSubHelper(kWord, false, false, rd, rn, o);
    }
    void addsw(Register rd, Register rn, Operand o) {
      AddSubHelper(kWord, true, false, rd, rn, o);
    }
    void sub(Register rd, Register rn, Operand o) {
      AddSubHelper(kDoubleWord, false, true, rd, rn, o);
    }
    void subs(Register rd, Register rn, Operand o) {
      AddSubHelper(kDoubleWord, true, true, rd, rn, o);
    }
    void subw(Register rd, Register rn, Operand o) {
      AddSubHelper(kWord, false, true, rd, rn, o);
    }
    void subsw(Register rd, Register rn, Operand o) {
      AddSubHelper(kWord, true, true, rd, rn, o);
    }
  
    // Addition and subtraction with carry.
    void adc(Register rd, Register rn, Register rm) {
      AddSubWithCarryHelper(kDoubleWord, false, false, rd, rn, rm);
    }
    void adcs(Register rd, Register rn, Register rm) {
      AddSubWithCarryHelper(kDoubleWord, true, false, rd, rn, rm);
    }
    void adcw(Register rd, Register rn, Register rm) {
      AddSubWithCarryHelper(kWord, false, false, rd, rn, rm);
    }
    void adcsw(Register rd, Register rn, Register rm) {
      AddSubWithCarryHelper(kWord, true, false, rd, rn, rm);
    }
    void sbc(Register rd, Register rn, Register rm) {
      AddSubWithCarryHelper(kDoubleWord, false, true, rd, rn, rm);
    }
    void sbcs(Register rd, Register rn, Register rm) {
      AddSubWithCarryHelper(kDoubleWord, true, true, rd, rn, rm);
    }
    void sbcw(Register rd, Register rn, Register rm) {
      AddSubWithCarryHelper(kWord, false, true, rd, rn, rm);
    }
    void sbcsw(Register rd, Register rn, Register rm) {
      AddSubWithCarryHelper(kWord, true, true, rd, rn, rm);
    }
  
    // PC relative immediate add. imm is in bytes.
    void adr(Register rd, const Immediate& imm) { EmitPCRelOp(ADR, rd, imm); }
  
    // Bitfield operations.
    // Bitfield move.
    // If s >= r then Rd[s-r:0] := Rn[s:r], else Rd[bitwidth+s-r:bitwidth-r] :=
    // Rn[s:0].
    void bfm(Register rd,
             Register rn,
             int r_imm,
             int s_imm,
             OperandSize size = kDoubleWord) {
      EmitBitfieldOp(BFM, rd, rn, r_imm, s_imm, size);
    }
  
    // Signed bitfield move.
    void sbfm(Register rd,
              Register rn,
              int r_imm,
              int s_imm,
              OperandSize size = kDoubleWord) {
      EmitBitfieldOp(SBFM, rd, rn, r_imm, s_imm, size);
    }
  
    // Unsigned bitfield move.
    void ubfm(Register rd,
              Register rn,
              int r_imm,
              int s_imm,
              OperandSize size = kDoubleWord) {
      EmitBitfieldOp(UBFM, rd, rn, r_imm, s_imm, size);
    }
  
    // Bitfield insert.  Takes the low width bits and replaces bits in rd with
    // them, starting at low_bit.
    void bfi(Register rd,
             Register rn,
             int low_bit,
             int width,
             OperandSize size = kDoubleWord) {
      int wordsize = size == kDoubleWord ? 64 : 32;
      EmitBitfieldOp(BFM, rd, rn, -low_bit & (wordsize - 1), width - 1, size);
    }
  
    // Bitfield extract and insert low.  Takes width bits, starting at low_bit and
    // replaces the low width bits of rd with them.
    void bfxil(Register rd,
               Register rn,
               int low_bit,
               int width,
               OperandSize size = kDoubleWord) {
      EmitBitfieldOp(BFM, rd, rn, low_bit, low_bit + width - 1, size);
    }
  
    // Signed bitfield insert in zero.  Takes the low width bits, sign extends
    // them and writes them to rd, starting at low_bit, and zeroing bits below
    // that.
    void sbfiz(Register rd,
               Register rn,
               int low_bit,
               int width,
               OperandSize size = kDoubleWord) {
      int wordsize = size == kDoubleWord ? 64 : 32;
      EmitBitfieldOp(SBFM, rd, rn, (wordsize - low_bit) & (wordsize - 1),
                     width - 1, size);
    }
  
    // Signed bitfield extract.  Takes width bits, starting at low_bit, sign
    // extends them and writes them to rd, starting at the lowest bit.
    void sbfx(Register rd,
              Register rn,
              int low_bit,
              int width,
              OperandSize size = kDoubleWord) {
      EmitBitfieldOp(SBFM, rd, rn, low_bit, low_bit + width - 1, size);
    }
  
    // Unsigned bitfield insert in zero.  Takes the low width bits and writes
    // them to rd, starting at low_bit, and zeroing bits above and below.
    void ubfiz(Register rd,
               Register rn,
               int low_bit,
               int width,
               OperandSize size = kDoubleWord) {
      int wordsize = size == kDoubleWord ? 64 : 32;
      ASSERT(width > 0);
      ASSERT(low_bit < wordsize);
      EmitBitfieldOp(UBFM, rd, rn, (-low_bit) & (wordsize - 1), width - 1, size);
    }
  
    // Unsigned bitfield extract.  Takes the width bits, starting at low_bit and
    // writes them to the low bits of rd zeroing bits above.
    void ubfx(Register rd,
              Register rn,
              int low_bit,
              int width,
              OperandSize size = kDoubleWord) {
      EmitBitfieldOp(UBFM, rd, rn, low_bit, low_bit + width - 1, size);
    }
  
    // Sign extend byte->64 bit.
    void sxtb(Register rd, Register rn) {
      EmitBitfieldOp(SBFM, rd, rn, 0, 7, kDoubleWord);
    }
  
    // Sign extend halfword->64 bit.
    void sxth(Register rd, Register rn) {
      EmitBitfieldOp(SBFM, rd, rn, 0, 15, kDoubleWord);
    }
  
    // Sign extend word->64 bit.
    void sxtw(Register rd, Register rn) {
      EmitBitfieldOp(SBFM, rd, rn, 0, 31, kDoubleWord);
    }
  
    // Zero/unsigned extend byte->64 bit.
    void uxtb(Register rd, Register rn) {
      EmitBitfieldOp(UBFM, rd, rn, 0, 7, kDoubleWord);
    }
  
    // Zero/unsigned extend halfword->64 bit.
    void uxth(Register rd, Register rn) {
      EmitBitfieldOp(UBFM, rd, rn, 0, 15, kDoubleWord);
    }
  
    // Zero/unsigned extend word->64 bit.
    void uxtw(Register rd, Register rn) {
      EmitBitfieldOp(UBFM, rd, rn, 0, 31, kDoubleWord);
    }
  
    // Logical immediate operations.
    void andi(Register rd, Register rn, const Immediate& imm) {
      Operand imm_op;
      const bool immok =
          Operand::IsImmLogical(imm.value(), kXRegSizeInBits, &imm_op);
      ASSERT(immok);
      EmitLogicalImmOp(ANDI, rd, rn, imm_op, kDoubleWord);
    }
    void orri(Register rd, Register rn, const Immediate& imm) {
      Operand imm_op;
      const bool immok =
          Operand::IsImmLogical(imm.value(), kXRegSizeInBits, &imm_op);
      ASSERT(immok);
      EmitLogicalImmOp(ORRI, rd, rn, imm_op, kDoubleWord);
    }
    void eori(Register rd, Register rn, const Immediate& imm) {
      Operand imm_op;
      const bool immok =
          Operand::IsImmLogical(imm.value(), kXRegSizeInBits, &imm_op);
      ASSERT(immok);
      EmitLogicalImmOp(EORI, rd, rn, imm_op, kDoubleWord);
    }
    void andis(Register rd, Register rn, const Immediate& imm) {
      Operand imm_op;
      const bool immok =
          Operand::IsImmLogical(imm.value(), kXRegSizeInBits, &imm_op);
      ASSERT(immok);
      EmitLogicalImmOp(ANDIS, rd, rn, imm_op, kDoubleWord);
    }
  
    // Logical (shifted) register operations.
    void and_(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(AND, rd, rn, o, kDoubleWord);
    }
    void andw_(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(AND, rd, rn, o, kWord);
    }
    void bic(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(BIC, rd, rn, o, kDoubleWord);
    }
    void orr(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(ORR, rd, rn, o, kDoubleWord);
    }
    void orrw(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(ORR, rd, rn, o, kWord);
    }
    void orn(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(ORN, rd, rn, o, kDoubleWord);
    }
    void ornw(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(ORN, rd, rn, o, kWord);
    }
    void eor(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(EOR, rd, rn, o, kDoubleWord);
    }
    void eorw(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(EOR, rd, rn, o, kWord);
    }
    void eon(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(EON, rd, rn, o, kDoubleWord);
    }
    void ands(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(ANDS, rd, rn, o, kDoubleWord);
    }
    void bics(Register rd, Register rn, Operand o) {
      EmitLogicalShiftOp(BICS, rd, rn, o, kDoubleWord);
    }
  
    // Count leading zero bits.
    void clz(Register rd, Register rn) {
      EmitMiscDP1Source(CLZ, rd, rn, kDoubleWord);
    }
  
    // Reverse bits.
    void rbit(Register rd, Register rn) {
      EmitMiscDP1Source(RBIT, rd, rn, kDoubleWord);
    }
  
    // Misc. arithmetic.
    void udiv(Register rd, Register rn, Register rm) {
      EmitMiscDP2Source(UDIV, rd, rn, rm, kDoubleWord);
    }
    void sdiv(Register rd, Register rn, Register rm) {
      EmitMiscDP2Source(SDIV, rd, rn, rm, kDoubleWord);
    }
    void lslv(Register rd, Register rn, Register rm) {
      EmitMiscDP2Source(LSLV, rd, rn, rm, kDoubleWord);
    }
    void lsrv(Register rd, Register rn, Register rm) {
      EmitMiscDP2Source(LSRV, rd, rn, rm, kDoubleWord);
    }
    void asrv(Register rd, Register rn, Register rm) {
      EmitMiscDP2Source(ASRV, rd, rn, rm, kDoubleWord);
    }
    void lslvw(Register rd, Register rn, Register rm) {
      EmitMiscDP2Source(LSLV, rd, rn, rm, kWord);
    }
    void lsrvw(Register rd, Register rn, Register rm) {
      EmitMiscDP2Source(LSRV, rd, rn, rm, kWord);
    }
    void asrvw(Register rd, Register rn, Register rm) {
      EmitMiscDP2Source(ASRV, rd, rn, rm, kWord);
    }
    void madd(Register rd,
              Register rn,
              Register rm,
              Register ra,
              OperandSize sz = kDoubleWord) {
      EmitMiscDP3Source(MADD, rd, rn, rm, ra, sz);
    }
    void msub(Register rd,
              Register rn,
              Register rm,
              Register ra,
              OperandSize sz = kDoubleWord) {
      EmitMiscDP3Source(MSUB, rd, rn, rm, ra, sz);
    }
    void smulh(Register rd,
               Register rn,
               Register rm,
               OperandSize sz = kDoubleWord) {
      EmitMiscDP3Source(SMULH, rd, rn, rm, R31, sz);
    }
    void umulh(Register rd,
               Register rn,
               Register rm,
               OperandSize sz = kDoubleWord) {
      EmitMiscDP3Source(UMULH, rd, rn, rm, R31, sz);
    }
    void umaddl(Register rd,
                Register rn,
                Register rm,
                Register ra,
                OperandSize sz = kDoubleWord) {
      EmitMiscDP3Source(UMADDL, rd, rn, rm, ra, sz);
    }
    void umull(Register rd,
               Register rn,
               Register rm,
               OperandSize sz = kDoubleWord) {
      EmitMiscDP3Source(UMADDL, rd, rn, rm, ZR, sz);
    }
    void smaddl(Register rd,
                Register rn,
                Register rm,
                Register ra,
                OperandSize sz = kDoubleWord) {
      EmitMiscDP3Source(SMADDL, rd, rn, rm, ra, sz);
    }
    void smull(Register rd,
               Register rn,
               Register rm,
               OperandSize sz = kDoubleWord) {
      EmitMiscDP3Source(SMADDL, rd, rn, rm, ZR, sz);
    }
  
    // Move wide immediate.
    void movk(Register rd, const Immediate& imm, int hw_idx) {
      ASSERT(rd != CSP);
      const Register crd = ConcreteRegister(rd);
      EmitMoveWideOp(MOVK, crd, imm, hw_idx, kDoubleWord);
    }
    void movn(Register rd, const Immediate& imm, int hw_idx) {
      ASSERT(rd != CSP);
      const Register crd = ConcreteRegister(rd);
      EmitMoveWideOp(MOVN, crd, imm, hw_idx, kDoubleWord);
    }
    void movz(Register rd, const Immediate& imm, int hw_idx) {
      ASSERT(rd != CSP);
      const Register crd = ConcreteRegister(rd);
      EmitMoveWideOp(MOVZ, crd, imm, hw_idx, kDoubleWord);
    }
  
    // Loads and Stores.
    void ldr(Register rt, Address a, OperandSize sz = kDoubleWord) {
      ASSERT((a.type() != Address::PairOffset) &&
             (a.type() != Address::PairPostIndex) &&
             (a.type() != Address::PairPreIndex));
      if (a.type() == Address::PCOffset) {
        ASSERT(sz == kDoubleWord);
        EmitLoadRegLiteral(LDRpc, rt, a, sz);
      } else {
        if (IsSignedOperand(sz)) {
          EmitLoadStoreReg(LDRS, rt, a, sz);
        } else {
          EmitLoadStoreReg(LDR, rt, a, sz);
        }
      }
    }
    void str(Register rt, Address a, OperandSize sz = kDoubleWord) {
      ASSERT((a.type() != Address::PairOffset) &&
             (a.type() != Address::PairPostIndex) &&
             (a.type() != Address::PairPreIndex));
      EmitLoadStoreReg(STR, rt, a, sz);
    }
  
    void ldp(Register rt, Register rt2, Address a, OperandSize sz = kDoubleWord) {
      ASSERT((a.type() == Address::PairOffset) ||
             (a.type() == Address::PairPostIndex) ||
             (a.type() == Address::PairPreIndex));
      EmitLoadStoreRegPair(LDP, rt, rt2, a, sz);
    }
    void stp(Register rt, Register rt2, Address a, OperandSize sz = kDoubleWord) {
      ASSERT((a.type() == Address::PairOffset) ||
             (a.type() == Address::PairPostIndex) ||
             (a.type() == Address::PairPreIndex));
      EmitLoadStoreRegPair(STP, rt, rt2, a, sz);
    }
  
    void ldxr(Register rt, Register rn, OperandSize size = kDoubleWord) {
      // rt = value
      // rn = address
      EmitLoadStoreExclusive(LDXR, R31, rn, rt, size);
    }
    void stxr(Register rs,
              Register rt,
              Register rn,
              OperandSize size = kDoubleWord) {
      // rs = status (1 = failure, 0 = success)
      // rt = value
      // rn = address
      EmitLoadStoreExclusive(STXR, rs, rn, rt, size);
    }
    void clrex() {
      const int32_t encoding = static_cast<int32_t>(CLREX);
      Emit(encoding);
    }
  
    void ldar(Register rt, Register rn, OperandSize sz = kDoubleWord) {
      EmitLoadStoreExclusive(LDAR, R31, rn, rt, sz);
    }
  
    void stlr(Register rt, Register rn, OperandSize sz = kDoubleWord) {
      EmitLoadStoreExclusive(STLR, R31, rn, rt, sz);
    }
  
    // Conditional select.
    void csel(Register rd, Register rn, Register rm, Condition cond) {
      EmitConditionalSelect(CSEL, rd, rn, rm, cond, kDoubleWord);
    }
    void csinc(Register rd, Register rn, Register rm, Condition cond) {
      EmitConditionalSelect(CSINC, rd, rn, rm, cond, kDoubleWord);
    }
    void cinc(Register rd, Register rn, Condition cond) {
      csinc(rd, rn, rn, InvertCondition(cond));
    }
    void cset(Register rd, Condition cond) {
      csinc(rd, ZR, ZR, InvertCondition(cond));
    }
    void csinv(Register rd, Register rn, Register rm, Condition cond) {
      EmitConditionalSelect(CSINV, rd, rn, rm, cond, kDoubleWord);
    }
    void cinv(Register rd, Register rn, Condition cond) {
      csinv(rd, rn, rn, InvertCondition(cond));
    }
    void csetm(Register rd, Condition cond) {
      csinv(rd, ZR, ZR, InvertCondition(cond));
    }
    void csneg(Register rd, Register rn, Register rm, Condition cond) {
      EmitConditionalSelect(CSNEG, rd, rn, rm, cond, kDoubleWord);
    }
    void cneg(Register rd, Register rn, Condition cond) {
      EmitConditionalSelect(CSNEG, rd, rn, rn, InvertCondition(cond),
                            kDoubleWord);
    }
  
    // Comparison.
    // rn cmp o.
    // For add and sub, to use CSP for rn, o must be of type Operand::Extend.
    // For an unmodified rm in this case, use Operand(rm, UXTX, 0);
    void cmp(Register rn, Operand o) { subs(ZR, rn, o); }
    void cmpw(Register rn, Operand o) { subsw(ZR, rn, o); }
    // rn cmp -o.
    void cmn(Register rn, Operand o) { adds(ZR, rn, o); }
  
    void CompareRegisters(Register rn, Register rm) {
      if (rn == CSP) {
        // UXTX 0 on a 64-bit register (rm) is a nop, but forces R31 to be
        // interpreted as CSP.
        cmp(CSP, Operand(rm, UXTX, 0));
      } else {
        cmp(rn, Operand(rm));
      }
    }
  
    // Conditional branch.
    void b(Label* label, Condition cond = AL) {
      EmitConditionalBranch(BCOND, cond, label);
    }
  
    void b(int32_t offset) { EmitUnconditionalBranchOp(B, offset); }
    void bl(int32_t offset) { EmitUnconditionalBranchOp(BL, offset); }
  
    void BranchIf(Condition condition, Label* label) { b(label, condition); }
  
    void cbz(Label* label, Register rt, OperandSize sz = kDoubleWord) {
      EmitCompareAndBranch(CBZ, rt, label, sz);
    }
  
    void cbnz(Label* label, Register rt, OperandSize sz = kDoubleWord) {
      EmitCompareAndBranch(CBNZ, rt, label, sz);
    }
  
    // Generate 64-bit compare with zero and branch when condition allows to use
    // a single instruction: cbz/cbnz/tbz/tbnz.
    bool CanGenerateXCbzTbz(Register rn, Condition cond);
    void GenerateXCbzTbz(Register rn, Condition cond, Label* label);
  
    // Test bit and branch if zero.
    void tbz(Label* label, Register rt, intptr_t bit_number) {
      EmitTestAndBranch(TBZ, rt, bit_number, label);
    }
    void tbnz(Label* label, Register rt, intptr_t bit_number) {
      EmitTestAndBranch(TBNZ, rt, bit_number, label);
    }
  
    // Branch, link, return.
    void br(Register rn) { EmitUnconditionalBranchRegOp(BR, rn); }
    void blr(Register rn) { EmitUnconditionalBranchRegOp(BLR, rn); }
    void ret(Register rn = R30) { EmitUnconditionalBranchRegOp(RET, rn); }
  
    // Breakpoint.
    void brk(uint16_t imm) { EmitExceptionGenOp(BRK, imm); }
  
    // Double floating point.
    bool fmovdi(VRegister vd, double immd) {
      int64_t imm64 = bit_cast<int64_t, double>(immd);
      const uint8_t bit7 = imm64 >> 63;
      const uint8_t bit6 = (~(imm64 >> 62)) & 0x1;
      const uint8_t bit54 = (imm64 >> 52) & 0x3;
      const uint8_t bit30 = (imm64 >> 48) & 0xf;
      const uint8_t imm8 = (bit7 << 7) | (bit6 << 6) | (bit54 << 4) | bit30;
      const int64_t expimm8 = Instr::VFPExpandImm(imm8);
      if (imm64 != expimm8) {
        return false;
      }
      EmitFPImm(FMOVDI, vd, imm8);
      return true;
    }
    void fmovsr(VRegister vd, Register rn) {
      ASSERT(rn != R31);
      ASSERT(rn != CSP);
      const Register crn = ConcreteRegister(rn);
      EmitFPIntCvtOp(FMOVSR, static_cast<Register>(vd), crn, kWord);
    }
    void fmovrs(Register rd, VRegister vn) {
      ASSERT(rd != R31);
      ASSERT(rd != CSP);
      const Register crd = ConcreteRegister(rd);
      EmitFPIntCvtOp(FMOVRS, crd, static_cast<Register>(vn), kWord);
    }
    void fmovdr(VRegister vd, Register rn) {
      ASSERT(rn != R31);
      ASSERT(rn != CSP);
      const Register crn = ConcreteRegister(rn);
      EmitFPIntCvtOp(FMOVDR, static_cast<Register>(vd), crn);
    }
    void fmovrd(Register rd, VRegister vn) {
      ASSERT(rd != R31);
      ASSERT(rd != CSP);
      const Register crd = ConcreteRegister(rd);
      EmitFPIntCvtOp(FMOVRD, crd, static_cast<Register>(vn));
    }
    void scvtfdx(VRegister vd, Register rn) {
      ASSERT(rn != R31);
      ASSERT(rn != CSP);
      const Register crn = ConcreteRegister(rn);
      EmitFPIntCvtOp(SCVTFD, static_cast<Register>(vd), crn);
    }
    void scvtfdw(VRegister vd, Register rn) {
      ASSERT(rn != R31);
      ASSERT(rn != CSP);
      const Register crn = ConcreteRegister(rn);
      EmitFPIntCvtOp(SCVTFD, static_cast<Register>(vd), crn, kWord);
    }
    void fcvtzds(Register rd, VRegister vn) {
      ASSERT(rd != R31);
      ASSERT(rd != CSP);
      const Register crd = ConcreteRegister(rd);
      EmitFPIntCvtOp(FCVTZDS, crd, static_cast<Register>(vn));
    }
    void fmovdd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FMOVDD, vd, vn); }
    void fabsd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FABSD, vd, vn); }
    void fnegd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FNEGD, vd, vn); }
    void fsqrtd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FSQRTD, vd, vn); }
    void fcvtsd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FCVTSD, vd, vn); }
    void fcvtds(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FCVTDS, vd, vn); }
    void fldrq(VRegister vt, Address a) {
      ASSERT(a.type() != Address::PCOffset);
      EmitLoadStoreReg(FLDRQ, static_cast<Register>(vt), a, kByte);
    }
    void fstrq(VRegister vt, Address a) {
      ASSERT(a.type() != Address::PCOffset);
      EmitLoadStoreReg(FSTRQ, static_cast<Register>(vt), a, kByte);
    }
    void fldrd(VRegister vt, Address a) {
      ASSERT(a.type() != Address::PCOffset);
      EmitLoadStoreReg(FLDR, static_cast<Register>(vt), a, kDWord);
    }
    void fstrd(VRegister vt, Address a) {
      ASSERT(a.type() != Address::PCOffset);
      EmitLoadStoreReg(FSTR, static_cast<Register>(vt), a, kDWord);
    }
    void fldrs(VRegister vt, Address a) {
      ASSERT(a.type() != Address::PCOffset);
      EmitLoadStoreReg(FLDR, static_cast<Register>(vt), a, kSWord);
    }
    void fstrs(VRegister vt, Address a) {
      ASSERT(a.type() != Address::PCOffset);
      EmitLoadStoreReg(FSTR, static_cast<Register>(vt), a, kSWord);
    }
    void fcmpd(VRegister vn, VRegister vm) { EmitFPCompareOp(FCMPD, vn, vm); }
    void fcmpdz(VRegister vn) { EmitFPCompareOp(FCMPZD, vn, V0); }
    void fmuld(VRegister vd, VRegister vn, VRegister vm) {
      EmitFPTwoSourceOp(FMULD, vd, vn, vm);
    }
    void fdivd(VRegister vd, VRegister vn, VRegister vm) {
      EmitFPTwoSourceOp(FDIVD, vd, vn, vm);
    }
    void faddd(VRegister vd, VRegister vn, VRegister vm) {
      EmitFPTwoSourceOp(FADDD, vd, vn, vm);
    }
    void fsubd(VRegister vd, VRegister vn, VRegister vm) {
      EmitFPTwoSourceOp(FSUBD, vd, vn, vm);
    }
  
    // SIMD operations.
    void vand(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VAND, vd, vn, vm);
    }
    void vorr(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VORR, vd, vn, vm);
    }
    void veor(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VEOR, vd, vn, vm);
    }
    void vaddw(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VADDW, vd, vn, vm);
    }
    void vaddx(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VADDX, vd, vn, vm);
    }
    void vsubw(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VSUBW, vd, vn, vm);
    }
    void vsubx(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VSUBX, vd, vn, vm);
    }
    void vadds(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VADDS, vd, vn, vm);
    }
    void vaddd(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VADDD, vd, vn, vm);
    }
    void vsubs(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VSUBS, vd, vn, vm);
    }
    void vsubd(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VSUBD, vd, vn, vm);
    }
    void vmuls(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VMULS, vd, vn, vm);
    }
    void vmuld(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VMULD, vd, vn, vm);
    }
    void vdivs(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VDIVS, vd, vn, vm);
    }
    void vdivd(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VDIVD, vd, vn, vm);
    }
    void vceqs(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VCEQS, vd, vn, vm);
    }
    void vceqd(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VCEQD, vd, vn, vm);
    }
    void vcgts(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VCGTS, vd, vn, vm);
    }
    void vcgtd(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VCGTD, vd, vn, vm);
    }
    void vcges(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VCGES, vd, vn, vm);
    }
    void vcged(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VCGED, vd, vn, vm);
    }
    void vmins(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VMINS, vd, vn, vm);
    }
    void vmind(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VMIND, vd, vn, vm);
    }
    void vmaxs(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VMAXS, vd, vn, vm);
    }
    void vmaxd(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VMAXD, vd, vn, vm);
    }
    void vrecpss(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VRECPSS, vd, vn, vm);
    }
    void vrsqrtss(VRegister vd, VRegister vn, VRegister vm) {
      EmitSIMDThreeSameOp(VRSQRTSS, vd, vn, vm);
    }
    void vnot(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VNOT, vd, vn); }
    void vabss(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VABSS, vd, vn); }
    void vabsd(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VABSD, vd, vn); }
    void vnegs(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VNEGS, vd, vn); }
    void vnegd(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VNEGD, vd, vn); }
    void vsqrts(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VSQRTS, vd, vn); }
    void vsqrtd(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VSQRTD, vd, vn); }
    void vrecpes(VRegister vd, VRegister vn) {
      EmitSIMDTwoRegOp(VRECPES, vd, vn);
    }
    void vrsqrtes(VRegister vd, VRegister vn) {
      EmitSIMDTwoRegOp(VRSQRTES, vd, vn);
    }
    void vdupw(VRegister vd, Register rn) {
      const VRegister vn = static_cast<VRegister>(rn);
      EmitSIMDCopyOp(VDUPI, vd, vn, kWord, 0, 0);
    }
    void vdupx(VRegister vd, Register rn) {
      const VRegister vn = static_cast<VRegister>(rn);
      EmitSIMDCopyOp(VDUPI, vd, vn, kDoubleWord, 0, 0);
    }
    void vdups(VRegister vd, VRegister vn, int32_t idx) {
      EmitSIMDCopyOp(VDUP, vd, vn, kSWord, 0, idx);
    }
    void vdupd(VRegister vd, VRegister vn, int32_t idx) {
      EmitSIMDCopyOp(VDUP, vd, vn, kDWord, 0, idx);
    }
    void vinsw(VRegister vd, int32_t didx, Register rn) {
      const VRegister vn = static_cast<VRegister>(rn);
      EmitSIMDCopyOp(VINSI, vd, vn, kWord, 0, didx);
    }
    void vinsx(VRegister vd, int32_t didx, Register rn) {
      const VRegister vn = static_cast<VRegister>(rn);
      EmitSIMDCopyOp(VINSI, vd, vn, kDoubleWord, 0, didx);
    }
    void vinss(VRegister vd, int32_t didx, VRegister vn, int32_t sidx) {
      EmitSIMDCopyOp(VINS, vd, vn, kSWord, sidx, didx);
    }
    void vinsd(VRegister vd, int32_t didx, VRegister vn, int32_t sidx) {
      EmitSIMDCopyOp(VINS, vd, vn, kDWord, sidx, didx);
    }
    void vmovrs(Register rd, VRegister vn, int32_t sidx) {
      const VRegister vd = static_cast<VRegister>(rd);
      EmitSIMDCopyOp(VMOVW, vd, vn, kWord, 0, sidx);
    }
    void vmovrd(Register rd, VRegister vn, int32_t sidx) {
      const VRegister vd = static_cast<VRegister>(rd);
      EmitSIMDCopyOp(VMOVX, vd, vn, kDoubleWord, 0, sidx);
    }
  
    // Aliases.
    void mov(Register rd, Register rn) {
      if ((rd == CSP) || (rn == CSP)) {
        add(rd, rn, Operand(0));
      } else {
        orr(rd, ZR, Operand(rn));
      }
    }
    void movw(Register rd, Register rn) {
      if ((rd == CSP) || (rn == CSP)) {
        addw(rd, rn, Operand(0));
      } else {
        orrw(rd, ZR, Operand(rn));
      }
    }
    void vmov(VRegister vd, VRegister vn) { vorr(vd, vn, vn); }
    void mvn(Register rd, Register rm) { orn(rd, ZR, Operand(rm)); }
    void mvnw(Register rd, Register rm) { ornw(rd, ZR, Operand(rm)); }
    void neg(Register rd, Register rm) { sub(rd, ZR, Operand(rm)); }
    void negs(Register rd, Register rm) { subs(rd, ZR, Operand(rm)); }
    void negsw(Register rd, Register rm) { subsw(rd, ZR, Operand(rm)); }
    void mul(Register rd, Register rn, Register rm) {
      madd(rd, rn, rm, ZR, kDoubleWord);
    }
    void mulw(Register rd, Register rn, Register rm) {
      madd(rd, rn, rm, ZR, kWord);
    }
    void Push(Register reg) {
      ASSERT(reg != PP);  // Only push PP with TagAndPushPP().
      str(reg, Address(SP, -1 * target::kWordSize, Address::PreIndex));
    }
    void Pop(Register reg) {
      ASSERT(reg != PP);  // Only pop PP with PopAndUntagPP().
      ldr(reg, Address(SP, 1 * target::kWordSize, Address::PostIndex));
    }
    void PushPair(Register low, Register high) {
      stp(low, high, Address(SP, -2 * target::kWordSize, Address::PairPreIndex));
    }
    void PopPair(Register low, Register high) {
      ldp(low, high, Address(SP, 2 * target::kWordSize, Address::PairPostIndex));
    }
    void PushFloat(VRegister reg) {
      fstrs(reg, Address(SP, -1 * kFloatSize, Address::PreIndex));
    }
    void PushDouble(VRegister reg) {
      fstrd(reg, Address(SP, -1 * kDoubleSize, Address::PreIndex));
    }
    void PushQuad(VRegister reg) {
      fstrq(reg, Address(SP, -1 * kQuadSize, Address::PreIndex));
    }
    void PopFloat(VRegister reg) {
      fldrs(reg, Address(SP, 1 * kFloatSize, Address::PostIndex));
    }
    void PopDouble(VRegister reg) {
      fldrd(reg, Address(SP, 1 * kDoubleSize, Address::PostIndex));
    }
    void PopQuad(VRegister reg) {
      fldrq(reg, Address(SP, 1 * kQuadSize, Address::PostIndex));
    }
    void TagAndPushPP() {
      // Add the heap object tag back to PP before putting it on the stack.
      add(TMP, PP, Operand(kHeapObjectTag));
      str(TMP, Address(SP, -1 * target::kWordSize, Address::PreIndex));
    }
    void TagAndPushPPAndPcMarker() {
      COMPILE_ASSERT(CODE_REG != TMP2);
      // Add the heap object tag back to PP before putting it on the stack.
      add(TMP2, PP, Operand(kHeapObjectTag));
      stp(TMP2, CODE_REG,
          Address(SP, -2 * target::kWordSize, Address::PairPreIndex));
    }
    void PopAndUntagPP() {
      ldr(PP, Address(SP, 1 * target::kWordSize, Address::PostIndex));
      sub(PP, PP, Operand(kHeapObjectTag));
      // The caller of PopAndUntagPP() must explicitly allow use of popped PP.
      set_constant_pool_allowed(false);
    }
    void tst(Register rn, Operand o) { ands(ZR, rn, o); }
    void tsti(Register rn, const Immediate& imm) { andis(ZR, rn, imm); }
  
    void LslImmediate(Register rd,
                      Register rn,
                      int shift,
                      OperandSize sz = kDoubleWord) {
      const int reg_size =
          (sz == kDoubleWord) ? kXRegSizeInBits : kWRegSizeInBits;
      ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1, sz);
    }
    void LsrImmediate(Register rd,
                      Register rn,
                      int shift,
                      OperandSize sz = kDoubleWord) {
      const int reg_size =
          (sz == kDoubleWord) ? kXRegSizeInBits : kWRegSizeInBits;
      ubfm(rd, rn, shift, reg_size - 1, sz);
    }
    void AsrImmediate(Register rd, Register rn, int shift) {
      const int reg_size = kXRegSizeInBits;
      sbfm(rd, rn, shift, reg_size - 1);
    }
  
    void VRecps(VRegister vd, VRegister vn);
    void VRSqrts(VRegister vd, VRegister vn);
  
    void SmiUntag(Register reg) { AsrImmediate(reg, reg, kSmiTagSize); }
    void SmiUntag(Register dst, Register src) {
      AsrImmediate(dst, src, kSmiTagSize);
    }
    void SmiTag(Register reg) { LslImmediate(reg, reg, kSmiTagSize); }
    void SmiTag(Register dst, Register src) {
      LslImmediate(dst, src, kSmiTagSize);
    }
  
    void BranchIfNotSmi(Register reg, Label* label) { tbnz(label, reg, kSmiTag); }
  
    void BranchIfSmi(Register reg, Label* label) { tbz(label, reg, kSmiTag); }
  
    void Branch(const Code& code,
                Register pp,
                ObjectPoolBuilderEntry::Patchability patchable =
                    ObjectPoolBuilderEntry::kNotPatchable);
  
    void BranchLink(const Code& code,
                    ObjectPoolBuilderEntry::Patchability patchable =
                        ObjectPoolBuilderEntry::kNotPatchable,
                    CodeEntryKind entry_kind = CodeEntryKind::kNormal);
  
    void BranchLinkPatchable(const Code& code,
                             CodeEntryKind entry_kind = CodeEntryKind::kNormal) {
      BranchLink(code, ObjectPoolBuilderEntry::kPatchable, entry_kind);
    }
    void BranchLinkToRuntime();
  
    // Emit a call that shares its object pool entries with other calls
    // that have the same equivalence marker.
    void BranchLinkWithEquivalence(
        const Code& code,
        const Object& equivalence,
        CodeEntryKind entry_kind = CodeEntryKind::kNormal);
  
    void Call(Address target) {
      ldr(LR, target);
      blr(LR);
    }
  
    void CallCFunction(Address target) { Call(target); }
  
    void AddImmediate(Register dest, int64_t imm) {
      AddImmediate(dest, dest, imm);
    }
  
    // Macros accepting a pp Register argument may attempt to load values from
    // the object pool when possible. Unless you are sure that the untagged object
    // pool pointer is in another register, or that it is not available at all,
    // PP should be passed for pp. `dest` can be TMP2, `rn` cannot.
    void AddImmediate(Register dest, Register rn, int64_t imm);
    void AddImmediateSetFlags(Register dest,
                              Register rn,
                              int64_t imm,
                              OperandSize sz = kDoubleWord);
    void SubImmediateSetFlags(Register dest,
                              Register rn,
                              int64_t imm,
                              OperandSize sz = kDoubleWord);
    void AndImmediate(Register rd, Register rn, int64_t imm);
    void OrImmediate(Register rd, Register rn, int64_t imm);
    void XorImmediate(Register rd, Register rn, int64_t imm);
    void TestImmediate(Register rn, int64_t imm);
    void CompareImmediate(Register rn, int64_t imm);
  
    void LoadFromOffset(Register dest,
                        Register base,
                        int32_t offset,
                        OperandSize sz = kDoubleWord);
    void LoadFieldFromOffset(Register dest,
                             Register base,
                             int32_t offset,
                             OperandSize sz = kDoubleWord) {
      LoadFromOffset(dest, base, offset - kHeapObjectTag, sz);
    }
    void LoadSFromOffset(VRegister dest, Register base, int32_t offset);
    void LoadDFromOffset(VRegister dest, Register base, int32_t offset);
    void LoadDFieldFromOffset(VRegister dest, Register base, int32_t offset) {
      LoadDFromOffset(dest, base, offset - kHeapObjectTag);
    }
    void LoadQFromOffset(VRegister dest, Register base, int32_t offset);
    void LoadQFieldFromOffset(VRegister dest, Register base, int32_t offset) {
      LoadQFromOffset(dest, base, offset - kHeapObjectTag);
    }
  
    void StoreToOffset(Register src,
                       Register base,
                       int32_t offset,
                       OperandSize sz = kDoubleWord);
    void StoreFieldToOffset(Register src,
                            Register base,
                            int32_t offset,
                            OperandSize sz = kDoubleWord) {
      StoreToOffset(src, base, offset - kHeapObjectTag, sz);
    }
  
    void StoreSToOffset(VRegister src, Register base, int32_t offset);
    void StoreDToOffset(VRegister src, Register base, int32_t offset);
    void StoreDFieldToOffset(VRegister src, Register base, int32_t offset) {
      StoreDToOffset(src, base, offset - kHeapObjectTag);
    }
    void StoreQToOffset(VRegister src, Register base, int32_t offset);
    void StoreQFieldToOffset(VRegister src, Register base, int32_t offset) {
      StoreQToOffset(src, base, offset - kHeapObjectTag);
    }
  
    enum CanBeSmi {
      kValueIsNotSmi,
      kValueCanBeSmi,
    };
  
    // Store into a heap object and apply the generational and incremental write
    // barriers. All stores into heap objects must pass through this function or,
    // if the value can be proven either Smi or old-and-premarked, its NoBarrier
    // variants.
    // Preserves object and value registers.
    void StoreIntoObject(Register object,
                         const Address& dest,
                         Register value,
                         CanBeSmi can_value_be_smi = kValueCanBeSmi,
                         bool lr_reserved = false);
    void StoreIntoArray(Register object,
                        Register slot,
                        Register value,
                        CanBeSmi can_value_be_smi = kValueCanBeSmi,
                        bool lr_reserved = false);
  
    void StoreIntoObjectOffset(Register object,
                               int32_t offset,
                               Register value,
                               CanBeSmi can_value_be_smi = kValueCanBeSmi,
                               bool lr_reserved = false);
    void StoreIntoObjectNoBarrier(Register object,
                                  const Address& dest,
                                  Register value);
    void StoreIntoObjectOffsetNoBarrier(Register object,
                                        int32_t offset,
                                        Register value);
    void StoreIntoObjectNoBarrier(Register object,
                                  const Address& dest,
                                  const Object& value);
    void StoreIntoObjectOffsetNoBarrier(Register object,
                                        int32_t offset,
                                        const Object& value);
  
    // Stores a non-tagged value into a heap object.
    void StoreInternalPointer(Register object,
                              const Address& dest,
                              Register value);
  
    // Object pool, loading from pool, etc.
    void LoadPoolPointer(Register pp = PP);
  
    bool constant_pool_allowed() const { return constant_pool_allowed_; }
    void set_constant_pool_allowed(bool b) { constant_pool_allowed_ = b; }
  
    intptr_t FindImmediate(int64_t imm);
    bool CanLoadFromObjectPool(const Object& object) const;
    void LoadNativeEntry(Register dst,
                         const ExternalLabel* label,
                         ObjectPoolBuilderEntry::Patchability patchable);
    void LoadIsolate(Register dst);
  
    // Note: the function never clobbers TMP, TMP2 scratch registers.
    void LoadObject(Register dst, const Object& obj);
    // Note: the function never clobbers TMP, TMP2 scratch registers.
    void LoadUniqueObject(Register dst, const Object& obj);
    // Note: the function never clobbers TMP, TMP2 scratch registers.
    void LoadImmediate(Register reg, int64_t imm);
  
    void LoadDImmediate(VRegister reg, double immd);
  
    // Load word from pool from the given offset using encoding that
    // InstructionPattern::DecodeLoadWordFromPool can decode.
    //
    // Note: the function never clobbers TMP, TMP2 scratch registers.
    void LoadWordFromPoolIndex(Register dst, intptr_t index, Register pp = PP);
  
    void LoadDoubleWordFromPoolIndex(Register lower,
                                     Register upper,
                                     intptr_t index);
  
    void PushObject(const Object& object) {
      if (IsSameObject(compiler::NullObject(), object)) {
        Push(NULL_REG);
      } else {
        LoadObject(TMP, object);
        Push(TMP);
      }
    }
    void PushImmediate(int64_t immediate) {
      LoadImmediate(TMP, immediate);
      Push(TMP);
    }
    void CompareObject(Register reg, const Object& object);
  
    void ExtractClassIdFromTags(Register result, Register tags);
    void ExtractInstanceSizeFromTags(Register result, Register tags);
  
    void LoadClassId(Register result, Register object);
    void LoadClassById(Register result, Register class_id);
    void CompareClassId(Register object,
                        intptr_t class_id,
                        Register scratch = kNoRegister);
    // Note: input and output registers must be different.
    void LoadClassIdMayBeSmi(Register result, Register object);
    void LoadTaggedClassIdMayBeSmi(Register result, Register object);
  
    // Reserve specifies how much space to reserve for the Dart stack.
    void SetupDartSP(intptr_t reserve = 4096);
    void SetupCSPFromThread(Register thr);
    void RestoreCSP();
  
    void EnterFrame(intptr_t frame_size);
    void LeaveFrame();
    void Ret() { ret(LR); }
  
    // Emit code to transition between generated mode and native mode.
    //
    // These require and ensure that CSP and SP are equal and aligned and require
    // a scratch register (in addition to TMP/TMP2).
  
    void TransitionGeneratedToNative(Register destination_address,
                                     Register new_exit_frame,
                                     Register new_exit_through_ffi,
                                     bool enter_safepoint);
    void TransitionNativeToGenerated(Register scratch, bool exit_safepoint);
    void EnterSafepoint(Register scratch);
    void ExitSafepoint(Register scratch);
  
    void CheckCodePointer();
    void RestoreCodePointer();
  
    // Restores the values of the registers that are blocked to cache some values
    // e.g. BARRIER_MASK and NULL_REG.
    void RestorePinnedRegisters();
  
    void SetupGlobalPoolAndDispatchTable();
  
    void EnterDartFrame(intptr_t frame_size, Register new_pp = kNoRegister);
    void EnterOsrFrame(intptr_t extra_size, Register new_pp = kNoRegister);
    void LeaveDartFrame(RestorePP restore_pp = kRestoreCallerPP);
  
    void CallRuntime(const RuntimeEntry& entry, intptr_t argument_count);
  
    // Helper method for performing runtime calls from callers requiring manual
    // register preservation is required (e.g. outside IL instructions marked
    // as calling).
    class CallRuntimeScope : public ValueObject {
     public:
      CallRuntimeScope(Assembler* assembler,
                       const RuntimeEntry& entry,
                       intptr_t frame_size,
                       bool preserve_registers = true)
          : CallRuntimeScope(assembler,
                             entry,
                             frame_size,
                             preserve_registers,
                             /*caller=*/nullptr) {}
  
      CallRuntimeScope(Assembler* assembler,
                       const RuntimeEntry& entry,
                       intptr_t frame_size,
                       Address caller,
                       bool preserve_registers = true)
          : CallRuntimeScope(assembler,
                             entry,
                             frame_size,
                             preserve_registers,
                             &caller) {}
  
      void Call(intptr_t argument_count);
  
      ~CallRuntimeScope();
  
     private:
      CallRuntimeScope(Assembler* assembler,
                       const RuntimeEntry& entry,
                       intptr_t frame_size,
                       bool preserve_registers,
                       const Address* caller);
  
      Assembler* const assembler_;
      const RuntimeEntry& entry_;
      const bool preserve_registers_;
      const bool restore_code_reg_;
    };
  
    // Set up a stub frame so that the stack traversal code can easily identify
    // a stub frame.
    void EnterStubFrame();
    void LeaveStubFrame();
  
    // Set up a frame for calling a C function.
    // Automatically save the pinned registers in Dart which are not callee-
    // saved in the native calling convention.
    // Use together with CallCFunction.
    void EnterCFrame(intptr_t frame_space);
    void LeaveCFrame();
  
    void MonomorphicCheckedEntryJIT();
    void MonomorphicCheckedEntryAOT();
    void BranchOnMonomorphicCheckedEntryJIT(Label* label);
  
    // If allocation tracing for |cid| is enabled, will jump to |trace| label,
    // which will allocate in the runtime where tracing occurs.
    void MaybeTraceAllocation(intptr_t cid, Register temp_reg, Label* trace);
  
    // Inlined allocation of an instance of class 'cls', code has no runtime
    // calls. Jump to 'failure' if the instance cannot be allocated here.
    // Allocated instance is returned in 'instance_reg'.
    // Only the tags field of the object is initialized.
    // Result:
    //   * [instance_reg] will contain allocated new-space object
    //   * [top_reg] will contain Thread::top_offset()
    void TryAllocate(const Class& cls,
                     Label* failure,
                     Register instance_reg,
                     Register top_reg,
                     bool tag_result = true);
  
    void TryAllocateArray(intptr_t cid,
                          intptr_t instance_size,
                          Label* failure,
                          Register instance,
                          Register end_address,
                          Register temp1,
                          Register temp2);
  
    // This emits an PC-relative call of the form "bl <offset>".  The offset
    // is not yet known and needs therefore relocation to the right place before
    // the code can be used.
    //
    // The neccessary information for the "linker" (i.e. the relocation
    // information) is stored in [CodeLayout::static_calls_target_table_]: an
    // entry of the form
    //
    //   (Code::kPcRelativeCall & pc_offset, <target-code>, <target-function>)
    //
    // will be used during relocation to fix the offset.
    //
    // The provided [offset_into_target] will be added to calculate the final
    // destination.  It can be used e.g. for calling into the middle of a
    // function.
    void GenerateUnRelocatedPcRelativeCall(intptr_t offset_into_target = 0);
  
    // This emits an PC-relative tail call of the form "b <offset>".
    //
    // See also above for the pc-relative call.
    void GenerateUnRelocatedPcRelativeTailCall(intptr_t offset_into_target = 0);
  
    Address ElementAddressForIntIndex(bool is_external,
                                      intptr_t cid,
                                      intptr_t index_scale,
                                      Register array,
                                      intptr_t index) const;
    void ComputeElementAddressForIntIndex(Register address,
                                          bool is_external,
                                          intptr_t cid,
                                          intptr_t index_scale,
                                          Register array,
                                          intptr_t index);
    Address ElementAddressForRegIndex(bool is_external,
                                      intptr_t cid,
                                      intptr_t index_scale,
                                      bool index_unboxed,
                                      Register array,
                                      Register index,
                                      Register temp);
  
    // Special version of ElementAddressForRegIndex for the case when cid and
    // operand size for the target load don't match (e.g. when loading a few
    // elements of the array with one load).
    Address ElementAddressForRegIndexWithSize(bool is_external,
                                              intptr_t cid,
                                              OperandSize size,
                                              intptr_t index_scale,
                                              bool index_unboxed,
                                              Register array,
                                              Register index,
                                              Register temp);
  
    void ComputeElementAddressForRegIndex(Register address,
                                          bool is_external,
                                          intptr_t cid,
                                          intptr_t index_scale,
                                          bool index_unboxed,
                                          Register array,
                                          Register index);
  
    void LoadFieldAddressForRegOffset(Register address,
                                      Register instance,
                                      Register offset_in_words_as_smi);
  
    // Returns object data offset for address calculation; for heap objects also
    // accounts for the tag.
    static int32_t HeapDataOffset(bool is_external, intptr_t cid) {
      return is_external
                 ? 0
                 : (target::Instance::DataOffsetFor(cid) - kHeapObjectTag);
    }
  
    static int32_t EncodeImm26BranchOffset(int64_t imm, int32_t instr) {
      const int32_t imm32 = static_cast<int32_t>(imm);
      const int32_t off = (((imm32 >> 2) << kImm26Shift) & kImm26Mask);
      return (instr & ~kImm26Mask) | off;
    }
  
    static int64_t DecodeImm26BranchOffset(int32_t instr) {
      const int32_t off = (((instr & kImm26Mask) >> kImm26Shift) << 6) >> 4;
      return static_cast<int64_t>(off);
    }
  
   private:
    bool use_far_branches_;
  
    bool constant_pool_allowed_;
  
    void LoadWordFromPoolIndexFixed(Register dst, intptr_t index);
  
    // Note: the function never clobbers TMP, TMP2 scratch registers.
    void LoadObjectHelper(Register dst, const Object& obj, bool is_unique);
  
    void AddSubHelper(OperandSize os,
                      bool set_flags,
                      bool subtract,
                      Register rd,
                      Register rn,
                      Operand o) {
      ASSERT((rd != R31) && (rn != R31));
      const Register crd = ConcreteRegister(rd);
      const Register crn = ConcreteRegister(rn);
      if (o.type() == Operand::Immediate) {
        ASSERT(rn != ZR);
        EmitAddSubImmOp(subtract ? SUBI : ADDI, crd, crn, o, os, set_flags);
      } else if (o.type() == Operand::Shifted) {
        ASSERT((rd != CSP) && (rn != CSP));
        EmitAddSubShiftExtOp(subtract ? SUB : ADD, crd, crn, o, os, set_flags);
      } else {
        ASSERT(o.type() == Operand::Extended);
        ASSERT((rd != CSP) && (rn != ZR));
        EmitAddSubShiftExtOp(subtract ? SUB : ADD, crd, crn, o, os, set_flags);
      }
    }
  
    void AddSubWithCarryHelper(OperandSize sz,
                               bool set_flags,
                               bool subtract,
                               Register rd,
                               Register rn,
                               Register rm) {
      ASSERT((rd != R31) && (rn != R31) && (rm != R31));
      ASSERT((rd != CSP) && (rn != CSP) && (rm != CSP));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t s = set_flags ? B29 : 0;
      const int32_t op = subtract ? SBC : ADC;
      const int32_t encoding = op | size | s | Arm64Encode::Rd(rd) |
                               Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm);
      Emit(encoding);
    }
  
    void EmitAddSubImmOp(AddSubImmOp op,
                         Register rd,
                         Register rn,
                         Operand o,
                         OperandSize sz,
                         bool set_flags) {
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t s = set_flags ? B29 : 0;
      const int32_t encoding = op | size | s | Arm64Encode::Rd(rd) |
                               Arm64Encode::Rn(rn) | o.encoding();
      Emit(encoding);
    }
  
    // Follows the *bfm instructions in taking r before s (unlike the Operand
    // constructor, which follows DecodeBitMasks from Appendix G).
    void EmitBitfieldOp(BitfieldOp op,
                        Register rd,
                        Register rn,
                        int r_imm,
                        int s_imm,
                        OperandSize size) {
      if (size != kDoubleWord) {
        ASSERT(size == kWord);
        ASSERT(r_imm < 32 && s_imm < 32);
      } else {
        ASSERT(r_imm < 64 && s_imm < 64);
      }
      const int32_t instr = op | (size == kDoubleWord ? Bitfield64 : 0);
      const int32_t encoding = instr | Operand(0, s_imm, r_imm).encoding() |
                               Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn);
      Emit(encoding);
    }
  
    void EmitLogicalImmOp(LogicalImmOp op,
                          Register rd,
                          Register rn,
                          Operand o,
                          OperandSize sz) {
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      ASSERT((rd != R31) && (rn != R31));
      ASSERT(rn != CSP);
      ASSERT((op == ANDIS) || (rd != ZR));   // op != ANDIS => rd != ZR.
      ASSERT((op != ANDIS) || (rd != CSP));  // op == ANDIS => rd != CSP.
      ASSERT(o.type() == Operand::BitfieldImm);
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding =
          op | size | Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn) | o.encoding();
      Emit(encoding);
    }
  
    void EmitLogicalShiftOp(LogicalShiftOp op,
                            Register rd,
                            Register rn,
                            Operand o,
                            OperandSize sz) {
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      ASSERT((rd != R31) && (rn != R31));
      ASSERT((rd != CSP) && (rn != CSP));
      ASSERT(o.type() == Operand::Shifted);
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding =
          op | size | Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn) | o.encoding();
      Emit(encoding);
    }
  
    void EmitAddSubShiftExtOp(AddSubShiftExtOp op,
                              Register rd,
                              Register rn,
                              Operand o,
                              OperandSize sz,
                              bool set_flags) {
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t s = set_flags ? B29 : 0;
      const int32_t encoding = op | size | s | Arm64Encode::Rd(rd) |
                               Arm64Encode::Rn(rn) | o.encoding();
      Emit(encoding);
    }
  
    int32_t BindImm19Branch(int64_t position, int64_t dest);
    int32_t BindImm14Branch(int64_t position, int64_t dest);
  
    int32_t EncodeImm19BranchOffset(int64_t imm, int32_t instr) {
      if (!CanEncodeImm19BranchOffset(imm)) {
        ASSERT(!use_far_branches());
        BailoutWithBranchOffsetError();
      }
      const int32_t imm32 = static_cast<int32_t>(imm);
      const int32_t off =
          ((static_cast<uint32_t>(imm32 >> 2) << kImm19Shift) & kImm19Mask);
      return (instr & ~kImm19Mask) | off;
    }
  
    int64_t DecodeImm19BranchOffset(int32_t instr) {
      int32_t insns = (static_cast<uint32_t>(instr) & kImm19Mask) >> kImm19Shift;
      const int32_t off = static_cast<int32_t>(insns << 13) >> 11;
      return static_cast<int64_t>(off);
    }
  
    int32_t EncodeImm14BranchOffset(int64_t imm, int32_t instr) {
      if (!CanEncodeImm14BranchOffset(imm)) {
        ASSERT(!use_far_branches());
        BailoutWithBranchOffsetError();
      }
      const int32_t imm32 = static_cast<int32_t>(imm);
      const int32_t off =
          ((static_cast<uint32_t>(imm32 >> 2) << kImm14Shift) & kImm14Mask);
      return (instr & ~kImm14Mask) | off;
    }
  
    int64_t DecodeImm14BranchOffset(int32_t instr) {
      int32_t insns = (static_cast<uint32_t>(instr) & kImm14Mask) >> kImm14Shift;
      const int32_t off = static_cast<int32_t>(insns << 18) >> 16;
      return static_cast<int64_t>(off);
    }
  
    bool IsConditionalBranch(int32_t instr) {
      return (instr & ConditionalBranchMask) ==
             (ConditionalBranchFixed & ConditionalBranchMask);
    }
  
    bool IsCompareAndBranch(int32_t instr) {
      return (instr & CompareAndBranchMask) ==
             (CompareAndBranchFixed & CompareAndBranchMask);
    }
  
    bool IsTestAndBranch(int32_t instr) {
      return (instr & TestAndBranchMask) ==
             (TestAndBranchFixed & TestAndBranchMask);
    }
  
    Condition DecodeImm19BranchCondition(int32_t instr) {
      if (IsConditionalBranch(instr)) {
        return static_cast<Condition>((instr & kCondMask) >> kCondShift);
      }
      ASSERT(IsCompareAndBranch(instr));
      return (instr & B24) ? EQ : NE;  // cbz : cbnz
    }
  
    int32_t EncodeImm19BranchCondition(Condition cond, int32_t instr) {
      if (IsConditionalBranch(instr)) {
        const int32_t c_imm = static_cast<int32_t>(cond);
        return (instr & ~kCondMask) | (c_imm << kCondShift);
      }
      ASSERT(IsCompareAndBranch(instr));
      return (instr & ~B24) | (cond == EQ ? B24 : 0);  // cbz : cbnz
    }
  
    Condition DecodeImm14BranchCondition(int32_t instr) {
      ASSERT(IsTestAndBranch(instr));
      return (instr & B24) ? EQ : NE;  // tbz : tbnz
    }
  
    int32_t EncodeImm14BranchCondition(Condition cond, int32_t instr) {
      ASSERT(IsTestAndBranch(instr));
      return (instr & ~B24) | (cond == EQ ? B24 : 0);  // tbz : tbnz
    }
  
    void EmitCompareAndBranchOp(CompareAndBranchOp op,
                                Register rt,
                                int64_t imm,
                                OperandSize sz) {
      // EncodeImm19BranchOffset will longjump out if the offset does not fit in
      // 19 bits.
      const int32_t encoded_offset = EncodeImm19BranchOffset(imm, 0);
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      ASSERT(Utils::IsInt(21, imm) && ((imm & 0x3) == 0));
      ASSERT((rt != CSP) && (rt != R31));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding = op | size | Arm64Encode::Rt(rt) | encoded_offset;
      Emit(encoding);
    }
  
    void EmitTestAndBranchOp(TestAndBranchOp op,
                             Register rt,
                             intptr_t bit_number,
                             int64_t imm) {
      // EncodeImm14BranchOffset will longjump out if the offset does not fit in
      // 14 bits.
      const int32_t encoded_offset = EncodeImm14BranchOffset(imm, 0);
      ASSERT((bit_number >= 0) && (bit_number <= 63));
      ASSERT(Utils::IsInt(16, imm) && ((imm & 0x3) == 0));
      ASSERT((rt != CSP) && (rt != R31));
      const Register crt = ConcreteRegister(rt);
      int32_t bit_number_low = bit_number & 0x1f;
      int32_t bit_number_hi = (bit_number & 0x20) >> 5;
      const int32_t encoding =
          op | (bit_number_low << 19) | (bit_number_hi << 31) |
          (static_cast<int32_t>(crt) << kRtShift) | encoded_offset;
      Emit(encoding);
    }
  
    void EmitConditionalBranchOp(ConditionalBranchOp op,
                                 Condition cond,
                                 int64_t imm) {
      const int32_t off = EncodeImm19BranchOffset(imm, 0);
      const int32_t encoding =
          op | (static_cast<int32_t>(cond) << kCondShift) | off;
      Emit(encoding);
    }
  
    bool CanEncodeImm19BranchOffset(int64_t offset) {
      ASSERT(Utils::IsAligned(offset, 4));
      return Utils::IsInt(21, offset);
    }
  
    bool CanEncodeImm14BranchOffset(int64_t offset) {
      ASSERT(Utils::IsAligned(offset, 4));
      return Utils::IsInt(16, offset);
    }
  
    void EmitConditionalBranch(ConditionalBranchOp op,
                               Condition cond,
                               Label* label) {
      if (label->IsBound()) {
        const int64_t dest = label->Position() - buffer_.Size();
        if (use_far_branches() && !CanEncodeImm19BranchOffset(dest)) {
          if (cond == AL) {
            // If the condition is AL, we must always branch to dest. There is
            // no need for a guard branch.
            b(dest);
          } else {
            EmitConditionalBranchOp(op, InvertCondition(cond),
                                    2 * Instr::kInstrSize);
            // Make a new dest that takes the new position into account after the
            // inverted test.
            const int64_t dest = label->Position() - buffer_.Size();
            b(dest);
          }
        } else {
          EmitConditionalBranchOp(op, cond, dest);
        }
      } else {
        const int64_t position = buffer_.Size();
        if (use_far_branches()) {
          // When cond is AL, this guard branch will be rewritten as a nop when
          // the label is bound. We don't write it as a nop initially because it
          // makes the decoding code in Bind simpler.
          EmitConditionalBranchOp(op, InvertCondition(cond),
                                  2 * Instr::kInstrSize);
          b(label->position_);
        } else {
          EmitConditionalBranchOp(op, cond, label->position_);
        }
        label->LinkTo(position);
      }
    }
  
    void EmitCompareAndBranch(CompareAndBranchOp op,
                              Register rt,
                              Label* label,
                              OperandSize sz) {
      if (label->IsBound()) {
        const int64_t dest = label->Position() - buffer_.Size();
        if (use_far_branches() && !CanEncodeImm19BranchOffset(dest)) {
          EmitCompareAndBranchOp(op == CBZ ? CBNZ : CBZ, rt,
                                 2 * Instr::kInstrSize, sz);
          // Make a new dest that takes the new position into account after the
          // inverted test.
          const int64_t dest = label->Position() - buffer_.Size();
          b(dest);
        } else {
          EmitCompareAndBranchOp(op, rt, dest, sz);
        }
      } else {
        const int64_t position = buffer_.Size();
        if (use_far_branches()) {
          EmitCompareAndBranchOp(op == CBZ ? CBNZ : CBZ, rt,
                                 2 * Instr::kInstrSize, sz);
          b(label->position_);
        } else {
          EmitCompareAndBranchOp(op, rt, label->position_, sz);
        }
        label->LinkTo(position);
      }
    }
  
    void EmitTestAndBranch(TestAndBranchOp op,
                           Register rt,
                           intptr_t bit_number,
                           Label* label) {
      if (label->IsBound()) {
        const int64_t dest = label->Position() - buffer_.Size();
        if (use_far_branches() && !CanEncodeImm14BranchOffset(dest)) {
          EmitTestAndBranchOp(op == TBZ ? TBNZ : TBZ, rt, bit_number,
                              2 * Instr::kInstrSize);
          // Make a new dest that takes the new position into account after the
          // inverted test.
          const int64_t dest = label->Position() - buffer_.Size();
          b(dest);
        } else {
          EmitTestAndBranchOp(op, rt, bit_number, dest);
        }
      } else {
        int64_t position = buffer_.Size();
        if (use_far_branches()) {
          EmitTestAndBranchOp(op == TBZ ? TBNZ : TBZ, rt, bit_number,
                              2 * Instr::kInstrSize);
          b(label->position_);
        } else {
          EmitTestAndBranchOp(op, rt, bit_number, label->position_);
        }
        label->LinkTo(position);
      }
    }
  
    bool CanEncodeImm26BranchOffset(int64_t offset) {
      ASSERT(Utils::IsAligned(offset, 4));
      return Utils::IsInt(26, offset);
    }
  
    void EmitUnconditionalBranchOp(UnconditionalBranchOp op, int64_t offset) {
      ASSERT(CanEncodeImm26BranchOffset(offset));
      const int32_t off = ((offset >> 2) << kImm26Shift) & kImm26Mask;
      const int32_t encoding = op | off;
      Emit(encoding);
    }
  
    void EmitUnconditionalBranchRegOp(UnconditionalBranchRegOp op, Register rn) {
      ASSERT((rn != CSP) && (rn != R31));
      const int32_t encoding = op | Arm64Encode::Rn(rn);
      Emit(encoding);
    }
  
    static int32_t ExceptionGenOpEncoding(ExceptionGenOp op, uint16_t imm) {
      return op | (static_cast<int32_t>(imm) << kImm16Shift);
    }
  
    void EmitExceptionGenOp(ExceptionGenOp op, uint16_t imm) {
      Emit(ExceptionGenOpEncoding(op, imm));
    }
  
    void EmitMoveWideOp(MoveWideOp op,
                        Register rd,
                        const Immediate& imm,
                        int hw_idx,
                        OperandSize sz) {
      ASSERT((hw_idx >= 0) && (hw_idx <= 3));
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding =
          op | size | Arm64Encode::Rd(rd) |
          (static_cast<int32_t>(hw_idx) << kHWShift) |
          (static_cast<int32_t>(imm.value() & 0xffff) << kImm16Shift);
      Emit(encoding);
    }
  
    void EmitLoadStoreExclusive(LoadStoreExclusiveOp op,
                                Register rs,
                                Register rn,
                                Register rt,
                                OperandSize sz = kDoubleWord) {
      ASSERT(sz == kDoubleWord || sz == kWord);
      const int32_t size = B31 | (sz == kDoubleWord ? B30 : 0);
  
      ASSERT((rs != kNoRegister) && (rs != ZR));
      ASSERT((rn != kNoRegister) && (rn != ZR));
      ASSERT((rt != kNoRegister) && (rt != ZR));
  
      const int32_t encoding = op | size | Arm64Encode::Rs(rs) |
                               Arm64Encode::Rt2(R31) | Arm64Encode::Rn(rn) |
                               Arm64Encode::Rt(rt);
  
      Emit(encoding);
    }
  
    void EmitLoadStoreReg(LoadStoreRegOp op,
                          Register rt,
                          Address a,
                          OperandSize sz) {
      // Unpredictable, illegal on some microarchitectures.
      ASSERT((op != LDR && op != STR && op != LDRS) || a.can_writeback_to(rt));
  
      const int32_t size = Log2OperandSizeBytes(sz);
      ASSERT(a.log2sz_ == -1 || a.log2sz_ == size);
      const int32_t encoding =
          op | ((size & 0x3) << kSzShift) | Arm64Encode::Rt(rt) | a.encoding();
      Emit(encoding);
    }
  
    void EmitLoadRegLiteral(LoadRegLiteralOp op,
                            Register rt,
                            Address a,
                            OperandSize sz) {
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      ASSERT(a.log2sz_ == -1 || a.log2sz_ == Log2OperandSizeBytes(sz));
      ASSERT((rt != CSP) && (rt != R31));
      const int32_t size = (sz == kDoubleWord) ? B30 : 0;
      const int32_t encoding = op | size | Arm64Encode::Rt(rt) | a.encoding();
      Emit(encoding);
    }
  
    void EmitLoadStoreRegPair(LoadStoreRegPairOp op,
                              Register rt,
                              Register rt2,
                              Address a,
                              OperandSize sz) {
      // Unpredictable, illegal on some microarchitectures.
      ASSERT(a.can_writeback_to(rt) && a.can_writeback_to(rt2));
      ASSERT(op != LDP || rt != rt2);
  
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      ASSERT(a.log2sz_ == -1 || a.log2sz_ == Log2OperandSizeBytes(sz));
      ASSERT((rt != CSP) && (rt != R31));
      ASSERT((rt2 != CSP) && (rt2 != R31));
      int32_t opc = 0;
      switch (sz) {
        case kDoubleWord:
          opc = B31;
          break;
        case kWord:
          opc = B30;
          break;
        case kUnsignedWord:
          opc = 0;
          break;
        default:
          UNREACHABLE();
          break;
      }
      const int32_t encoding =
          opc | op | Arm64Encode::Rt(rt) | Arm64Encode::Rt2(rt2) | a.encoding();
      Emit(encoding);
    }
  
    void EmitPCRelOp(PCRelOp op, Register rd, const Immediate& imm) {
      ASSERT(Utils::IsInt(21, imm.value()));
      ASSERT((rd != R31) && (rd != CSP));
      const int32_t loimm = (imm.value() & 0x3) << 29;
      const int32_t hiimm =
          (static_cast<uint32_t>(imm.value() >> 2) << kImm19Shift) & kImm19Mask;
      const int32_t encoding = op | loimm | hiimm | Arm64Encode::Rd(rd);
      Emit(encoding);
    }
  
    void EmitMiscDP1Source(MiscDP1SourceOp op,
                           Register rd,
                           Register rn,
                           OperandSize sz) {
      ASSERT((rd != CSP) && (rn != CSP));
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding =
          op | size | Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn);
      Emit(encoding);
    }
  
    void EmitMiscDP2Source(MiscDP2SourceOp op,
                           Register rd,
                           Register rn,
                           Register rm,
                           OperandSize sz) {
      ASSERT((rd != CSP) && (rn != CSP) && (rm != CSP));
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding = op | size | Arm64Encode::Rd(rd) |
                               Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm);
      Emit(encoding);
    }
  
    void EmitMiscDP3Source(MiscDP3SourceOp op,
                           Register rd,
                           Register rn,
                           Register rm,
                           Register ra,
                           OperandSize sz) {
      ASSERT((rd != CSP) && (rn != CSP) && (rm != CSP) && (ra != CSP));
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding = op | size | Arm64Encode::Rd(rd) |
                               Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm) |
                               Arm64Encode::Ra(ra);
      Emit(encoding);
    }
  
    void EmitConditionalSelect(ConditionalSelectOp op,
                               Register rd,
                               Register rn,
                               Register rm,
                               Condition cond,
                               OperandSize sz) {
      ASSERT((rd != CSP) && (rn != CSP) && (rm != CSP));
      ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
      const int32_t size = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding = op | size | Arm64Encode::Rd(rd) |
                               Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm) |
                               (static_cast<int32_t>(cond) << kSelCondShift);
      Emit(encoding);
    }
  
    void EmitFPImm(FPImmOp op, VRegister vd, uint8_t imm8) {
      const int32_t encoding =
          op | (static_cast<int32_t>(vd) << kVdShift) | (imm8 << kImm8Shift);
      Emit(encoding);
    }
  
    void EmitFPIntCvtOp(FPIntCvtOp op,
                        Register rd,
                        Register rn,
                        OperandSize sz = kDoubleWord) {
      ASSERT((sz == kDoubleWord) || (sz == kWord));
      const int32_t sfield = (sz == kDoubleWord) ? B31 : 0;
      const int32_t encoding =
          op | Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn) | sfield;
      Emit(encoding);
    }
  
    void EmitFPOneSourceOp(FPOneSourceOp op, VRegister vd, VRegister vn) {
      const int32_t encoding = op | (static_cast<int32_t>(vd) << kVdShift) |
                               (static_cast<int32_t>(vn) << kVnShift);
      Emit(encoding);
    }
  
    void EmitFPTwoSourceOp(FPTwoSourceOp op,
                           VRegister vd,
                           VRegister vn,
                           VRegister vm) {
      const int32_t encoding = op | (static_cast<int32_t>(vd) << kVdShift) |
                               (static_cast<int32_t>(vn) << kVnShift) |
                               (static_cast<int32_t>(vm) << kVmShift);
      Emit(encoding);
    }
  
    void EmitFPCompareOp(FPCompareOp op, VRegister vn, VRegister vm) {
      const int32_t encoding = op | (static_cast<int32_t>(vn) << kVnShift) |
                               (static_cast<int32_t>(vm) << kVmShift);
      Emit(encoding);
    }
  
    void EmitSIMDThreeSameOp(SIMDThreeSameOp op,
                             VRegister vd,
                             VRegister vn,
                             VRegister vm) {
      const int32_t encoding = op | (static_cast<int32_t>(vd) << kVdShift) |
                               (static_cast<int32_t>(vn) << kVnShift) |
                               (static_cast<int32_t>(vm) << kVmShift);
      Emit(encoding);
    }
  
    void EmitSIMDCopyOp(SIMDCopyOp op,
                        VRegister vd,
                        VRegister vn,
                        OperandSize sz,
                        int32_t idx4,
                        int32_t idx5) {
      const int32_t shift = Log2OperandSizeBytes(sz);
      const int32_t imm5 = ((idx5 << (shift + 1)) | (1 << shift)) & 0x1f;
      const int32_t imm4 = (idx4 << shift) & 0xf;
      const int32_t encoding = op | (imm5 << kImm5Shift) | (imm4 << kImm4Shift) |
                               (static_cast<int32_t>(vd) << kVdShift) |
                               (static_cast<int32_t>(vn) << kVnShift);
      Emit(encoding);
    }
  
    void EmitSIMDTwoRegOp(SIMDTwoRegOp op, VRegister vd, VRegister vn) {
      const int32_t encoding = op | (static_cast<int32_t>(vd) << kVdShift) |
                               (static_cast<int32_t>(vn) << kVnShift);
      Emit(encoding);
    }
  
    enum BarrierFilterMode {
      // Filter falls through into the barrier update code. Target label
      // is a "after-store" label.
      kJumpToNoUpdate,
  
      // Filter falls through to the "after-store" code. Target label
      // is barrier update code label.
      kJumpToBarrier,
    };
  
    void StoreIntoObjectFilter(Register object,
                               Register value,
                               Label* label,
                               CanBeSmi can_be_smi,
                               BarrierFilterMode barrier_filter_mode);
  
    // Note: leaf call sequence uses some abi callee save registers as scratch
    // so they should be manually preserved.
    void EnterCallRuntimeFrame(intptr_t frame_size, bool is_leaf);
    void LeaveCallRuntimeFrame(bool is_leaf);
  
    friend class dart::FlowGraphCompiler;
    std::function<void(Register reg)> generate_invoke_write_barrier_wrapper_;
    std::function<void()> generate_invoke_array_write_barrier_;
  
    DISALLOW_ALLOCATION();
    DISALLOW_COPY_AND_ASSIGN(Assembler);
  };
  
  }  // namespace compiler
  }  // namespace dart
  
  #endif  // RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_ARM64_H_
  