text: // Copyright 2016 the V8 project authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  
  #include "src/compiler/simd-scalar-lowering.h"
  
  #include "src/codegen/machine-type.h"
  #include "src/common/globals.h"
  #include "src/compiler/diamond.h"
  #include "src/compiler/linkage.h"
  #include "src/compiler/machine-operator.h"
  #include "src/compiler/node-matchers.h"
  #include "src/compiler/node-properties.h"
  #include "src/compiler/node.h"
  #include "src/compiler/wasm-compiler.h"
  
  namespace v8 {
  namespace internal {
  namespace compiler {
  
  namespace {
  static const int kNumLanes64 = 2;
  static const int kNumLanes32 = 4;
  static const int kNumLanes16 = 8;
  static const int kNumLanes8 = 16;
  static const int32_t kMask16 = 0xFFFF;
  static const int32_t kMask8 = 0xFF;
  static const int32_t kShift16 = 16;
  static const int32_t kShift8 = 24;
  static const int32_t kShiftMask8 = 0x7;
  static const int32_t kShiftMask16 = 0xF;
  static const int32_t kShiftMask32 = 0x1F;
  static const int32_t kShiftMask64 = 0x3F;
  
  // Shift values are taken modulo lane size. This helper calculates the mask
  // required for different shift opcodes.
  int GetMaskForShift(Node* node) {
    switch (node->opcode()) {
      case IrOpcode::kI8x16Shl:
      case IrOpcode::kI8x16ShrS:
      case IrOpcode::kI8x16ShrU:
        return kShiftMask8;
      case IrOpcode::kI16x8Shl:
      case IrOpcode::kI16x8ShrS:
      case IrOpcode::kI16x8ShrU:
        return kShiftMask16;
      case IrOpcode::kI32x4Shl:
      case IrOpcode::kI32x4ShrS:
      case IrOpcode::kI32x4ShrU:
        return kShiftMask32;
      case IrOpcode::kI64x2Shl:
      case IrOpcode::kI64x2ShrS:
      case IrOpcode::kI64x2ShrU:
        return kShiftMask64;
      default:
        UNIMPLEMENTED();
    }
  }
  }  // anonymous namespace
  
  SimdScalarLowering::SimdScalarLowering(
      MachineGraph* mcgraph, Signature<MachineRepresentation>* signature)
      : mcgraph_(mcgraph),
        state_(mcgraph->graph(), 3),
        stack_(mcgraph_->zone()),
        replacements_(nullptr),
        signature_(signature),
        placeholder_(graph()->NewNode(common()->Parameter(-2, "placeholder"),
                                      graph()->start())),
        parameter_count_after_lowering_(-1) {
    DCHECK_NOT_NULL(graph());
    DCHECK_NOT_NULL(graph()->end());
    replacements_ = zone()->NewArray<Replacement>(graph()->NodeCount());
    memset(static_cast<void*>(replacements_), 0,
           sizeof(Replacement) * graph()->NodeCount());
  }
  
  void SimdScalarLowering::LowerGraph() {
    stack_.push_back({graph()->end(), 0});
    state_.Set(graph()->end(), State::kOnStack);
    replacements_[graph()->end()->id()].type = SimdType::kInt32x4;
  
    while (!stack_.empty()) {
      NodeState& top = stack_.back();
      if (top.input_index == top.node->InputCount()) {
        // All inputs of top have already been lowered, now lower top.
        stack_.pop_back();
        state_.Set(top.node, State::kVisited);
        LowerNode(top.node);
      } else {
        // Push the next input onto the stack.
        Node* input = top.node->InputAt(top.input_index++);
        if (state_.Get(input) == State::kUnvisited) {
          SetLoweredType(input, top.node);
          if (input->opcode() == IrOpcode::kPhi) {
            // To break cycles with phi nodes we push phis on a separate stack so
            // that they are processed after all other nodes.
            PreparePhiReplacement(input);
            stack_.push_front({input, 0});
          } else if (input->opcode() == IrOpcode::kEffectPhi ||
                     input->opcode() == IrOpcode::kLoop) {
            stack_.push_front({input, 0});
          } else {
            stack_.push_back({input, 0});
          }
          state_.Set(input, State::kOnStack);
        }
      }
    }
  }
  
  #define FOREACH_INT64X2_OPCODE(V) \
    V(I64x2Splat)                   \
    V(I64x2ExtractLane)             \
    V(I64x2ReplaceLane)             \
    V(I64x2Neg)                     \
    V(I64x2Shl)                     \
    V(I64x2ShrS)                    \
    V(I64x2ShrU)                    \
    V(I64x2Add)                     \
    V(I64x2Sub)                     \
    V(I64x2Mul)
  
  #define FOREACH_INT32X4_OPCODE(V) \
    V(I32x4Splat)                   \
    V(I32x4ExtractLane)             \
    V(I32x4ReplaceLane)             \
    V(I32x4SConvertF32x4)           \
    V(I32x4UConvertF32x4)           \
    V(I32x4SConvertI16x8Low)        \
    V(I32x4SConvertI16x8High)       \
    V(I32x4Neg)                     \
    V(I32x4Shl)                     \
    V(I32x4ShrS)                    \
    V(I32x4Add)                     \
    V(I32x4AddHoriz)                \
    V(I32x4Sub)                     \
    V(I32x4Mul)                     \
    V(I32x4MinS)                    \
    V(I32x4MaxS)                    \
    V(I32x4ShrU)                    \
    V(I32x4MinU)                    \
    V(I32x4MaxU)                    \
    V(I32x4Eq)                      \
    V(I32x4Ne)                      \
    V(I32x4LtS)                     \
    V(I32x4LtU)                     \
    V(I32x4GtS)                     \
    V(I32x4GtU)                     \
    V(I32x4LeS)                     \
    V(I32x4LeU)                     \
    V(I32x4GeS)                     \
    V(I32x4GeU)                     \
    V(I32x4UConvertI16x8Low)        \
    V(I32x4UConvertI16x8High)       \
    V(I32x4Abs)                     \
    V(S128And)                      \
    V(S128Or)                       \
    V(S128Xor)                      \
    V(S128Not)                      \
    V(V32x4AnyTrue)                 \
    V(V32x4AllTrue)                 \
    V(V16x8AnyTrue)                 \
    V(V16x8AllTrue)                 \
    V(V8x16AnyTrue)                 \
    V(V8x16AllTrue)                 \
    V(I32x4BitMask)
  
  #define FOREACH_FLOAT64X2_OPCODE(V) \
    V(F64x2Splat)                     \
    V(F64x2ExtractLane)               \
    V(F64x2ReplaceLane)               \
    V(F64x2Abs)                       \
    V(F64x2Neg)                       \
    V(F64x2Sqrt)                      \
    V(F64x2Add)                       \
    V(F64x2Sub)                       \
    V(F64x2Mul)                       \
    V(F64x2Div)                       \
    V(F64x2Min)                       \
    V(F64x2Max)
  
  #define FOREACH_FLOAT32X4_OPCODE(V) \
    V(F32x4Splat)                     \
    V(F32x4ExtractLane)               \
    V(F32x4ReplaceLane)               \
    V(F32x4SConvertI32x4)             \
    V(F32x4UConvertI32x4)             \
    V(F32x4Abs)                       \
    V(F32x4Neg)                       \
    V(F32x4Sqrt)                      \
    V(F32x4RecipApprox)               \
    V(F32x4RecipSqrtApprox)           \
    V(F32x4Add)                       \
    V(F32x4AddHoriz)                  \
    V(F32x4Sub)                       \
    V(F32x4Mul)                       \
    V(F32x4Div)                       \
    V(F32x4Min)                       \
    V(F32x4Max)
  
  #define FOREACH_FLOAT64x2_TO_INT64x2OPCODE(V) \
    V(F64x2Eq)                                  \
    V(F64x2Ne)                                  \
    V(F64x2Lt)                                  \
    V(F64x2Le)
  
  #define FOREACH_FLOAT32X4_TO_INT32X4OPCODE(V) \
    V(F32x4Eq)                                  \
    V(F32x4Ne)                                  \
    V(F32x4Lt)                                  \
    V(F32x4Le)                                  \
    V(F32x4Gt)                                  \
    V(F32x4Ge)
  
  #define FOREACH_INT16X8_OPCODE(V) \
    V(I16x8Splat)                   \
    V(I16x8ExtractLaneU)            \
    V(I16x8ExtractLaneS)            \
    V(I16x8ReplaceLane)             \
    V(I16x8SConvertI8x16Low)        \
    V(I16x8SConvertI8x16High)       \
    V(I16x8Neg)                     \
    V(I16x8Shl)                     \
    V(I16x8ShrS)                    \
    V(I16x8SConvertI32x4)           \
    V(I16x8Add)                     \
    V(I16x8AddSaturateS)            \
    V(I16x8AddHoriz)                \
    V(I16x8Sub)                     \
    V(I16x8SubSaturateS)            \
    V(I16x8Mul)                     \
    V(I16x8MinS)                    \
    V(I16x8MaxS)                    \
    V(I16x8UConvertI8x16Low)        \
    V(I16x8UConvertI8x16High)       \
    V(I16x8ShrU)                    \
    V(I16x8UConvertI32x4)           \
    V(I16x8AddSaturateU)            \
    V(I16x8SubSaturateU)            \
    V(I16x8MinU)                    \
    V(I16x8MaxU)                    \
    V(I16x8Eq)                      \
    V(I16x8Ne)                      \
    V(I16x8LtS)                     \
    V(I16x8LtU)                     \
    V(I16x8GtS)                     \
    V(I16x8GtU)                     \
    V(I16x8LeS)                     \
    V(I16x8LeU)                     \
    V(I16x8GeS)                     \
    V(I16x8GeU)                     \
    V(I16x8RoundingAverageU)        \
    V(I16x8Abs)                     \
    V(I16x8BitMask)
  
  #define FOREACH_INT8X16_OPCODE(V) \
    V(I8x16Splat)                   \
    V(I8x16ExtractLaneU)            \
    V(I8x16ExtractLaneS)            \
    V(I8x16ReplaceLane)             \
    V(I8x16SConvertI16x8)           \
    V(I8x16Neg)                     \
    V(I8x16Shl)                     \
    V(I8x16ShrS)                    \
    V(I8x16Add)                     \
    V(I8x16AddSaturateS)            \
    V(I8x16Sub)                     \
    V(I8x16SubSaturateS)            \
    V(I8x16Mul)                     \
    V(I8x16MinS)                    \
    V(I8x16MaxS)                    \
    V(I8x16ShrU)                    \
    V(I8x16UConvertI16x8)           \
    V(I8x16AddSaturateU)            \
    V(I8x16SubSaturateU)            \
    V(I8x16MinU)                    \
    V(I8x16MaxU)                    \
    V(I8x16Eq)                      \
    V(I8x16Ne)                      \
    V(I8x16LtS)                     \
    V(I8x16LtU)                     \
    V(I8x16GtS)                     \
    V(I8x16GtU)                     \
    V(I8x16LeS)                     \
    V(I8x16LeU)                     \
    V(I8x16GeS)                     \
    V(I8x16GeU)                     \
    V(S8x16Swizzle)                 \
    V(S8x16Shuffle)                 \
    V(I8x16RoundingAverageU)        \
    V(I8x16Abs)                     \
    V(I8x16BitMask)
  
  MachineType SimdScalarLowering::MachineTypeFrom(SimdType simdType) {
    switch (simdType) {
      case SimdType::kFloat64x2:
        return MachineType::Float64();
      case SimdType::kFloat32x4:
        return MachineType::Float32();
      case SimdType::kInt64x2:
        return MachineType::Int64();
      case SimdType::kInt32x4:
        return MachineType::Int32();
      case SimdType::kInt16x8:
        return MachineType::Int16();
      case SimdType::kInt8x16:
        return MachineType::Int8();
    }
    return MachineType::None();
  }
  
  void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
    switch (node->opcode()) {
  #define CASE_STMT(name) case IrOpcode::k##name:
      FOREACH_FLOAT64X2_OPCODE(CASE_STMT) {
        replacements_[node->id()].type = SimdType::kFloat64x2;
        break;
      }
      FOREACH_INT64X2_OPCODE(CASE_STMT) {
        replacements_[node->id()].type = SimdType::kInt64x2;
        break;
      }
      FOREACH_INT32X4_OPCODE(CASE_STMT)
      case IrOpcode::kReturn:
      case IrOpcode::kParameter:
      case IrOpcode::kCall: {
        replacements_[node->id()].type = SimdType::kInt32x4;
        break;
      }
        FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
          replacements_[node->id()].type = SimdType::kFloat32x4;
          break;
        }
        FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT) {
          replacements_[node->id()].type = SimdType::kInt32x4;
          break;
        }
        FOREACH_FLOAT64x2_TO_INT64x2OPCODE(CASE_STMT) {
          replacements_[node->id()].type = SimdType::kInt64x2;
          break;
        }
        FOREACH_INT16X8_OPCODE(CASE_STMT) {
          replacements_[node->id()].type = SimdType::kInt16x8;
          break;
        }
        FOREACH_INT8X16_OPCODE(CASE_STMT) {
          replacements_[node->id()].type = SimdType::kInt8x16;
          break;
        }
      case IrOpcode::kLoadTransform: {
        LoadTransformParameters params = LoadTransformParametersOf(node->op());
        switch (params.transformation) {
          case LoadTransformation::kS8x16LoadSplat:
            replacements_[node->id()].type = SimdType::kInt8x16;
            break;
          case LoadTransformation::kS16x8LoadSplat:
            replacements_[node->id()].type = SimdType::kInt16x8;
            break;
          case LoadTransformation::kS32x4LoadSplat:
            replacements_[node->id()].type = SimdType::kInt32x4;
            break;
          case LoadTransformation::kI16x8Load8x8S:
          case LoadTransformation::kI16x8Load8x8U:
            replacements_[node->id()].type = SimdType::kInt16x8;
            break;
          case LoadTransformation::kI32x4Load16x4S:
          case LoadTransformation::kI32x4Load16x4U:
            replacements_[node->id()].type = SimdType::kInt32x4;
            break;
          case LoadTransformation::kI64x2Load32x2S:
          case LoadTransformation::kI64x2Load32x2U:
            replacements_[node->id()].type = SimdType::kInt64x2;
            break;
          default:
            UNIMPLEMENTED();
        }
        break;
      }
      default: {
        switch (output->opcode()) {
          case IrOpcode::kF32x4SConvertI32x4:
          case IrOpcode::kF32x4UConvertI32x4:
          case IrOpcode::kI16x8SConvertI32x4:
          case IrOpcode::kI16x8UConvertI32x4: {
            replacements_[node->id()].type = SimdType::kInt32x4;
            break;
          }
          case IrOpcode::kI8x16SConvertI16x8:
          case IrOpcode::kI8x16UConvertI16x8:
          case IrOpcode::kI32x4SConvertI16x8Low:
          case IrOpcode::kI32x4SConvertI16x8High:
          case IrOpcode::kI32x4UConvertI16x8Low:
          case IrOpcode::kI32x4UConvertI16x8High: {
            replacements_[node->id()].type = SimdType::kInt16x8;
            break;
          }
          case IrOpcode::kI16x8SConvertI8x16Low:
          case IrOpcode::kI16x8SConvertI8x16High:
          case IrOpcode::kI16x8UConvertI8x16Low:
          case IrOpcode::kI16x8UConvertI8x16High: {
            replacements_[node->id()].type = SimdType::kInt8x16;
            break;
          }
            FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT)
          case IrOpcode::kI32x4SConvertF32x4:
          case IrOpcode::kI32x4UConvertF32x4: {
            replacements_[node->id()].type = SimdType::kFloat32x4;
            break;
          }
          case IrOpcode::kS128Select: {
            replacements_[node->id()].type = SimdType::kInt32x4;
            break;
          }
          default: {
            replacements_[node->id()].type = replacements_[output->id()].type;
          }
        }
      }
  #undef CASE_STMT
    }
  }
  
  static int GetParameterIndexAfterLoweringSimd128(
      Signature<MachineRepresentation>* signature, int old_index) {
    // In function calls, the simd128 types are passed as 4 Int32 types. The
    // parameters are typecast to the types as needed for various operations.
    int result = old_index;
    for (int i = 0; i < old_index; ++i) {
      if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
        result += 3;
      }
    }
    return result;
  }
  
  int SimdScalarLowering::GetParameterCountAfterLowering() {
    if (parameter_count_after_lowering_ == -1) {
      // GetParameterIndexAfterLoweringSimd128(parameter_count) returns the
      // parameter count after lowering.
      parameter_count_after_lowering_ = GetParameterIndexAfterLoweringSimd128(
          signature(), static_cast<int>(signature()->parameter_count()));
    }
    return parameter_count_after_lowering_;
  }
  
  static int GetReturnCountAfterLoweringSimd128(
      Signature<MachineRepresentation>* signature) {
    int result = static_cast<int>(signature->return_count());
    for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
      if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
        result += 3;
      }
    }
    return result;
  }
  
  int GetReturnIndexAfterLowering(const CallDescriptor* call_descriptor,
                                  int old_index) {
    int result = old_index;
    for (int i = 0; i < old_index; ++i) {
      if (call_descriptor->GetReturnType(i).representation() ==
          MachineRepresentation::kSimd128) {
        result += kNumLanes32 - 1;
      }
    }
    return result;
  }
  
  static int GetReturnCountAfterLoweringSimd128(
      const CallDescriptor* call_descriptor) {
    return GetReturnIndexAfterLowering(
        call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
  }
  
  int SimdScalarLowering::NumLanes(SimdType type) {
    int num_lanes = 0;
    if (type == SimdType::kFloat64x2 || type == SimdType::kInt64x2) {
      num_lanes = kNumLanes64;
    } else if (type == SimdType::kFloat32x4 || type == SimdType::kInt32x4) {
      num_lanes = kNumLanes32;
    } else if (type == SimdType::kInt16x8) {
      num_lanes = kNumLanes16;
    } else if (type == SimdType::kInt8x16) {
      num_lanes = kNumLanes8;
    } else {
      UNREACHABLE();
    }
    return num_lanes;
  }
  
  constexpr int SimdScalarLowering::kLaneOffsets[];
  
  void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices,
                                         SimdType type) {
    int num_lanes = NumLanes(type);
    int lane_width = kSimd128Size / num_lanes;
    int laneIndex = kLaneOffsets[0] / lane_width;
    new_indices[laneIndex] = index;
    for (int i = 1; i < num_lanes; ++i) {
      laneIndex = kLaneOffsets[i * lane_width] / lane_width;
      new_indices[laneIndex] = graph()->NewNode(
          machine()->Int32Add(), index,
          graph()->NewNode(
              common()->Int32Constant(static_cast<int>(i) * lane_width)));
    }
  }
  
  void SimdScalarLowering::LowerLoadOp(Node* node, SimdType type) {
    MachineRepresentation rep = LoadRepresentationOf(node->op()).representation();
    const Operator* load_op;
    switch (node->opcode()) {
      case IrOpcode::kLoad:
        load_op = machine()->Load(MachineTypeFrom(type));
        break;
      case IrOpcode::kUnalignedLoad:
        load_op = machine()->UnalignedLoad(MachineTypeFrom(type));
        break;
      case IrOpcode::kProtectedLoad:
        load_op = machine()->ProtectedLoad(MachineTypeFrom(type));
        break;
      default:
        UNREACHABLE();
    }
    if (rep == MachineRepresentation::kSimd128) {
      Node* base = node->InputAt(0);
      Node* index = node->InputAt(1);
      int num_lanes = NumLanes(type);
      Node** indices = zone()->NewArray<Node*>(num_lanes);
      GetIndexNodes(index, indices, type);
      Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
      rep_nodes[0] = node;
      rep_nodes[0]->ReplaceInput(1, indices[0]);
      NodeProperties::ChangeOp(rep_nodes[0], load_op);
      if (node->InputCount() > 2) {
        DCHECK_LT(3, node->InputCount());
        Node* effect_input = node->InputAt(2);
        Node* control_input = node->InputAt(3);
        for (int i = num_lanes - 1; i > 0; --i) {
          rep_nodes[i] = graph()->NewNode(load_op, base, indices[i], effect_input,
                                          control_input);
          effect_input = rep_nodes[i];
        }
        rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
      } else {
        for (int i = 1; i < num_lanes; ++i) {
          rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
        }
      }
      ReplaceNode(node, rep_nodes, num_lanes);
    } else {
      DefaultLowering(node);
    }
  }
  
  void SimdScalarLowering::LowerLoadTransformOp(Node* node, SimdType type) {
    LoadTransformParameters params = LoadTransformParametersOf(node->op());
    MachineType load_rep = MachineType::None();
    SimdType load_type = type;
  
    // Load extends have a different machine type for loading.
    switch (params.transformation) {
      case LoadTransformation::kI16x8Load8x8S:
        load_rep = MachineType::Int8();
        load_type = SimdType::kInt8x16;
        break;
      case LoadTransformation::kI16x8Load8x8U:
        load_rep = MachineType::Uint8();
        load_type = SimdType::kInt8x16;
        break;
      case LoadTransformation::kI32x4Load16x4S:
        load_rep = MachineType::Int16();
        load_type = SimdType::kInt16x8;
        break;
      case LoadTransformation::kI32x4Load16x4U:
        load_rep = MachineType::Uint16();
        load_type = SimdType::kInt16x8;
        break;
      case LoadTransformation::kI64x2Load32x2S:
        load_rep = MachineType::Int32();
        load_type = SimdType::kInt32x4;
        break;
      case LoadTransformation::kI64x2Load32x2U:
        load_rep = MachineType::Uint32();
        load_type = SimdType::kInt32x4;
        break;
      case LoadTransformation::kS8x16LoadSplat:
      case LoadTransformation::kS16x8LoadSplat:
      case LoadTransformation::kS32x4LoadSplat:
        load_rep = MachineTypeFrom(type);
        break;
      default:
        // Lowering for s64x2 is not implemented since lowering for 64x2
        // operations doesn't work properly yet.
        UNIMPLEMENTED();
    }
  
    DCHECK_NE(load_rep, MachineType::None());
  
    const Operator* load_op;
    switch (params.kind) {
      case LoadKind::kNormal:
        load_op = machine()->Load(load_rep);
        break;
      case LoadKind::kUnaligned:
        load_op = machine()->UnalignedLoad(load_rep);
        break;
      case LoadKind::kProtected:
        load_op = machine()->ProtectedLoad(load_rep);
        break;
    }
  
    Node* base = node->InputAt(0);
    Node* index = node->InputAt(1);
    int num_lanes = NumLanes(type);
    Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
    Node* effect_input = node->InputAt(2);
    Node* control_input = node->InputAt(3);
  
    if (type != load_type) {
      // We load a smaller lane size, then extend to a larger lane size. So use
      // the smaller lane size to calculte the index nodes for loads, but only
      // actually load half of those lanes.
      Node** indices = zone()->NewArray<Node*>(num_lanes * 2);
      GetIndexNodes(index, indices, load_type);
      for (int i = num_lanes - 1; i >= 0; --i) {
        rep_nodes[i] = graph()->NewNode(load_op, base, indices[i], effect_input,
                                        control_input);
        effect_input = rep_nodes[i];
  
        // Load operations are Word32 nodes, change them to Word64.
        if (params.transformation == LoadTransformation::kI64x2Load32x2S) {
          rep_nodes[i] =
              graph()->NewNode(machine()->ChangeInt32ToInt64(), rep_nodes[i]);
        } else if (params.transformation == LoadTransformation::kI64x2Load32x2U) {
          rep_nodes[i] =
              graph()->NewNode(machine()->ChangeUint32ToUint64(), rep_nodes[i]);
        }
      }
    } else {
      // Load splat, load from the same index for every lane.
      for (int i = num_lanes - 1; i >= 0; --i) {
        rep_nodes[i] =
            graph()->NewNode(load_op, base, index, effect_input, control_input);
        effect_input = rep_nodes[i];
      }
    }
    ReplaceNode(node, rep_nodes, num_lanes);
  }
  
  void SimdScalarLowering::LowerStoreOp(Node* node) {
    // For store operation, use replacement type of its input instead of the
    // one of its effected node.
    DCHECK_LT(2, node->InputCount());
    SimdType rep_type = ReplacementType(node->InputAt(2));
    replacements_[node->id()].type = rep_type;
    const Operator* store_op;
    MachineRepresentation rep;
    switch (node->opcode()) {
      case IrOpcode::kStore: {
        rep = StoreRepresentationOf(node->op()).representation();
        WriteBarrierKind write_barrier_kind =
            StoreRepresentationOf(node->op()).write_barrier_kind();
        store_op = machine()->Store(StoreRepresentation(
            MachineTypeFrom(rep_type).representation(), write_barrier_kind));
        break;
      }
      case IrOpcode::kUnalignedStore: {
        rep = UnalignedStoreRepresentationOf(node->op());
        store_op =
            machine()->UnalignedStore(MachineTypeFrom(rep_type).representation());
        break;
      }
      case IrOpcode::kProtectedStore: {
        rep = StoreRepresentationOf(node->op()).representation();
        store_op =
            machine()->ProtectedStore(MachineTypeFrom(rep_type).representation());
        break;
      }
      default:
        UNREACHABLE();
    }
    if (rep == MachineRepresentation::kSimd128) {
      Node* base = node->InputAt(0);
      Node* index = node->InputAt(1);
      int num_lanes = NumLanes(rep_type);
      Node** indices = zone()->NewArray<Node*>(num_lanes);
      GetIndexNodes(index, indices, rep_type);
      Node* value = node->InputAt(2);
      DCHECK(HasReplacement(1, value));
      Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
      rep_nodes[0] = node;
      Node** rep_inputs = GetReplacementsWithType(value, rep_type);
      rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
      rep_nodes[0]->ReplaceInput(1, indices[0]);
      NodeProperties::ChangeOp(node, store_op);
      if (node->InputCount() > 3) {
        DCHECK_LT(4, node->InputCount());
        Node* effect_input = node->InputAt(3);
        Node* control_input = node->InputAt(4);
        for (int i = num_lanes - 1; i > 0; --i) {
          rep_nodes[i] =
              graph()->NewNode(store_op, base, indices[i], rep_inputs[i],
                               effect_input, control_input);
          effect_input = rep_nodes[i];
        }
        rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
      } else {
        for (int i = 1; i < num_lanes; ++i) {
          rep_nodes[i] =
              graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
        }
      }
      ReplaceNode(node, rep_nodes, num_lanes);
    } else {
      DefaultLowering(node);
    }
  }
  
  void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType input_rep_type,
                                         const Operator* op,
                                         bool not_horizontal) {
    DCHECK_EQ(2, node->InputCount());
    Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
    Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
    int num_lanes = NumLanes(input_rep_type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    if (not_horizontal) {
      for (int i = 0; i < num_lanes; ++i) {
        rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
      }
    } else {
      for (int i = 0; i < num_lanes / 2; ++i) {
        rep_node[i] = graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]);
        rep_node[i + num_lanes / 2] =
            graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]);
      }
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  void SimdScalarLowering::LowerCompareOp(Node* node, SimdType input_rep_type,
                                          const Operator* op,
                                          bool invert_inputs) {
    DCHECK_EQ(2, node->InputCount());
    Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
    Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
    int num_lanes = NumLanes(input_rep_type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    for (int i = 0; i < num_lanes; ++i) {
      Node* cmp_result = nullptr;
      if (invert_inputs) {
        cmp_result = graph()->NewNode(op, rep_right[i], rep_left[i]);
      } else {
        cmp_result = graph()->NewNode(op, rep_left[i], rep_right[i]);
      }
      Diamond d_cmp(graph(), common(), cmp_result);
      rep_node[i] = ConstructPhiForComparison(d_cmp, input_rep_type, -1, 0);
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  Node* SimdScalarLowering::FixUpperBits(Node* input, int32_t shift) {
    return graph()->NewNode(machine()->Word32Sar(),
                            graph()->NewNode(machine()->Word32Shl(), input,
                                             mcgraph_->Int32Constant(shift)),
                            mcgraph_->Int32Constant(shift));
  }
  
  void SimdScalarLowering::LowerBinaryOpForSmallInt(Node* node,
                                                    SimdType input_rep_type,
                                                    const Operator* op,
                                                    bool not_horizontal) {
    DCHECK_EQ(2, node->InputCount());
    DCHECK(input_rep_type == SimdType::kInt16x8 ||
           input_rep_type == SimdType::kInt8x16);
    Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
    Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
    int num_lanes = NumLanes(input_rep_type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    int32_t shift_val =
        (input_rep_type == SimdType::kInt16x8) ? kShift16 : kShift8;
    if (not_horizontal) {
      for (int i = 0; i < num_lanes; ++i) {
        rep_node[i] = FixUpperBits(
            graph()->NewNode(op, rep_left[i], rep_right[i]), shift_val);
      }
    } else {
      for (int i = 0; i < num_lanes / 2; ++i) {
        rep_node[i] = FixUpperBits(
            graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]),
            shift_val);
        rep_node[i + num_lanes / 2] = FixUpperBits(
            graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]),
            shift_val);
      }
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  Node* SimdScalarLowering::Mask(Node* input, int32_t mask) {
    return graph()->NewNode(machine()->Word32And(), input,
                            mcgraph_->Int32Constant(mask));
  }
  
  void SimdScalarLowering::LowerSaturateBinaryOp(Node* node,
                                                 SimdType input_rep_type,
                                                 const Operator* op,
                                                 bool is_signed) {
    DCHECK_EQ(2, node->InputCount());
    DCHECK(input_rep_type == SimdType::kInt16x8 ||
           input_rep_type == SimdType::kInt8x16);
    Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
    Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
    int32_t min = 0;
    int32_t max = 0;
    int32_t mask = 0;
    int32_t shift_val = 0;
    MachineRepresentation phi_rep;
    if (input_rep_type == SimdType::kInt16x8) {
      if (is_signed) {
        min = std::numeric_limits<int16_t>::min();
        max = std::numeric_limits<int16_t>::max();
      } else {
        min = std::numeric_limits<uint16_t>::min();
        max = std::numeric_limits<uint16_t>::max();
      }
      mask = kMask16;
      shift_val = kShift16;
      phi_rep = MachineRepresentation::kWord16;
    } else {
      if (is_signed) {
        min = std::numeric_limits<int8_t>::min();
        max = std::numeric_limits<int8_t>::max();
      } else {
        min = std::numeric_limits<uint8_t>::min();
        max = std::numeric_limits<uint8_t>::max();
      }
      mask = kMask8;
      shift_val = kShift8;
      phi_rep = MachineRepresentation::kWord8;
    }
    int num_lanes = NumLanes(input_rep_type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    for (int i = 0; i < num_lanes; ++i) {
      Node* op_result = nullptr;
      Node* left = is_signed ? rep_left[i] : Mask(rep_left[i], mask);
      Node* right = is_signed ? rep_right[i] : Mask(rep_right[i], mask);
      op_result = graph()->NewNode(op, left, right);
      Diamond d_min(graph(), common(),
                    graph()->NewNode(machine()->Int32LessThan(), op_result,
                                     mcgraph_->Int32Constant(min)));
      rep_node[i] = d_min.Phi(phi_rep, mcgraph_->Int32Constant(min), op_result);
      Diamond d_max(graph(), common(),
                    graph()->NewNode(machine()->Int32LessThan(),
                                     mcgraph_->Int32Constant(max), rep_node[i]));
      rep_node[i] = d_max.Phi(phi_rep, mcgraph_->Int32Constant(max), rep_node[i]);
      rep_node[i] =
          is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  void SimdScalarLowering::LowerUnaryOp(Node* node, SimdType input_rep_type,
                                        const Operator* op) {
    DCHECK_EQ(1, node->InputCount());
    Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
    int num_lanes = NumLanes(input_rep_type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    for (int i = 0; i < num_lanes; ++i) {
      rep_node[i] = graph()->NewNode(op, rep[i]);
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  void SimdScalarLowering::LowerIntMinMax(Node* node, const Operator* op,
                                          bool is_max, SimdType type) {
    DCHECK_EQ(2, node->InputCount());
    Node** rep_left = GetReplacementsWithType(node->InputAt(0), type);
    Node** rep_right = GetReplacementsWithType(node->InputAt(1), type);
    int num_lanes = NumLanes(type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    MachineRepresentation rep = MachineRepresentation::kNone;
    if (type == SimdType::kInt32x4) {
      rep = MachineRepresentation::kWord32;
    } else if (type == SimdType::kInt16x8) {
      rep = MachineRepresentation::kWord16;
    } else if (type == SimdType::kInt8x16) {
      rep = MachineRepresentation::kWord8;
    } else {
      UNREACHABLE();
    }
    for (int i = 0; i < num_lanes; ++i) {
      Diamond d(graph(), common(),
                graph()->NewNode(op, rep_left[i], rep_right[i]));
      if (is_max) {
        rep_node[i] = d.Phi(rep, rep_right[i], rep_left[i]);
      } else {
        rep_node[i] = d.Phi(rep, rep_left[i], rep_right[i]);
      }
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  Node* SimdScalarLowering::BuildF64Trunc(Node* input) {
    if (machine()->Float64RoundTruncate().IsSupported()) {
      return graph()->NewNode(machine()->Float64RoundTruncate().op(), input);
    } else {
      ExternalReference ref = ExternalReference::wasm_f64_trunc();
      Node* stack_slot =
          graph()->NewNode(machine()->StackSlot(MachineRepresentation::kFloat64));
      const Operator* store_op = machine()->Store(
          StoreRepresentation(MachineRepresentation::kFloat64, kNoWriteBarrier));
      Node* effect =
          graph()->NewNode(store_op, stack_slot, mcgraph_->Int32Constant(0),
                           input, graph()->start(), graph()->start());
      Node* function = graph()->NewNode(common()->ExternalConstant(ref));
      Node** args = zone()->NewArray<Node*>(4);
      args[0] = function;
      args[1] = stack_slot;
      args[2] = effect;
      args[3] = graph()->start();
      Signature<MachineType>::Builder sig_builder(zone(), 0, 1);
      sig_builder.AddParam(MachineType::Pointer());
      auto call_descriptor =
          Linkage::GetSimplifiedCDescriptor(zone(), sig_builder.Build());
      Node* call = graph()->NewNode(common()->Call(call_descriptor), 4, args);
      return graph()->NewNode(machine()->Load(LoadRepresentation::Float64()),
                              stack_slot, mcgraph_->Int32Constant(0), call,
                              graph()->start());
    }
  }
  
  void SimdScalarLowering::LowerConvertFromFloat(Node* node, bool is_signed) {
    DCHECK_EQ(1, node->InputCount());
    Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kFloat32x4);
    Node* rep_node[kNumLanes32];
    Node* double_zero = graph()->NewNode(common()->Float64Constant(0.0));
    Node* min = graph()->NewNode(
        common()->Float64Constant(static_cast<double>(is_signed ? kMinInt : 0)));
    Node* max = graph()->NewNode(common()->Float64Constant(
        static_cast<double>(is_signed ? kMaxInt : 0xFFFFFFFFu)));
    for (int i = 0; i < kNumLanes32; ++i) {
      Node* double_rep =
          graph()->NewNode(machine()->ChangeFloat32ToFloat64(), rep[i]);
      Diamond nan_d(graph(), common(), graph()->NewNode(machine()->Float64Equal(),
                                                        double_rep, double_rep));
      Node* temp =
          nan_d.Phi(MachineRepresentation::kFloat64, double_rep, double_zero);
      Diamond min_d(graph(), common(),
                    graph()->NewNode(machine()->Float64LessThan(), temp, min));
      temp = min_d.Phi(MachineRepresentation::kFloat64, min, temp);
      Diamond max_d(graph(), common(),
                    graph()->NewNode(machine()->Float64LessThan(), max, temp));
      temp = max_d.Phi(MachineRepresentation::kFloat64, max, temp);
      Node* trunc = BuildF64Trunc(temp);
      if (is_signed) {
        rep_node[i] = graph()->NewNode(machine()->ChangeFloat64ToInt32(), trunc);
      } else {
        rep_node[i] =
            graph()->NewNode(machine()->TruncateFloat64ToUint32(), trunc);
      }
    }
    ReplaceNode(node, rep_node, kNumLanes32);
  }
  
  void SimdScalarLowering::LowerConvertFromInt(Node* node,
                                               SimdType input_rep_type,
                                               SimdType output_rep_type,
                                               bool is_signed, int start_index) {
    DCHECK_EQ(1, node->InputCount());
    Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
  
    int32_t mask = 0;
    if (input_rep_type == SimdType::kInt16x8) {
      DCHECK_EQ(output_rep_type, SimdType::kInt32x4);
      mask = kMask16;
    } else {
      DCHECK_EQ(output_rep_type, SimdType::kInt16x8);
      DCHECK_EQ(input_rep_type, SimdType::kInt8x16);
      mask = kMask8;
    }
  
    int num_lanes = NumLanes(output_rep_type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    for (int i = 0; i < num_lanes; ++i) {
      rep_node[i] =
          is_signed ? rep[i + start_index] : Mask(rep[i + start_index], mask);
    }
  
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  void SimdScalarLowering::LowerPack(Node* node, SimdType input_rep_type,
                                     SimdType output_rep_type, bool is_signed) {
    DCHECK_EQ(2, node->InputCount());
    Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
    Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
    const Operator* less_op = machine()->Int32LessThan();
    Node* min = nullptr;
    Node* max = nullptr;
    MachineRepresentation phi_rep;
    if (output_rep_type == SimdType::kInt16x8) {
      DCHECK(input_rep_type == SimdType::kInt32x4);
      if (is_signed) {
        min = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::min());
        max = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::max());
      } else {
        min = mcgraph_->Int32Constant(std::numeric_limits<uint16_t>::min());
        max = mcgraph_->Uint32Constant(std::numeric_limits<uint16_t>::max());
      }
      phi_rep = MachineRepresentation::kWord16;
    } else {
      DCHECK(output_rep_type == SimdType::kInt8x16 &&
             input_rep_type == SimdType::kInt16x8);
      if (is_signed) {
        min = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::min());
        max = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::max());
      } else {
        min = mcgraph_->Int32Constant(std::numeric_limits<uint8_t>::min());
        max = mcgraph_->Uint32Constant(std::numeric_limits<uint8_t>::max());
      }
      phi_rep = MachineRepresentation::kWord8;
    }
    int num_lanes = NumLanes(output_rep_type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    for (int i = 0; i < num_lanes; ++i) {
      Node* input = nullptr;
      if (i < num_lanes / 2)
        input = rep_left[i];
      else
        input = rep_right[i - num_lanes / 2];
      Diamond d_min(graph(), common(), graph()->NewNode(less_op, input, min));
      input = d_min.Phi(phi_rep, min, input);
      Diamond d_max(graph(), common(), graph()->NewNode(less_op, max, input));
      rep_node[i] = d_max.Phi(phi_rep, max, input);
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  void SimdScalarLowering::LowerShiftOp(Node* node, SimdType type) {
    DCHECK_EQ(2, node->InputCount());
    Node* shift_node = Mask(node->InputAt(1), GetMaskForShift(node));
    Node** rep = GetReplacementsWithType(node->InputAt(0), type);
    int num_lanes = NumLanes(type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    for (int i = 0; i < num_lanes; ++i) {
      rep_node[i] = rep[i];
      switch (node->opcode()) {
        case IrOpcode::kI8x16ShrU:
          rep_node[i] = Mask(rep_node[i], kMask8);
          rep_node[i] =
              graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
          break;
        case IrOpcode::kI16x8ShrU:
          rep_node[i] = Mask(rep_node[i], kMask16);
          V8_FALLTHROUGH;
        case IrOpcode::kI32x4ShrU:
          rep_node[i] =
              graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
          break;
        case IrOpcode::kI64x2ShrU:
          rep_node[i] =
              graph()->NewNode(machine()->Word64Shr(), rep_node[i], shift_node);
          break;
        case IrOpcode::kI64x2Shl:
          rep_node[i] =
              graph()->NewNode(machine()->Word64Shl(), rep_node[i], shift_node);
          break;
        case IrOpcode::kI32x4Shl:
          rep_node[i] =
              graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
          break;
        case IrOpcode::kI16x8Shl:
          rep_node[i] =
              graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
          rep_node[i] = FixUpperBits(rep_node[i], kShift16);
          break;
        case IrOpcode::kI8x16Shl:
          rep_node[i] =
              graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
          rep_node[i] = FixUpperBits(rep_node[i], kShift8);
          break;
        case IrOpcode::kI64x2ShrS:
          rep_node[i] =
              graph()->NewNode(machine()->Word64Sar(), rep_node[i], shift_node);
          break;
        case IrOpcode::kI32x4ShrS:
        case IrOpcode::kI16x8ShrS:
        case IrOpcode::kI8x16ShrS:
          rep_node[i] =
              graph()->NewNode(machine()->Word32Sar(), rep_node[i], shift_node);
          break;
        default:
          UNREACHABLE();
      }
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  Node* SimdScalarLowering::ConstructPhiForComparison(Diamond d,
                                                      SimdType rep_type,
                                                      int true_value,
                                                      int false_value) {
    // Close the given Diamond d using a Phi node, taking care of constructing the
    // right kind of constants (Int32 or Int64) based on rep_type.
    if (rep_type == SimdType::kFloat64x2) {
      MachineRepresentation rep = MachineRepresentation::kWord64;
      return d.Phi(rep, mcgraph_->Int64Constant(true_value),
                   mcgraph_->Int64Constant(false_value));
    } else {
      MachineRepresentation rep =
          (rep_type == SimdType::kFloat32x4)
              ? MachineRepresentation::kWord32
              : MachineTypeFrom(rep_type).representation();
      return d.Phi(rep, mcgraph_->Int32Constant(true_value),
                   mcgraph_->Int32Constant(false_value));
    }
  }
  
  void SimdScalarLowering::LowerNotEqual(Node* node, SimdType input_rep_type,
                                         const Operator* op) {
    DCHECK_EQ(2, node->InputCount());
    Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
    Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
    int num_lanes = NumLanes(input_rep_type);
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    for (int i = 0; i < num_lanes; ++i) {
      Diamond d(graph(), common(),
                graph()->NewNode(op, rep_left[i], rep_right[i]));
      rep_node[i] = ConstructPhiForComparison(d, input_rep_type, 0, -1);
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  void SimdScalarLowering::LowerBitMaskOp(Node* node, SimdType rep_type,
                                          int msb_index) {
    Node** reps = GetReplacementsWithType(node->InputAt(0), rep_type);
    int num_lanes = NumLanes(rep_type);
    Node** rep_node = zone()->NewArray<Node*>(1);
    Node* result = mcgraph_->Int32Constant(0);
    uint32_t mask = 1 << msb_index;
  
    for (int i = 0; i < num_lanes; ++i) {
      // Lane i should end up at bit i in the final result.
      // +-----------------------------------------------------------------+
      // |       | msb_index |   (i < msb_index)    |    (i > msb_index)   |
      // +-------+-----------+----------------------+----------------------+
      // | i8x16 |     7     | msb >> (msb_index-i) | msb << (i-msb_index) |
      // | i16x8 |    15     | msb >> (msb_index-i) |         n/a          |
      // | i32x4 |    31     | msb >> (msb_index-i) |         n/a          |
      // +-------+-----------+----------------------+----------------------+
      Node* msb = Mask(reps[i], mask);
  
      if (i < msb_index) {
        int shift = msb_index - i;
        Node* shifted = graph()->NewNode(machine()->Word32Shr(), msb,
                                         mcgraph_->Int32Constant(shift));
        result = graph()->NewNode(machine()->Word32Or(), shifted, result);
      } else if (i > msb_index) {
        int shift = i - msb_index;
        Node* shifted = graph()->NewNode(machine()->Word32Shl(), msb,
                                         mcgraph_->Int32Constant(shift));
        result = graph()->NewNode(machine()->Word32Or(), shifted, result);
      } else {
        result = graph()->NewNode(machine()->Word32Or(), msb, result);
      }
    }
  
    rep_node[0] = result;
    ReplaceNode(node, rep_node, 1);
  }
  
  void SimdScalarLowering::LowerAllTrueOp(Node* node, SimdType rep_type) {
    // AllTrue ops require the input to be of a particular SimdType, but the op
    // itself is always replaced by a Int32x4 with 1 node.
    int num_lanes = NumLanes(rep_type);
    DCHECK_EQ(1, node->InputCount());
    Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
  
    Node** rep_node = zone()->NewArray<Node*>(num_lanes);
    Node* zero = mcgraph_->Int32Constant(0);
    Node* tmp_result = mcgraph_->Int32Constant(1);
    for (int i = 0; i < num_lanes; ++i) {
      Diamond d(graph(), common(),
                graph()->NewNode(machine()->Word32Equal(), rep[i], zero));
      tmp_result = d.Phi(MachineRepresentation::kWord32, zero, tmp_result);
    }
    rep_node[0] = tmp_result;
    for (int i = 1; i < num_lanes; ++i) {
      rep_node[i] = nullptr;
    }
    ReplaceNode(node, rep_node, num_lanes);
  }
  
  void SimdScalarLowering::LowerNode(Node* node) {
    SimdType rep_type = ReplacementType(node);
    int num_lanes = NumLanes(rep_type);
    switch (node->opcode()) {
      case IrOpcode::kS128Const: {
        // We could use GetReplacementsWithType for all this, but it adds a lot of
        // nodes, so sign extend the immediates ourselves here.
        DCHECK_EQ(0, node->InputCount());
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        S128ImmediateParameter params = S128ImmediateParameterOf(node->op());
  
        // For all the small ints below, we have a choice of static_cast or bit
        // twiddling, clang seems to be able to optimize either
        // (https://godbolt.org/z/9c65o8) so use static_cast for clarity.
        switch (rep_type) {
          case SimdType::kInt8x16: {
            for (int i = 0; i < num_lanes; ++i) {
              Address data_address = reinterpret_cast<Address>(params.data() + i);
              rep_node[i] = mcgraph_->Int32Constant(
                  static_cast<int32_t>(static_cast<int8_t>(
                      base::ReadLittleEndianValue<int8_t>(data_address))));
            }
            break;
          }
          case SimdType::kInt16x8: {
            int16_t val[kNumLanes16];
            memcpy(val, params.data(), kSimd128Size);
            for (int i = 0; i < num_lanes; ++i) {
              rep_node[i] = mcgraph_->Int32Constant(static_cast<int32_t>(
                  base::ReadLittleEndianValue<int16_t>(&val[i])));
            }
            break;
          }
          case SimdType::kInt32x4: {
            uint32_t val[kNumLanes32];
            memcpy(val, params.data(), kSimd128Size);
            for (int i = 0; i < num_lanes; ++i) {
              rep_node[i] = mcgraph_->Int32Constant(
                  base::ReadLittleEndianValue<uint32_t>(&val[i]));
            }
            break;
          }
          case SimdType::kInt64x2: {
            uint64_t val[kNumLanes64];
            memcpy(val, params.data(), kSimd128Size);
            for (int i = 0; i < num_lanes; ++i) {
              rep_node[i] = mcgraph_->Int64Constant(
                  base::ReadLittleEndianValue<uint64_t>(&val[i]));
            }
            break;
          }
          case SimdType::kFloat32x4: {
            float val[kNumLanes32];
            memcpy(val, params.data(), kSimd128Size);
            for (int i = 0; i < num_lanes; ++i) {
              rep_node[i] = mcgraph_->Float32Constant(
                  base::ReadLittleEndianValue<float>(&val[i]));
            }
            break;
          }
          default: {
            UNIMPLEMENTED();
          }
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kStart: {
        int parameter_count = GetParameterCountAfterLowering();
        // Only exchange the node if the parameter count actually changed.
        if (parameter_count != static_cast<int>(signature()->parameter_count())) {
          int delta =
              parameter_count - static_cast<int>(signature()->parameter_count());
          int new_output_count = node->op()->ValueOutputCount() + delta;
          NodeProperties::ChangeOp(node, common()->Start(new_output_count));
        }
        break;
      }
      case IrOpcode::kParameter: {
        DCHECK_EQ(1, node->InputCount());
        int param_count = static_cast<int>(signature()->parameter_count());
        // Only exchange the node if the parameter count actually changed. We do
        // not even have to do the default lowering because the start node,
        // the only input of a parameter node, only changes if the parameter count
        // changes.
        if (GetParameterCountAfterLowering() != param_count) {
          int old_index = ParameterIndexOf(node->op());
          // Parameter index 0 is the instance parameter, we will use old_index to
          // index into the function signature, so we need to decrease it by 1.
          --old_index;
          int new_index =
              GetParameterIndexAfterLoweringSimd128(signature(), old_index);
          // Similarly, the index into function signature needs to account for the
          // instance parameter, so increase it by 1.
          ++new_index;
          NodeProperties::ChangeOp(node, common()->Parameter(new_index));
  
          if (old_index < 0) {
            break;
          }
  
          DCHECK(old_index < param_count);
  
          if (signature()->GetParam(old_index) ==
              MachineRepresentation::kSimd128) {
            Node* new_node[kNumLanes32];
            new_node[0] = node;
            for (int i = 1; i < kNumLanes32; ++i) {
              new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
                                             graph()->start());
            }
            ReplaceNode(node, new_node, kNumLanes32);
          }
        }
        break;
      }
      case IrOpcode::kSimd128ReverseBytes: {
        DCHECK_EQ(1, node->InputCount());
        SimdType input_type = ReplacementType(node->InputAt(0));
        bool is_float = input_type == SimdType::kFloat32x4 ||
                        input_type == SimdType::kFloat64x2;
        replacements_[node->id()].type =
            is_float ? SimdType::kFloat32x4 : SimdType::kInt32x4;
        Node** rep = GetReplacementsWithType(
            node->InputAt(0),
            is_float ? SimdType::kFloat32x4 : SimdType::kInt32x4);
        Node* rep_node[kNumLanes32];
        for (int i = 0; i < kNumLanes32; ++i) {
          Node* temp = is_float ? graph()->NewNode(
                                      machine()->BitcastFloat32ToInt32(), rep[i])
                                : rep[i];
          temp = graph()->NewNode(machine()->Word32ReverseBytes(), temp);
          rep_node[kNumLanes32 - 1 - i] =
              is_float
                  ? graph()->NewNode(machine()->BitcastInt32ToFloat32(), temp)
                  : temp;
        }
        ReplaceNode(node, rep_node, kNumLanes32);
        break;
      }
      case IrOpcode::kLoad:
      case IrOpcode::kUnalignedLoad:
      case IrOpcode::kProtectedLoad: {
        LowerLoadOp(node, rep_type);
        break;
      }
      case IrOpcode::kLoadTransform: {
        LowerLoadTransformOp(node, rep_type);
        break;
      }
      case IrOpcode::kStore:
      case IrOpcode::kUnalignedStore:
      case IrOpcode::kProtectedStore: {
        LowerStoreOp(node);
        break;
      }
      case IrOpcode::kReturn: {
        // V128 return types are lowered to i32x4, so if the inputs to kReturn are
        // not Word32, we need to convert them.
        int return_arity = static_cast<int>(signature()->return_count());
        for (int i = 0; i < return_arity; i++) {
          if (signature()->GetReturn(i) != MachineRepresentation::kSimd128) {
            continue;
          }
  
          // Return nodes have a hidden input at value 0.
          Node* input = node->InputAt(i + 1);
          if (!HasReplacement(0, input)) {
            continue;
          }
  
          switch (ReplacementType(input)) {
            case SimdType::kInt8x16:
            case SimdType::kInt16x8:
            case SimdType::kInt64x2:
            case SimdType::kFloat32x4: {
              Node** reps = GetReplacementsWithType(input, rep_type);
              ReplaceNode(input, reps, NumLanes(rep_type));
              break;
            }
            case SimdType::kInt32x4: {
              // No action needed.
              break;
            }
            default: {
              // i64x2 and f64x2 aren't handled anywhere yet, ignore it here.
              break;
              UNIMPLEMENTED();
            }
          }
        }
  
        DefaultLowering(node);
        int new_return_count = GetReturnCountAfterLoweringSimd128(signature());
        if (static_cast<int>(signature()->return_count()) != new_return_count) {
          NodeProperties::ChangeOp(node, common()->Return(new_return_count));
        }
        break;
      }
      case IrOpcode::kCall: {
        // TODO(turbofan): Make wasm code const-correct wrt. CallDescriptor.
        auto call_descriptor =
            const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
        bool returns_require_lowering =
            GetReturnCountAfterLoweringSimd128(call_descriptor) !=
            static_cast<int>(call_descriptor->ReturnCount());
  
        // All call arguments are lowered to i32x4 in the call descriptor, so the
        // arguments need to be converted to i32x4 as well.
        for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
          Node* input = node->InputAt(i);
          if (HasReplacement(0, input)) {
            Node** reps = GetReplacementsWithType(input, SimdType::kInt32x4);
            ReplaceNode(input, reps, NumLanes(SimdType::kInt32x4));
          }
        }
  
        if (DefaultLowering(node) || returns_require_lowering) {
          // We have to adjust the call descriptor.
          const Operator* op = common()->Call(
              GetI32WasmCallDescriptorForSimd(zone(), call_descriptor));
          NodeProperties::ChangeOp(node, op);
        }
  
        if (!returns_require_lowering) {
          break;
        }
  
        size_t return_arity = call_descriptor->ReturnCount();
  
        if (return_arity == 1) {
          // We access the additional return values through projections.
          // Special case for return_arity 1, with multi-returns, we would have
          // already built projections for each return value, and will be handled
          // by the following code.
          Node* rep_node[kNumLanes32];
          for (int i = 0; i < kNumLanes32; ++i) {
            rep_node[i] =
                graph()->NewNode(common()->Projection(i), node, graph()->start());
          }
          ReplaceNode(node, rep_node, kNumLanes32);
          break;
        }
  
        ZoneVector<Node*> projections(return_arity, zone());
        NodeProperties::CollectValueProjections(node, projections.data(),
                                                return_arity);
  
        for (size_t old_index = 0, new_index = 0; old_index < return_arity;
             ++old_index, ++new_index) {
          Node* use_node = projections[old_index];
          DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
          DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
                                                static_cast<int>(old_index)),
                    static_cast<int>(new_index));
          if (new_index != old_index) {
            NodeProperties::ChangeOp(use_node, common()->Projection(new_index));
          }
          if (call_descriptor->GetReturnType(old_index).representation() ==
              MachineRepresentation::kSimd128) {
            Node* rep_node[kNumLanes32];
            for (int i = 0; i < kNumLanes32; ++i) {
              rep_node[i] = graph()->NewNode(common()->Projection(new_index + i),
                                             node, graph()->start());
            }
            ReplaceNode(use_node, rep_node, kNumLanes32);
            new_index += kNumLanes32 - 1;
          }
        }
        break;
      }
      case IrOpcode::kPhi: {
        MachineRepresentation rep = PhiRepresentationOf(node->op());
        if (rep == MachineRepresentation::kSimd128) {
          // The replacement nodes have already been created, we only have to
          // replace placeholder nodes.
          Node** rep_node = GetReplacements(node);
          for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
            Node** rep_input =
                GetReplacementsWithType(node->InputAt(i), rep_type);
            for (int j = 0; j < num_lanes; j++) {
              rep_node[j]->ReplaceInput(i, rep_input[j]);
            }
          }
        } else {
          DefaultLowering(node);
        }
        break;
      }
      case IrOpcode::kI64x2Add: {
        LowerBinaryOp(node, rep_type, machine()->Int64Add());
        break;
      }
      case IrOpcode::kI64x2Sub: {
        LowerBinaryOp(node, rep_type, machine()->Int64Sub());
        break;
      }
      case IrOpcode::kI64x2Mul: {
        LowerBinaryOp(node, rep_type, machine()->Int64Mul());
        break;
      }
  #define I32X4_BINOP_CASE(opcode, instruction)                \
    case IrOpcode::opcode: {                                   \
      LowerBinaryOp(node, rep_type, machine()->instruction()); \
      break;                                                   \
    }
        I32X4_BINOP_CASE(kI32x4Add, Int32Add)
        I32X4_BINOP_CASE(kI32x4Sub, Int32Sub)
        I32X4_BINOP_CASE(kI32x4Mul, Int32Mul)
        I32X4_BINOP_CASE(kS128And, Word32And)
        I32X4_BINOP_CASE(kS128Or, Word32Or)
        I32X4_BINOP_CASE(kS128Xor, Word32Xor)
  #undef I32X4_BINOP_CASE
      case IrOpcode::kI32x4AddHoriz: {
        LowerBinaryOp(node, rep_type, machine()->Int32Add(), false);
        break;
      }
      case IrOpcode::kI16x8AddHoriz: {
        LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add(), false);
        break;
      }
      case IrOpcode::kI16x8Add:
      case IrOpcode::kI8x16Add: {
        LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add());
        break;
      }
      case IrOpcode::kI16x8Sub:
      case IrOpcode::kI8x16Sub: {
        LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Sub());
        break;
      }
      case IrOpcode::kI16x8Mul:
      case IrOpcode::kI8x16Mul: {
        LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Mul());
        break;
      }
      case IrOpcode::kI16x8AddSaturateS:
      case IrOpcode::kI8x16AddSaturateS: {
        LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), true);
        break;
      }
      case IrOpcode::kI16x8SubSaturateS:
      case IrOpcode::kI8x16SubSaturateS: {
        LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), true);
        break;
      }
      case IrOpcode::kI16x8AddSaturateU:
      case IrOpcode::kI8x16AddSaturateU: {
        LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), false);
        break;
      }
      case IrOpcode::kI16x8SubSaturateU:
      case IrOpcode::kI8x16SubSaturateU: {
        LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), false);
        break;
      }
      case IrOpcode::kI32x4MaxS:
      case IrOpcode::kI16x8MaxS:
      case IrOpcode::kI8x16MaxS: {
        LowerIntMinMax(node, machine()->Int32LessThan(), true, rep_type);
        break;
      }
      case IrOpcode::kI32x4MinS:
      case IrOpcode::kI16x8MinS:
      case IrOpcode::kI8x16MinS: {
        LowerIntMinMax(node, machine()->Int32LessThan(), false, rep_type);
        break;
      }
      case IrOpcode::kI32x4MaxU:
      case IrOpcode::kI16x8MaxU:
      case IrOpcode::kI8x16MaxU: {
        LowerIntMinMax(node, machine()->Uint32LessThan(), true, rep_type);
        break;
      }
      case IrOpcode::kI32x4MinU:
      case IrOpcode::kI16x8MinU:
      case IrOpcode::kI8x16MinU: {
        LowerIntMinMax(node, machine()->Uint32LessThan(), false, rep_type);
        break;
      }
      case IrOpcode::kI64x2Neg: {
        DCHECK_EQ(1, node->InputCount());
        Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
        int num_lanes = NumLanes(rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        Node* zero = graph()->NewNode(common()->Int64Constant(0));
        for (int i = 0; i < num_lanes; ++i) {
          rep_node[i] = graph()->NewNode(machine()->Int64Sub(), zero, rep[i]);
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kI32x4Neg:
      case IrOpcode::kI16x8Neg:
      case IrOpcode::kI8x16Neg: {
        DCHECK_EQ(1, node->InputCount());
        Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
        int num_lanes = NumLanes(rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        Node* zero = graph()->NewNode(common()->Int32Constant(0));
        for (int i = 0; i < num_lanes; ++i) {
          rep_node[i] = graph()->NewNode(machine()->Int32Sub(), zero, rep[i]);
          if (node->opcode() == IrOpcode::kI16x8Neg) {
            rep_node[i] = FixUpperBits(rep_node[i], kShift16);
          } else if (node->opcode() == IrOpcode::kI8x16Neg) {
            rep_node[i] = FixUpperBits(rep_node[i], kShift8);
          }
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kI32x4Abs:
      case IrOpcode::kI16x8Abs:
      case IrOpcode::kI8x16Abs: {
        // From https://stackoverflow.com/a/14194764
        // abs(x) = (x XOR y) - y
        Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        for (int i = 0; i < num_lanes; ++i) {
          // It's fine to shift by 31 even for i8x16 since each node is
          // effectively expanded to 32 bits.
          Node* y = graph()->NewNode(machine()->Word32Sar(), rep[i],
                                     mcgraph_->Int32Constant(31));
          rep_node[i] = graph()->NewNode(
              machine()->Int32Sub(),
              graph()->NewNode(machine()->Word32Xor(), rep[i], y), y);
          if (node->opcode() == IrOpcode::kI16x8Neg) {
            rep_node[i] = FixUpperBits(rep_node[i], kShift16);
          } else if (node->opcode() == IrOpcode::kI8x16Neg) {
            rep_node[i] = FixUpperBits(rep_node[i], kShift8);
          }
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kS128Zero: {
        DCHECK_EQ(0, node->InputCount());
        Node* rep_node[kNumLanes32];
        for (int i = 0; i < kNumLanes32; ++i) {
          rep_node[i] = mcgraph_->Int32Constant(0);
        }
        ReplaceNode(node, rep_node, kNumLanes32);
        break;
      }
      case IrOpcode::kS128Not: {
        DCHECK_EQ(1, node->InputCount());
        Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node* rep_node[kNumLanes32];
        Node* mask = graph()->NewNode(common()->Int32Constant(0xFFFFFFFF));
        for (int i = 0; i < kNumLanes32; ++i) {
          rep_node[i] = graph()->NewNode(machine()->Word32Xor(), rep[i], mask);
        }
        ReplaceNode(node, rep_node, kNumLanes32);
        break;
      }
      case IrOpcode::kS128AndNot: {
        DCHECK_EQ(2, node->InputCount());
        Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
        int num_lanes = NumLanes(rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        Node* mask = graph()->NewNode(common()->Int32Constant(0xFFFFFFFF));
        for (int i = 0; i < num_lanes; ++i) {
          Node* not_rep_right =
              graph()->NewNode(machine()->Word32Xor(), rep_right[i], mask);
          rep_node[i] = graph()->NewNode(machine()->Word32And(), rep_left[i],
                                         not_rep_right);
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kI32x4SConvertF32x4: {
        LowerConvertFromFloat(node, true);
        break;
      }
      case IrOpcode::kI32x4UConvertF32x4: {
        LowerConvertFromFloat(node, false);
        break;
      }
      case IrOpcode::kI32x4SConvertI16x8Low: {
        LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
                            0);
        break;
      }
      case IrOpcode::kI32x4SConvertI16x8High: {
        LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
                            4);
        break;
      }
      case IrOpcode::kI32x4UConvertI16x8Low: {
        LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
                            0);
        break;
      }
      case IrOpcode::kI32x4UConvertI16x8High: {
        LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
                            4);
        break;
      }
      case IrOpcode::kI16x8SConvertI8x16Low: {
        LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
                            0);
        break;
      }
      case IrOpcode::kI16x8SConvertI8x16High: {
        LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
                            8);
        break;
      }
      case IrOpcode::kI16x8UConvertI8x16Low: {
        LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
                            0);
        break;
      }
      case IrOpcode::kI16x8UConvertI8x16High: {
        LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
                            8);
        break;
      }
      case IrOpcode::kI16x8SConvertI32x4: {
        LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, true);
        break;
      }
      case IrOpcode::kI16x8UConvertI32x4: {
        LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, false);
        break;
      }
      case IrOpcode::kI8x16SConvertI16x8: {
        LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, true);
        break;
      }
      case IrOpcode::kI8x16UConvertI16x8: {
        LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, false);
        break;
      }
      case IrOpcode::kI64x2Shl:
      case IrOpcode::kI32x4Shl:
      case IrOpcode::kI16x8Shl:
      case IrOpcode::kI8x16Shl:
      case IrOpcode::kI64x2ShrS:
      case IrOpcode::kI32x4ShrS:
      case IrOpcode::kI16x8ShrS:
      case IrOpcode::kI8x16ShrS:
      case IrOpcode::kI64x2ShrU:
      case IrOpcode::kI32x4ShrU:
      case IrOpcode::kI16x8ShrU:
      case IrOpcode::kI8x16ShrU: {
        LowerShiftOp(node, rep_type);
        break;
      }
      case IrOpcode::kF32x4AddHoriz: {
        LowerBinaryOp(node, rep_type, machine()->Float32Add(), false);
        break;
      }
  #define F32X4_BINOP_CASE(name)                                 \
    case IrOpcode::kF32x4##name: {                               \
      LowerBinaryOp(node, rep_type, machine()->Float32##name()); \
      break;                                                     \
    }
        F32X4_BINOP_CASE(Add)
        F32X4_BINOP_CASE(Sub)
        F32X4_BINOP_CASE(Mul)
        F32X4_BINOP_CASE(Div)
        F32X4_BINOP_CASE(Min)
        F32X4_BINOP_CASE(Max)
  #undef F32X4_BINOP_CASE
  #define F32X4_UNOP_CASE(name)                                 \
    case IrOpcode::kF32x4##name: {                              \
      LowerUnaryOp(node, rep_type, machine()->Float32##name()); \
      break;                                                    \
    }
        F32X4_UNOP_CASE(Abs)
        F32X4_UNOP_CASE(Neg)
        F32X4_UNOP_CASE(Sqrt)
  #undef F32X4_UNOP_CASE
      case IrOpcode::kF32x4RecipApprox:
      case IrOpcode::kF32x4RecipSqrtApprox: {
        DCHECK_EQ(1, node->InputCount());
        Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        Node* float_one = graph()->NewNode(common()->Float32Constant(1.0));
        for (int i = 0; i < num_lanes; ++i) {
          Node* tmp = rep[i];
          if (node->opcode() == IrOpcode::kF32x4RecipSqrtApprox) {
            tmp = graph()->NewNode(machine()->Float32Sqrt(), rep[i]);
          }
          rep_node[i] = graph()->NewNode(machine()->Float32Div(), float_one, tmp);
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kF32x4SConvertI32x4: {
        LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundInt32ToFloat32());
        break;
      }
      case IrOpcode::kF32x4UConvertI32x4: {
        LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundUint32ToFloat32());
        break;
      }
      case IrOpcode::kF64x2Abs: {
        LowerUnaryOp(node, rep_type, machine()->Float64Abs());
        break;
      }
      case IrOpcode::kF64x2Neg: {
        LowerUnaryOp(node, rep_type, machine()->Float64Neg());
        break;
      }
      case IrOpcode::kF64x2Sqrt: {
        LowerUnaryOp(node, rep_type, machine()->Float64Sqrt());
        break;
      }
      case IrOpcode::kF64x2Add: {
        LowerBinaryOp(node, rep_type, machine()->Float64Add());
        break;
      }
      case IrOpcode::kF64x2Sub: {
        LowerBinaryOp(node, rep_type, machine()->Float64Sub());
        break;
      }
      case IrOpcode::kF64x2Mul: {
        LowerBinaryOp(node, rep_type, machine()->Float64Mul());
        break;
      }
      case IrOpcode::kF64x2Div: {
        LowerBinaryOp(node, rep_type, machine()->Float64Div());
        break;
      }
      case IrOpcode::kF64x2Min: {
        LowerBinaryOp(node, rep_type, machine()->Float64Min());
        break;
      }
      case IrOpcode::kF64x2Max: {
        LowerBinaryOp(node, rep_type, machine()->Float64Max());
        break;
      }
      case IrOpcode::kF64x2Splat:
      case IrOpcode::kF32x4Splat:
      case IrOpcode::kI64x2Splat:
      case IrOpcode::kI32x4Splat:
      case IrOpcode::kI16x8Splat:
      case IrOpcode::kI8x16Splat: {
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        for (int i = 0; i < num_lanes; ++i) {
          if (HasReplacement(0, node->InputAt(0))) {
            rep_node[i] = GetReplacements(node->InputAt(0))[0];
          } else {
            rep_node[i] = node->InputAt(0);
          }
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kF64x2ExtractLane:
      case IrOpcode::kF32x4ExtractLane:
      case IrOpcode::kI64x2ExtractLane:
      case IrOpcode::kI32x4ExtractLane:
      case IrOpcode::kI16x8ExtractLaneU:
      case IrOpcode::kI16x8ExtractLaneS:
      case IrOpcode::kI8x16ExtractLaneU:
      case IrOpcode::kI8x16ExtractLaneS: {
        int32_t lane = OpParameter<int32_t>(node->op());
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        rep_node[0] = GetReplacementsWithType(node->InputAt(0), rep_type)[lane];
        for (int i = 1; i < num_lanes; ++i) {
          rep_node[i] = nullptr;
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kF64x2ReplaceLane:
      case IrOpcode::kF32x4ReplaceLane:
      case IrOpcode::kI64x2ReplaceLane:
      case IrOpcode::kI32x4ReplaceLane:
      case IrOpcode::kI16x8ReplaceLane:
      case IrOpcode::kI8x16ReplaceLane: {
        DCHECK_EQ(2, node->InputCount());
        Node* repNode = node->InputAt(1);
        int32_t lane = OpParameter<int32_t>(node->op());
        Node** old_rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        for (int i = 0; i < num_lanes; ++i) {
          rep_node[i] = old_rep_node[i];
        }
        if (HasReplacement(0, repNode)) {
          rep_node[lane] = GetReplacements(repNode)[0];
        } else {
          rep_node[lane] = repNode;
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
  #define COMPARISON_CASE(type, simd_op, lowering_op, invert)                    \
    case IrOpcode::simd_op: {                                                    \
      LowerCompareOp(node, SimdType::k##type, machine()->lowering_op(), invert); \
      break;                                                                     \
    }
        COMPARISON_CASE(Float64x2, kF64x2Eq, Float64Equal, false)
        COMPARISON_CASE(Float64x2, kF64x2Lt, Float64LessThan, false)
        COMPARISON_CASE(Float64x2, kF64x2Le, Float64LessThanOrEqual, false)
        COMPARISON_CASE(Float32x4, kF32x4Eq, Float32Equal, false)
        COMPARISON_CASE(Float32x4, kF32x4Lt, Float32LessThan, false)
        COMPARISON_CASE(Float32x4, kF32x4Le, Float32LessThanOrEqual, false)
        COMPARISON_CASE(Float32x4, kF32x4Gt, Float32LessThan, true)
        COMPARISON_CASE(Float32x4, kF32x4Ge, Float32LessThanOrEqual, true)
        COMPARISON_CASE(Int32x4, kI32x4Eq, Word32Equal, false)
        COMPARISON_CASE(Int32x4, kI32x4LtS, Int32LessThan, false)
        COMPARISON_CASE(Int32x4, kI32x4LeS, Int32LessThanOrEqual, false)
        COMPARISON_CASE(Int32x4, kI32x4GtS, Int32LessThan, true)
        COMPARISON_CASE(Int32x4, kI32x4GeS, Int32LessThanOrEqual, true)
        COMPARISON_CASE(Int32x4, kI32x4LtU, Uint32LessThan, false)
        COMPARISON_CASE(Int32x4, kI32x4LeU, Uint32LessThanOrEqual, false)
        COMPARISON_CASE(Int32x4, kI32x4GtU, Uint32LessThan, true)
        COMPARISON_CASE(Int32x4, kI32x4GeU, Uint32LessThanOrEqual, true)
        COMPARISON_CASE(Int16x8, kI16x8Eq, Word32Equal, false)
        COMPARISON_CASE(Int16x8, kI16x8LtS, Int32LessThan, false)
        COMPARISON_CASE(Int16x8, kI16x8LeS, Int32LessThanOrEqual, false)
        COMPARISON_CASE(Int16x8, kI16x8GtS, Int32LessThan, true)
        COMPARISON_CASE(Int16x8, kI16x8GeS, Int32LessThanOrEqual, true)
        COMPARISON_CASE(Int16x8, kI16x8LtU, Uint32LessThan, false)
        COMPARISON_CASE(Int16x8, kI16x8LeU, Uint32LessThanOrEqual, false)
        COMPARISON_CASE(Int16x8, kI16x8GtU, Uint32LessThan, true)
        COMPARISON_CASE(Int16x8, kI16x8GeU, Uint32LessThanOrEqual, true)
        COMPARISON_CASE(Int8x16, kI8x16Eq, Word32Equal, false)
        COMPARISON_CASE(Int8x16, kI8x16LtS, Int32LessThan, false)
        COMPARISON_CASE(Int8x16, kI8x16LeS, Int32LessThanOrEqual, false)
        COMPARISON_CASE(Int8x16, kI8x16GtS, Int32LessThan, true)
        COMPARISON_CASE(Int8x16, kI8x16GeS, Int32LessThanOrEqual, true)
        COMPARISON_CASE(Int8x16, kI8x16LtU, Uint32LessThan, false)
        COMPARISON_CASE(Int8x16, kI8x16LeU, Uint32LessThanOrEqual, false)
        COMPARISON_CASE(Int8x16, kI8x16GtU, Uint32LessThan, true)
        COMPARISON_CASE(Int8x16, kI8x16GeU, Uint32LessThanOrEqual, true)
  #undef COMPARISON_CASE
      case IrOpcode::kF64x2Ne: {
        LowerNotEqual(node, SimdType::kFloat64x2, machine()->Float64Equal());
        break;
      }
      case IrOpcode::kF32x4Ne: {
        LowerNotEqual(node, SimdType::kFloat32x4, machine()->Float32Equal());
        break;
      }
      case IrOpcode::kI32x4Ne: {
        LowerNotEqual(node, SimdType::kInt32x4, machine()->Word32Equal());
        break;
      }
      case IrOpcode::kI16x8Ne: {
        LowerNotEqual(node, SimdType::kInt16x8, machine()->Word32Equal());
        break;
      }
      case IrOpcode::kI8x16Ne: {
        LowerNotEqual(node, SimdType::kInt8x16, machine()->Word32Equal());
        break;
      }
      case IrOpcode::kS128Select: {
        DCHECK_EQ(3, node->InputCount());
        DCHECK(ReplacementType(node->InputAt(0)) == SimdType::kInt32x4 ||
               ReplacementType(node->InputAt(0)) == SimdType::kInt16x8 ||
               ReplacementType(node->InputAt(0)) == SimdType::kInt8x16);
        Node** boolean_input =
            GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** rep_left = GetReplacementsWithType(node->InputAt(1), rep_type);
        Node** rep_right = GetReplacementsWithType(node->InputAt(2), rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        for (int i = 0; i < num_lanes; ++i) {
          Node* tmp1 =
              graph()->NewNode(machine()->Word32Xor(), rep_left[i], rep_right[i]);
          Node* tmp2 =
              graph()->NewNode(machine()->Word32And(), boolean_input[i], tmp1);
          rep_node[i] =
              graph()->NewNode(machine()->Word32Xor(), rep_right[i], tmp2);
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kS8x16Swizzle: {
        DCHECK_EQ(2, node->InputCount());
        Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** indices = GetReplacementsWithType(node->InputAt(1), rep_type);
        Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
        Node* stack_slot = graph()->NewNode(
            machine()->StackSlot(MachineRepresentation::kSimd128));
  
        // Push all num_lanes values into stack slot.
        const Operator* store_op = machine()->Store(
            StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
        Node* effect_input = graph()->start();
        for (int i = num_lanes - 1; i >= 0; i--) {
          // We want all the stores to happen first before any of the loads
          // below, so connect them via effect edge from i-1 to i.
          Node* store =
              graph()->NewNode(store_op, stack_slot, mcgraph_->Int32Constant(i),
                               rep_left[i], effect_input, graph()->start());
          effect_input = store;
        }
  
        for (int i = num_lanes - 1; i >= 0; i--) {
          // Only select lane when index is < num_lanes, otherwise write 0 to
          // lane. Use Uint32 to take care of negative indices.
          Diamond d(graph(), common(),
                    graph()->NewNode(machine()->Uint32LessThan(), indices[i],
                                     mcgraph_->Int32Constant(num_lanes)));
  
          Node* load =
              graph()->NewNode(machine()->Load(LoadRepresentation::Uint8()),
                               stack_slot, indices[i], effect_input, d.if_true);
  
          rep_nodes[i] = d.Phi(MachineRepresentation::kWord8, load,
                               mcgraph_->Int32Constant(0));
        }
  
        ReplaceNode(node, rep_nodes, num_lanes);
        break;
      }
      case IrOpcode::kS8x16Shuffle: {
        DCHECK_EQ(2, node->InputCount());
        S128ImmediateParameter shuffle = S128ImmediateParameterOf(node->op());
        Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
        Node** rep_node = zone()->NewArray<Node*>(16);
        for (int i = 0; i < 16; i++) {
          int lane = shuffle[i];
          rep_node[i] = lane < 16 ? rep_left[lane] : rep_right[lane - 16];
        }
        ReplaceNode(node, rep_node, 16);
        break;
      }
      case IrOpcode::kV32x4AnyTrue:
      case IrOpcode::kV16x8AnyTrue:
      case IrOpcode::kV8x16AnyTrue: {
        DCHECK_EQ(1, node->InputCount());
        // AnyTrue always returns a I32x4, and can work with inputs of any shape,
        // but we still need GetReplacementsWithType if input is float.
        DCHECK_EQ(ReplacementType(node), SimdType::kInt32x4);
        Node** reps = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        Node* true_node = mcgraph_->Int32Constant(1);
        Node* zero = mcgraph_->Int32Constant(0);
        Node* tmp_result = zero;
        for (int i = 0; i < num_lanes; ++i) {
          Diamond d(graph(), common(),
                    graph()->NewNode(machine()->Word32Equal(), reps[i], zero));
          tmp_result =
              d.Phi(MachineRepresentation::kWord32, tmp_result, true_node);
        }
        rep_node[0] = tmp_result;
        for (int i = 1; i < num_lanes; ++i) {
          rep_node[i] = nullptr;
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      case IrOpcode::kV32x4AllTrue: {
        LowerAllTrueOp(node, SimdType::kInt32x4);
        break;
      }
      case IrOpcode::kV16x8AllTrue: {
        LowerAllTrueOp(node, SimdType::kInt16x8);
        break;
      }
      case IrOpcode::kV8x16AllTrue: {
        LowerAllTrueOp(node, SimdType::kInt8x16);
        break;
      }
      case IrOpcode::kI8x16BitMask: {
        LowerBitMaskOp(node, rep_type, 7);
        break;
      }
      case IrOpcode::kI16x8BitMask: {
        LowerBitMaskOp(node, rep_type, 15);
        break;
      }
      case IrOpcode::kI32x4BitMask: {
        LowerBitMaskOp(node, rep_type, 31);
        break;
      }
      case IrOpcode::kI8x16RoundingAverageU:
      case IrOpcode::kI16x8RoundingAverageU: {
        DCHECK_EQ(2, node->InputCount());
        Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
        Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
        int num_lanes = NumLanes(rep_type);
        Node** rep_node = zone()->NewArray<Node*>(num_lanes);
        // Nodes are stored signed, so mask away the top bits.
        // rounding_average(left, right) = (left + right + 1) >> 1
        const int bit_mask = num_lanes == 16 ? kMask8 : kMask16;
        for (int i = 0; i < num_lanes; ++i) {
          Node* mask_left = graph()->NewNode(machine()->Word32And(), rep_left[i],
                                             mcgraph_->Int32Constant(bit_mask));
          Node* mask_right =
              graph()->NewNode(machine()->Word32And(), rep_right[i],
                               mcgraph_->Int32Constant(bit_mask));
          Node* left_plus_right_plus_one = graph()->NewNode(
              machine()->Int32Add(),
              graph()->NewNode(machine()->Int32Add(), mask_left, mask_right),
              mcgraph_->Int32Constant(1));
          rep_node[i] =
              graph()->NewNode(machine()->Word32Shr(), left_plus_right_plus_one,
                               mcgraph_->Int32Constant(1));
        }
        ReplaceNode(node, rep_node, num_lanes);
        break;
      }
      default: { DefaultLowering(node); }
    }
  }
  
  bool SimdScalarLowering::DefaultLowering(Node* node) {
    bool something_changed = false;
    for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
      Node* input = node->InputAt(i);
      if (HasReplacement(0, input)) {
        something_changed = true;
        node->ReplaceInput(i, GetReplacements(input)[0]);
      }
      if (ReplacementCount(input) > 1 && HasReplacement(1, input)) {
        something_changed = true;
        for (int j = 1; j < ReplacementCount(input); ++j) {
          node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
        }
      }
    }
    return something_changed;
  }
  
  void SimdScalarLowering::ReplaceNode(Node* old, Node** new_nodes, int count) {
    replacements_[old->id()].node = zone()->NewArray<Node*>(count);
    for (int i = 0; i < count; ++i) {
      replacements_[old->id()].node[i] = new_nodes[i];
    }
    replacements_[old->id()].num_replacements = count;
  }
  
  bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
    return replacements_[node->id()].node != nullptr &&
           replacements_[node->id()].node[index] != nullptr;
  }
  
  SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
    return replacements_[node->id()].type;
  }
  
  Node** SimdScalarLowering::GetReplacements(Node* node) {
    Node** result = replacements_[node->id()].node;
    DCHECK(result);
    return result;
  }
  
  int SimdScalarLowering::ReplacementCount(Node* node) {
    return replacements_[node->id()].num_replacements;
  }
  
  void SimdScalarLowering::Int32ToFloat32(Node** replacements, Node** result) {
    for (int i = 0; i < kNumLanes32; ++i) {
      if (replacements[i] != nullptr) {
        result[i] =
            graph()->NewNode(machine()->BitcastInt32ToFloat32(), replacements[i]);
      } else {
        result[i] = nullptr;
      }
    }
  }
  
  void SimdScalarLowering::Float64ToInt64(Node** replacements, Node** result) {
    for (int i = 0; i < kNumLanes64; ++i) {
      if (replacements[i] != nullptr) {
        result[i] =
            graph()->NewNode(machine()->BitcastFloat64ToInt64(), replacements[i]);
      } else {
        result[i] = nullptr;
      }
    }
  }
  
  void SimdScalarLowering::Float32ToInt32(Node** replacements, Node** result) {
    for (int i = 0; i < kNumLanes32; ++i) {
      if (replacements[i] != nullptr) {
        result[i] =
            graph()->NewNode(machine()->BitcastFloat32ToInt32(), replacements[i]);
      } else {
        result[i] = nullptr;
      }
    }
  }
  
  void SimdScalarLowering::Int64ToInt32(Node** replacements, Node** result) {
    const int num_ints = sizeof(int64_t) / sizeof(int32_t);
    const int bit_size = sizeof(int32_t) * 8;
    const Operator* truncate = machine()->TruncateInt64ToInt32();
  
    for (int i = 0; i < kNumLanes64; i++) {
      if (replacements[i] != nullptr) {
        for (int j = 0; j < num_ints; j++) {
          result[num_ints * i + j] = graph()->NewNode(
              truncate, graph()->NewNode(machine()->Word64Sar(), replacements[i],
                                         mcgraph_->Int32Constant(j * bit_size)));
        }
      } else {
        for (int j = 0; j < num_ints; j++) {
          result[num_ints * i + j] = nullptr;
        }
      }
    }
  }
  
  template <typename T>
  void SimdScalarLowering::Int32ToSmallerInt(Node** replacements, Node** result) {
    const int num_ints = sizeof(int32_t) / sizeof(T);
    const int bit_size = sizeof(T) * 8;
    const Operator* sign_extend;
    switch (sizeof(T)) {
      case 1:
        sign_extend = machine()->SignExtendWord8ToInt32();
        break;
      case 2:
        sign_extend = machine()->SignExtendWord16ToInt32();
        break;
      default:
        UNREACHABLE();
    }
  
    for (int i = 0; i < kNumLanes32; i++) {
      if (replacements[i] != nullptr) {
        for (int j = 0; j < num_ints; j++) {
          result[num_ints * i + j] = graph()->NewNode(
              sign_extend,
              graph()->NewNode(machine()->Word32Sar(), replacements[i],
                               mcgraph_->Int32Constant(j * bit_size)));
        }
      } else {
        for (int j = 0; j < num_ints; j++) {
          result[num_ints * i + j] = nullptr;
        }
      }
    }
  }
  
  template <typename T>
  void SimdScalarLowering::SmallerIntToInt32(Node** replacements, Node** result) {
    const int num_ints = sizeof(int32_t) / sizeof(T);
    const int bit_size = sizeof(T) * 8;
    const int bit_mask = (1 << bit_size) - 1;
  
    for (int i = 0; i < kNumLanes32; ++i) {
      result[i] = mcgraph_->Int32Constant(0);
      for (int j = 0; j < num_ints; j++) {
        if (replacements[num_ints * i + j] != nullptr) {
          Node* clean_bits = graph()->NewNode(machine()->Word32And(),
                                              replacements[num_ints * i + j],
                                              mcgraph_->Int32Constant(bit_mask));
          Node* shift = graph()->NewNode(machine()->Word32Shl(), clean_bits,
                                         mcgraph_->Int32Constant(j * bit_size));
          result[i] = graph()->NewNode(machine()->Word32Or(), result[i], shift);
        }
      }
    }
  }
  
  void SimdScalarLowering::Int32ToInt64(Node** replacements, Node** result) {
    const int num_ints = sizeof(int64_t) / sizeof(int32_t);
  
    for (int i = 0; i < kNumLanes64; i++) {
      Node* i64 = graph()->NewNode(machine()->ChangeUint32ToUint64(),
                                   replacements[num_ints * i + 1]);
      Node* high = graph()->NewNode(machine()->Word64Shl(), i64,
                                    mcgraph_->Int32Constant(32));
      Node* i64_low = graph()->NewNode(machine()->ChangeUint32ToUint64(),
                                       replacements[num_ints * i]);
      result[i] = graph()->NewNode(machine()->Word64Or(), high, i64_low);
    }
  }
  
  Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
    Node** replacements = GetReplacements(node);
    if (ReplacementType(node) == type) {
      return GetReplacements(node);
    }
    int num_lanes = NumLanes(type);
    Node** result = zone()->NewArray<Node*>(num_lanes);
    if (type == SimdType::kInt64x2) {
      if (ReplacementType(node) == SimdType::kInt32x4) {
        Int32ToInt64(replacements, result);
      }
    } else if (type == SimdType::kInt32x4) {
      if (ReplacementType(node) == SimdType::kInt64x2) {
        Int64ToInt32(replacements, result);
      } else if (ReplacementType(node) == SimdType::kFloat32x4) {
        Float32ToInt32(replacements, result);
      } else if (ReplacementType(node) == SimdType::kInt16x8) {
        SmallerIntToInt32<int16_t>(replacements, result);
      } else if (ReplacementType(node) == SimdType::kInt8x16) {
        SmallerIntToInt32<int8_t>(replacements, result);
      } else {
        UNREACHABLE();
      }
    } else if (type == SimdType::kFloat32x4) {
      if (ReplacementType(node) == SimdType::kFloat64x2) {
        Node** float64_to_int64 = zone()->NewArray<Node*>(kNumLanes64);
        Node** int64_to_int32 = zone()->NewArray<Node*>(kNumLanes32);
        Float64ToInt64(replacements, float64_to_int64);
        Int64ToInt32(float64_to_int64, int64_to_int32);
        Int32ToFloat32(int64_to_int32, result);
      } else if (ReplacementType(node) == SimdType::kInt32x4) {
        Int32ToFloat32(replacements, result);
      } else if (ReplacementType(node) == SimdType::kInt16x8) {
        UNIMPLEMENTED();
      } else if (ReplacementType(node) == SimdType::kInt8x16) {
        SmallerIntToInt32<int8_t>(replacements, result);
        Int32ToFloat32(result, result);
      } else {
        UNREACHABLE();
      }
    } else if (type == SimdType::kInt16x8) {
      if (ReplacementType(node) == SimdType::kInt32x4) {
        Int32ToSmallerInt<int16_t>(replacements, result);
      } else if (ReplacementType(node) == SimdType::kFloat32x4) {
        UNIMPLEMENTED();
      } else {
        UNREACHABLE();
      }
    } else if (type == SimdType::kInt8x16) {
      if (ReplacementType(node) == SimdType::kInt32x4) {
        Int32ToSmallerInt<int8_t>(replacements, result);
      } else {
        UNIMPLEMENTED();
      }
    } else {
      UNREACHABLE();
    }
    return result;
  }
  
  void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
    MachineRepresentation rep = PhiRepresentationOf(phi->op());
    if (rep == MachineRepresentation::kSimd128) {
      // We have to create the replacements for a phi node before we actually
      // lower the phi to break potential cycles in the graph. The replacements of
      // input nodes do not exist yet, so we use a placeholder node to pass the
      // graph verifier.
      int value_count = phi->op()->ValueInputCount();
      SimdType type = ReplacementType(phi);
      int num_lanes = NumLanes(type);
      Node*** inputs_rep = zone()->NewArray<Node**>(num_lanes);
      for (int i = 0; i < num_lanes; ++i) {
        inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
        inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
      }
      for (int i = 0; i < value_count; ++i) {
        for (int j = 0; j < num_lanes; ++j) {
          inputs_rep[j][i] = placeholder_;
        }
      }
      Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
      for (int i = 0; i < num_lanes; ++i) {
        rep_nodes[i] = graph()->NewNode(
            common()->Phi(MachineTypeFrom(type).representation(), value_count),
            value_count + 1, inputs_rep[i], false);
      }
      ReplaceNode(phi, rep_nodes, num_lanes);
    }
  }
  }  // namespace compiler
  }  // namespace internal
  }  // namespace v8
  