<s> if ( ! ST -> isThumb2 ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; UP . UnrollAndJam = true ;

UP . UnrollAndJamInnerLoopThreshold = <NUM_LIT> ;


<s> EVT ValVT = TLI -> getValueType ( DL , ValTy ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ValVT . isSimple ( ) || ISD != <STR_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , IsPairwiseForm , CostKind ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ;


<s> if ( Opcode == Instruction :: And ) { if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; }

if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return <NUM_LIT> ;


<s> if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return <NUM_LIT> ; } return getIntImmCost ( Imm , Ty , CostKind ) ;


<s> <ISA_LIT>TTI ( const <ISA_LIT>BaseTargetMachine * TM ) : ImmutablePass ( ID ) , TM ( TM ) , ST ( TM -> getSubtargetImpl ( ) ) { initialize<ISA_LIT>TTIPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> if ( Kind == TTI :: SK_Reverse ) { static const CostTblEntry NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; } if ( Kind == TTI :: SK_Select ) { static const CostTblEntry NEONSelShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONSelShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ;


<s> const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < <STR_LIT> > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ;


<s> unsigned <ISA_LIT>TTIImpl :: getCmpSelInstrCost ( unsigned Opcode , Type * ValTy , Type * CondTy ) { int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry < <STR_LIT> > NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT>


<s> if ( ST -> hasVFP2 ( ) && ! ST -> isThumb1Only ( ) ) { if ( Ty -> isFloatTy ( ) ) { return TargetTransformInfo :: TCC_Basic ; } if ( Ty -> isDoubleTy ( ) ) { return ST -> isFPOnlySP ( ) ? TargetTransformInfo :: TCC_Expensive : TargetTransformInfo :: TCC_Basic ; } }

return TargetTransformInfo :: TCC_Expensive ;


<s> return TargetTransformInfo :: TCC_Basic ; } if ( Ty -> isDoubleTy ( ) ) { return ST -> isFPOnlySP ( ) ? TargetTransformInfo :: TCC_Expensive :


<s> assert ( isa < VectorType > ( VecTy ) && <STR_LIT> ) ; bool EltIs64Bits = DL . getTypeAllocSizeInBits ( VecTy -> getScalarType ( ) ) == <NUM_LIT> ; if ( Factor <= TLI -> getMaxSupportedInterleaveFactor ( ) && ! EltIs64Bits ) { unsigned NumElts = VecTy -> getVectorNumElements ( ) ;

Type * SubVecTy = VectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ;


<s> std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; if ( Src -> isVectorTy ( ) && Alignment != <NUM_LIT> && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) { return LT . first * <NUM_LIT> ; } return LT . first ;


<s> if ( Idx == - <NUM_LIT> ) return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; return LT . first * NEONShuffleTbl [ Idx ] . Cost ; } if ( Kind == TTI :: SK_Alternate ) { static const CostTblEntry < <STR_LIT> > NEONAltShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ;

std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ;


<s> if ( Kind == TTI :: SK_Reverse ) { static const CostTblEntry < <STR_LIT> > NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; int Idx = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; return LT . first * NEONShuffleTbl [ Idx ] . Cost ; } if ( Kind == TTI :: SK_Alternate ) { static const CostTblEntry < <STR_LIT> > NEONAltShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; int Idx = CostTableLookup ( NEONAltShuffleTbl , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; return LT . first * NEONAltShuffleTbl [ Idx ] . Cost ;


<s> int <ISA_LIT>TTIImpl :: getArithmeticInstrCost ( unsigned Opcode , Type * Ty , TTI :: OperandValueKind Op1Info , TTI :: OperandValueKind Op2Info , TTI :: OperandValueProperties Opd1PropInfo , TTI :: OperandValueProperties Opd2PropInfo ) { int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < <STR_LIT> > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; int Idx = - <NUM_LIT> ;

if ( ST -> hasNEON ( ) ) Idx = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ;


<s> int <ISA_LIT>TTIImpl :: getCmpSelInstrCost ( unsigned Opcode , Type * ValTy , Type * CondTy ) { int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry < <STR_LIT> > NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ;

EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ;


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; int Idx = CostTableLookup ( NEONAltShuffleTbl , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ;

return LT . first * NEONAltShuffleTbl [ Idx ] . Cost ;


<s> default : break ; case Instruction :: FPToSI : case Instruction :: FPToUI : case Instruction :: SIToFP : case Instruction :: UIToFP : case Instruction :: FPTrunc : case Instruction :: FPExt : return ! ST -> hasFP<ISA_LIT>v8Base ( ) ; } if ( VT . isInteger ( ) && VT . getSizeInBits ( ) >= <NUM_LIT> ) { switch ( ISD ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } if ( ! VT . isFloatingPoint ( ) ) return false ; if ( TLI -> useSoftFloat ( ) ) { switch ( I . getOpcode ( ) ) { default : return true ; case Instruction :: Alloca : case Instruction :: Load : case Instruction :: Store : case Instruction :: Select : case Instruction :: PHI : return false ; } } if ( I . getType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( I . getType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return false ; } ;

auto IsHardwareLoopIntrinsic = [ ] ( Instruction & I ) {


<s> if ( I . getType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return false ; } ; auto IsHardwareLoopIntrinsic = [ ] ( Instruction & I ) { if ( auto * Call = dyn_cast < IntrinsicInst > ( & I ) ) { switch ( Call -> getIntrinsicID ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } return false ; } ; auto ScanLoop = [ & ] ( Loop * L ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( MaybeCall ( I ) || IsHardwareLoopIntrinsic ( I ) ) return false ; } } return true ; } ; for ( auto Inner : * L ) if ( ! ScanLoop ( Inner ) ) return false ; if ( ! ScanLoop ( L ) ) return false ; LLVMContext & C = L -> getHeader ( ) -> getContext ( ) ; HWLoopInfo . CounterInReg = true ; HWLoopInfo . IsNestingLegal = false ; HWLoopInfo . CountType = Type :: getInt32Ty ( C ) ;

HWLoopInfo . LoopDecrement = ConstantInt :: get ( HWLoopInfo . CountType , <NUM_LIT> ) ;


<s> return ST -> isMClass ( ) && ST -> isThumb2 ( ) && L -> getNumBlocks ( ) == <NUM_LIT> ;


<s> unsigned SubVecSize = DL . getTypeSizeInBits ( SubVecTy ) ; if ( NumElts % Factor == <NUM_LIT> && ( SubVecSize == <NUM_LIT> || SubVecSize % <NUM_LIT> == <NUM_LIT> ) && ! VecTy -> getScalarType ( ) -> isHalfTy ( ) ) return Factor * ( ( SubVecSize + <NUM_LIT> ) / <NUM_LIT> ) ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ;


<s> assert ( isa < VectorType > ( VecTy ) && <STR_LIT> ) ; bool EltIs64Bits = DL . getTypeSizeInBits ( VecTy -> getScalarType ( ) ) == <NUM_LIT> ; if ( Factor <= TLI -> getMaxSupportedInterleaveFactor ( ) && ! EltIs64Bits ) { unsigned NumElts = VecTy -> getVectorNumElements ( ) ; Type * SubVecTy = VectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; unsigned SubVecSize = DL . getTypeSizeInBits ( SubVecTy ) ; if ( NumElts % Factor == <NUM_LIT> && ( SubVecSize == <NUM_LIT> || SubVecSize % <NUM_LIT> == <NUM_LIT> ) && ! VecTy -> getScalarType ( ) -> isHalfTy ( ) ) return Factor * ( ( SubVecSize + <NUM_LIT> ) / <NUM_LIT> ) ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ;


<s> unsigned <STR_LIT> ( ShuffleKind Kind , Type * Tp , int Index , Type * SubTp ) const { if ( Kind != SK_Reverse && Kind != SK_Alternate ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; if ( Kind == SK_Reverse ) { static const CostTblEntry < <STR_LIT> > NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ; int Idx = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp , Index , SubTp ) ;

return LT . first * NEONShuffleTbl [ Idx ] . Cost ;


<s> static const CostTblEntry < <STR_LIT> > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; int Idx = - <NUM_LIT> ; if ( ST -> hasNEON ( ) ) Idx = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ; if ( Idx != - <NUM_LIT> ) return LT . first * CostTbl [ Idx ] . Cost ; return TargetTransformInfo :: getArithmeticInstrCost ( Opcode , Ty , Op1Info , Op2Info ) ;


<s> <ISA_LIT>TTI ( const <ISA_LIT>BaseTargetMachine * TM ) : ImmutablePass ( ID ) , TM ( TM ) , ST ( TM -> getSubtargetImpl ( ) ) , TLI ( TM -> getSubtargetImpl ( ) -> getTargetLowering ( ) ) { initialize<ISA_LIT>TTIPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * Ty , OperandValueKind Op1Info , OperandValueKind Op2Info , OperandValueProperties Opd1PropInfo , OperandValueProperties Opd2PropInfo ) const { int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < <STR_LIT> > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ;


<s> if ( ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) && ValTy -> getVectorElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ;

return TargetTransformInfo :: getVectorInstrCost ( Opcode , ValTy , Index ) ;


<s> if ( ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) && ValTy -> getVectorElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ; return TargetTransformInfo :: getVectorInstrCost ( Opcode , ValTy , Index ) ;


<s> case Instruction :: FMul : case Instruction :: And : case Instruction :: Or : case Instruction :: Xor : case Instruction :: Mul : case Instruction :: ICmp : case Instruction :: FCmp : return false ;


<s> bool <ISA_LIT>TTIImpl :: useReductionIntrinsic ( unsigned Opcode , Type * Ty , TTI :: ReductionFlags Flags ) const { assert ( isa < VectorType > ( Ty ) && <STR_LIT> ) ; unsigned ScalarBits = Ty -> getScalarSizeInBits ( ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; switch ( Opcode ) {


<s> if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first ;


<s> bool shouldExpandReduction ( const IntrinsicInst * II ) const { switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : return ! II -> getFastMathFlags ( ) . allowReassoc ( ) ; default : return false ; }


<s> <ISA_LIT>TTI ( const <ISA_LIT>BaseTargetMachine * TM ) : ImmutablePass ( ID ) , TM ( TM ) , ST ( TM -> getSubtargetImpl ( ) ) , TLI ( TM -> getTargetLowering ( ) ) { initialize<ISA_LIT>TTIPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> ImmutablePass * llvm :: create<ISA_LIT>TargetTransformInfoPass ( const <ISA_LIT>BaseTargetMachine * TM ) {


<s> unsigned <STR_LIT> ( Type * Ty ) const {


<s> TargetTransformInfo :: getAnalysisUsage ( AU ) ;


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( CondTy ) ; EVT SelValTy = TLI -> getValueType ( ValTy ) ; int Idx = ConvertCostTableLookup < MVT > ( NEONVectorSelectTbl , array_lengthof ( NEONVectorSelectTbl ) , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ; if ( Idx != - <NUM_LIT> ) return NEONVectorSelectTbl [ Idx ] . Cost ;

std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( ValTy ) ;


<s> if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } else if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } else {

if ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) return <NUM_LIT> ;


<s> unsigned getMaximumUnrollFactor ( ) const { if ( ST -> isCortexA15 ( ) || ST -> isSwift ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> if ( ST -> isThumb1Only ( ) ) return <NUM_LIT> ;


<s> if ( ST -> hasNEON ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ; int Idx = CostTableLookup < MVT > ( NEONShuffleTbl , array_lengthof ( NEONShuffleTbl ) , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp , Index , SubTp ) ;


<s> static const CostTblEntry < MVT > NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ;

int Idx = CostTableLookup < MVT > ( NEONShuffleTbl , array_lengthof ( NEONShuffleTbl ) , <STR_LIT> , LT . second ) ;


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * ValTy , unsigned Index ) const { if ( ST -> isSwift ( ) && Opcode == Instruction :: InsertElement && ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return <NUM_LIT> ; return TargetTransformInfo :: getVectorInstrCost ( Opcode , ValTy , Index ) ;


<s> bool isFPVectorizationPotentiallyUnsafe ( ) {


<s> bool shouldBuildLookupTablesForConstant ( Constant * C ) const { if ( ST -> isROPI ( ) || ST -> isRWPI ( ) || ST -> isPIP ( ) ) return ! C -> needsRelocation ( ) ; return true ;


<s> size_t <ISA_LIT>To<ISA_LIT>Stub :: alignment ( ) const {


<s> <ISA_LIT>To<ISA_LIT>Stub :: <ISA_LIT>To<ISA_LIT>Stub ( const uint32_t * pData , size_t pSize , const_fixup_iterator pBegin , const_fixup_iterator pEnd ) : m_pData ( pData ) , m_Name ( <STR_LIT> ) , m_Size ( pSize ) {

for ( const_fixup_iterator it = pBegin , ie = pEnd ; it != ie ; ++ it ) addFixup ( * * it ) ;


<s> <ISA_LIT>To<ISA_LIT>Stub :: <ISA_LIT>To<ISA_LIT>Stub ( const uint32_t * pData , size_t pSize , const_fixup_iterator pBegin , const_fixup_iterator pEnd ) : m_pData ( pData ) , m_Name ( <STR_LIT> ) , m_Size ( pSize ) { for ( const_fixup_iterator it = pBegin , ie = pEnd ; it != ie ; ++ it ) addFixup ( * * it ) ;


<s> Stub * <ISA_LIT>To<ISA_LIT>Stub :: doClone ( ) { return new <ISA_LIT>To<ISA_LIT>Stub ( m_pData , m_Size , fixup_begin ( ) , fixup_end ( ) ) ;


<s> const uint8_t * <ISA_LIT>To<ISA_LIT>Stub :: getContent ( ) const { return reinterpret_cast < const uint8_t * > ( m_pData ) ;


<s> case llvm :: ELF :: R_<ISA_LIT>_JUMP24 : case llvm :: ELF :: R_<ISA_LIT>_PLT32 : { uint64_t dest = pTargetSymValue + pReloc . addend ( ) + <NUM_LIT> ; int64_t branch_offset = static_cast < int64_t > ( dest ) - pSource ;

if ( ( branch_offset > <ISA_LIT>GNULDBackend :: <ISA_LIT>_MAX_FWD_BRANCH_OFFSET ) || ( branch_offset < <ISA_LIT>GNULDBackend :: <ISA_LIT>_MAX_BWD_BRANCH_OFFSET ) ) {


<s> const std :: string & <ISA_LIT>To<ISA_LIT>Stub :: name ( ) const {


<s> <ISA_LIT>To<ISA_LIT>Stub :: ~ <ISA_LIT>To<ISA_LIT>Stub ( ) {


<s> for ( const_fixup_iterator it = pBegin , ie = pEnd ; it != ie ; ++ it ) addFixup ( * * it ) ;


<s> Stub * <ISA_LIT>ToTHMStub :: doClone ( ) {

return new <ISA_LIT>ToTHMStub ( m_pData , m_Size , fixup_begin ( ) , fixup_end ( ) ) ;


<s> if ( ( pTargetSymValue & <NUM_LIT> ) != <NUM_LIT> ) { switch ( pReloc . type ( ) ) { case llvm :: ELF :: R_<ISA_LIT>_CALL : { uint64_t dest = pTargetSymValue + pReloc . addend ( ) + <NUM_LIT> ; int64_t branch_offset = static_cast < int64_t > ( dest ) - pSource ; if ( ( branch_offset > <ISA_LIT>GNULDBackend :: <ISA_LIT>_MAX_FWD_BRANCH_OFFSET ) || ( branch_offset < <ISA_LIT>GNULDBackend :: <ISA_LIT>_MAX_BWD_BRANCH_OFFSET ) ) { result = true ; break ; } break ; } case llvm :: ELF :: R_<ISA_LIT>_PC24 : case llvm :: ELF :: R_<ISA_LIT>_JUMP24 : case llvm :: ELF :: R_<ISA_LIT>_PLT32 : { result = true ; break ; }


<s> const std :: string & <ISA_LIT>ToTHMStub :: name ( ) const { return m_Name ;


<s> size_t <ISA_LIT>ToTHMStub :: size ( ) const { return m_Size ;


<s> <ISA_LIT>ToTHMStub :: ~ <ISA_LIT>ToTHMStub ( ) {


<s> Pos = ( ( ( Pos ^ <NUM_LIT> ) + <NUM_LIT> ) ^ <NUM_LIT> ) ;


<s> void EmitPersonalityIndex ( unsigned PI ) { assert ( PI < <STR_LIT> :: NUM_PERSONALITY_INDEX && <STR_LIT> ) ;

EmitByte ( <STR_LIT> :: EHT_COMPACT | PI ) ;


<s> assert ( PI < <STR_LIT> :: NUM_PERSONALITY_INDEX && <STR_LIT> ) ;


<s> EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_POP_REG_RANGE_R4_R14 | Range ) ; RegSave &= <NUM_LIT> ; } } if ( ( RegSave & <NUM_LIT> ) != <NUM_LIT> ) EmitInt16 ( <STR_LIT> :: UNWIND_OPCODE_POP_REG_MASK_R4 | ( RegSave >> <NUM_LIT> ) ) ; if ( ( RegSave & <NUM_LIT> ) != <NUM_LIT> ) EmitInt16 ( <STR_LIT> :: UNWIND_OPCODE_POP_REG_MASK | ( RegSave & <NUM_LIT> ) ) ;


<s> EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_SET_VSP | Reg ) ;


<s> EmitByte ( static_cast < uint8_t > ( SizeInWords - <NUM_LIT> ) ) ;


<s> uint8_t Buff [ <NUM_LIT> ] ; Buff [ <NUM_LIT> ] = <STR_LIT> :: UNWIND_OPCODE_INC_VSP_ULEB128 ; size_t ULEBSize = encodeULEB128 ( ( Offset - <NUM_LIT> ) >> <NUM_LIT> , Buff + <NUM_LIT> ) ; EmitBytes ( Buff , ULEBSize + <NUM_LIT> ) ; } else if ( Offset > <NUM_LIT> ) { if ( Offset > <NUM_LIT> ) { EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_INC_VSP | <NUM_LIT> ) ; Offset -= <NUM_LIT> ; } EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_INC_VSP | static_cast < uint8_t > ( ( Offset - <NUM_LIT> ) >> <NUM_LIT> ) ) ; } else if ( Offset < <NUM_LIT> ) { while ( Offset < - <NUM_LIT> ) {


<s> if ( Offset > <NUM_LIT> ) { EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_INC_VSP | <NUM_LIT> ) ; Offset -= <NUM_LIT> ; } EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_INC_VSP | static_cast < uint8_t > ( ( Offset - <NUM_LIT> ) >> <NUM_LIT> ) ) ; } else if ( Offset < <NUM_LIT> ) { while ( Offset < - <NUM_LIT> ) { EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_DEC_VSP | <NUM_LIT> ) ; Offset += <NUM_LIT> ;


EmitInt16 ( Opcode | ( ( RangeLSB % <NUM_LIT> ) << <NUM_LIT> ) | ( RangeLen - <NUM_LIT> ) ) ;


<s> void FillFinishOpcode ( ) { while ( Pos < Vec . size ( ) ) EmitByte ( <STR_LIT> :: UNWIND_OPCODE_FINISH ) ;


<s> while ( Pos < Vec . size ( ) ) EmitByte ( <STR_LIT> :: UNWIND_OPCODE_FINISH ) ;


<s> PersonalityIndex = <STR_LIT> :: NUM_PERSONALITY_INDEX ; size_t TotalSize = Ops . size ( ) + <NUM_LIT> ; size_t RoundUpSize = ( TotalSize + <NUM_LIT> ) / <NUM_LIT> * <NUM_LIT> ; Result . resize ( RoundUpSize ) ; OpStreamer . EmitSize ( RoundUpSize ) ; } else { if ( PersonalityIndex == <STR_LIT> :: NUM_PERSONALITY_INDEX ) PersonalityIndex = ( Ops . size ( ) <= <NUM_LIT> ) ? <STR_LIT> :: AEABI_UNWIND_CPP_PR0 : <STR_LIT> :: AEABI_UNWIND_CPP_PR1 ;

if ( PersonalityIndex == <STR_LIT> :: AEABI_UNWIND_CPP_PR0 ) {


<s> UnwindOpcodeStreamer ( SmallVectorImpl < uint8_t > & V ) : Vec ( V ) {


<s> OpBegins . push_back ( <NUM_LIT> ) ;


<s> Ops . clear ( ) ; OpBegins . clear ( ) ; OpBegins . push_back ( <NUM_LIT> ) ; HasPersonality = false ;


<s> if ( Offset > <NUM_LIT> ) { uint8_t Buff [ <NUM_LIT> ] ; Buff [ <NUM_LIT> ] = <STR_LIT> :: UNWIND_OPCODE_INC_VSP_ULEB128 ; size_t ULEBSize = encodeULEB128 ( ( Offset - <NUM_LIT> ) >> <NUM_LIT> , Buff + <NUM_LIT> ) ; emitBytes ( Buff , ULEBSize + <NUM_LIT> ) ; } else if ( Offset > <NUM_LIT> ) { if ( Offset > <NUM_LIT> ) { EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_INC_VSP | <NUM_LIT> ) ; Offset -= <NUM_LIT> ; } EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_INC_VSP | static_cast < uint8_t > ( ( Offset - <NUM_LIT> ) >> <NUM_LIT> ) ) ;


<s> uint32_t UnmaskedReg = RegSave & <NUM_LIT> & ( ~ Mask ) ; if ( UnmaskedReg == <NUM_LIT> ) { EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_POP_REG_RANGE_R4 | Range ) ; RegSave &= <NUM_LIT> ; } else if ( UnmaskedReg == ( <NUM_LIT> << <NUM_LIT> ) ) { EmitInt8 ( <STR_LIT> :: UNWIND_OPCODE_POP_REG_RANGE_R4_R14 | Range ) ; RegSave &= <NUM_LIT> ; } }

if ( ( RegSave & <NUM_LIT> ) != <NUM_LIT> ) EmitInt16 ( <STR_LIT> :: UNWIND_OPCODE_POP_REG_MASK_R4 | ( RegSave >> <NUM_LIT> ) ) ;


Pos = ( ( ( Pos ^ <NUM_LIT> ) + <NUM_LIT> ) ^ <NUM_LIT> ) ;


<s> OpBegins . push_back ( OpBegins . back ( ) + Size ) ;


<s> OpBegins . push_back ( OpBegins . back ( ) + <NUM_LIT> ) ;


<s> inline void EmitPersonalityIndex ( unsigned PI ) { assert ( PI < <STR_LIT> :: NUM_PERSONALITY_INDEX && <STR_LIT> ) ;


<s> EmitByte ( <STR_LIT> :: EHT_COMPACT | PI ) ;


<s> OpBegins . push_back ( OpBegins . back ( ) + Opcodes . size ( ) ) ;


<s> Ops . insert ( Ops . end ( ) , Opcodes . begin ( ) , Opcodes . end ( ) ) ;

OpBegins . push_back ( OpBegins . back ( ) + Opcodes . size ( ) ) ;


<s> size_t SizeInWords = ( Size + <NUM_LIT> ) / <NUM_LIT> ; assert ( SizeInWords <= <NUM_LIT> && <STR_LIT> ) ;

EmitByte ( static_cast < uint8_t > ( SizeInWords - <NUM_LIT> ) ) ;


while ( Pos < Vec . size ( ) ) EmitByte ( <STR_LIT> :: UNWIND_OPCODE_FINISH ) ;


<s> OpBegins . clear ( ) ; OpBegins . push_back ( <NUM_LIT> ) ; HasPersonality = <NUM_LIT> ;


<s> void setPersonality ( const MCSymbol * Per ) { HasPersonality = <NUM_LIT> ;


<s> UnwindOpcodeAssembler ( ) : HasPersonality ( <NUM_LIT> ) {


<s> UnwindOpcodeStreamer ( SmallVectorImpl < uint8_t > & V ) : Vec ( V ) , Pos ( <NUM_LIT> ) {


<s> assert ( getMachine ( ) == COFF :: IMAGE_FILE_MACHINE_<ISA_LIT>NT && <STR_LIT> ) ; MCSymbolRefExpr :: VariantKind Modifier = Target . isAbsolute ( ) ? MCSymbolRefExpr :: VK_None : Target . getSymA ( ) -> getKind ( ) ; switch ( static_cast < unsigned > ( Fixup . getKind ( ) ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case FK_Data_4 : switch ( Modifier ) { case MCSymbolRefExpr :: VK_COFF_IMGREL32 : return COFF :: IMAGE_REL_<ISA_LIT>_ADDR32NB ; case MCSymbolRefExpr :: VK_SECREL :

return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ;


<s> MCSymbolRefExpr :: VariantKind Modifier = Target . isAbsolute ( ) ? MCSymbolRefExpr :: VK_None : Target . getSymA ( ) -> getKind ( ) ; switch ( static_cast < unsigned > ( Fixup . getKind ( ) ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case FK_Data_4 : switch ( Modifier ) { case MCSymbolRefExpr :: VK_COFF_IMGREL32 : return COFF :: IMAGE_REL_<ISA_LIT>_ADDR32NB ; case MCSymbolRefExpr :: VK_SECREL : return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ; default : return COFF :: IMAGE_REL_<ISA_LIT>_ADDR32 ; } case FK_SecRel_4 : return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BRANCH20T ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BRANCH24T ; case <STR_LIT> : case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BLX23T ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_MOV32T ; case <STR_LIT> :


<s> <ISA_LIT>WinCOFFObjectWriter ( ) : MCWinCOFFObjectTargetWriter ( COFF :: IMAGE_FILE_MACHINE_<ISA_LIT>NT ) {


<s> return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ; default : return COFF :: IMAGE_REL_<ISA_LIT>_ADDR32 ; } case FK_PCRel_4 : return COFF :: IMAGE_REL_<ISA_LIT>_REL32 ; case FK_SecRel_2 : return COFF :: IMAGE_REL_<ISA_LIT>_SECTION ; case FK_SecRel_4 : return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BRANCH20T ; case <STR_LIT> : case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BRANCH24T ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BLX23T ; case <STR_LIT> : case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_MOV32T ;


<s> MCObjectWriter * create<ISA_LIT>WinCOFFObjectWriter ( raw_pwrite_stream & OS , bool Is64Bit ) { MCWinCOFFObjectTargetWriter * MOTW = new <ISA_LIT>WinCOFFObjectWriter ( Is64Bit ) ; return createWinCOFFObjectWriter ( MOTW , OS ) ;


<s> MCObjectWriter * create<ISA_LIT>WinCOFFObjectWriter ( raw_pwrite_stream & OS , bool Is64Bit ) {

MCWinCOFFObjectTargetWriter * MOTW = new <ISA_LIT>WinCOFFObjectWriter ( Is64Bit ) ;


<s> ~ <ISA_LIT>WinCOFFObjectWriter ( ) override {


<s> std :: unique_ptr < MCObjectTargetWriter > create<ISA_LIT>WinCOFFObjectWriter ( bool Is64Bit ) { return llvm :: make_unique < <ISA_LIT>WinCOFFObjectWriter > ( Is64Bit ) ;


<s> std :: unique_ptr < MCObjectTargetWriter > create<ISA_LIT>WinCOFFObjectWriter ( bool Is64Bit ) {

return llvm :: make_unique < <ISA_LIT>WinCOFFObjectWriter > ( Is64Bit ) ;


<s> switch ( Modifier ) { case MCSymbolRefExpr :: VK_COFF_IMGREL32 : return COFF :: IMAGE_REL_<ISA_LIT>_ADDR32NB ; case MCSymbolRefExpr :: VK_SECREL : return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ; default : return COFF :: IMAGE_REL_<ISA_LIT>_ADDR32 ; } case FK_SecRel_2 : return COFF :: IMAGE_REL_<ISA_LIT>_SECTION ; case FK_SecRel_4 : return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BRANCH20T ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BRANCH24T ; case <STR_LIT> : case <STR_LIT> :

return COFF :: IMAGE_REL_<ISA_LIT>_BLX23T ;


<s> std :: unique_ptr < MCObjectWriter > create<ISA_LIT>WinCOFFObjectWriter ( raw_pwrite_stream & OS , bool Is64Bit ) { auto MOTW = llvm :: make_unique < <ISA_LIT>WinCOFFObjectWriter > ( Is64Bit ) ; return createWinCOFFObjectWriter ( std :: move ( MOTW ) , OS ) ;


<s> <ISA_LIT>WinCOFFObjectWriter ( bool Is64Bit ) : MCWinCOFFObjectTargetWriter ( COFF :: IMAGE_FILE_MACHINE_<ISA_LIT>NT ) {


<s> MCWinCOFFObjectTargetWriter * MOTW = new <ISA_LIT>WinCOFFObjectWriter ( Is64Bit ) ;

return createWinCOFFObjectWriter ( MOTW , OS ) ;


<s> MCWinCOFFObjectTargetWriter * MOTW = new <ISA_LIT>WinCOFFObjectWriter ( Is64Bit ) ; return createWinCOFFObjectWriter ( MOTW , OS ) ;


<s> case FK_Data_4 : switch ( Modifier ) { case MCSymbolRefExpr :: VK_COFF_IMGREL32 : return COFF :: IMAGE_REL_<ISA_LIT>_ADDR32NB ; case MCSymbolRefExpr :: VK_SECREL : return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ; default : return COFF :: IMAGE_REL_<ISA_LIT>_ADDR32 ; } case FK_SecRel_2 : return COFF :: IMAGE_REL_<ISA_LIT>_SECTION ; case FK_SecRel_4 : return COFF :: IMAGE_REL_<ISA_LIT>_SECREL ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BRANCH20T ; case <STR_LIT> : return COFF :: IMAGE_REL_<ISA_LIT>_BRANCH24T ; case <STR_LIT> :


<s> bool <ISA_LIT>WinCOFFObjectWriter :: recordRelocation ( const MCFixup & Fixup ) const { return static_cast < unsigned > ( Fixup . getKind ( ) ) != <STR_LIT> ;


<s> return static_cast < unsigned > ( Fixup . getKind ( ) ) != <STR_LIT> ;


<s> virtual ~ <ISA_LIT>WinCOFFObjectWriter ( ) {


<s> MCStreamer * llvm :: create<ISA_LIT>WinCOFFStreamer ( MCContext & Context , MCAsmBackend & MAB , raw_pwrite_stream & OS , MCCodeEmitter * Emitter , bool RelaxAll ) {

return new <ISA_LIT>WinCOFFStreamer ( Context , MAB , * Emitter , OS ) ;


<s> <ISA_LIT>WinCOFFStreamer ( MCContext & C , std :: unique_ptr < MCAsmBackend > AB , std :: unique_ptr < MCCodeEmitter > CE , raw_pwrite_stream & OS ) : MCWinCOFFStreamer ( C , std :: move ( AB ) , std :: move ( CE ) , OS ) {


<s> auto * S = new <ISA_LIT>WinCOFFStreamer ( Context , std :: move ( MAB ) , std :: move ( Emitter ) , OS ) ; S -> getAssembler ( ) . setIncrementalLinkerCompatible ( IncrementalLinkerCompatible ) ; return S ;


<s> <ISA_LIT>WinCOFFStreamer ( MCContext & C , MCAsmBackend & AB , MCCodeEmitter & CE , raw_pwrite_stream & OS ) : MCWinCOFFStreamer ( C , AB , CE , OS ) {


<s> auto * S = new <ISA_LIT>WinCOFFStreamer ( Context , MAB , * Emitter , OS ) ; S -> getAssembler ( ) . setIncrementalLinkerCompatible ( IncrementalLinkerCompatible ) ; return S ;


<s> switch ( Flag ) { default : llvm_unreachable ( <STR_LIT> ) ; case MCAF_SyntaxUnified :


<s> getAssembler ( ) . setIsThumbFunc ( Symbol ) ;


<s> <ISA_LIT>TargetWinCOFFStreamer ( llvm :: MCStreamer & S ) : <ISA_LIT>TargetStreamer ( S ) {


<s> unsigned Op = Win64EH :: UOP_AllocSmall ; if ( ! Wide ) { if ( Size / <NUM_LIT> > <NUM_LIT> ) Op = Win64EH :: UOP_AllocHuge ; else if ( Size / <NUM_LIT> > <NUM_LIT> ) Op = Win64EH :: UOP_AllocLarge ; } else { Op = Win64EH :: UOP_WideAllocMedium ;

if ( Size / <NUM_LIT> > <NUM_LIT> ) Op = Win64EH :: UOP_WideAllocHuge ;


<s> unsigned Op = Win64EH :: UOP_AllocSmall ; if ( ! Wide ) { if ( Size / <NUM_LIT> > <NUM_LIT> ) Op = Win64EH :: UOP_AllocHuge ; else if ( Size / <NUM_LIT> > <NUM_LIT> ) Op = Win64EH :: UOP_AllocLarge ; } else { Op = Win64EH :: UOP_WideAllocMedium ; if ( Size / <NUM_LIT> > <NUM_LIT> ) Op = Win64EH :: UOP_WideAllocHuge ; else if ( Size / <NUM_LIT> > <NUM_LIT> ) Op = Win64EH :: UOP_WideAllocLarge ;


<s> std :: vector < WinEH :: Instruction > & Epilog = CurFrame -> EpilogMap [ CurrentEpilog ] . Instructions ; unsigned UnwindCode = Win64EH :: UOP_End ; if ( ! Epilog . empty ( ) ) { WinEH :: Instruction EndInstr = Epilog . back ( ) ; if ( EndInstr . Operation == Win64EH :: UOP_Nop ) { UnwindCode = Win64EH :: UOP_EndNop ; Epilog . pop_back ( ) ; } else if ( EndInstr . Operation == Win64EH :: UOP_WideNop ) {

UnwindCode = Win64EH :: UOP_WideEndNop ;


<s> auto & S = getStreamer ( ) ; WinEH :: FrameInfo * CurFrame = S . EnsureValidWinFrameInfo ( SMLoc ( ) ) ; if ( ! CurFrame ) return ; InEpilogCFI = true ; CurrentEpilog = S . emitCFILabel ( ) ;

CurFrame -> EpilogMap [ CurrentEpilog ] . Condition = Condition ;


<s> if ( ! CurFrame ) return ; InEpilogCFI = true ; CurrentEpilog = S . emitCFILabel ( ) ; CurFrame -> EpilogMap [ CurrentEpilog ] . Condition = Condition ;


<s> void <ISA_LIT>TargetWinCOFFStreamer :: emit<ISA_LIT>WinCFINop ( bool Wide ) { if ( Wide ) emit<ISA_LIT>WinUnwindCode ( Win64EH :: UOP_WideNop , - <NUM_LIT> , <NUM_LIT> ) ;


<s> auto & S = getStreamer ( ) ; WinEH :: FrameInfo * CurFrame = S . EnsureValidWinFrameInfo ( SMLoc ( ) ) ; if ( ! CurFrame ) return ; MCSymbol * Label = S . emitCFILabel ( ) ; CurFrame -> PrologEnd = Label ; WinEH :: Instruction Inst = WinEH :: Instruction ( Win64EH :: UOP_End , nullptr , - <NUM_LIT> , <NUM_LIT> ) ; auto it = CurFrame -> Instructions . begin ( ) ;

CurFrame -> Instructions . insert ( it , Inst ) ;


<s> assert ( First <= <NUM_LIT> && Last <= <NUM_LIT> ) ; if ( First == <NUM_LIT> ) emit<ISA_LIT>WinUnwindCode ( Win64EH :: UOP_SaveFRegD8D15 , Last , <NUM_LIT> ) ; else if ( First <= <NUM_LIT> ) emit<ISA_LIT>WinUnwindCode ( Win64EH :: UOP_SaveFRegD0D15 , First , Last ) ; else emit<ISA_LIT>WinUnwindCode ( Win64EH :: UOP_SaveFRegD16D31 , First , Last ) ;


<s> else if ( First <= <NUM_LIT> ) emit<ISA_LIT>WinUnwindCode ( Win64EH :: UOP_SaveFRegD0D15 , First , Last ) ;

else emit<ISA_LIT>WinUnwindCode ( Win64EH :: UOP_SaveFRegD16D31 , First , Last ) ;


<s> if ( Wide ) assert ( ( Mask & ~ <NUM_LIT> ) == <NUM_LIT> ) ; else assert ( ( Mask & ~ <NUM_LIT> ) == <NUM_LIT> ) ; if ( Mask && ( ( Mask + ( <NUM_LIT> << <NUM_LIT> ) ) & Mask ) == <NUM_LIT> ) { if ( Wide && ( Mask & <NUM_LIT> ) == <NUM_LIT> && ( Mask & <NUM_LIT> ) == <NUM_LIT> ) { for ( int I = <NUM_LIT> ; I >= <NUM_LIT> ; I -- ) { if ( Mask & ( <NUM_LIT> << I ) ) { emit<ISA_LIT>WinUnwindCode ( Win64EH :: UOP_WideSaveRegsR4R11LR , I , Lr ) ; return ; } } } else if ( ! Wide ) { for ( int I = <NUM_LIT> ; I >= <NUM_LIT> ; I -- ) { if ( Mask & ( <NUM_LIT> << I ) ) { emit<ISA_LIT>WinUnwindCode ( Win64EH :: UOP_SaveRegsR4R7LR , I , Lr ) ;


<s> void <ISA_LIT>TargetWinCOFFStreamer :: emit<ISA_LIT>WinUnwindCode ( unsigned UnwindCode , int Reg , int Offset ) { auto & S = getStreamer ( ) ;

WinEH :: FrameInfo * CurFrame = S . EnsureValidWinFrameInfo ( SMLoc ( ) ) ;


<s> void <ISA_LIT>TargetWinCOFFStreamer :: emit<ISA_LIT>WinUnwindCode ( unsigned UnwindCode , int Reg , int Offset ) { auto & S = getStreamer ( ) ; WinEH :: FrameInfo * CurFrame = S . EnsureValidWinFrameInfo ( SMLoc ( ) ) ; if ( ! CurFrame ) return ; MCSymbol * Label = S . emitCFILabel ( ) ;

auto Inst = WinEH :: Instruction ( UnwindCode , Label , Reg , Offset ) ;


<s> void <ISA_LIT>WinCOFFStreamer :: emitWinEHHandlerData ( SMLoc Loc ) {

MCStreamer :: emitWinEHHandlerData ( Loc ) ;


<s> MCStreamer :: emitWinEHHandlerData ( Loc ) ; EHStreamer . EmitUnwindInfo ( * this , getCurrentWinFrameInfo ( ) , true ) ;


<s> emitWindowsUnwindTables ( ) ; MCWinCOFFStreamer :: finishImpl ( ) ;


<s> void <ISA_LIT>WinCOFFStreamer :: finishImpl ( ) { emitFrames ( nullptr ) ;


<s> <ISA_LIT>WinCOFFStreamer ( MCContext & C , MCAsmBackend & AB , MCCodeEmitter & CE , raw_ostream & OS ) : MCWinCOFFStreamer ( C , AB , CE , OS ) {


<s> MCStreamer * create<ISA_LIT>WinCOFFStreamer ( MCContext & Context , MCAsmBackend & MAB , MCCodeEmitter & Emitter , raw_ostream & OS ) {

return new <ISA_LIT>WinCOFFStreamer ( Context , MAB , Emitter , OS ) ;


<s> return new <ISA_LIT>WinCOFFStreamer ( Context , MAB , * Emitter , OS ) ;


<s> <ISA_LIT>WinCOFFStreamer ( MCContext & C , std :: unique_ptr < MCAsmBackend > AB , std :: unique_ptr < MCCodeEmitter > CE , std :: unique_ptr < MCObjectWriter > OW ) : MCWinCOFFStreamer ( C , std :: move ( AB ) , std :: move ( CE ) , std :: move ( OW ) ) {


<s> MCStreamer * llvm :: create<ISA_LIT>WinCOFFStreamer ( MCContext & Context , std :: unique_ptr < MCAsmBackend > && MAB , std :: unique_ptr < MCObjectWriter > && OW , std :: unique_ptr < MCCodeEmitter > && Emitter , bool RelaxAll , bool IncrementalLinkerCompatible ) { auto * S = new <ISA_LIT>WinCOFFStreamer ( Context , std :: move ( MAB ) , std :: move ( Emitter ) , std :: move ( OW ) ) ; S -> getAssembler ( ) . setIncrementalLinkerCompatible ( IncrementalLinkerCompatible ) ; return S ;


<s> void <ISA_LIT>WinCOFFStreamer :: emitThumbFunc ( MCSymbol * Symbol ) { getAssembler ( ) . setIsThumbFunc ( Symbol ) ;


<s> MCWinCOFFStreamer :: finishImpl ( ) ;


<s> std :: fill ( LastMIs , LastMIs + <NUM_LIT> , nullptr ) ;


<s> bool MLxExpansion :: ExpandFPMLxInstructions ( MachineBasicBlock & MBB ) { bool Changed = false ; clearStack ( ) ; IgnoreStall . clear ( ) ; unsigned Skip = <NUM_LIT> ; MachineBasicBlock :: reverse_iterator MII = MBB . rbegin ( ) , E = MBB . rend ( ) ; while ( MII != E ) { MachineInstr * MI = & * MII ; if ( MI -> isPosition ( ) || MI -> isImplicitDef ( ) || MI -> isCopy ( ) ) { ++ MII ; continue ; } const MCInstrDesc & MCID = MI -> getDesc ( ) ;


<s> if ( Domain == <STR_LIT> ) { if ( ++ Skip == <NUM_LIT> ) pushStack ( nullptr ) ; } else { Skip = <NUM_LIT> ; unsigned MulOpc , AddSubOpc ; bool NegAcc , HasLane ; if ( ! TII -> isFpMLxInstruction ( MCID . getOpcode ( ) , MulOpc , AddSubOpc , NegAcc , HasLane ) || ! FindMLxHazard ( MI ) ) pushStack ( MI ) ; else { ExpandFPMLxInstruction ( MBB , MI , MulOpc , AddSubOpc , NegAcc , HasLane ) ; E = MBB . rend ( ) ; Changed = true ; continue ; } } ++ MII ;


<s> MachineInstr * MLxExpansion :: getAccDefMI ( MachineInstr * MI ) const { unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) ) return nullptr ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * DefMI = MRI -> getVRegDef ( Reg ) ; while ( true ) { if ( DefMI -> getParent ( ) != MBB ) break ; if ( DefMI -> isCopyLike ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ; continue ; } } else if ( DefMI -> isInsertSubreg ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ; continue ; } } break ;


<s> TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( Fn . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getSubtarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const <ISA_LIT>Subtarget * STI = & Fn . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! STI -> expandMLx ( ) ) return false ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ;


<s> void MLxExpansion :: clearStack ( ) {

std :: fill ( LastMIs , LastMIs + <NUM_LIT> , ( MachineInstr * ) <NUM_LIT> ) ;


<s> FunctionPass * llvm :: createMLxExpansionPass ( ) {


<s> const MCInstrDesc & MCID2 = TII -> get ( AddSubOpc ) ; const MachineFunction & MF = * MI -> getParent ( ) -> getParent ( ) ; unsigned TmpReg = MRI -> createVirtualRegister ( TII -> getRegClass ( MCID1 , <NUM_LIT> , TRI , MF ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ; if ( HasLane ) MIB . addImm ( LaneImm ) ; MIB . addImm ( Pred ) . addReg ( PredReg ) ; MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID2 ) . addReg ( DstReg , getDefRegState ( true ) | getDeadRegState ( DstDead ) ) ; if ( NegAcc ) { bool AccKill = MRI -> hasOneNonDBGUse ( AccReg ) ; MIB . addReg ( TmpReg , getKillRegState ( true ) ) . addReg ( AccReg , getKillRegState ( AccKill ) ) ; } else { MIB . addReg ( AccReg ) . addReg ( TmpReg , getKillRegState ( true ) ) ; } MIB . addImm ( Pred ) . addReg ( PredReg ) ; DEBUG ( { dbgs ( ) << <STR_LIT> << * MI ; dbgs ( ) << <STR_LIT> ; MachineBasicBlock :: iterator MII = MI ; MII = std :: prev ( MII ) ; MachineInstr & MI2 = * MII ; MII = std :: prev ( MII ) ; MachineInstr & MI1 = * MII ;


<s> } const MCInstrDesc & MCID = MI -> getDesc ( ) ; if ( MI -> isBarrier ( ) ) { clearStack ( ) ; Skip = <NUM_LIT> ; ++ MII ; continue ; } unsigned Domain = MCID . TSFlags & <STR_LIT> ; if ( Domain == <STR_LIT> ) { if ( ++ Skip == <NUM_LIT> ) pushStack ( <NUM_LIT> ) ; } else { Skip = <NUM_LIT> ; unsigned MulOpc , AddSubOpc ; bool NegAcc , HasLane ; if ( ! TII -> isFpMLxInstruction ( MCID . getOpcode ( ) , MulOpc , AddSubOpc , NegAcc , HasLane ) || ! FindMLxHazard ( MI ) ) pushStack ( MI ) ; else { ExpandFPMLxInstruction ( MBB , MI , MulOpc , AddSubOpc , NegAcc , HasLane ) ; E = MBB . rend ( ) ; Changed = true ; continue ; } } ++ MII ; }


<s> int Idx = ( ( int ) MIIdx - i + <NUM_LIT> ) % <NUM_LIT> ; MachineInstr * NextMI = LastMIs [ Idx ] ; if ( ! NextMI ) continue ; if ( TII -> canCauseFpMLxStall ( NextMI -> getOpcode ( ) ) ) { if ( i <= Limit1 ) return true ; }

if ( i <= Limit2 && hasRAWHazard ( getDefReg ( MI ) , NextMI ) ) return true ;


<s> while ( true ) { if ( DefMI -> getParent ( ) != MBB ) break ; if ( DefMI -> isCopyLike ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) {


<s> MachineInstr * DefMI = MRI -> getVRegDef ( Reg ) ; while ( true ) { if ( DefMI -> getParent ( ) != MBB ) break ; if ( DefMI -> isCopyLike ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ; continue ; } } else if ( DefMI -> isInsertSubreg ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ;


<s> unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) || ! MRI -> hasOneNonDBGUse ( Reg ) ) return Reg ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * UseMI = & * MRI -> use_nodbg_begin ( Reg ) ; if ( UseMI -> getParent ( ) != MBB ) return Reg ;

while ( UseMI -> isCopy ( ) || UseMI -> isInsertSubreg ( ) ) {


<s> if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) || ! MRI -> hasOneNonDBGUse ( Reg ) ) return Reg ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * UseMI = & * MRI -> use_nodbg_begin ( Reg ) ; if ( UseMI -> getParent ( ) != MBB ) return Reg ; while ( UseMI -> isCopy ( ) || UseMI -> isInsertSubreg ( ) ) { Reg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) || ! MRI -> hasOneNonDBGUse ( Reg ) ) return Reg ; UseMI = & * MRI -> use_nodbg_begin ( Reg ) ; if ( UseMI -> getParent ( ) != MBB ) return Reg ; }


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> if ( DefMI -> getParent ( ) != MBB ) break ; if ( DefMI -> isPHI ( ) ) { for ( unsigned i = <NUM_LIT> , e = DefMI -> getNumOperands ( ) ; i < e ; i += <NUM_LIT> ) { if ( DefMI -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) == MBB ) { unsigned SrcReg = DefMI -> getOperand ( i ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { DefMI = MRI -> getVRegDef ( SrcReg ) ; goto outer_continue ; } }


<s> if ( MI -> mayStore ( ) ) return false ; unsigned Opcode = MCID . getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return false ; if ( ( Domain & <STR_LIT> ) || ( Domain & <STR_LIT> ) ) return MI -> readsRegister ( Reg , TRI ) ;


<s> if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return false ;

if ( ( Domain & <STR_LIT> ) || ( Domain & <STR_LIT> ) ) return MI -> readsRegister ( Reg , TRI ) ;


<s> MLxExpansion ( ) : MachineFunctionPass ( ID ) {


<s> if ( ++ MIIdx == <NUM_LIT> ) MIIdx = <NUM_LIT> ;


<s> void MLxExpansion :: pushStack ( MachineInstr * MI ) { LastMIs [ MIIdx ] = MI ;

if ( ++ MIIdx == <NUM_LIT> ) MIIdx = <NUM_LIT> ;


<s> const <ISA_LIT>Subtarget * STI = & Fn . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ;


<s> unsigned Src1Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src2Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool Src1Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool Src2Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; unsigned LaneImm = HasLane ? MI -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; unsigned NextOp = HasLane ? <NUM_LIT> : <NUM_LIT> ; <STR_LIT> Pred = ( <STR_LIT> ) MI -> getOperand ( NextOp ) . getImm ( ) ; unsigned PredReg = MI -> getOperand ( ++ NextOp ) . getReg ( ) ; const TargetInstrDesc & TID1 = TII -> get ( MulOpc ) ; const TargetInstrDesc & TID2 = TII -> get ( AddSubOpc ) ; unsigned TmpReg = MRI -> createVirtualRegister ( TID1 . getRegClass ( <NUM_LIT> , TRI ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , * MI , MI -> getDebugLoc ( ) , TID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ; if ( HasLane ) MIB . addImm ( LaneImm ) ; MIB . addImm ( Pred ) . addReg ( PredReg ) ; MIB = BuildMI ( MBB , * MI , MI -> getDebugLoc ( ) , TID2 ) . addReg ( DstReg , getDefRegState ( true ) | getDeadRegState ( DstDead ) ) ; if ( NegAcc ) { bool AccKill = MRI -> hasOneNonDBGUse ( AccReg ) ;

MIB . addReg ( TmpReg , getKillRegState ( true ) ) . addReg ( AccReg , getKillRegState ( AccKill ) ) ;


<s> Skip = <NUM_LIT> ; unsigned MulOpc , AddSubOpc ; bool NegAcc , HasLane ; if ( ! TII -> isFpMLxInstruction ( TID . getOpcode ( ) , MulOpc , AddSubOpc , NegAcc , HasLane ) || ! FindMLxHazard ( MI ) ) pushStack ( MI ) ; else { ExpandFPMLxInstruction ( MBB , MI , MulOpc , AddSubOpc , NegAcc , HasLane ) ; E = MBB . rend ( ) ; Changed = true ; continue ; } } ++ MII ;


<s> if ( TII -> isFpMLxInstruction ( DefMI -> getOpcode ( ) ) ) return true ; for ( unsigned i = <NUM_LIT> ; i <= <NUM_LIT> ; ++ i ) { int Idx = ( ( int ) MIIdx - i + <NUM_LIT> ) % <NUM_LIT> ; MachineInstr * NextMI = LastMIs [ Idx ] ; if ( ! NextMI ) continue ; if ( TII -> canCauseFpMLxStall ( NextMI -> getOpcode ( ) ) ) return true ; if ( hasRAWHazard ( getDefReg ( MI ) , NextMI ) ) return true ;


<s> virtual const char * getPassName ( ) const {


<s> } else if ( Domain == <STR_LIT> ) { if ( TID . mayStore ( ) || TID . mayLoad ( ) ) return false ; } else { return false ; }

return MI -> readsRegister ( Reg , TRI ) ;


<s> if ( Domain == <STR_LIT> ) { unsigned Opcode = TID . getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return false ; } else if ( Domain == <STR_LIT> ) { if ( TID . mayStore ( ) || TID . mayLoad ( ) ) return false ; } else {


<s> bool Modified = false ; for ( MachineFunction :: iterator MFI = Fn . begin ( ) , E = Fn . end ( ) ; MFI != E ; ++ MFI ) { MachineBasicBlock & MBB = * MFI ; Modified |= ExpandFPMLxInstructions ( MBB ) ; }


<s> void MLxExpansion :: ExpandFPMLxInstruction ( MachineBasicBlock & MBB , MachineInstr * MI , unsigned MulOpc , unsigned AddSubOpc , bool NegAcc , bool HasLane ) { Register DstReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool DstDead = MI -> getOperand ( <NUM_LIT> ) . isDead ( ) ; Register AccReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src2Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool Src1Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool Src2Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; unsigned LaneImm = HasLane ? MI -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; unsigned NextOp = HasLane ? <NUM_LIT> : <NUM_LIT> ; <STR_LIT> Pred = ( <STR_LIT> ) MI -> getOperand ( NextOp ) . getImm ( ) ; Register PredReg = MI -> getOperand ( ++ NextOp ) . getReg ( ) ; const MCInstrDesc & MCID1 = TII -> get ( MulOpc ) ; const MCInstrDesc & MCID2 = TII -> get ( AddSubOpc ) ; const MachineFunction & MF = * MI -> getParent ( ) -> getParent ( ) ; Register TmpReg = MRI -> createVirtualRegister ( TII -> getRegClass ( MCID1 , <NUM_LIT> , TRI , MF ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ; if ( HasLane ) MIB . addImm ( LaneImm ) ; MIB . addImm ( Pred ) . addReg ( PredReg ) ; MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID2 ) . addReg ( DstReg , getDefRegState ( true ) | getDeadRegState ( DstDead ) ) ; if ( NegAcc ) { bool AccKill = MRI -> hasOneNonDBGUse ( AccReg ) ; MIB . addReg ( TmpReg , getKillRegState ( true ) ) . addReg ( AccReg , getKillRegState ( AccKill ) ) ; } else { MIB . addReg ( AccReg ) . addReg ( TmpReg , getKillRegState ( true ) ) ; }


<s> const MCInstrDesc & MCID2 = TII -> get ( AddSubOpc ) ; const MachineFunction & MF = * MI -> getParent ( ) -> getParent ( ) ; Register TmpReg = MRI -> createVirtualRegister ( TII -> getRegClass ( MCID1 , <NUM_LIT> , TRI , MF ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ; if ( HasLane ) MIB . addImm ( LaneImm ) ; MIB . addImm ( Pred ) . addReg ( PredReg ) ; MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID2 ) . addReg ( DstReg , getDefRegState ( true ) | getDeadRegState ( DstDead ) ) ; if ( NegAcc ) {

bool AccKill = MRI -> hasOneNonDBGUse ( AccReg ) ;


<s> continue ; } unsigned Domain = MCID . TSFlags & <STR_LIT> ; if ( Domain == <STR_LIT> ) { if ( ++ Skip == <NUM_LIT> ) pushStack ( nullptr ) ; } else { Skip = <NUM_LIT> ; unsigned MulOpc , AddSubOpc ; bool NegAcc , HasLane ; if ( ! TII -> isFpMLxInstruction ( MCID . getOpcode ( ) , MulOpc , AddSubOpc , NegAcc , HasLane ) || ! FindMLxHazard ( MI ) ) pushStack ( MI ) ; else { ExpandFPMLxInstruction ( MBB , MI , MulOpc , AddSubOpc , NegAcc , HasLane ) ; Changed = true ; } } }


<s> MachineInstr * DefMI = MRI -> getVRegDef ( Reg ) ; while ( true ) { if ( DefMI -> getParent ( ) != MBB ) break ; if ( DefMI -> isCopyLike ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ; continue ; } } else if ( DefMI -> isInsertSubreg ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ;


<s> Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isPhysicalRegister ( Reg ) ) return nullptr ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * DefMI = MRI -> getVRegDef ( Reg ) ; while ( true ) { if ( DefMI -> getParent ( ) != MBB ) break ; if ( DefMI -> isCopyLike ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ;

if ( Register :: isVirtualRegister ( Reg ) ) {


<s> if ( UseMI -> getParent ( ) != MBB ) return Reg ; while ( UseMI -> isCopy ( ) || UseMI -> isInsertSubreg ( ) ) { Reg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isPhysicalRegister ( Reg ) || ! MRI -> hasOneNonDBGUse ( Reg ) ) return Reg ; UseMI = & * MRI -> use_instr_nodbg_begin ( Reg ) ; if ( UseMI -> getParent ( ) != MBB ) return Reg ;


<s> unsigned MLxExpansion :: getDefReg ( MachineInstr * MI ) const { Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isPhysicalRegister ( Reg ) || ! MRI -> hasOneNonDBGUse ( Reg ) ) return Reg ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * UseMI = & * MRI -> use_instr_nodbg_begin ( Reg ) ;

if ( UseMI -> getParent ( ) != MBB ) return Reg ;


<s> } } } } else if ( DefMI -> isCopyLike ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ; continue ; } } else if ( DefMI -> isInsertSubreg ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> if ( DefMI -> getParent ( ) != MBB ) break ; if ( DefMI -> isPHI ( ) ) { for ( unsigned i = <NUM_LIT> , e = DefMI -> getNumOperands ( ) ; i < e ; i += <NUM_LIT> ) { if ( DefMI -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) == MBB ) { Register SrcReg = DefMI -> getOperand ( i ) . getReg ( ) ; if ( Register :: isVirtualRegister ( SrcReg ) ) { DefMI = MRI -> getVRegDef ( SrcReg ) ; goto outer_continue ; } } } } else if ( DefMI -> isCopyLike ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ; continue ; } } else if ( DefMI -> isInsertSubreg ( ) ) { Reg = DefMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isVirtualRegister ( Reg ) ) { DefMI = MRI -> getVRegDef ( Reg ) ; continue ; }


<s> unsigned AccReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src1Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src2Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool Src1Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool Src2Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; unsigned LaneImm = HasLane ? MI -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; unsigned NextOp = HasLane ? <NUM_LIT> : <NUM_LIT> ; <STR_LIT> Pred = ( <STR_LIT> ) MI -> getOperand ( NextOp ) . getImm ( ) ; unsigned PredReg = MI -> getOperand ( ++ NextOp ) . getReg ( ) ; const MCInstrDesc & MCID1 = TII -> get ( MulOpc ) ; const MCInstrDesc & MCID2 = TII -> get ( AddSubOpc ) ; const MachineFunction & MF = * MI -> getParent ( ) -> getParent ( ) ; unsigned TmpReg = MRI -> createVirtualRegister ( TII -> getRegClass ( MCID1 , <NUM_LIT> , TRI , MF ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ; if ( HasLane ) MIB . addImm ( LaneImm ) ; MIB . addImm ( Pred ) . addReg ( PredReg ) ; MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID2 ) . addReg ( DstReg , getDefRegState ( true ) | getDeadRegState ( DstDead ) ) ;


<s> <STR_LIT> Pred = ( <STR_LIT> ) MI -> getOperand ( NextOp ) . getImm ( ) ; unsigned PredReg = MI -> getOperand ( ++ NextOp ) . getReg ( ) ; const MCInstrDesc & MCID1 = TII -> get ( MulOpc ) ; const MCInstrDesc & MCID2 = TII -> get ( AddSubOpc ) ; const MachineFunction & MF = * MI -> getParent ( ) -> getParent ( ) ; unsigned TmpReg = MRI -> createVirtualRegister ( TII -> getRegClass ( MCID1 , <NUM_LIT> , TRI , MF ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ; if ( HasLane ) MIB . addImm ( LaneImm ) ; MIB . addImm ( Pred ) . addReg ( PredReg ) ; MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID2 ) . addReg ( DstReg , getDefRegState ( true ) | getDeadRegState ( DstDead ) ) ; if ( NegAcc ) { bool AccKill = MRI -> hasOneNonDBGUse ( AccReg ) ; MIB . addReg ( TmpReg , getKillRegState ( true ) ) . addReg ( AccReg , getKillRegState ( AccKill ) ) ; } else { MIB . addReg ( AccReg ) . addReg ( TmpReg , getKillRegState ( true ) ) ; } MIB . addImm ( Pred ) . addReg ( PredReg ) ; DEBUG ( { dbgs ( ) << <STR_LIT> << * MI ; dbgs ( ) << <STR_LIT> ; MachineBasicBlock :: iterator MII = MI ;


<s> bool DstDead = MI -> getOperand ( <NUM_LIT> ) . isDead ( ) ; unsigned AccReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src1Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src2Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool Src1Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool Src2Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; unsigned LaneImm = HasLane ? MI -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; unsigned NextOp = HasLane ? <NUM_LIT> : <NUM_LIT> ; <STR_LIT> Pred = ( <STR_LIT> ) MI -> getOperand ( NextOp ) . getImm ( ) ; unsigned PredReg = MI -> getOperand ( ++ NextOp ) . getReg ( ) ; const MCInstrDesc & MCID1 = TII -> get ( MulOpc ) ; const MCInstrDesc & MCID2 = TII -> get ( AddSubOpc ) ; const MachineFunction & MF = * MI -> getParent ( ) -> getParent ( ) ; unsigned TmpReg = MRI -> createVirtualRegister ( TII -> getRegClass ( MCID1 , <NUM_LIT> , TRI , MF ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ; if ( HasLane ) MIB . addImm ( LaneImm ) ;


<s> ++ MII ; continue ; } unsigned Domain = MCID . TSFlags & <STR_LIT> ; if ( Domain == <STR_LIT> ) { if ( ++ Skip == <NUM_LIT> ) pushStack ( <NUM_LIT> ) ; } else { Skip = <NUM_LIT> ; unsigned MulOpc , AddSubOpc ; bool NegAcc , HasLane ; if ( ! TII -> isFpMLxInstruction ( MCID . getOpcode ( ) , MulOpc , AddSubOpc , NegAcc , HasLane ) || ! FindMLxHazard ( MI ) ) pushStack ( MI ) ; else { ExpandFPMLxInstruction ( MBB , MI , MulOpc , AddSubOpc , NegAcc , HasLane ) ; E = MBB . rend ( ) ; Changed = true ; continue ; } } ++ MII ; } return Changed ;


<s> const MCInstrDesc & MCID = MI -> getDesc ( ) ; if ( MI -> isBarrier ( ) ) { clearStack ( ) ; Skip = <NUM_LIT> ; ++ MII ; continue ; } unsigned Domain = MCID . TSFlags & <STR_LIT> ; if ( Domain == <STR_LIT> ) { if ( ++ Skip == <NUM_LIT> ) pushStack ( <NUM_LIT> ) ; } else { Skip = <NUM_LIT> ; unsigned MulOpc , AddSubOpc ; bool NegAcc , HasLane ; if ( ! TII -> isFpMLxInstruction ( MCID . getOpcode ( ) , MulOpc , AddSubOpc , NegAcc , HasLane ) || ! FindMLxHazard ( MI ) ) pushStack ( MI ) ; else { ExpandFPMLxInstruction ( MBB , MI , MulOpc , AddSubOpc , NegAcc , HasLane ) ;


<s> TRI = Fn . getTarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const <ISA_LIT>Subtarget * STI = & Fn . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ;


<s> unsigned TmpReg = MRI -> createVirtualRegister ( TII -> getRegClass ( MCID1 , <NUM_LIT> , TRI ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ; if ( HasLane ) MIB . addImm ( LaneImm ) ; MIB . addImm ( Pred ) . addReg ( PredReg ) ; MIB = BuildMI ( MBB , MI , MI -> getDebugLoc ( ) , MCID2 ) . addReg ( DstReg , getDefRegState ( true ) | getDeadRegState ( DstDead ) ) ; if ( NegAcc ) { bool AccKill = MRI -> hasOneNonDBGUse ( AccReg ) ; MIB . addReg ( TmpReg , getKillRegState ( true ) ) . addReg ( AccReg , getKillRegState ( AccKill ) ) ; } else { MIB . addReg ( AccReg ) . addReg ( TmpReg , getKillRegState ( true ) ) ; } MIB . addImm ( Pred ) . addReg ( PredReg ) ; DEBUG ( { dbgs ( ) << <STR_LIT> << * MI ; dbgs ( ) << <STR_LIT> ; MachineBasicBlock :: iterator MII = MI ; MII = llvm :: prior ( MII ) ; MachineInstr & MI2 = * MII ; MII = llvm :: prior ( MII ) ; MachineInstr & MI1 = * MII ; dbgs ( ) << <STR_LIT> << MI1 ;


<s> if ( ForceExapnd ) return true ; MachineInstr * DefMI = getAccDefMI ( MI ) ; if ( TII -> isFpMLxInstruction ( DefMI -> getOpcode ( ) ) ) { IgnoreStall . insert ( DefMI ) ; return true ; } if ( IgnoreStall . count ( MI ) ) return false ; unsigned Limit1 = isA9 ? <NUM_LIT> : <NUM_LIT> ; unsigned Limit2 = isA9 ? <NUM_LIT> : <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i <= <NUM_LIT> ; ++ i ) { int Idx = ( ( int ) MIIdx - i + <NUM_LIT> ) % <NUM_LIT> ; MachineInstr * NextMI = LastMIs [ Idx ] ; if ( ! NextMI ) continue ;

if ( TII -> canCauseFpMLxStall ( NextMI -> getOpcode ( ) ) ) {


<s> MachineInstr * DefMI = getAccDefMI ( MI ) ; if ( TII -> isFpMLxInstruction ( DefMI -> getOpcode ( ) ) ) { IgnoreStall . insert ( DefMI ) ; return true ; } if ( IgnoreStall . count ( MI ) ) return false ; unsigned Limit1 = isA9 ? <NUM_LIT> : <NUM_LIT> ;


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( Fn . getTarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getTarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const <ISA_LIT>Subtarget * STI = & Fn . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) ; isA9 = STI -> isCortexA9 ( ) ; bool Modified = false ;

for ( MachineFunction :: iterator MFI = Fn . begin ( ) , E = Fn . end ( ) ; MFI != E ;


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( Fn . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getSubtarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const <ISA_LIT>Subtarget * STI = & Fn . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ;


<s> const <ISA_LIT>Subtarget * STI = & Fn . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! STI -> isCortexA9 ( ) ) return false ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ;

for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ;


<s> unsigned AccReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src1Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src2Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool Src1Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool Src2Kill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; unsigned LaneImm = HasLane ? MI -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; unsigned NextOp = HasLane ? <NUM_LIT> : <NUM_LIT> ; <STR_LIT> Pred = ( <STR_LIT> ) MI -> getOperand ( NextOp ) . getImm ( ) ; unsigned PredReg = MI -> getOperand ( ++ NextOp ) . getReg ( ) ; const MCInstrDesc & MCID1 = TII -> get ( MulOpc ) ; const MCInstrDesc & MCID2 = TII -> get ( AddSubOpc ) ; unsigned TmpReg = MRI -> createVirtualRegister ( TII -> getRegClass ( MCID1 , <NUM_LIT> , TRI ) ) ; MachineInstrBuilder MIB = BuildMI ( MBB , * MI , MI -> getDebugLoc ( ) , MCID1 , TmpReg ) . addReg ( Src1Reg , getKillRegState ( Src1Kill ) ) . addReg ( Src2Reg , getKillRegState ( Src2Kill ) ) ;

if ( HasLane ) MIB . addImm ( LaneImm ) ;


<s> if ( Domain == <STR_LIT> ) { if ( ++ Skip == <NUM_LIT> ) pushStack ( <NUM_LIT> ) ; } else { Skip = <NUM_LIT> ; unsigned MulOpc , AddSubOpc ; bool NegAcc , HasLane ; if ( ! TII -> isFpMLxInstruction ( MCID . getOpcode ( ) , MulOpc , AddSubOpc , NegAcc , HasLane ) || ! FindMLxHazard ( MI ) ) pushStack ( MI ) ; else { ExpandFPMLxInstruction ( MBB , MI , MulOpc , AddSubOpc , NegAcc , HasLane ) ; E = MBB . rend ( ) ; Changed = true ; continue ; } }


<s> Skip = <NUM_LIT> ; ++ MII ; continue ; } unsigned Domain = MCID . TSFlags & <STR_LIT> ; if ( Domain == <STR_LIT> ) { if ( ++ Skip == <NUM_LIT> ) pushStack ( <NUM_LIT> ) ; } else { Skip = <NUM_LIT> ; unsigned MulOpc , AddSubOpc ; bool NegAcc , HasLane ; if ( ! TII -> isFpMLxInstruction ( MCID . getOpcode ( ) , MulOpc , AddSubOpc , NegAcc , HasLane ) || ! FindMLxHazard ( MI ) ) pushStack ( MI ) ; else {

ExpandFPMLxInstruction ( MBB , MI , MulOpc , AddSubOpc , NegAcc , HasLane ) ;


<s> unsigned Domain = MCID . TSFlags & <STR_LIT> ; if ( MCID . mayStore ( ) ) return false ; unsigned Opcode = MCID . getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return false ; if ( ( Domain & <STR_LIT> ) || ( Domain & <STR_LIT> ) ) return MI -> readsRegister ( Reg , TRI ) ; return false ;


<s> isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ;


<s> bool MLxExpansion :: hasRAWHazard ( unsigned Reg , MachineInstr * MI ) const { const TargetInstrDesc & TID = MI -> getDesc ( ) ; unsigned Domain = TID . TSFlags & <STR_LIT> ; if ( TID . mayStore ( ) ) return false ; unsigned Opcode = TID . getOpcode ( ) ;

if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return false ;


<s> ConstantInt * Const ; if ( ( Const = dyn_cast < ConstantInt > ( NonVectorVal ) ) && VT -> getElementType ( ) != NonVectorVal -> getType ( ) ) { unsigned TargetElemSize = VT -> getElementType ( ) -> getPrimitiveSizeInBits ( ) ; uint64_t N = Const -> getZExtValue ( ) ; if ( N < ( unsigned ) ( <NUM_LIT> << ( TargetElemSize - <NUM_LIT> ) ) ) { NonVectorVal = Builder . CreateVectorSplat ( VT -> getNumElements ( ) , Builder . getIntN ( TargetElemSize , N ) ) ; return ; } } NonVectorVal = Builder . CreateVectorSplat ( VT -> getNumElements ( ) , NonVectorVal ) ; } ; FixedVectorType * XElType = dyn_cast < FixedVectorType > ( X -> getType ( ) ) ; FixedVectorType * YElType = dyn_cast < FixedVectorType > ( Y -> getType ( ) ) ; if ( XElType && ! YElType ) { FixSummands ( XElType , Y ) ; YElType = cast < FixedVectorType > ( Y -> getType ( ) ) ; } else if ( YElType && ! XElType ) { FixSummands ( YElType , X ) ; XElType = cast < FixedVectorType > ( X -> getType ( ) ) ; } assert ( XElType && YElType && <STR_LIT> ) ; if ( XElType != YElType ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } if ( XElType -> getElementType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) {

Constant * ConstX = dyn_cast < Constant > ( X ) ;


<s> } ; if ( isa < FixedVectorType > ( ConstOff -> getType ( ) ) ) { for ( unsigned i = <NUM_LIT> ; i < TargetElemCount ; i ++ ) { if ( ! CheckValueSize ( ConstOff -> getAggregateElement ( i ) ) ) return false ; } } else { if ( ! CheckValueSize ( ConstOff ) ) return false ; } } return true ;


<s> auto CheckValueSize = [ TargetElemMaxSize ] ( Value * OffsetElem ) { ConstantInt * OConst = dyn_cast < ConstantInt > ( OffsetElem ) ; if ( ! OConst ) return false ; int SExtValue = OConst -> getSExtValue ( ) ; if ( SExtValue >= TargetElemMaxSize || SExtValue < <NUM_LIT> ) return false ; return true ; } ; if ( isa < FixedVectorType > ( ConstOff -> getType ( ) ) ) { for ( unsigned i = <NUM_LIT> ; i < TargetElemCount ; i ++ ) {

if ( ! CheckValueSize ( ConstOff -> getAggregateElement ( i ) ) ) return false ;


<s> else if ( GEPElemSize == <NUM_LIT> ) return <NUM_LIT> ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> else if ( GEPElemSize == <NUM_LIT> && MemoryElemSize == <NUM_LIT> ) return <NUM_LIT> ; else if ( GEPElemSize == <NUM_LIT> ) return <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return - <NUM_LIT> ;


<s> Pass * llvm :: createMVEGatherScatterLoweringPass ( ) {

return new MVEGatherScatterLowering ( ) ;


<s> ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ; if ( ZextOffs ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; FixedVectorType * OffsetType = cast < FixedVectorType > ( Offsets -> getType ( ) ) ; if ( ! ZextOffs || cast < FixedVectorType > ( ZextOffs -> getDestTy ( ) ) -> getElementType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) if ( ! checkOffsetSize ( Offsets , OffsetsElemCount ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) { if ( ( Ty -> getElementType ( ) -> getScalarSizeInBits ( ) < OffsetType -> getElementType ( ) -> getScalarSizeInBits ( ) ) ) {

Offsets = Builder . CreateTrunc ( Offsets , Ty ) ;


<s> if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Offsets = GEP -> getOperand ( <NUM_LIT> ) ; unsigned OffsetsElemCount = cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ; assert ( Ty -> getNumElements ( ) == OffsetsElemCount ) ; ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ; if ( ZextOffs ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; FixedVectorType * OffsetType = cast < FixedVectorType > ( Offsets -> getType ( ) ) ; if ( ! ZextOffs || cast < FixedVectorType > ( ZextOffs -> getDestTy ( ) ) -> getElementType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) if ( ! checkOffsetSize ( Offsets , OffsetsElemCount ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) {

if ( ( Ty -> getElementType ( ) -> getScalarSizeInBits ( ) < OffsetType -> getElementType ( ) -> getScalarSizeInBits ( ) ) ) {


<s> Scale = computeScale ( GEP -> getSourceElementType ( ) -> getPrimitiveSizeInBits ( ) , MemoryTy -> getScalarSizeInBits ( ) ) ; return Scale == - <NUM_LIT> ? nullptr : V ; } } FixedVectorType * PtrTy = cast < FixedVectorType > ( Ptr -> getType ( ) ) ; if ( PtrTy -> getNumElements ( ) != <NUM_LIT> || MemoryTy -> getScalarSizeInBits ( ) == <NUM_LIT> ) return nullptr ; Value * Zero = ConstantInt :: get ( Builder . getInt32Ty ( ) , <NUM_LIT> ) ; Value * BasePtr = Builder . CreateIntToPtr ( Zero , Builder . getInt8PtrTy ( ) ) ; Offsets = Builder . CreatePtrToInt ( Ptr , FixedVectorType :: get ( Builder . getInt32Ty ( ) , <NUM_LIT> ) ) ; Scale = <NUM_LIT> ; return BasePtr ;


<s> } FixedVectorType * PtrTy = cast < FixedVectorType > ( Ptr -> getType ( ) ) ; if ( PtrTy -> getNumElements ( ) != <NUM_LIT> || MemoryTy -> getScalarSizeInBits ( ) == <NUM_LIT> ) return nullptr ; Value * Zero = ConstantInt :: get ( Builder . getInt32Ty ( ) , <NUM_LIT> ) ; Value * BasePtr = Builder . CreateIntToPtr ( Zero , Builder . getInt8PtrTy ( ) ) ;

Offsets = Builder . CreatePtrToInt ( Ptr , FixedVectorType :: get ( Builder . getInt32Ty ( ) , <NUM_LIT> ) ) ;


<s> Value * BaseBasePtr = foldGEP ( BaseGEP , Offsets , Builder ) ; if ( ! BaseBasePtr ) return nullptr ; Offsets = CheckAndCreateOffsetAdd ( Offsets , GEP -> getOperand ( <NUM_LIT> ) , GEP , Builder ) ; if ( Offsets == nullptr ) return nullptr ; return BaseBasePtr ; }


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ; AU . addRequired < TargetPassConfig > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ;

FunctionPass :: getAnalysisUsage ( AU ) ;


<s> const Constant * C = dyn_cast < Constant > ( V ) ; if ( C && C -> getSplatValue ( ) ) return Optional < int64_t > { C -> getUniqueInteger ( ) . getSExtValue ( ) } ; if ( ! isa < Instruction > ( V ) ) return Optional < int64_t > { } ; const Instruction * I = cast < Instruction > ( V ) ; if ( I -> getOpcode ( ) == Instruction :: Add || I -> getOpcode ( ) == Instruction :: Or || I -> getOpcode ( ) == Instruction :: Mul || I -> getOpcode ( ) == Instruction :: Shl ) { Optional < int64_t > Op0 = getIfConst ( I -> getOperand ( <NUM_LIT> ) ) ; Optional < int64_t > Op1 = getIfConst ( I -> getOperand ( <NUM_LIT> ) ) ; if ( ! Op0 || ! Op1 ) return Optional < int64_t > { } ; if ( I -> getOpcode ( ) == Instruction :: Add ) return Optional < int64_t > { Op0 . getValue ( ) + Op1 . getValue ( ) } ;

if ( I -> getOpcode ( ) == Instruction :: Mul ) return Optional < int64_t > { Op0 . getValue ( ) * Op1 . getValue ( ) } ;


<s> if ( C && C -> getSplatValue ( ) ) return Optional < int64_t > { C -> getUniqueInteger ( ) . getSExtValue ( ) } ; if ( ! isa < Instruction > ( V ) ) return Optional < int64_t > { } ; const Instruction * I = cast < Instruction > ( V ) ; if ( I -> getOpcode ( ) == Instruction :: Add || I -> getOpcode ( ) == Instruction :: Or || I -> getOpcode ( ) == Instruction :: Mul || I -> getOpcode ( ) == Instruction :: Shl ) { Optional < int64_t > Op0 = getIfConst ( I -> getOperand ( <NUM_LIT> ) ) ;

Optional < int64_t > Op1 = getIfConst ( I -> getOperand ( <NUM_LIT> ) ) ;


<s> StringRef getPassName ( ) const override {


<s> Value * Summand ; Optional < int64_t > Const ; if ( ( Const = getIfConst ( Add -> getOperand ( <NUM_LIT> ) ) ) ) Summand = Add -> getOperand ( <NUM_LIT> ) ; else if ( ( Const = getIfConst ( Add -> getOperand ( <NUM_LIT> ) ) ) ) Summand = Add -> getOperand ( <NUM_LIT> ) ; else return ReturnFalse ; int64_t Immediate = Const . getValue ( ) << TypeScale ;

if ( Immediate > <NUM_LIT> || Immediate < - <NUM_LIT> || Immediate % <NUM_LIT> != <NUM_LIT> ) return ReturnFalse ;


<s> Optional < int64_t > Const ; if ( ( Const = getIfConst ( Add -> getOperand ( <NUM_LIT> ) ) ) ) Summand = Add -> getOperand ( <NUM_LIT> ) ; else if ( ( Const = getIfConst ( Add -> getOperand ( <NUM_LIT> ) ) ) ) Summand = Add -> getOperand ( <NUM_LIT> ) ; else return ReturnFalse ;

int64_t Immediate = Const . getValue ( ) << TypeScale ;


<s> return false ; } bool Gatscat = true ; for ( User * U : I -> users ( ) ) { if ( ! isa < Instruction > ( U ) ) return false ; if ( isa < GetElementPtrInst > ( U ) || isGatherScatter ( dyn_cast < IntrinsicInst > ( U ) ) ) { return Gatscat ; } else { unsigned OpCode = cast < Instruction > ( U ) -> getOpcode ( ) ;

if ( ( OpCode == Instruction :: Add || OpCode == Instruction :: Mul || OpCode == Instruction :: Shl || isAddLikeOr ( cast < Instruction > ( U ) , DL ) ) && hasAllGatScatUsers ( cast < Instruction > ( U ) , DL ) ) {


<s> } bool Gatscat = true ; for ( User * U : I -> users ( ) ) { if ( ! isa < Instruction > ( U ) ) return false ; if ( isa < GetElementPtrInst > ( U ) || isGatherScatter ( dyn_cast < IntrinsicInst > ( U ) ) ) { return Gatscat ; } else {

unsigned OpCode = cast < Instruction > ( U ) -> getOpcode ( ) ;


<s> return I -> getOpcode ( ) == Instruction :: Or && haveNoCommonBitsSet ( I -> getOperand ( <NUM_LIT> ) , I -> getOperand ( <NUM_LIT> ) , DL ) ;


<s> if ( ( ( NumElements == <NUM_LIT> && ( ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> ) ) || ( NumElements == <NUM_LIT> && ( ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> ) ) || ( NumElements == <NUM_LIT> && ElemSize == <NUM_LIT> ) ) && Alignment >= ElemSize / <NUM_LIT> ) return true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> if ( BCTy -> getNumElements ( ) == BCSrcTy -> getNumElements ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Ptr = BitCast -> getOperand ( <NUM_LIT> ) ; } }


<s> lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Instruction * Load = tryCreateIncrementingGatScat ( I , Ptr , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ;

if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) {


<s> Instruction * Load = tryCreateIncrementingGatScat ( I , Ptr , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = SelectInst :: Create ( Mask , Load , PassThru ) ; Builder . Insert ( Load ) ; } Root -> replaceAllUsesWith ( Load ) ;


<s> auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Store = tryCreateIncrementingGatScat ( I , Ptr , Builder ) ; if ( ! Store ) Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ;

if ( ! Store ) Store = tryCreateMaskedScatterBase ( I , Ptr , Builder ) ;


<s> Align Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getAlignValue ( ) ; auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ;

Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ;


<s> initializeMVEGatherScatterLoweringPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> Builder . SetInsertPoint ( GEP ) ; Builder . SetCurrentDebugLocation ( GEP -> getDebugLoc ( ) ) ; Value * Offsets ; Value * Base = foldGEP ( GEP , Offsets , Builder ) ; if ( Offsets && Base && Base != GEP ) { GetElementPtrInst * NewAddress = GetElementPtrInst :: Create ( GEP -> getSourceElementType ( ) , Base , Offsets , <STR_LIT> , GEP ) ; GEP -> replaceAllUsesWith ( NewAddress ) ; GEP = NewAddress ; Changed = true ; } } Changed |= optimiseOffsets ( GEP -> getOperand ( <NUM_LIT> ) , GEP -> getParent ( ) , LI ) ; return Changed ;


<s> Phi = cast < PHINode > ( Offs -> getOperand ( <NUM_LIT> ) ) ; OffsSecondOp = <NUM_LIT> ; } else if ( isa < PHINode > ( Offs -> getOperand ( <NUM_LIT> ) ) ) { Phi = cast < PHINode > ( Offs -> getOperand ( <NUM_LIT> ) ) ; OffsSecondOp = <NUM_LIT> ; } else { return false ; } } if ( Phi -> getParent ( ) != L -> getHeader ( ) ) return false ; BinaryOperator * IncInstruction ; Value * Start , * IncrementPerRound ; if ( ! matchSimpleRecurrence ( Phi , IncInstruction , Start , IncrementPerRound ) || IncInstruction -> getOpcode ( ) != Instruction :: Add ) return false ; int IncrementingBlock = Phi -> getIncomingValue ( <NUM_LIT> ) == IncInstruction ? <NUM_LIT> : <NUM_LIT> ; Value * OffsSecondOperand = Offs -> getOperand ( OffsSecondOp ) ; if ( IncrementPerRound -> getType ( ) != OffsSecondOperand -> getType ( ) || ! L -> isLoopInvariant ( OffsSecondOperand ) ) return false ; if ( ! isa < Constant > ( IncrementPerRound ) && ! ( isa < Instruction > ( IncrementPerRound ) && ! L -> contains ( cast < Instruction > ( IncrementPerRound ) ) ) ) return false ; PHINode * NewPhi ; if ( Phi -> getNumUses ( ) == <NUM_LIT> ) { if ( IncInstruction -> getNumUses ( ) != <NUM_LIT> ) { IncInstruction = BinaryOperator :: Create ( Instruction :: BinaryOps ( IncInstruction -> getOpcode ( ) ) , Phi , IncrementPerRound , <STR_LIT> , IncInstruction ) ; Phi -> setIncomingValue ( IncrementingBlock , IncInstruction ) ; } NewPhi = Phi ; } else { NewPhi = PHINode :: Create ( Phi -> getType ( ) , <NUM_LIT> , <STR_LIT> , Phi ) ; NewPhi -> addIncoming ( Phi -> getIncomingValue ( IncrementingBlock == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , Phi -> getIncomingBlock ( IncrementingBlock == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) ) ; IncInstruction = BinaryOperator :: Create ( Instruction :: BinaryOps ( IncInstruction -> getOpcode ( ) ) , NewPhi , IncrementPerRound , <STR_LIT> , IncInstruction ) ; NewPhi -> addIncoming ( IncInstruction , Phi -> getIncomingBlock ( IncrementingBlock ) ) ; IncrementingBlock = <NUM_LIT> ; } IRBuilder < > Builder ( BB -> getContext ( ) ) ; Builder . SetInsertPoint ( Phi ) ; Builder . SetCurrentDebugLocation ( Offs -> getDebugLoc ( ) ) ; switch ( Offs -> getOpcode ( ) ) { case Instruction :: Add : case Instruction :: Or :

pushOutAdd ( NewPhi , OffsSecondOperand , IncrementingBlock == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) ;


<s> if ( IncInstruction -> getNumUses ( ) != <NUM_LIT> ) { IncInstruction = BinaryOperator :: Create ( Instruction :: BinaryOps ( IncInstruction -> getOpcode ( ) ) , Phi , IncrementPerRound , <STR_LIT> , IncInstruction ) ; Phi -> setIncomingValue ( IncrementingBlock , IncInstruction ) ; } NewPhi = Phi ; } else { NewPhi = PHINode :: Create ( Phi -> getType ( ) , <NUM_LIT> , <STR_LIT> , Phi ) ; NewPhi -> addIncoming ( Phi -> getIncomingValue ( IncrementingBlock == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , Phi -> getIncomingBlock ( IncrementingBlock == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) ) ; IncInstruction = BinaryOperator :: Create ( Instruction :: BinaryOps ( IncInstruction -> getOpcode ( ) ) , NewPhi , IncrementPerRound , <STR_LIT> , IncInstruction ) ; NewPhi -> addIncoming ( IncInstruction , Phi -> getIncomingBlock ( IncrementingBlock ) ) ; IncrementingBlock = <NUM_LIT> ; } IRBuilder < > Builder ( BB -> getContext ( ) ) ; Builder . SetInsertPoint ( Phi ) ; Builder . SetCurrentDebugLocation ( Offs -> getDebugLoc ( ) ) ; switch ( Offs -> getOpcode ( ) ) { case Instruction :: Add : case Instruction :: Or : pushOutAdd ( NewPhi , OffsSecondOperand , IncrementingBlock == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) ; break ; case Instruction :: Mul : case Instruction :: Shl : pushOutMulShl ( Offs -> getOpcode ( ) , NewPhi , IncrementPerRound , OffsSecondOperand , IncrementingBlock , Builder ) ; break ; default : return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> unsigned IncrementIndex = StartIndex == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; Phi -> addIncoming ( NewIndex , Phi -> getIncomingBlock ( StartIndex ) ) ;

Phi -> addIncoming ( Phi -> getIncomingValue ( IncrementIndex ) , Phi -> getIncomingBlock ( IncrementIndex ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Instruction * InsertionPoint = & cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) -> back ( ) ) ; Value * StartIndex = BinaryOperator :: Create ( ( Instruction :: BinaryOps ) Opcode , Phi -> getIncomingValue ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ; Instruction * Product = BinaryOperator :: Create ( ( Instruction :: BinaryOps ) Opcode , IncrementPerRound , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ; Instruction * NewIncrement = BinaryOperator :: Create ( Instruction :: Add , Phi , Product , <STR_LIT> , cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement ) -> back ( ) ) . getPrevNode ( ) ) ; Phi -> addIncoming ( StartIndex , Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) ) ; Phi -> addIncoming ( NewIncrement , Phi -> getIncomingBlock ( LoopIncrement ) ) ; Phi -> removeIncomingValue ( ( unsigned ) <NUM_LIT> ) ;


<s> Value * StartIndex = BinaryOperator :: Create ( ( Instruction :: BinaryOps ) Opcode , Phi -> getIncomingValue ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ; Instruction * Product = BinaryOperator :: Create ( ( Instruction :: BinaryOps ) Opcode , IncrementPerRound , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ; Instruction * NewIncrement = BinaryOperator :: Create ( Instruction :: Add , Phi , Product , <STR_LIT> , cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement ) -> back ( ) ) . getPrevNode ( ) ) ; Phi -> addIncoming ( StartIndex , Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) ) ; Phi -> addIncoming ( NewIncrement , Phi -> getIncomingBlock ( LoopIncrement ) ) ; Phi -> removeIncomingValue ( ( unsigned ) <NUM_LIT> ) ;


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; DL = & F . getParent ( ) -> getDataLayout ( ) ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { Changed |= SimplifyInstructionsInBlock ( & BB ) ; for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ;

Changed |= optimiseAddress ( II -> getArgOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ;


<s> Changed |= SimplifyInstructionsInBlock ( & BB ) ; for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == <STR_LIT> && isa < FixedVectorType > ( II -> getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ) { Scatters . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } } } for ( unsigned i = <NUM_LIT> ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Instruction * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( L -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = <NUM_LIT> ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Instruction * S = lowerScatter ( I ) ;


<s> Loop * L = LI -> getLoopFor ( I -> getParent ( ) ) ; if ( L == nullptr ) return nullptr ; GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; Value * Offsets ; Value * BasePtr = decomposeGEP ( Offsets , Ty , GEP , Builder ) ; if ( ! BasePtr ) return nullptr ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; int TypeScale = computeScale ( DL -> getTypeSizeInBits ( GEP -> getOperand ( <NUM_LIT> ) -> getType ( ) ) , DL -> getTypeSizeInBits ( GEP -> getType ( ) ) / cast < FixedVectorType > ( GEP -> getType ( ) ) -> getNumElements ( ) ) ;

if ( TypeScale == - <NUM_LIT> ) return nullptr ;


<s> } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; std :: pair < Value * , int64_t > Add = getVarAndConst ( Offsets , TypeScale ) ; if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , OffsetsIncoming , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , I ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , I ) ;

if ( I -> getIntrinsicID ( ) == <STR_LIT> ) return tryCreateMaskedGatherBase ( I , OffsetsIncoming , Builder , Immediate ) ;