<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : if ( CxtI && CxtI -> hasOneUse ( ) ) if ( const auto * FAdd = dyn_cast < BinaryOperator > ( * CxtI -> user_begin ( ) ) ) { const int OPC = TLI -> InstructionOpcodeToISD ( FAdd -> getOpcode ( ) ) ; if ( OPC == <STR_LIT> || OPC == <STR_LIT> ) { if ( ST -> hasMadMacF32Insts ( ) && SLT == <STR_LIT> && ! HasFP32Denormals ) return TargetTransformInfo :: TCC_Free ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> && ! HasFP64FP16Denormals ) return TargetTransformInfo :: TCC_Free ; const TargetOptions & Options = TLI -> getTargetMachine ( ) . Options ; if ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( FAdd -> hasAllowContract ( ) && CxtI -> hasAllowContract ( ) ) ) return TargetTransformInfo :: TCC_Free ; } } LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ;

if ( ! ST -> hasUsableDivScaleConditionOutput ( ) ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ;


<s> if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , Ty , IsPairwise ) ;

std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ;


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ; if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , Ty , IsPairwise ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ;

return LT . first * getFullRateInstrCost ( ) ;


<s> if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; unsigned InstRate = getQuarterRateInstrCost ( ) ; if ( ID == <STR_LIT> ) { InstRate = ST -> hasFastFMAF32 ( ) ? getHalfRateInstrCost ( ) : getQuarterRateInstrCost ( ) ; } return LT . first * NElts * InstRate ;


<s> if ( MinAlign == <NUM_LIT> ) return Type :: getInt16Ty ( Context ) ;

if ( SrcAddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAddrSpace == <ISA_LIT>AS :: REGION_ADDRESS || DestAddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) {


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ; if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getMinMaxReductionCost ( Ty , CondTy , IsPairwise , IsUnsigned ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ;

return LT . first * getHalfRateInstrCost ( ) ;


<s> switch ( Kind ) { case TTI :: SK_Broadcast : case TTI :: SK_Reverse : case TTI :: SK_PermuteSingleSrc : return <NUM_LIT> ; default : break ; } } } return BaseT :: getShuffleCost ( Kind , VT , Index , SubTp ) ;


<s> } else { return BaseT :: getUserCost ( U , Operands , CostKind ) ; } } case Instruction :: ShuffleVector : { const ShuffleVectorInst * Shuffle = cast < ShuffleVectorInst > ( I ) ; auto * Ty = cast < VectorType > ( Shuffle -> getType ( ) ) ; auto * SrcTy = cast < VectorType > ( Shuffle -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; int SubIndex ; if ( Shuffle -> isExtractSubvectorMask ( SubIndex ) ) return getShuffleCost ( TTI :: SK_ExtractSubvector , SrcTy , SubIndex , Ty ) ; if ( Shuffle -> changesLength ( ) ) return BaseT :: getUserCost ( U , Operands , CostKind ) ; if ( Shuffle -> isIdentity ( ) ) return <NUM_LIT> ; if ( Shuffle -> isReverse ( ) ) return getShuffleCost ( TTI :: SK_Reverse , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isSelect ( ) ) return getShuffleCost ( TTI :: SK_Select , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isTranspose ( ) ) return getShuffleCost ( TTI :: SK_Transpose , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isZeroEltSplat ( ) ) return getShuffleCost ( TTI :: SK_Broadcast , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isSingleSource ( ) ) return getShuffleCost ( TTI :: SK_PermuteSingleSrc , Ty , <NUM_LIT> , nullptr ) ; return getShuffleCost ( TTI :: SK_PermuteTwoSrc , Ty , <NUM_LIT> , nullptr ) ; } case Instruction :: ZExt : case Instruction :: SExt : case Instruction :: FPToUI : case Instruction :: FPTo<ISA_LIT> : case Instruction :: FPExt :


<s> const ShuffleVectorInst * Shuffle = cast < ShuffleVectorInst > ( I ) ; auto * Ty = cast < VectorType > ( Shuffle -> getType ( ) ) ; auto * SrcTy = cast < VectorType > ( Shuffle -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; int SubIndex ; if ( Shuffle -> isExtractSubvectorMask ( SubIndex ) ) return getShuffleCost ( TTI :: SK_ExtractSubvector , SrcTy , SubIndex , Ty ) ; if ( Shuffle -> changesLength ( ) ) return BaseT :: getUserCost ( U , Operands , CostKind ) ; if ( Shuffle -> isIdentity ( ) ) return <NUM_LIT> ; if ( Shuffle -> isReverse ( ) ) return getShuffleCost ( TTI :: SK_Reverse , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isSelect ( ) ) return getShuffleCost ( TTI :: SK_Select , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isTranspose ( ) ) return getShuffleCost ( TTI :: SK_Transpose , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isZeroEltSplat ( ) ) return getShuffleCost ( TTI :: SK_Broadcast , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isSingleSource ( ) ) return getShuffleCost ( TTI :: SK_PermuteSingleSrc , Ty , <NUM_LIT> , nullptr ) ; return getShuffleCost ( TTI :: SK_PermuteTwoSrc , Ty , <NUM_LIT> , nullptr ) ; } case Instruction :: ZExt : case Instruction :: SExt : case Instruction :: FPToUI : case Instruction :: FPTo<ISA_LIT> : case Instruction :: FPExt : case Instruction :: PtrToInt : case Instruction :: IntToPtr : case Instruction :: <ISA_LIT>ToFP : case Instruction :: UIToFP : case Instruction :: Trunc : case Instruction :: FPTrunc : case Instruction :: BitCast : case Instruction :: AddrSpaceCast : { return getCastInstrCost ( I -> getOpcode ( ) , I -> getType ( ) , I -> getOperand ( <NUM_LIT> ) -> getType ( ) , I ) ; } case Instruction :: Add : case Instruction :: FAdd : case Instruction :: Sub : case Instruction :: FSub : case Instruction :: Mul : case Instruction :: FMul :


<s> for ( const Instruction & I : * BB ) { if ( const BranchInst * Br = dyn_cast < BranchInst > ( & I ) ) { if ( UP . Threshold < MaxBoost && Br -> isConditional ( ) ) { if ( L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) || L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) ) continue ; if ( dependsOnLocalPhi ( L , Br -> getCondition ( ) ) ) { UP . Threshold += UnrollThresholdIf ; DEBUG ( dbgs ( ) << <STR_LIT> << UP . Threshold << <STR_LIT> << * L << <STR_LIT> << * Br << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } } continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == ASST . PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == ASST . LOCAL_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == ASST . PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == ASST . LOCAL_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; } bool HasLoopDef = false ; for ( const Value * Op : GEP -> operands ( ) ) { const Instruction * Inst = dyn_cast < Instruction > ( Op ) ; if ( ! Inst || L -> isLoopInvariant ( Op ) ) continue ; if ( any_of ( L -> getSubLoops ( ) , [ Inst ] ( const Loop * SubLoop ) {

return SubLoop -> contains ( Inst ) ; } ) ) continue ;


<s> for ( const Instruction & I : * BB ) { if ( const BranchInst * Br = dyn_cast < BranchInst > ( & I ) ) { if ( UP . Threshold < MaxBoost && Br -> isConditional ( ) ) { if ( L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) || L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) ) continue ; if ( dependsOnLocalPhi ( L , Br -> getCondition ( ) ) ) { UP . Threshold += UnrollThresholdIf ; DEBUG ( dbgs ( ) << <STR_LIT> << UP . Threshold << <STR_LIT> << * L << <STR_LIT> << * Br << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } } continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == ASST . PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == ASST . LOCAL_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == ASST . PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == ASST . LOCAL_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; } bool HasLoopDef = false ; for ( const Value * Op : GEP -> operands ( ) ) { const Instruction * Inst = dyn_cast < Instruction > ( Op ) ; if ( ! Inst || L -> isLoopInvariant ( Op ) ) continue ; if ( any_of ( L -> getSubLoops ( ) , [ Inst ] ( const Loop * SubLoop ) { return SubLoop -> contains ( Inst ) ; } ) ) continue ; HasLoopDef = true ;


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ; if ( ! OrigTy . isSimple ( ) ) { return BaseT :: getArithmeticInstrCost ( Opcode , Ty , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo ) ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; switch ( ISD ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( SLT == <STR_LIT> ) return get64BitInstrCost ( ) * LT . first * NElts ; return getFullRateInstrCost ( ) * LT . first * NElts ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } return LT . first * NElts * getFullRateInstrCost ( ) ; } case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ST -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; return LT . first * Cost * NElts ; }

if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) {


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } StringRef Name = I -> getCalledFunction ( ) -> getName ( ) ; switch ( TII -> lookupName ( ( const char * ) Name . bytes_begin ( ) , Name . size ( ) ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; }


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ; if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , Ty , IsPairwise , CostKind ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ;


<s> assert ( ( I == nullptr || I -> getOpcode ( ) == Opcode ) && <STR_LIT> ) ; const bool SCost = ( CostKind == TTI :: TCK_CodeSize || CostKind == TTI :: TCK_SizeAndLatency ) ; const int CBrCost = SCost ? <NUM_LIT> : <NUM_LIT> ; switch ( Opcode ) { case Instruction :: Br : { auto BI = dyn_cast_or_null < BranchInst > ( I ) ; if ( BI && BI -> isUnconditional ( ) ) return SCost ? <NUM_LIT> : <NUM_LIT> ; return CBrCost ; } case Instruction :: Switch : { auto <ISA_LIT> = dyn_cast_or_null < SwitchInst > ( I ) ; return ( <ISA_LIT> ? ( <ISA_LIT> -> getNumCases ( ) + <NUM_LIT> ) : <NUM_LIT> ) * ( CBrCost + <NUM_LIT> ) ; } case Instruction :: Ret : return SCost ? <NUM_LIT> : <NUM_LIT> ; case Instruction :: PHI : return <NUM_LIT> ;


<s> case Instruction :: Switch : { auto <ISA_LIT> = dyn_cast_or_null < SwitchInst > ( I ) ; return ( <ISA_LIT> ? ( <ISA_LIT> -> getNumCases ( ) + <NUM_LIT> ) : <NUM_LIT> ) * ( CBrCost + <NUM_LIT> ) ; } case Instruction :: Ret : return SCost ? <NUM_LIT> : <NUM_LIT> ; case Instruction :: PHI : return <NUM_LIT> ;


<s> if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getMinMaxReductionCost ( Ty , CondTy , IsPairwise , IsUnsigned , CostKind ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; return LT . first * getHalfRateInstrCost ( CostKind ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } StringRef Name = I -> getCalledFunction ( ) -> getName ( ) ; switch ( TII -> lookupName ( ( const char * ) Name . bytes_begin ( ) , Name . size ( ) ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : return true ;


<s> case CallingConv :: <ISA_LIT>_CS : return F -> getAttributes ( ) . hasParamAttribute ( A -> getArgNo ( ) , Attribute :: InReg ) || F -> getAttributes ( ) . hasParamAttribute ( A -> getArgNo ( ) , Attribute :: ByVal ) ; default :


<s> case <STR_LIT> : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } StringRef Name = I -> getCalledFunction ( ) -> getName ( ) ; switch ( TII -> lookupName ( ( const char * ) Name . bytes_begin ( ) , Name . size ( ) ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : return true ; }


<s> unsigned ThresholdPrivate = UnrollThresholdPrivate ; unsigned ThresholdLocal = UnrollThresholdLocal ; unsigned MaxBoost = std :: max ( ThresholdPrivate , ThresholdLocal ) ; for ( const BasicBlock * BB : L -> getBlocks ( ) ) { const DataLayout & DL = BB -> getModule ( ) -> getDataLayout ( ) ; unsigned LocalGEPsSeen = <NUM_LIT> ; if ( llvm :: any_of ( L -> getSubLoops ( ) , [ BB ] ( const Loop * SubLoop ) { return SubLoop -> contains ( BB ) ; } ) ) continue ; for ( const Instruction & I : * BB ) { if ( const BranchInst * Br = dyn_cast < BranchInst > ( & I ) ) { if ( UP . Threshold < MaxBoost && Br -> isConditional ( ) ) { BasicBlock * Succ0 = Br -> getSuccessor ( <NUM_LIT> ) ; BasicBlock * Succ1 = Br -> getSuccessor ( <NUM_LIT> ) ; if ( ( L -> contains ( Succ0 ) && L -> isLoopExiting ( Succ0 ) ) || ( L -> contains ( Succ1 ) && L -> isLoopExiting ( Succ1 ) ) ) continue ; if ( dependsOnLocalPhi ( L , Br -> getCondition ( ) ) ) { UP . Threshold += UnrollThresholdIf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << UP . Threshold << <STR_LIT> << * L << <STR_LIT> << * Br << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } } continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ;

else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) Threshold = ThresholdLocal ;


<s> return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ! ST -> hasUsableDivScaleConditionOutput ( ) ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( ! Args . empty ( ) && match ( Args [ <NUM_LIT> ] , PatternMatch :: m_FPOne ( ) ) ) { if ( ( SLT == <STR_LIT> && ! HasFP32Denormals ) || ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) ) { return LT . first * getQuarterRateInstrCost ( ) * NElts ; } } if ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) { int Cost = <NUM_LIT> * getFullRateInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * getFullRateInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ! HasFP32Denormals ) { Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; } return LT . first * NElts * Cost ; } break ; default : break ; } return BaseT :: getArithmeticInstrCost ( Opcode , Ty , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo ) ;


<s> <STR_LIT> CalleeMode ( * Callee ) ; if ( ! CallerMode . isInlineCompatible ( CalleeMode ) ) return false ; if ( InlineMaxBB && ! Callee -> hasFnAttribute ( Attribute :: InlineHint ) ) { size_t BBSize = Caller -> size ( ) + Callee -> size ( ) - <NUM_LIT> ; return BBSize <= InlineMaxBB ; }


<s> if ( ( RealCallerBits & RealCalleeBits ) != RealCalleeBits ) return false ; <STR_LIT> CallerMode ( * Caller ) ; <STR_LIT> CalleeMode ( * Callee ) ; if ( ! CallerMode . isInlineCompatible ( CalleeMode ) ) return false ; if ( InlineMaxBB && ! Callee -> hasFnAttribute ( Attribute :: InlineHint ) ) { size_t BBSize = Caller -> size ( ) + Callee -> size ( ) - <NUM_LIT> ; return BBSize <= InlineMaxBB ; } return true ;


<s> return ST -> hasHalfRate64Ops ( ) ? getHalfRateInstrCost ( CostKind ) : getQuarterRateInstrCost ( CostKind ) ;


<s> const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == ASST . PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == ASST . LOCAL_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == ASST . PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == ASST . LOCAL_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; } bool HasLoopDef = false ; for ( const Value * Op : GEP -> operands ( ) ) {

const Instruction * Inst = dyn_cast < Instruction > ( Op ) ;


<s> static bool isIntrinsicSourceOfDivergence ( const TargetIntrinsicInfo * TII , const IntrinsicInst * I ) { switch ( I -> getIntrinsicID ( ) ) { default : return false ; case <STR_LIT> : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> bool GCNTTIImpl :: isSourceOfDivergence ( const Value * V ) const { if ( const Argument * A = dyn_cast < Argument > ( V ) ) return ! isArgPassedInSGPR ( A ) ; if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == ST -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ; if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ;

if ( isa < CallInst > ( V ) || isa < InvokeInst > ( V ) ) return true ;


<s> if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ; if ( isa < CallInst > ( V ) || isa < InvokeInst > ( V ) ) return true ;


<s> if ( ! OrigTy . isSimple ( ) ) { return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , RetTy ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; unsigned InstRate = getQuarterRateInstrCost ( ) ; if ( ICA . getID ( ) == <STR_LIT> ) {

InstRate = ST -> hasFastFMAF32 ( ) ? getHalfRateInstrCost ( ) : getQuarterRateInstrCost ( ) ;


<s> bool GCNTTIImpl :: collectFlatAddressOperands ( SmallVectorImpl < int > & OpIndexes , <STR_LIT> IID ) const { switch ( IID ) { case <STR_LIT> :


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; switch ( ISD ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return get64BitInstrCost ( ) * LT . first * NElts ; return getFullRateInstrCost ( ) * LT . first * NElts ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ! ST -> hasUsableDivScaleConditionOutput ( ) ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( ! Args . empty ( ) && match ( Args [ <NUM_LIT> ] , PatternMatch :: m_FPOne ( ) ) ) { if ( ( SLT == <STR_LIT> && ! HasFP32Denormals ) || ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) ) { return LT . first * getQuarterRateInstrCost ( ) * NElts ; } } if ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) { int Cost = <NUM_LIT> * getFullRateInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * getFullRateInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ! HasFP32Denormals ) { Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; }


<s> if ( AddrSpace == AS . GLOBAL_ADDRESS || AddrSpace == AS . CONSTANT_ADDRESS || AddrSpace == AS . CONSTANT_ADDRESS_32BIT || AddrSpace == AS . FLAT_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == AS . LOCAL_ADDRESS || AddrSpace == AS . REGION_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == AS . PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ; if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == AS . PARAM_D_ADDRESS || AddrSpace == AS . PARAM_I_ADDRESS || ( AddrSpace >= AS . CONSTANT_BUFFER_0 && AddrSpace <= AS . CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ;


<s> default : if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == <ISA_LIT>AS :: PARAM_D_ADDRESS || AddrSpace == <ISA_LIT>AS :: PARAM_I_ADDRESS || ( AddrSpace >= <ISA_LIT>AS :: CONSTANT_BUFFER_0 && AddrSpace <= <ISA_LIT>AS :: CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ;


<s> } } if ( const CallInst * CI = dyn_cast < CallInst > ( V ) ) { if ( isa < InlineAsm > ( CI -> getCalledValue ( ) ) ) return ! isInlineAsmSourceOfDivergence ( CI ) ; return false ; } const ExtractValueInst * ExtValue = dyn_cast < ExtractValueInst > ( V ) ; if ( ! ExtValue ) return false ; const CallInst * CI = dyn_cast < CallInst > ( ExtValue -> getOperand ( <NUM_LIT> ) ) ; if ( ! CI ) return false ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( CI ) ) { switch ( Intrinsic -> getIntrinsicID ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : { ArrayRef < unsigned > Indices = ExtValue -> getIndices ( ) ; return Indices . size ( ) == <NUM_LIT> && Indices [ <NUM_LIT> ] == <NUM_LIT> ; } } } if ( isa < InlineAsm > ( CI -> getCalledValue ( ) ) ) return ! isInlineAsmSourceOfDivergence ( CI , ExtValue -> getIndices ( ) ) ;


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , TargetTriple ( TM -> getTargetTriple ( ) ) {


<s> explicit GCNTTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const GCNSubtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) , IsGraphicsShader ( <STR_LIT> ( F . getCallingConv ( ) ) ) {


<s> unsigned getFlatAddressSpace ( ) const { if ( IsGraphicsShader ) return - <NUM_LIT> ;

return ST -> hasFlatAddressSpace ( ) ? <ISA_LIT>AS :: FLAT_ADDRESS : <ISA_LIT>AS :: UNKNOWN_ADDRESS_SPACE ;


<s> unsigned getFlatAddressSpace ( ) const { if ( IsGraphicsShader ) return - <NUM_LIT> ; return ST -> hasFlatAddressSpace ( ) ? <ISA_LIT>AS :: FLAT_ADDRESS : <ISA_LIT>AS :: UNKNOWN_ADDRESS_SPACE ;


<s> unsigned getInliningThresholdMultiplier ( ) {


<s> const <ISA_LIT>Subtarget * getST ( ) const {


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const <ISA_LIT>Subtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) {


<s> AU . addPreserved < DivergenceAnalysis > ( ) ; AU . addPreservedID ( BreakCriticalEdgesID ) ;


<s> while ( ! Stack . empty ( ) ) { BasicBlock * Top = Stack . pop_back_val ( ) ; if ( ! DA . isUniform ( Top -> getTerminator ( ) ) ) return false ; for ( BasicBlock * Pred : predecessors ( Top ) ) { if ( Visited . insert ( Pred ) . second ) Stack . push_back ( Pred ) ; } }


<s> if ( ! DA . isUniform ( Top -> getTerminator ( ) ) ) return false ; for ( BasicBlock * Pred : predecessors ( Top ) ) { if ( Visited . insert ( Pred ) . second ) Stack . push_back ( Pred ) ; } } return true ;


<s> if ( isa < ReturnInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) ReturningBlocks . push_back ( BB ) ; } else if ( isa < UnreachableInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) UnreachableBlocks . push_back ( BB ) ; } else if ( BranchInst * BI = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ) { ConstantInt * BoolTrue = ConstantInt :: getTrue ( F . getContext ( ) ) ; if ( DummyReturnBB == nullptr ) { DummyReturnBB = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , DummyReturnBB ) ; ReturningBlocks . push_back ( DummyReturnBB ) ; } if ( BI -> isUnconditional ( ) ) { BasicBlock * LoopHeaderBB = BI -> getSuccessor ( <NUM_LIT> ) ;


<s> ReturnInst :: Create ( F . getContext ( ) , PN , NewRetBlock ) ; } for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ;


<s> ReturnInst :: Create ( F . getContext ( ) , PN , NewRetBlock ) ; } for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ; } for ( BasicBlock * BB : ReturningBlocks ) { simplifyCFG ( BB , TTI , { <NUM_LIT> } ) ;


<s> } else { PN = B . CreatePHI ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ; assert ( ! InsertExport ) ; B . CreateRet ( PN ) ; } for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ; }


<s> B . getInt32 ( <STR_LIT> :: ET_NULL ) , B . getInt32 ( <NUM_LIT> ) , Undef , Undef , Undef , Undef , B . getTrue ( ) , B . getTrue ( ) , } ) ; } PHINode * PN = nullptr ; if ( F . getReturnType ( ) -> isVoidTy ( ) ) { B . CreateRetVoid ( ) ; } else { PN = B . CreatePHI ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ; assert ( ! InsertExport ) ; B . CreateRet ( PN ) ; } for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ; }


<s> BasicBlock * TransitionBB = BB -> splitBasicBlock ( BI , <STR_LIT> ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ; } } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; } } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ; ReturningBlocks . push_back ( UnreachableBlock ) ;


<s> if ( PDT . getRoots ( ) . size ( ) <= <NUM_LIT> ) return false ; DivergenceAnalysis & DA = getAnalysis < DivergenceAnalysis > ( ) ; SmallVector < BasicBlock * , <NUM_LIT> > ReturningBlocks ; SmallVector < BasicBlock * , <NUM_LIT> > UnreachableBlocks ; for ( BasicBlock * BB : PDT . getRoots ( ) ) { if ( isa < ReturnInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) ReturningBlocks . push_back ( BB ) ; } else if ( isa < UnreachableInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) UnreachableBlocks . push_back ( BB ) ; } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getInstList ( ) . pop_back ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ;


<s> PHINode * PN = nullptr ; if ( F . getReturnType ( ) -> isVoidTy ( ) ) { ReturnInst :: Create ( F . getContext ( ) , nullptr , NewRetBlock ) ; } else { PN = PHINode :: Create ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ; NewRetBlock -> getInstList ( ) . push_back ( PN ) ; ReturnInst :: Create ( F . getContext ( ) , PN , NewRetBlock ) ; } for ( BasicBlock * BB : ReturningBlocks ) {

if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ;


<s> if ( DummyReturnBB == nullptr ) { DummyReturnBB = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && RetTy -> isVoidTy ( ) ) { InsertExport = true ; } ReturnInst :: Create ( F . getContext ( ) , RetVal , DummyReturnBB ) ; ReturningBlocks . push_back ( DummyReturnBB ) ; } if ( BI -> isUnconditional ( ) ) { BasicBlock * LoopHeaderBB = BI -> getSuccessor ( <NUM_LIT> ) ; BI -> eraseFromParent ( ) ; BranchInst :: Create ( LoopHeaderBB , DummyReturnBB , BoolTrue , BB ) ; } else { BasicBlock * TransitionBB = BB -> splitBasicBlock ( BI , <STR_LIT> ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ; } } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; } } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ;

ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ;


<s> SmallVector < BasicBlock * , <NUM_LIT> > ReturningBlocks ; SmallVector < BasicBlock * , <NUM_LIT> > UnreachableBlocks ; BasicBlock * DummyReturnBB = nullptr ; bool InsertExport = false ; for ( BasicBlock * BB : PDT . getRoots ( ) ) { if ( isa < ReturnInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) ReturningBlocks . push_back ( BB ) ; } else if ( isa < UnreachableInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) UnreachableBlocks . push_back ( BB ) ; } else if ( BranchInst * BI = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ) { ConstantInt * BoolTrue = ConstantInt :: getTrue ( F . getContext ( ) ) ; if ( DummyReturnBB == nullptr ) { DummyReturnBB = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && RetTy -> isVoidTy ( ) ) { InsertExport = true ; } ReturnInst :: Create ( F . getContext ( ) , RetVal , DummyReturnBB ) ; ReturningBlocks . push_back ( DummyReturnBB ) ; } if ( BI -> isUnconditional ( ) ) { BasicBlock * LoopHeaderBB = BI -> getSuccessor ( <NUM_LIT> ) ; BI -> eraseFromParent ( ) ; BranchInst :: Create ( LoopHeaderBB , DummyReturnBB , BoolTrue , BB ) ; } else { BasicBlock * TransitionBB = BB -> splitBasicBlock ( BI , <STR_LIT> ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ; } } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ;


<s> UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; } } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ; ReturningBlocks . push_back ( UnreachableBlock ) ; } } if ( ReturningBlocks . empty ( ) ) return false ; if ( ReturningBlocks . size ( ) == <NUM_LIT> ) return false ; const TargetTransformInfo & TTI = getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ;

unifyReturnBlockSet ( F , ReturningBlocks , TTI , <STR_LIT> ) ;


<s> } else { BasicBlock * TransitionBB = BB -> splitBasicBlock ( BI , <STR_LIT> ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ; } } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; } } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ; ReturningBlocks . push_back ( UnreachableBlock ) ; } } if ( ReturningBlocks . empty ( ) ) return false ; if ( ReturningBlocks . size ( ) == <NUM_LIT> ) return false ; const TargetTransformInfo & TTI = getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; unifyReturnBlockSet ( F , ReturningBlocks , TTI , <STR_LIT> ) ; return true ;


<s> } Changed = true ; } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; } Changed = true ; } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ; ReturningBlocks . push_back ( UnreachableBlock ) ; Changed = true ; } } if ( ReturningBlocks . empty ( ) ) return Changed ; if ( ReturningBlocks . size ( ) == <NUM_LIT> && ! InsertExport ) return Changed ; const TargetTransformInfo & TTI = getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ;

auto BlocksToUnify = std :: move ( ReturningBlocks ) ;


<s> if ( ! isUniformlyReached ( DA , * BB ) ) ReturningBlocks . push_back ( BB ) ; else UniformlyReachedRetBlocks . push_back ( BB ) ; } else if ( isa < UnreachableInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) UnreachableBlocks . push_back ( BB ) ; } else if ( BranchInst * BI = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ) { ConstantInt * BoolTrue = ConstantInt :: getTrue ( F . getContext ( ) ) ; if ( DummyReturnBB == nullptr ) { DummyReturnBB = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && RetTy -> isVoidTy ( ) ) { InsertExport = true ; } ReturnInst :: Create ( F . getContext ( ) , RetVal , DummyReturnBB ) ; ReturningBlocks . push_back ( DummyReturnBB ) ; } if ( BI -> isUnconditional ( ) ) { BasicBlock * LoopHeaderBB = BI -> getSuccessor ( <NUM_LIT> ) ; BI -> eraseFromParent ( ) ; BranchInst :: Create ( LoopHeaderBB , DummyReturnBB , BoolTrue , BB ) ; } else { BasicBlock * TransitionBB = BB -> splitBasicBlock ( BI , <STR_LIT> ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ; } Changed = true ; } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; } Changed = true ; } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ; ReturningBlocks . push_back ( UnreachableBlock ) ; Changed = true ; } }

if ( ReturningBlocks . empty ( ) ) return Changed ;


<s> PHINode * PN = nullptr ; if ( F . getReturnType ( ) -> isVoidTy ( ) ) { B . CreateRetVoid ( ) ; } else { PN = B . CreatePHI ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ; assert ( ! InsertExport ) ; B . CreateRet ( PN ) ; } for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ; } for ( BasicBlock * BB : ReturningBlocks ) { simplifyCFG ( BB , TTI , SimplifyCFGOptions ( ) . bonusInstThreshold ( <NUM_LIT> ) ) ; } return NewRetBlock ;


<s> } else if ( BranchInst * BI = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ) { ConstantInt * BoolTrue = ConstantInt :: getTrue ( F . getContext ( ) ) ; if ( DummyReturnBB == nullptr ) { DummyReturnBB = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , DummyReturnBB ) ; ReturningBlocks . push_back ( DummyReturnBB ) ; } if ( BI -> isUnconditional ( ) ) { BasicBlock * LoopHeaderBB = BI -> getSuccessor ( <NUM_LIT> ) ; BI -> eraseFromParent ( ) ; BranchInst :: Create ( LoopHeaderBB , DummyReturnBB , BoolTrue , BB ) ; } else { BasicBlock * TransitionBB = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; BI -> removeFromParent ( ) ; TransitionBB -> getInstList ( ) . push_back ( BI ) ;

BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ;


<s> } } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; } } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ; ReturningBlocks . push_back ( UnreachableBlock ) ; } }

if ( ReturningBlocks . empty ( ) ) return false ;


<s> void <ISA_LIT>UnifyDivergentExitNodes :: getAnalysisUsage ( AnalysisUsage & AU ) const { if ( RequireAndPreserveDomTree ) AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < PostDominatorTreeWrapperPass > ( ) ; AU . addRequired < LegacyDivergenceAnalysis > ( ) ; if ( RequireAndPreserveDomTree ) { AU . addPreserved < DominatorTreeWrapperPass > ( ) ; } AU . addPreserved < LegacyDivergenceAnalysis > ( ) ;

AU . addPreservedID ( BreakCriticalEdgesID ) ;


<s> if ( ! DA . isUniform ( Top -> getTerminator ( ) ) ) return false ; for ( BasicBlock * Pred : predecessors ( Top ) ) {

if ( Visited . insert ( Pred ) . second ) Stack . push_back ( Pred ) ;


<s> if ( isa < ReturnInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) ReturningBlocks . push_back ( BB ) ; } else if ( isa < UnreachableInst > ( BB -> getTerminator ( ) ) ) { if ( ! isUniformlyReached ( DA , * BB ) ) UnreachableBlocks . push_back ( BB ) ; } else if ( BranchInst * BI = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ) { ConstantInt * BoolTrue = ConstantInt :: getTrue ( F . getContext ( ) ) ; if ( DummyReturnBB == nullptr ) { DummyReturnBB = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , DummyReturnBB ) ; ReturningBlocks . push_back ( DummyReturnBB ) ; } if ( BI -> isUnconditional ( ) ) { BasicBlock * LoopHeaderBB = BI -> getSuccessor ( <NUM_LIT> ) ; BI -> eraseFromParent ( ) ; BranchInst :: Create ( LoopHeaderBB , DummyReturnBB , BoolTrue , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , DummyReturnBB } ) ; } else { SmallVector < BasicBlock * , <NUM_LIT> > Successors ( successors ( BB ) ) ; BasicBlock * TransitionBB = BB -> splitBasicBlock ( BI , <STR_LIT> ) ; Updates . reserve ( Updates . size ( ) + <NUM_LIT> * Successors . size ( ) + <NUM_LIT> ) ; Updates . push_back ( { DominatorTree :: Insert , BB , TransitionBB } ) ; for ( BasicBlock * Successor : Successors ) { Updates . push_back ( { DominatorTree :: Insert , TransitionBB , Successor } ) ; Updates . push_back ( { DominatorTree :: Delete , BB , Successor } ) ; } BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , DummyReturnBB } ) ; } Changed = true ; } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else {

UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ;


<s> B . CreateRet ( PN ) ; } std :: vector < DominatorTree :: UpdateType > Updates ; Updates . reserve ( ReturningBlocks . size ( ) ) ; for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , NewRetBlock } ) ; } if ( RequireAndPreserveDomTree ) DTU . applyUpdates ( Updates ) ;


<s> PN = B . CreatePHI ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ; assert ( ! InsertExport ) ; B . CreateRet ( PN ) ; } std :: vector < DominatorTree :: UpdateType > Updates ; Updates . reserve ( ReturningBlocks . size ( ) ) ; for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , NewRetBlock } ) ; } if ( RequireAndPreserveDomTree ) DTU . applyUpdates ( Updates ) ; Updates . clear ( ) ; for ( BasicBlock * BB : ReturningBlocks ) { simplifyCFG ( BB , TTI , RequireAndPreserveDomTree ? & DTU : nullptr , SimplifyCFGOptions ( ) . bonusInstThreshold ( <NUM_LIT> ) ) ;


<s> PN = B . CreatePHI ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ; assert ( ! InsertExport ) ; B . CreateRet ( PN ) ; } std :: vector < DominatorTree :: UpdateType > Updates ; Updates . reserve ( ReturningBlocks . size ( ) ) ; for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , NewRetBlock } ) ; } if ( RequireAndPreserveDomTree ) DTU . applyUpdates ( Updates ) ; Updates . clear ( ) ; for ( BasicBlock * BB : ReturningBlocks ) { simplifyCFG ( BB , TTI , RequireAndPreserveDomTree ? & DTU : nullptr , SimplifyCFGOptions ( ) . bonusInstThreshold ( <NUM_LIT> ) ) ; }


<s> ReturnInst :: Create ( F . getContext ( ) , PN , NewRetBlock ) ; } for ( BasicBlock * BB : ReturningBlocks ) { if ( PN ) PN -> addIncoming ( BB -> getTerminator ( ) -> getOperand ( <NUM_LIT> ) , BB ) ; BB -> getInstList ( ) . pop_back ( ) ; BranchInst :: Create ( NewRetBlock , BB ) ; } for ( BasicBlock * BB : ReturningBlocks ) { SimplifyCFG ( BB , TTI , nullptr , { <NUM_LIT> } ) ; } return NewRetBlock ;


<s> static BasicBlock * unifyReturnBlockSet ( Function & F , ArrayRef < BasicBlock * > ReturningBlocks , const TargetTransformInfo & TTI , StringRef Name ) { BasicBlock * NewRetBlock = BasicBlock :: Create ( F . getContext ( ) , Name , & F ) ; PHINode * PN = nullptr ; if ( F . getReturnType ( ) -> isVoidTy ( ) ) { ReturnInst :: Create ( F . getContext ( ) , nullptr , NewRetBlock ) ; } else { PN = PHINode :: Create ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ; NewRetBlock -> getInstList ( ) . push_back ( PN ) ; ReturnInst :: Create ( F . getContext ( ) , PN , NewRetBlock ) ; }


<s> InsertExport = true ; } ReturnInst :: Create ( F . getContext ( ) , RetVal , DummyReturnBB ) ; ReturningBlocks . push_back ( DummyReturnBB ) ; } if ( BI -> isUnconditional ( ) ) { BasicBlock * LoopHeaderBB = BI -> getSuccessor ( <NUM_LIT> ) ; BI -> eraseFromParent ( ) ; BranchInst :: Create ( LoopHeaderBB , DummyReturnBB , BoolTrue , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , DummyReturnBB } ) ; } else { SmallVector < BasicBlock * , <NUM_LIT> > Successors ( succ_begin ( BB ) , succ_end ( BB ) ) ; BasicBlock * TransitionBB = BB -> splitBasicBlock ( BI , <STR_LIT> ) ; Updates . reserve ( Updates . size ( ) + <NUM_LIT> * Successors . size ( ) + <NUM_LIT> ) ; Updates . push_back ( { DominatorTree :: Insert , BB , TransitionBB } ) ; for ( BasicBlock * Successor : Successors ) { Updates . push_back ( { DominatorTree :: Insert , TransitionBB , Successor } ) ; Updates . push_back ( { DominatorTree :: Delete , BB , Successor } ) ; } BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , DummyReturnBB } ) ; } Changed = true ; } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; Updates . reserve ( Updates . size ( ) + UnreachableBlocks . size ( ) ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , UnreachableBlock } ) ; } Changed = true ; } if ( ! ReturningBlocks . empty ( ) ) {


<s> Value * Undef = UndefValue :: get ( B . getFloatTy ( ) ) ; B . CreateIntrinsic ( <STR_LIT> , { B . getFloatTy ( ) } , { B . getInt32 ( <STR_LIT> :: ET_NULL ) , B . getInt32 ( <NUM_LIT> ) , Undef , Undef , Undef , Undef , B . getTrue ( ) , B . getTrue ( ) , } ) ; } PHINode * PN = nullptr ; if ( F . getReturnType ( ) -> isVoidTy ( ) ) { B . CreateRetVoid ( ) ; } else {

PN = B . CreatePHI ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ;


<s> UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; Updates . reserve ( Updates . size ( ) + UnreachableBlocks . size ( ) ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , UnreachableBlock } ) ; } Changed = true ; } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ; ReturningBlocks . push_back ( UnreachableBlock ) ; Changed = true ; } } DomTreeUpdater DTU ( DT , DomTreeUpdater :: UpdateStrategy :: Eager ) ; if ( RequireAndPreserveDomTree ) DTU . applyUpdates ( Updates ) ; Updates . clear ( ) ; if ( ReturningBlocks . empty ( ) ) return Changed ; if ( ReturningBlocks . size ( ) == <NUM_LIT> ) return Changed ; unifyReturnBlockSet ( F , DTU , ReturningBlocks , <STR_LIT> ) ; return true ;


<s> if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; Updates . reserve ( Updates . size ( ) + UnreachableBlocks . size ( ) ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; Updates . push_back ( { DominatorTree :: Insert , BB , UnreachableBlock } ) ; } Changed = true ; } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; UnreachableBlock -> getTerminator ( ) -> eraseFromParent ( ) ; Function * UnreachableIntrin = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; CallInst :: Create ( UnreachableIntrin , { } , <STR_LIT> , UnreachableBlock ) ; ReturnInst :: Create ( F . getContext ( ) , RetVal , UnreachableBlock ) ; ReturningBlocks . push_back ( UnreachableBlock ) ; Changed = true ; } } DomTreeUpdater DTU ( DT , DomTreeUpdater :: UpdateStrategy :: Eager ) ; if ( RequireAndPreserveDomTree ) DTU . applyUpdates ( Updates ) ; Updates . clear ( ) ; if ( ReturningBlocks . empty ( ) ) return Changed ;

if ( ReturningBlocks . size ( ) == <NUM_LIT> ) return Changed ;


<s> void <ISA_LIT>UnifyDivergentExitNodes :: getAnalysisUsage ( AnalysisUsage & AU ) const { AU . addRequired < PostDominatorTreeWrapperPass > ( ) ; AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addPreserved < LegacyDivergenceAnalysis > ( ) ; AU . addPreservedID ( BreakCriticalEdgesID ) ; AU . addPreservedID ( LowerSwitchID ) ;

FunctionPass :: getAnalysisUsage ( AU ) ;


<s> while ( ! Stack . empty ( ) ) { BasicBlock * Top = Stack . pop_back_val ( ) ; if ( ! DA . isUniform ( Top -> getTerminator ( ) ) ) return false ; for ( BasicBlock * Pred : predecessors ( Top ) ) { if ( Visited . insert ( Pred ) . second ) Stack . push_back ( Pred ) ;


<s> if ( IntrinsicInst * Intrin = llvm :: dyn_cast < IntrinsicInst > ( & I ) ) { if ( Intrin -> getIntrinsicID ( ) == <STR_LIT> ) { Intrin -> setArgOperand ( <NUM_LIT> , BoolFalse ) ;

} else if ( Intrin -> getIntrinsicID ( ) == <STR_LIT> ) {


<s> Type * RetTy = F . getReturnType ( ) ; Value * RetVal = RetTy -> isVoidTy ( ) ? nullptr : UndefValue :: get ( RetTy ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && RetTy -> isVoidTy ( ) ) { InsertExport = true ; } ReturnInst :: Create ( F . getContext ( ) , RetVal , DummyReturnBB ) ; ReturningBlocks . push_back ( DummyReturnBB ) ; } if ( BI -> isUnconditional ( ) ) { BasicBlock * LoopHeaderBB = BI -> getSuccessor ( <NUM_LIT> ) ; BI -> eraseFromParent ( ) ; BranchInst :: Create ( LoopHeaderBB , DummyReturnBB , BoolTrue , BB ) ; } else { BasicBlock * TransitionBB = BB -> splitBasicBlock ( BI , <STR_LIT> ) ; BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( TransitionBB , DummyReturnBB , BoolTrue , BB ) ; } Changed = true ; } } if ( ! UnreachableBlocks . empty ( ) ) { BasicBlock * UnreachableBlock = nullptr ; if ( UnreachableBlocks . size ( ) == <NUM_LIT> ) { UnreachableBlock = UnreachableBlocks . front ( ) ; } else { UnreachableBlock = BasicBlock :: Create ( F . getContext ( ) , <STR_LIT> , & F ) ; new UnreachableInst ( F . getContext ( ) , UnreachableBlock ) ; for ( BasicBlock * BB : UnreachableBlocks ) { BB -> getTerminator ( ) -> eraseFromParent ( ) ; BranchInst :: Create ( UnreachableBlock , BB ) ; } Changed = true ; } if ( ! ReturningBlocks . empty ( ) ) { Type * RetTy = F . getReturnType ( ) ;


<s> if ( InsertExport ) { removeDoneExport ( F ) ; Value * Undef = UndefValue :: get ( B . getFloatTy ( ) ) ; B . CreateIntrinsic ( <STR_LIT> , { B . getFloatTy ( ) } , { B . getInt32 ( <NUM_LIT> ) , B . getInt32 ( <NUM_LIT> ) , Undef , Undef , Undef , Undef , B . getTrue ( ) , B . getTrue ( ) , } ) ; } PHINode * PN = nullptr ; if ( F . getReturnType ( ) -> isVoidTy ( ) ) { B . CreateRetVoid ( ) ; } else { PN = B . CreatePHI ( F . getReturnType ( ) , ReturningBlocks . size ( ) , <STR_LIT> ) ; assert ( ! InsertExport ) ; B . CreateRet ( PN ) ; } for ( BasicBlock * BB : ReturningBlocks ) {


<s> auto NamedMD = M . getNamedMetadata ( Name ) ; if ( ! NamedMD || NamedMD -> getNumOperands ( ) == <NUM_LIT> ) return false ; SmallVector < Metadata * , <NUM_LIT> > All ; for ( const auto & MD : NamedMD -> operands ( ) ) for ( const auto & Op : MD -> operands ( ) ) if ( std :: find ( All . begin ( ) , All . end ( ) , Op . get ( ) ) == All . end ( ) ) All . push_back ( Op . get ( ) ) ;


<s> auto CMajor = mdconst :: extract < ConstantInt > ( VersionMD -> getOperand ( <NUM_LIT> ) ) ; auto VersionMajor = CMajor -> getZExtValue ( ) ; auto CMinor = mdconst :: extract < ConstantInt > ( VersionMD -> getOperand ( <NUM_LIT> ) ) ; auto VersionMinor = CMinor -> getZExtValue ( ) ; auto Ver = ( VersionMajor * <NUM_LIT> ) + ( VersionMinor * <NUM_LIT> ) ; if ( Ver > MaxVer ) { MaxVer = Ver ; MaxMD = VersionMD ; } if ( PickFirst ) break ; } NamedMD -> eraseFromParent ( ) ;

NamedMD = M . getOrInsertNamedMetadata ( Name ) ;


<s> auto CMajor = mdconst :: extract < ConstantInt > ( VersionMD -> getOperand ( <NUM_LIT> ) ) ; auto VersionMajor = CMajor -> getZExtValue ( ) ; auto CMinor = mdconst :: extract < ConstantInt > ( VersionMD -> getOperand ( <NUM_LIT> ) ) ; auto VersionMinor = CMinor -> getZExtValue ( ) ; auto Ver = ( VersionMajor * <NUM_LIT> ) + ( VersionMinor * <NUM_LIT> ) ; if ( Ver > MaxVer ) { MaxVer = Ver ; MaxMD = VersionMD ; } if ( PickFirst ) break ; } NamedMD -> eraseFromParent ( ) ; NamedMD = M . getOrInsertNamedMetadata ( Name ) ;


<s> ModulePass * llvm :: create<ISA_LIT>UnifyMetadataPass ( ) {


<s> return unifyMetadataImpl ( M ) ? PreservedAnalyses :: none ( ) : PreservedAnalyses :: all ( ) ;


<s> bool unifyMetadataImpl ( Module & M ) { const char * Vers [ ] = { kOCLMD :: SpirVer , kOCLMD :: OCLVer } ; const char * Exts [ ] = { kOCLMD :: UsedExt , kOCLMD :: UsedOptCoreFeat , kOCLMD :: CompilerOptions , kOCLMD :: LLVMIdent } ;


<s> bool Changed = false ; for ( auto & I : Vers ) Changed |= unifyVersionMD ( M , I , true ) ; for ( auto & I : Exts ) Changed |= unifyExtensionMD ( M , I ) ;


<s> return new <ISA_LIT>UnifyMetadata ( ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>UnifyMetadataPass ( ) { return new <ISA_LIT>UnifyMetadata ( ) ;


<s> bool unifyExtensionMD ( Module & M , StringRef Name ) { auto NamedMD = M . getNamedMetadata ( Name ) ; if ( ! NamedMD || NamedMD -> getNumOperands ( ) == <NUM_LIT> ) return false ; SmallVector < Metadata * , <NUM_LIT> > All ; for ( auto MD : NamedMD -> operands ( ) ) for ( const auto & Op : MD -> operands ( ) ) if ( std :: find ( All . begin ( ) , All . end ( ) , Op . get ( ) ) == All . end ( ) ) All . push_back ( Op . get ( ) ) ; NamedMD -> eraseFromParent ( ) ; NamedMD = M . getOrInsertNamedMetadata ( Name ) ;

for ( const auto & MD : All ) NamedMD -> addOperand ( MDNode :: get ( M . getContext ( ) , MD ) ) ;


<s> SmallVector < Metadata * , <NUM_LIT> > All ; for ( auto MD : NamedMD -> operands ( ) ) for ( const auto & Op : MD -> operands ( ) ) if ( std :: find ( All . begin ( ) , All . end ( ) , Op . get ( ) ) == All . end ( ) ) All . push_back ( Op . get ( ) ) ; NamedMD -> eraseFromParent ( ) ; NamedMD = M . getOrInsertNamedMetadata ( Name ) ; for ( const auto & MD : All ) NamedMD -> addOperand ( MDNode :: get ( M . getContext ( ) , MD ) ) ; return true ;


<s> explicit <ISA_LIT>UnifyMetadata ( ) : ModulePass ( ID ) {


<s> const char * Vers [ ] = { kOCLMD :: SpirVer , kOCLMD :: OCLVer } ; const char * Exts [ ] = { kOCLMD :: UsedExt , kOCLMD :: UsedOptCoreFeat , kOCLMD :: CompilerOptions , kOCLMD :: LLVMIdent } ; bool Changed = false ; for ( auto & I : Vers ) Changed |= unifyVersionMD ( M , I , true ) ; for ( auto & I : Exts ) Changed |= unifyExtensionMD ( M , I ) ;


<s> for ( auto & I : Vers ) Changed |= unifyVersionMD ( M , I , true ) ; for ( auto & I : Exts ) Changed |= unifyExtensionMD ( M , I ) ; return Changed ;


<s> for ( auto MD : NamedMD -> operands ( ) ) for ( const auto & Op : MD -> operands ( ) ) if ( ! llvm :: is_contained ( All , Op . get ( ) ) ) All . push_back ( Op . get ( ) ) ; NamedMD -> eraseFromParent ( ) ;


<s> for ( auto VersionMD : NamedMD -> operands ( ) ) { assert ( VersionMD -> getNumOperands ( ) == <NUM_LIT> ) ; auto CMajor = mdconst :: extract < ConstantInt > ( VersionMD -> getOperand ( <NUM_LIT> ) ) ; auto VersionMajor = CMajor -> getZExtValue ( ) ; auto CMinor = mdconst :: extract < ConstantInt > ( VersionMD -> getOperand ( <NUM_LIT> ) ) ; auto VersionMinor = CMinor -> getZExtValue ( ) ; auto Ver = ( VersionMajor * <NUM_LIT> ) + ( VersionMinor * <NUM_LIT> ) ;


<s> assert ( VersionMD -> getNumOperands ( ) == <NUM_LIT> ) ; auto CMajor = mdconst :: extract < ConstantInt > ( VersionMD -> getOperand ( <NUM_LIT> ) ) ; auto VersionMajor = CMajor -> getZExtValue ( ) ; auto CMinor = mdconst :: extract < ConstantInt > ( VersionMD -> getOperand ( <NUM_LIT> ) ) ; auto VersionMinor = CMinor -> getZExtValue ( ) ; auto Ver = ( VersionMajor * <NUM_LIT> ) + ( VersionMinor * <NUM_LIT> ) ;


<s> insertInstrEnd ( DummyExitBlk , <STR_LIT> ) ; for ( MachineBasicBlock * MBB : RetMBB ) { if ( MachineInstr * MI = getReturnInstr ( MBB ) ) MI -> eraseFromParent ( ) ;


<s> insertInstrEnd ( DummyExitBlk , <STR_LIT> ) ; for ( MachineBasicBlock * MBB : RetMBB ) { if ( MachineInstr * MI = getReturnInstr ( MBB ) ) MI -> eraseFromParent ( ) ; MBB -> addSuccessor ( DummyExitBlk ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; }

SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ;


<s> <ISA_LIT>CFGStructurizer ( ) : MachineFunctionPass ( ID ) {

initialize<ISA_LIT>CFGStructurizerPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: clone ( MachineBasicBlock * MBB ) { MachineFunction * Func = MBB -> getParent ( ) ; MachineBasicBlock * NewMBB = Func -> CreateMachineBasicBlock ( ) ;


<s> MachineFunction * Func = MBB -> getParent ( ) ;

MachineBasicBlock * NewMBB = Func -> CreateMachineBasicBlock ( ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> size ( ) << <STR_LIT> ; ) ; SHOWNEWBLK ( CloneMBB , <STR_LIT> ) ; return CloneMBB ;


<s> MachineBasicBlock * CloneMBB = clone ( MBB ) ; replaceInstrUseOfBlockWith ( PredMBB , MBB , CloneMBB ) ; PredMBB -> replaceSuccessor ( MBB , CloneMBB ) ; cloneSuccessorList ( CloneMBB , MBB ) ; numClonedInstr += MBB -> size ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> size ( ) << <STR_LIT> ; ) ; SHOWNEWBLK ( CloneMBB , <STR_LIT> ) ;


<s> void <ISA_LIT>CFGStructurizer :: cloneSuccessorList ( MachineBasicBlock * DstMBB , MachineBasicBlock * SrcMBB ) { for ( MachineBasicBlock * Succ : SrcMBB -> successors ( ) ) DstMBB -> addSuccessor ( Succ ) ;


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ;

default : llvm_unreachable ( <STR_LIT> ) ;


<s> default : llvm_unreachable ( <STR_LIT> ) ; } return - <NUM_LIT> ;


<s> int <ISA_LIT>CFGStructurizer :: getContinueZeroOpcode ( int OldOpcode ) { switch ( OldOpcode ) { case <STR_LIT> :


<s> DebugLoc <ISA_LIT>CFGStructurizer :: getLastDebugLocInBB ( MachineBasicBlock * MBB ) { DebugLoc DL ; for ( MachineInstr & MI : * MBB ) if ( MI . getDebugLoc ( ) ) DL = MI . getDebugLoc ( ) ;


<s> DebugLoc DL ; for ( MachineInstr & MI : * MBB ) if ( MI . getDebugLoc ( ) ) DL = MI . getDebugLoc ( ) ;


<s> if ( Num == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ; ) ;

Num = handleJumpintoIfImp ( HeadMBB , FalseMBB , TrueMBB ) ;


<s> } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { report_fatal_error ( <STR_LIT> ) ; Register CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; report_fatal_error ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } Register InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } if ( LandBlkHasOtherPred ) {


<s> if ( TrueMBB == FalseMBB ) return <NUM_LIT> ; MigrateTrue = needMigrateBlock ( TrueMBB ) ; MigrateFalse = needMigrateBlock ( FalseMBB ) ; if ( ! MigrateTrue && ! MigrateFalse ) return <NUM_LIT> ; if ( ! MigrateTrue && TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) MigrateTrue = true ; if ( ! MigrateFalse && FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) MigrateFalse = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { report_fatal_error ( <STR_LIT> ) ; Register CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; report_fatal_error ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } Register InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; }


<s> MachineFunction * MF = blk -> getParent ( ) ; MachineInstr * NewInstr = MF -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ; blk -> insert ( I , NewInstr ) ; MachineInstrBuilder ( * MF , NewInstr ) . addReg ( RegNum , false ) ;


<s> MachineInstr * NewInstr = MF -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ;


<s> MachineInstr * MI = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ;

if ( ! MBB -> empty ( ) ) MBB -> insert ( MBB -> begin ( ) , MI ) ;


<s> MachineInstr * MI = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ; MBB -> push_back ( MI ) ;


<s> void <ISA_LIT>CFGStructurizer :: insertInstrEnd ( MachineBasicBlock * MBB , int NewOpcode , const DebugLoc & DL ) {

MachineInstr * MI = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ;


<s> switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ; }


<s> bool <ISA_LIT>CFGStructurizer :: isCondBranch ( MachineInstr * MI ) { switch ( MI -> getOpcode ( ) ) {


<s> MachineInstr * MI = getReturnInstr ( MBB ) ; bool IsReturn = MBB -> succ_empty ( ) ;


<s> MachineLoop * LoopRep = MLI -> getLoopFor ( Src1MBB ) ; if ( LoopRep && LoopRep == MLI -> getLoopFor ( Src2MBB ) ) { MachineBasicBlock * & TheEntry = LLInfoMap [ LoopRep ] ; if ( TheEntry ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Src1MBB -> getNumber ( ) << <STR_LIT> << Src2MBB -> getNumber ( ) << <STR_LIT> ; ) ; return true ;


<s> case <STR_LIT> : return true ; default : return false ; }


<s> for ( auto & It : * MLI ) for ( MachineLoop * ML : depth_first ( It ) ) NestedLoops . push_front ( ML ) ; if ( NestedLoops . empty ( ) ) return <NUM_LIT> ; int Num = <NUM_LIT> ; for ( MachineLoop * ExaminedLoop : NestedLoops ) { if ( ExaminedLoop -> getNumBlocks ( ) == <NUM_LIT> || Visited [ ExaminedLoop ] ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ExaminedLoop -> dump ( ) ; ) ; int NumBreak = mergeLoop ( ExaminedLoop ) ; if ( NumBreak == - <NUM_LIT> ) break ;


<s> insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ; if ( TrueMBB ) { MBB -> splice ( I , TrueMBB , TrueMBB -> begin ( ) , TrueMBB -> end ( ) ) ; MBB -> removeSuccessor ( TrueMBB , true ) ; if ( LandMBB && TrueMBB -> succ_size ( ) != <NUM_LIT> ) TrueMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( TrueMBB ) ; MLI -> removeBlock ( TrueMBB ) ; } if ( FalseMBB ) { insertInstrBefore ( I , <STR_LIT> ) ; MBB -> splice ( I , FalseMBB , FalseMBB -> begin ( ) , FalseMBB -> end ( ) ) ; MBB -> removeSuccessor ( FalseMBB , true ) ; if ( LandMBB && ! FalseMBB -> succ_empty ( ) ) FalseMBB -> removeSuccessor ( LandMBB , true ) ;


<s> MBBVector ExitBlks ; LoopRep -> getExitBlocks ( ExitBlks ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > ExitBlkSet ; for ( unsigned i = <NUM_LIT> , e = ExitBlks . size ( ) ; i < e ; ++ i ) ExitBlkSet . insert ( ExitBlks [ i ] ) ; assert ( ExitBlkSet . size ( ) == <NUM_LIT> ) ; MachineBasicBlock * ExitBlk = * ExitBlks . begin ( ) ; assert ( ExitBlk && <STR_LIT> ) ; MBBVector LatchBlks ; for ( auto * LB : inverse_children < MachineBasicBlock * > ( LoopHeader ) ) if ( LoopRep -> contains ( LB ) ) LatchBlks . push_back ( LB ) ; for ( unsigned i = <NUM_LIT> , e = ExitingMBBs . size ( ) ; i < e ; ++ i ) mergeLoopbreakBlock ( ExitingMBBs [ i ] , ExitBlk ) ; for ( unsigned i = <NUM_LIT> , e = LatchBlks . size ( ) ; i < e ; ++ i ) settleLoopcontBlock ( LatchBlks [ i ] , LoopHeader ) ; int Match = <NUM_LIT> ; do { Match = <NUM_LIT> ;

Match += serialPatternMatch ( LoopHeader ) ;


<s> int <ISA_LIT>CFGStructurizer :: mergeLoop ( MachineLoop * LoopRep ) { MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; MBBVector ExitingMBBs ; LoopRep -> getExitingBlocks ( ExitingMBBs ) ; assert ( ! ExitingMBBs . empty ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ExitingMBBs . size ( ) << <STR_LIT> ; ) ; MBBVector ExitBlks ; LoopRep -> getExitBlocks ( ExitBlks ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > ExitBlkSet ; for ( unsigned i = <NUM_LIT> , e = ExitBlks . size ( ) ; i < e ; ++ i ) ExitBlkSet . insert ( ExitBlks [ i ] ) ; assert ( ExitBlkSet . size ( ) == <NUM_LIT> ) ; MachineBasicBlock * ExitBlk = * ExitBlks . begin ( ) ; assert ( ExitBlk && <STR_LIT> ) ; MBBVector LatchBlks ; for ( auto * LB : inverse_children < MachineBasicBlock * > ( LoopHeader ) ) if ( LoopRep -> contains ( LB ) ) LatchBlks . push_back ( LB ) ; for ( unsigned i = <NUM_LIT> , e = ExitingMBBs . size ( ) ; i < e ; ++ i ) mergeLoopbreakBlock ( ExitingMBBs [ i ] , ExitBlk ) ; for ( unsigned i = <NUM_LIT> , e = LatchBlks . size ( ) ; i < e ; ++ i ) settleLoopcontBlock ( LatchBlks [ i ] , LoopHeader ) ; int Match = <NUM_LIT> ; do { Match = <NUM_LIT> ; Match += serialPatternMatch ( LoopHeader ) ; Match += ifPatternMatch ( LoopHeader ) ; } while ( Match > <NUM_LIT> ) ; mergeLooplandBlock ( LoopHeader , ExitBlk ) ; MachineLoop * ParentLoop = LoopRep -> getParentLoop ( ) ; if ( ParentLoop ) MLI -> changeLoopFor ( LoopHeader , ParentLoop ) ; else MLI -> removeBlock ( LoopHeader ) ; Visited [ LoopRep ] = true ; return <NUM_LIT> ;


<s> void <ISA_LIT>CFGStructurizer :: mergeLooplandBlock ( MachineBasicBlock * DstBlk , MachineBasicBlock * LandMBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstBlk -> getNumber ( ) << <STR_LIT> << LandMBB -> getNumber ( ) << <STR_LIT> ; ) ; insertInstrBefore ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ;

insertInstrEnd ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ;


<s> void <ISA_LIT>CFGStructurizer :: mergeSerialBlock ( MachineBasicBlock * DstMBB , MachineBasicBlock * SrcMBB ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> getNumber ( ) << <STR_LIT> << SrcMBB -> getNumber ( ) << <STR_LIT> ; ) ;


<s> void <ISA_LIT>CFGStructurizer :: mergeSerialBlock ( MachineBasicBlock * DstMBB , MachineBasicBlock * SrcMBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> getNumber ( ) << <STR_LIT> << SrcMBB -> getNumber ( ) << <STR_LIT> ; ) ; DstMBB -> splice ( DstMBB -> end ( ) , SrcMBB , SrcMBB -> begin ( ) , SrcMBB -> end ( ) ) ; DstMBB -> removeSuccessor ( SrcMBB , true ) ; cloneSuccessorList ( DstMBB , SrcMBB ) ; removeSuccessor ( SrcMBB ) ;


<s> MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ;


<s> if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ; FuncRep -> push_back ( DummyExitBlk ) ; SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI << <STR_LIT> ; ) ; LLVMContext & Ctx = LoopHeader -> getParent ( ) -> getFunction ( ) . getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ;


<s> ++ It , ++ SccNum ) { const std :: vector < MachineBasicBlock * > & SccNext = * It ; for ( MachineBasicBlock * MBB : SccNext ) { OrderedBlks . push_back ( MBB ) ; recordSccnum ( MBB , SccNum ) ; } }


<s> void <ISA_LIT>CFGStructurizer :: orderBlocks ( MachineFunction * MF ) { int SccNum = <NUM_LIT> ; for ( scc_iterator < MachineFunction * > It = scc_begin ( MF ) ; ! It . isAtEnd ( ) ; ++ It , ++ SccNum ) { const std :: vector < MachineBasicBlock * > & SccNext = * It ; for ( MachineBasicBlock * MBB : SccNext ) { OrderedBlks . push_back ( MBB ) ; recordSccnum ( MBB , SccNum ) ; } } for ( auto * MBB : nodes ( MF ) ) { SccNum = getSCCNum ( MBB ) ; if ( SccNum == INVALIDSCCNUM ) dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ;


<s> while ( ( CurMatch = patternMatchGroup ( MBB ) ) > <NUM_LIT> ) NumMatch += CurMatch ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << NumMatch << <STR_LIT> ; ) ;


<s> int NumMatch = <NUM_LIT> ; int CurMatch ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; while ( ( CurMatch = patternMatchGroup ( MBB ) ) > <NUM_LIT> ) NumMatch += CurMatch ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << NumMatch << <STR_LIT> ; ) ;


<s> MachineLoop * LoopRep = ( * It ) ; MBBVector ExitingMBBs ; LoopRep -> getExitingBlocks ( ExitingMBBs ) ; if ( ExitingMBBs . size ( ) == <NUM_LIT> ) { MachineBasicBlock * DummyExitBlk = normalizeInfiniteLoopExit ( LoopRep ) ; if ( DummyExitBlk ) RetBlks . push_back ( DummyExitBlk ) ; } } for ( MachineBasicBlock * MBB : OrderedBlks ) { removeUnconditionalBranch ( MBB ) ; removeRedundantConditionalBranch ( MBB ) ; if ( isReturnBlock ( MBB ) ) { RetBlks . push_back ( MBB ) ; } assert ( MBB -> succ_size ( ) <= <NUM_LIT> ) ;


<s> orderBlocks ( FuncRep ) ; SmallVector < MachineBasicBlock * , DEFAULT_VEC_SLOTS > RetBlks ; for ( MachineLoopInfo :: iterator It = MLI -> begin ( ) , E = MLI -> end ( ) ; It != E ; ++ It ) { MachineLoop * LoopRep = ( * It ) ; MBBVector ExitingMBBs ; LoopRep -> getExitingBlocks ( ExitingMBBs ) ; if ( ExitingMBBs . size ( ) == <NUM_LIT> ) { MachineBasicBlock * DummyExitBlk = normalizeInfiniteLoopExit ( LoopRep ) ; if ( DummyExitBlk ) RetBlks . push_back ( DummyExitBlk ) ; } } for ( MachineBasicBlock * MBB : OrderedBlks ) { removeUnconditionalBranch ( MBB ) ;

removeRedundantConditionalBranch ( MBB ) ;


<s> if ( MBB1 != MBB2 ) return ; MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ;


<s> if ( MBB -> succ_size ( ) != <NUM_LIT> ) return ; MachineBasicBlock * MBB1 = * MBB -> succ_begin ( ) ; MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ; if ( MBB1 != MBB2 ) return ; MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; BranchMI -> eraseFromParent ( ) ; SHOWNEWBLK ( MBB1 , <STR_LIT> ) ;

MBB -> removeSuccessor ( MBB1 , true ) ;


<s> void <ISA_LIT>CFGStructurizer :: removeUnconditionalBranch ( MachineBasicBlock * MBB ) { MachineInstr * BranchMI ;

while ( ( BranchMI = getLoopendBlockBranchInstr ( MBB ) ) && isUncondBranch ( BranchMI ) ) {


<s> if ( ! SrcBlkInfo ) SrcBlkInfo = new BlockInformation ( ) ; SrcBlkInfo -> IsRetired = true ; assert ( MBB -> succ_empty ( ) && MBB -> pred_empty ( ) && <STR_LIT> ) ;


<s> void <ISA_LIT>CFGStructurizer :: reversePredicateSetter ( MachineBasicBlock :: iterator I , MachineBasicBlock & MBB ) { assert ( I . isValid ( ) && <STR_LIT> ) ; for ( ; ; -- I ) { if ( I == MBB . end ( ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( I -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; default :


<s> assert ( I . isValid ( ) && <STR_LIT> ) ; for ( ; ; -- I ) { if ( I == MBB . end ( ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( I -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; default :


<s> } } else { ContNextScc = false ; } if ( ContNextScc ) SccBeginMBB = nullptr ; } MachineBasicBlock * EntryMBB = * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ; if ( EntryMBB -> succ_empty ( ) ) { Finish = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ; for ( MBBInfoMap :: iterator It = BlockInfoMap . begin ( ) , E = BlockInfoMap . end ( ) ; It != E ; ++ It ) { if ( ( * It ) . second && ( * It ) . second -> IsRetired ) {

assert ( ( ( * It ) . first ) -> getNumber ( ) != - <NUM_LIT> ) ;


<s> FuncRep = & MF ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; PrintLoopinfo ( * MLI ) ; ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; LLVM_DEBUG ( MDT -> print ( dbgs ( ) , ( const Module * ) nullptr ) ; ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; LLVM_DEBUG ( PDT -> print ( dbgs ( ) ) ; ) ; prepare ( ) ; run ( ) ; LLVM_DEBUG ( MF . dump ( ) ; ) ; return true ;


<s> LLVM_DEBUG ( MF . dump ( ) ; ) ; OrderedBlks . clear ( ) ; Visited . clear ( ) ; FuncRep = & MF ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; PrintLoopinfo ( * MLI ) ; ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; LLVM_DEBUG ( MDT -> print ( dbgs ( ) , ( const Module * ) nullptr ) ; ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; LLVM_DEBUG ( PDT -> print ( dbgs ( ) ) ; ) ;


<s> assert ( isCondBranch ( MI ) ) ; MachineBasicBlock :: iterator I = MI ; MachineBasicBlock * TrueBranch = getTrueBranch ( MI ) ; int OldOpcode = MI -> getOpcode ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool UseContinueLogical = ( ( & * ContingMBB -> rbegin ( ) ) == MI ) ; if ( ! UseContinueLogical ) { int BranchOpcode = TrueBranch == ContMBB ? getBranchNzeroOpcode ( OldOpcode ) : getBranchZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; } else { int BranchOpcode = TrueBranch == ContMBB ? getContinueNzeroOpcode ( OldOpcode ) : getContinueZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ;


<s> MachineInstr * MI = getLoopendBlockBranchInstr ( ContingMBB ) ; if ( MI ) { assert ( isCondBranch ( MI ) ) ; MachineBasicBlock :: iterator I = MI ; MachineBasicBlock * TrueBranch = getTrueBranch ( MI ) ; int OldOpcode = MI -> getOpcode ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ;

bool UseContinueLogical = ( ( & * ContingMBB -> rbegin ( ) ) == MI ) ;


<s> SmallVector < MachineInstr * , DEFAULT_VEC_SLOTS > ContInstr ; MachineBasicBlock :: iterator Pre = MBB -> begin ( ) ; MachineBasicBlock :: iterator E = MBB -> end ( ) ; MachineBasicBlock :: iterator It = Pre ; while ( It != E ) {

if ( Pre -> getOpcode ( ) == <STR_LIT> && It -> getOpcode ( ) == <STR_LIT> ) ContInstr . push_back ( & * Pre ) ;


<s> for ( SmallVectorImpl < MachineBasicBlock * > :: iterator It = RetMBB . begin ( ) , E = RetMBB . end ( ) ; It != E ; ++ It ) { MachineBasicBlock * MBB = * It ; MachineInstr * MI = getReturnInstr ( MBB ) ; if ( MI ) MI -> eraseFromParent ( ) ; MBB -> addSuccessor ( DummyExitBlk ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ;


<s> MachineInstr * MI = getReturnInstr ( MBB ) ; if ( MI ) MI -> eraseFromParent ( ) ; MBB -> addSuccessor ( DummyExitBlk ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ;


<s> showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { report_fatal_error ( <STR_LIT> ) ; unsigned CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; report_fatal_error ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } unsigned InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk , I ) ;


<s> if ( ! MigrateTrue && ! MigrateFalse ) return <NUM_LIT> ; if ( ! MigrateTrue && TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) MigrateTrue = true ; if ( ! MigrateFalse && FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) MigrateFalse = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { report_fatal_error ( <STR_LIT> ) ;

unsigned CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ;


<s> bool IsReturn = ( MBB -> succ_size ( ) == <NUM_LIT> ) ;


<s> MachineBasicBlock * & TheEntry = LLInfoMap [ LoopRep ] ; if ( TheEntry ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Src1MBB -> getNumber ( ) << <STR_LIT> << Src2MBB -> getNumber ( ) << <STR_LIT> ; ) ; return true ; } }


<s> int OldOpcode = BranchMI -> getOpcode ( ) ; DebugLoc BranchDL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock :: iterator I = BranchMI ; insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ; if ( TrueMBB ) { MBB -> splice ( I , TrueMBB , TrueMBB -> begin ( ) , TrueMBB -> end ( ) ) ; MBB -> removeSuccessor ( TrueMBB , true ) ; if ( LandMBB && TrueMBB -> succ_size ( ) != <NUM_LIT> ) TrueMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( TrueMBB ) ; MLI -> removeBlock ( TrueMBB ) ; } if ( FalseMBB ) { insertInstrBefore ( I , <STR_LIT> ) ; MBB -> splice ( I , FalseMBB , FalseMBB -> begin ( ) , FalseMBB -> end ( ) ) ; MBB -> removeSuccessor ( FalseMBB , true ) ; if ( LandMBB && FalseMBB -> succ_size ( ) != <NUM_LIT> ) FalseMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( FalseMBB ) ; MLI -> removeBlock ( FalseMBB ) ; }


<s> MachineBasicBlock * TrueBranch = getTrueBranch ( BranchMI ) ; MachineBasicBlock :: iterator I = BranchMI ; if ( TrueBranch != LandMBB ) reversePredicateSetter ( I , * I -> getParent ( ) ) ;

insertCondBranchBefore ( ExitingMBB , I , <STR_LIT> , <STR_LIT> , DL ) ;


<s> MachineBasicBlock :: iterator I = BranchMI ; if ( TrueBranch != LandMBB ) reversePredicateSetter ( I , * I -> getParent ( ) ) ; insertCondBranchBefore ( ExitingMBB , I , <STR_LIT> , <STR_LIT> , DL ) ; insertInstrBefore ( I , <STR_LIT> ) ; insertInstrBefore ( I , <STR_LIT> ) ; BranchMI -> eraseFromParent ( ) ; ExitingMBB -> removeSuccessor ( LandMBB , true ) ;


<s> void <ISA_LIT>CFGStructurizer :: mergeLooplandBlock ( MachineBasicBlock * DstBlk , MachineBasicBlock * LandMBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstBlk -> getNumber ( ) << <STR_LIT> << LandMBB -> getNumber ( ) << <STR_LIT> ; ) ; insertInstrBefore ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ;


<s> void <ISA_LIT>CFGStructurizer :: mergeLooplandBlock ( MachineBasicBlock * DstBlk , MachineBasicBlock * LandMBB ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstBlk -> getNumber ( ) << <STR_LIT> << LandMBB -> getNumber ( ) << <STR_LIT> ; ) ;


<s> ++ It , ++ SccNum ) { const std :: vector < MachineBasicBlock * > & SccNext = * It ; for ( std :: vector < MachineBasicBlock * > :: const_iterator blockIter = SccNext . begin ( ) , blockEnd = SccNext . end ( ) ; blockIter != blockEnd ; ++ blockIter ) { MBB = * blockIter ; OrderedBlks . push_back ( MBB ) ; recordSccnum ( MBB , SccNum ) ; } } for ( auto * MBB : nodes ( MF ) ) {


<s> bool Changed = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; orderBlocks ( FuncRep ) ; SmallVector < MachineBasicBlock * , DEFAULT_VEC_SLOTS > RetBlks ; for ( MachineLoopInfo :: iterator It = MLI -> begin ( ) , E = MLI -> end ( ) ; It != E ; ++ It ) { MachineLoop * LoopRep = ( * It ) ; MBBVector ExitingMBBs ; LoopRep -> getExitingBlocks ( ExitingMBBs ) ; if ( ExitingMBBs . size ( ) == <NUM_LIT> ) { MachineBasicBlock * DummyExitBlk = normalizeInfiniteLoopExit ( LoopRep ) ; if ( DummyExitBlk ) RetBlks . push_back ( DummyExitBlk ) ; } } for ( SmallVectorImpl < MachineBasicBlock * > :: const_iterator It = OrderedBlks . begin ( ) , E = OrderedBlks . end ( ) ; It != E ; ++ It ) { MachineBasicBlock * MBB = * It ; removeUnconditionalBranch ( MBB ) ; removeRedundantConditionalBranch ( MBB ) ;


<s> void <ISA_LIT>CFGStructurizer :: retireBlock ( MachineBasicBlock * MBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ;

BlockInformation * & SrcBlkInfo = BlockInfoMap [ MBB ] ;


<s> void <ISA_LIT>CFGStructurizer :: retireBlock ( MachineBasicBlock * MBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; BlockInformation * & SrcBlkInfo = BlockInfoMap [ MBB ] ; if ( ! SrcBlkInfo ) SrcBlkInfo = new BlockInformation ( ) ;


<s> for ( ; ; -- I ) { if ( I == MBB . end ( ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( I -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> :


<s> bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else { ContNextScc = false ; } if ( ContNextScc ) SccBeginMBB = nullptr ; } MachineBasicBlock * EntryMBB = * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ; if ( EntryMBB -> succ_size ( ) == <NUM_LIT> ) { Finish = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ; for ( MBBInfoMap :: iterator It = BlockInfoMap . begin ( ) , E = BlockInfoMap . end ( ) ; It != E ; ++ It ) { if ( ( * It ) . second && ( * It ) . second -> IsRetired ) { assert ( ( ( * It ) . first ) -> getNumber ( ) != - <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( ( * It ) . first ) -> getNumber ( ) << <STR_LIT> ; ) ; ( * It ) . first -> eraseFromParent ( ) ; } delete ( * It ) . second ; } BlockInfoMap . clear ( ) ; LLInfoMap . clear ( ) ; if ( ! Finish ) {


<s> OrderedBlks . clear ( ) ; Visited . clear ( ) ; FuncRep = & MF ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; PrintLoopinfo ( * MLI ) ; ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; LLVM_DEBUG ( MDT -> print ( dbgs ( ) , ( const Module * ) nullptr ) ; ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; LLVM_DEBUG ( PDT -> print ( dbgs ( ) ) ; ) ; prepare ( ) ; run ( ) ;


<s> MachineInstr * MI = getLoopendBlockBranchInstr ( ContingMBB ) ; if ( MI ) { assert ( isCondBranch ( MI ) ) ; MachineBasicBlock :: iterator I = MI ; MachineBasicBlock * TrueBranch = getTrueBranch ( MI ) ; int OldOpcode = MI -> getOpcode ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ;


<s> void <ISA_LIT>CFGStructurizer :: wrapup ( MachineBasicBlock * MBB ) { assert ( ( ! MBB -> getParent ( ) -> getJumpTableInfo ( ) || MBB -> getParent ( ) -> getJumpTableInfo ( ) -> isEmpty ( ) ) && <STR_LIT> ) ; SmallVector < MachineInstr * , DEFAULT_VEC_SLOTS > ContInstr ; MachineBasicBlock :: iterator Pre = MBB -> begin ( ) ; MachineBasicBlock :: iterator E = MBB -> end ( ) ; MachineBasicBlock :: iterator It = Pre ; while ( It != E ) {


<s> MachineBasicBlock * CloneMBB = clone ( MBB ) ; replaceInstrUseOfBlockWith ( PredMBB , MBB , CloneMBB ) ; PredMBB -> replaceSuccessor ( MBB , CloneMBB ) ; cloneSuccessorList ( CloneMBB , MBB ) ; numClonedInstr += MBB -> size ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> size ( ) << <STR_LIT> ; ) ; SHOWNEWBLK ( CloneMBB , <STR_LIT> ) ; return CloneMBB ;


<s> dbgs ( ) << <STR_LIT> ; if ( ! LandMBB ) { dbgs ( ) << <STR_LIT> ; } else { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; ) ; int OldOpcode = BranchMI -> getOpcode ( ) ; DebugLoc BranchDL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock :: iterator I = BranchMI ; insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ; if ( TrueMBB ) { MBB -> splice ( I , TrueMBB , TrueMBB -> begin ( ) , TrueMBB -> end ( ) ) ;

MBB -> removeSuccessor ( TrueMBB , true ) ;


<s> insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ; if ( TrueMBB ) { MBB -> splice ( I , TrueMBB , TrueMBB -> begin ( ) , TrueMBB -> end ( ) ) ; MBB -> removeSuccessor ( TrueMBB , true ) ; if ( LandMBB && TrueMBB -> succ_size ( ) != <NUM_LIT> ) TrueMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( TrueMBB ) ; MLI -> removeBlock ( TrueMBB ) ; } if ( FalseMBB ) { insertInstrBefore ( I , <STR_LIT> ) ; MBB -> splice ( I , FalseMBB , FalseMBB -> begin ( ) , FalseMBB -> end ( ) ) ; MBB -> removeSuccessor ( FalseMBB , true ) ; if ( LandMBB && FalseMBB -> succ_size ( ) != <NUM_LIT> ) FalseMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( FalseMBB ) ; MLI -> removeBlock ( FalseMBB ) ; } insertInstrBefore ( I , <STR_LIT> ) ;


<s> insertInstrBefore ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ; insertInstrEnd ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ; DstBlk -> replaceSuccessor ( DstBlk , LandMBB ) ;


<s> int SccNum = <NUM_LIT> ; MachineBasicBlock * MBB ; for ( scc_iterator < MachineFunction * > It = scc_begin ( MF ) ; ! It . isAtEnd ( ) ; ++ It , ++ SccNum ) { const std :: vector < MachineBasicBlock * > & SccNext = * It ; for ( std :: vector < MachineBasicBlock * > :: const_iterator blockIter = SccNext . begin ( ) , blockEnd = SccNext . end ( ) ; blockIter != blockEnd ; ++ blockIter ) { MBB = * blockIter ; OrderedBlks . push_back ( MBB ) ; recordSccnum ( MBB , SccNum ) ; } } typedef GraphTraits < MachineFunction * > GTM ; auto It = GTM :: nodes_begin ( MF ) , E = GTM :: nodes_end ( MF ) ; for ( ; It != E ; ++ It ) {


<s> void <ISA_LIT>CFGStructurizer :: orderBlocks ( MachineFunction * MF ) { int SccNum = <NUM_LIT> ; MachineBasicBlock * MBB ; for ( scc_iterator < MachineFunction * > It = scc_begin ( MF ) ; ! It . isAtEnd ( ) ; ++ It , ++ SccNum ) {

const std :: vector < MachineBasicBlock * > & SccNext = * It ;