<s> Num = Builder . CreateSExt ( Num , I32Ty ) ; Den = Builder . CreateSExt ( Den , I32Ty ) ; } else { Num = Builder . CreateZExt ( Num , I32Ty ) ; Den = Builder . CreateZExt ( Den , I32Ty ) ; } } if ( Value * Res = expandDivRem24 ( Builder , I , Num , Den , IsDiv , IsSigned ) ) { Res = Builder . CreateTrunc ( Res , Ty ) ; return Res ; } ConstantInt * Zero = Builder . getInt32 ( <NUM_LIT> ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; ConstantInt * MinusOne = Builder . getInt32 ( ~ <NUM_LIT> ) ; Value * Sign = nullptr ; if ( IsSigned ) { ConstantInt * K31 = Builder . getInt32 ( <NUM_LIT> ) ; Value * LHSign = Builder . CreateAShr ( Num , K31 ) ; Value * RHSign = Builder . CreateAShr ( Den , K31 ) ; Sign = IsDiv ? Builder . CreateXor ( LHSign , RHSign ) : LHSign ; Num = Builder . CreateAdd ( Num , LHSign ) ; Den = Builder . CreateAdd ( Den , RHSign ) ; Num = Builder . CreateXor ( Num , LHSign ) ; Den = Builder . CreateXor ( Den , RHSign ) ; } Value * DEN_F32 = Builder . CreateUIToFP ( Den , F32Ty ) ; Value * RCP_F32 = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , DEN_F32 ) ; Constant * UINT_MAX_PLUS_1 = ConstantFP :: get ( F32Ty , BitsToFloat ( <NUM_LIT> ) ) ; Value * RCP_SCALE = Builder . CreateFMul ( RCP_F32 , UINT_MAX_PLUS_1 ) ;

Value * RCP = Builder . CreateFPToUI ( RCP_SCALE , I32Ty ) ;


<s> AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . setPreservesAll ( ) ;


<s> if ( T -> isIntegerTy ( ) ) return T -> getIntegerBitWidth ( ) ; return cast < VectorType > ( T ) -> getElementType ( ) -> getIntegerBitWidth ( ) ;


<s> Type * <ISA_LIT>CodeGenPrepare :: getI32Ty ( IRBuilder < > & B , const Type * T ) const { assert ( needsPromotionToI32 ( T ) && <STR_LIT> ) ;

if ( T -> isIntegerTy ( ) ) return B . getInt32Ty ( ) ;


<s> Value * Lo = Builder . CreateTrunc ( MUL64 , I32Ty ) ; Value * Hi = Builder . CreateLShr ( MUL64 , Builder . getInt64 ( <NUM_LIT> ) ) ; Hi = Builder . CreateTrunc ( Hi , I32Ty ) ;


<s> static Value * getMulHu ( IRBuilder < > & Builder , Value * LHS , Value * RHS ) { return getMul64 ( Builder , LHS , RHS ) . second ;


<s> StringRef getPassName ( ) const override {


<s> return Attr . getValueAsString ( ) == <STR_LIT> ;


<s> return isa < ICmpInst > ( I . getOperand ( <NUM_LIT> ) ) ? cast < ICmpInst > ( I . getOperand ( <NUM_LIT> ) ) -> isSigned ( ) : false ;


<s> if ( IntTy && IntTy -> getBitWidth ( ) > <NUM_LIT> && IntTy -> getBitWidth ( ) <= <NUM_LIT> ) return true ; if ( const VectorType * VT = dyn_cast < VectorType > ( T ) ) { if ( ST -> hasVOP3PInsts ( ) ) return false ; return needsPromotionToI32 ( VT -> getElementType ( ) ) ; } return false ;


<s> case Instruction :: Add : case Instruction :: Sub : return true ;


<s> static bool promotedOpIsNSW ( const Instruction & I ) { switch ( I . getOpcode ( ) ) { case Instruction :: Shl : case Instruction :: Add : case Instruction :: Sub : return true ; case Instruction :: Mul : return I . hasNoUnsignedWrap ( ) ; default : return false ; }


<s> switch ( I . getOpcode ( ) ) { case Instruction :: Shl : case Instruction :: Add : case Instruction :: Mul : return true ; case Instruction :: Sub : return I . hasNoUnsignedWrap ( ) ; default : return false ;


<s> case Instruction :: Mul : return true ; case Instruction :: Sub : return I . hasNoUnsignedWrap ( ) ;


<s> Value * ExtRes = Builder . CreateCall ( I32 , { ExtOp } ) ; Value * LShrOp = Builder . CreateLShr ( ExtRes , <NUM_LIT> - getBaseElementBitWidth ( I . getType ( ) ) ) ; Value * TruncRes = Builder . CreateTrunc ( LShrOp , I . getType ( ) ) ; I . replaceAllUsesWith ( TruncRes ) ; I . eraseFromParent ( ) ;


<s> Value * TruncRes = nullptr ; if ( isSigned ( I ) ) { ExtOp1 = Builder . CreateSExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ; ExtOp2 = Builder . CreateSExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ; } else { ExtOp1 = Builder . CreateZExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ; ExtOp2 = Builder . CreateZExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ; } ExtRes = Builder . CreateSelect ( I . getOperand ( <NUM_LIT> ) , ExtOp1 , ExtOp2 ) ; TruncRes = Builder . CreateTrunc ( ExtRes , I . getType ( ) ) ; I . replaceAllUsesWith ( TruncRes ) ; I . eraseFromParent ( ) ; return true ;


<s> ExtOp2 = Builder . CreateZExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ; } ExtRes = Builder . CreateSelect ( I . getOperand ( <NUM_LIT> ) , ExtOp1 , ExtOp2 ) ; TruncRes = Builder . CreateTrunc ( ExtRes , I . getType ( ) ) ; I . replaceAllUsesWith ( TruncRes ) ; I . eraseFromParent ( ) ;


<s> if ( ! TPC ) return false ; const <ISA_LIT>TargetMachine & TM = TPC -> getTM < <ISA_LIT>TargetMachine > ( ) ; ST = & TM . getSubtarget < GCNSubtarget > ( F ) ; AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) {


<s> auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; if ( ! TPC ) return false ; const <ISA_LIT>TargetMachine & TM = TPC -> getTM < <ISA_LIT>TargetMachine > ( ) ; ST = & TM . getSubtarget < GCNSubtarget > ( F ) ; AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ;

HasUnsafeFPMath = hasUnsafeFPMath ( F ) ;


<s> bool IsOne = CNum -> isExactlyValue ( + <NUM_LIT> ) || CNum -> isExactlyValue ( - <NUM_LIT> ) ; return HasDenormals ^ IsOne ;


<s> Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; } if ( NewDiv ) { I . replaceAllUsesWith ( NewDiv ) ; I . eraseFromParent ( ) ; Changed = true ; } }


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformBitreverseToI32 ( I ) ; return Changed ;


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformBitreverseToI32 ( I ) ;


<s> Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; bool HasDenormals = ST -> hasFP32Denormals ( ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv , HasDenormals ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else { NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } ) ; } NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; }


<s> Type * Ty = FDiv . getType ( ) ; if ( ! Ty -> getScalarType ( ) -> isFloatTy ( ) ) return false ; MDNode * FPMath = FDiv . getMetadata ( LLVMContext :: MD_fpmath ) ; if ( ! FPMath ) return false ; const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; float ULP = FPOp -> getFPAccuracy ( ) ; if ( ULP < <NUM_LIT> ) return false ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; bool UnsafeDiv = HasUnsafeFPMath || FMF . isFast ( ) || FMF . allowReciprocal ( ) ; if ( UnsafeDiv ) return false ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) , FPMath ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ;

Value * Num = FDiv . getOperand ( <NUM_LIT> ) ;


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getOperand ( <NUM_LIT> ) -> getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformOpToI32 ( I ) ;


<s> bool visitInstruction ( Instruction & I ) {


<s> return visitBitreverseIntrinsicInst ( I ) ; default : return false ; }


<s> Metadata * LowAndHigh [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , Lower -> getValue ( ) . zext ( <NUM_LIT> ) ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , <NUM_LIT> ) ) } ; WidenLoad -> setMetadata ( LLVMContext :: MD_range , MDNode :: get ( Mod -> getContext ( ) , LowAndHigh ) ) ; } } int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ;

Value * ValOrig = Builder . CreateBitCast ( ValTrunc , I . getType ( ) ) ;


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformOpToI32 ( I ) ;


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformOpToI32 ( I ) ; return Changed ;


<s> if ( Instruction * Inst = dyn_cast < Instruction > ( ExtRes ) ) { if ( promotedOpIsNSW ( cast < Instruction > ( I ) ) ) Inst -> setHasNoSignedWrap ( ) ; if ( promotedOpIsNUW ( cast < Instruction > ( I ) ) ) Inst -> setHasNoUnsignedWrap ( ) ; if ( const auto * ExactOp = dyn_cast < PossiblyExactOperator > ( & I ) ) Inst -> setIsExact ( ExactOp -> isExact ( ) ) ; } TruncRes = Builder . CreateTrunc ( ExtRes , I . getType ( ) ) ; I . replaceAllUsesWith ( TruncRes ) ; I . eraseFromParent ( ) ; return true ;


<s> bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MadeChange |= visit ( * I ) ; } } return MadeChange ;


<s> if ( ! CNum ) return false ; return UnsafeDiv || CNum -> isExactlyValue ( + <NUM_LIT> ) ;


return UnsafeDiv || CNum -> isExactlyValue ( + <NUM_LIT> ) ;


<s> MDNode * FPMath = FDiv . getMetadata ( LLVMContext :: MD_fpmath ) ; if ( ! FPMath ) return false ; const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; float ULP = FPOp -> getFPAccuracy ( ) ; if ( ULP < <NUM_LIT> ) return false ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; bool UnsafeDiv = HasUnsafeFPMath || FMF . unsafeAlgebra ( ) || FMF . allowReciprocal ( ) ; if ( ST -> hasFP32Denormals ( ) && ! UnsafeDiv ) return false ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) , FPMath ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else { NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } ) ; } NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else { if ( ! shouldKeepFDivF32 ( Num , UnsafeDiv ) ) NewFDiv = Builder . CreateCall ( Decl , { Num , Den } ) ; } if ( NewFDiv ) { FDiv . replaceAllUsesWith ( NewFDiv ) ; NewFDiv -> takeName ( & FDiv ) ; FDiv . eraseFromParent ( ) ; }


<s> Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else { NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } ) ; } NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else { if ( ! shouldKeepFDivF32 ( Num , UnsafeDiv ) ) NewFDiv = Builder . CreateCall ( Decl , { Num , Den } ) ; } if ( NewFDiv ) { FDiv . replaceAllUsesWith ( NewFDiv ) ; NewFDiv -> takeName ( & FDiv ) ; FDiv . eraseFromParent ( ) ;


<s> } assert ( NumBits <= <NUM_LIT> ) ; <STR_LIT> LoID = IsSigned ? <STR_LIT> : <STR_LIT> ; <STR_LIT> HiID = IsSigned ? <STR_LIT> : <STR_LIT> ; Value * Lo = Builder . CreateIntrinsic ( LoID , { } , { LHS , RHS } ) ; Value * Hi = Builder . CreateIntrinsic ( HiID , { } , { LHS , RHS } ) ; IntegerType * I64Ty = Builder . getInt64Ty ( ) ; Lo = Builder . CreateZExtOrTrunc ( Lo , I64Ty ) ; Hi = Builder . CreateZExtOrTrunc ( Hi , I64Ty ) ;

return Builder . CreateOr ( Lo , Builder . CreateShl ( Hi , <NUM_LIT> ) ) ;


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsUnsigned ( Value * Op ) const { return computeKnownBits ( Op , * DL , <NUM_LIT> , AC ) . countMaxActiveBits ( ) ;


<s> SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IsSigned ) { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; }

Value * Result = getMul24 ( Builder , LHS , RHS , Size , LHSBits + RHSBits , IsSigned ) ;


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsSigned ( Value * Op , unsigned ScalarSize ) const {

return ScalarSize - ComputeNumSignBits ( Op , * DL , <NUM_LIT> , AC ) + <NUM_LIT> ;


<s> if ( Size <= <NUM_LIT> && ST -> has16BitInsts ( ) ) return false ; if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; unsigned LHSBits = <NUM_LIT> , RHSBits = <NUM_LIT> ; bool IsSigned = false ; if ( ST -> hasMulU24 ( ) && ( LHSBits = numBitsUnsigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsUnsigned ( RHS , Size ) ) <= <NUM_LIT> ) { IsSigned = false ; } else if ( ST -> hasMulI24 ( ) && ( LHSBits = numBitsSigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsSigned ( RHS , Size ) ) <= <NUM_LIT> ) { IsSigned = true ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IsSigned ) { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = getMul24 ( Builder , LHS , RHS , Size , LHSBits + RHSBits , IsSigned ) ; if ( IsSigned ) { ResultVals . push_back ( Builder . CreateSExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } else { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } } Value * NewVal = insertValues ( Builder , Ty , ResultVals ) ; NewVal -> takeName ( & I ) ;


<s> unsigned LHSBits = <NUM_LIT> , RHSBits = <NUM_LIT> ; bool IsSigned = false ; if ( ST -> hasMulU24 ( ) && ( LHSBits = numBitsUnsigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsUnsigned ( RHS , Size ) ) <= <NUM_LIT> ) { IsSigned = false ; } else if ( ST -> hasMulI24 ( ) && ( LHSBits = numBitsSigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsSigned ( RHS , Size ) ) <= <NUM_LIT> ) { IsSigned = true ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IsSigned ) { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = getMul24 ( Builder , LHS , RHS , Size , LHSBits + RHSBits , IsSigned ) ; if ( IsSigned ) { ResultVals . push_back ( Builder . CreateSExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } else { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; }


<s> if ( I . getPointerAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS && canWidenScalarExtLoad ( I ) ) { IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * PT = PointerType :: get ( I32Ty , I . getPointerAddressSpace ( ) ) ; Value * BitCast = Builder . CreateBitCast ( I . getPointerOperand ( ) , PT ) ; Value * WidenLoad = Builder . CreateLoad ( BitCast ) ; int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ; Value * ValOrig = Builder . CreateBitCast ( ValTrunc , I . getType ( ) ) ; I . replaceAllUsesWith ( ValOrig ) ; I . eraseFromParent ( ) ; return true ; }


<s> Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else { NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } ) ; }

NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ;


<s> FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; bool UnsafeDiv = HasUnsafeFPMath || FMF . isFast ( ) || FMF . allowReciprocal ( ) ; if ( ST -> hasFP32Denormals ( ) || UnsafeDiv ) return false ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) , FPMath ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ;

Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ;


<s> Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * PT = PointerType :: get ( I32Ty , I . getPointerAddressSpace ( ) ) ; Value * BitCast = Builder . CreateBitCast ( I . getPointerOperand ( ) , PT ) ; Value * WidenLoad = Builder . CreateLoad ( BitCast ) ; int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ;

Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ;


<s> return ScalarSize >= <NUM_LIT> && numBitsSigned ( V , ScalarSize ) < <NUM_LIT> ;


<s> bool <ISA_LIT>CodeGenPrepare :: isI24 ( Value * V , unsigned ScalarSize ) const { return ScalarSize >= <NUM_LIT> && numBitsSigned ( V , ScalarSize ) < <NUM_LIT> ;


<s> bool <ISA_LIT>CodeGenPrepare :: isU24 ( Value * V , unsigned ScalarSize ) const { return numBitsUnsigned ( V , ScalarSize ) <= <NUM_LIT> ;


<s> for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IntrID == <STR_LIT> ) { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = Builder . CreateCall ( Intrin , { LHS , RHS } ) ; if ( IntrID == <STR_LIT> ) { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } else { ResultVals . push_back ( Builder . CreateSExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; }


<s> const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; const float ReqdAccuracy = FPOp -> getFPAccuracy ( ) ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; const bool AllowInaccurateRcp = HasUnsafeFPMath || FMF . approxFunc ( ) ; const bool RcpIsAccurate = ( Ty -> isHalfTy ( ) && ReqdAccuracy >= <NUM_LIT> ) || ( Ty -> isFloatTy ( ) && ! HasFP32Denormals && ReqdAccuracy >= <NUM_LIT> ) ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( auto * VT = dyn_cast < FixedVectorType > ( FDiv . getType ( ) ) ) { NewFDiv = UndefValue :: get ( VT ) ;

for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) {


<s> WidenLoad -> copyMetadata ( I ) ; if ( auto * Range = WidenLoad -> getMetadata ( LLVMContext :: MD_range ) ) { ConstantInt * Lower = mdconst :: extract < ConstantInt > ( Range -> getOperand ( <NUM_LIT> ) ) ; if ( Lower -> getValue ( ) . isNullValue ( ) ) { WidenLoad -> setMetadata ( LLVMContext :: MD_range , nullptr ) ; } else { Metadata * LowAndHigh [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , Lower -> getValue ( ) . zext ( <NUM_LIT> ) ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , <NUM_LIT> ) ) } ; WidenLoad -> setMetadata ( LLVMContext :: MD_range , MDNode :: get ( Mod -> getContext ( ) , LowAndHigh ) ) ; } } int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ; Value * ValOrig = Builder . CreateBitCast ( ValTrunc , I . getType ( ) ) ;


<s> <ISA_LIT>CodeGenPrepare ( const TargetMachine * TM = nullptr ) :

FunctionPass ( ID ) , TM ( static_cast < const GCNTargetMachine * > ( TM ) ) {


<s> FunctionPass ( ID ) , TM ( static_cast < const GCNTargetMachine * > ( TM ) ) {


<s> BinOp -> setHasNoSignedWrap ( I . hasNoSignedWrap ( ) ) ; BinOp -> setHasNoUnsignedWrap ( I . hasNoUnsignedWrap ( ) ) ;

} else if ( isa < PossiblyExactOperator > ( BinOp ) ) BinOp -> setIsExact ( I . isExact ( ) ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>CodeGenPreparePass ( const GCNTargetMachine * TM ) {

return new <ISA_LIT>CodeGenPrepare ( TM ) ;


<s> if ( ! T -> isVectorTy ( ) ) return false ; return needsPromotionToI32 ( cast < VectorType > ( T ) -> getElementType ( ) ) ;


<s> Type * I32Ty = getI32Ty ( Builder , I . getType ( ) ) ; Value * ExtOp0 = nullptr ; Value * ExtOp1 = nullptr ; Value * ExtRes = nullptr ; Value * TruncRes = nullptr ; if ( isSigned ( I ) ) { ExtOp0 = Builder . CreateSExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ;

ExtOp1 = Builder . CreateSExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ;


<s> Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; Type * I32Ty = getI32Ty ( Builder , I . getType ( ) ) ; Value * ExtOp0 = nullptr ; Value * ExtOp1 = nullptr ; Value * ExtRes = nullptr ; Value * TruncRes = nullptr ; if ( isSigned ( I ) ) {

ExtOp0 = Builder . CreateSExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ;


<s> DA = & getAnalysis < DivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator Next ;

for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) {


<s> BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ;


<s> const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; float ULP = FPOp -> getFPAccuracy ( ) ; if ( ULP < <NUM_LIT> ) return false ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; bool UnsafeDiv = HasUnsafeFPMath || FMF . unsafeAlgebra ( ) || FMF . allowReciprocal ( ) ; if ( ST -> hasFP32Denormals ( ) && ! UnsafeDiv ) return false ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) , FPMath ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; const <ISA_LIT>IntrinsicInfo * II = TM -> getIntrinsicInfo ( ) ; Function * Decl = II -> getDeclaration ( Mod , <STR_LIT> , { } ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ;


<s> bool <ISA_LIT>CodeGenPrepare :: canWidenScalarExtLoad ( LoadInst & I ) const { Type * Ty = I . getType ( ) ; const DataLayout & DL = Mod -> getDataLayout ( ) ;

int TySize = DL . getTypeSizeInBits ( Ty ) ;


<s> bool <ISA_LIT>CodeGenPrepare :: canWidenScalarExtLoad ( LoadInst & I ) const { Type * Ty = I . getType ( ) ; const DataLayout & DL = Mod -> getDataLayout ( ) ; int TySize = DL . getTypeSizeInBits ( Ty ) ; Align Alignment = DL . getValueOrABITypeAlignment ( I . getAlign ( ) , Ty ) ;

return I . isSimple ( ) && TySize < <NUM_LIT> && Alignment >= <NUM_LIT> && DA -> isUniform ( & I ) ;


<s> return false ; } if ( BinaryOperator * BinOpDen = dyn_cast < BinaryOperator > ( Den ) ) {

if ( BinOpDen -> getOpcode ( ) == Instruction :: Shl && isa < Constant > ( BinOpDen -> getOperand ( <NUM_LIT> ) ) && isKnownToBeAPowerOfTwo ( BinOpDen -> getOperand ( <NUM_LIT> ) , * DL , true , <NUM_LIT> , AC , & I , DT ) ) {


<s> } if ( BinaryOperator * BinOpDen = dyn_cast < BinaryOperator > ( Den ) ) { if ( BinOpDen -> getOpcode ( ) == Instruction :: Shl && isa < Constant > ( BinOpDen -> getOperand ( <NUM_LIT> ) ) && isKnownToBeAPowerOfTwo ( BinOpDen -> getOperand ( <NUM_LIT> ) , * DL , true , <NUM_LIT> , AC , & I , DT ) ) { return true ; }


<s> bool <ISA_LIT>CodeGenPrepare :: doInitialization ( Module & M ) {


<s> Value * <ISA_LIT>CodeGenPrepare :: expandDivRem24 ( IRBuilder < > & Builder , BinaryOperator & I , Value * Num , Value * Den , bool IsDiv , bool IsSigned ) const { int DivBits = getDivNumBits ( I , Num , Den , <NUM_LIT> , IsSigned ) ;

if ( DivBits == - <NUM_LIT> ) return nullptr ;


<s> Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Function * RcpDecl = <STR_LIT> ( Mod , <STR_LIT> , Builder . getFloatTy ( ) ) ; Value * RCP = Builder . CreateCall ( RcpDecl , { FB } ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FQM ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; auto FMAD = ! ST -> hasMadMacF32Insts ( ) ? <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; Value * FR = Builder . CreateIntrinsic ( FMAD , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) { Value * Rem = Builder . CreateMul ( Div , Den ) ; Res = Builder . CreateSub ( Num , Rem ) ; } if ( DivBits != <NUM_LIT> && DivBits < <NUM_LIT> ) { if ( IsSigned ) { int InRegBits = <NUM_LIT> - DivBits ; Res = Builder . CreateShl ( Res , InRegBits ) ; Res = Builder . CreateAShr ( Res , InRegBits ) ; } else { ConstantInt * TruncMask = Builder . getInt32 ( ( UINT64_C ( <NUM_LIT> ) << DivBits ) - <NUM_LIT> ) ; Res = Builder . CreateAnd ( Res , TruncMask ) ; }


<s> X = Builder . CreateSExt ( X , I32Ty ) ; Y = Builder . CreateSExt ( Y , I32Ty ) ; } else { X = Builder . CreateZExt ( X , I32Ty ) ; Y = Builder . CreateZExt ( Y , I32Ty ) ; } } if ( Value * Res = expandDivRem24 ( Builder , I , X , Y , IsDiv , IsSigned ) ) { return IsSigned ? Builder . CreateSExtOrTrunc ( Res , Ty ) : Builder . CreateZExtOrTrunc ( Res , Ty ) ; } ConstantInt * Zero = Builder . getInt32 ( <NUM_LIT> ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * Sign = nullptr ; if ( IsSigned ) { Value * SignX = getSign32 ( X , Builder , DL ) ; Value * SignY = getSign32 ( Y , Builder , DL ) ; Sign = IsDiv ? Builder . CreateXor ( SignX , SignY ) : SignX ; X = Builder . CreateAdd ( X , SignX ) ; Y = Builder . CreateAdd ( Y , SignY ) ; X = Builder . CreateXor ( X , SignX ) ; Y = Builder . CreateXor ( Y , SignY ) ; } Value * FloatY = Builder . CreateUIToFP ( Y , F32Ty ) ; Function * Rcp = <STR_LIT> ( Mod , <STR_LIT> , F32Ty ) ; Value * RcpY = Builder . CreateCall ( Rcp , { FloatY } ) ; Constant * Scale = ConstantFP :: get ( F32Ty , BitsToFloat ( <NUM_LIT> ) ) ; Value * ScaledY = Builder . CreateFMul ( RcpY , Scale ) ; Value * Z = Builder . CreateFPToUI ( ScaledY , I32Ty ) ; Value * NegY = Builder . CreateSub ( Zero , Y ) ; Value * NegYZ = Builder . CreateMul ( NegY , Z ) ; Z = Builder . CreateAdd ( Z , getMulHu ( Builder , Z , NegYZ ) ) ;

Value * Q = getMulHu ( Builder , X , Z ) ;


<s> Value * Z = Builder . CreateFPToUI ( ScaledY , I32Ty ) ; Value * NegY = Builder . CreateSub ( Zero , Y ) ; Value * NegYZ = Builder . CreateMul ( NegY , Z ) ; Z = Builder . CreateAdd ( Z , getMulHu ( Builder , Z , NegYZ ) ) ; Value * Q = getMulHu ( Builder , X , Z ) ; Value * R = Builder . CreateSub ( X , Builder . CreateMul ( Q , Y ) ) ; Value * Cond = Builder . CreateICmpUGE ( R , Y ) ; if ( IsDiv ) Q = Builder . CreateSelect ( Cond , Builder . CreateAdd ( Q , One ) , Q ) ; R = Builder . CreateSelect ( Cond , Builder . CreateSub ( R , Y ) , R ) ; Cond = Builder . CreateICmpUGE ( R , Y ) ; Value * Res ; if ( IsDiv ) Res = Builder . CreateSelect ( Cond , Builder . CreateAdd ( Q , One ) , Q ) ; else Res = Builder . CreateSelect ( Cond , Builder . CreateSub ( R , Y ) , R ) ; if ( IsSigned ) { Res = Builder . CreateXor ( Res , Sign ) ; Res = Builder . CreateSub ( Res , Sign ) ; } Res = Builder . CreateTrunc ( Res , Ty ) ; return Res ;


<s> if ( Opc == Instruction :: URem || Opc == Instruction :: SRem ) { expandRemainderUpTo64Bits ( & I ) ; return ;


<s> static void extractValues ( IRBuilder < > & Builder , SmallVectorImpl < Value * > & Values , Value * V ) { auto * VT = dyn_cast < FixedVectorType > ( V -> getType ( ) ) ; if ( ! VT ) { Values . push_back ( V ) ;


<s> } for ( int I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) Values . push_back ( Builder . CreateExtractElement ( V , I ) ) ;


<s> if ( SelectInst * Sel = dyn_cast < SelectInst > ( V ) ) return Sel ; if ( ( Cast = dyn_cast < CastInst > ( V ) ) ) { if ( SelectInst * Sel = dyn_cast < SelectInst > ( Cast -> getOperand ( <NUM_LIT> ) ) ) return Sel ;


<s> bool <ISA_LIT>CodeGenPrepare :: foldBinOpIntoSelect ( BinaryOperator & BO ) const { int SelOpNo = <NUM_LIT> ; CastInst * CastOp ; SelectInst * Sel = findSelectThroughCast ( BO . getOperand ( <NUM_LIT> ) , CastOp ) ; if ( ! Sel || ! Sel -> hasOneUse ( ) ) { SelOpNo = <NUM_LIT> ; Sel = findSelectThroughCast ( BO . getOperand ( <NUM_LIT> ) , CastOp ) ; } if ( ! Sel || ! Sel -> hasOneUse ( ) ) return false ; Constant * CT = dyn_cast < Constant > ( Sel -> getTrueValue ( ) ) ; Constant * CF = dyn_cast < Constant > ( Sel -> getFalseValue ( ) ) ; Constant * CBO = dyn_cast < Constant > ( BO . getOperand ( SelOpNo ^ <NUM_LIT> ) ) ; if ( ! CBO || ! CT || ! CF ) return false ; if ( CastOp ) { if ( ! CastOp -> hasOneUse ( ) ) return false ; CT = ConstantFoldCastOperand ( CastOp -> getOpcode ( ) , CT , BO . getType ( ) , * DL ) ; CF = ConstantFoldCastOperand ( CastOp -> getOpcode ( ) , CF , BO . getType ( ) , * DL ) ; } Constant * FoldedT = SelOpNo ? ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CBO , CT , * DL ) : ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CT , CBO , * DL ) ; if ( isa < ConstantExpr > ( FoldedT ) ) return false ; Constant * FoldedF = SelOpNo ? ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CBO , CF , * DL ) : ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CF , CBO , * DL ) ; if ( isa < ConstantExpr > ( FoldedF ) ) return false ;


<s> AU . addRequired < LegacyDivergenceAnalysis > ( ) ;

if ( ! ExpandDiv64InIR ) AU . setPreservesAll ( ) ;


<s> const DataLayout & DL = Mod -> getDataLayout ( ) ; unsigned LHSSignBits = ComputeNumSignBits ( Num , DL , <NUM_LIT> , AC , & I ) ; if ( LHSSignBits < AtLeast ) return - <NUM_LIT> ; unsigned RHSSignBits = ComputeNumSignBits ( Den , DL , <NUM_LIT> , AC , & I ) ;

if ( RHSSignBits < AtLeast ) return - <NUM_LIT> ;


<s> Type * <ISA_LIT>CodeGenPrepare :: getI32Ty ( IRBuilder < > & B , const Type * T ) const { assert ( needsPromotionToI32 ( T ) && <STR_LIT> ) ; if ( T -> isIntegerTy ( ) ) return B . getInt32Ty ( ) ;

return FixedVectorType :: get ( B . getInt32Ty ( ) , cast < FixedVectorType > ( T ) ) ;


<s> if ( Known . isNonNegative ( ) ) return Constant :: getNullValue ( V -> getType ( ) ) ;

return Builder . CreateAShr ( V , Builder . getInt32 ( <NUM_LIT> ) ) ;


<s> if ( Known . isNegative ( ) ) return Constant :: getAllOnesValue ( V -> getType ( ) ) ; if ( Known . isNonNegative ( ) ) return Constant :: getNullValue ( V -> getType ( ) ) ; return Builder . CreateAShr ( V , Builder . getInt32 ( <NUM_LIT> ) ) ;


<s> if ( Values . size ( ) == <NUM_LIT> ) return Values [ <NUM_LIT> ] ; Value * NewVal = UndefValue :: get ( Ty ) ; for ( int I = <NUM_LIT> , E = Values . size ( ) ; I != E ; ++ I ) NewVal = Builder . CreateInsertElement ( NewVal , Values [ I ] , I ) ; return NewVal ;


<s> if ( IntTy && IntTy -> getBitWidth ( ) > <NUM_LIT> && IntTy -> getBitWidth ( ) <= <NUM_LIT> ) return true ; if ( const VectorType * VT = dyn_cast < VectorType > ( T ) ) { if ( ST -> hasVOP3PInsts ( ) ) return false ; return needsPromotionToI32 ( VT -> getElementType ( ) ) ; } return false ;


<s> if ( ! Widen16BitOps ) return false ; const IntegerType * IntTy = dyn_cast < IntegerType > ( T ) ; if ( IntTy && IntTy -> getBitWidth ( ) > <NUM_LIT> && IntTy -> getBitWidth ( ) <= <NUM_LIT> ) return true ;

if ( const VectorType * VT = dyn_cast < VectorType > ( T ) ) {


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsSigned ( Value * Op , unsigned ScalarSize ) const {

return ScalarSize - ComputeNumSignBits ( Op , * DL , <NUM_LIT> , AC ) ;


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsUnsigned ( Value * Op , unsigned ScalarSize ) const { KnownBits Known = computeKnownBits ( Op , * DL , <NUM_LIT> , AC ) ; return ScalarSize - Known . countMinLeadingZeros ( ) ;


<s> Type * Ty = Den -> getType ( ) ; if ( ! Ty -> isFloatTy ( ) ) return nullptr ; bool NumIsOne = false ; if ( const ConstantFP * CNum = dyn_cast < ConstantFP > ( Num ) ) {

if ( CNum -> isExactlyValue ( + <NUM_LIT> ) || CNum -> isExactlyValue ( - <NUM_LIT> ) ) NumIsOne = true ;


<s> if ( Size <= <NUM_LIT> && ST -> has16BitInsts ( ) ) return false ; if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; unsigned LHSBits = <NUM_LIT> , RHSBits = <NUM_LIT> ; if ( ST -> hasMulU24 ( ) && ( LHSBits = numBitsUnsigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsUnsigned ( RHS , Size ) ) <= <NUM_LIT> ) { if ( Size > <NUM_LIT> && LHSBits + RHSBits > <NUM_LIT> ) return false ; IntrID = <STR_LIT> ; } else if ( ST -> hasMulI24 ( ) && ( LHSBits = numBitsSigned ( LHS , Size ) ) < <NUM_LIT> && ( RHSBits = numBitsSigned ( RHS , Size ) ) < <NUM_LIT> ) { if ( Size > <NUM_LIT> && LHSBits + RHSBits > <NUM_LIT> ) return false ; IntrID = <STR_LIT> ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ;

FunctionCallee Intrin = <STR_LIT> ( Mod , IntrID ) ;


<s> if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; unsigned LHSBits = <NUM_LIT> , RHSBits = <NUM_LIT> ; if ( ST -> hasMulU24 ( ) && ( LHSBits = numBitsUnsigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsUnsigned ( RHS , Size ) ) <= <NUM_LIT> ) { if ( Size > <NUM_LIT> && LHSBits + RHSBits > <NUM_LIT> ) return false ; IntrID = <STR_LIT> ; } else if ( ST -> hasMulI24 ( ) && ( LHSBits = numBitsSigned ( LHS , Size ) ) < <NUM_LIT> && ( RHSBits = numBitsSigned ( RHS , Size ) ) < <NUM_LIT> ) { if ( Size > <NUM_LIT> && LHSBits + RHSBits > <NUM_LIT> ) return false ; IntrID = <STR_LIT> ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; FunctionCallee Intrin = <STR_LIT> ( Mod , IntrID ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IntrID == <STR_LIT> ) { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = Builder . CreateCall ( Intrin , { LHS , RHS } ) ; if ( IntrID == <STR_LIT> ) { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ;


<s> NextBB = std :: next ( FI ) ; BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB -> begin ( ) , E = BB -> end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MadeChange |= visit ( * I ) ; if ( Next != E ) { BasicBlock * NextInstBB = Next -> getParent ( ) ; if ( NextInstBB != BB ) { BB = NextInstBB ; E = BB -> end ( ) ;


<s> int NumDivBits = getDivNumBits ( I , Num , Den , <NUM_LIT> , IsSigned ) ; if ( NumDivBits == - <NUM_LIT> ) return nullptr ; Value * Narrowed = nullptr ; if ( NumDivBits <= <NUM_LIT> ) { Narrowed = expandDivRem24Impl ( Builder , I , Num , Den , NumDivBits , IsDiv , IsSigned ) ; } else if ( NumDivBits <= <NUM_LIT> ) { Narrowed = expandDivRem32 ( Builder , I , Num , Den ) ; } if ( Narrowed ) { return IsSigned ? Builder . CreateSExt ( Narrowed , Num -> getType ( ) ) : Builder . CreateZExt ( Narrowed , Num -> getType ( ) ) ;


<s> NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; } else { NewElt = shrinkDivRem64 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) { NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; Div64ToExpand . push_back ( cast < BinaryOperator > ( NewElt ) ) ; } } NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { if ( ScalarSize <= <NUM_LIT> ) NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; else { NewDiv = shrinkDivRem64 ( Builder , I , Num , Den ) ; if ( ! NewDiv ) Div64ToExpand . push_back ( & I ) ; } } if ( NewDiv ) { I . replaceAllUsesWith ( NewDiv ) ; I . eraseFromParent ( ) ; Changed = true ; } } if ( ExpandDiv64InIR ) { for ( BinaryOperator * Div : Div64ToExpand ) { expandDivRem64 ( * Div ) ; Changed = true ; } } return Changed ;


<s> if ( Ty -> isHalfTy ( ) && ! ST -> has16BitInsts ( ) ) return false ; const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; const float ReqdAccuracy = FPOp -> getFPAccuracy ( ) ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; const bool AllowInaccurateRcp = HasUnsafeFPMath || FMF . approxFunc ( ) ; const bool RcpIsAccurate = ( Ty -> isHalfTy ( ) && ReqdAccuracy >= <NUM_LIT> ) || ( Ty -> isFloatTy ( ) && ! HasFP32Denormals && ReqdAccuracy >= <NUM_LIT> ) ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( auto * VT = dyn_cast < FixedVectorType > ( FDiv . getType ( ) ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt = optimizeWithRcp ( NumEltI , DenEltI , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ; if ( ! NewElt ) NewElt = optimizeWithFDivFast ( NumEltI , DenEltI , ReqdAccuracy , HasFP32Denormals , Builder , Mod ) ; if ( ! NewElt ) NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; }


<s> LoadInst * WidenLoad = Builder . CreateLoad ( I32Ty , BitCast ) ; WidenLoad -> copyMetadata ( I ) ; if ( auto * Range = WidenLoad -> getMetadata ( LLVMContext :: MD_range ) ) { ConstantInt * Lower = mdconst :: extract < ConstantInt > ( Range -> getOperand ( <NUM_LIT> ) ) ; if ( Lower -> isNullValue ( ) ) { WidenLoad -> setMetadata ( LLVMContext :: MD_range , nullptr ) ; } else { Metadata * LowAndHigh [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , Lower -> getValue ( ) . zext ( <NUM_LIT> ) ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , <NUM_LIT> ) ) } ; WidenLoad -> setMetadata ( LLVMContext :: MD_range , MDNode :: get ( Mod -> getContext ( ) , LowAndHigh ) ) ; } } int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ; Value * ValOrig = Builder . CreateBitCast ( ValTrunc , I . getType ( ) ) ;


<s> ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * JQ = One ; if ( IsSigned ) { JQ = Builder . CreateXor ( Num , Den ) ; JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateIntrinsic ( <STR_LIT> , { FQM } ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateIntrinsic ( <STR_LIT> , { FR } , FQ ) ; FB = Builder . CreateIntrinsic ( <STR_LIT> , { FB } , FQ ) ;

Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ;


<s> AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < DivergenceAnalysis > ( ) ;


<s> LoadInst * WidenLoad = Builder . CreateLoad ( BitCast ) ; WidenLoad -> copyMetadata ( I ) ; if ( auto * Range = WidenLoad -> getMetadata ( LLVMContext :: MD_range ) ) { ConstantInt * Lower = mdconst :: extract < ConstantInt > ( Range -> getOperand ( <NUM_LIT> ) ) ; if ( Lower -> getValue ( ) . isNullValue ( ) ) { WidenLoad -> setMetadata ( LLVMContext :: MD_range , nullptr ) ; } else { Metadata * LowAndHigh [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , Lower -> getValue ( ) . zext ( <NUM_LIT> ) ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , <NUM_LIT> ) ) } ; WidenLoad -> setMetadata ( LLVMContext :: MD_range , MDNode :: get ( Mod -> getContext ( ) , LowAndHigh ) ) ;


<s> JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Function * RcpDecl = <STR_LIT> ( Mod , <STR_LIT> , Builder . getFloatTy ( ) ) ; Value * RCP = Builder . CreateCall ( RcpDecl , { FB } ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FQM ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) {

Value * Rem = Builder . CreateMul ( Div , Den ) ;


<s> FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) { Value * Rem = Builder . CreateMul ( Div , Den ) ; Res = Builder . CreateSub ( Num , Rem ) ; } if ( DivBits != <NUM_LIT> && DivBits < <NUM_LIT> ) { if ( IsSigned ) { int InRegBits = <NUM_LIT> - DivBits ; Res = Builder . CreateShl ( Res , InRegBits ) ; Res = Builder . CreateAShr ( Res , InRegBits ) ; } else {

ConstantInt * TruncMask = Builder . getInt32 ( ( UINT64_C ( <NUM_LIT> ) << DivBits ) - <NUM_LIT> ) ;


<s> Value * <ISA_LIT>CodeGenPrepare :: expandDivRem32 ( IRBuilder < > & Builder , BinaryOperator & I , Value * Num , Value * Den ) const { Instruction :: BinaryOps Opc = I . getOpcode ( ) ; assert ( Opc == Instruction :: URem || Opc == Instruction :: UDiv || Opc == Instruction :: SRem || Opc == Instruction :: SDiv ) ; FastMathFlags FMF ; FMF . setFast ( ) ; Builder . setFastMathFlags ( FMF ) ; if ( divHasSpecialOptimization ( I , Num , Den ) ) return nullptr ; bool IsDiv = Opc == Instruction :: UDiv || Opc == Instruction :: SDiv ; bool IsSigned = Opc == Instruction :: SRem || Opc == Instruction :: SDiv ; Type * Ty = Num -> getType ( ) ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * F32Ty = Builder . getFloatTy ( ) ; if ( Ty -> getScalarSizeInBits ( ) < <NUM_LIT> ) { if ( IsSigned ) { Num = Builder . CreateSExt ( Num , I32Ty ) ; Den = Builder . CreateSExt ( Den , I32Ty ) ; } else { Num = Builder . CreateZExt ( Num , I32Ty ) ; Den = Builder . CreateZExt ( Den , I32Ty ) ; } } if ( Value * Res = expandDivRem24 ( Builder , I , Num , Den , IsDiv , IsSigned ) ) { return IsSigned ? Builder . CreateSExtOrTrunc ( Res , Ty ) : Builder . CreateZExtOrTrunc ( Res , Ty ) ; } ConstantInt * Zero = Builder . getInt32 ( <NUM_LIT> ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * Sign = nullptr ; if ( IsSigned ) { ConstantInt * K31 = Builder . getInt32 ( <NUM_LIT> ) ; Value * LHSign = Builder . CreateAShr ( Num , K31 ) ; Value * RHSign = Builder . CreateAShr ( Den , K31 ) ; Sign = IsDiv ? Builder . CreateXor ( LHSign , RHSign ) : LHSign ; Num = Builder . CreateAdd ( Num , LHSign ) ; Den = Builder . CreateAdd ( Den , RHSign ) ;

Num = Builder . CreateXor ( Num , LHSign ) ;


<s> static void extractValues ( IRBuilder < > & Builder , SmallVectorImpl < Value * > & Values , Value * V ) {

VectorType * VT = dyn_cast < VectorType > ( V -> getType ( ) ) ;


<s> DT = DTWP ? & DTWP -> getDomTree ( ) : nullptr ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; HasFP32Denormals = ST -> hasFP32Denormals ( F ) ; bool MadeChange = false ; Function :: iterator NextBB ; for ( Function :: iterator FI = F . begin ( ) , FE = F . end ( ) ; FI != FE ; FI = NextBB ) { BasicBlock * BB = & * FI ; NextBB = std :: next ( FI ) ; BasicBlock :: iterator Next ;

for ( BasicBlock :: iterator I = BB -> begin ( ) , E = BB -> end ( ) ; I != E ; I = Next ) {


<s> if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt ; if ( ScalarSize <= <NUM_LIT> ) { NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; } else { NewElt = shrinkDivRem64 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) { NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; Div64ToExpand . push_back ( cast < BinaryOperator > ( NewElt ) ) ; } } NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { if ( ScalarSize <= <NUM_LIT> ) NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; else { NewDiv = shrinkDivRem64 ( Builder , I , Num , Den ) ; if ( ! NewDiv ) Div64ToExpand . push_back ( & I ) ; } } if ( NewDiv ) {


<s> if ( foldBinOpIntoSelect ( I ) ) return true ; if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) && promoteUniformOpToI32 ( I ) ) return true ; if ( UseMul24Intrin && replaceMulWithMul24 ( I ) ) return true ; bool Changed = false ; Instruction :: BinaryOps Opc = I . getOpcode ( ) ; Type * Ty = I . getType ( ) ; Value * NewDiv = nullptr ; unsigned ScalarSize = Ty -> getScalarSizeInBits ( ) ; SmallVector < BinaryOperator * , <NUM_LIT> > Div64ToExpand ; if ( ( Opc == Instruction :: URem || Opc == Instruction :: UDiv || Opc == Instruction :: SRem || Opc == Instruction :: SDiv ) && ScalarSize <= <NUM_LIT> && ! DisableIDivExpand ) { Value * Num = I . getOperand ( <NUM_LIT> ) ; Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ;

Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ;


<s> Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt = optimizeWithRcp ( NumEltI , DenEltI , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ; if ( ! NewElt ) NewElt = optimizeWithFDivFast ( NumEltI , DenEltI , ReqdAccuracy , HasFP32Denormals , Builder , Mod ) ; if ( ! NewElt ) NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else { NewFDiv = optimizeWithRcp ( Num , Den , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ; if ( ! NewFDiv ) { NewFDiv = optimizeWithFDivFast ( Num , Den , ReqdAccuracy , HasFP32Denormals , Builder , Mod ) ; } }


<s> Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( FDiv . getType ( ) ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt = optimizeWithRcp ( NumEltI , DenEltI , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ; if ( ! NewElt ) NewElt = optimizeWithFDivFast ( NumEltI , DenEltI , ReqdAccuracy , HasFP32Denormals , Builder , Mod ) ; if ( ! NewElt ) NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else {

NewFDiv = optimizeWithRcp ( Num , Den , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ;


<s> Value * RCP_F32 = Builder . CreateCall ( RcpDecl , { DEN_F32 } ) ; Constant * UINT_MAX_PLUS_1 = ConstantFP :: get ( F32Ty , BitsToFloat ( <NUM_LIT> ) ) ; Value * RCP_SCALE = Builder . CreateFMul ( RCP_F32 , UINT_MAX_PLUS_1 ) ; Value * RCP = Builder . CreateFPToUI ( RCP_SCALE , I32Ty ) ; Value * RCP_LO , * RCP_HI ; std :: tie ( RCP_LO , RCP_HI ) = getMul64 ( Builder , RCP , Den ) ; Value * NEG_RCP_LO = Builder . CreateNeg ( RCP_LO ) ; Value * RCP_HI_0_CC = Builder . CreateICmpEQ ( RCP_HI , Zero ) ; Value * ABS_RCP_LO = Builder . CreateSelect ( RCP_HI_0_CC , NEG_RCP_LO , RCP_LO ) ; Value * E = getMulHu ( Builder , ABS_RCP_LO , RCP ) ; Value * RCP_A_E = Builder . CreateAdd ( RCP , E ) ; Value * RCP_S_E = Builder . CreateSub ( RCP , E ) ; Value * Tmp0 = Builder . CreateSelect ( RCP_HI_0_CC , RCP_A_E , RCP_S_E ) ; Value * Quotient = getMulHu ( Builder , Tmp0 , Num ) ; Value * Num_S_Remainder = Builder . CreateMul ( Quotient , Den ) ; Value * Remainder = Builder . CreateSub ( Num , Num_S_Remainder ) ; Value * Remainder_GE_Den = Builder . CreateICmpUGE ( Remainder , Den ) ; Value * Remainder_GE_Zero = Builder . CreateICmpUGE ( Num , Num_S_Remainder ) ; Value * Tmp1 = Builder . CreateAnd ( Remainder_GE_Den , Remainder_GE_Zero ) ; Value * Res ; if ( IsDiv ) { Value * Quotient_A_One = Builder . CreateAdd ( Quotient , One ) ; Value * Quotient_S_One = Builder . CreateSub ( Quotient , One ) ; Value * Div = Builder . CreateSelect ( Tmp1 , Quotient_A_One , Quotient ) ; Res = Builder . CreateSelect ( Remainder_GE_Zero , Div , Quotient_S_One ) ; } else { Value * Remainder_S_Den = Builder . CreateSub ( Remainder , Den ) ; Value * Remainder_A_Den = Builder . CreateAdd ( Remainder , Den ) ; Value * Rem = Builder . CreateSelect ( Tmp1 , Remainder_S_Den , Remainder ) ; Res = Builder . CreateSelect ( Remainder_GE_Zero , Rem , Remainder_A_Den ) ; }


<s> const ConstantFP * CNum = dyn_cast < ConstantFP > ( Num ) ; if ( ! CNum ) return false ; return UnsafeDiv && CNum -> isExactlyValue ( + <NUM_LIT> ) ;


<s> const ConstantFP * CNum = dyn_cast < ConstantFP > ( Num ) ;


<s> ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * JQ = One ; if ( IsSigned ) { JQ = Builder . CreateXor ( Num , Den ) ; JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateIntrinsic ( <STR_LIT> , { FQM } ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg , FB , FA } , FQ ) ;

Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ;


<s> JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateIntrinsic ( <STR_LIT> , { FQM } ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateIntrinsic ( <STR_LIT> , { FR } , FQ ) ; FB = Builder . CreateIntrinsic ( <STR_LIT> , { FB } , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) {


<s> const <ISA_LIT>TargetMachine & TM = TPC -> getTM < <ISA_LIT>TargetMachine > ( ) ; ST = & TM . getSubtarget < GCNSubtarget > ( F ) ; DA = & getAnalysis < DivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; AMDGPUA<ISA_LIT> = TM . get<ISA_LIT>AS ( ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) {


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) && promoteUniformOpToI32 ( I ) ) return true ; bool Changed = false ; Instruction :: BinaryOps Opc = I . getOpcode ( ) ; Type * Ty = I . getType ( ) ; Value * NewDiv = nullptr ; if ( ( Opc == Instruction :: URem || Opc == Instruction :: UDiv || Opc == Instruction :: SRem || Opc == Instruction :: SDiv ) && Ty -> getScalarSizeInBits ( ) <= <NUM_LIT> ) { Value * Num = I . getOperand ( <NUM_LIT> ) ; Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ;

Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ;


<s> Instruction :: BinaryOps Opc = I . getOpcode ( ) ; Type * Ty = I . getType ( ) ; Value * NewDiv = nullptr ; if ( ( Opc == Instruction :: URem || Opc == Instruction :: UDiv || Opc == Instruction :: SRem || Opc == Instruction :: SDiv ) && Ty -> getScalarSizeInBits ( ) <= <NUM_LIT> ) { Value * Num = I . getOperand ( <NUM_LIT> ) ; Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt = expandDivRem32 ( Builder , Opc , NumEltI , DenEltI ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltI , DenEltI ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , I ) ; } } else { NewDiv = expandDivRem32 ( Builder , Opc , Num , Den ) ;


<s> for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MadeChange |= visit ( * I ) ; }


<s> } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { Function * Decl = <STR_LIT> ( Mod , <STR_LIT> , Ty ) ; Value * FNeg = Builder . CreateFNeg ( Den ) ; return Builder . CreateCall ( Decl , { FNeg } ) ; } } } if ( AllowInaccurateRcp ) { Function * Decl = <STR_LIT> ( Mod , <STR_LIT> , Ty ) ;

Value * Recip = Builder . CreateCall ( Decl , { Den } ) ;


<s> <ISA_LIT>CodeGenPrepare ( const TargetMachine * TM = nullptr ) : FunctionPass ( ID ) , TM ( static_cast < const GCNTargetMachine * > ( TM ) ) , ST ( nullptr ) , DA ( nullptr ) , Mod ( nullptr ) , HasUnsafeFPMath ( false ) {


<s> if ( Size <= <NUM_LIT> && ST -> has16BitInsts ( ) ) return false ; if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; if ( ST -> hasMulU24 ( ) && isU24 ( LHS , Size ) && isU24 ( RHS , Size ) ) { IntrID = <STR_LIT> ; } else if ( ST -> hasMulI24 ( ) && isI24 ( LHS , Size ) && isI24 ( RHS , Size ) ) { IntrID = <STR_LIT> ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; FunctionCallee Intrin = <STR_LIT> ( Mod , IntrID ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IntrID == <STR_LIT> ) { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = Builder . CreateCall ( Intrin , { LHS , RHS } ) ; if ( IntrID == <STR_LIT> ) { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } else { ResultVals . push_back ( Builder . CreateSExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; }


<s> unsigned Size = Ty -> getScalarSizeInBits ( ) ; if ( Size <= <NUM_LIT> && ST -> has16BitInsts ( ) ) return false ; if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; if ( ST -> hasMulU24 ( ) && isU24 ( LHS , Size ) && isU24 ( RHS , Size ) ) { IntrID = <STR_LIT> ; } else if ( ST -> hasMulI24 ( ) && isI24 ( LHS , Size ) && isI24 ( RHS , Size ) ) { IntrID = <STR_LIT> ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; FunctionCallee Intrin = <STR_LIT> ( Mod , IntrID ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IntrID == <STR_LIT> ) { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = Builder . CreateCall ( Intrin , { LHS , RHS } ) ; if ( IntrID == <STR_LIT> ) { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } else {

ResultVals . push_back ( Builder . CreateSExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ;


<s> Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else {

NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ;


<s> Value * Num = I . getOperand ( <NUM_LIT> ) ; Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; } if ( NewDiv ) {


<s> if ( IsSigned ) ++ DivBits ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * F32Ty = Builder . getFloatTy ( ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * JQ = One ; if ( IsSigned ) { JQ = Builder . CreateXor ( Num , Den ) ; JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FQM ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) { Value * Rem = Builder . CreateMul ( Div , Den ) ; Res = Builder . CreateSub ( Num , Rem ) ; } if ( IsSigned ) { Res = Builder . CreateShl ( Res , <NUM_LIT> - DivBits ) ; Res = Builder . CreateAShr ( Res , <NUM_LIT> - DivBits ) ; } else { ConstantInt * TruncMask = Builder . getInt32 ( ( UINT64_C ( <NUM_LIT> ) << DivBits ) - <NUM_LIT> ) ; Res = Builder . CreateAnd ( Res , TruncMask ) ;


<s> Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FQM ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) { Value * Rem = Builder . CreateMul ( Div , Den ) ; Res = Builder . CreateSub ( Num , Rem ) ; } if ( IsSigned ) { Res = Builder . CreateShl ( Res , <NUM_LIT> - DivBits ) ;

Res = Builder . CreateAShr ( Res , <NUM_LIT> - DivBits ) ;


<s> Type * Ty = Den -> getType ( ) ; if ( ! FastUnsafeRcpLegal && Ty -> isFloatTy ( ) && ( HasDenormals || NeedHighAccuracy ) ) return nullptr ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> , Ty ) ; if ( const ConstantFP * CLHS = dyn_cast < ConstantFP > ( Num ) ) { if ( FastUnsafeRcpLegal || Ty -> isFloatTy ( ) || Ty -> isHalfTy ( ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { return Builder . CreateCall ( Decl , { Den } ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { Value * FNeg = Builder . CreateFNeg ( Den ) ; return Builder . CreateCall ( Decl , { FNeg } ) ; } } } if ( FastUnsafeRcpLegal ) {


<s> return Builder . CreateCall ( Decl , { Den } ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { Value * FNeg = Builder . CreateFNeg ( Den ) ; return Builder . CreateCall ( Decl , { FNeg } ) ; } } } if ( FastUnsafeRcpLegal ) { Value * Recip = Builder . CreateCall ( Decl , { Den } ) ; return Builder . CreateFMul ( Num , Recip , <STR_LIT> , FPMath ) ; }


<s> if ( skipFunction ( F ) ) return false ; auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; if ( ! TPC ) return false ; const <ISA_LIT>TargetMachine & TM = TPC -> getTM < <ISA_LIT>TargetMachine > ( ) ; ST = & TM . getSubtarget < GCNSubtarget > ( F ) ; AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; HasFP32Denormals = ST -> hasFP32Denormals ( F ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MadeChange |= visit ( * I ) ; } } return MadeChange ;


<s> AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; HasFP32Denormals = ST -> hasFP32Denormals ( F ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MadeChange |= visit ( * I ) ; } }


<s> if ( ! CNum ) return HasDenormals ; if ( FastUnsafeRcpLegal ) return true ; bool IsOne = CNum -> isExactlyValue ( + <NUM_LIT> ) || CNum -> isExactlyValue ( - <NUM_LIT> ) ; return HasDenormals ^ IsOne ;


<s> if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; } if ( NewDiv ) { I . replaceAllUsesWith ( NewDiv ) ; I . eraseFromParent ( ) ;


<s> if ( ( Opc == Instruction :: URem || Opc == Instruction :: UDiv || Opc == Instruction :: SRem || Opc == Instruction :: SDiv ) && Ty -> getScalarSizeInBits ( ) <= <NUM_LIT> ) { Value * Num = I . getOperand ( <NUM_LIT> ) ; Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ;

if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ;


<s> Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( FDiv . getType ( ) ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt = nullptr ; if ( UseFDivFast && ! shouldKeepFDivF32 ( NumEltI , FastUnsafeRcpLegal , HasFP32Denormals ) ) { Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } , <STR_LIT> , FPMath ) ; } if ( ! NewElt ) NewElt = performRCPOpt ( NumEltI , DenEltI , FastUnsafeRcpLegal , Builder , FPMath , Mod , HasFP32Denormals , NeedHighAccuracy ) ; if ( ! NewElt ) NewElt = Builder . CreateFDiv ( NumEltI , DenEltI , <STR_LIT> , FPMath ) ; NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else { if ( UseFDivFast && ! shouldKeepFDivF32 ( Num , FastUnsafeRcpLegal , HasFP32Denormals ) ) { Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; NewFDiv = Builder . CreateCall ( Decl , { Num , Den } , <STR_LIT> , FPMath ) ; }


<s> Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv , HasFP32Denormals ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else { NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } ) ; } NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else { if ( ! shouldKeepFDivF32 ( Num , UnsafeDiv , HasFP32Denormals ) ) NewFDiv = Builder . CreateCall ( Decl , { Num , Den } ) ; } if ( NewFDiv ) {


<s> <ISA_LIT>CodeGenPrepare ( const TargetMachine * TM = nullptr ) : FunctionPass ( ID ) , TM ( TM ) {


<s> bool <ISA_LIT>CodeGenPrepare :: doInitialization ( Module & M ) { return false ;


<s> bool <ISA_LIT>CodeGenPrepare :: runOnFunction ( Function & F ) {

if ( ! TM || skipFunction ( F ) ) return false ;


<s> DA = & getAnalysis < DivergenceAnalysis > ( ) ;


<s> if ( ! Sel || ! Sel -> hasOneUse ( ) ) { SelOpNo = <NUM_LIT> ; Sel = findSelectThroughCast ( BO . getOperand ( <NUM_LIT> ) , CastOp ) ; } if ( ! Sel || ! Sel -> hasOneUse ( ) ) return false ; Constant * CT = dyn_cast < Constant > ( Sel -> getTrueValue ( ) ) ; Constant * CF = dyn_cast < Constant > ( Sel -> getFalseValue ( ) ) ; Constant * CBO = dyn_cast < Constant > ( BO . getOperand ( SelOpNo ^ <NUM_LIT> ) ) ; if ( ! CBO || ! CT || ! CF ) return false ; if ( CastOp ) { CT = ConstantFoldCastOperand ( CastOp -> getOpcode ( ) , CT , BO . getType ( ) , * DL ) ; CF = ConstantFoldCastOperand ( CastOp -> getOpcode ( ) , CF , BO . getType ( ) , * DL ) ; } Constant * FoldedT = SelOpNo ? ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CBO , CT , * DL ) : ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CT , CBO , * DL ) ; if ( isa < ConstantExpr > ( FoldedT ) ) return false ;

Constant * FoldedF = SelOpNo ? ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CBO , CF , * DL ) :


<s> ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CT , CBO , * DL ) ; if ( isa < ConstantExpr > ( FoldedT ) ) return false ; Constant * FoldedF = SelOpNo ? ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CBO , CF , * DL ) : ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CF , CBO , * DL ) ; if ( isa < ConstantExpr > ( FoldedF ) ) return false ; IRBuilder < > Builder ( & BO ) ; Builder . SetCurrentDebugLocation ( BO . getDebugLoc ( ) ) ; if ( const FPMathOperator * FPOp = dyn_cast < const FPMathOperator > ( & BO ) ) Builder . setFastMathFlags ( FPOp -> getFastMathFlags ( ) ) ; Value * NewSelect = Builder . CreateSelect ( Sel -> getCondition ( ) , FoldedT , FoldedF ) ; NewSelect -> takeName ( & BO ) ;


<s> YIO . enumCase ( EN , <STR_LIT> , ValueType :: I8 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: U8 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: I16 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: U16 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: F16 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: I32 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: U32 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: F32 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: I64 ) ; YIO . enumCase ( EN , <STR_LIT> , ValueType :: U64 ) ;

YIO . enumCase ( EN , <STR_LIT> , ValueType :: F64 ) ;


<s> if ( TypeQual . find ( <STR_LIT> ) != StringRef :: npos ) return ValueKind :: Pipe ; return StringSwitch < ValueKind > ( BaseTypeName ) . Case ( <STR_LIT> , ValueKind :: Sampler ) . Case ( <STR_LIT> , ValueKind :: Queue ) . Cases ( <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> <STR_LIT> <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , ValueKind :: Image ) . Default ( isa < PointerType > ( Ty ) ? ( Ty -> getPointerAddressSpace ( ) == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ? ValueKind :: DynamicSharedPointer : ValueKind :: GlobalBuffer ) :


<s> static void mapping ( IO & YIO , CodeObject :: Metadata & MD ) { YIO . mapRequired ( Key :: Version , MD . mVersion ) ; YIO . mapOptional ( Key :: Printf , MD . mPrintf , std :: vector < std :: string > ( ) ) ;

if ( ! MD . mKernels . empty ( ) || ! YIO . outputting ( ) ) YIO . mapOptional ( Key :: Kernels , MD . mKernels ) ;


<s> void MetadataStreamer :: emitKernelArg ( const Argument & Arg ) { auto Func = Arg . getParent ( ) ; auto ArgNo = Arg . getArgNo ( ) ; const MDNode * Node ; StringRef TypeQual ; Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) TypeQual = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; StringRef BaseTypeName ;


<s> errs ( ) << <STR_LIT> << YamlString << '\n' ;


<s> auto & Kernel = CodeObjectMetadata . mKernels . back ( ) ; Kernel . mName = Func . getName ( ) ; emitKernelLanguage ( Func ) ;


<s> } Arg . mAccQual = getAccessQualifier ( AccQual ) ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) Arg . mAddrSpaceQual = getAddressSpaceQualifer ( PtrTy -> getAddressSpace ( ) ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ; TypeQual . split ( SplitTypeQuals , <STR_LIT> , - <NUM_LIT> , false ) ; for ( StringRef Key : SplitTypeQuals ) {

auto P = StringSwitch < bool * > ( Key ) . Case ( <STR_LIT> , & Arg . mIsConst ) . Case ( <STR_LIT> , & Arg . mIsPipe ) . Case ( <STR_LIT> , & Arg . mIsRestrict ) . Case ( <STR_LIT> , & Arg . mIsVolatile ) . Default ( nullptr ) ;


<s> void MetadataStreamer :: emitKernelArgs ( const Function & Func ) { for ( auto & Arg : Func . args ( ) ) emitKernelArg ( Arg ) ; if ( ! Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) return ; auto & DL = Func . getParent ( ) -> getDataLayout ( ) ; auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ;

emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetX ) ;


<s> if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Attrs . mWorkGroupSizeHint = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) {

Attrs . mVecTypeHint = getTypeName ( cast < ValueAsMetadata > ( Node -> getOperand ( <NUM_LIT> ) ) -> getType ( ) , mdconst :: extract < ConstantInt > ( Node -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ;


<s> CodeProps . mWorkitemPrivateSegmentSize = KernelCode . workitem_private_segment_byte_size ; CodeProps . mWavefrontNumSGPRs = KernelCode . wavefront_sgpr_count ; CodeProps . mWorkitemNumVGPRs = KernelCode . workitem_vgpr_count ;

CodeProps . mKernargSegmentAlign = KernelCode . kernarg_segment_alignment ;


<s> if ( ! ( KernelCode . code_properties & AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED ) ) return ; auto & DebugProps = CodeObjectMetadata . mKernels . back ( ) . mDebugProps ;

DebugProps . mDebuggerABIVersion . push_back ( <NUM_LIT> ) ;


<s> void MetadataStreamer :: emitKernelDebugProps ( const amd_kernel_code_t & KernelCode ) { if ( ! ( KernelCode . code_properties & AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED ) ) return ; auto & DebugProps = CodeObjectMetadata . mKernels . back ( ) . mDebugProps ; DebugProps . mDebuggerABIVersion . push_back ( <NUM_LIT> ) ; DebugProps . mDebuggerABIVersion . push_back ( <NUM_LIT> ) ; DebugProps . mReservedNumVGPRs = KernelCode . reserved_vgpr_count ; DebugProps . mReservedFirstVGPR = KernelCode . reserved_vgpr_first ; DebugProps . mPrivateSegmentBufferSGPR = KernelCode . debug_private_segment_buffer_sgpr ;


<s> if ( Op0 -> getNumOperands ( ) <= <NUM_LIT> ) return ; Kernel . mLanguage = <STR_LIT> ; Kernel . mLanguageVersion . push_back ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; Kernel . mLanguageVersion . push_back ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ;


<s> void MetadataStreamer :: emitVersion ( ) { auto & Version = CodeObjectMetadata . mVersion ; Version . push_back ( MetadataVersionMajor ) ;

Version . push_back ( MetadataVersionMinor ) ;


<s> AccessQualifier MetadataStreamer :: getAccessQualifier ( StringRef AccQual ) const { if ( AccQual . empty ( ) ) return AccessQualifier :: Unknown ; return StringSwitch < AccessQualifier > ( AccQual ) . Case ( <STR_LIT> , AccessQualifier :: ReadOnly ) . Case ( <STR_LIT> , AccessQualifier :: WriteOnly ) . Case ( <STR_LIT> , AccessQualifier :: ReadWrite ) . Default ( AccessQualifier :: Default ) ;


<s> return StringSwitch < AccessQualifier > ( AccQual ) . Case ( <STR_LIT> , AccessQualifier :: ReadOnly ) . Case ( <STR_LIT> , AccessQualifier :: WriteOnly ) . Case ( <STR_LIT> , AccessQualifier :: ReadWrite ) . Default ( AccessQualifier :: Default ) ;


<s> if ( AddressSpace == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) return AddressSpaceQualifier :: Private ; if ( AddressSpace == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) return AddressSpaceQualifier :: Global ;

if ( AddressSpace == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) return AddressSpaceQualifier :: Constant ;


<s> if ( AddressSpace == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) return AddressSpaceQualifier :: Private ; if ( AddressSpace == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) return AddressSpaceQualifier :: Global ; if ( AddressSpace == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) return AddressSpaceQualifier :: Constant ;

if ( AddressSpace == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) return AddressSpaceQualifier :: Local ;


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : return ( Twine ( 'i' ) + Twine ( BitWidth ) ) . str ( ) ; } } case Type :: HalfTyID : return <STR_LIT> ; case Type :: FloatTyID : return <STR_LIT> ; case Type :: DoubleTyID : return <STR_LIT> ; case Type :: VectorTyID : { auto VecTy = cast < VectorType > ( Ty ) ; auto ElTy = VecTy -> getElementType ( ) ; auto NumElements = VecTy -> getVectorNumElements ( ) ; return ( Twine ( getTypeName ( ElTy , Signed ) ) + Twine ( NumElements ) ) . str ( ) ;


<s> case Type :: DoubleTyID : return <STR_LIT> ; case Type :: VectorTyID : { auto VecTy = cast < VectorType > ( Ty ) ; auto ElTy = VecTy -> getElementType ( ) ; auto NumElements = VecTy -> getVectorNumElements ( ) ; return ( Twine ( getTypeName ( ElTy , Signed ) ) + Twine ( NumElements ) ) . str ( ) ; } default :


<s> return StringSwitch < ValueKind > ( BaseTypeName ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Image ) . Case ( <STR_LIT> , ValueKind :: Sampler ) . Case ( <STR_LIT> , ValueKind :: Queue ) . Default ( isa < PointerType > ( Ty ) ? ( Ty -> getPointerAddressSpace ( ) == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ? ValueKind :: DynamicSharedPointer : ValueKind :: GlobalBuffer ) : ValueKind :: ByValue ) ;


<s> case <NUM_LIT> : return Signed ? ValueType :: I8 : ValueType :: U8 ; case <NUM_LIT> : return Signed ? ValueType :: I16 : ValueType :: U16 ; case <NUM_LIT> : return Signed ? ValueType :: I32 : ValueType :: U32 ; case <NUM_LIT> : return Signed ? ValueType :: I64 : ValueType :: U64 ;


<s> if ( auto Error = Metadata :: fromYamlString ( YamlString , CodeObjectMetadata ) ) return Error ; return toYamlString ( ) ;


<s> if ( auto Error = Metadata :: fromYamlString ( YamlString , CodeObjectMetadata ) ) return Error ;


<s> std :: string ToYamlString ; if ( Metadata :: toYamlString ( FromYamlString , ToYamlString ) ) { errs ( ) << <STR_LIT> ; return ; } errs ( ) << ( YamlString == ToYamlString ? <STR_LIT> : <STR_LIT> ) << '\n' ; if ( YamlString != ToYamlString ) {

errs ( ) << <STR_LIT> << YamlString << '\n' << <STR_LIT> << ToYamlString << '\n' ;


<s> bool empty ( ) const { return ! notEmpty ( ) ;


<s> return ! mDebuggerABIVersion . empty ( ) ;


<s> Builder . setInstrAndDebugLoc ( * MatchInfo ) ; switch ( MatchInfo -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; unsigned Opposite = inverseMinMax ( MatchInfo -> getOpcode ( ) ) ; replaceOpcodeWith ( * MatchInfo , Opposite ) ; break ; } case <STR_LIT> : case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : { unsigned IntrinsicID = MatchInfo -> getIntrinsicID ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } break ; } default : llvm_unreachable ( <STR_LIT> ) ; } Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register MatchInfoDst = MatchInfo -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( MRI . hasOneNonDBGUse ( MatchInfoDst ) ) { replaceRegWith ( MRI , Dst , MatchInfoDst ) ; } else { LLT Type = MRI . getType ( Dst ) ; Register NegatedMatchInfo = MRI . createGenericVirtualRegister ( Type ) ; replaceRegOpWith ( MRI , MatchInfo -> getOperand ( <NUM_LIT> ) , NegatedMatchInfo ) ; replaceRegWith ( MRI , Dst , NegatedMatchInfo ) ; Builder . setInstrAndDebugLoc ( MI ) ; Builder . buildFNeg ( MatchInfoDst , NegatedMatchInfo , MI . getFlags ( ) ) ; } MI . eraseFromParent ( ) ; return ;


<s> } ; Builder . setInstrAndDebugLoc ( * MatchInfo ) ; switch ( MatchInfo -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; unsigned Opposite = inverseMinMax ( MatchInfo -> getOpcode ( ) ) ; replaceOpcodeWith ( * MatchInfo , Opposite ) ; break ; } case <STR_LIT> : case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : { unsigned IntrinsicID = MatchInfo -> getIntrinsicID ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ;


<s> switch ( MatchInfo -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; unsigned Opposite = inverseMinMax ( MatchInfo -> getOpcode ( ) ) ; replaceOpcodeWith ( * MatchInfo , Opposite ) ; break ; } case <STR_LIT> : case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : { unsigned IntrinsicID = MatchInfo -> getIntrinsicID ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } break ; } default :


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( const MachineInstr & Use : MRI . use_nodbg_instructions ( Dst ) ) { if ( ! hasSourceMods ( Use ) ) return false ; if ( ! opMustUseVOP3Encoding ( Use , MRI ) ) { if ( ++ NumMayIncreaseSize > CostThreshold ) return false ; } } return true ;


<s> NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; unsigned Opposite = inverseMinMax ( MatchInfo -> getOpcode ( ) ) ; replaceOpcodeWith ( * MatchInfo , Opposite ) ; break ; } case <STR_LIT> : case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : { unsigned IntrinsicID = MatchInfo -> getIntrinsicID ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : NegateEitherOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) , MatchInfo -> getOperand ( <NUM_LIT> ) ) ; NegateOperand ( MatchInfo -> getOperand ( <NUM_LIT> ) ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } break ; } default : llvm_unreachable ( <STR_LIT> ) ; } Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register MatchInfoDst = MatchInfo -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( MRI . hasOneNonDBGUse ( MatchInfoDst ) ) { replaceRegWith ( MRI , Dst , MatchInfoDst ) ; } else { LLT Type = MRI . getType ( Dst ) ; Register NegatedMatchInfo = MRI . createGenericVirtualRegister ( Type ) ; replaceRegOpWith ( MRI , MatchInfo -> getOperand ( <NUM_LIT> ) , NegatedMatchInfo ) ; replaceRegWith ( MRI , Dst , NegatedMatchInfo ) ; Builder . setInstrAndDebugLoc ( MI ) ; Builder . buildFNeg ( MatchInfoDst , NegatedMatchInfo , MI . getFlags ( ) ) ; } MI . eraseFromParent ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : { unsigned IntrinsicID = MI . getIntrinsicID ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ; } }


<s> case <STR_LIT> : return false ; case <STR_LIT> : { unsigned IntrinsicID = MI . getIntrinsicID ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; default : return true ; } }


<s> static unsigned inverseMinMax ( unsigned Opc ) { switch ( Opc ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> if ( mi_match ( Reg , MRI , m_GFCstOrSplat ( FPValReg ) ) ) { if ( FPValReg -> Value . isZero ( ) && ! FPValReg -> Value . isNegative ( ) ) return true ; const GCNSubtarget & ST = MI . getMF ( ) -> getSubtarget < GCNSubtarget > ( ) ;

if ( ST . hasInv2PiInlineImm ( ) && isInv2Pi ( FPValReg -> Value ) ) return true ;


<s> const GCNSubtarget & ST = MI . getMF ( ) -> getSubtarget < GCNSubtarget > ( ) ; if ( ST . hasInv2PiInlineImm ( ) && isInv2Pi ( FPValReg -> Value ) ) return true ; } return false ;


<s> static bool isInv2Pi ( const APFloat & APF ) {

static const APFloat KF16 ( APFloat :: IEEEhalf ( ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ! isConstantCostlierToNegate ( * MatchInfo , MatchInfo -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return mayIgnoreSignedZero ( * MatchInfo ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : { unsigned IntrinsicID = MatchInfo -> getIntrinsicID ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : return mayIgnoreSignedZero ( * MatchInfo ) ; default : return false ; } } default :


<s> static bool mayIgnoreSignedZero ( MachineInstr & MI ) { const TargetOptions & Options = MI . getMF ( ) -> getTarget ( ) . Options ; return Options . NoSignedZerosFPMath || MI . getFlag ( MachineInstr :: MIFlag :: FmNsz ) ;


<s> LLVM_READONLY static bool opMustUseVOP3Encoding ( const MachineInstr & MI , const MachineRegisterInfo & MRI ) {

return MI . getNumOperands ( ) > ( MI . getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ) || MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . getScalarSizeInBits ( ) == <NUM_LIT> ;


<s> if ( ! GV ) return false ; ConstantArray * GA = dyn_cast < ConstantArray > ( GV -> getInitializer ( ) ) ; if ( ! GA || GA -> getNumOperands ( ) == <NUM_LIT> ) return false ; Function * InitOrFiniKernel = createInitOrFiniKernelFunction ( M , IsCtor ) ;

IRBuilder < > IRB ( InitOrFiniKernel -> getEntryBlock ( ) . getTerminator ( ) ) ;


<s> for ( Value * V : GA -> operands ( ) ) { auto * CS = cast < ConstantStruct > ( V ) ; if ( Function * F = dyn_cast < Function > ( CS -> getOperand ( <NUM_LIT> ) ) ) { FunctionCallee Ctor = M . getOrInsertFunction ( F -> getName ( ) , IRB . getVoidTy ( ) ) ; IRB . CreateCall ( Ctor ) ;