<s> const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsGraphics && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( Register R : VRegs [ Idx ] ) B . buildUndef ( R ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg , Idx ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , CC ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; if ( ! IsEntryFunc && ! IsGraphics ) { TLI . allocateSpecialInputVGPRsFixed ( CCInfo , MF , * TRI , * Info ) ; } IncomingValueAssigner Assigner ( AssignFn ) ; if ( ! determineAssignments ( Assigner , SplitArgs , CCInfo ) ) return false ; FormalArgHandler Handler ( B ,


<s> for ( auto & Arg : F . args ( ) ) { const bool IsByRef = Arg . hasByRefAttr ( ) ; Type * ArgTy = IsByRef ? Arg . getParamByRefType ( ) : Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; MaybeAlign ABIAlign = IsByRef ? Arg . getParamAlign ( ) : None ; if ( ! ABIAlign ) ABIAlign = DL . getABITypeAlign ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; if ( Arg . use_empty ( ) ) { ++ i ; continue ; } Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; if ( IsByRef ) { unsigned ByRefAS = cast < PointerType > ( Arg . getType ( ) ) -> getAddressSpace ( ) ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; if ( ByRefAS == <ISA_LIT>AS :: CONSTANT_ADDRESS ) { lowerParameterPtr ( VRegs [ i ] [ <NUM_LIT> ] , B , ArgOffset ) ; } else { const LLT ConstPtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register PtrReg = MRI . createGenericVirtualRegister ( ConstPtrTy ) ; lowerParameterPtr ( PtrReg , B , ArgOffset ) ; B . buildAddrSpaceCast ( VRegs [ i


<s> MachineFunction & MF = B . getMF ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant


<s> unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Align ) ; B . buildLoad ( DstReg ,


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; Register DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; Register OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildConstant ( OffsetReg , Offset ) ; B . buildPtrAdd ( DstReg , KernArgSegmentVReg , OffsetReg ) ; return


<s> auto const & ST = MIRBuilder . getMF ( ) . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = MIRBuilder . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! lowerReturnVal ( MIRBuilder , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LiveInReturn = MF . addLiveIn ( TRI -> getReturnAddressReg ( MF ) , & <STR_LIT> ) ; MIRBuilder . buildCopy ( ReturnAddrVReg , LiveInReturn ) ; } MIRBuilder . insertInstr ( Ret ) ; return


<s> unsigned Align = inferAlignmentFromPtrInfo ( MF , MPO ) ; auto MMO = MF . getMachineMemOperand ( MPO , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant , Size , Align ) ; MIRBuilder . buildLoad ( ValVReg , Addr , *


<s> assert ( OrigArg . Regs . size ( ) == SplitVTs . size ( ) ) ; int SplitIdx = <NUM_LIT> ; for ( EVT VT : SplitVTs ) { Register Reg = OrigArg . Regs [ SplitIdx ] ; Type * Ty = VT . getTypeForEVT ( Ctx ) ; LLT LLTy = getLLTForType ( * Ty , DL ) ; if ( OrigArgIdx == AttributeList :: ReturnIndex && VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( OrigArg . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( OrigArg . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { VT = ExtVT ; Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLTy = getLLTForType ( * Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { LLTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } unsigned NumParts = TLI . getNumRegistersForCallingConv ( Ctx , CallConv , VT ) ; MVT RegVT = TLI . getRegisterTypeForCallingConv ( Ctx , CallConv , VT ) ; if ( NumParts == <NUM_LIT> ) { SplitArgs . emplace_back ( Reg , Ty , OrigArg . Flags , OrigArg . IsFixed ) ; ++ SplitIdx ; continue ; } SmallVector < Register , <NUM_LIT> > SplitRegs ; Type * PartTy = EVT ( RegVT ) . getTypeForEVT ( Ctx ) ; LLT PartLLT = getLLTForType ( * PartTy , DL ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; for ( unsigned i = <NUM_LIT> ; i <


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; CallingConv :: ID CallConv = F . getCallingConv ( ) ; if ( ! <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && CallConv != CallingConv :: <ISA_LIT>_Gfx ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( <STR_LIT> ( CallConv ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } SmallVector < ArgInfo , <NUM_LIT> > OutArgs ; for ( auto & OrigArg : Info . OrigArgs ) splitToValueTypes ( OrigArg , OutArgs , DL , Info . CallConv ) ; SmallVector < ArgInfo , <NUM_LIT> > InArgs ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) splitToValueTypes ( Info . OrigRet , InArgs , DL , Info . CallConv ) ; bool CanTailCallOpt = isEligibleForTailCallOptimization ( MIRBuilder , Info , InArgs , OutArgs ) ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( CanTailCallOpt ) return lowerTailCall ( MIRBuilder , Info , OutArgs ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs


<s> Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && ! OrigArg . Flags . isInReg ( ) && ! OrigArg . Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { if ( CurOrigArg -> use_empty ( ) && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ) { Skipped . set ( i ) ; ++ P<ISA_LIT>nputNum ; continue ; } Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! CurOrigArg -> use_empty ( ) ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; ++ P<ISA_LIT>nputNum ; } CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; if ( ValEVT . isVector ( ) ) { EVT ElemVT = ValEVT . getVectorElementType ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ElemVT . getSimpleVT ( ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( ! Res ) return false ; } else { MVT ValVT = ValEVT . getSimpleVT ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS || F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ) { for ( unsigned i = <NUM_LIT> , OrigArgIdx = <NUM_LIT> ; OrigArgIdx != NumArgs && i != ArgLocs . size ( ) ; ++ Arg , ++ OrigArgIdx ) { if ( Skipped . test ( OrigArgIdx ) ) continue ; CCValAssign & VA = ArgLocs [ i ++ ] ; MRI . addLiveIn ( VA . getLocReg ( ) , VRegs [ OrigArgIdx ] ) ; MIRBuilder . getMBB ( ) . addLiveIn ( VA . getLocReg ( ) ) ; MIRBuilder . buildCopy ( VRegs [ OrigArgIdx ] , VA .


<s> B . buildMerge ( OrigRegs [ <NUM_LIT> ] , Regs ) ; return ; } if ( LLTy . isVector ( ) && PartLLT . isVector ( ) ) { assert ( LLTy . getElementType ( ) == PartLLT . getElementType ( ) ) ; int DstElts = LLTy . getNumElements ( ) ; int PartElts = PartLLT . getNumElements ( ) ; if ( DstElts % PartElts == <NUM_LIT> ) B . buildConcatVectors ( OrigRegs [ <NUM_LIT> ] , Regs ) ; else { assert ( PartElts == <NUM_LIT> && DstElts % <NUM_LIT> != <NUM_LIT> ) ; int RoundedElts = PartElts * ( ( DstElts + PartElts - <NUM_LIT> ) / PartElts ) ; LLT RoundedDestTy = LLT :: vector ( RoundedElts , PartLLT . getElementType ( ) ) ; auto RoundedConcat = B . buildConcatVectors ( RoundedDestTy , Regs ) ; B . buildExtract ( OrigRegs [ <NUM_LIT> ] , RoundedConcat , <NUM_LIT> ) ; } return ; } MachineRegisterInfo & MRI = * B . getMRI ( ) ; assert ( LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) ; LLT DstEltTy = LLTy . getElementType ( ) ; LLT RealDstEltTy = MRI . getType ( OrigRegs [ <NUM_LIT> ] ) . getElementType ( ) ; assert ( DstEltTy . getSizeInBits ( ) == RealDstEltTy . getSizeInBits ( ) ) ; if ( DstEltTy == PartLLT ) { if ( RealDstEltTy . isPointer ( ) ) { for ( Register Reg : Regs ) MRI . setType ( Reg , RealDstEltTy ) ; } B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , Regs ) ; } else if ( DstEltTy . getSizeInBits ( ) > PartLLT . getSizeInBits ( ) ) { SmallVector < Register , <NUM_LIT> > EltMerges ; int PartsPerElt = DstEltTy . getSizeInBits ( ) / PartLLT . getSizeInBits ( ) ; assert ( DstEltTy . getSizeInBits ( ) % PartLLT . getSizeInBits ( ) == <NUM_LIT> ) ; for ( int I = <NUM_LIT> , NumElts = LLTy . getNumElements ( ) ; I != NumElts ; ++ I ) { auto Merge = B . buildMerge ( RealDstEltTy , Regs . take_front ( PartsPerElt ) ) ; MRI . setType ( Merge . getReg ( <NUM_LIT> ) , RealDstEltTy ) ; EltMerges . push_back ( Merge . getReg ( <NUM_LIT> ) ) ; Regs = Regs . drop_front ( PartsPerElt ) ; } B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , EltMerges ) ; } else


<s> } if ( SrcSize % PartSize == <NUM_LIT> ) { B . buildUnmerge ( DstRegs , SrcReg ) ; return ; } const int NumRoundedParts = ( SrcSize + PartSize - <NUM_LIT> ) / PartSize ; LLT BigTy = getMultipleType ( PartTy , NumRoundedParts ) ; auto ImpDef = B . buildUndef ( BigTy ) ; auto Big = B . buildInsert ( BigTy , ImpDef . getReg ( <NUM_LIT> ) , SrcReg , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; int64_t Offset = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i , Offset += PartSize ) B

. buildExtract ( DstRegs [ i ] , Big , Offset ) ;


<s> LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar (


<s> } bool CanTailCallOpt = false ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , AssignFnFixed , AssignFnVarArg , false ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , MIRBuilder , OutArgs , Handler ) ) return false ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! ST . enableFlatScratch ( ) ) { auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) , MFI -> getScratchRSrcReg ( ) ) ; MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) { MIRBuilder . buildCopy ( ( Register ) ArgReg . first , ArgReg . second ) ; MIB . addReg ( ArgReg . first , RegState :: Implicit ) ; } unsigned NumBytes = CCInfo . getNextStackOffset ( ) ; if ( MIB -> getOperand ( <NUM_LIT>


<s> void assignValueToAddress ( unsigned ValVReg , unsigned Addr , uint64_t Size , MachinePointerInfo &


<s> void assignValueToReg ( unsigned ValVReg , unsigned PhysReg , CCValAssign & VA ) override { MIB . addUse (


<s> unsigned getStackAddress ( uint64_t Size , int64_t Offset , MachinePointerInfo & MPO ) override { llvm_unreachable ( <STR_LIT> )


<s> bool <ISA_LIT>CallLowering :: lowerFormalArguments ( MachineIRBuilder & MIRBuilder , const Function :: ArgumentListType & Args , ArrayRef < unsigned > VRegs ) const { return


<s> bool <ISA_LIT>CallLowering :: lowerFormalArguments ( MachineIRBuilder & MIRBuilder , const Function :: ArgumentListType & Args , ArrayRef < unsigned > VRegs ) const { return true


<s> MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true


<s> MachineFunction & MF = MIRBuilder . getMF ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>RegisterInfo * TRI = MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; bool IsShader = <STR_LIT> ( F . getCallingConv ( ) ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; if ( Info -> hasImplicitBufferPtr ( ) ) { unsigned ImplicitBufferPtrReg = Info -> addImplicitBufferPtr ( * TRI ) ; MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; BitVector Skipped ( NumArgs ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; <STR_LIT> Flags ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ArgInfo OrigArg { VRegs [ i ] [ <NUM_LIT> ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && ! OrigArg . Flags . isInReg ( ) && ! OrigArg . Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { if ( CurOrigArg -> use_empty ( ) && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ) { Skipped . set ( i ) ; ++ P<ISA_LIT>nputNum ; continue ; } Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! CurOrigArg -> use_empty ( ) ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; ++ P<ISA_LIT>nputNum ; } CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; if ( ValEVT . isVector ( ) ) { EVT ElemVT = ValEVT . getVectorElementType ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ElemVT . getSimpleVT ( ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( ! Res ) return false ; } else


<s> uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; unsigned Align = MinAlign ( KernArgBaseAlign , ArgOffset ) ; ArgOffset = alignTo ( ArgOffset , DL . getABITypeAlignment ( ArgTy ) ) ; lowerParameter ( MIRBuilder , ArgTy , ArgOffset , Align , ArgReg ) ; if ( OrigArgRegs . size ( ) > <NUM_LIT> ) unpackRegs ( OrigArgRegs


<s> bool <ISA_LIT>CallLowering :: lowerFormalArguments ( MachineIRBuilder & MIRBuilder , const Function :: ArgumentListType & Args , const

SmallVectorImpl < unsigned > & VRegs ) const {


<s> bool <ISA_LIT>CallLowering :: lowerFormalArguments ( MachineIRBuilder & MIRBuilder , const Function :: ArgumentListType & Args , const SmallVectorImpl < unsigned > & VRegs ) const { return


<s> bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( int I = <NUM_LIT> , E = VRegs [ Idx ] . size ( ) ; I != E ; ++ I ) MIRBuilder . buildUndef ( VRegs [ Idx ] [ I ] ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; setArgFlags ( OrigArg , Idx + AttributeList :: FirstArgIndex , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { packSplitRegsToOrigType ( MIRBuilder , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) MIRBuilder . setInstr ( * MBB . begin ( ) ) ; FormalArgHandler Handler ( MIRBuilder , MRI , AssignFn ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , MIRBuilder , SplitArgs , Handler ) ) return false ; if ( ! IsEntryFunc ) { TLI . allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsShader ) ; } else { CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; CCInfo . AllocateReg ( Info -> getScratchWaveOffsetReg ( ) ) ; CCInfo . AllocateReg ( Info ->


<s> MachineFunction & MF = B . getMF ( ) ; MachineBasicBlock & MBB = B . getMBB ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget . getRegisterInfo ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CC , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; if ( ! IsEntryFunc ) { Register ReturnAddrReg = TRI -> getReturnAddressReg ( MF ) ; Register LiveInReturn = MF . addLiveIn ( ReturnAddrReg , & <STR_LIT> ) ; MBB . addLiveIn ( ReturnAddrReg ) ; B . buildCopy ( LiveInReturn , ReturnAddrReg ) ; } if ( Info -> hasImplicitBufferPtr ( ) ) { Register ImplicitBufferPtrReg = Info -> addImplicitBufferPtr ( * TRI ) ; MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; unsigned Idx = <NUM_LIT> ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsShader && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg )


<s> MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; unsigned Idx = <NUM_LIT> ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsShader && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( int I = <NUM_LIT> , E = VRegs [ Idx ] . size ( ) ; I != E ; ++ I ) B . buildUndef ( VRegs [ Idx ] [ I ] ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( B , OrigArg , OrigArgIdx , SplitArgs , DL , CC , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT ,


<s> if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) splitToValueTypes ( Info . OrigRet , InArgs , DL , Info . CallConv ) ; bool CanTailCallOpt = isEligibleForTailCallOptimization ( MIRBuilder , Info , InArgs , OutArgs ) ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( CanTailCallOpt ) return lowerTailCall ( MIRBuilder , Info , OutArgs ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , false ) ; if ( ! handleAssignments ( Handler , OutArgs , CCInfo , ArgLocs , MIRBuilder ) ) return false ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; handleImplicitCallArguments ( MIRBuilder , MIB , ST , * MFI , ImplicitArgRegs ) ; unsigned NumBytes = CCInfo . getNextStackOffset ( ) ; if ( MIB -> getOperand ( <NUM_LIT> ) . isReg (


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo ::


<s> MachineFunction & MF = MIRBuilder . getMF ( ) ; const <ISA_LIT>RegisterInfo * TRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = * MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( OffsetReg , Offset )


<s> CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; BitVector Skipped ( NumArgs ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; <STR_LIT> Flags ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ArgInfo OrigArg { VRegs [ i ] [ <NUM_LIT> ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && ! OrigArg . Flags . isInReg ( ) && ! OrigArg . Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { if ( CurOrigArg -> use_empty ( ) && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ) { Skipped . set ( i ) ; ++ P<ISA_LIT>nputNum ; continue ; } Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! CurOrigArg -> use_empty ( ) ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; ++ P<ISA_LIT>nputNum ; } CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; if ( ValEVT . isVector ( ) ) { EVT ElemVT = ValEVT . getVectorElementType ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ElemVT . getSimpleVT ( ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( ! Res ) return false ; } else { MVT ValVT = ValEVT . getSimpleVT ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS || F . getCallingConv ( ) == CallingConv


<s> static bool addCallTargetOperands ( MachineInstrBuilder & CallInst , MachineIRBuilder & MIRBuilder , <ISA_LIT>CallLowering :: CallLoweringInfo & Info ) { if ( Info . Callee . isReg ( ) ) { CallInst . addReg ( Info . Callee . getReg ( ) ) ; CallInst . addImm ( <NUM_LIT> ) ; } else if ( Info . Callee . isGlobal ( ) && Info . Callee . getOffset ( ) == <NUM_LIT> ) { const GlobalValue * GV = Info . Callee . getGlobal ( ) ; auto Ptr = MIRBuilder . buildGlobalValue ( LLT :: pointer ( GV -> getAddressSpace ( ) , <NUM_LIT> ) , GV ) ; CallInst . addReg ( Ptr . getReg ( <NUM_LIT> ) ) ; CallInst . add ( Info . Callee ) ; } else return false ; return true


<s> } if ( Info . hasKernargSegmentPtr ( ) ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Register InputPtrReg = Info . addKernargSegmentPtr ( TRI ) ; const LLT P4 = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register VReg = MRI . createGenericVirtualRegister ( P4 ) ; MRI . addLiveIn ( InputPtrReg , VReg ) ; B . getMBB ( ) . addLiveIn ( InputPtrReg ) ; B . buildCopy ( VReg , InputPtrReg ) ; CCInfo . AllocateReg ( InputPtrReg ) ; } if ( Info . hasDispatchID ( ) ) { Register DispatchIDReg = Info . addDispatchID ( TRI


<s> CallingConv :: ID CallerCC = CallerF . getCallingConv ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( CalleeCC , TLI ) ; SmallVector < CCValAssign , <NUM_LIT> > OutLocs ; CCState OutInfo ( CalleeCC , false , MF , OutLocs , CallerF . getContext ( ) ) ; OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , OutInfo ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( OutInfo . getNextStackOffset ( ) > FuncInfo -> getBytesInStackArgArea ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget


<s> Register ValVReg = VA . getLocInfo ( ) != CCValAssign :: LocInfo :: FPExt ? extendRegister ( Arg . Regs [ ValRegIndex ] , VA ) :


<s> Register ValVReg = VA . getLocInfo ( ) != CCValAssign :: LocInfo :: FPExt ? extendRegister ( Arg . Regs

[ ValRegIndex ] , VA ) : Arg . Regs [ ValRegIndex ] ;


<s> void assignValueToReg ( Register ValVReg , Register PhysReg , CCValAssign VA ) override { MIB . addUse ( PhysReg , RegState :: Implicit ) ; Register ExtReg = extendRegisterMin32 ( * this , ValVReg , VA ) ; MIRBuilder . buildCopy ( PhysReg ,


<s> bool <ISA_LIT>CallLowering :: canLowerReturn ( MachineFunction & MF , CallingConv :: ID CallConv , SmallVectorImpl < BaseArgInfo > & Outs , bool IsVarArg ) const { if ( <STR_LIT> ( CallConv ) ) return true ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering >


<s> bool <ISA_LIT>CallLowering :: canLowerReturn ( MachineFunction & MF , CallingConv :: ID CallConv , SmallVectorImpl < BaseArgInfo > & Outs , bool IsVarArg ) const { if ( <STR_LIT> ( CallConv ) ) return true ; SmallVector < CCValAssign ,


<s> const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * CalleeAssignFnFixed ; CCAssignFn * CalleeAssignFnVarArg ; std :: tie ( CalleeAssignFnFixed , CalleeAssignFnVarArg ) = getAssignFnsForCC ( CalleeCC , TLI ) ; CCAssignFn * CallerAssignFnFixed ; CCAssignFn * CallerAssignFnVarArg ; std :: tie ( CallerAssignFnFixed , CallerAssignFnVarArg ) = getAssignFnsForCC ( CallerCC , TLI ) ; IncomingValueAssigner CalleeAssigner ( CalleeAssignFnFixed ,


<s> case TargetOpcode :: G_ZEXT : return <STR_LIT> ; case TargetOpcode :: G_ANYEXT : return <STR_LIT> ; default : llvm_unreachable (


<s> static std :: pair < CCAssignFn * , CCAssignFn * > getAssignFnsForCC ( CallingConv :: ID CC , const <ISA_LIT>TargetLowering & TLI ) { return { TLI . CCAssignFnForCall ( CC , false ) , TLI . CCAssignFnForCall


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! SPReg ) SPReg = MIRBuilder . buildCopy ( PtrTy , MFI -> getStackPtrOffsetReg ( ) ) . getReg ( <NUM_LIT> ) ; auto OffsetReg = MIRBuilder . buildConstant ( S32 , Offset ) ; auto AddrReg = MIRBuilder . buildPtrAdd ( PtrTy , SPReg , OffsetReg ) ; MPO = MachinePointerInfo :: getStack ( MF , Offset ) ; return AddrReg . getReg


<s> if ( ! ST . enableFlatScratch ( ) ) { auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) , FuncInfo . getScratchRSrcReg ( ) ) ; MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; CallInst . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) { MIRBuilder . buildCopy ( ( Register ) ArgReg . first


<s> } if ( any_of ( CallerF . args ( ) , [ ] ( const Argument & A ) { return A . hasByValAttr ( ) || A . hasSwiftErrorAttr ( ) ; } ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( MF . getTarget ( ) . Options . GuaranteedTailCallOpt ) return canGuaranteeTCO ( CalleeCC ) && CalleeCC == CallerF . getCallingConv ( ) ; if ( ! doCallerAndCalleePassArgsTheSameWay ( Info , MF , InArgs ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! areCalleeOutgoingArgsTailCallable ( Info , MF , OutArgs ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> return false ; } SmallVector < ArgInfo , <NUM_LIT> > OutArgs ; for ( auto & OrigArg : Info . OrigArgs ) splitToValueTypes ( OrigArg , OutArgs , DL , Info . CallConv ) ; SmallVector < ArgInfo , <NUM_LIT> > InArgs ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) splitToValueTypes ( Info . OrigRet , InArgs , DL , Info . CallConv ) ; bool CanTailCallOpt = isEligibleForTailCallOptimization ( MIRBuilder , Info , InArgs , OutArgs ) ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( CanTailCallOpt ) return lowerTailCall ( MIRBuilder , Info , OutArgs ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs ,


<s> CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI ,


<s> <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget . getRegisterInfo ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; Info -> allocateModuleLDSGlobal ( F . getParent ( ) ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CC , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; if ( ! IsEntryFunc ) { Register ReturnAddrReg = TRI -> getReturnAddressReg ( MF ) ; Register LiveInReturn = MF . addLiveIn ( ReturnAddrReg , & <STR_LIT> ) ; MBB . addLiveIn ( ReturnAddrReg ) ; B . buildCopy ( LiveInReturn , ReturnAddrReg ) ; } if ( Info -> hasImplicitBufferPtr ( ) ) { Register ImplicitBufferPtrReg = Info -> addImplicitBufferPtr ( * TRI ) ; MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; unsigned Idx = <NUM_LIT> ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; if ( ! FLI . CanLowerReturn ) insertSRetIncomingArgument ( F , SplitArgs , FLI . DemoteRegister , MRI , DL ) ; for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsGraphics && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty (


<s> LLT ArgTy = getLLTForType ( * SplitArg . Ty , DL ) ; if ( SplitArg . Flags [ <NUM_LIT> ] . isPointer ( ) ) { LLT PtrTy = LLT :: pointer ( SplitArg . Flags [ <NUM_LIT> ] . getPointerAddrSpace ( ) , ArgTy . getScalarSizeInBits ( ) ) ; ArgTy = ArgTy . isVector ( ) ? LLT :: vector ( ArgTy . getElementCount ( ) , PtrTy ) : PtrTy ; } MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , ArgTy , commonAlignment ( Alignment , FieldOffsets [ Idx ] ) ) ; assert ( SplitArg . Regs . size ( ) == <NUM_LIT> ) ; B . buildLoad ( SplitArg . Regs [ <NUM_LIT> ] ,


<s> void <ISA_LIT>CallLowering :: lowerParameter ( MachineIRBuilder & B , ArgInfo & OrigArg , uint64_t Offset , Align Alignment ) const { MachineFunction & MF = B . getMF ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; SmallVector < uint64_t > FieldOffsets ; splitToValueTypes ( OrigArg , SplitArgs , DL , F . getCallingConv ( ) , & FieldOffsets ) ; unsigned Idx = <NUM_LIT> ; for ( ArgInfo & SplitArg : SplitArgs ) { Register PtrReg = B . getMRI ( ) -> createGenericVirtualRegister ( PtrTy ) ; lowerParameterPtr ( PtrReg , B , Offset + FieldOffsets [ Idx ] ) ; LLT ArgTy = getLLTForType ( * SplitArg . Ty , DL ) ; if ( SplitArg . Flags [ <NUM_LIT> ] . isPointer ( ) ) { LLT PtrTy = LLT :: pointer ( SplitArg . Flags [ <NUM_LIT> ] . getPointerAddrSpace ( ) , ArgTy . getScalarSizeInBits ( ) ) ; ArgTy = ArgTy . isVector ( ) ? LLT :: vector ( ArgTy . getElementCount ( ) , PtrTy ) : PtrTy ; } MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable

| MachineMemOperand :: MOInvariant , ArgTy , commonAlignment ( Alignment , FieldOffsets [ Idx ] ) ) ;


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; auto OffsetReg = B . buildConstant ( LLT :: scalar


<s> void <ISA_LIT>CallLowering :: lowerParameterPtr ( Register DstReg , MachineIRBuilder & B , uint64_t Offset ) const { MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF


<s> MIB . addDef ( PhysReg , RegState ::


<s> FunctionPass * llvm :: create<ISA_LIT>CodeGenPreparePass ( ) { return new <ISA_LIT>CodeGenPrepare


<s> return new <ISA_LIT>CodeGenPrepare (


<s> bool <ISA_LIT>CodeGenPrepare :: doInitialization ( Module & M ) { Mod = & M


<s> if ( RHSSignBits < <NUM_LIT> ) return nullptr ; unsigned SignBits = std :: min ( LHSSignBits , RHSSignBits ) ; unsigned DivBits = <NUM_LIT> - SignBits ; if ( IsSigned ) ++ DivBits ; Type * Ty = Num -> getType ( ) ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * F32Ty = Builder . getFloatTy ( ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * JQ = One ; if ( IsSigned ) { JQ = Builder . CreateXor ( Num , Den ) ; JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FQM ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE (


<s> Num = Builder . CreateSExt ( Num , I32Ty ) ; Den = Builder . CreateSExt ( Den , I32Ty ) ; } else { Num = Builder . CreateZExt ( Num , I32Ty ) ; Den = Builder . CreateZExt ( Den , I32Ty ) ; } } if ( Value * Res = expandDivRem24 ( Builder , I , Num , Den , IsDiv , IsSigned ) ) { Res = Builder . CreateTrunc ( Res , Ty ) ; return Res ; } ConstantInt * Zero = Builder . getInt32 ( <NUM_LIT> ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; ConstantInt * MinusOne = Builder . getInt32 ( ~ <NUM_LIT> ) ; Value * Sign = nullptr ; if ( IsSigned ) { ConstantInt * K31 = Builder . getInt32 ( <NUM_LIT> ) ; Value * LHSign = Builder . CreateAShr ( Num , K31 ) ; Value * RHSign = Builder . CreateAShr ( Den , K31 ) ; Sign = IsDiv ? Builder . CreateXor ( LHSign , RHSign ) : LHSign ; Num = Builder . CreateAdd ( Num , LHSign ) ; Den = Builder . CreateAdd ( Den , RHSign ) ; Num = Builder . CreateXor ( Num , LHSign ) ; Den = Builder . CreateXor ( Den , RHSign ) ; } Value * DEN_F32 = Builder . CreateUIToFP ( Den , F32Ty ) ; Value * RCP_F32 = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , DEN_F32 ) ; Constant * UINT_MAX_PLUS_1 = ConstantFP :: get ( F32Ty , BitsToFloat ( <NUM_LIT> ) ) ; Value * RCP_SCALE = Builder . CreateFMul ( RCP_F32 , UINT_MAX_PLUS_1 ) ; Value * RCP = Builder . CreateFPToUI (


<s> return cast < VectorType > ( T ) -> getElementType ( ) -> getIntegerBitWidth (


<s> return VectorType :: get ( B . getInt32Ty ( ) , cast < VectorType > ( T ) ->


<s> Type * <ISA_LIT>CodeGenPrepare :: getI32Ty ( IRBuilder < > & B , const Type * T ) const { assert ( needsPromotionToI32 ( T ) && <STR_LIT> ) ; if ( T -> isIntegerTy ( ) ) return B . getInt32Ty


<s> Value * Lo = Builder . CreateTrunc ( MUL64 , I32Ty ) ; Value * Hi = Builder . CreateLShr ( MUL64 , Builder . getInt64 ( <NUM_LIT> ) ) ; Hi = Builder . CreateTrunc ( Hi , I32Ty ) ; return std :: make_pair ( Lo


<s> return getMul64 ( Builder , LHS , RHS )


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> case Instruction :: Add : case Instruction :: Sub : return true ; case Instruction :: Mul


<s> Value * ExtRes = Builder . CreateCall ( I32 , { ExtOp } ) ; Value * LShrOp = Builder . CreateLShr ( ExtRes , <NUM_LIT> - getBaseElementBitWidth ( I . getType ( ) ) ) ; Value * TruncRes = Builder . CreateTrunc ( LShrOp , I . getType ( ) ) ; I . replaceAllUsesWith ( TruncRes ) ; I . eraseFromParent ( ) ; return


<s> ExtOp2 = Builder . CreateZExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ; } ExtRes = Builder . CreateSelect ( I . getOperand ( <NUM_LIT> ) , ExtOp1 , ExtOp2 ) ; TruncRes = Builder . CreateTrunc ( ExtRes , I . getType ( ) ) ; I . replaceAllUsesWith ( TruncRes ) ; I . eraseFromParent ( ) ; return true


<s> if ( ! TPC ) return false ; const <ISA_LIT>TargetMachine & TM = TPC -> getTM < <ISA_LIT>TargetMachine > ( ) ; ST = & TM . getSubtarget < GCNSubtarget > ( F ) ; AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator


<s> auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; if ( ! TPC ) return false ; const <ISA_LIT>TargetMachine & TM = TPC -> getTM < <ISA_LIT>TargetMachine > ( ) ; ST = & TM . getSubtarget < GCNSubtarget > ( F ) ; AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F


<s> Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; } if ( NewDiv ) { I . replaceAllUsesWith ( NewDiv ) ; I . eraseFromParent ( ) ; Changed = true ; } } return Changed


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformBitreverseToI32 ( I ) ; return Changed


<s> Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; bool HasDenormals = ST -> hasFP32Denormals ( ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv , HasDenormals ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else { NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } ) ; } NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } }


<s> Type * Ty = FDiv . getType ( ) ; if ( ! Ty -> getScalarType ( ) -> isFloatTy ( ) ) return false ; MDNode * FPMath = FDiv . getMetadata ( LLVMContext :: MD_fpmath ) ; if ( ! FPMath ) return false ; const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; float ULP = FPOp -> getFPAccuracy ( ) ; if ( ULP < <NUM_LIT> ) return false ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; bool UnsafeDiv = HasUnsafeFPMath || FMF . isFast ( ) || FMF . allowReciprocal ( ) ; if ( UnsafeDiv ) return false ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) , FPMath ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getOperand ( <NUM_LIT> ) -> getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformOpToI32 ( I ) ; return Changed


<s> bool visitInstruction ( Instruction & I ) { return


<s> Metadata * LowAndHigh [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , Lower -> getValue ( ) . zext ( <NUM_LIT> ) ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , <NUM_LIT> ) ) } ; WidenLoad -> setMetadata ( LLVMContext :: MD_range , MDNode :: get ( Mod -> getContext ( ) , LowAndHigh ) ) ; } } int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ; Value * ValOrig = Builder . CreateBitCast ( ValTrunc , I . getType


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformOpToI32 ( I ) ; return Changed


<s> if ( ! CNum ) return false ; return UnsafeDiv || CNum -> isExactlyValue (


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) ) Changed |= promoteUniformOpToI32 ( I ) ; return


<s> MDNode * FPMath = FDiv . getMetadata ( LLVMContext :: MD_fpmath ) ; if ( ! FPMath ) return false ; const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; float ULP = FPOp -> getFPAccuracy ( ) ; if ( ULP < <NUM_LIT> ) return false ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; bool UnsafeDiv = HasUnsafeFPMath || FMF . unsafeAlgebra ( ) || FMF . allowReciprocal ( ) ; if ( ST -> hasFP32Denormals ( ) && ! UnsafeDiv ) return false ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) , FPMath ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else { NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } ) ; } NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else { if ( ! shouldKeepFDivF32 ( Num , UnsafeDiv ) ) NewFDiv = Builder . CreateCall ( Decl , { Num , Den } ) ; } if ( NewFDiv ) { FDiv . replaceAllUsesWith ( NewFDiv ) ; NewFDiv -> takeName ( & FDiv ) ; FDiv . eraseFromParent ( ) ; } return true


<s> } assert ( NumBits <= <NUM_LIT> ) ; <STR_LIT> LoID = IsSigned ? <STR_LIT> : <STR_LIT> ; <STR_LIT> HiID = IsSigned ? <STR_LIT> : <STR_LIT> ; Value * Lo = Builder . CreateIntrinsic ( LoID , { } , { LHS , RHS } ) ; Value * Hi = Builder . CreateIntrinsic ( HiID , { } , { LHS , RHS } ) ; IntegerType * I64Ty = Builder . getInt64Ty ( ) ; Lo = Builder . CreateZExtOrTrunc ( Lo , I64Ty ) ; Hi = Builder . CreateZExtOrTrunc ( Hi , I64Ty ) ; return Builder . CreateOr ( Lo , Builder .


<s> return ComputeMaxSignificantBits ( Op , * DL , <NUM_LIT>


<s> return ComputeMaxSignificantBits ( Op , * DL


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsUnsigned ( Value * Op )


<s> SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IsSigned ) { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = getMul24 ( Builder , LHS , RHS , Size


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsSigned ( Value * Op , unsigned ScalarSize ) const { return ScalarSize - ComputeNumSignBits ( Op , * DL , <NUM_LIT> , AC


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsSigned ( Value * Op ,


<s> if ( Size <= <NUM_LIT> && ST -> has16BitInsts ( ) ) return false ; if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; unsigned LHSBits = <NUM_LIT> , RHSBits = <NUM_LIT> ; bool IsSigned = false ; if ( ST -> hasMulU24 ( ) && ( LHSBits = numBitsUnsigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsUnsigned ( RHS , Size ) ) <= <NUM_LIT> ) { IsSigned = false ; } else if ( ST -> hasMulI24 ( ) && ( LHSBits = numBitsSigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsSigned ( RHS , Size ) ) <= <NUM_LIT> ) { IsSigned = true ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IsSigned ) { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = getMul24 ( Builder , LHS , RHS , Size , LHSBits + RHSBits , IsSigned ) ; if ( IsSigned ) { ResultVals . push_back ( Builder . CreateSExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } else { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } } Value * NewVal = insertValues ( Builder , Ty , ResultVals ) ; NewVal -> takeName ( & I ) ; I . replaceAllUsesWith


<s> if ( I . getPointerAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS && canWidenScalarExtLoad ( I ) ) { IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * PT = PointerType :: get ( I32Ty , I . getPointerAddressSpace ( ) ) ; Value * BitCast = Builder . CreateBitCast ( I . getPointerOperand ( ) , PT ) ; Value * WidenLoad = Builder . CreateLoad ( BitCast ) ; int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ; Value * ValOrig = Builder . CreateBitCast ( ValTrunc , I . getType ( ) ) ; I . replaceAllUsesWith ( ValOrig ) ; I . eraseFromParent ( ) ; return true ; } return


<s> Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else { NewElt = Builder . CreateCall ( Decl , { NumEltI , DenEltI } ) ; } NewFDiv = Builder . CreateInsertElement ( NewFDiv ,


<s> FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; bool UnsafeDiv = HasUnsafeFPMath || FMF . isFast ( ) || FMF . allowReciprocal ( ) ; if ( ST -> hasFP32Denormals ( ) || UnsafeDiv ) return false ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) , FPMath ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement


<s> Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * PT = PointerType :: get ( I32Ty , I . getPointerAddressSpace ( ) ) ; Value * BitCast = Builder . CreateBitCast ( I . getPointerOperand ( ) , PT ) ; Value * WidenLoad = Builder . CreateLoad ( BitCast ) ; int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc (


<s> bool <ISA_LIT>CodeGenPrepare :: isU24 ( Value *


<s> const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; const float ReqdAccuracy = FPOp -> getFPAccuracy ( ) ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; const bool AllowInaccurateRcp = HasUnsafeFPMath || FMF . approxFunc ( ) ; const bool RcpIsAccurate = ( Ty -> isHalfTy ( ) && ReqdAccuracy >= <NUM_LIT> ) || ( Ty -> isFloatTy ( ) && ! HasFP32Denormals && ReqdAccuracy >= <NUM_LIT> ) ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( auto * VT = dyn_cast < FixedVectorType > ( FDiv . getType ( ) ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E


<s> WidenLoad -> copyMetadata ( I ) ; if ( auto * Range = WidenLoad -> getMetadata ( LLVMContext :: MD_range ) ) { ConstantInt * Lower = mdconst :: extract < ConstantInt > ( Range -> getOperand ( <NUM_LIT> ) ) ; if ( Lower -> getValue ( ) . isNullValue ( ) ) { WidenLoad -> setMetadata ( LLVMContext :: MD_range , nullptr ) ; } else { Metadata * LowAndHigh [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , Lower -> getValue ( ) . zext ( <NUM_LIT> ) ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , <NUM_LIT> ) ) } ; WidenLoad -> setMetadata ( LLVMContext :: MD_range , MDNode :: get ( Mod -> getContext ( ) , LowAndHigh ) ) ; } } int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ; Value * ValOrig = Builder . CreateBitCast ( ValTrunc , I . getType ( ) ) ; I . replaceAllUsesWith


<s> <ISA_LIT>CodeGenPrepare ( const TargetMachine * TM = nullptr ) : FunctionPass ( ID ) , TM ( static_cast < const GCNTargetMachine


<s> BinOp -> setHasNoSignedWrap ( I . hasNoSignedWrap ( ) ) ; BinOp -> setHasNoUnsignedWrap ( I . hasNoUnsignedWrap ( ) ) ; } else if ( isa < PossiblyExactOperator > ( BinOp ) ) BinOp -> setIsExact ( I


<s> FunctionPass * llvm :: create<ISA_LIT>CodeGenPreparePass ( const GCNTargetMachine * TM ) { return new <ISA_LIT>CodeGenPrepare (


<s> return new <ISA_LIT>CodeGenPrepare (


<s> Type * I32Ty = getI32Ty ( Builder , I . getType ( ) ) ; Value * ExtOp0 = nullptr ; Value * ExtOp1 = nullptr ; Value * ExtRes = nullptr ; Value * TruncRes = nullptr ; if ( isSigned ( I ) ) { ExtOp0 = Builder . CreateSExt ( I . getOperand ( <NUM_LIT> ) , I32Ty ) ; ExtOp1 = Builder . CreateSExt ( I .


<s> Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; Type * I32Ty = getI32Ty ( Builder , I . getType ( ) ) ; Value * ExtOp0 = nullptr ; Value * ExtOp1 = nullptr ; Value * ExtRes = nullptr ; Value * TruncRes = nullptr ; if ( isSigned ( I ) ) { ExtOp0 = Builder . CreateSExt ( I . getOperand


<s> DA = & getAnalysis < DivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB .


<s> BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MadeChange |= visit ( *


<s> const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; float ULP = FPOp -> getFPAccuracy ( ) ; if ( ULP < <NUM_LIT> ) return false ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; bool UnsafeDiv = HasUnsafeFPMath || FMF . unsafeAlgebra ( ) || FMF . allowReciprocal ( ) ; if ( ST -> hasFP32Denormals ( ) && ! UnsafeDiv ) return false ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) , FPMath ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; const <ISA_LIT>IntrinsicInfo * II = TM -> getIntrinsicInfo ( ) ; Function * Decl = II -> getDeclaration ( Mod , <STR_LIT> , { } ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt ; if ( shouldKeepFDivF32 ( NumEltI , UnsafeDiv ) ) { NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; } else


<s> bool <ISA_LIT>CodeGenPrepare :: canWidenScalarExtLoad ( LoadInst & I ) const { Type * Ty = I . getType ( ) ; const DataLayout & DL = Mod -> getDataLayout ( ) ; int TySize = DL .


<s> bool <ISA_LIT>CodeGenPrepare :: canWidenScalarExtLoad ( LoadInst & I ) const { Type * Ty = I . getType ( ) ; const DataLayout & DL = Mod -> getDataLayout ( ) ; int TySize = DL . getTypeSizeInBits ( Ty ) ; Align Alignment = DL . getValueOrABITypeAlignment ( I . getAlign ( ) , Ty ) ; return I . isSimple ( ) && TySize < <NUM_LIT> && Alignment

>= <NUM_LIT> && DA -> isUniform ( & I ) ;


<s> return false ; } if ( BinaryOperator * BinOpDen = dyn_cast < BinaryOperator > ( Den ) ) { if ( BinOpDen -> getOpcode ( ) == Instruction :: Shl && isa < Constant > ( BinOpDen -> getOperand ( <NUM_LIT> ) ) && isKnownToBeAPowerOfTwo ( BinOpDen ->

getOperand ( <NUM_LIT> ) , * DL , true , <NUM_LIT> , AC , & I , DT ) ) {


<s> bool <ISA_LIT>CodeGenPrepare :: doInitialization ( Module & M ) { Mod =


<s> bool <ISA_LIT>CodeGenPrepare :: doInitialization ( Module & M ) { Mod = &


<s> Value * <ISA_LIT>CodeGenPrepare :: expandDivRem24 ( IRBuilder < > & Builder , BinaryOperator & I , Value * Num , Value * Den , bool IsDiv , bool IsSigned ) const { int DivBits = getDivNumBits ( I , Num , Den , <NUM_LIT> , IsSigned ) ; if ( DivBits == - <NUM_LIT> ) return


<s> X = Builder . CreateSExt ( X , I32Ty ) ; Y = Builder . CreateSExt ( Y , I32Ty ) ; } else { X = Builder . CreateZExt ( X , I32Ty ) ; Y = Builder . CreateZExt ( Y , I32Ty ) ; } } if ( Value * Res = expandDivRem24 ( Builder , I , X , Y , IsDiv , IsSigned ) ) { return IsSigned ? Builder . CreateSExtOrTrunc ( Res , Ty ) : Builder . CreateZExtOrTrunc ( Res , Ty ) ; } ConstantInt * Zero = Builder . getInt32 ( <NUM_LIT> ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * Sign = nullptr ; if ( IsSigned ) { Value * SignX = getSign32 ( X , Builder , DL ) ; Value * SignY = getSign32 ( Y , Builder , DL ) ; Sign = IsDiv ? Builder . CreateXor ( SignX , SignY ) : SignX ; X = Builder . CreateAdd ( X , SignX ) ; Y = Builder . CreateAdd ( Y , SignY ) ; X = Builder . CreateXor ( X , SignX ) ; Y = Builder . CreateXor ( Y , SignY ) ; } Value * FloatY = Builder . CreateUIToFP ( Y , F32Ty ) ; Function * Rcp = <STR_LIT> ( Mod , <STR_LIT> , F32Ty ) ; Value * RcpY = Builder . CreateCall ( Rcp , { FloatY } ) ; Constant * Scale = ConstantFP :: get ( F32Ty , BitsToFloat ( <NUM_LIT> ) ) ; Value * ScaledY = Builder . CreateFMul ( RcpY , Scale ) ; Value * Z = Builder . CreateFPToUI ( ScaledY , I32Ty ) ; Value * NegY = Builder . CreateSub ( Zero , Y ) ; Value * NegYZ = Builder . CreateMul ( NegY , Z ) ; Z = Builder . CreateAdd ( Z , getMulHu ( Builder , Z , NegYZ ) ) ; Value * Q = getMulHu ( Builder , X , Z


<s> bool <ISA_LIT>CodeGenPrepare :: foldBinOpIntoSelect ( BinaryOperator & BO ) const { int SelOpNo = <NUM_LIT> ; CastInst * CastOp ; SelectInst * Sel = findSelectThroughCast ( BO . getOperand ( <NUM_LIT> ) , CastOp ) ; if ( ! Sel || ! Sel -> hasOneUse ( ) ) { SelOpNo = <NUM_LIT> ; Sel = findSelectThroughCast ( BO . getOperand ( <NUM_LIT> ) , CastOp ) ; } if ( ! Sel || ! Sel -> hasOneUse ( ) ) return false ; Constant * CT = dyn_cast < Constant > ( Sel -> getTrueValue ( ) ) ; Constant * CF = dyn_cast < Constant > ( Sel -> getFalseValue ( ) ) ; Constant * CBO = dyn_cast < Constant > ( BO . getOperand ( SelOpNo ^ <NUM_LIT> ) ) ; if ( ! CBO || ! CT || ! CF ) return false ; if ( CastOp ) { if ( ! CastOp -> hasOneUse ( ) ) return false ; CT = ConstantFoldCastOperand ( CastOp -> getOpcode ( ) , CT , BO . getType ( ) , * DL ) ; CF = ConstantFoldCastOperand ( CastOp -> getOpcode ( ) , CF , BO . getType ( ) , * DL ) ; } Constant * FoldedT = SelOpNo ? ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CBO , CT , * DL ) : ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CT , CBO , * DL ) ; if ( isa < ConstantExpr > ( FoldedT ) ) return false ; Constant * FoldedF = SelOpNo ? ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CBO , CF , * DL ) : ConstantFoldBinaryOpOperands ( BO . getOpcode ( ) , CF , CBO , * DL ) ; if ( isa < ConstantExpr > ( FoldedF ) ) return false ; IRBuilder < > Builder (


<s> AU . addRequired < LegacyDivergenceAnalysis > ( ) ; if ( ! ExpandDiv64InIR )


<s> const DataLayout & DL = Mod -> getDataLayout ( ) ; unsigned LHSSignBits = ComputeNumSignBits ( Num , DL , <NUM_LIT> , AC , & I ) ; if ( LHSSignBits < AtLeast ) return - <NUM_LIT> ; unsigned RHSSignBits = ComputeNumSignBits ( Den , DL , <NUM_LIT> , AC , & I ) ; if ( RHSSignBits < AtLeast )


<s> Type * <ISA_LIT>CodeGenPrepare :: getI32Ty ( IRBuilder < > & B , const Type * T ) const { assert ( needsPromotionToI32 ( T ) && <STR_LIT> ) ; if ( T -> isIntegerTy ( ) ) return B . getInt32Ty ( ) ; return FixedVectorType :: get ( B . getInt32Ty ( )


<s> return FixedVectorType :: get ( B . getInt32Ty ( ) , cast < FixedVectorType > (


<s> if ( Known . isNonNegative ( ) ) return Constant :: getNullValue ( V -> getType ( ) ) ; return Builder . CreateAShr ( V , Builder


<s> Attribute Attr = F . getFnAttribute ( <STR_LIT>


<s> if ( ! Widen16BitOps ) return false ; const IntegerType * IntTy = dyn_cast < IntegerType > ( T ) ; if ( IntTy && IntTy -> getBitWidth ( ) > <NUM_LIT> && IntTy -> getBitWidth ( ) <= <NUM_LIT> ) return true ; if ( const VectorType * VT = dyn_cast <


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsSigned ( Value *


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsSigned ( Value * Op , unsigned ScalarSize ) const { return ScalarSize - ComputeNumSignBits ( Op , * DL


<s> unsigned <ISA_LIT>CodeGenPrepare :: numBitsUnsigned ( Value * Op ,


<s> Type * Ty = Den -> getType ( ) ; if ( ! Ty -> isFloatTy ( ) ) return nullptr ; bool NumIsOne = false ; if ( const ConstantFP * CNum = dyn_cast < ConstantFP > ( Num ) ) { if ( CNum -> isExactlyValue ( + <NUM_LIT> ) || CNum ->

isExactlyValue ( - <NUM_LIT> ) ) NumIsOne = true ;


<s> if ( Size <= <NUM_LIT> && ST -> has16BitInsts ( ) ) return false ; if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; unsigned LHSBits = <NUM_LIT> , RHSBits = <NUM_LIT> ; if ( ST -> hasMulU24 ( ) && ( LHSBits = numBitsUnsigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsUnsigned ( RHS , Size ) ) <= <NUM_LIT> ) { if ( Size > <NUM_LIT> && LHSBits + RHSBits > <NUM_LIT> ) return false ; IntrID = <STR_LIT> ; } else if ( ST -> hasMulI24 ( ) && ( LHSBits = numBitsSigned ( LHS , Size ) ) < <NUM_LIT> && ( RHSBits = numBitsSigned ( RHS , Size ) ) < <NUM_LIT> ) { if ( Size > <NUM_LIT> && LHSBits + RHSBits > <NUM_LIT> ) return false ; IntrID = <STR_LIT> ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; FunctionCallee Intrin = <STR_LIT> ( Mod ,


<s> if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; unsigned LHSBits = <NUM_LIT> , RHSBits = <NUM_LIT> ; if ( ST -> hasMulU24 ( ) && ( LHSBits = numBitsUnsigned ( LHS , Size ) ) <= <NUM_LIT> && ( RHSBits = numBitsUnsigned ( RHS , Size ) ) <= <NUM_LIT> ) { if ( Size > <NUM_LIT> && LHSBits + RHSBits > <NUM_LIT> ) return false ; IntrID = <STR_LIT> ; } else if ( ST -> hasMulI24 ( ) && ( LHSBits = numBitsSigned ( LHS , Size ) ) < <NUM_LIT> && ( RHSBits = numBitsSigned ( RHS , Size ) ) < <NUM_LIT> ) { if ( Size > <NUM_LIT> && LHSBits + RHSBits > <NUM_LIT> ) return false ; IntrID = <STR_LIT> ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; FunctionCallee Intrin = <STR_LIT> ( Mod , IntrID ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IntrID == <STR_LIT> ) { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = Builder . CreateCall ( Intrin , { LHS , RHS } ) ; if ( IntrID == <STR_LIT> ) { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } else


<s> NextBB = std :: next ( FI ) ; BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB -> begin ( ) , E = BB -> end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MadeChange |= visit ( * I ) ; if ( Next != E ) { BasicBlock * NextInstBB = Next -> getParent ( ) ; if ( NextInstBB != BB ) { BB = NextInstBB ; E = BB -> end ( ) ; FE = F .


<s> if ( Ty -> isHalfTy ( ) && ! ST -> has16BitInsts ( ) ) return false ; const FPMathOperator * FPOp = cast < const FPMathOperator > ( & FDiv ) ; const float ReqdAccuracy = FPOp -> getFPAccuracy ( ) ; FastMathFlags FMF = FPOp -> getFastMathFlags ( ) ; const bool AllowInaccurateRcp = HasUnsafeFPMath || FMF . approxFunc ( ) ; const bool RcpIsAccurate = ( Ty -> isHalfTy ( ) && ReqdAccuracy >= <NUM_LIT> ) || ( Ty -> isFloatTy ( ) && ! HasFP32Denormals && ReqdAccuracy >= <NUM_LIT> ) ; IRBuilder < > Builder ( FDiv . getParent ( ) , std :: next ( FDiv . getIterator ( ) ) ) ; Builder . setFastMathFlags ( FMF ) ; Builder . SetCurrentDebugLocation ( FDiv . getDebugLoc ( ) ) ; Value * Num = FDiv . getOperand ( <NUM_LIT> ) ; Value * Den = FDiv . getOperand ( <NUM_LIT> ) ; Value * NewFDiv = nullptr ; if ( auto * VT = dyn_cast < FixedVectorType > ( FDiv . getType ( ) ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt = optimizeWithRcp ( NumEltI , DenEltI , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ; if ( ! NewElt ) NewElt = optimizeWithFDivFast ( NumEltI , DenEltI , ReqdAccuracy , HasFP32Denormals , Builder , Mod ) ; if ( ! NewElt ) NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } }


<s> LoadInst * WidenLoad = Builder . CreateLoad ( I32Ty , BitCast ) ; WidenLoad -> copyMetadata ( I ) ; if ( auto * Range = WidenLoad -> getMetadata ( LLVMContext :: MD_range ) ) { ConstantInt * Lower = mdconst :: extract < ConstantInt > ( Range -> getOperand ( <NUM_LIT> ) ) ; if ( Lower -> isNullValue ( ) ) { WidenLoad -> setMetadata ( LLVMContext :: MD_range , nullptr ) ; } else { Metadata * LowAndHigh [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , Lower -> getValue ( ) . zext ( <NUM_LIT> ) ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( I32Ty , <NUM_LIT> ) ) } ; WidenLoad -> setMetadata ( LLVMContext :: MD_range , MDNode :: get ( Mod -> getContext ( ) , LowAndHigh ) ) ; } } int TySize = Mod -> getDataLayout ( ) . getTypeSizeInBits ( I . getType ( ) ) ; Type * IntNTy = Builder . getIntNTy ( TySize ) ; Value * ValTrunc = Builder . CreateTrunc ( WidenLoad , IntNTy ) ; Value * ValOrig = Builder . CreateBitCast ( ValTrunc , I . getType ( ) ) ; I . replaceAllUsesWith (


<s> ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * JQ = One ; if ( IsSigned ) { JQ = Builder . CreateXor ( Num , Den ) ; JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateIntrinsic ( <STR_LIT> , { FQM } ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateIntrinsic ( <STR_LIT> , { FR } , FQ ) ; FB = Builder . CreateIntrinsic ( <STR_LIT> , { FB } , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB


<s> AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < DivergenceAnalysis > ( ) ; AU . setPreservesAll (


<s> AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < DivergenceAnalysis > ( ) ; AU . setPreservesAll


<s> JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Function * RcpDecl = <STR_LIT> ( Mod , <STR_LIT> , Builder . getFloatTy ( ) ) ; Value * RCP = Builder . CreateCall ( RcpDecl , { FB } ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FQM ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) { Value * Rem = Builder . CreateMul (


<s> FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) { Value * Rem = Builder . CreateMul ( Div , Den ) ; Res = Builder . CreateSub ( Num , Rem ) ; } if ( DivBits != <NUM_LIT> && DivBits < <NUM_LIT> ) { if ( IsSigned ) { int InRegBits = <NUM_LIT> - DivBits ; Res = Builder . CreateShl ( Res , InRegBits ) ; Res = Builder . CreateAShr ( Res , InRegBits ) ; } else { ConstantInt * TruncMask = Builder . getInt32 ( ( UINT64_C ( <NUM_LIT>


<s> Value * <ISA_LIT>CodeGenPrepare :: expandDivRem32 ( IRBuilder < > & Builder , BinaryOperator & I , Value * Num , Value * Den ) const { Instruction :: BinaryOps Opc = I . getOpcode ( ) ; assert ( Opc == Instruction :: URem || Opc == Instruction :: UDiv || Opc == Instruction :: SRem || Opc == Instruction :: SDiv ) ; FastMathFlags FMF ; FMF . setFast ( ) ; Builder . setFastMathFlags ( FMF ) ; if ( divHasSpecialOptimization ( I , Num , Den ) ) return nullptr ; bool IsDiv = Opc == Instruction :: UDiv || Opc == Instruction :: SDiv ; bool IsSigned = Opc == Instruction :: SRem || Opc == Instruction :: SDiv ; Type * Ty = Num -> getType ( ) ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * F32Ty = Builder . getFloatTy ( ) ; if ( Ty -> getScalarSizeInBits ( ) < <NUM_LIT> ) { if ( IsSigned ) { Num = Builder . CreateSExt ( Num , I32Ty ) ; Den = Builder . CreateSExt ( Den , I32Ty ) ; } else { Num = Builder . CreateZExt ( Num , I32Ty ) ; Den = Builder . CreateZExt ( Den , I32Ty ) ; } } if ( Value * Res = expandDivRem24 ( Builder , I , Num , Den , IsDiv , IsSigned ) ) { return IsSigned ? Builder . CreateSExtOrTrunc ( Res , Ty ) : Builder . CreateZExtOrTrunc ( Res , Ty ) ; } ConstantInt * Zero = Builder . getInt32 ( <NUM_LIT> ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * Sign = nullptr ; if ( IsSigned ) { ConstantInt * K31 = Builder . getInt32 ( <NUM_LIT> ) ; Value * LHSign = Builder . CreateAShr ( Num , K31 ) ; Value * RHSign = Builder . CreateAShr ( Den , K31 ) ; Sign = IsDiv ? Builder . CreateXor ( LHSign , RHSign ) : LHSign ; Num = Builder . CreateAdd ( Num , LHSign ) ; Den = Builder . CreateAdd ( Den , RHSign ) ; Num = Builder . CreateXor ( Num


<s> for ( int I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I

) Values . push_back ( Builder . CreateExtractElement ( V , I ) ) ;


<s> static void extractValues ( IRBuilder < > & Builder , SmallVectorImpl < Value * > & Values , Value * V ) { VectorType * VT = dyn_cast < VectorType > ( V


<s> DT = DTWP ? & DTWP -> getDomTree ( ) : nullptr ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; HasFP32Denormals = ST -> hasFP32Denormals ( F ) ; bool MadeChange = false ; Function :: iterator NextBB ; for ( Function :: iterator FI = F . begin ( ) , FE = F . end ( ) ; FI != FE ; FI = NextBB ) { BasicBlock * BB = & * FI ; NextBB = std :: next ( FI ) ; BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB -> begin ( ) , E = BB -> end ( ) ; I != E ; I


<s> if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt ; if ( ScalarSize <= <NUM_LIT> ) { NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; } else { NewElt = shrinkDivRem64 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) { NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; Div64ToExpand . push_back ( cast < BinaryOperator > ( NewElt ) ) ; } } NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { if ( ScalarSize <= <NUM_LIT> ) NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; else { NewDiv = shrinkDivRem64 ( Builder , I , Num , Den ) ; if ( ! NewDiv ) Div64ToExpand . push_back ( & I ) ; } } if ( NewDiv ) { I . replaceAllUsesWith ( NewDiv


<s> if ( foldBinOpIntoSelect ( I ) ) return true ; if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) && promoteUniformOpToI32 ( I ) ) return true ; if ( UseMul24Intrin && replaceMulWithMul24 ( I ) ) return true ; bool Changed = false ; Instruction :: BinaryOps Opc = I . getOpcode ( ) ; Type * Ty = I . getType ( ) ; Value * NewDiv = nullptr ; unsigned ScalarSize = Ty -> getScalarSizeInBits ( ) ; SmallVector < BinaryOperator * , <NUM_LIT> > Div64ToExpand ; if ( ( Opc == Instruction :: URem || Opc == Instruction :: UDiv || Opc == Instruction :: SRem || Opc == Instruction :: SDiv ) && ScalarSize <= <NUM_LIT> && ! DisableIDivExpand ) { Value * Num = I . getOperand ( <NUM_LIT> ) ; Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den


<s> Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt = optimizeWithRcp ( NumEltI , DenEltI , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ; if ( ! NewElt ) NewElt = optimizeWithFDivFast ( NumEltI , DenEltI , ReqdAccuracy , HasFP32Denormals , Builder , Mod ) ; if ( ! NewElt ) NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else { NewFDiv = optimizeWithRcp ( Num , Den , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ; if ( ! NewFDiv ) { NewFDiv = optimizeWithFDivFast ( Num , Den , ReqdAccuracy , HasFP32Denormals , Builder , Mod ) ; } } if ( NewFDiv )


<s> Value * NewFDiv = nullptr ; if ( VectorType * VT = dyn_cast < VectorType > ( FDiv . getType ( ) ) ) { NewFDiv = UndefValue :: get ( VT ) ; for ( unsigned I = <NUM_LIT> , E = VT -> getNumElements ( ) ; I != E ; ++ I ) { Value * NumEltI = Builder . CreateExtractElement ( Num , I ) ; Value * DenEltI = Builder . CreateExtractElement ( Den , I ) ; Value * NewElt = optimizeWithRcp ( NumEltI , DenEltI , AllowInaccurateRcp , RcpIsAccurate , Builder , Mod ) ; if ( ! NewElt ) NewElt = optimizeWithFDivFast ( NumEltI , DenEltI , ReqdAccuracy , HasFP32Denormals , Builder , Mod ) ; if ( ! NewElt ) NewElt = Builder . CreateFDiv ( NumEltI , DenEltI ) ; NewFDiv = Builder . CreateInsertElement ( NewFDiv , NewElt , I ) ; } } else { NewFDiv = optimizeWithRcp ( Num , Den , AllowInaccurateRcp , RcpIsAccurate


<s> Value * RCP_F32 = Builder . CreateCall ( RcpDecl , { DEN_F32 } ) ; Constant * UINT_MAX_PLUS_1 = ConstantFP :: get ( F32Ty , BitsToFloat ( <NUM_LIT> ) ) ; Value * RCP_SCALE = Builder . CreateFMul ( RCP_F32 , UINT_MAX_PLUS_1 ) ; Value * RCP = Builder . CreateFPToUI ( RCP_SCALE , I32Ty ) ; Value * RCP_LO , * RCP_HI ; std :: tie ( RCP_LO , RCP_HI ) = getMul64 ( Builder , RCP , Den ) ; Value * NEG_RCP_LO = Builder . CreateNeg ( RCP_LO ) ; Value * RCP_HI_0_CC = Builder . CreateICmpEQ ( RCP_HI , Zero ) ; Value * ABS_RCP_LO = Builder . CreateSelect ( RCP_HI_0_CC , NEG_RCP_LO , RCP_LO ) ; Value * E = getMulHu ( Builder , ABS_RCP_LO , RCP ) ; Value * RCP_A_E = Builder . CreateAdd ( RCP , E ) ; Value * RCP_S_E = Builder . CreateSub ( RCP , E ) ; Value * Tmp0 = Builder . CreateSelect ( RCP_HI_0_CC , RCP_A_E , RCP_S_E ) ; Value * Quotient = getMulHu ( Builder , Tmp0 , Num ) ; Value * Num_S_Remainder = Builder . CreateMul ( Quotient , Den ) ; Value * Remainder = Builder . CreateSub ( Num , Num_S_Remainder ) ; Value * Remainder_GE_Den = Builder . CreateICmpUGE ( Remainder , Den ) ; Value * Remainder_GE_Zero = Builder . CreateICmpUGE ( Num , Num_S_Remainder ) ; Value * Tmp1 = Builder . CreateAnd ( Remainder_GE_Den , Remainder_GE_Zero ) ; Value * Res ; if ( IsDiv ) { Value * Quotient_A_One = Builder . CreateAdd ( Quotient , One ) ; Value * Quotient_S_One = Builder . CreateSub ( Quotient , One ) ; Value * Div = Builder . CreateSelect ( Tmp1 , Quotient_A_One , Quotient ) ; Res = Builder . CreateSelect ( Remainder_GE_Zero , Div , Quotient_S_One ) ; } else { Value * Remainder_S_Den = Builder . CreateSub ( Remainder , Den ) ; Value * Remainder_A_Den = Builder . CreateAdd ( Remainder , Den ) ; Value * Rem = Builder . CreateSelect ( Tmp1 , Remainder_S_Den , Remainder ) ; Res = Builder . CreateSelect ( Remainder_GE_Zero , Rem , Remainder_A_Den ) ; } if ( IsSigned )


<s> const ConstantFP * CNum = dyn_cast < ConstantFP > ( Num ) ; if ( ! CNum


<s> ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * JQ = One ; if ( IsSigned ) { JQ = Builder . CreateXor ( Num , Den ) ; JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateIntrinsic ( <STR_LIT> , { FQM } ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty

) : Builder . CreateFPToUI ( FQ , I32Ty ) ;


<s> JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateIntrinsic ( <STR_LIT> , { FQM } ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateIntrinsic ( <STR_LIT> , { FR } , FQ ) ; FB = Builder . CreateIntrinsic ( <STR_LIT> , { FB } , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) { Value * Rem = Builder . CreateMul (


<s> const <ISA_LIT>TargetMachine & TM = TPC -> getTM < <ISA_LIT>TargetMachine > ( ) ; ST = & TM . getSubtarget < GCNSubtarget > ( F ) ; DA = & getAnalysis < DivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; AMDGPUA<ISA_LIT> = TM . get<ISA_LIT>AS ( ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) { Next = std :: next ( I


<s> if ( ST -> has16BitInsts ( ) && needsPromotionToI32 ( I . getType ( ) ) && DA -> isUniform ( & I ) && promoteUniformOpToI32 ( I ) ) return true ; bool Changed = false ; Instruction :: BinaryOps Opc = I . getOpcode ( ) ; Type * Ty = I . getType ( ) ; Value * NewDiv = nullptr ; if ( ( Opc == Instruction :: URem || Opc == Instruction :: UDiv || Opc == Instruction :: SRem || Opc == Instruction :: SDiv ) && Ty -> getScalarSizeInBits ( ) <= <NUM_LIT> ) { Value * Num = I . getOperand ( <NUM_LIT> ) ; Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( )


<s> } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { Function * Decl = <STR_LIT> ( Mod , <STR_LIT> , Ty ) ; Value * FNeg = Builder . CreateFNeg ( Den ) ; return Builder . CreateCall ( Decl , { FNeg } ) ; } } } if ( AllowInaccurateRcp ) { Function * Decl = <STR_LIT> ( Mod , <STR_LIT> , Ty ) ; Value * Recip = Builder . CreateCall ( Decl , { Den


<s> FunctionPass ( ID ) , TM ( static_cast < const GCNTargetMachine * > ( TM ) ) , ST (

nullptr ) , DA ( nullptr ) , Mod ( nullptr ) , HasUnsafeFPMath ( false ) {


<s> unsigned Size = Ty -> getScalarSizeInBits ( ) ; if ( Size <= <NUM_LIT> && ST -> has16BitInsts ( ) ) return false ; if ( DA -> isUniform ( & I ) ) return false ; Value * LHS = I . getOperand ( <NUM_LIT> ) ; Value * RHS = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; if ( ST -> hasMulU24 ( ) && isU24 ( LHS , Size ) && isU24 ( RHS , Size ) ) { IntrID = <STR_LIT> ; } else if ( ST -> hasMulI24 ( ) && isI24 ( LHS , Size ) && isI24 ( RHS , Size ) ) { IntrID = <STR_LIT> ; } else return false ; SmallVector < Value * , <NUM_LIT> > LHSVals ; SmallVector < Value * , <NUM_LIT> > RHSVals ; SmallVector < Value * , <NUM_LIT> > ResultVals ; extractValues ( Builder , LHSVals , LHS ) ; extractValues ( Builder , RHSVals , RHS ) ; IntegerType * I32Ty = Builder . getInt32Ty ( ) ; FunctionCallee Intrin = <STR_LIT> ( Mod , IntrID ) ; for ( int I = <NUM_LIT> , E = LHSVals . size ( ) ; I != E ; ++ I ) { Value * LHS , * RHS ; if ( IntrID == <STR_LIT> ) { LHS = Builder . CreateZExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateZExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } else { LHS = Builder . CreateSExtOrTrunc ( LHSVals [ I ] , I32Ty ) ; RHS = Builder . CreateSExtOrTrunc ( RHSVals [ I ] , I32Ty ) ; } Value * Result = Builder . CreateCall ( Intrin , { LHS , RHS } ) ; if ( IntrID == <STR_LIT> ) { ResultVals . push_back ( Builder . CreateZExtOrTrunc ( Result , LHSVals [ I ] -> getType ( ) ) ) ; } else { ResultVals . push_back ( Builder . CreateSExtOrTrunc ( Result , LHSVals [ I ]


<s> Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { NewDiv = expandDivRem32 ( Builder , I , Num ,


<s> Value * Num = I . getOperand ( <NUM_LIT> ) ; Value * Den = I . getOperand ( <NUM_LIT> ) ; IRBuilder < > Builder ( & I ) ; Builder . SetCurrentDebugLocation ( I . getDebugLoc ( ) ) ; if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; } if ( NewDiv ) { I . replaceAllUsesWith (


<s> Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FQM ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ; Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ; JQ = Builder . CreateSelect ( CV , JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; Value * Div = Builder . CreateAdd ( IQ , JQ ) ; Value * Res = Div ; if ( ! IsDiv ) { Value * Rem = Builder . CreateMul ( Div , Den ) ; Res = Builder . CreateSub ( Num , Rem ) ; } if ( IsSigned ) { Res = Builder . CreateShl ( Res , <NUM_LIT> - DivBits ) ; Res = Builder . CreateAShr ( Res , <NUM_LIT>


<s> Type * Ty = Den -> getType ( ) ; if ( ! FastUnsafeRcpLegal && Ty -> isFloatTy ( ) && ( HasDenormals || NeedHighAccuracy ) ) return nullptr ; Function * Decl = <STR_LIT> ( Mod , <STR_LIT> , Ty ) ; if ( const ConstantFP * CLHS = dyn_cast < ConstantFP > ( Num ) ) { if ( FastUnsafeRcpLegal || Ty -> isFloatTy ( ) || Ty -> isHalfTy ( ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { return Builder . CreateCall ( Decl , { Den } ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { Value * FNeg = Builder . CreateFNeg ( Den ) ; return Builder . CreateCall ( Decl , { FNeg } ) ; } } } if ( FastUnsafeRcpLegal ) { Value * Recip = Builder . CreateCall ( Decl , { Den


<s> return Builder . CreateCall ( Decl , { Den } ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { Value * FNeg = Builder . CreateFNeg ( Den ) ; return Builder . CreateCall ( Decl , { FNeg } ) ; } } } if ( FastUnsafeRcpLegal ) { Value * Recip = Builder . CreateCall ( Decl , { Den } ) ; return Builder . CreateFMul ( Num , Recip , <STR_LIT> , FPMath ) ; } return nullptr


<s> AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; HasUnsafeFPMath = hasUnsafeFPMath ( F ) ; HasFP32Denormals = ST -> hasFP32Denormals ( F ) ; bool MadeChange = false ; for ( BasicBlock & BB : F ) { BasicBlock :: iterator Next ; for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MadeChange |= visit ( * I ) ; } } return MadeChange


<s> if ( VectorType * VT = dyn_cast < VectorType > ( Ty ) ) { NewDiv = UndefValue :: get ( VT ) ; for ( unsigned N = <NUM_LIT> , E = VT -> getNumElements ( ) ; N != E ; ++ N ) { Value * NumEltN = Builder . CreateExtractElement ( Num , N ) ; Value * DenEltN = Builder . CreateExtractElement ( Den , N ) ; Value * NewElt = expandDivRem32 ( Builder , I , NumEltN , DenEltN ) ; if ( ! NewElt ) NewElt = Builder . CreateBinOp ( Opc , NumEltN , DenEltN ) ; NewDiv = Builder . CreateInsertElement ( NewDiv , NewElt , N ) ; } } else { NewDiv = expandDivRem32 ( Builder , I , Num , Den ) ; } if ( NewDiv ) { I . replaceAllUsesWith ( NewDiv ) ; I . eraseFromParent ( ) ; Changed =