<s> return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI ,


<s> Type * Ty = MemVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; SDValue Ptr = LowerParameterPtr ( DAG , SL , Chain , Offset ) ; SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Align , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; SDValue Val = Load ; if ( Arg && ( Arg -> Flags . isSExt ( ) || Arg -> Flags . isZExt ( ) ) && VT . bitsLT ( MemVT ) ) { unsigned Opc = Arg -> Flags . isZExt ( ) ? <STR_LIT> : <STR_LIT> ; Val = DAG . getNode ( Opc , SL , MemVT , Val , DAG . getValueType ( VT ) ) ; } if ( MemVT . isFloatingPoint ( ) ) Val = getFPExtOrFPTrunc ( DAG , Val , SL , VT ) ; else if ( Signed ) Val = DAG . getSExtOrTrunc ( Val , SL , VT ) ; else Val = DAG . getZExtOrTrunc ( Val ,


<s> MVT PtrVT = getPointerTy ( DL , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg ) , PtrVT ) ; return DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL ,


<s> if ( IsFixed || Info . hasWorkGroupIDX ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDX ) ; if ( IsFixed || Info . hasWorkGroupIDY ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDY ) ; if ( IsFixed || Info . hasWorkGroupIDZ ( ) ) allocateSGPR32Input


<s> SDValue One = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; SDValue LHS = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Op . getOperand ( <NUM_LIT> ) ) ; SDValue RHS = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Op . getOperand ( <NUM_LIT> ) ) ; SDValue Lo0 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , LHS , Zero ) ; SDValue Lo1 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , RHS , Zero ) ; SDValue Lo = DAG . getSelect ( DL , <STR_LIT> , Cond , Lo0 , Lo1 ) ; SDValue Hi0 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , LHS , One ) ; SDValue Hi1 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , RHS , One ) ; SDValue Hi = DAG . getSelect ( DL , <STR_LIT> , Cond , Hi0 , Hi1 ) ; SDValue Res = DAG . getBuildVector ( <STR_LIT> , DL ,


<s> if ( IncomingArg ) { InputReg = loadInputValue ( DAG , ArgRC , ArgVT , DL , * IncomingArg ) ; } else if ( InputID == <STR_LIT> ) { InputReg = getImplicitArgPtr ( DAG , DL ) ; } else { InputReg = DAG . getUNDEF ( ArgVT ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; if ( ! CCInfo . AllocateReg ( OutgoingArg -> getRegister ( ) ) ) report_fatal_error ( <STR_LIT> ) ; } else { unsigned SpecialArgOffset = CCInfo . AllocateStack ( ArgVT . getStoreSize ( ) , Align ( <NUM_LIT> ) ) ; SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain , InputReg , SpecialArgOffset ) ; MemOpChains . push_back ( ArgStore ) ; } } const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; LLT Ty ; std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return ; const ArgDescriptor * IncomingArgX = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgY = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgZ = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; SDValue InputReg ; SDLoc SL ; const bool NeedWorkItemIDX = ! CLI . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDY = ! CLI . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDZ = ! CLI . CB -> hasFnAttr ( <STR_LIT> ) ; if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX && NeedWorkItemIDX ) InputReg = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgX ) ; if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY && NeedWorkItemIDY ) { SDValue Y = loadInputValue ( DAG , ArgRC , <STR_LIT>


<s> uint64_t CRHSVal = CRHS -> getZExtValue ( ) ; if ( ( CC == <STR_LIT> || CC == <STR_LIT> ) && LHS . getOpcode ( ) == <STR_LIT> && isa < ConstantSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) && isa < ConstantSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) && LHS . getConstantOperandVal ( <NUM_LIT> ) != LHS . getConstantOperandVal ( <NUM_LIT> ) && isBoolSGPR ( LHS . getOperand ( <NUM_LIT> ) ) ) { uint64_t CT = LHS . getConstantOperandVal ( <NUM_LIT> ) ; uint64_t CF = LHS . getConstantOperandVal ( <NUM_LIT> ) ; if ( ( CF == CRHSVal && CC == <STR_LIT> ) || ( CT == CRHSVal && CC == <STR_LIT> ) ) return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS . getOperand ( <NUM_LIT> ) , DAG . getConstant ( - <NUM_LIT> , SL , <STR_LIT> ) ) ; if ( ( CF == CRHSVal && CC == <STR_LIT> ) || ( CT == CRHSVal && CC == <STR_LIT> ) ) return LHS . getOperand ( <NUM_LIT> ) ; } } if ( VT != <STR_LIT> && VT != <STR_LIT> && ( Subtarget -> has16BitInsts ( ) && VT != <STR_LIT> ) ) return SDValue ( ) ; if ( ( CC == <STR_LIT> || CC == <STR_LIT> ) && LHS . getOpcode ( ) == <STR_LIT> ) { const ConstantFPSDNode * CRHS = dyn_cast < ConstantFPSDNode > ( RHS ) ; if ( ! CRHS ) return SDValue ( ) ; const APFloat & APF = CRHS -> getValueAPF ( ) ; if ( APF . isInfinity ( ) && ! APF . isNegative ( ) ) { const unsigned IsInfMask = <STR_LIT> | <STR_LIT> ; const unsigned IsFiniteMask = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ; unsigned Mask = CC == <STR_LIT> ? IsInfMask : IsFiniteMask ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS . getOperand (

<NUM_LIT> ) , DAG . getConstant ( Mask , SL , <STR_LIT> ) ) ;


<s> if ( IsD16 ) Data = adjustLoadValueTypeImpl ( Data , ReqRetVT , DL , DAG , Unpacked ) ; EVT LegalReqRetVT = ReqRetVT ; if ( ! ReqRetVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; } else { if ( ( ReqRetVT . getVectorNumElements ( ) % <NUM_LIT> ) == <NUM_LIT> && ReqRetVT . getVectorElementType ( ) . getSizeInBits ( ) == <NUM_LIT> ) { LegalReqRetVT = EVT :: getVectorVT ( * DAG . getContext ( ) , ReqRetVT . getVectorElementType ( ) , ReqRetVT . getVectorNumElements ( ) + <NUM_LIT> ) ; } } Data = DAG . getNode ( <STR_LIT> , DL , LegalReqRetVT , Data ) ; if ( IsTexFail ) { TexFail = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , SDValue ( Result , <NUM_LIT> )

, DAG . getConstant ( MaskPopDwords , DL , <STR_LIT> ) ) ;


<s> reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ; if ( Info -> getFrameOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ; Info -> limitOccupancy ( MF ) ; if ( ST . isWave32 ( ) && ! MF . empty ( ) ) { for ( auto & MBB : MF


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; if ( Info -> isEntryFunction ( ) ) { reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ; if ( Info -> getFrameOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ; Info -> limitOccupancy ( MF ) ; if ( ST . isWave32 ( ) && ! MF . empty ( ) ) { for ( auto & MBB : MF ) { for ( auto & MI : MBB ) { TII -> fixImplicitOperands ( MI )


<s> LLT Ty ; std :: tie ( Reg , RC , Ty ) = MFI . getPreloadedValue ( PVID ) ; return CreateLiveInRegister ( DAG , RC , Reg -> getRegister ( ) ,


<s> case <NUM_LIT> : RC = & <STR_LIT> ; break ; default : RC = TRI -> getAGPRClassForBitWidth ( BitWidth ) ; if ( ! RC ) return std :: make_pair ( <NUM_LIT> , nullptr ) ; break ; } break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ; } if ( Constraint . size ( ) > <NUM_LIT> ) { if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 'a' ) { RC = & <STR_LIT>


<s> <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; SDValue EncodingImm = DAG . getTargetConstant ( Encoding , DL , <STR_LIT> ) ; SDValue ApertureReg = SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , EncodingImm ) , <NUM_LIT> ) ; SDValue ShiftAmount = DAG . getTargetConstant ( WidthM1 + <NUM_LIT> , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , ApertureReg , ShiftAmount ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getObjectPtrOffset ( DL , QueuePtr , TypeSize :: Fixed ( StructOffset ) ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; return DAG . getLoad ( <STR_LIT> , DL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr , PtrInfo , commonAlignment ( Align ( <NUM_LIT> )

, StructOffset ) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> std :: pair < InstructionCost , MVT > Cost = TargetLoweringBase :: getTypeLegalizationCost ( DL , Ty ) ; auto Size = DL . getTypeSizeInBits ( Ty ) ; if ( Size <= <NUM_LIT> ) return


<s> auto F = MI -> getOperand ( <NUM_LIT> ) . getFPImm ( ) -> getValueAPF ( ) ; if ( F . isNaN ( ) && F . isSignaling ( ) ) return false ; return ! F . isDenormal ( ) || denormalsEnabledForType ( MRI . getType ( Reg ) , MF ) ; } if ( MaxDepth == <NUM_LIT> ) return false ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> :


<s> return AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS

:: REGION_ADDRESS || AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ;


<s> if ( SrcAS == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( SrcAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , SegmentNullPtr , <STR_LIT> ) ; SDValue Aperture = getSegmentAperture ( ASC -> getSrcAddressSpace ( ) , SL , DAG ) ; SDValue CvtPtr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src , Aperture ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , CvtPtr ) , FlatNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT && Src . getValueType ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; const MachineFunction & MF = DAG . getMachineFunction ( ) ; DiagnosticInfoUnsupported InvalidAddrSpaceCast ( MF . getFunction ( ) , <STR_LIT> , SL . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( InvalidAddrSpaceCast ) ; return DAG . getUNDEF ( ASC -> getValueType ( <NUM_LIT>


<s> SDValue Src = ASC -> getOperand ( <NUM_LIT> ) ; SDValue FlatNullPtr = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( getTargetMachine ( ) ) ; if ( ASC -> getSrcAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) { unsigned DestAS = ASC -> getDestAddressSpace ( ) ; if ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , FlatNullPtr , <STR_LIT> ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , Ptr , SegmentNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) { unsigned SrcAS = ASC -> getSrcAddressSpace ( ) ; if ( SrcAS == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( SrcAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , SegmentNullPtr , <STR_LIT> ) ; SDValue Aperture = getSegmentAperture ( ASC -> getSrcAddressSpace ( ) , SL , DAG ) ; SDValue CvtPtr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src , Aperture ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , CvtPtr ) , FlatNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT && Src . getValueType ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; const MachineFunction & MF = DAG . getMachineFunction ( ) ; DiagnosticInfoUnsupported InvalidAddrSpaceCast ( MF . getFunction ( ) , <STR_LIT> , SL .


<s> return DAG . getNode ( <STR_LIT> , SL , VT , Src . getOperand ( <NUM_LIT> ) , Src . getOperand ( <NUM_LIT> ) , Src . getOperand ( <NUM_LIT> ) ) ; } if ( const ConstantSDNode * Arg = dyn_cast < ConstantSDNode > ( Src ) ) { if ( Arg -> isNullValue ( ) ) return DAG . getConstant ( <NUM_LIT> , SL , VT ) ; if ( Arg -> isOne ( ) ) { Register Exec


<s> static SDValue lowerBALLOTIntrinsic ( const <ISA_LIT>TargetLowering & TLI , SDNode * N , SelectionDAG & DAG ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; if ( Src . getOpcode ( ) == <STR_LIT> ) { return DAG . getNode ( <STR_LIT> , SL , VT , Src . getOperand ( <NUM_LIT> ) , Src . getOperand ( <NUM_LIT> ) , Src . getOperand ( <NUM_LIT> ) ) ; } if ( const ConstantSDNode * Arg = dyn_cast < ConstantSDNode > ( Src ) ) { if ( Arg -> isNullValue ( ) ) return DAG . getConstant ( <NUM_LIT> , SL , VT ) ; if ( Arg -> isOne ( ) ) { Register Exec ; if ( VT . getScalarSizeInBits ( ) == <NUM_LIT> )


<s> const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * InputPtrReg ; const TargetRegisterClass * RC ; LLT ArgTy ; std :: tie ( InputPtrReg , RC , ArgTy ) = Info -> getPreloadedValue ( <STR_LIT> ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI .

getLiveInVirtReg ( InputPtrReg -> getRegister ( ) ) , PtrVT ) ;


<s> return lowerTrapHsaQueuePtr ( Op , DAG ) ; case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V4 : return Subtarget -> supportsGetDoorbellID ( ) ? lowerTrapHsa ( Op , DAG ) : lowerTrapHsaQueuePtr ( Op , DAG ) ; } } llvm_unreachable ( <STR_LIT> )


<s> SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; uint64_t TrapID = static_cast < uint64_t > ( GCNSubtarget :: TrapID :: LLVMAMDHSATrap ) ; SDValue Ops [ ] = { ToReg , DAG . getTargetConstant ( TrapID , SL , <STR_LIT> ) , SGPR01 ,


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; uint64_t TrapID = static_cast < uint64_t > ( GCNSubtarget :: TrapID :: LLVMAMDHSATrap ) ; SDValue Ops [ ] = { ToReg , DAG . getTargetConstant ( TrapID , SL , <STR_LIT> ) , SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ; return DAG . getNode ( <STR_LIT> ,


<s> const TargetRegisterClass * ArgRC ; LLT Ty ; std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return ; const ArgDescriptor * IncomingArgX = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgY = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgZ = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; SDValue InputReg ; SDLoc SL ; if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX ) InputReg = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgX ) ; if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY ) { SDValue Y = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgY ) ; Y = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Y , DAG . getShiftAmountConstant ( <NUM_LIT> , <STR_LIT> , SL ) ) ; InputReg = InputReg . getNode ( ) ? DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InputReg , Y ) : Y ; } if ( IncomingArgZ && ! IncomingArgZ -> isMasked ( ) && CalleeArgInfo -> WorkItemIDZ ) { SDValue Z = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgZ ) ; Z = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Z , DAG . getShiftAmountConstant ( <NUM_LIT> , <STR_LIT> , SL ) ) ; InputReg = InputReg . getNode ( ) ? DAG . getNode ( <STR_LIT> ,


<s> if ( Shift . getOpcode ( ) == <STR_LIT> ) ShiftOffset -= C -> getZExtValue ( ) ; else ShiftOffset += C -> getZExtValue ( ) ; if ( ShiftOffset < <NUM_LIT> && ( ShiftOffset % <NUM_LIT> ) == <NUM_LIT> ) { return DAG . getNode ( <STR_LIT> + ShiftOffset / <NUM_LIT> , SL , <STR_LIT> , Shift ) ; } } } const TargetLowering & TLI = DAG . getTargetLoweringInfo ( ) ; APInt DemandedBits = APInt :: getBitsSet ( <NUM_LIT> , <NUM_LIT> * Offset , <NUM_LIT> * Offset + <NUM_LIT> ) ; if ( TLI . SimplifyDemandedBits ( Src , DemandedBits , DCI ) ) { if ( N -> getOpcode ( ) != <STR_LIT> ) DCI . AddToWorklist ( N ) ; return SDValue ( N , <NUM_LIT> ) ; } if ( SDValue DemandedSrc = TLI . SimplifyMultipleUseDemandedBits ( Src , DemandedBits , DAG ) ) return DAG

. getNode ( N -> getOpcode ( ) , SL , <STR_LIT> , DemandedSrc ) ;


<s> const <ISA_LIT>RegisterInfo * <ISA_LIT>RI = Subtarget -> getRegisterInfo ( ) ; TargetLowering :: AsmOperandInfoVector TargetConstraints = ParseConstraints ( MF . getDataLayout ( ) , Subtarget -> getRegisterInfo ( ) , * CI ) ; for ( auto & TC : TargetConstraints ) { if ( TC . Type == InlineAsm :: isOutput ) { ComputeConstraintToUse ( TC , SDValue ( ) ) ; unsigned AssignedReg ; const TargetRegisterClass * RC ; std :: tie ( AssignedReg , RC ) = getRegForInlineAsmConstraint ( <ISA_LIT>RI , TC . ConstraintCode , TC . ConstraintVT ) ; if ( RC ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; if ( AssignedReg != <NUM_LIT> && <ISA_LIT>RI -> isSGPRReg ( MRI , AssignedReg ) ) return true ; else if ( <ISA_LIT>RI -> isSGPRClass ( RC )


<s> if ( ! Ty -> isFloatTy ( ) && ( ! Subtarget -> hasGFX90AInsts ( ) || ! Ty -> isDoubleTy ( ) ) ) return AtomicExpansionKind :: CmpXChg ; unsigned AS = RMW -> getPointerAddressSpace ( ) ; if ( ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) && Subtarget -> hasAtomicFaddInsts ( ) ) { if ( RMW -> getFunction ( ) -> getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) != <STR_LIT> ) return AtomicExpansionKind :: CmpXChg ; if ( Subtarget -> hasGFX90AInsts ( ) ) { if ( Ty -> isFloatTy ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return AtomicExpansionKind :: CmpXChg ; auto S<ISA_LIT>D = RMW -> getSyncScopeID ( ) ; if ( S<ISA_LIT>D == SyncScope :: System || S<ISA_LIT>D == RMW -> getContext ( ) . getOrInsertSyncScopeID ( <STR_LIT> ) ) return AtomicExpansionKind :: CmpXChg ; return AtomicExpansionKind ::


<s> EVT VT = Op . getValueType ( ) ; assert ( VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> ) ; SDValue Lo0 , Hi0 ; std :: tie ( Lo0 , Hi0 ) = DAG . SplitVectorOperand ( Op . getNode ( ) , <NUM_LIT> ) ; SDValue Lo1 , Hi1 ; std :: tie ( Lo1 , Hi1 ) = DAG . SplitVectorOperand ( Op . getNode ( ) , <NUM_LIT> ) ; SDLoc SL ( Op


<s> std :: tie ( Lo1 , Hi1 ) = DAG . SplitVectorOperand ( Op . getNode ( ) , <NUM_LIT> ) ; SDValue Lo2 , Hi2 ; std :: tie ( Lo2 , Hi2 ) = DAG . SplitVectorOperand ( Op . getNode ( ) , <NUM_LIT> ) ; SDLoc SL ( Op ) ; SDValue OpLo = DAG . getNode ( Opc , SL , Lo0 . getValueType ( ) , Lo0 , Lo1 , Lo2 , Op


<s> if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) { if ( Subtarget -> useDS128 ( ) && Store -> getAlignment ( ) >= <NUM_LIT> && VT . getStoreSize ( ) == <NUM_LIT> && NumElements != <NUM_LIT> ) return SDValue ( ) ; if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && NumElements == <NUM_LIT> && VT . getStoreSize ( ) == <NUM_LIT> && Store -> getAlignment ( ) < <NUM_LIT> ) { return SplitVectorStore ( Op , DAG


<s> auto CachePolicyConst = cast < ConstantSDNode > ( CachePolicy . getNode ( ) ) ; uint64_t Value = CachePolicyConst -> getZExtValue ( ) ; SDLoc DL ( CachePolicy ) ; if ( GLC ) { * GLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ; Value &= ~ ( uint64_t ) <NUM_LIT> ; } if ( SLC ) { * SLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ; Value &= ~ ( uint64_t ) <NUM_LIT> ; } return Value ==


<s> if ( GLC ) { * GLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ; Value &= ~ ( uint64_t ) <NUM_LIT> ; } if ( SLC ) { * SLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ?


<s> } else { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } } if ( DataDwordVT . isVector ( ) ) Data = padEltsToUndef ( DAG , DL , DataDwordVT , Data , NumDataDwords - MaskPopDwords ) ; if ( IsD16 ) Data = adjustLoadValueTypeImpl ( Data , ReqRetVT , DL , DAG , Unpacked ) ; EVT LegalReqRetVT = ReqRetVT ; if ( ! ReqRetVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; } else { if ( ( ReqRetVT . getVectorNumElements ( ) % <NUM_LIT> ) == <NUM_LIT> && ReqRetVT . getVectorElementType ( ) . getSizeInBits ( ) == <NUM_LIT> ) { LegalReqRetVT = EVT :: getVectorVT ( * DAG . getContext ( ) , ReqRetVT . getVectorElementType ( ) , ReqRetVT . getVectorNumElements


<s> } if ( ImageStore && Subtarget -> hasImageStoreD16Bug ( ) ) { EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; SmallVector < SDValue , <NUM_LIT> > Elts ; DAG . ExtractVectorElements ( IntVData , Elts ) ; SmallVector < SDValue , <NUM_LIT> > PackedElts ; for ( unsigned I = <NUM_LIT> ; I < Elts . size ( ) / <NUM_LIT> ; I += <NUM_LIT> ) { SDValue Pair = DAG . getBuildVector ( <STR_LIT> , DL , { Elts [ I * <NUM_LIT> ] , Elts [ I * <NUM_LIT> + <NUM_LIT> ] } ) ; SDValue IntPair = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Pair ) ; PackedElts . push_back ( IntPair ) ; } PackedElts . resize ( PackedElts . size ( ) * <NUM_LIT> , DAG . getUNDEF ( <STR_LIT> ) ) ; EVT VecVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , PackedElts . size


<s> return DAG . getMergeValues ( Ops , DL ) ; } SmallVector < SDValue , <NUM_LIT> > Elts ; for ( unsigned I = <NUM_LIT> , N = MemVT . getVectorNumElements ( ) ; I != N ; ++ I ) { SDValue Elt = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , NewLD , DAG . getConstant ( I , DL , <STR_LIT> ) ) ; Elts . push_back ( DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Elt ) ) ; } SDValue Ops [ ] = { DAG . getBuildVector ( MemVT , DL , Elts ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned Alignment = Load -> getAlignment ( ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Alignment < MemVT . getStoreSize ( ) && MemVT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorLoad ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasMultiDwordFlatScratchAddressing ( ) ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && Load -> isSimple ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op ,


<s> Arg = allocateVGPR32Input ( CCInfo , Mask ) ; Info . setWorkItemIDX ( Arg ) ; } if ( Info . hasWorkItemIDY ( ) ) { Arg = allocateVGPR32Input ( CCInfo , Mask << <NUM_LIT> , Arg ) ; Info . setWorkItemIDY (


<s> BuildMI ( * LoopBB , LoopBB -> begin ( ) , DL , TII -> get ( <STR_LIT> ) , Src -> getReg ( ) ) . addReg ( Data0 ) ; MRI . setSimpleHint ( Data0 , Src -> getReg ( ) ) ; } BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( EncodedReg ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopBB ) ; return RemainderBB


<s> MRI . setSimpleHint ( Data0 , Src -> getReg ( ) ) ; } BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( EncodedReg ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopBB ) ; return RemainderBB


<s> break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; } break ; case 'a' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; return std :: make_pair ( <NUM_LIT> , RC ) ; } break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ; } if ( Constraint . size ( ) > <NUM_LIT> ) { if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 'a' ) { RC = & <STR_LIT> ; } if ( RC ) { uint32_t Idx ; bool Failed = Constraint . substr ( <NUM_LIT> ) . getAsInteger


<s> return CreateLiveInRegister ( DAG , & <STR_LIT> , RegNo , <STR_LIT> ) ; } SDLoc SL ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , QueuePtr , DAG . getConstant ( StructOffset , SL


<s> <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , QueuePtr , DAG . getConstant ( StructOffset , SL , <STR_LIT> ) ) ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( * DAG . getContext ( ) ) , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ; return DAG . getLoad ( <STR_LIT> , SL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr , PtrInfo

, MinAlign ( <NUM_LIT> , StructOffset ) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; SDLoc DL ( Op ) ; SDValue Chain = Op . getOperand ( <NUM_LIT> ) ; unsigned IntrinsicID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : { const ConstantSDNode * Tgt = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * En = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * Done = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * VM = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const SDValue Ops [ ] = { Chain , DAG . getTargetConstant ( Tgt -> getZExtValue ( ) , DL , <STR_LIT> ) , DAG . getTargetConstant ( En -> getZExtValue ( ) , DL , <STR_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getTargetConstant ( VM -> getZExtValue ( ) , DL , <STR_LIT> ) } ; unsigned Opc = Done -> isNullValue ( ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( Opc , DL , Op -> getVTList ( ) , Ops ) ; } case <STR_LIT> : { const ConstantSDNode * Tgt = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * En = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; SDValue Src0 = Op .


<s> MachineBasicBlock * <ISA_LIT>TargetLowering :: EmitInstrWithCustomInserter ( MachineInstr * MI , MachineBasicBlock * BB ) const { switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; BuildMI ( * BB , MI -> getIterator ( ) , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addOperand ( MI -> getOperand ( <NUM_LIT> ) ) ; MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : return BB ; case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo


<s> static SDValue emitNonHSAIntrinsicError ( SelectionDAG & DAG , EVT VT ) { DiagnosticInfoUnsupported BadIntrin ( * DAG . getMachineFunction ( ) . getFunction ( ) , <STR_LIT> ) ; DAG . getContext ( ) -> diagnose ( BadIntrin ) ; return DAG . getUNDEF


<s> SDLoc DL ( GSD ) ; const GlobalValue * GV = GSD -> getGlobal ( ) ; EVT PtrVT = Op . getValueType ( ) ; SDValue GA = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , GSD -> getOffset ( ) + <NUM_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , PtrVT ,


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( Subtarget -> getRegisterInfo ( ) ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; Type * Ty = VT . getTypeForEVT ( * DAG . getContext ( ) ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg ) , PtrVT ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL , PtrVT ) ) ; SDValue PtrOffset = DAG . getUNDEF ( PtrVT ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL .


<s> SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL , PtrVT ) ) ; SDValue PtrOffset = DAG . getUNDEF ( PtrVT ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; <STR_LIT> ExtTy = Signed ? <STR_LIT> : <STR_LIT> ; if ( MemVT . isFloatingPoint ( ) ) ExtTy = <STR_LIT> ; return DAG . getLoad ( <STR_LIT> , ExtTy , VT , SL , Chain , Ptr , PtrOffset , PtrInfo , MemVT ,


<s> DAG . getContext ( ) -> diagnose ( NoTrap ) ; return DAG . getNode ( <STR_LIT> , SDLoc ( Op ) , <STR_LIT> , Op


<s> DiagnosticInfoUnsupported NoTrap ( * MF . getFunction ( ) , <STR_LIT> , Op . getDebugLoc ( ) , DS_Warning ) ; DAG . getContext ( )


<s> } assert ( DCI . isBeforeLegalize ( ) && <STR_LIT> ) ; unsigned NElts = SrcVT . getVectorNumElements ( ) ; if ( ! SrcVT . isSimple ( ) && NElts != <NUM_LIT> ) return SDValue ( ) ; if ( <STR_LIT> ( Src . getNode ( ) ) && Src . hasOneUse ( ) ) { EVT LoadVT = getEquivalentMemType ( * DAG . getContext ( ) , SrcVT ) ; EVT RegVT = getEquivalentLoadRegType ( * DAG . getContext ( ) , SrcVT ) ; EVT FloatVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , NElts ) ; LoadSDNode * Load = cast < LoadSDNode > ( Src ) ; unsigned AS = Load -> getAddressSpace ( ) ; unsigned Align = Load -> getAlignment ( ) ; Type * Ty = LoadVT . getTypeForEVT ( * DAG . getContext ( ) ) ; unsigned ABIAlignment = DAG . getDataLayout ( ) . getABITypeAlignment ( Ty ) ; if ( Align < ABIAlignment && ! allowsMisalignedMemoryAccesses ( LoadVT , AS , Align , nullptr ) ) { return SDValue ( ) ; } SDValue NewLoad = DAG . getExtLoad ( <STR_LIT> , DL , RegVT , Load -> getChain ( ) , Load -> getBasePtr ( ) , LoadVT , Load -> getMemOperand ( ) ) ; DAG . ReplaceAllUsesOfValueWith ( SDValue ( Load , <NUM_LIT> ) , NewLoad . getValue ( <NUM_LIT> ) ) ; SmallVector < SDValue , <NUM_LIT> > Elts ; if ( RegVT . isVector ( ) ) DAG . ExtractVectorElements ( NewLoad , Elts ) ; else Elts . push_back ( NewLoad ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; unsigned EltIdx = <NUM_LIT> ; for ( SDValue Elt : Elts ) { unsigned ComponentsInElt = std :: min ( <NUM_LIT> , NElts - <NUM_LIT> * EltIdx ) ; for ( unsigned I = <NUM_LIT> ; I < ComponentsInElt ; ++ I ) { unsigned Opc = <STR_LIT> + I ; SDValue Cvt = DAG . getNode ( Opc , DL , <STR_LIT> , Elt ) ; DCI . AddToWorklist ( Cvt . getNode ( ) ) ; Ops . push_back ( Cvt


<s> unsigned NElts = SrcVT . getVectorNumElements ( ) ; if ( ! SrcVT . isSimple ( ) && NElts != <NUM_LIT> ) return SDValue ( ) ; if ( <STR_LIT> ( Src . getNode ( ) ) && Src . hasOneUse ( ) ) { EVT LoadVT = getEquivalentMemType ( * DAG . getContext ( ) , SrcVT ) ; EVT RegVT = getEquivalentLoadRegType ( * DAG . getContext ( ) , SrcVT ) ; EVT FloatVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , NElts ) ; LoadSDNode * Load = cast < LoadSDNode > ( Src ) ; unsigned AS = Load -> getAddressSpace ( ) ; unsigned Align = Load -> getAlignment ( ) ; Type * Ty = LoadVT . getTypeForEVT ( * DAG . getContext ( ) ) ; unsigned ABIAlignment = DAG . getDataLayout ( ) . getABITypeAlignment ( Ty ) ; if ( Align < ABIAlignment && ! allowsMisalignedMemoryAccesses ( LoadVT , AS , Align , nullptr ) ) { return SDValue ( ) ; } SDValue NewLoad = DAG . getExtLoad ( <STR_LIT> , DL , RegVT , Load -> getChain ( ) , Load -> getBasePtr ( ) , LoadVT , Load ->


<s> unsigned Opcode = Node -> getMachineOpcode ( ) ; if ( TII -> isMIMG ( Opcode ) && ! TII -> get ( Opcode ) . mayStore ( ) ) adjustWritemask ( Node , DAG ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { legalizeTargetIndependentNode ( Node , DAG ) ; return Node ; } return


<s> MachineSDNode * <ISA_LIT>TargetLowering :: wrapAddr64Rsrc ( SelectionDAG & DAG , const SDLoc & DL , SDValue Ptr ) const { const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; const SDValue Ops0 [ ] = { DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , buildSMovImm32 ( DAG , DL , <NUM_LIT> ) , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , buildSMovImm32 ( DAG , DL , TII -> getDefaultRsrcDataFormat

( ) >> <NUM_LIT> ) , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) } ;


<s> Info . memVT = <STR_LIT> ( CI . getType ( ) , true ) ; if ( Info . memVT == <STR_LIT> ) { Info . memVT = memVTFromAggregate ( CI . getType ( ) ) ; } Info . flags |= MachineMemOperand :: MOLoad ; } else if ( Attr . hasFnAttribute ( Attribute :: WriteOnly ) ) { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . flags |= MachineMemOperand :: MOStore ; } else { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; } return true ; } switch ( IntrID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT>


<s> EVT IntLoadVT = LoadVT . changeTypeToInteger ( ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , DL , IntLoadVT , Result ) ; return DAG . getNode ( <STR_LIT> , DL , LoadVT , Trunc ) ; } return DAG . getNode ( <STR_LIT> , DL , LoadVT , Result


<s> if ( const <STR_LIT> * RsrcIntr = <STR_LIT> ( IntrID ) ) { AttributeList Attr = <STR_LIT> ( CI . getContext ( ) , ( <STR_LIT> ) IntrID ) ; if ( Attr . hasFnAttribute ( Attribute :: ReadNone ) ) return false ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RsrcIntr -> IsImage ) { Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( RsrcIntr -> RsrcArg ) ) ; Info . align = <NUM_LIT> ; } else { Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( RsrcIntr -> RsrcArg ) ) ; } Info . flags = MachineMemOperand :: MODereferenceable ; if ( Attr . hasFnAttribute ( Attribute :: ReadOnly ) ) { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . flags |= MachineMemOperand ::


<s> } else { EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = ( StoreVT == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( <STR_LIT> , DL , EquivStoreVT ,


<s> if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { return isLegalMUBUFAddressingMode ( AM ) ; } else if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . REGION_ADDRESS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false


<s> if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale ==


<s> switch ( IID ) { case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; Res = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; Res = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; unsigned Opc = getImageOpcode ( IID ) ; Res = DAG . getMemIntrinsicNode ( Opc , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> SDVTList VTList = DAG . getVTList ( EquivLoadVT , <STR_LIT> ) ; SDValue Res ; SDLoc DL ( Op ) ; MemSDNode * M = cast < MemSDNode > ( Op ) ; unsigned IID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IID ) { case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; Res = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; Res = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; unsigned Opc = getImageOpcode ( IID ) ; Res = DAG . getMemIntrinsicNode ( Opc , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : case <STR_LIT>


<s> SDValue <ISA_LIT>TargetLowering :: lowerKernargMemParameter ( SelectionDAG & DAG , EVT VT , EVT MemVT , const SDLoc & SL , SDValue Chain , uint64_t Offset , bool Signed , const <STR_LIT> * Arg ) const { const DataLayout & DL = DAG . getDataLayout ( ) ; Type * Ty = MemVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy )


<s> PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , Offset ) ; SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Align , MachineMemOperand :: MODereferenceable | MachineMemOperand


<s> EVT VT = Store -> getMemoryVT ( ) ; if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue (


<s> return ; } case <STR_LIT> : { if ( SDValue Res = lowerEXTRACT_VECTOR_ELT ( SDValue ( N , <NUM_LIT> ) , DAG ) ) Results . push_back ( Res ) ; return ; } case <STR_LIT> : { unsigned IID = cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IID ) { case <STR_LIT> : { SDValue Src0 = N -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; SDValue Cvt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src0 = N -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; unsigned Opcode ; if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else Opcode = <STR_LIT> ; SDValue Cvt = DAG . getNode ( Opcode , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } } break ; } case <STR_LIT> : { SDValue Chain ; if ( SDValue Res = lowerIntrinsicWChain_IllegalReturnType ( SDValue ( N , <NUM_LIT> ) , Chain , DAG ) ) { Results . push_back ( Res ) ; Results . push_back ( Chain ) ; return ; } break ; } case <STR_LIT> : { SDLoc SL ( N ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ; SDValue LHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N ->


<s> bool <ISA_LIT>TargetLowering :: allowsMisalignedMemoryAccessesImpl ( unsigned Size , unsigned AddrSpace , unsigned Align , MachineMemOperand :: Flags Flags , bool * IsFast ) const { if ( IsFast ) * IsFast = false ; if ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { bool AlignedBy4 = ( Align % <NUM_LIT> == <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) ) { bool AlignedBy4 = Align >= <NUM_LIT> ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccess ( ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? Align >= <NUM_LIT> : Align != <NUM_LIT> ; } return true


<s> if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccess ( ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? Align >= <NUM_LIT> : Align != <NUM_LIT> ; } return true ; } if ( Size < <NUM_LIT> ) return false ; if ( IsFast ) * IsFast = true ; return Size >= <NUM_LIT> &&


<s> if ( MaskPopVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } else { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } TexFail = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , SDValue ( Result , <NUM_LIT> ) , DAG . getConstant ( MaskPopDwords , DL , <STR_LIT> ) ) ; } if ( DataDwordVT . isVector ( ) ) Data = padEltsToUndef ( DAG , DL , DataDwordVT , Data , NumDataDwords - MaskPopDwords ) ; if ( IsD16 ) Data = adjustLoadValueTypeImpl ( Data , ReqRetVT , DL , DAG , Unpacked ) ; if ( ! ReqRetVT . isVector ( ) ) Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT , Data ) ; if ( TexFail ) return DAG . getMergeValues ( { Data , TexFail

, SDValue ( Result , <NUM_LIT> ) } , DL ) ;


<s> case 'a' : return C_RegisterClass ; case 'A' : return C_Other ; } } return TargetLowering :: getConstraintType (


<s> if ( Subtarget -> hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; SDValue EncodingImm = DAG . getTargetConstant ( Encoding , DL , <STR_LIT> ) ; SDValue ApertureReg = SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , EncodingImm ) , <NUM_LIT> ) ; SDValue ShiftAmount = DAG . getTargetConstant ( WidthM1 + <NUM_LIT> , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , ApertureReg , ShiftAmount ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS


<s> Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol || ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerElementType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . ptrVal = MFI -> getGWSPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ) ; Info . memVT = <STR_LIT> ; Info . size = <NUM_LIT> ; Info . align = Align ( <NUM_LIT> ) ; Info . flags = MachineMemOperand :: MOStore ; if ( IntrID == <STR_LIT> ) Info . flags = MachineMemOperand :: MOLoad ; return true


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return Op . getValueType ( ) . getScalarType ( ) != <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( DAG , Op . getValueType ( ) ) ) return true ; for ( unsigned I = <NUM_LIT> , E = Op . getNumOperands ( ) ; I != E ; ++ I ) { if ( ! isCanonicalized ( DAG , Op . getOperand ( I ) , MaxDepth - <NUM_LIT> ) ) return false ; } return true ; } case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { for ( unsigned i = <NUM_LIT> , e = Op . getNumOperands ( ) ; i != e ; ++ i ) { SDValue SrcOp = Op . getOperand ( i ) ; if ( ! isCanonicalized ( DAG , SrcOp , MaxDepth - <NUM_LIT> ) ) return false ; } return true ; } case <STR_LIT> : case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : return false ; case <STR_LIT> : { SDValue Src = Op . getOperand ( <NUM_LIT> ) ; if ( Src . getValueType ( ) == <STR_LIT> && Src . getOpcode ( ) == <STR_LIT> ) { SDValue TruncSrc = Src . getOperand ( <NUM_LIT> ) ; if ( TruncSrc . getValueType ( ) == <STR_LIT> && TruncSrc . getOpcode ( ) == <STR_LIT> && TruncSrc . getOperand ( <NUM_LIT> ) . getValueType ( ) == <STR_LIT> ) { return isCanonicalized ( DAG , TruncSrc . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } } return false ; } case <STR_LIT> : { unsigned IntrinsicID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> void <ISA_LIT>TargetLowering :: LowerAsmOperandForConstraint ( SDValue Op , std :: string & Constraint , std :: vector < SDValue > & Ops , SelectionDAG & DAG ) const { if ( Constraint . length ( ) == <NUM_LIT> && Constraint


<s> if ( Size > <NUM_LIT> ) return ; uint64_t Val ; bool IsConst = false ; if ( ConstantSDNode * C = dyn_cast < ConstantSDNode > ( Op ) ) { Val = C -> getSExtValue ( ) ; IsConst = true ; } else if ( ConstantFPSDNode * C = dyn_cast < ConstantFPSDNode > ( Op ) ) { Val = C -> getValueAPF ( ) . bitcastToAPInt ( ) . getSExtValue ( ) ; IsConst = true ; } else if ( BuildVectorSDNode * V = dyn_cast < BuildVectorSDNode > ( Op ) ) { if ( Size != <NUM_LIT> || Op . getNumOperands ( ) != <NUM_LIT> ) return ; if ( Op . getOperand ( <NUM_LIT> ) . isUndef ( ) || Op . getOperand ( <NUM_LIT> ) . isUndef ( ) ) return ; if ( ConstantSDNode * C = V -> getConstantSplatNode ( ) ) { Val = C -> getSExtValue ( ) ; IsConst = true ; } else if ( ConstantFPSDNode * C = V -> getConstantFPSplatNode ( ) ) { Val = C -> getValueAPF ( ) . bitcastToAPInt ( )


<s> if ( ( GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS && shouldUseLDSConstAddress ( GV ) ) || GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: REGION_ADDRESS || GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; EVT PtrVT = Op . getValueType ( ) ; if ( GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { SDValue GA = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , GSD -> getOffset ( ) , <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , GA ) ; } if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty ,


<s> NumLoads = NumElts / <NUM_LIT> ; LoadVT = <STR_LIT> ( LoadVT . getScalarType ( ) , <NUM_LIT> ) ; } SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ; SDValue Ops [ ] = { DAG . getEntryNode ( ) , Rsrc , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , { } , { } , { } , CachePolicy , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ; setBufferOffsets ( Offset , DAG , & Ops [ <NUM_LIT> ] , NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ) ; uint64_t InstOffset = cast < ConstantSDNode > ( Ops [ <NUM_LIT> ] ) -> getZExtValue ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumLoads ; ++ i ) { Ops [ <NUM_LIT> ] = DAG . getTargetConstant ( InstOffset + <NUM_LIT> * i , DL , <STR_LIT> ) ; Loads . push_back ( getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , LoadVT , MMO , DAG ) ) ; } if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) return DAG . getNode (


<s> SDLoc SL ( N ) ; SDValue Ins = N -> getOperand ( <NUM_LIT> ) ; EVT IdxVT = Idx . getValueType ( ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; for ( unsigned I = <NUM_LIT> , E = VecVT . getVectorNumElements ( ) ; I < E ; ++ I ) { SDValue IC = DAG . getConstant ( I , SL , IdxVT ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec , IC ) ; SDValue V = DAG . getSelectCC ( SL , Idx , IC , Ins , Elt , <STR_LIT> ) ; Ops . push_back ( V ) ; } return DAG . getBuildVector ( VecVT


<s> unsigned NumElem = VecVT . getVectorNumElements ( ) ; if ( VecSize <= <NUM_LIT> && EltSize < <NUM_LIT> ) return false ; if ( EltSize < <NUM_LIT> ) return true ; if ( Idx -> isDivergent ( ) ) return true ; unsigned NumInsts = NumElem + ( ( EltSize + <NUM_LIT> ) /


<s> assert ( Offset == <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , Dst ) . add ( * SrcVec ) . add ( * Val ) . addImm ( SubReg ) ; MI . eraseFromParent ( ) ; return & MBB ; } if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , false ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( SrcVec -> getReg ( ) , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( SrcVec -> getReg ( ) , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { const MCInstrDesc & MovRelDesc = TII -> get ( getMOVRELDPseudo ( TRI , VecRC ) ) ; BuildMI ( MBB , I , DL , MovRelDesc ) . addReg ( Dst , RegState :: Define ) . addReg ( SrcVec -> getReg ( ) ) . add ( * Val ) . addImm ( SubReg - <STR_LIT> ) ; } MI . eraseFromParent ( ) ; return & MBB ; } if ( Val -> isReg ( ) ) MRI . clearKillFlags ( Val -> getReg ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; Register PhiReg = MRI . createVirtualRegister ( VecRC ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , SrcVec -> getReg ( ) , PhiReg , Offset , UseGPRIdxMode , false ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode


<s> unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; bool UseGPRIdxMode = ST . useVGPRIndexMode ( EnableVGPRIndexMode ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return & MBB ; } const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; Register PhiReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register InitReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , InitReg ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , InitReg , PhiReg , Offset , UseGPRIdxMode , true ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode


<s> int DPDenormModeDefault = ST -> hasFP64Denormals ( ) ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT ; int Mode = SPDenormMode | ( DPDenormModeDefault << <NUM_LIT> ) ; return DAG . getTargetConstant ( Mode , SL


<s> const SDValue One = DAG . getConstantFP ( <NUM_LIT> , SL , <STR_LIT> ) ; SDVTList ScaleVT = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue DenominatorScaled = DAG . getNode ( <STR_LIT> , SL , ScaleVT , RHS , RHS , LHS ) ; SDValue NumeratorScaled = DAG . getNode ( <STR_LIT> , SL , ScaleVT , LHS , RHS , LHS ) ; SDValue ApproxRcp = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , DenominatorScaled ) ; SDValue NegDivScale0 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , DenominatorScaled ) ; const unsigned Denorm32Reg = <STR_LIT> :: ID_MODE | ( <NUM_LIT> << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ; const SDValue BitField = DAG . getTargetConstant ( Denorm32Reg , SL , <STR_LIT> ) ; if ( ! Subtarget -> hasFP32Denormals ( ) ) { SDVTList BindParamVTs = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue EnableDenorm ; if ( Subtarget -> hasDenormModeInst ( ) ) { const SDValue EnableDenormValue = getSPDenormModeValue ( FP_DENORM_FLUSH_NONE , DAG , SL , Subtarget ) ; EnableDenorm = DAG . getNode ( <STR_LIT> , SL , BindParamVTs , DAG . getEntryNode ( ) , EnableDenormValue ) ; } else { const SDValue EnableDenormValue = DAG . getConstant ( FP_DENORM_FLUSH_NONE , SL , <STR_LIT> ) ; EnableDenorm = DAG . getNode ( <STR_LIT> , SL , BindParamVTs , DAG . getEntryNode ( ) , EnableDenormValue , BitField ) ; } SDValue Ops [ <NUM_LIT> ] = { NegDivScale0 , EnableDenorm . getValue ( <NUM_LIT> ) , EnableDenorm . getValue (


<s> SDNode * <ISA_LIT>TargetLowering :: PostISelFolding ( MachineSDNode * Node , SelectionDAG & DAG ) const { const <ISA_LIT>InstrInfo * TII = getSubtarget ( )


<s> unsigned Opcode = Node -> getMachineOpcode ( ) ; if ( TII -> isMIMG ( Opcode ) && ! TII -> get ( Opcode ) . mayStore ( ) ) adjustWritemask ( Node , DAG ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { legalizeTargetIndependentNode ( Node , DAG ) ; return Node


<s> Lane = SubIdx2Lane ( I -> getConstantOperandVal ( <NUM_LIT> ) ) ; unsigned Comp ; for ( unsigned i = <NUM_LIT> , Dmask = OldDmask ; i <= Lane ; i ++ ) { Comp = countTrailingZeros ( Dmask ) ; Dmask &= ~ ( <NUM_LIT> << Comp ) ; } if ( Users [ Lane ] ) return Node ; Users [ Lane ] = * I ; NewDmask |= <NUM_LIT> << Comp ; } if ( NewDmask == OldDmask ) return Node ; unsigned BitsSet = countPopulation ( NewDmask ) ; int NewOpcode = <STR_LIT> ( Node -> getMachineOpcode ( ) , BitsSet ) ; assert ( NewOpcode != - <NUM_LIT> && NewOpcode != static_cast < int > ( Node -> getMachineOpcode ( ) ) && <STR_LIT> ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; Ops . insert ( Ops . end ( ) , Node -> op_begin ( ) , Node -> op_begin ( ) + DmaskIdx ) ; Ops . push_back ( DAG . getTargetConstant ( NewDmask , SDLoc ( Node ) , <STR_LIT> ) ) ; Ops . insert ( Ops . end ( ) , Node -> op_begin ( ) + DmaskIdx + <NUM_LIT> , Node -> op_end ( ) ) ; MVT SVT = Node -> getValueType ( <NUM_LIT> ) . getVectorElementType ( ) . getSimpleVT ( ) ; MVT ResultVT = BitsSet == <NUM_LIT> ? SVT : <STR_LIT> ( SVT , BitsSet == <NUM_LIT> ? <NUM_LIT> : BitsSet ) ; SDVTList NewVTList = HasChain ? DAG . getVTList ( ResultVT , <STR_LIT> ) : DAG . getVTList ( ResultVT ) ; MachineSDNode * NewNode = DAG . getMachineNode ( NewOpcode , SDLoc ( Node ) , NewVTList , Ops ) ; if ( HasChain ) { DAG . setNodeMemRefs ( NewNode , Node -> memoperands ( ) ) ; DAG . ReplaceAllUsesOfValueWith ( SDValue ( Node , <NUM_LIT> ) , SDValue ( NewNode , <NUM_LIT> ) ) ; } if ( BitsSet == <NUM_LIT> ) { assert ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) ) ; SDNode * Copy = DAG . getMachineNode ( TargetOpcode :: COPY , SDLoc ( Node ) , Users [ Lane ] -> getValueType ( <NUM_LIT> ) , SDValue ( NewNode , <NUM_LIT> ) ) ; DAG . ReplaceAllUsesWith ( Users [ Lane ] , Copy ) ; return nullptr ; } for ( unsigned i = <NUM_LIT> , Idx = <STR_LIT> ; i < <NUM_LIT> ; ++ i ) { SDNode * User = Users [ i ] ; if ( ! User ) continue ; SDValue Op = DAG . getTargetConstant ( Idx , SDLoc ( User ) , <STR_LIT> ) ; DAG . UpdateNodeOperands ( User , SDValue ( NewNode , <NUM_LIT> ) , Op ) ; switch ( Idx ) { default : break


<s> else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; } return ! KDA || KDA -> isDivergent ( FLI -> getValueFromVirtualReg ( Reg ) ) ; } } break ; case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ; } break ; case <STR_LIT> : return true ; break ; case <STR_LIT> : { } return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true


<s> SDValue <ISA_LIT>TargetLowering :: LowerSTORE ( SDValue Op , SelectionDAG & DAG ) const { SDLoc DL ( Op ) ; StoreSDNode * Store = cast < StoreSDNode > ( Op ) ; EVT VT = Store -> getMemoryVT ( ) ; if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable (


<s> case <STR_LIT> : { SDValue Elt0 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; SDValue Elt1 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; DCI . AddToWorklist ( Elt0 . getNode ( ) ) ; DCI . AddToWorklist ( Elt1 . getNode ( ) ) ; return DAG . getNode ( Opc , SL , EltVT , Elt0 , Elt1 , Vec -> getFlags ( ) ) ; } } } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ; unsigned LeftoverBitIdx = BitIndex % <NUM_LIT> ; SDLoc SL ( N ) ; SDValue Cast = DAG . getNode ( <STR_LIT> , SL , NewVT ,


<s> if ( ConstantFPSDNode * CFP = isConstOrConstSplatFP ( N0 ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; return getCanonicalConstantFP ( DAG , SDLoc ( N ) , VT , CFP -> getValueAPF ( ) ) ; } if ( N0 . getOpcode ( ) == <STR_LIT> && VT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) { SDLoc SL ( N ) ; SDValue NewElts [ <NUM_LIT> ] ; SDValue Lo = N0 . getOperand ( <NUM_LIT> ) ; SDValue Hi = N0 . getOperand ( <NUM_LIT> ) ; EVT EltVT = Lo . getValueType ( ) ; if ( vectorEltWillFoldAway ( Lo ) || vectorEltWillFoldAway ( Hi ) ) { for ( unsigned I = <NUM_LIT> ; I != <NUM_LIT> ; ++ I ) { SDValue Op = N0 . getOperand ( I ) ; if ( ConstantFPSDNode * CFP = dyn_cast < ConstantFPSDNode > ( Op ) ) { NewElts [ I ] = getCanonicalConstantFP ( DAG , SL , EltVT , CFP -> getValueAPF ( ) ) ; } else if ( Op .


<s> SelectionDAG & DAG = DCI . DAG ; SDValue N0 = N -> getOperand ( <NUM_LIT> ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( N0 . isUndef ( ) ) { APFloat QNaN = APFloat :: getQNaN ( SelectionDAG :: EVTToAPFloatSemantics ( VT ) ) ; return DAG . getConstantFP ( QNaN , SDLoc ( N ) , VT ) ; } if ( ConstantFPSDNode * CFP = isConstOrConstSplatFP ( N0 ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; return getCanonicalConstantFP ( DAG , SDLoc ( N ) , VT , CFP -> getValueAPF ( ) ) ; } if ( N0 . getOpcode ( ) == <STR_LIT> && VT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) { SDLoc SL ( N ) ; SDValue NewElts [ <NUM_LIT> ] ; SDValue Lo = N0 . getOperand ( <NUM_LIT> ) ; SDValue Hi = N0 . getOperand ( <NUM_LIT> ) ; EVT EltVT = Lo . getValueType ( ) ; if ( vectorEltWillFoldAway ( Lo ) || vectorEltWillFoldAway ( Hi ) ) { for ( unsigned I = <NUM_LIT> ; I != <NUM_LIT> ; ++ I ) { SDValue Op = N0 . getOperand ( I ) ; if ( ConstantFPSDNode * CFP = dyn_cast <


<s> ConstantSDNode * C1 = nullptr ; if ( N0 . getOpcode ( ) == <STR_LIT> ) { if ( ( C1 = dyn_cast < ConstantSDNode > ( N0 . getOperand ( <NUM_LIT> ) ) ) ) N0 = N0 . getOperand ( <NUM_LIT> ) ; } else if ( ( C1 = dyn_cast < ConstantSDNode > ( N0 ) ) ) N0 = SDValue ( ) ; if ( C1 ) { unsigned ImmOffset = C1 -> getZExtValue ( ) ; unsigned Overflow = ImmOffset & ~ MaxImm ; ImmOffset -=


<s> case <STR_LIT> : { if ( getTargetMachine ( ) . getOptLevel ( ) > CodeGenOpt :: None ) return performMinMaxCombine ( N , DCI ) ; break ; } case <STR_LIT> : return performFMACombine ( N , DCI ) ; case <STR_LIT> : { if ( SDValue Widended = widenLoad ( cast < LoadSDNode > ( N ) , DCI ) ) return Widended ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : return performRcpCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> : return performFMed3Combine ( N , DCI ) ; case <STR_LIT> : return performCvtPkRTZCombine ( N , DCI ) ; case <STR_LIT> : return performClampCombine ( N , DCI ) ; case <STR_LIT> : { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> || VT == <STR_LIT> ) { SDLoc SL ( N ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ; EVT EltVT = Src . getValueType ( ) ; if ( EltVT == <STR_LIT> ) Src = DAG . getNode ( <STR_LIT> , SL , <STR_LIT>


<s> case <STR_LIT> : return performAddCombine ( N , DCI ) ; case <STR_LIT> : return performSubCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : return performAddCarrySubCarryCombine ( N , DCI ) ; case <STR_LIT> : return performFAddCombine ( N , DCI ) ; case <STR_LIT> : return performFSubCombine ( N , DCI ) ; case <STR_LIT> : return performSetCCCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( getTargetMachine ( ) . getOptLevel ( ) > CodeGenOpt :: None ) return performMinMaxCombine ( N , DCI ) ; break ; } case <STR_LIT> : return performFMACombine ( N , DCI ) ; case <STR_LIT> : { if ( SDValue Widended = widenLoad ( cast < LoadSDNode > ( N ) , DCI ) ) return Widended ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N ,


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Elt0 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; SDValue Elt1 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; DCI . AddToWorklist ( Elt0 . getNode ( ) ) ; DCI . AddToWorklist ( Elt1 . getNode ( ) ) ; return DAG . getNode ( Opc , SL , EltVT , Elt0 , Elt1 , Vec -> getFlags ( ) ) ; } } } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ; unsigned LeftoverBitIdx =


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Elt0 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; SDValue Elt1 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; DCI . AddToWorklist ( Elt0 . getNode ( ) ) ; DCI . AddToWorklist ( Elt1 . getNode ( ) ) ; return DAG . getNode ( Opc , SL , EltVT , Elt0 , Elt1 , Vec -> getFlags ( ) ) ; } } } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( )


<s> } Info . flags |= MachineMemOperand :: MOLoad ; } else if ( Attr . hasFnAttribute ( Attribute :: WriteOnly ) ) { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . flags |= MachineMemOperand :: MOStore ; } else { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; } return true ; } switch ( IntrID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align = <NUM_LIT> ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align = <NUM_LIT> ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . ptrVal = MFI -> getGWSPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ) ; Info . memVT =


<s> unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned MovExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , TmpExec ) ; BuildMI ( MBB , I , DL , TII -> get ( MovExecOpc ) , SaveExec ) . addReg ( Exec ) ; MachineBasicBlock * LoopBB = MF -> CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF -> CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF -> insert ( MBBI , LoopBB ) ; MF -> insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor ( RemainderBB ) ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; MBB . addSuccessor ( LoopBB ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; auto InsPt = emitLoadM0FromVGPRLoop ( TII , MRI , MBB , * LoopBB , DL , * Idx , InitResultReg , DstReg , PhiReg , TmpExec , Offset , UseGPRIdxMode , IsIndirectSrc ) ; MachineBasicBlock :: iterator First = RemainderBB -> begin ( ) ; BuildMI ( * RemainderBB , First , DL , TII -> get ( MovExecOpc )


<s> if ( Info -> isEntryFunction ( ) ) { reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT>


<s> case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol || ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerElementType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT>


<s> return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol || ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerElementType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |=


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( VT . getTypeForEVT ( * DAG . getContext ( ) ) ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , VT . getStoreSize ( ) , Align ) ; if ( ! Offset -> isDivergent ( ) ) { SDValue Ops [ ] = { Rsrc , Offset , CachePolicy } ; if ( VT . isVector ( ) && VT . getVectorNumElements ( ) == <NUM_LIT> ) { EVT WidenedVT = EVT :: getVectorVT ( * DAG . getContext ( ) , VT . getVectorElementType ( ) , <NUM_LIT> ) ; auto WidenedOp = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( WidenedVT ) , Ops , WidenedVT , MF . getMachineMemOperand ( MMO , <NUM_LIT> , WidenedVT . getStoreSize ( ) ) ) ; auto Subvector = DAG . getNode ( <STR_LIT> , DL , VT , WidenedOp , DAG . getVectorIdxConstant ( <NUM_LIT> , DL ) ) ; return Subvector ; } return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( VT ) , Ops , VT , MMO ) ; } SmallVector < SDValue , <NUM_LIT> > Loads ; unsigned NumLoads = <NUM_LIT> ; MVT LoadVT = VT . getSimpleVT ( ) ; unsigned NumElts = LoadVT . isVector ( ) ? LoadVT . getVectorNumElements ( ) : <NUM_LIT> ; assert ( ( LoadVT . getScalarType ( ) == <STR_LIT> || LoadVT . getScalarType ( ) == <STR_LIT> ) ) ; if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) { NumLoads = NumElts / <NUM_LIT> ; LoadVT = <STR_LIT> ( LoadVT . getScalarType ( ) , <NUM_LIT> ) ; } SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ; SDValue Ops [


<s> unsigned EltSize = EltVT . getSizeInBits ( ) ; if ( VecSize <= <NUM_LIT> && ( VecSize > <NUM_LIT> || EltSize >= <NUM_LIT> ) && ! isa < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ) { SDLoc SL ( N ) ; SDValue Idx = N -> getOperand ( <NUM_LIT> ) ; SDValue V ; for ( unsigned I = <NUM_LIT> , E = VecVT . getVectorNumElements ( ) ; I < E ; ++ I ) { SDValue IC = DAG . getVectorIdxConstant ( I , SL ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec , IC ) ; if ( I == <NUM_LIT> ) V = Elt ; else V = DAG . getSelectCC ( SL , Idx , IC , Elt , V , <STR_LIT> ) ; } return V ; } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ; unsigned LeftoverBitIdx = BitIndex % <NUM_LIT> ; SDLoc SL ( N ) ; SDValue Cast = DAG . getNode ( <STR_LIT> , SL , NewVT , Vec ) ; DCI . AddToWorklist ( Cast . getNode ( ) ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cast , DAG . getConstant ( EltIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Elt . getNode ( ) ) ; SDValue Srl = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Elt , DAG . getConstant ( LeftoverBitIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Srl . getNode ( ) ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , SL , EltVT . changeTypeToInteger ( ) , Srl ) ; DCI . AddToWorklist ( Trunc . getNode ( ) ) ; return DAG . getNode ( <STR_LIT> , SL , EltVT , Trunc ) ; } return SDValue


<s> for ( auto U : V -> users ( ) ) { if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( U ) ) { if ( V == U -> getOperand ( <NUM_LIT> ) ) { switch ( Intrinsic -> getIntrinsicID ( ) ) { default : Result = false ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result = true ; break ; } } if ( V == U -> getOperand (


<s> unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS ==


<s> Opc = ( Opc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; } else { Args [ <NUM_LIT> ] = LHS ; Args [ <NUM_LIT> ] = Zero ; Opc = ( Opc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; } return DAG . getNode ( Opc , SL , VTList , Args ) ; } } if ( LHS . getOpcode ( ) == <STR_LIT> ) { auto C = dyn_cast < ConstantSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) ; if ( ! C || ! C -> isNullValue ( ) ) return SDValue ( ) ; SDValue Args [ ] = { LHS . getOperand ( <NUM_LIT> ) , RHS , LHS . getOperand ( <NUM_LIT> ) } ; return DAG . getNode ( <STR_LIT> , SDLoc ( N ) ,


<s> MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , VT . getStoreSize ( ) , Align ) ; if ( ! Offset -> isDivergent ( ) ) { SDValue Ops [ ] = { Rsrc , Offset , GLC , DLC , } ; if ( VT . isVector ( ) && VT . getVectorNumElements ( ) == <NUM_LIT> ) { EVT WidenedVT = EVT :: getVectorVT ( * DAG . getContext ( ) , VT . getVectorElementType ( ) , <NUM_LIT> ) ; auto WidenedOp = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( WidenedVT ) , Ops , WidenedVT , MF . getMachineMemOperand ( MMO , <NUM_LIT> , WidenedVT . getStoreSize ( ) ) ) ; auto Subvector = DAG . getNode ( <STR_LIT> , DL , VT , WidenedOp , DAG . getVectorIdxConstant ( <NUM_LIT> , DL ) ) ; return Subvector ; } return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( VT ) , Ops , VT , MMO ) ; } SmallVector < SDValue , <NUM_LIT> > Loads ; unsigned NumLoads = <NUM_LIT> ; MVT LoadVT = VT . getSimpleVT ( ) ; unsigned NumElts = LoadVT . isVector ( ) ? LoadVT . getVectorNumElements ( ) : <NUM_LIT> ; assert ( ( LoadVT . getScalarType ( ) == <STR_LIT> || LoadVT . getScalarType ( ) == <STR_LIT> ) ) ; if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) { NumLoads = NumElts / <NUM_LIT> ; LoadVT = <STR_LIT> ; } SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ; unsigned CachePolicy = cast < ConstantSDNode > ( GLC ) -> getZExtValue ( ) ; SDValue Ops [ ] = { DAG . getEntryNode ( ) , Rsrc , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , { } , { } , { } , DAG . getTargetConstant

( CachePolicy , DL , <STR_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ;


<s> EVT WidenedVT = EVT :: getVectorVT ( * DAG . getContext ( ) , VT . getVectorElementType ( ) , <NUM_LIT> ) ; auto WidenedOp = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( WidenedVT ) , Ops , WidenedVT , MF . getMachineMemOperand ( MMO , <NUM_LIT> , WidenedVT . getStoreSize ( ) ) ) ; auto Subvector = DAG . getNode ( <STR_LIT> , DL , VT , WidenedOp , DAG . getVectorIdxConstant ( <NUM_LIT> , DL ) ) ; return Subvector ; } return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( VT ) , Ops , VT , MMO ) ; } SmallVector < SDValue , <NUM_LIT> > Loads ; unsigned NumLoads = <NUM_LIT> ; MVT LoadVT = VT . getSimpleVT ( ) ; unsigned NumElts = LoadVT . isVector ( ) ? LoadVT . getVectorNumElements ( ) : <NUM_LIT> ; assert ( ( LoadVT . getScalarType ( ) == <STR_LIT> || LoadVT . getScalarType ( ) == <STR_LIT> ) ) ; if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) { NumLoads = NumElts /


<s> return BB ; case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( MFI -> LDSSize ) ; MI . eraseFromParent ( ) ; return BB ; } case <STR_LIT> : return splitKillBlock ( MI , BB ) ; default : return <ISA_LIT>TargetLowering :: EmitInstrWithCustomInserter ( MI , BB ) ; } return BB


<s> case <STR_LIT> : return BB ; case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( MFI -> LDSSize ) ; MI . eraseFromParent ( ) ; return BB ; } case <STR_LIT>


<s> EVT PtrVT = Op . getValueType ( ) ; if ( ! shouldEmitGOTReloc ( GV , getTargetMachine ( ) ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( PtrTy ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy


<s> SDValue Glue = Chain . getValue ( <NUM_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Chain , Op . getOperand ( <NUM_LIT> ) , Glue ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Chain , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getOperand ( <NUM_LIT> ) . getValueType ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOStore , VT . getStoreSize ( ) , <NUM_LIT> ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; } case <STR_LIT>