<s> bool Unsafe = DAG . getTarget ( ) . Options . UnsafeFPMath ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( ( Unsafe || ( VT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) {

if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ;


<s> } unsigned NumElements = MemVT . getVectorNumElements ( ) ; switch ( AS ) { case <ISA_LIT>AS :: CONSTANT_ADDRESS : if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; LLVM_FALLTHROUGH ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : case <ISA_LIT>AS :: FLAT_ADDRESS : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <ISA_LIT>AS :: PRIVATE_ADDRESS : { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } case <ISA_LIT>AS :: LOCAL_ADDRESS : { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> ) return SDValue ( ) ; return SplitVectorLoad ( Op , DAG ) ; } default :


<s> SDValue BasePtr = Load -> getBasePtr ( ) ; MachineMemOperand * MMO = Load -> getMemOperand ( ) ; SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , <STR_LIT> , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } unsigned NumElements = MemVT . getVectorNumElements ( ) ; switch ( AS ) { case <ISA_LIT>AS :: CONSTANT_ADDRESS : if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; LLVM_FALLTHROUGH ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : case <ISA_LIT>AS :: FLAT_ADDRESS : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <ISA_LIT>AS :: PRIVATE_ADDRESS : { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } case <ISA_LIT>AS :: LOCAL_ADDRESS : { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> ) return SDValue ( ) ; return SplitVectorLoad ( Op , DAG ) ; } default :


<s> return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

return LowerGlobalAddress ( MFI , Op , DAG ) ;


<s> Type * Ty = MemVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; SDValue Ptr = LowerParameterPtr ( DAG , SL , Chain , Offset ) ; SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Align , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; SDValue Val = Load ; if ( Arg && ( Arg -> Flags . isSExt ( ) || Arg -> Flags . isZExt ( ) ) && VT . bitsLT ( MemVT ) ) { unsigned Opc = Arg -> Flags . isZExt ( ) ? <STR_LIT> : <STR_LIT> ; Val = DAG . getNode ( Opc , SL , MemVT , Val , DAG . getValueType ( VT ) ) ; } if ( MemVT . isFloatingPoint ( ) ) Val = getFPExtOrFPTrunc ( DAG , Val , SL , VT ) ; else if ( Signed ) Val = DAG . getSExtOrTrunc ( Val , SL , VT ) ;

else Val = DAG . getZExtOrTrunc ( Val , SL , VT ) ;


<s> MVT PtrVT = getPointerTy ( DL , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg ) , PtrVT ) ;

return DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL , PtrVT ) ) ;


<s> void <ISA_LIT>TargetLowering :: ReplaceNodeResults ( SDNode * N , SmallVectorImpl < SDValue > & Results , SelectionDAG & DAG ) const { switch ( N -> getOpcode ( ) ) { case <STR_LIT> : { if ( SDValue Res = lowerINSERT_VECTOR_ELT ( SDValue ( N , <NUM_LIT> ) , DAG ) ) Results . push_back ( Res ) ; return ; } case <STR_LIT> : { if ( SDValue Res = lowerEXTRACT_VECTOR_ELT ( SDValue ( N , <NUM_LIT> ) , DAG ) ) Results . push_back ( Res ) ; return ; } case <STR_LIT> : { unsigned IID = cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IID ) { case <STR_LIT> : { SDValue Src0 = N -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; SDValue Cvt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ;


<s> if ( IsFixed || Info . hasWorkGroupIDX ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDX ) ; if ( IsFixed || Info . hasWorkGroupIDY ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDY ) ;

if ( IsFixed || Info . hasWorkGroupIDZ ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDZ ) ;


<s> SDValue One = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; SDValue LHS = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Op . getOperand ( <NUM_LIT> ) ) ; SDValue RHS = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Op . getOperand ( <NUM_LIT> ) ) ; SDValue Lo0 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , LHS , Zero ) ; SDValue Lo1 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , RHS , Zero ) ; SDValue Lo = DAG . getSelect ( DL , <STR_LIT> , Cond , Lo0 , Lo1 ) ; SDValue Hi0 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , LHS , One ) ; SDValue Hi1 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , RHS , One ) ; SDValue Hi = DAG . getSelect ( DL , <STR_LIT> , Cond , Hi0 , Hi1 ) ;

SDValue Res = DAG . getBuildVector ( <STR_LIT> , DL , { Lo , Hi } ) ;


<s> if ( IncomingArg ) { InputReg = loadInputValue ( DAG , ArgRC , ArgVT , DL , * IncomingArg ) ; } else if ( InputID == <STR_LIT> ) { InputReg = getImplicitArgPtr ( DAG , DL ) ; } else { InputReg = DAG . getUNDEF ( ArgVT ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; if ( ! CCInfo . AllocateReg ( OutgoingArg -> getRegister ( ) ) ) report_fatal_error ( <STR_LIT> ) ; } else { unsigned SpecialArgOffset = CCInfo . AllocateStack ( ArgVT . getStoreSize ( ) , Align ( <NUM_LIT> ) ) ; SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain , InputReg , SpecialArgOffset ) ; MemOpChains . push_back ( ArgStore ) ; } } const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; LLT Ty ; std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return ; const ArgDescriptor * IncomingArgX = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgY = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgZ = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; SDValue InputReg ; SDLoc SL ; const bool NeedWorkItemIDX = ! CLI . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDY = ! CLI . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDZ = ! CLI . CB -> hasFnAttr ( <STR_LIT> ) ; if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX && NeedWorkItemIDX ) InputReg = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgX ) ; if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY && NeedWorkItemIDY ) {

SDValue Y = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgY ) ;


<s> uint64_t CRHSVal = CRHS -> getZExtValue ( ) ; if ( ( CC == <STR_LIT> || CC == <STR_LIT> ) && LHS . getOpcode ( ) == <STR_LIT> && isa < ConstantSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) && isa < ConstantSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) && LHS . getConstantOperandVal ( <NUM_LIT> ) != LHS . getConstantOperandVal ( <NUM_LIT> ) && isBoolSGPR ( LHS . getOperand ( <NUM_LIT> ) ) ) { uint64_t CT = LHS . getConstantOperandVal ( <NUM_LIT> ) ; uint64_t CF = LHS . getConstantOperandVal ( <NUM_LIT> ) ; if ( ( CF == CRHSVal && CC == <STR_LIT> ) || ( CT == CRHSVal && CC == <STR_LIT> ) ) return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS . getOperand ( <NUM_LIT> ) , DAG . getConstant ( - <NUM_LIT> , SL , <STR_LIT> ) ) ; if ( ( CF == CRHSVal && CC == <STR_LIT> ) || ( CT == CRHSVal && CC == <STR_LIT> ) ) return LHS . getOperand ( <NUM_LIT> ) ; } } if ( VT != <STR_LIT> && VT != <STR_LIT> && ( Subtarget -> has16BitInsts ( ) && VT != <STR_LIT> ) ) return SDValue ( ) ; if ( ( CC == <STR_LIT> || CC == <STR_LIT> ) && LHS . getOpcode ( ) == <STR_LIT> ) { const ConstantFPSDNode * CRHS = dyn_cast < ConstantFPSDNode > ( RHS ) ; if ( ! CRHS ) return SDValue ( ) ; const APFloat & APF = CRHS -> getValueAPF ( ) ; if ( APF . isInfinity ( ) && ! APF . isNegative ( ) ) { const unsigned IsInfMask = <STR_LIT> | <STR_LIT> ; const unsigned IsFiniteMask = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ; unsigned Mask = CC == <STR_LIT> ? IsInfMask : IsFiniteMask ;

return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS . getOperand ( <NUM_LIT> ) , DAG . getConstant ( Mask , SL , <STR_LIT> ) ) ;


<s> MachineOperand & Op = MI . getOperand ( I ) ; if ( ( OpInfo [ I ] . RegClass != llvm :: <STR_LIT> && OpInfo [ I ] . RegClass != llvm :: <STR_LIT> ) || ! Op . getReg ( ) . isVirtual ( ) || ! TRI -> isAGPR ( MRI , Op . getReg ( ) ) ) continue ; auto * Src = MRI . getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Src || ! Src -> isCopy ( ) || ! TRI -> isSGPRReg ( MRI , Src -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; auto * RC = TRI -> getRegClassForReg ( MRI , Op . getReg ( ) ) ; auto * NewRC = TRI -> getEquivalentVGPRClass ( RC ) ; MRI . setRegClass ( Op . getReg ( ) , NewRC ) ; } } return ; } int NoRetAtomicOp = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( NoRetAtomicOp != - <NUM_LIT> ) { if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { int CPolIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: cpol ) ; if ( CPolIdx != - <NUM_LIT> ) { MachineOperand & CPol = MI . getOperand ( CPolIdx ) ; CPol . setImm ( CPol . getImm ( ) & ~ <STR_LIT> :: GLC ) ; } MI . RemoveOperand ( <NUM_LIT> ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; return ; } if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) -> getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) { Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Def ) ;


<s> if ( ! ReqRetVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; } else { if ( ( ReqRetVT . getVectorNumElements ( ) % <NUM_LIT> ) == <NUM_LIT> && ReqRetVT . getVectorElementType ( ) . getSizeInBits ( ) == <NUM_LIT> ) { LegalReqRetVT = EVT :: getVectorVT ( * DAG . getContext ( ) , ReqRetVT . getVectorElementType ( ) , ReqRetVT . getVectorNumElements ( ) + <NUM_LIT> ) ; } } Data = DAG . getNode ( <STR_LIT> , DL , LegalReqRetVT , Data ) ; if ( IsTexFail ) { TexFail = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , SDValue ( Result , <NUM_LIT> ) , DAG . getConstant ( MaskPopDwords , DL , <STR_LIT> ) ) ; return DAG . getMergeValues ( { Data , TexFail , SDValue ( Result , <NUM_LIT> ) } , DL ) ; } if ( Result -> getNumValues ( ) == <NUM_LIT> ) return Data ; return DAG . getMergeValues ( { Data , SDValue ( Result , <NUM_LIT> ) } , DL ) ;


<s> if ( IsD16 ) Data = adjustLoadValueTypeImpl ( Data , ReqRetVT , DL , DAG , Unpacked ) ; EVT LegalReqRetVT = ReqRetVT ; if ( ! ReqRetVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; } else { if ( ( ReqRetVT . getVectorNumElements ( ) % <NUM_LIT> ) == <NUM_LIT> && ReqRetVT . getVectorElementType ( ) . getSizeInBits ( ) == <NUM_LIT> ) { LegalReqRetVT = EVT :: getVectorVT ( * DAG . getContext ( ) , ReqRetVT . getVectorElementType ( ) , ReqRetVT . getVectorNumElements ( ) + <NUM_LIT> ) ; } } Data = DAG . getNode ( <STR_LIT> , DL , LegalReqRetVT , Data ) ; if ( IsTexFail ) {

TexFail = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , SDValue ( Result , <NUM_LIT> ) , DAG . getConstant ( MaskPopDwords , DL , <STR_LIT> ) ) ;


<s> reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ; if ( Info -> getFrameOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ; Info -> limitOccupancy ( MF ) ; if ( ST . isWave32 ( ) && ! MF . empty ( ) ) {


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; if ( Info -> isEntryFunction ( ) ) { reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ; if ( Info -> getFrameOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ; Info -> limitOccupancy ( MF ) ; if ( ST . isWave32 ( ) && ! MF . empty ( ) ) { for ( auto & MBB : MF ) { for ( auto & MI : MBB ) {


<s> LLT Ty ; std :: tie ( Reg , RC , Ty ) = MFI . getPreloadedValue ( PVID ) ;

return CreateLiveInRegister ( DAG , RC , Reg -> getRegister ( ) , VT ) ;


<s> case <NUM_LIT> : RC = & <STR_LIT> ; break ; default : RC = TRI -> getAGPRClassForBitWidth ( BitWidth ) ; if ( ! RC ) return std :: make_pair ( <NUM_LIT> , nullptr ) ; break ; } break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ; } if ( Constraint . size ( ) > <NUM_LIT> ) { if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 'a' ) {


<s> <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; SDValue EncodingImm = DAG . getTargetConstant ( Encoding , DL , <STR_LIT> ) ; SDValue ApertureReg = SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , EncodingImm ) , <NUM_LIT> ) ; SDValue ShiftAmount = DAG . getTargetConstant ( WidthM1 + <NUM_LIT> , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , ApertureReg , ShiftAmount ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getObjectPtrOffset ( DL , QueuePtr , TypeSize :: Fixed ( StructOffset ) ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ;

return DAG . getLoad ( <STR_LIT> , DL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr , PtrInfo , commonAlignment ( Align ( <NUM_LIT> ) , StructOffset ) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> std :: pair < InstructionCost , MVT > Cost = TargetLoweringBase :: getTypeLegalizationCost ( DL , Ty ) ; auto Size = DL . getTypeSizeInBits ( Ty ) ;


<s> case <STR_LIT> : case <STR_LIT> : { if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( MRI . getType ( Reg ) , MF ) ) return true ; for ( unsigned I = <NUM_LIT> , E = MI -> getNumOperands ( ) ; I != E ; ++ I ) { if ( ! isCanonicalized ( MI -> getOperand ( I ) . getReg ( ) , MF , MaxDepth - <NUM_LIT> ) ) return false ; } return true ;


<s> auto F = MI -> getOperand ( <NUM_LIT> ) . getFPImm ( ) -> getValueAPF ( ) ; if ( F . isNaN ( ) && F . isSignaling ( ) ) return false ; return ! F . isDenormal ( ) || denormalsEnabledForType ( MRI . getType ( Reg ) , MF ) ; } if ( MaxDepth == <NUM_LIT> ) return false ; switch ( Opcode ) { case <STR_LIT> :


<s> return AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS || AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ;


<s> if ( SrcAS == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( SrcAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , SegmentNullPtr , <STR_LIT> ) ; SDValue Aperture = getSegmentAperture ( ASC -> getSrcAddressSpace ( ) , SL , DAG ) ; SDValue CvtPtr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src , Aperture ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , CvtPtr ) , FlatNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT && Src . getValueType ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; const MachineFunction & MF = DAG . getMachineFunction ( ) ; DiagnosticInfoUnsupported InvalidAddrSpaceCast ( MF . getFunction ( ) , <STR_LIT> , SL . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( InvalidAddrSpaceCast ) ;

return DAG . getUNDEF ( ASC -> getValueType ( <NUM_LIT> ) ) ;


<s> SDValue Src = ASC -> getOperand ( <NUM_LIT> ) ; SDValue FlatNullPtr = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( getTargetMachine ( ) ) ; if ( ASC -> getSrcAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) { unsigned DestAS = ASC -> getDestAddressSpace ( ) ; if ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , FlatNullPtr , <STR_LIT> ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , Ptr , SegmentNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) { unsigned SrcAS = ASC -> getSrcAddressSpace ( ) ; if ( SrcAS == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( SrcAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , SegmentNullPtr , <STR_LIT> ) ; SDValue Aperture = getSegmentAperture ( ASC -> getSrcAddressSpace ( ) , SL , DAG ) ; SDValue CvtPtr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src , Aperture ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , CvtPtr ) , FlatNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT && Src . getValueType ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; const MachineFunction & MF = DAG . getMachineFunction ( ) ;

DiagnosticInfoUnsupported InvalidAddrSpaceCast ( MF . getFunction ( ) , <STR_LIT> , SL . getDebugLoc ( ) ) ;


<s> return DAG . getNode ( <STR_LIT> , SL , VT , Src . getOperand ( <NUM_LIT> ) , Src . getOperand ( <NUM_LIT> ) , Src . getOperand ( <NUM_LIT> ) ) ; } if ( const ConstantSDNode * Arg = dyn_cast < ConstantSDNode > ( Src ) ) { if ( Arg -> isNullValue ( ) ) return DAG . getConstant ( <NUM_LIT> , SL , VT ) ; if ( Arg -> isOne ( ) ) {


<s> static SDValue lowerBALLOTIntrinsic ( const <ISA_LIT>TargetLowering & TLI , SDNode * N , SelectionDAG & DAG ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; if ( Src . getOpcode ( ) == <STR_LIT> ) { return DAG . getNode ( <STR_LIT> , SL , VT , Src . getOperand ( <NUM_LIT> ) , Src . getOperand ( <NUM_LIT> ) , Src . getOperand ( <NUM_LIT> ) ) ; } if ( const ConstantSDNode * Arg = dyn_cast < ConstantSDNode > ( Src ) ) { if ( Arg -> isNullValue ( ) ) return DAG . getConstant ( <NUM_LIT> , SL , VT ) ; if ( Arg -> isOne ( ) ) { Register Exec ;

if ( VT . getScalarSizeInBits ( ) == <NUM_LIT> ) Exec = <STR_LIT> ;


<s> const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * InputPtrReg ; const TargetRegisterClass * RC ; LLT ArgTy ; std :: tie ( InputPtrReg , RC , ArgTy ) = Info -> getPreloadedValue ( <STR_LIT> ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ;

SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg -> getRegister ( ) ) , PtrVT ) ;


<s> return lowerTrapHsaQueuePtr ( Op , DAG ) ; case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V4 : return Subtarget -> supportsGetDoorbellID ( ) ? lowerTrapHsa ( Op , DAG ) : lowerTrapHsaQueuePtr ( Op , DAG ) ; } }


<s> SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; uint64_t TrapID = static_cast < uint64_t > ( GCNSubtarget :: TrapID :: LLVMAMDHSATrap ) ; SDValue Ops [ ] = {

ToReg , DAG . getTargetConstant ( TrapID , SL , <STR_LIT> ) , SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ;


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; uint64_t TrapID = static_cast < uint64_t > ( GCNSubtarget :: TrapID :: LLVMAMDHSATrap ) ; SDValue Ops [ ] = { ToReg , DAG . getTargetConstant ( TrapID , SL , <STR_LIT> ) , SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ;

return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Ops ) ;


<s> const TargetRegisterClass * ArgRC ; LLT Ty ; std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return ; const ArgDescriptor * IncomingArgX = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgY = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgZ = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; SDValue InputReg ; SDLoc SL ; if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX ) InputReg = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgX ) ; if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY ) { SDValue Y = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgY ) ; Y = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Y , DAG . getShiftAmountConstant ( <NUM_LIT> , <STR_LIT> , SL ) ) ; InputReg = InputReg . getNode ( ) ? DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InputReg , Y ) : Y ; } if ( IncomingArgZ && ! IncomingArgZ -> isMasked ( ) && CalleeArgInfo -> WorkItemIDZ ) { SDValue Z = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgZ ) ; Z = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Z , DAG . getShiftAmountConstant ( <NUM_LIT> , <STR_LIT> , SL ) ) ;

InputReg = InputReg . getNode ( ) ? DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InputReg , Z ) : Z ;


<s> if ( Shift . getOpcode ( ) == <STR_LIT> ) ShiftOffset -= C -> getZExtValue ( ) ; else ShiftOffset += C -> getZExtValue ( ) ; if ( ShiftOffset < <NUM_LIT> && ( ShiftOffset % <NUM_LIT> ) == <NUM_LIT> ) { return DAG . getNode ( <STR_LIT> + ShiftOffset / <NUM_LIT> , SL , <STR_LIT> , Shift ) ; } } } const TargetLowering & TLI = DAG . getTargetLoweringInfo ( ) ; APInt DemandedBits = APInt :: getBitsSet ( <NUM_LIT> , <NUM_LIT> * Offset , <NUM_LIT> * Offset + <NUM_LIT> ) ; if ( TLI . SimplifyDemandedBits ( Src , DemandedBits , DCI ) ) { if ( N -> getOpcode ( ) != <STR_LIT> ) DCI . AddToWorklist ( N ) ; return SDValue ( N , <NUM_LIT> ) ; }

if ( SDValue DemandedSrc = TLI . SimplifyMultipleUseDemandedBits ( Src , DemandedBits , DAG ) ) return DAG . getNode ( N -> getOpcode ( ) , SL , <STR_LIT> , DemandedSrc ) ;


<s> const <ISA_LIT>RegisterInfo * <ISA_LIT>RI = Subtarget -> getRegisterInfo ( ) ; TargetLowering :: AsmOperandInfoVector TargetConstraints = ParseConstraints ( MF . getDataLayout ( ) , Subtarget -> getRegisterInfo ( ) , * CI ) ; for ( auto & TC : TargetConstraints ) { if ( TC . Type == InlineAsm :: isOutput ) { ComputeConstraintToUse ( TC , SDValue ( ) ) ; unsigned AssignedReg ; const TargetRegisterClass * RC ; std :: tie ( AssignedReg , RC ) = getRegForInlineAsmConstraint ( <ISA_LIT>RI , TC . ConstraintCode , TC . ConstraintVT ) ; if ( RC ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; if ( AssignedReg != <NUM_LIT> && <ISA_LIT>RI -> isSGPRReg ( MRI , AssignedReg ) ) return true ;

else if ( <ISA_LIT>RI -> isSGPRClass ( RC ) ) return true ;


<s> if ( ! Ty -> isFloatTy ( ) && ( ! Subtarget -> hasGFX90AInsts ( ) || ! Ty -> isDoubleTy ( ) ) ) return AtomicExpansionKind :: CmpXChg ; unsigned AS = RMW -> getPointerAddressSpace ( ) ; if ( ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) && Subtarget -> hasAtomicFaddInsts ( ) ) { if ( RMW -> getFunction ( ) -> getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) != <STR_LIT> ) return AtomicExpansionKind :: CmpXChg ; if ( Subtarget -> hasGFX90AInsts ( ) ) { if ( Ty -> isFloatTy ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return AtomicExpansionKind :: CmpXChg ; auto S<ISA_LIT>D = RMW -> getSyncScopeID ( ) ; if ( S<ISA_LIT>D == SyncScope :: System || S<ISA_LIT>D == RMW -> getContext ( ) . getOrInsertSyncScopeID ( <STR_LIT> ) ) return AtomicExpansionKind :: CmpXChg ;


<s> EVT VT = Op . getValueType ( ) ; assert ( VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> ) ; SDValue Lo0 , Hi0 ; std :: tie ( Lo0 , Hi0 ) = DAG . SplitVectorOperand ( Op . getNode ( ) , <NUM_LIT> ) ; SDValue Lo1 , Hi1 ; std :: tie ( Lo1 , Hi1 ) = DAG . SplitVectorOperand ( Op . getNode ( ) , <NUM_LIT> ) ;


<s> std :: tie ( Lo1 , Hi1 ) = DAG . SplitVectorOperand ( Op . getNode ( ) , <NUM_LIT> ) ; SDValue Lo2 , Hi2 ; std :: tie ( Lo2 , Hi2 ) = DAG . SplitVectorOperand ( Op . getNode ( ) , <NUM_LIT> ) ; SDLoc SL ( Op ) ;

SDValue OpLo = DAG . getNode ( Opc , SL , Lo0 . getValueType ( ) , Lo0 , Lo1 , Lo2 , Op -> getFlags ( ) ) ;


<s> MMO -> setValue ( ( Value * ) nullptr ) ; return ; } MMO -> setOffset ( cast < ConstantSDNode > ( VOffset ) -> getSExtValue ( ) + cast < ConstantSDNode > ( SOffset ) -> getSExtValue ( ) + cast < ConstantSDNode > ( Offset ) -> getSExtValue ( ) ) ;


<s> MMO -> setValue ( ( Value * ) nullptr ) ; return ; } if ( VIndex && ( ! isa < ConstantSDNode > ( VIndex ) || ! cast < ConstantSDNode > ( VIndex ) -> isNullValue ( ) ) ) { MMO -> setValue ( ( Value * ) nullptr ) ; return ; } MMO -> setOffset ( cast < ConstantSDNode > ( VOffset ) -> getSExtValue ( ) + cast < ConstantSDNode > ( SOffset ) -> getSExtValue ( ) + cast < ConstantSDNode > ( Offset ) -> getSExtValue ( ) ) ;


<s> if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) { if ( Subtarget -> useDS128 ( ) && Store -> getAlignment ( ) >= <NUM_LIT> && VT . getStoreSize ( ) == <NUM_LIT> && NumElements != <NUM_LIT> ) return SDValue ( ) ; if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && NumElements == <NUM_LIT> && VT . getStoreSize ( ) == <NUM_LIT> && Store -> getAlignment ( ) < <NUM_LIT> ) {


<s> auto CachePolicyConst = cast < ConstantSDNode > ( CachePolicy . getNode ( ) ) ; uint64_t Value = CachePolicyConst -> getZExtValue ( ) ; SDLoc DL ( CachePolicy ) ; if ( GLC ) { * GLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ; Value &= ~ ( uint64_t ) <NUM_LIT> ; } if ( SLC ) { * SLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ; Value &= ~ ( uint64_t ) <NUM_LIT> ; }


<s> if ( GLC ) { * GLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ; Value &= ~ ( uint64_t ) <NUM_LIT> ; } if ( SLC ) {

* SLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ;


<s> } else { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } } if ( DataDwordVT . isVector ( ) ) Data = padEltsToUndef ( DAG , DL , DataDwordVT , Data , NumDataDwords - MaskPopDwords ) ; if ( IsD16 ) Data = adjustLoadValueTypeImpl ( Data , ReqRetVT , DL , DAG , Unpacked ) ; EVT LegalReqRetVT = ReqRetVT ; if ( ! ReqRetVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; } else { if ( ( ReqRetVT . getVectorNumElements ( ) % <NUM_LIT> ) == <NUM_LIT> && ReqRetVT . getVectorElementType ( ) . getSizeInBits ( ) == <NUM_LIT> ) {

LegalReqRetVT = EVT :: getVectorVT ( * DAG . getContext ( ) , ReqRetVT . getVectorElementType ( ) , ReqRetVT . getVectorNumElements ( ) + <NUM_LIT> ) ;


<s> } if ( ImageStore && Subtarget -> hasImageStoreD16Bug ( ) ) { EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; SmallVector < SDValue , <NUM_LIT> > Elts ; DAG . ExtractVectorElements ( IntVData , Elts ) ; SmallVector < SDValue , <NUM_LIT> > PackedElts ; for ( unsigned I = <NUM_LIT> ; I < Elts . size ( ) / <NUM_LIT> ; I += <NUM_LIT> ) { SDValue Pair = DAG . getBuildVector ( <STR_LIT> , DL , { Elts [ I * <NUM_LIT> ] , Elts [ I * <NUM_LIT> + <NUM_LIT> ] } ) ; SDValue IntPair = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Pair ) ; PackedElts . push_back ( IntPair ) ; } PackedElts . resize ( PackedElts . size ( ) * <NUM_LIT> , DAG . getUNDEF ( <STR_LIT> ) ) ;

EVT VecVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , PackedElts . size ( ) ) ;


<s> return DAG . getMergeValues ( Ops , DL ) ; } SmallVector < SDValue , <NUM_LIT> > Elts ; for ( unsigned I = <NUM_LIT> , N = MemVT . getVectorNumElements ( ) ; I != N ; ++ I ) { SDValue Elt = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , NewLD , DAG . getConstant ( I , DL , <STR_LIT> ) ) ; Elts . push_back ( DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Elt ) ) ; } SDValue Ops [ ] = { DAG . getBuildVector ( MemVT , DL , Elts ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned Alignment = Load -> getAlignment ( ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Alignment < MemVT . getStoreSize ( ) && MemVT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorLoad ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasMultiDwordFlatScratchAddressing ( ) ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && Load -> isSimple ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ;

if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ;


<s> if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; if ( ! allowsMemoryAccessForAlignment ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) return expandUnalignedStore ( Store , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( Subtarget -> hasDS96AndDS128 ( ) && ( ( Subtarget -> useDS128 ( ) && VT . getStoreSize ( ) == <NUM_LIT> ) || ( VT . getStoreSize ( ) == <NUM_LIT> ) ) && allowsMisalignedMemoryAccessesImpl ( VT . getSizeInBits ( ) , AS , Store -> getAlign ( ) ) ) return SDValue ( ) ; if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( ! Subtarget -> hasUsableDSOffset ( ) && NumElements == <NUM_LIT> && VT . getStoreSize ( ) == <NUM_LIT> && Store -> getAlignment ( ) < <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } if ( ! allowsMemoryAccessForAlignment ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) { if ( VT . isVector ( ) ) return SplitVectorStore ( Op , DAG ) ; return expandUnalignedStore ( Store , DAG ) ;


<s> Arg = allocateVGPR32Input ( CCInfo , Mask ) ; Info . setWorkItemIDX ( Arg ) ; } if ( Info . hasWorkItemIDY ( ) ) { Arg = allocateVGPR32Input ( CCInfo , Mask << <NUM_LIT> , Arg ) ;


<s> BuildMI ( * LoopBB , LoopBB -> begin ( ) , DL , TII -> get ( <STR_LIT> ) , Src -> getReg ( ) ) . addReg ( Data0 ) ; MRI . setSimpleHint ( Data0 , Src -> getReg ( ) ) ; } BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( EncodedReg ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopBB ) ;


<s> MRI . setSimpleHint ( Data0 , Src -> getReg ( ) ) ; } BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( EncodedReg ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopBB ) ;


<s> break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; } break ; case 'a' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; return std :: make_pair ( <NUM_LIT> , RC ) ; } break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ; } if ( Constraint . size ( ) > <NUM_LIT> ) { if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 'a' ) { RC = & <STR_LIT> ; } if ( RC ) { uint32_t Idx ;

bool Failed = Constraint . substr ( <NUM_LIT> ) . getAsInteger ( <NUM_LIT> , Idx ) ;


<s> for ( unsigned I = <NUM_LIT> , E = Ins . size ( ) , P<ISA_LIT>nputNum = <NUM_LIT> ; I != E ; ++ I ) { const <STR_LIT> * Arg = & Ins [ I ] ; assert ( ( ! Arg -> VT . isVector ( ) || Arg -> VT . getScalarSizeInBits ( ) == <NUM_LIT> ) && <STR_LIT> ) ; if ( CallConv == CallingConv :: <ISA_LIT>_PS && ! Arg -> Flags . isInReg ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { bool SkipArg = ! Arg -> Used && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( Arg -> Flags . isSplit ( ) ) { while ( ! Arg -> Flags . isSplitEnd ( ) ) { assert ( ! Arg -> VT . isVector ( ) && <STR_LIT> ) ; if ( ! SkipArg ) Splits . push_back ( * Arg ) ; Arg = & Ins [ ++ I ] ; } } if ( SkipArg ) { Skipped . set ( Arg -> getOrigArgIndex ( ) ) ; ++ P<ISA_LIT>nputNum ;


<s> switch ( Opc ) { case <STR_LIT> : case <STR_LIT> : return false ; default : return true ; }


<s> return false ; default : return true ; }


<s> return CreateLiveInRegister ( DAG , & <STR_LIT> , RegNo , <STR_LIT> ) ; } SDLoc SL ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ;

SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , QueuePtr , DAG . getConstant ( StructOffset , SL , <STR_LIT> ) ) ;


<s> <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , QueuePtr , DAG . getConstant ( StructOffset , SL , <STR_LIT> ) ) ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( * DAG . getContext ( ) ) , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ;

return DAG . getLoad ( <STR_LIT> , SL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr , PtrInfo , MinAlign ( <NUM_LIT> , StructOffset ) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , Ptr , SegmentNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { unsigned SrcAS = ASC -> getSrcAddressSpace ( ) ; if ( SrcAS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || SrcAS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( SrcAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , SegmentNullPtr , <STR_LIT> ) ; SDValue Aperture = getSegmentAperture ( ASC -> getSrcAddressSpace ( ) , DAG ) ; SDValue CvtPtr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src , Aperture ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , CvtPtr ) , FlatNullPtr ) ; } } const MachineFunction & MF = DAG . getMachineFunction ( ) ; DiagnosticInfoUnsupported InvalidAddrSpaceCast ( * MF . getFunction ( ) , <STR_LIT> , SL . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( InvalidAddrSpaceCast ) ; return DAG . getUNDEF ( ASC -> getValueType ( <NUM_LIT> ) ) ;


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; SDLoc DL ( Op ) ; SDValue Chain = Op . getOperand ( <NUM_LIT> ) ; unsigned IntrinsicID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : { const ConstantSDNode * Tgt = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * En = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * Done = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * VM = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const SDValue Ops [ ] = { Chain , DAG . getTargetConstant ( Tgt -> getZExtValue ( ) , DL , <STR_LIT> ) , DAG . getTargetConstant ( En -> getZExtValue ( ) , DL , <STR_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getTargetConstant ( VM -> getZExtValue ( ) , DL , <STR_LIT> ) } ; unsigned Opc = Done -> isNullValue ( ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( Opc , DL , Op -> getVTList ( ) , Ops ) ; } case <STR_LIT> : { const ConstantSDNode * Tgt = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * En = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ;

SDValue Src0 = Op . getOperand ( <NUM_LIT> ) ;


<s> } if ( TII -> isMIMG ( * MI ) ) { unsigned VReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned DmaskIdx = MI -> getNumOperands ( ) == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; unsigned Writemask = MI -> getOperand ( DmaskIdx ) . getImm ( ) ; unsigned BitsSet = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) BitsSet += Writemask & ( <NUM_LIT> << i ) ? <NUM_LIT> : <NUM_LIT> ; const TargetRegisterClass * RC ; switch ( BitsSet ) { default : return ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; } unsigned NewOpcode = TII -> getMaskedMIMGOp ( MI -> getOpcode ( ) , BitsSet ) ; MI -> setDesc ( TII -> get ( NewOpcode ) ) ; MRI . setRegClass ( VReg , RC ) ; return ;


<s> case <STR_LIT> : return BB ; case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI -> getOperand ( <NUM_LIT> ) ) . addImm ( MFI -> LDSSize ) ; MI -> eraseFromParent ( ) ; return BB ; }


<s> MachineBasicBlock * <ISA_LIT>TargetLowering :: EmitInstrWithCustomInserter ( MachineInstr * MI , MachineBasicBlock * BB ) const { switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; BuildMI ( * BB , MI -> getIterator ( ) , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addOperand ( MI -> getOperand ( <NUM_LIT> ) ) ; MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : return BB ; case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; MachineFunction * MF = BB -> getParent ( ) ;

<ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> static SDValue emitNonHSAIntrinsicError ( SelectionDAG & DAG , EVT VT ) { DiagnosticInfoUnsupported BadIntrin ( * DAG . getMachineFunction ( ) . getFunction ( ) , <STR_LIT> ) ; DAG . getContext ( ) -> diagnose ( BadIntrin ) ; return DAG . getUNDEF ( VT ) ;


<s> static SDValue emitNonHSAIntrinsicError ( SelectionDAG & DAG , EVT VT ) { DiagnosticInfoUnsupported BadIntrin ( * DAG . getMachineFunction ( ) . getFunction ( ) , <STR_LIT> ) ; DAG . getContext ( ) -> diagnose ( BadIntrin ) ;


<s> if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } case <ISA_LIT>AS :: PRIVATE_ADDRESS : return isLegalMUBUFAddressingMode ( AM ) ; case <ISA_LIT>AS :: LOCAL_ADDRESS : case <ISA_LIT>AS :: REGION_ADDRESS : { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ;


<s> if ( GSD -> getAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; const GlobalValue * GV = GSD -> getGlobal ( ) ; EVT PtrVT = Op . getValueType ( ) ; SDValue GA = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , GSD -> getOffset ( ) + <NUM_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , PtrVT , GA ) ;


<s> SDLoc DL ( GSD ) ; const GlobalValue * GV = GSD -> getGlobal ( ) ; EVT PtrVT = Op . getValueType ( ) ; SDValue GA = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , GSD -> getOffset ( ) + <NUM_LIT> ) ;

return DAG . getNode ( <STR_LIT> , DL , PtrVT , GA ) ;


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( Subtarget -> getRegisterInfo ( ) ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; Type * Ty = VT . getTypeForEVT ( * DAG . getContext ( ) ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg ) , PtrVT ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL , PtrVT ) ) ; SDValue PtrOffset = DAG . getUNDEF ( PtrVT ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ;

unsigned Align = DL . getABITypeAlignment ( Ty ) ;


<s> SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL , PtrVT ) ) ; SDValue PtrOffset = DAG . getUNDEF ( PtrVT ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; <STR_LIT> ExtTy = Signed ? <STR_LIT> : <STR_LIT> ; if ( MemVT . isFloatingPoint ( ) ) ExtTy = <STR_LIT> ;

return DAG . getLoad ( <STR_LIT> , ExtTy , VT , SL , Chain , Ptr , PtrOffset , PtrInfo , MemVT , false , true , true , Align ) ;


<s> DAG . getContext ( ) -> diagnose ( NoTrap ) ;

return DAG . getNode ( <STR_LIT> , SDLoc ( Op ) , <STR_LIT> , Op . getOperand ( <NUM_LIT> ) ) ;


<s> DiagnosticInfoUnsupported NoTrap ( * MF . getFunction ( ) , <STR_LIT> , Op . getDebugLoc ( ) , DS_Warning ) ;

DAG . getContext ( ) -> diagnose ( NoTrap ) ;


<s> } assert ( DCI . isBeforeLegalize ( ) && <STR_LIT> ) ; unsigned NElts = SrcVT . getVectorNumElements ( ) ; if ( ! SrcVT . isSimple ( ) && NElts != <NUM_LIT> ) return SDValue ( ) ; if ( <STR_LIT> ( Src . getNode ( ) ) && Src . hasOneUse ( ) ) { EVT LoadVT = getEquivalentMemType ( * DAG . getContext ( ) , SrcVT ) ; EVT RegVT = getEquivalentLoadRegType ( * DAG . getContext ( ) , SrcVT ) ; EVT FloatVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , NElts ) ; LoadSDNode * Load = cast < LoadSDNode > ( Src ) ; unsigned AS = Load -> getAddressSpace ( ) ; unsigned Align = Load -> getAlignment ( ) ; Type * Ty = LoadVT . getTypeForEVT ( * DAG . getContext ( ) ) ; unsigned ABIAlignment = DAG . getDataLayout ( ) . getABITypeAlignment ( Ty ) ; if ( Align < ABIAlignment && ! allowsMisalignedMemoryAccesses ( LoadVT , AS , Align , nullptr ) ) { return SDValue ( ) ; } SDValue NewLoad = DAG . getExtLoad ( <STR_LIT> , DL , RegVT , Load -> getChain ( ) , Load -> getBasePtr ( ) , LoadVT , Load -> getMemOperand ( ) ) ; DAG . ReplaceAllUsesOfValueWith ( SDValue ( Load , <NUM_LIT> ) , NewLoad . getValue ( <NUM_LIT> ) ) ; SmallVector < SDValue , <NUM_LIT> > Elts ; if ( RegVT . isVector ( ) ) DAG . ExtractVectorElements ( NewLoad , Elts ) ; else Elts . push_back ( NewLoad ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; unsigned EltIdx = <NUM_LIT> ; for ( SDValue Elt : Elts ) { unsigned ComponentsInElt = std :: min ( <NUM_LIT> , NElts - <NUM_LIT> * EltIdx ) ; for ( unsigned I = <NUM_LIT> ; I < ComponentsInElt ; ++ I ) { unsigned Opc = <STR_LIT> + I ; SDValue Cvt = DAG . getNode ( Opc , DL , <STR_LIT> , Elt ) ; DCI . AddToWorklist ( Cvt . getNode ( ) ) ;


<s> unsigned NElts = SrcVT . getVectorNumElements ( ) ; if ( ! SrcVT . isSimple ( ) && NElts != <NUM_LIT> ) return SDValue ( ) ; if ( <STR_LIT> ( Src . getNode ( ) ) && Src . hasOneUse ( ) ) { EVT LoadVT = getEquivalentMemType ( * DAG . getContext ( ) , SrcVT ) ; EVT RegVT = getEquivalentLoadRegType ( * DAG . getContext ( ) , SrcVT ) ; EVT FloatVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , NElts ) ; LoadSDNode * Load = cast < LoadSDNode > ( Src ) ; unsigned AS = Load -> getAddressSpace ( ) ; unsigned Align = Load -> getAlignment ( ) ; Type * Ty = LoadVT . getTypeForEVT ( * DAG . getContext ( ) ) ; unsigned ABIAlignment = DAG . getDataLayout ( ) . getABITypeAlignment ( Ty ) ; if ( Align < ABIAlignment && ! allowsMisalignedMemoryAccesses ( LoadVT , AS , Align , nullptr ) ) { return SDValue ( ) ; }

SDValue NewLoad = DAG . getExtLoad ( <STR_LIT> , DL , RegVT , Load -> getChain ( ) , Load -> getBasePtr ( ) , LoadVT , Load -> getMemOperand ( ) ) ;


<s> unsigned Opcode = Node -> getMachineOpcode ( ) ; if ( TII -> isMIMG ( Opcode ) && ! TII -> get ( Opcode ) . mayStore ( ) ) adjustWritemask ( Node , DAG ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { legalizeTargetIndependentNode ( Node , DAG ) ; return Node ; }


<s> MachineSDNode * <ISA_LIT>TargetLowering :: wrapAddr64Rsrc ( SelectionDAG & DAG , const SDLoc & DL , SDValue Ptr ) const { const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; const SDValue Ops0 [ ] = {

DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , buildSMovImm32 ( DAG , DL , <NUM_LIT> ) , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , buildSMovImm32 ( DAG , DL , TII -> getDefaultRsrcDataFormat ( ) >> <NUM_LIT> ) , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) } ;


<s> Info . memVT = <STR_LIT> ( CI . getType ( ) , true ) ; if ( Info . memVT == <STR_LIT> ) { Info . memVT = memVTFromAggregate ( CI . getType ( ) ) ; } Info . flags |= MachineMemOperand :: MOLoad ; } else if ( Attr . hasFnAttribute ( Attribute :: WriteOnly ) ) { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . flags |= MachineMemOperand :: MOStore ; } else { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; } return true ; } switch ( IntrID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : {


<s> if ( Unpacked ) { EVT IntLoadVT = LoadVT . changeTypeToInteger ( ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , DL , IntLoadVT , Result ) ; return DAG . getNode ( <STR_LIT> , DL , LoadVT , Trunc ) ;


<s> EVT IntLoadVT = LoadVT . changeTypeToInteger ( ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , DL , IntLoadVT , Result ) ; return DAG . getNode ( <STR_LIT> , DL , LoadVT , Trunc ) ; }

return DAG . getNode ( <STR_LIT> , DL , LoadVT , Result ) ;


<s> if ( const <STR_LIT> * RsrcIntr = <STR_LIT> ( IntrID ) ) { AttributeList Attr = <STR_LIT> ( CI . getContext ( ) , ( <STR_LIT> ) IntrID ) ; if ( Attr . hasFnAttribute ( Attribute :: ReadNone ) ) return false ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RsrcIntr -> IsImage ) { Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( RsrcIntr -> RsrcArg ) ) ; Info . align = <NUM_LIT> ; } else { Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( RsrcIntr -> RsrcArg ) ) ; } Info . flags = MachineMemOperand :: MODereferenceable ; if ( Attr . hasFnAttribute ( Attribute :: ReadOnly ) ) { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ;

Info . flags |= MachineMemOperand :: MOLoad ;


<s> } else { EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = ( StoreVT == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ;

return DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ;


<s> if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { return isLegalMUBUFAddressingMode ( AM ) ; } else if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . REGION_ADDRESS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ;


<s> if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ;

if ( AM . Scale == <NUM_LIT> ) return true ;


<s> switch ( IID ) { case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; Res = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; Res = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; unsigned Opc = getImageOpcode ( IID ) ; Res = DAG . getMemIntrinsicNode ( Opc , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> SDVTList VTList = DAG . getVTList ( EquivLoadVT , <STR_LIT> ) ; SDValue Res ; SDLoc DL ( Op ) ; MemSDNode * M = cast < MemSDNode > ( Op ) ; unsigned IID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IID ) { case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; Res = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; Res = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; unsigned Opc = getImageOpcode ( IID ) ; Res = DAG . getMemIntrinsicNode ( Opc , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> :


<s> SDValue <ISA_LIT>TargetLowering :: lowerKernargMemParameter ( SelectionDAG & DAG , EVT VT , EVT MemVT , const SDLoc & SL , SDValue Chain , uint64_t Offset , bool Signed , const <STR_LIT> * Arg ) const { const DataLayout & DL = DAG . getDataLayout ( ) ; Type * Ty = MemVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ;

MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ;


<s> PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , Offset ) ;

SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Align , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> EVT VT = Store -> getMemoryVT ( ) ; if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ;


<s> case <STR_LIT> : return performSubCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : return performAddCarrySubCarryCombine ( N , DCI ) ; case <STR_LIT> : return performFAddCombine ( N , DCI ) ; case <STR_LIT> : return performFSubCombine ( N , DCI ) ; case <STR_LIT> : return performSetCCCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( DCI . getDAGCombineLevel ( ) >= AfterLegalizeDAG && getTargetMachine ( ) . getOptLevel ( ) > CodeGenOpt :: None ) return performMinMaxCombine ( N , DCI ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> : return performFMed3Combine ( N , DCI ) ; case <STR_LIT> : return performCvtPkRTZCombine ( N , DCI ) ; case <STR_LIT> : { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> || VT == <STR_LIT> ) { SDLoc SL ( N ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ; EVT EltVT = Src . getValueType ( ) ; if ( EltVT == <STR_LIT> ) Src = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; SDValue Ext = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , VT , Ext ) ;


<s> SDValue Src1 = Node -> getOperand ( <NUM_LIT> ) ; SDValue Src2 = Node -> getOperand ( <NUM_LIT> ) ; if ( ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) != <STR_LIT> ) && ( Src0 == Src1 || Src0 == Src2 ) ) break ; MVT VT = Src0 . getValueType ( ) . getSimpleVT ( ) ; const TargetRegisterClass * RC = getRegClassFor ( VT ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; SDValue UndefReg = DAG . getRegister ( MRI . createVirtualRegister ( RC ) , VT ) ; SDValue ImpDef = DAG . getCopyToReg ( DAG . getEntryNode ( ) , SDLoc ( Node ) , UndefReg , Src0 , SDValue ( ) ) ; if ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) == <STR_LIT> ) { if ( Src1 . isMachineOpcode ( ) && Src1 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src1 ; else if ( Src2 . isMachineOpcode ( ) && Src2 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src2 ; else { assert ( Src1 . getMachineOpcode ( ) == <STR_LIT> ) ; Src0 = UndefReg ; Src1 = UndefReg ; } } else break ; SmallVector < SDValue , <NUM_LIT> > Ops = { Src0 , Src1 , Src2 } ; for ( unsigned I = <NUM_LIT> , N = Node -> getNumOperands ( ) ; I != N ; ++ I ) Ops . push_back ( Node -> getOperand ( I ) ) ; Ops . push_back ( ImpDef . getValue ( <NUM_LIT> ) ) ; return DAG . getMachineNode ( Opcode , SDLoc ( Node ) , Node -> getVTList ( ) , Ops ) ; } default :


<s> return ; } case <STR_LIT> : { if ( SDValue Res = lowerEXTRACT_VECTOR_ELT ( SDValue ( N , <NUM_LIT> ) , DAG ) ) Results . push_back ( Res ) ; return ; } case <STR_LIT> : { unsigned IID = cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IID ) { case <STR_LIT> : { SDValue Src0 = N -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; SDValue Cvt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src0 = N -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; unsigned Opcode ; if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else Opcode = <STR_LIT> ; SDValue Cvt = DAG . getNode ( Opcode , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } } break ; } case <STR_LIT> : { SDValue Chain ; if ( SDValue Res = lowerIntrinsicWChain_IllegalReturnType ( SDValue ( N , <NUM_LIT> ) , Chain , DAG ) ) { Results . push_back ( Res ) ; Results . push_back ( Chain ) ; return ; } break ; } case <STR_LIT> : { SDLoc SL ( N ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ;

SDValue LHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ;


<s> bool <ISA_LIT>TargetLowering :: allowsMisalignedMemoryAccessesImpl ( unsigned Size , unsigned AddrSpace , unsigned Align , MachineMemOperand :: Flags Flags , bool * IsFast ) const { if ( IsFast ) * IsFast = false ; if ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { bool AlignedBy4 = ( Align % <NUM_LIT> == <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) ) { bool AlignedBy4 = Align >= <NUM_LIT> ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccess ( ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? Align >= <NUM_LIT> : Align != <NUM_LIT> ; }


<s> if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccess ( ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? Align >= <NUM_LIT> : Align != <NUM_LIT> ; } return true ; } if ( Size < <NUM_LIT> ) return false ; if ( IsFast ) * IsFast = true ;

return Size >= <NUM_LIT> && Align >= <NUM_LIT> ;


<s> if ( MaskPopVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } else { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } TexFail = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , SDValue ( Result , <NUM_LIT> ) , DAG . getConstant ( MaskPopDwords , DL , <STR_LIT> ) ) ; } if ( DataDwordVT . isVector ( ) ) Data = padEltsToUndef ( DAG , DL , DataDwordVT , Data , NumDataDwords - MaskPopDwords ) ; if ( IsD16 ) Data = adjustLoadValueTypeImpl ( Data , ReqRetVT , DL , DAG , Unpacked ) ; if ( ! ReqRetVT . isVector ( ) ) Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT , Data ) ;

if ( TexFail ) return DAG . getMergeValues ( { Data , TexFail , SDValue ( Result , <NUM_LIT> ) } , DL ) ;


<s> switch ( Constraint [ <NUM_LIT> ] ) { default : break ; case 's' : case 'v' : case 'a' : return C_RegisterClass ; case 'A' : return C_Other ; }


<s> case 'a' : return C_RegisterClass ; case 'A' : return C_Other ; } }

return TargetLowering :: getConstraintType ( Constraint ) ;


<s> assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getObjectPtrOffset ( DL , QueuePtr , StructOffset ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; return DAG . getLoad ( <STR_LIT> , DL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr , PtrInfo , MinAlign ( <NUM_LIT> , StructOffset ) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> if ( Subtarget -> hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; SDValue EncodingImm = DAG . getTargetConstant ( Encoding , DL , <STR_LIT> ) ; SDValue ApertureReg = SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , EncodingImm ) , <NUM_LIT> ) ; SDValue ShiftAmount = DAG . getTargetConstant ( WidthM1 + <NUM_LIT> , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , ApertureReg , ShiftAmount ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ;


<s> Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol || ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerElementType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . ptrVal = MFI -> getGWSPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ) ; Info . memVT = <STR_LIT> ; Info . size = <NUM_LIT> ; Info . align = Align ( <NUM_LIT> ) ; Info . flags = MachineMemOperand :: MOStore ; if ( IntrID == <STR_LIT> ) Info . flags = MachineMemOperand :: MOLoad ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return Op . getValueType ( ) . getScalarType ( ) != <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( DAG , Op . getValueType ( ) ) ) return true ; for ( unsigned I = <NUM_LIT> , E = Op . getNumOperands ( ) ; I != E ; ++ I ) { if ( ! isCanonicalized ( DAG , Op . getOperand ( I ) , MaxDepth - <NUM_LIT> ) ) return false ; } return true ; } case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { for ( unsigned i = <NUM_LIT> , e = Op . getNumOperands ( ) ; i != e ; ++ i ) { SDValue SrcOp = Op . getOperand ( i ) ; if ( ! isCanonicalized ( DAG , SrcOp , MaxDepth - <NUM_LIT> ) ) return false ; } return true ; } case <STR_LIT> : case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : return false ; case <STR_LIT> : { SDValue Src = Op . getOperand ( <NUM_LIT> ) ; if ( Src . getValueType ( ) == <STR_LIT> && Src . getOpcode ( ) == <STR_LIT> ) { SDValue TruncSrc = Src . getOperand ( <NUM_LIT> ) ; if ( TruncSrc . getValueType ( ) == <STR_LIT> && TruncSrc . getOpcode ( ) == <STR_LIT> && TruncSrc . getOperand ( <NUM_LIT> ) . getValueType ( ) == <STR_LIT> ) { return isCanonicalized ( DAG , TruncSrc . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } } return false ; } case <STR_LIT> : { unsigned IntrinsicID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> if ( VT == <STR_LIT> ) { return Subtarget -> hasMadF16 ( ) && ! hasFP64FP16Denormals ( DAG . getMachineFunction ( ) ) ;


<s> void <ISA_LIT>TargetLowering :: LowerAsmOperandForConstraint ( SDValue Op , std :: string & Constraint , std :: vector < SDValue > & Ops , SelectionDAG & DAG ) const {

if ( Constraint . length ( ) == <NUM_LIT> && Constraint [ <NUM_LIT> ] == 'A' ) {


<s> if ( Size > <NUM_LIT> ) return ; uint64_t Val ; bool IsConst = false ; if ( ConstantSDNode * C = dyn_cast < ConstantSDNode > ( Op ) ) { Val = C -> getSExtValue ( ) ; IsConst = true ; } else if ( ConstantFPSDNode * C = dyn_cast < ConstantFPSDNode > ( Op ) ) { Val = C -> getValueAPF ( ) . bitcastToAPInt ( ) . getSExtValue ( ) ; IsConst = true ; } else if ( BuildVectorSDNode * V = dyn_cast < BuildVectorSDNode > ( Op ) ) { if ( Size != <NUM_LIT> || Op . getNumOperands ( ) != <NUM_LIT> ) return ; if ( Op . getOperand ( <NUM_LIT> ) . isUndef ( ) || Op . getOperand ( <NUM_LIT> ) . isUndef ( ) ) return ; if ( ConstantSDNode * C = V -> getConstantSplatNode ( ) ) { Val = C -> getSExtValue ( ) ; IsConst = true ; } else if ( ConstantFPSDNode * C = V -> getConstantFPSplatNode ( ) ) {


<s> } else if ( ConstantFPSDNode * C = dyn_cast < ConstantFPSDNode > ( Op ) ) { Val = C -> getValueAPF ( ) . bitcastToAPInt ( ) . getSExtValue ( ) ; IsConst = true ; } else if ( BuildVectorSDNode * V = dyn_cast < BuildVectorSDNode > ( Op ) ) { if ( Size != <NUM_LIT> || Op . getNumOperands ( ) != <NUM_LIT> ) return ; if ( Op . getOperand ( <NUM_LIT> ) . isUndef ( ) || Op . getOperand ( <NUM_LIT> ) . isUndef ( ) ) return ; if ( ConstantSDNode * C = V -> getConstantSplatNode ( ) ) { Val = C -> getSExtValue ( ) ; IsConst = true ; } else if ( ConstantFPSDNode * C = V -> getConstantFPSplatNode ( ) ) { Val = C -> getValueAPF ( ) . bitcastToAPInt ( ) . getSExtValue ( ) ; IsConst = true ; }


<s> if ( ( GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS && shouldUseLDSConstAddress ( GV ) ) || GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: REGION_ADDRESS || GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; EVT PtrVT = Op . getValueType ( ) ; if ( GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { SDValue GA = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , GSD -> getOffset ( ) , <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , GA ) ; } if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ;

PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ;


<s> } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> : return lowerBUILD_VECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerFP_ROUND ( Op , DAG ) ; case <STR_LIT> : return lowerTRAP ( Op , DAG ) ; case <STR_LIT> : return lowerDEBUGTRAP ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return splitUnaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return lowerFMINNUM_FMAXNUM ( Op , DAG ) ; case <STR_LIT> : return splitTernaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return splitBinaryVectorOp ( Op , DAG ) ;


<s> NumLoads = NumElts / <NUM_LIT> ; LoadVT = <STR_LIT> ( LoadVT . getScalarType ( ) , <NUM_LIT> ) ; } SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ; SDValue Ops [ ] = { DAG . getEntryNode ( ) , Rsrc , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , { } , { } , { } , CachePolicy , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ; setBufferOffsets ( Offset , DAG , & Ops [ <NUM_LIT> ] , NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ) ; uint64_t InstOffset = cast < ConstantSDNode > ( Ops [ <NUM_LIT> ] ) -> getZExtValue ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumLoads ; ++ i ) { Ops [ <NUM_LIT> ] = DAG . getTargetConstant ( InstOffset + <NUM_LIT> * i , DL , <STR_LIT> ) ; Loads . push_back ( getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , LoadVT , MMO , DAG ) ) ; }

if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) return DAG . getNode ( <STR_LIT> , DL , VT , Loads ) ;


<s> if ( ! allowsMemoryAccessForAlignment ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } unsigned AS = Store -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Store -> getAlignment ( ) < VT . getStoreSize ( ) && VT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasMultiDwordFlatScratchAddressing ( ) ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( Subtarget -> useDS128 ( ) && Store -> getAlignment ( ) >= <NUM_LIT> && VT . getStoreSize ( ) == <NUM_LIT> && NumElements != <NUM_LIT> ) return SDValue ( ) ; if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( ! Subtarget -> hasUsableDSOffset ( ) && NumElements == <NUM_LIT> && VT . getStoreSize ( ) == <NUM_LIT> && Store -> getAlignment ( ) < <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } return SDValue ( ) ; } else { llvm_unreachable ( <STR_LIT> ) ; }


<s> if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX ) InputReg = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgX ) ; if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY ) { SDValue Y = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgY ) ; Y = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Y , DAG . getShiftAmountConstant ( <NUM_LIT> , <STR_LIT> , SL ) ) ; InputReg = InputReg . getNode ( ) ? DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InputReg , Y ) : Y ; } if ( IncomingArgZ && ! IncomingArgZ -> isMasked ( ) && CalleeArgInfo -> WorkItemIDZ ) { SDValue Z = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgZ ) ; Z = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Z , DAG . getShiftAmountConstant ( <NUM_LIT> , <STR_LIT> , SL ) ) ; InputReg = InputReg . getNode ( ) ? DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InputReg , Z ) : Z ; } if ( ! InputReg . getNode ( ) ) { ArgDescriptor IncomingArg = ArgDescriptor :: createArg ( IncomingArgX ? * IncomingArgX : IncomingArgY ? * IncomingArgY : * IncomingArgZ , ~ <NUM_LIT> ) ; InputReg = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , IncomingArg ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; CCInfo . AllocateReg ( OutgoingArg -> getRegister ( ) ) ; } else { unsigned SpecialArgOffset = CCInfo . AllocateStack ( <NUM_LIT> , <NUM_LIT> ) ; SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain , InputReg , SpecialArgOffset ) ; MemOpChains . push_back ( ArgStore ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performSignExtendInRegCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : return performRcpCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> : return performFMed3Combine ( N , DCI ) ; case <STR_LIT> : return performCvtPkRTZCombine ( N , DCI ) ; case <STR_LIT> : return performClampCombine ( N , DCI ) ; case <STR_LIT> : { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> || VT == <STR_LIT> ) { SDLoc SL ( N ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ; EVT EltVT = Src . getValueType ( ) ; if ( EltVT == <STR_LIT> ) Src = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; SDValue Ext = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , VT , Ext ) ; } break ; } case <STR_LIT> : return performExtractVectorEltCombine ( N , DCI ) ; case <STR_LIT> : return performInsertVectorEltCombine ( N , DCI ) ; } return <ISA_LIT>TargetLowering :: PerformDAGCombine ( N , DCI ) ;


<s> SDValue IC = DAG . getVectorIdxConstant ( I , SL ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec , IC ) ; if ( I == <NUM_LIT> ) V = Elt ; else V = DAG . getSelectCC ( SL , Idx , IC , Elt , V , <STR_LIT> ) ; } return V ; } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ; unsigned LeftoverBitIdx = BitIndex % <NUM_LIT> ; SDLoc SL ( N ) ; SDValue Cast = DAG . getNode ( <STR_LIT> , SL , NewVT , Vec ) ; DCI . AddToWorklist ( Cast . getNode ( ) ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cast , DAG . getConstant ( EltIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Elt . getNode ( ) ) ; SDValue Srl = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Elt , DAG . getConstant ( LeftoverBitIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Srl . getNode ( ) ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , SL , EltVT . changeTypeToInteger ( ) , Srl ) ; DCI . AddToWorklist ( Trunc . getNode ( ) ) ; return DAG . getNode ( <STR_LIT> , SL , EltVT , Trunc ) ;


<s> SDLoc SL ( N ) ; SDValue Ins = N -> getOperand ( <NUM_LIT> ) ; EVT IdxVT = Idx . getValueType ( ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; for ( unsigned I = <NUM_LIT> , E = VecVT . getVectorNumElements ( ) ; I < E ; ++ I ) { SDValue IC = DAG . getConstant ( I , SL , IdxVT ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec , IC ) ; SDValue V = DAG . getSelectCC ( SL , Idx , IC , Ins , Elt , <STR_LIT> ) ; Ops . push_back ( V ) ; }

return DAG . getBuildVector ( VecVT , SL , Ops ) ;


<s> unsigned NumElem = VecVT . getVectorNumElements ( ) ; if ( VecSize <= <NUM_LIT> && EltSize < <NUM_LIT> ) return false ; if ( EltSize < <NUM_LIT> ) return true ; if ( Idx -> isDivergent ( ) ) return true ;

unsigned NumInsts = NumElem + ( ( EltSize + <NUM_LIT> ) / <NUM_LIT> ) * NumElem ;


<s> assert ( Offset == <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , Dst ) . add ( * SrcVec ) . add ( * Val ) . addImm ( SubReg ) ; MI . eraseFromParent ( ) ; return & MBB ; } if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , false ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( SrcVec -> getReg ( ) , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( SrcVec -> getReg ( ) , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { const MCInstrDesc & MovRelDesc = TII -> get ( getMOVRELDPseudo ( TRI , VecRC ) ) ; BuildMI ( MBB , I , DL , MovRelDesc ) . addReg ( Dst , RegState :: Define ) . addReg ( SrcVec -> getReg ( ) ) . add ( * Val ) . addImm ( SubReg - <STR_LIT> ) ; } MI . eraseFromParent ( ) ; return & MBB ; } if ( Val -> isReg ( ) ) MRI . clearKillFlags ( Val -> getReg ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; Register PhiReg = MRI . createVirtualRegister ( VecRC ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , SrcVec -> getReg ( ) , PhiReg , Offset , UseGPRIdxMode , false ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; MachineFunction * MF = MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) -> getReg ( ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcReg ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; bool UseGPRIdxMode = ST . useVGPRIndexMode ( EnableVGPRIndexMode ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return & MBB ; } const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; Register PhiReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register InitReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , InitReg ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , InitReg , PhiReg , Offset , UseGPRIdxMode , true ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode ) { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ;


<s> unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; bool UseGPRIdxMode = ST . useVGPRIndexMode ( EnableVGPRIndexMode ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return & MBB ; } const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; Register PhiReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register InitReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , InitReg ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , InitReg , PhiReg , Offset , UseGPRIdxMode , true ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ;


<s> int DPDenormModeDefault = ST -> hasFP64Denormals ( ) ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT ; int Mode = SPDenormMode | ( DPDenormModeDefault << <NUM_LIT> ) ; return DAG . getTargetConstant ( Mode , SL , <STR_LIT> ) ;


<s> int DPDenormModeDefault = ST -> hasFP64Denormals ( ) ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT ; int Mode = SPDenormMode | ( DPDenormModeDefault << <NUM_LIT> ) ;

return DAG . getTargetConstant ( Mode , SL , <STR_LIT> ) ;


<s> const MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; unsigned Reg = R -> getReg ( ) ; if ( Register :: isPhysicalRegister ( Reg ) ) return ! TRI . isSGPRReg ( MRI , Reg ) ; if ( MRI . isLiveIn ( Reg ) ) { if ( ! TRI . isSGPRReg ( MRI , Reg ) ) return true ; else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; return false ; } const Value * V = FLI -> getValueFromVirtualReg ( Reg ) ; if ( V ) return KDA -> isDivergent ( V ) ; assert ( Reg == FLI -> DemoteRegister || isCopyFromRegOfInlineAsm ( N ) ) ; return ! TRI . isSGPRReg ( MRI , Reg ) ; } break ; case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ; } break ; case <STR_LIT> : return true ;


<s> const SDValue One = DAG . getConstantFP ( <NUM_LIT> , SL , <STR_LIT> ) ; SDVTList ScaleVT = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue DenominatorScaled = DAG . getNode ( <STR_LIT> , SL , ScaleVT , RHS , RHS , LHS ) ; SDValue NumeratorScaled = DAG . getNode ( <STR_LIT> , SL , ScaleVT , LHS , RHS , LHS ) ; SDValue ApproxRcp = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , DenominatorScaled ) ; SDValue NegDivScale0 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , DenominatorScaled ) ; const unsigned Denorm32Reg = <STR_LIT> :: ID_MODE | ( <NUM_LIT> << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ; const SDValue BitField = DAG . getTargetConstant ( Denorm32Reg , SL , <STR_LIT> ) ; if ( ! Subtarget -> hasFP32Denormals ( ) ) { SDVTList BindParamVTs = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue EnableDenorm ; if ( Subtarget -> hasDenormModeInst ( ) ) { const SDValue EnableDenormValue = getSPDenormModeValue ( FP_DENORM_FLUSH_NONE , DAG , SL , Subtarget ) ; EnableDenorm = DAG . getNode ( <STR_LIT> , SL , BindParamVTs , DAG . getEntryNode ( ) , EnableDenormValue ) ; } else { const SDValue EnableDenormValue = DAG . getConstant ( FP_DENORM_FLUSH_NONE , SL , <STR_LIT> ) ; EnableDenorm = DAG . getNode ( <STR_LIT> , SL , BindParamVTs , DAG . getEntryNode ( ) , EnableDenormValue , BitField ) ; } SDValue Ops [ <NUM_LIT> ] = {

NegDivScale0 , EnableDenorm . getValue ( <NUM_LIT> ) , EnableDenorm . getValue ( <NUM_LIT> ) } ;


<s> SDNode * <ISA_LIT>TargetLowering :: PostISelFolding ( MachineSDNode * Node , SelectionDAG & DAG ) const {

const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ;