<s> EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) { SDValue LHS = N -> getOperand ( <NUM_LIT> ) ; SDValue RHS = N -> getOperand ( <NUM_LIT> ) ; if ( LHS . getOpcode ( ) == <STR_LIT> ) { SDValue A = LHS . getOperand ( <NUM_LIT> ) ; if ( A == LHS . getOperand ( <NUM_LIT> ) ) { const SDValue Two = DAG . getConstantFP ( <NUM_LIT> , DL , <STR_LIT> ) ; SDValue NegRHS = DAG . getNode ( <STR_LIT> , DL , VT , RHS ) ; return DAG . getNode ( <STR_LIT> , DL , VT , Two , A , NegRHS ) ; } } if ( RHS . getOpcode ( ) == <STR_LIT> ) { SDValue A = RHS . getOperand ( <NUM_LIT> ) ; if ( A == RHS . getOperand ( <NUM_LIT> ) ) { const SDValue NegTwo = DAG . getConstantFP ( - <NUM_LIT> , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , VT , NegTwo , A , LHS ) ; } } return SDValue ( ) ; } break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( DCI . isBeforeLegalize ( ) ) break ; MemSDNode * MemNode = cast < MemSDNode > ( N ) ; SDValue Ptr = MemNode -> getBasePtr ( ) ; unsigned AS = MemNode -> getAddressSpace ( ) ; if ( Ptr . getOpcode ( ) == <STR_LIT> && AS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) { SDValue NewPtr = performSHLPtrCombine ( Ptr . getNode ( ) , AS , DCI ) ; if ( NewPtr ) { SmallVector < SDValue , <NUM_LIT> > NewOps ( MemNode -> op_begin ( ) , MemNode -> op_end ( ) ) ; NewOps [ N -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ] = NewPtr ; return SDValue ( DAG . UpdateNodeOperands ( MemNode , NewOps ) , <NUM_LIT> ) ; } }


<s> if ( Info -> getFrameOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ; Info -> limitOccupancy ( MF ) ; if ( ST . isWave32 ( ) && ! MF . empty ( ) ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DebugLoc DL ; MachineBasicBlock & MBB = MF . front ( ) ; MachineBasicBlock :: iterator I = MBB . getFirstNonDebugInstr ( ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , <STR_LIT> ) ; for ( auto & MBB : MF ) { for ( auto & MI : MBB ) {


<s> Info -> limitOccupancy ( MF ) ; if ( ST . isWave32 ( ) && ! MF . empty ( ) ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DebugLoc DL ; MachineBasicBlock & MBB = MF . front ( ) ; MachineBasicBlock :: iterator I = MBB . getFirstNonDebugInstr ( ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , <STR_LIT> ) ; for ( auto & MBB : MF ) { for ( auto & MI : MBB ) { TII -> fixImplicitOperands ( MI ) ; }


<s> if ( Reg . isPhysical ( ) ) return ! TRI . isSGPRReg ( MRI , Reg ) ; if ( MRI . isLiveIn ( Reg ) ) { if ( ! TRI . isSGPRReg ( MRI , Reg ) ) return true ; else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; return false ; } const Value * V = FLI -> getValueFromVirtualReg ( Reg ) ; if ( V ) return KDA -> isDivergent ( V ) ; assert ( Reg == FLI -> DemoteRegister || isCopyFromRegOfInlineAsm ( N ) ) ; return ! TRI . isSGPRReg ( MRI , Reg ) ; } break ; case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ; } break ; case <STR_LIT> : return true ;


<s> SDNode * Result = DAG . getNode ( CFNode , DL , DAG . getVTList ( Res ) , Ops ) . getNode ( ) ; if ( ! HaveChain ) { SDValue Ops [ ] = { SDValue ( Result , <NUM_LIT> ) , BRCOND . getOperand ( <NUM_LIT> ) } ; Result = DAG . getMergeValues ( Ops , DL ) . getNode ( ) ; } if ( BR ) { SDValue Ops [ ] = { BR -> getOperand ( <NUM_LIT> ) , BRCOND . getOperand ( <NUM_LIT> ) } ; SDValue NewBR = DAG . getNode ( <STR_LIT> , DL , BR -> getVTList ( ) , Ops ) ; DAG . ReplaceAllUsesWith ( BR , NewBR . getNode ( ) ) ; } SDValue Chain = SDValue ( Result , Result -> getNumValues ( ) - <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Intr -> getNumValues ( ) - <NUM_LIT> ; i != e ; ++ i ) {

SDNode * CopyToReg = findUser ( SDValue ( Intr , i ) , <STR_LIT> ) ;


<s> assert ( DMaskLanes != <NUM_LIT> ) ; if ( auto * VT = dyn_cast < VectorType > ( Ty ) ) { unsigned NumElts = std :: min ( DMaskLanes , static_cast < unsigned > ( VT -> getNumElements ( ) ) ) ;

return EVT :: getVectorVT ( Ty -> getContext ( ) , EVT :: getEVT ( VT -> getElementType ( ) ) , NumElts ) ;


<s> assert ( DMaskLanes != <NUM_LIT> ) ; if ( auto * VT = dyn_cast < VectorType > ( Ty ) ) { unsigned NumElts = std :: min ( DMaskLanes , static_cast < unsigned > ( VT -> getNumElements ( ) ) ) ; return EVT :: getVectorVT ( Ty -> getContext ( ) , EVT :: getEVT ( VT -> getElementType ( ) ) , NumElts ) ; }


<s> SDValue InputReg ; if ( IncomingArg ) { InputReg = loadInputValue ( DAG , ArgRC , ArgVT , DL , * IncomingArg ) ; } else { assert ( InputID == <STR_LIT> ) ; InputReg = getImplicitArgPtr ( DAG , DL ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; if ( ! CCInfo . AllocateReg ( OutgoingArg -> getRegister ( ) ) ) report_fatal_error ( <STR_LIT> ) ; } else { unsigned SpecialArgOffset = CCInfo . AllocateStack ( ArgVT . getStoreSize ( ) , <NUM_LIT> ) ; SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain , InputReg , SpecialArgOffset ) ; MemOpChains . push_back ( ArgStore ) ; } } const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return ; const ArgDescriptor * IncomingArgX = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) . first ; const ArgDescriptor * IncomingArgY = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) . first ; const ArgDescriptor * IncomingArgZ = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) . first ; SDValue InputReg ; SDLoc SL ; if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX ) InputReg = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgX ) ; if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY ) { SDValue Y = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgY ) ; Y = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Y , DAG . getShiftAmountConstant ( <NUM_LIT> , <STR_LIT> , SL ) ) ; InputReg = InputReg . getNode ( ) ? DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InputReg , Y ) : Y ; } if ( IncomingArgZ && ! IncomingArgZ -> isMasked ( ) && CalleeArgInfo -> WorkItemIDZ ) { SDValue Z = loadInputValue ( DAG , ArgRC , <STR_LIT> , DL , * IncomingArgZ ) ; Z = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Z , DAG . getShiftAmountConstant ( <NUM_LIT> , <STR_LIT> , SL ) ) ; InputReg = InputReg . getNode ( ) ? DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InputReg , Z ) : Z ; }


<s> void <ISA_LIT>TargetLowering :: passSpecialInputs ( CallLoweringInfo & CLI , CCState & CCInfo , const <ISA_LIT>MachineFunctionInfo & Info , SmallVectorImpl < std :: pair < unsigned , SDValue >> & RegsToPass , SmallVectorImpl < SDValue > & MemOpChains , SDValue Chain ) const { if ( ! CLI . CS ) return ; SelectionDAG & DAG = CLI . DAG ; const SDLoc & DL = CLI . DL ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; const <ISA_LIT>FunctionArgInfo & CallerArgInfo = Info . getArgInfo ( ) ; const <ISA_LIT>FunctionArgInfo * CalleeArgInfo = & <ISA_LIT>ArgumentUsageInfo :: FixedABIFunctionInfo ; if ( const Function * CalleeFunc = CLI . CS . getCalledFunction ( ) ) { auto & ArgUsageInfo = DAG . getPass ( ) -> getAnalysis < <ISA_LIT>ArgumentUsageInfo > ( ) ; CalleeArgInfo = & ArgUsageInfo . lookupFuncArgInfo ( * CalleeFunc ) ; } <STR_LIT> InputRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; for ( auto InputID : InputRegs ) { const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo -> getPreloadedValue ( InputID ) ; if ( ! OutgoingArg ) continue ; const ArgDescriptor * IncomingArg ; const TargetRegisterClass * IncomingArgRC ; std :: tie ( IncomingArg , IncomingArgRC ) = CallerArgInfo . getPreloadedValue ( InputID ) ; assert ( IncomingArgRC == ArgRC ) ; EVT ArgVT = TRI -> getSpillSize ( * ArgRC ) == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue InputReg ; if ( IncomingArg ) { InputReg = loadInputValue ( DAG , ArgRC , ArgVT , DL , * IncomingArg ) ; } else { assert ( InputID == <STR_LIT> ) ; InputReg = getImplicitArgPtr ( DAG , DL ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; if ( ! CCInfo . AllocateReg ( OutgoingArg -> getRegister ( ) ) ) report_fatal_error ( <STR_LIT> ) ; } else { unsigned SpecialArgOffset = CCInfo . AllocateStack ( ArgVT . getStoreSize ( ) , <NUM_LIT> ) ; SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain , InputReg , SpecialArgOffset ) ; MemOpChains . push_back ( ArgStore ) ; } } const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return ; const ArgDescriptor * IncomingArgX = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) . first ; const ArgDescriptor * IncomingArgY = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) . first ; const ArgDescriptor * IncomingArgZ = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) . first ;


<s> TargetLowering :: AsmOperandInfoVector TargetConstraints = ParseConstraints ( MF . getDataLayout ( ) , Subtarget -> getRegisterInfo ( ) , CS ) ; for ( auto & TC : TargetConstraints ) { if ( TC . Type == InlineAsm :: isOutput ) { ComputeConstraintToUse ( TC , SDValue ( ) ) ; unsigned AssignedReg ; const TargetRegisterClass * RC ; std :: tie ( AssignedReg , RC ) = getRegForInlineAsmConstraint ( <ISA_LIT>RI , TC . ConstraintCode , TC . ConstraintVT ) ; if ( RC ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; if ( AssignedReg != <NUM_LIT> && <ISA_LIT>RI -> isSGPRReg ( MRI , AssignedReg ) ) return true ; else if ( <ISA_LIT>RI -> isSGPRClass ( RC ) ) return true ;


<s> if ( VT == <STR_LIT> ) return ! Subtarget -> hasFP16Denormals ( ) && Subtarget -> hasMadF16 ( ) ;


<s> bool <ISA_LIT>TargetLowering :: isFPExtFoldable ( const SelectionDAG & DAG , unsigned Opcode , EVT DestVT , EVT SrcVT ) const { return ( ( Opcode == <STR_LIT> && Subtarget -> hasMadMixInsts ( ) ) || ( Opcode == <STR_LIT> && Subtarget -> hasFmaMixInsts ( ) ) ) && DestVT . getScalarType ( ) == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) && SrcVT . getScalarType ( ) == <STR_LIT> ;


<s> } SmallVector < SDValue , <NUM_LIT> > Loads ; unsigned NumLoads = <NUM_LIT> ; MVT LoadVT = VT . getSimpleVT ( ) ; unsigned NumElts = LoadVT . isVector ( ) ? LoadVT . getVectorNumElements ( ) : <NUM_LIT> ; assert ( ( LoadVT . getScalarType ( ) == <STR_LIT> || LoadVT . getScalarType ( ) == <STR_LIT> ) && isPowerOf2_32 ( NumElts ) ) ; if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) { NumLoads = NumElts == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; LoadVT = <STR_LIT> ; } SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ; unsigned CachePolicy = cast < ConstantSDNode > ( GLC ) -> getZExtValue ( ) ; SDValue Ops [ ] = { DAG . getEntryNode ( ) , Rsrc , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , { } , { } , { } , DAG . getTargetConstant ( CachePolicy , DL , <STR_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ;

setBufferOffsets ( Offset , DAG , & Ops [ <NUM_LIT> ] , NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ) ;


<s> SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ; unsigned CachePolicy = cast < ConstantSDNode > ( GLC ) -> getZExtValue ( ) ; SDValue Ops [ ] = { DAG . getEntryNode ( ) , Rsrc , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , { } , { } , { } , DAG . getTargetConstant ( CachePolicy , DL , <STR_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ; setBufferOffsets ( Offset , DAG , & Ops [ <NUM_LIT> ] , NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ) ; uint64_t InstOffset = cast < ConstantSDNode > ( Ops [ <NUM_LIT> ] ) -> getZExtValue ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumLoads ; ++ i ) { Ops [ <NUM_LIT> ] = DAG . getTargetConstant ( InstOffset + <NUM_LIT> * i , DL , <STR_LIT> ) ; Loads . push_back ( DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , LoadVT , MMO ) ) ;


<s> switch ( TRI . getRegSizeInBits ( * VecRC ) ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default :


<s> SDValue Glue = Chain . getValue ( <NUM_LIT> ) ; return DAG . getNode ( NodeOp , DL , <STR_LIT> , Chain , Op . getOperand ( <NUM_LIT> ) , Glue ) ; } case <STR_LIT> : { return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Chain , Op . getOperand ( <NUM_LIT> ) ) ; } case <STR_LIT> : { return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Chain , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Chain , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getOperand ( <NUM_LIT> ) . getValueType ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOStore , VT . getStoreSize ( ) , <NUM_LIT> ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; } case <STR_LIT> : { SDValue Src = Op . getOperand ( <NUM_LIT> ) ; if ( const ConstantFPSDNode * K = dyn_cast < ConstantFPSDNode > ( Src ) ) { if ( ! K -> isNegative ( ) ) return Chain ; SDValue NegOne = DAG . getTargetConstant ( FloatToBits ( - <NUM_LIT> ) , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Chain , NegOne ) ; }

SDValue Cast = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Src ) ;


<s> std :: tie ( Reg , RC , Ty ) = MFI . getPreloadedValue ( PVID ) ; if ( ! Reg ) { assert ( PVID == <STR_LIT> :: KERNARG_SEGMENT_PTR ) ; return DAG . getConstant ( <NUM_LIT> , SDLoc ( ) , VT ) ;


<s> const <ISA_LIT>FunctionArgInfo & CallerArgInfo = Info . getArgInfo ( ) ; const <ISA_LIT>FunctionArgInfo * CalleeArgInfo = & <ISA_LIT>ArgumentUsageInfo :: FixedABIFunctionInfo ; if ( const Function * CalleeFunc = CLI . CB -> getCalledFunction ( ) ) { auto & ArgUsageInfo = DAG . getPass ( ) -> getAnalysis < <ISA_LIT>ArgumentUsageInfo > ( ) ; CalleeArgInfo = & ArgUsageInfo . lookupFuncArgInfo ( * CalleeFunc ) ; } <STR_LIT> InputRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; for ( auto InputID : InputRegs ) { const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; LLT ArgTy ; std :: tie ( OutgoingArg , ArgRC , ArgTy ) = CalleeArgInfo -> getPreloadedValue ( InputID ) ; if ( ! OutgoingArg ) continue ; const ArgDescriptor * IncomingArg ; const TargetRegisterClass * IncomingArgRC ; LLT Ty ; std :: tie ( IncomingArg , IncomingArgRC , Ty ) = CallerArgInfo . getPreloadedValue ( InputID ) ; assert ( IncomingArgRC == ArgRC ) ; EVT ArgVT = TRI -> getSpillSize ( * ArgRC ) == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue InputReg ; if ( IncomingArg ) { InputReg = loadInputValue ( DAG , ArgRC , ArgVT , DL , * IncomingArg ) ; } else if ( InputID == <STR_LIT> ) { InputReg = getImplicitArgPtr ( DAG , DL ) ; } else { InputReg = DAG . getUNDEF ( ArgVT ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; if ( ! CCInfo . AllocateReg ( OutgoingArg -> getRegister ( ) ) ) report_fatal_error ( <STR_LIT> ) ; } else { unsigned SpecialArgOffset = CCInfo . AllocateStack ( ArgVT . getStoreSize ( ) , Align ( <NUM_LIT> ) ) ; SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain , InputReg , SpecialArgOffset ) ; MemOpChains . push_back ( ArgStore ) ; } } const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; LLT Ty ; std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , Ty ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return ; const ArgDescriptor * IncomingArgX = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgY = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; const ArgDescriptor * IncomingArgZ = std :: get < <NUM_LIT> > ( CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ) ; SDValue InputReg ;


<s> switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Value * Ptr = II -> getArgOperand ( <NUM_LIT> ) ; AccessTy = II -> getType ( ) ; Ops . push_back ( Ptr ) ; return true ;


<s> case <STR_LIT> : { Value * Ptr = II -> getArgOperand ( <NUM_LIT> ) ; AccessTy = II -> getType ( ) ; Ops . push_back ( Ptr ) ; return true ; } default : return false ;


<s> Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . align = <NUM_LIT> ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable ; return true ; } case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . flags = MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . align = <NUM_LIT> ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ;


<s> default : llvm_unreachable ( <STR_LIT> ) ; } SDValue Ops [ ] = { M -> getOperand ( <NUM_LIT> ) , M -> getOperand ( <NUM_LIT> ) , M -> getOperand ( <NUM_LIT> ) } ; return DAG . getMemIntrinsicNode ( Opc , SDLoc ( Op ) , M -> getVTList ( ) , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; } case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; unsigned Opc = ( IntrID == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; EVT VT = Op . getValueType ( ) ; EVT IntVT = VT . changeTypeToInteger ( ) ; auto * M = cast < MemSDNode > ( Op ) ; return DAG . getMemIntrinsicNode ( Opc , DL , Op -> getVTList ( ) , Ops , IntVT , M -> getMemOperand ( ) ) ; } case <STR_LIT> : { MemSDNode * M = cast < MemSDNode > ( Op ) ; SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getValueType ( ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , M -> getMemOperand ( ) ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getValueType ( ) ; auto * M = cast < MemSDNode > ( Op ) ; unsigned Opcode = <NUM_LIT> ; switch ( IntrID ) {


<s> return ; } case <STR_LIT> : { unsigned IID = cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; if ( IID == <STR_LIT> ) { SDValue Src0 = N -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; SDValue Cvt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } break ; } case <STR_LIT> : { SDValue Chain ; if ( SDValue Res = lowerIntrinsicWChain_IllegalReturnType ( SDValue ( N , <NUM_LIT> ) , Chain , DAG ) ) { Results . push_back ( Res ) ; Results . push_back ( Chain ) ; return ; } break ; } case <STR_LIT> : { SDLoc SL ( N ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ;

SDValue LHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ;


<s> break ; } case <STR_LIT> : { SDLoc SL ( N ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ; SDValue LHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; SDValue RHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; EVT SelectVT = NewVT ; if ( NewVT . bitsLT ( <STR_LIT> ) ) { LHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS ) ; RHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , RHS ) ; SelectVT = <STR_LIT> ; } SDValue NewSelect = DAG . getNode ( <STR_LIT> , SL , SelectVT , N -> getOperand ( <NUM_LIT> ) , LHS , RHS ) ;

if ( NewVT != SelectVT ) NewSelect = DAG . getNode ( <STR_LIT> , SL , NewVT , NewSelect ) ;


<s> switch ( IntrID ) { case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align = <NUM_LIT> ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ;

Info . vol = ! Vol || ! Vol -> isZero ( ) ;


<s> Opc = ( Opc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( Opc , SL , VTList , Args ) ; } case <STR_LIT> : { auto C = dyn_cast < ConstantSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) ; if ( ! C || C -> getZExtValue ( ) != <NUM_LIT> ) break ; SDValue Args [ ] = { LHS , RHS . getOperand ( <NUM_LIT> ) , RHS . getOperand ( <NUM_LIT> ) } ; return DAG . getNode ( <STR_LIT> , SDLoc ( N ) , RHS -> getVTList ( ) , Args ) ; }


<s> bool HasStackObjects = MFI . hasStackObjects ( ) ; if ( HasStackObjects ) Info . setHasNonSpillStackObjects ( true ) ; if ( TM . getOptLevel ( ) == CodeGenOpt :: None ) HasStackObjects = true ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . isAmdCodeObjectV2 ( MF ) ) { if ( HasStackObjects ) { unsigned PrivateSegmentBufferReg = TRI . getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_BUFFER ) ; Info . setScratchRSrcReg ( PrivateSegmentBufferReg ) ; unsigned PrivateSegmentWaveByteOffsetReg = TRI . getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET ) ; Info . setScratchWaveOffsetReg ( PrivateSegmentWaveByteOffsetReg ) ; } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ;

Info . setScratchRSrcReg ( ReservedBufferReg ) ;


<s> unsigned Opc = MI . getOpcode ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; for ( auto I : { <STR_LIT> ( Opc , <STR_LIT> :: src0 ) , <STR_LIT> ( Opc , <STR_LIT> :: src1 ) } ) { if ( I == - <NUM_LIT> ) break ; MachineOperand & Op = MI . getOperand ( I ) ; if ( ! Op . isReg ( ) || ! Op . getReg ( ) . isVirtual ( ) ) continue ; auto * RC = TRI -> getRegClassForReg ( MRI , Op . getReg ( ) ) ; if ( ! TRI -> hasAGPRs ( RC ) ) continue ; auto * Src = MRI . getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Src || ! Src -> isCopy ( ) || ! TRI -> isSGPRReg ( MRI , Src -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; auto * NewRC = TRI -> getEquivalentVGPRClass ( RC ) ; MRI . setRegClass ( Op . getReg ( ) , NewRC ) ;


<s> if ( Opcode == <STR_LIT> ) { auto F = MI -> getOperand ( <NUM_LIT> ) . getFPImm ( ) -> getValueAPF ( ) ; if ( F . isNaN ( ) && F . isSignaling ( ) ) return false ; return ! F . isDenormal ( ) || denormalsEnabledForType ( MRI . getType ( Reg ) , MF ) ; } if ( MaxDepth == <NUM_LIT> ) return false ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : { if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( MRI . getType ( Reg ) , MF ) ) return true ; for ( const MachineOperand & MO : llvm :: drop_begin ( MI -> operands ( ) ) ) if ( ! isCanonicalized ( MO . getReg ( ) , MF , MaxDepth - <NUM_LIT> ) ) return false ; return true ; } default : return denormalsEnabledForType ( MRI . getType ( Reg ) , MF ) && isKnownNeverSNaN ( Reg , MRI ) ; } llvm_unreachable ( <STR_LIT> ) ;


<s> unsigned Opcode = MI -> getOpcode ( ) ; if ( Opcode == <STR_LIT> ) return true ; if ( Opcode == <STR_LIT> ) { auto F = MI -> getOperand ( <NUM_LIT> ) . getFPImm ( ) -> getValueAPF ( ) ; if ( F . isNaN ( ) && F . isSignaling ( ) ) return false ; return ! F . isDenormal ( ) || denormalsEnabledForType ( MRI . getType ( Reg ) , MF ) ; } if ( MaxDepth == <NUM_LIT> ) return false ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : {

if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( MRI . getType ( Reg ) , MF ) ) return true ;


<s> } LLVMContext & Ctx = * DAG . getContext ( ) ; if ( ! CCState :: resultsCompatible ( CalleeCC , CallerCC , MF , Ctx , Ins , CCAssignFnForCall ( CalleeCC , IsVarArg ) , CCAssignFnForCall ( CallerCC , IsVarArg ) ) ) return false ; if ( ! CCMatch ) { const uint32_t * CalleePreserved = TRI -> getCallPreservedMask ( MF , CalleeCC ) ; if ( ! TRI -> regmaskSubsetEqual ( CallerPreserved , CalleePreserved ) ) return false ; } if ( Outs . empty ( ) ) return true ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CalleeCC , IsVarArg , MF , ArgLocs , Ctx ) ; CCInfo . AnalyzeCallOperands ( Outs , CCAssignFnForCall ( CalleeCC , IsVarArg ) ) ; const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

if ( CCInfo . getNextStackOffset ( ) > FuncInfo -> getBytesInStackArgArea ( ) ) return false ;


<s> CallingConv :: ID CallerCC = CallerF -> getCallingConv ( ) ; const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; const uint32_t * CallerPreserved = TRI -> getCallPreservedMask ( MF , CallerCC ) ; if ( ! CallerPreserved ) return false ; bool CCMatch = CallerCC == CalleeCC ; if ( DAG . getTarget ( ) . Options . GuaranteedTailCallOpt ) { if ( canGuaranteeTCO ( CalleeCC ) && CCMatch ) return true ; return false ; } if ( IsVarArg ) return false ; for ( const Argument & Arg : CallerF -> args ( ) ) { if ( Arg . hasByValAttr ( ) ) return false ; } LLVMContext & Ctx = * DAG . getContext ( ) ;

if ( ! CCState :: resultsCompatible ( CalleeCC , CallerCC , MF , Ctx , Ins , CCAssignFnForCall ( CalleeCC , IsVarArg ) , CCAssignFnForCall ( CallerCC , IsVarArg ) ) ) return false ;


<s> switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : { SDValue Src0 = Node -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = Node -> getOperand ( <NUM_LIT> ) ; SDValue Src2 = Node -> getOperand ( <NUM_LIT> ) ; if ( ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) != <STR_LIT> ) && ( Src0 == Src1 || Src0 == Src2 ) ) break ; MVT VT = Src0 . getValueType ( ) . getSimpleVT ( ) ; const TargetRegisterClass * RC = getRegClassFor ( VT ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; SDValue UndefReg = DAG . getRegister ( MRI . createVirtualRegister ( RC ) , VT ) ; SDValue ImpDef = DAG . getCopyToReg ( DAG . getEntryNode ( ) , SDLoc ( Node ) , UndefReg , Src0 , SDValue ( ) ) ; if ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) == <STR_LIT> ) { if ( Src1 . isMachineOpcode ( ) && Src1 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src1 ; else if ( Src2 . isMachineOpcode ( ) && Src2 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src2 ; else { assert ( Src1 . getMachineOpcode ( ) == <STR_LIT> ) ; Src0 = UndefReg ; Src1 = UndefReg ; } } else break ;

SmallVector < SDValue , <NUM_LIT> > Ops = { Src0 , Src1 , Src2 } ;


<s> if ( TII -> isMIMG ( Opcode ) && ! TII -> get ( Opcode ) . mayStore ( ) && ! TII -> isGather4 ( Opcode ) ) adjustWritemask ( Node , DAG ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { legalizeTargetIndependentNode ( Node , DAG ) ; return Node ; } switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : { SDValue Src0 = Node -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = Node -> getOperand ( <NUM_LIT> ) ; SDValue Src2 = Node -> getOperand ( <NUM_LIT> ) ; if ( ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) != <STR_LIT> ) && ( Src0 == Src1 || Src0 == Src2 ) ) break ; MVT VT = Src0 . getValueType ( ) . getSimpleVT ( ) ; const TargetRegisterClass * RC = getRegClassFor ( VT ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; SDValue UndefReg = DAG . getRegister ( MRI . createVirtualRegister ( RC ) , VT ) ; SDValue ImpDef = DAG . getCopyToReg ( DAG . getEntryNode ( ) , SDLoc ( Node ) , UndefReg , Src0 , SDValue ( ) ) ; if ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) == <STR_LIT> ) { if ( Src1 . isMachineOpcode ( ) && Src1 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src1 ; else if ( Src2 . isMachineOpcode ( ) && Src2 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src2 ; else { assert ( Src1 . getMachineOpcode ( ) == <STR_LIT> ) ; Src0 = UndefReg ;


<s> if ( LoadVT . isVector ( ) ) { EquivLoadVT = Unpacked ? UnpackedLoadVT : getEquivalentMemType ( * DAG . getContext ( ) , LoadVT ) ; } SDVTList VTList = DAG . getVTList ( EquivLoadVT , <STR_LIT> ) ; SDValue Load = DAG . getMemIntrinsicNode ( IsIntrinsic ? <STR_LIT> : Opcode , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; SDValue Adjusted = adjustLoadValueTypeImpl ( Load , LoadVT , DL , DAG , Unpacked ) ;

return DAG . getMergeValues ( { Adjusted , Load . getValue ( <NUM_LIT> ) } , DL ) ;


<s> SDValue <ISA_LIT>TargetLowering :: handleD16VData ( SDValue VData , SelectionDAG & DAG ) const { EVT StoreVT = VData . getValueType ( ) ; if ( ! StoreVT . isVector ( ) ) return VData ; SDLoc DL ( VData ) ; assert ( ( StoreVT . getVectorNumElements ( ) != <NUM_LIT> ) && <STR_LIT> ) ; if ( Subtarget -> hasUnpackedD16VMem ( ) ) { EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , StoreVT . getVectorNumElements ( ) ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ; return DAG . UnrollVectorOp ( ZExt . getNode ( ) ) ; } if ( isTypeLegal ( StoreVT ) ) return VData ;

EVT EquivStoreVT = getEquivalentMemType ( * DAG . getContext ( ) , StoreVT ) ;


<s> if ( ! StoreVT . isVector ( ) ) return VData ; SDLoc DL ( VData ) ; assert ( ( StoreVT . getVectorNumElements ( ) != <NUM_LIT> ) && <STR_LIT> ) ; if ( Subtarget -> hasUnpackedD16VMem ( ) ) { EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , StoreVT . getVectorNumElements ( ) ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ;

return DAG . UnrollVectorOp ( ZExt . getNode ( ) ) ;


<s> Hi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi ) ; SDValue ShlHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ; SDValue Or = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Lo , ShlHi ) ;

return DAG . getNode ( <STR_LIT> , SL , VT , Or ) ;


<s> EVT VecVT = Vec . getValueType ( ) ; unsigned NumElts = VecVT . getVectorNumElements ( ) ; assert ( VecVT . getScalarSizeInBits ( ) == <NUM_LIT> && ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) ) ; DAGCombinerInfo DCI ( DAG , AfterLegalizeVectorOps , true , nullptr ) ; if ( SDValue Combined = performExtractVectorEltCombine ( Op . getNode ( ) , DCI ) ) return Combined ; EVT IntVT = NumElts == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue Four = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue ScaledIdx = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Idx , Four ) ; SDValue BC = DAG . getNode ( <STR_LIT> , SL , IntVT , Vec ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , IntVT , BC , ScaledIdx ) ; if ( ResultVT == <STR_LIT> ) { SDValue Result = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Elt ) ; return DAG . getNode ( <STR_LIT> , SL , ResultVT , Result ) ;


<s> SDValue HiHalf = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , BCVec , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ; SDValue LoVec = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LoHalf ) ; SDValue HiVec = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , HiHalf ) ; unsigned Idx = KIdx -> getZExtValue ( ) ; bool InsertLo = Idx < <NUM_LIT> ; SDValue InsHalf = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InsertLo ? LoVec : HiVec , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InsVal ) , DAG . getConstant ( InsertLo ? Idx : ( Idx - <NUM_LIT> ) , SL , <STR_LIT> ) ) ; InsHalf = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InsHalf ) ; SDValue Concat = InsertLo ? DAG . getBuildVector ( <STR_LIT> , SL , { InsHalf , HiHalf } ) : DAG . getBuildVector ( <STR_LIT> , SL , { LoHalf , InsHalf } ) ; return DAG . getNode ( <STR_LIT> , SL , VecVT , Concat ) ; } assert ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) ; if ( isa < ConstantSDNode > ( Idx ) ) return SDValue ( ) ; EVT IntVT = NumElts == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ;

SDValue Val = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InsVal ) ;


<s> SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; return DAG . getNode ( Vec . getOpcode ( ) , SL , EltVT , Elt ) ; } if ( Vec . hasOneUse ( ) && DCI . isBeforeLegalize ( ) ) { SDLoc SL ( N ) ; EVT EltVT = N -> getValueType ( <NUM_LIT> ) ; SDValue Idx = N -> getOperand ( <NUM_LIT> ) ; unsigned Opc = Vec . getOpcode ( ) ; switch ( Opc ) { default :


<s> case <STR_LIT> : { SDValue Src0 = N -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; SDLoc SL ( N ) ; unsigned Opcode ; if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else if ( IID == <STR_LIT> ) Opcode = <STR_LIT> ; else Opcode = <STR_LIT> ; SDValue Cvt = DAG . getNode ( Opcode , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } } break ; } case <STR_LIT> : { if ( SDValue Res = LowerINTRIN<ISA_LIT>C_W_CHAIN ( SDValue ( N , <NUM_LIT> ) , DAG ) ) { Results . push_back ( Res ) ; Results . push_back ( Res . getValue ( <NUM_LIT> ) ) ; return ; } break ; } case <STR_LIT> : { SDLoc SL ( N ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ; SDValue LHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; SDValue RHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; EVT SelectVT = NewVT ; if ( NewVT . bitsLT ( <STR_LIT> ) ) { LHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS ) ; RHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , RHS ) ; SelectVT = <STR_LIT> ;


<s> if ( GAFlags == <ISA_LIT>InstrInfo :: MO_NONE ) { PtrHi = DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } else { PtrHi = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , GAFlags + <NUM_LIT> ) ;


<s> } if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ;

Align Alignment = DataLayout . getABITypeAlign ( PtrTy ) ;


<s> SDValue ArgVal = DAG . getNode ( <STR_LIT> , SL , IntVT , Extract ) ; ArgVal = DAG . getNode ( <STR_LIT> , SL , MemVT , ArgVal ) ; ArgVal = convertArgType ( DAG , VT , MemVT , SL , ArgVal , Signed , Arg ) ; return DAG . getMergeValues ( { ArgVal , Load . getValue ( <NUM_LIT> ) } , SL ) ; } SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , Offset ) ; SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Alignment , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;

SDValue Val = convertArgType ( DAG , VT , MemVT , SL , Load , Signed , Arg ) ;


<s> SDValue <ISA_LIT>TargetLowering :: lowerKernArgParameterPtr ( SelectionDAG & DAG , const SDLoc & SL , SDValue Chain , uint64_t Offset ) const { const DataLayout & DL = DAG . getDataLayout ( ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * InputPtrReg ; const TargetRegisterClass * RC ; LLT ArgTy ; std :: tie ( InputPtrReg , RC , ArgTy ) = Info -> getPreloadedValue ( <STR_LIT> ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ;

MVT PtrVT = getPointerTy ( DL , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ;


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasMultiDwordFlatScratchAddressing ( ) ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && Load -> isSimple ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = scalarizeVectorLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ;


<s> case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> : return lowerBUILD_VECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerFP_ROUND ( Op , DAG ) ; case <STR_LIT> : return lowerTRAP ( Op , DAG ) ; case <STR_LIT> : return lowerDEBUGTRAP ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return splitUnaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return lowerFMINNUM_FMAXNUM ( Op , DAG ) ; case <STR_LIT> : return splitTernaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return splitBinaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return lowerXMULO ( Op , DAG ) ; case <STR_LIT> : return LowerDYNAMIC_STACKALLOC ( Op , DAG ) ; } return SDValue ( ) ;


<s> } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ; unsigned LeftoverBitIdx = BitIndex % <NUM_LIT> ; SDLoc SL ( N ) ; SDValue Cast = DAG . getNode ( <STR_LIT> , SL , NewVT , Vec ) ; DCI . AddToWorklist ( Cast . getNode ( ) ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cast , DAG . getConstant ( EltIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Elt . getNode ( ) ) ; SDValue Srl = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Elt , DAG . getConstant ( LeftoverBitIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Srl . getNode ( ) ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , SL , EltVT . changeTypeToInteger ( ) , Srl ) ; DCI . AddToWorklist ( Trunc . getNode ( ) ) ; return DAG . getNode ( <STR_LIT> , SL , EltVT , Trunc ) ; } return SDValue ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return emitIndirectDst ( MI , * BB , * getSubtarget ( ) ) ; case <STR_LIT> : return splitKillBlock ( MI , BB ) ; case <STR_LIT> : { MachineRegisterInfo & MRI = BB -> getParent ( ) -> getRegInfo ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned SrcCond = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned DstLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned DstHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) , DstLo ) . addReg ( Src0 , <NUM_LIT> , <STR_LIT> ) . addReg ( Src1 , <NUM_LIT> , <STR_LIT> ) . addReg ( SrcCond ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) , DstHi ) . addReg ( Src0 , <NUM_LIT> , <STR_LIT> ) . addReg ( Src1 , <NUM_LIT> , <STR_LIT> ) . addReg ( SrcCond ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( DstLo ) . addImm ( <STR_LIT> ) . addReg ( DstHi ) . addImm ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; return BB ; } case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineInstr * Br = BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;

Br -> getOperand ( <NUM_LIT> ) . setIsUndef ( true ) ;


<s> unsigned Opc = TFL -> getStackGrowthDirection ( ) == TargetFrameLowering :: StackGrowsUp ? <STR_LIT> : <STR_LIT> ; SDValue ScaledSize = DAG . getNode ( <STR_LIT> , dl , VT , Size , DAG . getConstant ( ST . getWavefrontSizeLog2 ( ) , dl , <STR_LIT> ) ) ; unsigned StackAlign = TFL -> getStackAlignment ( ) ; Tmp1 = DAG . getNode ( Opc , dl , VT , SP , ScaledSize ) ; if ( Align > StackAlign ) {

Tmp1 = DAG . getNode ( <STR_LIT> , dl , VT , Tmp1 , DAG . getConstant ( - ( uint64_t ) Align << ST . getWavefrontSizeLog2 ( ) , dl , VT ) ) ;


<s> case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( MFI -> LDSSize ) ; MI . eraseFromParent ( ) ; return BB ; } case <STR_LIT> : return splitKillBlock ( MI , BB ) ;


<s> static bool isDwordAligned ( unsigned Alignment ) { return Alignment % <NUM_LIT> == <NUM_LIT> ;


<s> if ( MemVT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) return SDValue ( ) ; SDValue Chain = Load -> getChain ( ) ; SDValue BasePtr = Load -> getBasePtr ( ) ; MachineMemOperand * MMO = Load -> getMemOperand ( ) ; EVT RealMemVT = ( MemVT == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , RealMemVT , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned Alignment = Load -> getAlignment ( ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Alignment ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ;

if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) {


<s> SDValue <ISA_LIT>TargetLowering :: LowerLOAD ( SDValue Op , SelectionDAG & DAG ) const { SDLoc DL ( Op ) ; LoadSDNode * Load = cast < LoadSDNode > ( Op ) ; <STR_LIT> ExtType = Load -> getExtensionType ( ) ; EVT MemVT = Load -> getMemoryVT ( ) ; if ( ExtType == <STR_LIT> && MemVT . getSizeInBits ( ) < <NUM_LIT> ) { if ( MemVT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) return SDValue ( ) ; SDValue Chain = Load -> getChain ( ) ; SDValue BasePtr = Load -> getBasePtr ( ) ; MachineMemOperand * MMO = Load -> getMemOperand ( ) ; EVT RealMemVT = ( MemVT == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , RealMemVT , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ;

unsigned Alignment = Load -> getAlignment ( ) ;


<s> bool <ISA_LIT>TargetLowering :: allowsMisalignedMemoryAccesses ( EVT VT , unsigned AddrSpace , Align Alignment , MachineMemOperand :: Flags Flags , bool * IsFast ) const { bool Allow = allowsMisalignedMemoryAccessesImpl ( VT . getSizeInBits ( ) , AddrSpace , Alignment , Flags , IsFast ) ; if ( Allow && IsFast && Subtarget -> hasUnalignedDSAccessEnabled ( ) && ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ) { * IsFast = true ; }


<s> bool <ISA_LIT>TargetLowering :: allowsMisalignedMemoryAccesses ( EVT VT , unsigned AddrSpace , Align Alignment , MachineMemOperand :: Flags Flags , bool * IsFast ) const { bool Allow = allowsMisalignedMemoryAccessesImpl ( VT . getSizeInBits ( ) , AddrSpace , Alignment , Flags , IsFast ) ;

if ( Allow && IsFast && Subtarget -> hasUnalignedDSAccessEnabled ( ) && ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ) {


<s> case <NUM_LIT> : if ( ! Subtarget -> hasUsableDSOffset ( ) && Alignment < Align ( <NUM_LIT> ) ) return false ; RequiredAlignment = Align ( <NUM_LIT> ) ; if ( Subtarget -> hasUnalignedDSAccessEnabled ( ) ) { if ( IsFast ) * IsFast = true ; return true ; } break ; case <NUM_LIT> : if ( ! Subtarget -> hasDS96AndDS128 ( ) ) return false ; if ( Subtarget -> hasUnalignedDSAccessEnabled ( ) ) { if ( IsFast ) * IsFast = Alignment >= RequiredAlignment || Alignment < Align ( <NUM_LIT> ) ; return true ; } break ; case <NUM_LIT> : if ( ! Subtarget -> hasDS96AndDS128 ( ) || ! Subtarget -> useDS128 ( ) ) return false ; RequiredAlignment = Align ( <NUM_LIT> ) ; if ( Subtarget -> hasUnalignedDSAccessEnabled ( ) ) { if ( IsFast ) * IsFast = Alignment >= RequiredAlignment || Alignment < Align ( <NUM_LIT> ) ; return true ; } break ; default :


<s> } } MachineBasicBlock * Pre = ML -> getLoopPreheader ( ) ; MachineBasicBlock * Exit = ML -> getExitBlock ( ) ; if ( Pre && Exit ) { auto PreTerm = Pre -> getFirstTerminator ( ) ; if ( PreTerm == Pre -> begin ( ) || std :: prev ( PreTerm ) -> getOpcode ( ) != <STR_LIT> ) BuildMI ( * Pre , PreTerm , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; auto ExitHead = Exit -> getFirstNonDebugInstr ( ) ; if ( ExitHead == Exit -> end ( ) || ExitHead -> getOpcode ( ) != <STR_LIT> ) BuildMI ( * Exit , ExitHead , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> SDValue Idx = Op . getOperand ( <NUM_LIT> ) ; EVT VecVT = Vec . getValueType ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; EVT EltVT = VecVT . getVectorElementType ( ) ; DAGCombinerInfo DCI ( DAG , AfterLegalizeVectorOps , true , nullptr ) ; if ( SDValue Combined = performExtractVectorEltCombine ( Op . getNode ( ) , DCI ) ) return Combined ; if ( VecSize == <NUM_LIT> ) { SDValue Lo , Hi ; EVT LoVT , HiVT ; SDValue V2 = DAG . getBitcast ( <STR_LIT> , Vec ) ; std :: tie ( LoVT , HiVT ) = DAG . GetSplitDestVTs ( VecVT ) ; Lo = DAG . getBitcast ( LoVT , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , V2 , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ) ; Hi = DAG . getBitcast ( HiVT , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , V2 , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ) ; EVT IdxVT = Idx . getValueType ( ) ; unsigned NElem = VecVT . getVectorNumElements ( ) ; assert ( isPowerOf2_32 ( NElem ) ) ; SDValue IdxMask = DAG . getConstant ( NElem / <NUM_LIT> - <NUM_LIT> , SL , IdxVT ) ; SDValue NewIdx = DAG . getNode ( <STR_LIT> , SL , IdxVT , Idx , IdxMask ) ; SDValue Half = DAG . getSelectCC ( SL , Idx , IdxMask , Hi , Lo , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , SL , EltVT , Half , NewIdx ) ; } assert ( VecSize <= <NUM_LIT> ) ; MVT IntVT = <STR_LIT> ( VecSize ) ; SDValue VecBC = peekThroughBitcasts ( Vec ) ; if ( VecBC . getOpcode ( ) == <STR_LIT> ) { SDValue Src = VecBC . getOperand ( <NUM_LIT> ) ; Src = DAG . getBitcast ( Src . getValueType ( ) . changeTypeToInteger ( ) , Src ) ; Vec = DAG . getAnyExtOrTrunc ( Src , SL , IntVT ) ; } unsigned EltSize = EltVT . getSizeInBits ( ) ; assert ( isPowerOf2_32 ( EltSize ) ) ; SDValue ScaleFactor = DAG . getConstant ( Log2_32 ( EltSize ) , SL , <STR_LIT> ) ; SDValue ScaledIdx = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Idx , ScaleFactor ) ; SDValue BC = DAG . getNode ( <STR_LIT> , SL , IntVT , Vec ) ;

SDValue Elt = DAG . getNode ( <STR_LIT> , SL , IntVT , BC , ScaledIdx ) ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerOperation ( SDValue Op , SelectionDAG & DAG ) const { switch ( Op . getOpcode ( ) ) { default : return <ISA_LIT>TargetLowering :: LowerOperation ( Op , DAG ) ; case <STR_LIT> : return LowerBRCOND ( Op , DAG ) ; case <STR_LIT> : return LowerRETURNADDR ( Op , DAG ) ; case <STR_LIT> : { SDValue Result = LowerLOAD ( Op , DAG ) ; assert ( ( ! Result . getNode ( ) || Result . getNode ( ) -> getNumValues ( ) == <NUM_LIT> ) && <STR_LIT> ) ; return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> :

return lowerSCALAR_TO_VECTOR ( Op , DAG ) ;


<s> EVT ResultVT = Op . getValueType ( ) ; EVT SValVT = SVal . getValueType ( ) ; SDValue UndefVal = DAG . getUNDEF ( SValVT ) ; SDLoc SL ( Op ) ;

SmallVector < SDValue , <NUM_LIT> > VElts ;


<s> unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; if ( ! allowsMemoryAccessForAlignment ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) return expandUnalignedStore ( Store , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || ( NumElements == <NUM_LIT> && ! Subtarget -> enableFlatScratch ( ) ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) {


<s> SDLoc SL ( Op ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; unsigned MaxID = Subtarget -> getMaxWorkitemID ( MF . getFunction ( ) , Dim ) ; if ( MaxID == <NUM_LIT> ) return DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue Val = loadInputValue ( DAG , & <STR_LIT> , <STR_LIT> , SDLoc ( DAG . getEntryNode ( ) ) , Arg ) ; if ( Arg . isMasked ( ) ) return Val ;

EVT SmallVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , <NUM_LIT> - countLeadingZeros ( MaxID ) ) ;


<s> if ( MaxID == <NUM_LIT> ) return DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue Val = loadInputValue ( DAG , & <STR_LIT> , <STR_LIT> , SDLoc ( DAG . getEntryNode ( ) ) , Arg ) ; if ( Arg . isMasked ( ) ) return Val ; EVT SmallVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , <NUM_LIT> - countLeadingZeros ( MaxID ) ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Val , DAG . getValueType ( SmallVT ) ) ;


<s> } if ( SDValue V = reassociateScalarOps ( N , DAG ) ) { return V ; } if ( VT != <STR_LIT> || ! DCI . isAfterLegalizeDAG ( ) ) return SDValue ( ) ; unsigned Opc = LHS . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) std :: swap ( RHS , LHS ) ; Opc = RHS . getOpcode ( ) ; switch ( Opc ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { auto Cond = RHS . getOperand ( <NUM_LIT> ) ; if ( ! isBoolSGPR ( Cond ) ) break ; SDVTList VTList = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue Args [ ] = { LHS , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) , Cond } ; Opc = ( Opc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( Opc , SL , VTList , Args ) ; } case <STR_LIT> : {

auto C = dyn_cast < ConstantSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) ;


<s> TargetLowering :: AtomicExpansionKind <ISA_LIT>TargetLowering :: shouldExpandAtomicCmpXchgInIR ( AtomicCmpXchgInst * CmpX ) const {

return CmpX -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ? AtomicExpansionKind :: NotAtomic : AtomicExpansionKind :: None ;


<s> TargetLowering :: AtomicExpansionKind <ISA_LIT>TargetLowering :: shouldExpandAtomicLoadInIR ( LoadInst * LI ) const { return LI -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ? AtomicExpansionKind :: NotAtomic : AtomicExpansionKind :: None ;


<s> return LI -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ? AtomicExpansionKind :: NotAtomic : AtomicExpansionKind :: None ;


<s> if ( Subtarget -> hasGFX940Insts ( ) ) return AtomicExpansionKind :: None ; if ( RMW -> getFunction ( ) -> getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) != <STR_LIT> ) return AtomicExpansionKind :: CmpXChg ; if ( Subtarget -> hasGFX90AInsts ( ) ) { if ( Ty -> isFloatTy ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return AtomicExpansionKind :: CmpXChg ; auto S<ISA_LIT>D = RMW -> getSyncScopeID ( ) ; if ( S<ISA_LIT>D == SyncScope :: System || S<ISA_LIT>D == RMW -> getContext ( ) . getOrInsertSyncScopeID ( <STR_LIT> ) ) return AtomicExpansionKind :: CmpXChg ; return ReportUnsafeHWInst ( AtomicExpansionKind :: None ) ; } if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return AtomicExpansionKind :: CmpXChg ; return RMW -> use_empty ( ) ? ReportUnsafeHWInst ( AtomicExpansionKind :: None ) : AtomicExpansionKind :: CmpXChg ; } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && Subtarget -> hasLDSFPAtomicAdd ( ) ) { if ( ! Ty -> isDoubleTy ( ) ) return AtomicExpansionKind :: None ; if ( fpModeMatchesGlobalFPAtomicMode ( RMW ) ) return AtomicExpansionKind :: None ; return RMW -> getFunction ( ) -> getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) == <STR_LIT> ? ReportUnsafeHWInst ( AtomicExpansionKind :: None ) : AtomicExpansionKind :: CmpXChg ; } return AtomicExpansionKind :: CmpXChg ; } default : break ;


<s> TargetLowering :: AtomicExpansionKind <ISA_LIT>TargetLowering :: shouldExpandAtomicStoreInIR ( StoreInst * <ISA_LIT> ) const { return <ISA_LIT> -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ? AtomicExpansionKind :: NotAtomic : AtomicExpansionKind :: None ;


<s> if ( Use -> getOpcode ( ) != <STR_LIT> ) return SDValue ( ) ; ++ NumUsers ; if ( NumUsers >= <NUM_LIT> ) return SDValue ( ) ; } } SDValue MulLHS = LHS . getOperand ( <NUM_LIT> ) ; SDValue MulRHS = LHS . getOperand ( <NUM_LIT> ) ; SDValue AddRHS = RHS ; bool MulLHSUnsigned32 = numBitsUnsigned ( MulLHS , DAG ) <= <NUM_LIT> ; bool MulRHSUnsigned32 = numBitsUnsigned ( MulRHS , DAG ) <= <NUM_LIT> ; bool MulSignedLo = false ; if ( ! MulLHSUnsigned32 || ! MulRHSUnsigned32 ) { MulSignedLo = numBitsSigned ( MulLHS , DAG ) <= <NUM_LIT> && numBitsSigned ( MulRHS , DAG ) <= <NUM_LIT> ; } if ( VT != <STR_LIT> ) { MulLHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHS ) ; MulRHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulRHS ) ; AddRHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , AddRHS ) ; } SDValue Zero = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue One = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; auto MulLHSLo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHS ) ; auto MulRHSLo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulRHS ) ; SDValue Accum = getMad64_32 ( DAG , SL , <STR_LIT> , MulLHSLo , MulRHSLo , AddRHS , MulSignedLo ) ; if ( ! MulSignedLo && ( ! MulLHSUnsigned32 || ! MulRHSUnsigned32 ) ) { auto AccumLo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Accum , Zero ) ; auto AccumHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Accum , One ) ; if ( ! MulLHSUnsigned32 ) { auto MulLHSHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHS , One ) ; SDValue MulHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHSHi , MulRHSLo ) ; AccumHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulHi , AccumHi ) ; } if ( ! MulRHSUnsigned32 ) { auto MulRHSHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulRHS , One ) ; SDValue MulHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHSLo , MulRHSHi ) ; AccumHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulHi , AccumHi ) ; } Accum = DAG . getBuildVector ( <STR_LIT> , SL , { AccumLo , AccumHi } ) ; Accum = DAG . getBitcast ( <STR_LIT> , Accum ) ;


<s> MulLHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHS ) ; MulRHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulRHS ) ; AddRHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , AddRHS ) ; } SDValue Zero = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue One = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; auto MulLHSLo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHS ) ; auto MulRHSLo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulRHS ) ; SDValue Accum = getMad64_32 ( DAG , SL , <STR_LIT> , MulLHSLo , MulRHSLo , AddRHS , MulSignedLo ) ; if ( ! MulSignedLo && ( ! MulLHSUnsigned32 || ! MulRHSUnsigned32 ) ) { auto AccumLo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Accum , Zero ) ; auto AccumHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Accum , One ) ; if ( ! MulLHSUnsigned32 ) { auto MulLHSHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHS , One ) ; SDValue MulHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulLHSHi , MulRHSLo ) ;

AccumHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , MulHi , AccumHi ) ;


<s> return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> :

if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ;


<s> StoreSDNode * Store = cast < StoreSDNode > ( Op ) ; EVT VT = Store -> getMemoryVT ( ) ; if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } unsigned AS = Store -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Store -> getAlignment ( ) < VT . getStoreSize ( ) && VT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> :


<s> return false ; } bool Allow = allowsMisalignedMemoryAccessesImpl ( VT . getSizeInBits ( ) , AddrSpace , Alignment , Flags , IsFast ) ; if ( Allow && IsFast && Subtarget -> hasUnalignedDSAccessEnabled ( ) && ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ) {


<s> if ( ! Subtarget -> hasUnalignedDSAccessEnabled ( ) && Alignment < Align ( <NUM_LIT> ) ) return false ; Align RequiredAlignment ( PowerOf2Ceil ( Size / <NUM_LIT> ) ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && Size > <NUM_LIT> && Alignment < RequiredAlignment ) return false ; switch ( Size ) { case <NUM_LIT> : if ( ! Subtarget -> hasUsableDSOffset ( ) && Alignment < Align ( <NUM_LIT> ) ) return false ; RequiredAlignment = Align ( <NUM_LIT> ) ; break ; case <NUM_LIT> : if ( ! Subtarget -> hasDS96AndDS128 ( ) ) return false ; if ( Subtarget -> hasUnalignedDSAccessEnabled ( ) ) { if ( IsFast ) * IsFast = Alignment >= RequiredAlignment || Alignment < Align ( <NUM_LIT> ) ; return true ; } break ; case <NUM_LIT> : if ( ! Subtarget -> hasDS96AndDS128 ( ) || ! Subtarget -> useDS128 ( ) ) return false ; RequiredAlignment = Align ( <NUM_LIT> ) ; if ( Subtarget -> hasUnalignedDSAccessEnabled ( ) ) { if ( IsFast ) * IsFast = Alignment >= RequiredAlignment || Alignment < Align ( <NUM_LIT> ) ; return true ; } break ; default : if ( Size > <NUM_LIT> ) return false ; break ; } if ( IsFast ) { * IsFast = Alignment >= RequiredAlignment || ( Subtarget -> hasUnalignedDSAccessEnabled ( ) && Size < <NUM_LIT> ) ; } return Alignment >= RequiredAlignment || Subtarget -> hasUnalignedDSAccessEnabled ( ) ; } if ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 || Subtarget -> enableFlatScratch ( ) || Subtarget -> hasUnalignedScratchAccess ( ) ; } if ( AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasUnalignedScratchAccess ( ) ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccessEnabled ( ) ) {


<s> case <STR_LIT> : case <STR_LIT> : return Op . getValueType ( ) . getScalarType ( ) != <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ST -> supportsMinMaxDenormModes ( ) && DAG . isKnownNeverNaN ( Op . getOperand ( <NUM_LIT> ) ) && DAG . isKnownNeverNaN ( Op . getOperand ( <NUM_LIT> ) ) ) return true ; return ( MaxDepth > <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , ST , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , ST , MaxDepth - <NUM_LIT> ) ;


<s> ConstantFPSDNode * CFP = isConstOrConstSplatFP ( N0 ) ; if ( ! CFP ) { SDValue N0 = N -> getOperand ( <NUM_LIT> ) ; EVT VT = N0 . getValueType ( ) . getScalarType ( ) ; auto ST = getSubtarget ( ) ; if ( ( ( VT == <STR_LIT> && ST -> hasFP32Denormals ( ) ) || ( VT == <STR_LIT> && ST -> hasFP64Denormals ( ) ) || ( VT == <STR_LIT> && ST -> hasFP16Denormals ( ) ) ) && DAG . isKnownNeverNaN ( N0 ) ) return N0 ; bool IsIEEEMode = Subtarget -> enableIEEEBit ( DAG . getMachineFunction ( ) ) ; if ( ( IsIEEEMode || isKnownNeverSNan ( DAG , N0 ) ) && isCanonicalized ( DAG , N0 , ST ) ) return N0 ; return SDValue ( ) ; } const APFloat & C = CFP -> getValueAPF ( ) ; if ( C . isDenormal ( ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT SVT = VT . getScalarType ( ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP64Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; } if ( C . isNaN ( ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; APFloat CanonicalQNaN = APFloat :: getQNaN ( C . getSemantics ( ) ) ; if ( C . isSignaling ( ) ) { return DAG . getConstantFP ( CanonicalQNaN , SDLoc ( N ) , VT ) ; } if ( C . bitcastToAPInt ( ) != CanonicalQNaN . bitcastToAPInt ( ) ) return DAG . getConstantFP ( CanonicalQNaN , SDLoc ( N ) , VT ) ; } return N0 ;


<s> const APFloat & C = CFP -> getValueAPF ( ) ; if ( C . isDenormal ( ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT SVT = VT . getScalarType ( ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP64Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; } if ( C . isNaN ( ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; APFloat CanonicalQNaN = APFloat :: getQNaN ( C . getSemantics ( ) ) ; if ( C . isSignaling ( ) ) { return DAG . getConstantFP ( CanonicalQNaN , SDLoc ( N ) , VT ) ; } if ( C . bitcastToAPInt ( ) != CanonicalQNaN . bitcastToAPInt ( ) ) return DAG . getConstantFP ( CanonicalQNaN , SDLoc ( N ) , VT ) ; }


<s> if ( CallConv == CallingConv :: <ISA_LIT>_PS && ! Arg -> Flags . isInReg ( ) && ! Arg -> Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { bool SkipArg = ! Arg -> Used && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( Arg -> Flags . isSplit ( ) ) { while ( ! Arg -> Flags . isSplitEnd ( ) ) { assert ( ! Arg -> VT . isVector ( ) && <STR_LIT> ) ; if ( ! SkipArg ) Splits . push_back ( * Arg ) ; Arg = & Ins [ ++ I ] ;


<s> unsigned RegNo = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <STR_LIT> : <STR_LIT> ; return CreateLiveInRegister ( DAG , & <STR_LIT> , RegNo , <STR_LIT> ) ; } SDLoc SL ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ;

SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ;


<s> switch ( AS ) { case <ISA_LIT>AS :: GLOBAL_ADDRESS : if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { return isLegalFlatAddressingMode ( AM ) ; } return isLegalMUBUFAddressingMode ( AM ) ; case <ISA_LIT>AS :: CONSTANT_ADDRESS : if ( AM . BaseOffs % <NUM_LIT> != <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( DL . getTypeStoreSize ( Ty ) < <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) {


<s> case <ISA_LIT>AS :: CONSTANT_ADDRESS : if ( AM . BaseOffs % <NUM_LIT> != <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( DL . getTypeStoreSize ( Ty ) < <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale == <NUM_LIT> ) return true ;

if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> : return performFMed3Combine ( N , DCI ) ; case <STR_LIT> : return performCvtPkRTZCombine ( N , DCI ) ; }

return <ISA_LIT>TargetLowering :: PerformDAGCombine ( N , DCI ) ;


<s> if ( K1 -> isExactlyValue ( <NUM_LIT> ) && K0 -> isExactlyValue ( <NUM_LIT> ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op0 . getOperand ( <NUM_LIT> ) ) ; } if ( VT == <STR_LIT> || VT == <STR_LIT> ) return SDValue ( ) ; SDValue Var = Op0 . getOperand ( <NUM_LIT> ) ; if ( ! isKnownNeverSNan ( DAG , Var ) ) return SDValue ( ) ; return DAG . getNode ( <STR_LIT> , SL , K0 -> getValueType ( <NUM_LIT> ) , Var , SDValue ( K0 , <NUM_LIT> ) , SDValue ( K1 , <NUM_LIT> ) ) ;


<s> ConstantFPSDNode * K1 = dyn_cast < ConstantFPSDNode > ( Op1 ) ; if ( ! K1 ) return SDValue ( ) ; ConstantFPSDNode * K0 = dyn_cast < ConstantFPSDNode > ( Op0 . getOperand ( <NUM_LIT> ) ) ; if ( ! K0 ) return SDValue ( ) ; APFloat :: cmpResult Cmp = K0 -> getValueAPF ( ) . compare ( K1 -> getValueAPF ( ) ) ; if ( Cmp == APFloat :: cmpGreaterThan ) return SDValue ( ) ;

EVT VT = K0 -> getValueType ( <NUM_LIT> ) ;


<s> if ( IsFast ) * IsFast = Aligned ; return Aligned ; } } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccessEnabled ( ) && ! ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ) { if ( IsFast ) {

* IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? Alignment >= Align ( <NUM_LIT> ) : Alignment != Align ( <NUM_LIT> ) ;


<s> bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccessEnabled ( ) && ! ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? Alignment >= Align ( <NUM_LIT> ) : Alignment != Align ( <NUM_LIT> ) ; } return true ; } if ( Size < <NUM_LIT> ) return false ;


<s> bool <ISA_LIT>TargetLowering :: allowsMisalignedMemoryAccessesImpl ( unsigned Size , unsigned AddrSpace , Align Alignment , MachineMemOperand :: Flags Flags , bool * IsFast ) const { if ( IsFast ) * IsFast = false ; if ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ;


<s> if ( ! ( Op1 -> isDivergent ( ) ^ Op2 -> isDivergent ( ) ) ) return SDValue ( ) ; if ( Op1 -> isDivergent ( ) ) std :: swap ( Op1 , Op2 ) ; if ( isa < ConstantSDNode > ( Op0 ) || isa < ConstantSDNode > ( Op1 ) ) return SDValue ( ) ; SDLoc SL ( N ) ; SDValue Add1 = DAG . getNode ( Opc , SL , VT , Op0 , Op1 ) ;

return DAG . getNode ( Opc , SL , VT , Add1 , Op2 ) ;


<s> return DAG . getNode ( <STR_LIT> + ShiftOffset / <NUM_LIT> , SL , <STR_LIT> , Shift ) ; } } } const TargetLowering & TLI = DAG . getTargetLoweringInfo ( ) ; APInt DemandedBits = APInt :: getBitsSet ( <NUM_LIT> , <NUM_LIT> * Offset , <NUM_LIT> * Offset + <NUM_LIT> ) ; if ( TLI . SimplifyDemandedBits ( Src , DemandedBits , DCI ) ) return SDValue ( N , <NUM_LIT> ) ;

if ( SDValue DemandedSrc = TLI . SimplifyMultipleUseDemandedBits ( Src , DemandedBits , DAG ) ) return DAG . getNode ( N -> getOpcode ( ) , SL , <STR_LIT> , DemandedSrc ) ;


<s> if ( S<ISA_LIT>D == SyncScope :: System || S<ISA_LIT>D == RMW -> getContext ( ) . getOrInsertSyncScopeID ( <STR_LIT> ) ) return AtomicExpansionKind :: CmpXChg ; return ReportUnsafeHWInst ( AtomicExpansionKind :: None ) ; } if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return AtomicExpansionKind :: CmpXChg ; return RMW -> use_empty ( ) ? ReportUnsafeHWInst ( AtomicExpansionKind :: None ) : AtomicExpansionKind :: CmpXChg ; } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && Subtarget -> hasLDSFPAtomics ( ) ) { if ( ! Ty -> isDoubleTy ( ) ) return AtomicExpansionKind :: None ; if ( fpModeMatchesGlobalFPAtomicMode ( RMW ) ) return AtomicExpansionKind :: None ; return RMW -> getFunction ( ) -> getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) == <STR_LIT> ? ReportUnsafeHWInst ( AtomicExpansionKind :: None ) : AtomicExpansionKind :: CmpXChg ; } return AtomicExpansionKind :: CmpXChg ;


<s> } break ; case 'v' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; } break ; case 'a' : if ( ! Subtarget -> hasMAIInsts ( ) ) break ; switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; return std :: make_pair ( <NUM_LIT> , RC ) ; } break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ;


<s> if ( Size == <NUM_LIT> ) { bool Aligned = Alignment >= Align ( ( Subtarget -> hasUnalignedDSAccess ( ) && ! Subtarget -> hasLDSMisalignedBug ( ) ) ? <NUM_LIT> : <NUM_LIT> ) ; if ( IsFast ) * IsFast = Aligned ; return Aligned ; } } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccessEnabled ( ) && ! ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? Alignment >= Align ( <NUM_LIT> ) : Alignment != Align ( <NUM_LIT> ) ; } return true ; } if ( Size < <NUM_LIT> ) return false ;


<s> Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } TexFail = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , SDValue ( Result , <NUM_LIT> ) , DAG . getConstant ( MaskPopDwords , DL , <STR_LIT> ) ) ; } if ( DataDwordVT . isVector ( ) ) Data = padEltsToUndef ( DAG , DL , DataDwordVT , Data , NumDataDwords - MaskPopDwords ) ; if ( IsD16 ) Data = adjustLoadValueTypeImpl ( Data , ReqRetVT , DL , DAG , Unpacked ) ; EVT LegalReqRetVT = ReqRetVT ; if ( ! ReqRetVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; } else { if ( ( ReqRetVT . getVectorNumElements ( ) % <NUM_LIT> ) == <NUM_LIT> ) {

LegalReqRetVT = EVT :: getVectorVT ( * DAG . getContext ( ) , ReqRetVT . getVectorElementType ( ) , ReqRetVT . getVectorNumElements ( ) + <NUM_LIT> ) ;


<s> Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol || ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOVolatile ; return true ; }


<s> Info . align . reset ( ) ; } else { Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( RsrcIntr -> RsrcArg ) ) ; } Info . flags = MachineMemOperand :: MODereferenceable ; if ( Attr . hasFnAttribute ( Attribute :: ReadOnly ) ) { unsigned DMaskLanes = <NUM_LIT> ; if ( RsrcIntr -> IsImage ) { const <STR_LIT> * Intr = <STR_LIT> ( IntrID ) ; const <STR_LIT> * BaseOpcode = <STR_LIT> ( Intr -> BaseOpcode ) ; if ( ! BaseOpcode -> Gather4 ) { unsigned DMask = cast < ConstantInt > ( CI . getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; DMaskLanes = DMask == <NUM_LIT> ? <NUM_LIT> : countPopulation ( DMask ) ; } Info . memVT = memVTFromImageReturn ( CI . getType ( ) , DMaskLanes ) ; } else Info . memVT = EVT :: getEVT ( CI . getType ( ) ) ; Info . opc = <STR_LIT> ; Info . flags |= MachineMemOperand :: MOLoad ; } else if ( Attr . hasFnAttribute ( Attribute :: WriteOnly ) ) { Info . opc = <STR_LIT> ; Type * DataTy = CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ; if ( RsrcIntr -> IsImage ) { unsigned DMask = cast < ConstantInt > ( CI . getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; unsigned DMaskLanes = DMask == <NUM_LIT> ? <NUM_LIT> : countPopulation ( DMask ) ; Info . memVT = memVTFromImageData ( DataTy , DMaskLanes ) ; } else Info . memVT = EVT :: getEVT ( DataTy ) ; Info . flags |= MachineMemOperand :: MOStore ; } else { Info . opc = CI . getType ( ) -> isVoidTy ( ) ? <STR_LIT> : <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ;


<s> unsigned <ISA_LIT>TargetLowering :: getNumRegistersForCallingConv ( LLVMContext & Context , CallingConv :: ID CC , EVT VT ) const { if ( CC != CallingConv :: <ISA_LIT>_KERNEL && VT . isVector ( ) && VT . getVectorNumElements ( ) == <NUM_LIT> ) {


<s> MVT <ISA_LIT>TargetLowering :: getRegisterTypeForCallingConv ( LLVMContext & Context , CallingConv :: ID CC , EVT VT ) const {

if ( CC != CallingConv :: <ISA_LIT>_KERNEL && VT . isVector ( ) && VT . getVectorNumElements ( ) == <NUM_LIT> ) {


<s> } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; if ( HasStackObjects ) { unsigned ScratchWaveOffsetReg = TRI . getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET ) ; Info . setScratchWaveOffsetReg ( ScratchWaveOffsetReg ) ;


<s> SDNode * M0 = DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , <STR_LIT> , V , Chain ) ;


<s> SDValue <ISA_LIT>TargetLowering :: getSegmentAperture ( unsigned AS , SelectionDAG & DAG ) const { SDLoc SL ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , Info -> getQueuePtrUserSGPR ( ) , <STR_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , QueuePtr , DAG . getConstant ( StructOffset , SL , <STR_LIT> ) ) ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ; return DAG . getLoad ( <STR_LIT> , SL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr , PtrInfo , false , false , true , MinAlign ( <NUM_LIT> , StructOffset ) ) ;


<s> SDValue <ISA_LIT>TargetLowering :: getSegmentAperture ( unsigned AS , SelectionDAG & DAG ) const { SDLoc SL ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , Info -> getQueuePtrUserSGPR ( ) , <STR_LIT> ) ;


<s> SDValue NewOld = DAG . getBuildVector ( VecType , DL , { New , Old } ) ; SDValue Ops [ ] = { ChainIn , Addr , NewOld } ; SDVTList VTList = DAG . getVTList ( VT , <STR_LIT> ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , VT , AtomicNode -> getMemOperand ( ) ) ;


<s> return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ;

<ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ;


<s> NewOut . PartOffset += NewOut . VT . getStoreSize ( ) ; } } else { SplitVals . push_back ( OutVals [ i ] ) ; Splits . push_back ( Out ) ; } } SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; AnalyzeReturn ( CCInfo , Splits ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; for ( unsigned i = <NUM_LIT> , realRVLocIdx = <NUM_LIT> ; i != RVLocs . size ( ) ; ++ i , ++ realRVLocIdx ) { CCValAssign & VA = RVLocs [ i ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = SplitVals [ realRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; } Chain = DAG . getCopyToReg ( Chain , DL , VA . getLocReg ( ) , Arg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( DAG . getRegister ( VA . getLocReg ( ) , VA . getLocVT ( ) ) ) ; } RetOps [ <NUM_LIT> ] = Chain ; if ( Flag . getNode ( ) ) RetOps . push_back ( Flag ) ;

return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , RetOps ) ;


<s> const <STR_LIT> & Out = Outs [ i ] ; if ( Out . VT . isVector ( ) ) { MVT VT = Out . VT . getVectorElementType ( ) ; <STR_LIT> NewOut = Out ; NewOut . Flags . setSplit ( ) ; NewOut . VT = VT ; unsigned NumElements = Out . ArgVT . getVectorNumElements ( ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { SDValue Elem = DAG . getNode ( <STR_LIT> , DL , VT , OutVals [ i ] , DAG . getConstant ( j , DL , <STR_LIT> ) ) ; SplitVals . push_back ( Elem ) ; Splits . push_back ( NewOut ) ; NewOut . PartOffset += NewOut . VT . getStoreSize ( ) ; } } else { SplitVals . push_back ( OutVals [ i ] ) ; Splits . push_back ( Out ) ; } } SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; AnalyzeReturn ( CCInfo , Splits ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; for ( unsigned i = <NUM_LIT> , realRVLocIdx = <NUM_LIT> ; i != RVLocs . size ( ) ; ++ i , ++ realRVLocIdx ) { CCValAssign & VA = RVLocs [ i ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = SplitVals [ realRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; } Chain = DAG . getCopyToReg ( Chain , DL , VA . getLocReg ( ) , Arg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( DAG . getRegister ( VA . getLocReg ( ) , VA . getLocVT ( ) ) ) ; }


<s> setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; for ( MVT VT : { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ) { for ( unsigned Op = <NUM_LIT> ; Op < <STR_LIT> ; ++ Op ) { switch ( Op ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; case <STR_LIT> : setOperationAction ( Op , VT , Custom ) ; break ; default : setOperationAction ( Op , VT , Expand ) ; break ; } } } for ( MVT Vec64 : { <STR_LIT> , <STR_LIT> } ) { setOperationAction ( <STR_LIT> , Vec64 , Promote ) ; AddPromotedToType ( <STR_LIT> , Vec64 , <STR_LIT> ) ; setOperationAction ( <STR_LIT> , Vec64 , Promote ) ; AddPromotedToType ( <STR_LIT> , Vec64 , <STR_LIT> ) ; setOperationAction ( <STR_LIT> , Vec64 , Promote ) ; AddPromotedToType ( <STR_LIT> , Vec64 , <STR_LIT> ) ; setOperationAction ( <STR_LIT> , Vec64 , Promote ) ; AddPromotedToType ( <STR_LIT> , Vec64 , <STR_LIT> ) ; } setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; if ( Subtarget -> hasFlatAddressSpace ( ) ) { setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; } setOperationAction ( <STR_LIT> , <STR_LIT> , Legal ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Legal ) ;