<s> return DAG . getNode ( <STR_LIT> + ShiftOffset / <NUM_LIT> , SL , <STR_LIT> , Shift ) ; } } } const TargetLowering & TLI = DAG . getTargetLoweringInfo ( ) ; APInt DemandedBits = APInt :: getBitsSet ( <NUM_LIT> , <NUM_LIT> * Offset , <NUM_LIT> * Offset + <NUM_LIT> ) ; if ( TLI . SimplifyDemandedBits ( Src , DemandedBits , DCI ) ) return SDValue ( N , <NUM_LIT> ) ; if ( SDValue DemandedSrc = TLI . SimplifyMultipleUseDemandedBits ( Src , DemandedBits , DAG ) ) return DAG . getNode ( N ->

getOpcode ( ) , SL , <STR_LIT> , DemandedSrc ) ;


<s> if ( Size == <NUM_LIT> ) { bool Aligned = Alignment >= Align ( ( Subtarget -> hasUnalignedDSAccess ( ) && ! Subtarget -> hasLDSMisalignedBug ( ) ) ? <NUM_LIT> : <NUM_LIT> ) ; if ( IsFast ) * IsFast = Aligned ; return Aligned ; } } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Subtarget -> hasUnalignedBufferAccessEnabled ( ) && ! ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? Alignment >= Align ( <NUM_LIT> ) : Alignment != Align ( <NUM_LIT> ) ; } return true ; } if ( Size < <NUM_LIT> ) return false ; if ( IsFast ) * IsFast


<s> Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } TexFail = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , SDValue ( Result , <NUM_LIT> ) , DAG . getConstant ( MaskPopDwords , DL , <STR_LIT> ) ) ; } if ( DataDwordVT . isVector ( ) ) Data = padEltsToUndef ( DAG , DL , DataDwordVT , Data , NumDataDwords - MaskPopDwords ) ; if ( IsD16 ) Data = adjustLoadValueTypeImpl ( Data , ReqRetVT , DL , DAG , Unpacked ) ; EVT LegalReqRetVT = ReqRetVT ; if ( ! ReqRetVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , ReqRetVT . changeTypeToInteger ( ) , Data ) ; } else { if ( ( ReqRetVT . getVectorNumElements ( ) % <NUM_LIT> ) == <NUM_LIT> ) { LegalReqRetVT = EVT :: getVectorVT ( * DAG . getContext ( ) , ReqRetVT

. getVectorElementType ( ) , ReqRetVT . getVectorNumElements ( ) + <NUM_LIT> ) ;


<s> Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol || ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT>


<s> unsigned <ISA_LIT>TargetLowering :: getNumRegistersForCallingConv ( LLVMContext & Context , CallingConv :: ID CC , EVT VT ) const { if ( CC != CallingConv :: <ISA_LIT>_KERNEL && VT . isVector ( ) && VT . getVectorNumElements ( ) == <NUM_LIT> ) { EVT ScalarVT = VT . getScalarType (


<s> MVT <ISA_LIT>TargetLowering :: getRegisterTypeForCallingConv ( LLVMContext & Context , CallingConv :: ID CC , EVT VT ) const { if ( CC != CallingConv :: <ISA_LIT>_KERNEL && VT . isVector (

) && VT . getVectorNumElements ( ) == <NUM_LIT> ) {


<s> } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; if ( HasStackObjects ) { unsigned ScratchWaveOffsetReg = TRI . getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET ) ; Info . setScratchWaveOffsetReg ( ScratchWaveOffsetReg ) ; } else


<s> SDNode * M0 = DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , <STR_LIT> , V , Chain ) ; return SDValue ( M0 ,


<s> SDValue <ISA_LIT>TargetLowering :: getSegmentAperture ( unsigned AS , SelectionDAG & DAG ) const { SDLoc SL ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , Info -> getQueuePtrUserSGPR ( ) , <STR_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS


<s> return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo <


<s> NewOut . PartOffset += NewOut . VT . getStoreSize ( ) ; } } else { SplitVals . push_back ( OutVals [ i ] ) ; Splits . push_back ( Out ) ; } } SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; AnalyzeReturn ( CCInfo , Splits ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; for ( unsigned i = <NUM_LIT> , realRVLocIdx = <NUM_LIT> ; i != RVLocs . size ( ) ; ++ i , ++ realRVLocIdx ) { CCValAssign & VA = RVLocs [ i ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = SplitVals [ realRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; } Chain = DAG . getCopyToReg ( Chain , DL , VA . getLocReg ( ) , Arg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( DAG . getRegister ( VA . getLocReg ( ) , VA . getLocVT ( ) ) ) ; } RetOps [ <NUM_LIT> ] = Chain ; if ( Flag . getNode ( ) ) RetOps . push_back ( Flag ) ; return DAG . getNode ( <STR_LIT> , DL


<s> const <STR_LIT> & Out = Outs [ i ] ; if ( Out . VT . isVector ( ) ) { MVT VT = Out . VT . getVectorElementType ( ) ; <STR_LIT> NewOut = Out ; NewOut . Flags . setSplit ( ) ; NewOut . VT = VT ; unsigned NumElements = Out . ArgVT . getVectorNumElements ( ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { SDValue Elem = DAG . getNode ( <STR_LIT> , DL , VT , OutVals [ i ] , DAG . getConstant ( j , DL , <STR_LIT> ) ) ; SplitVals . push_back ( Elem ) ; Splits . push_back ( NewOut ) ; NewOut . PartOffset += NewOut . VT . getStoreSize ( ) ; } } else { SplitVals . push_back ( OutVals [ i ] ) ; Splits . push_back ( Out ) ; } } SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; AnalyzeReturn ( CCInfo , Splits ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; for ( unsigned i = <NUM_LIT> , realRVLocIdx = <NUM_LIT> ; i != RVLocs . size ( ) ; ++ i , ++ realRVLocIdx ) { CCValAssign & VA = RVLocs [ i ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = SplitVals [ realRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; } Chain = DAG . getCopyToReg ( Chain , DL , VA . getLocReg ( ) , Arg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( DAG . getRegister ( VA . getLocReg ( ) , VA . getLocVT ( ) ) ) ; } RetOps [ <NUM_LIT> ]


<s> setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; for ( MVT VT : { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ) { for ( unsigned Op = <NUM_LIT> ; Op < <STR_LIT> ; ++ Op ) { switch ( Op ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; case <STR_LIT> : setOperationAction ( Op , VT , Custom ) ; break ; default : setOperationAction ( Op , VT , Expand ) ; break ; } } } for ( MVT Vec64 : { <STR_LIT> , <STR_LIT> } ) { setOperationAction ( <STR_LIT> , Vec64 , Promote ) ; AddPromotedToType ( <STR_LIT> , Vec64 , <STR_LIT> ) ; setOperationAction ( <STR_LIT> , Vec64 , Promote ) ; AddPromotedToType ( <STR_LIT> , Vec64 , <STR_LIT> ) ; setOperationAction ( <STR_LIT> , Vec64 , Promote ) ; AddPromotedToType ( <STR_LIT> , Vec64 , <STR_LIT> ) ; setOperationAction ( <STR_LIT> , Vec64 , Promote ) ; AddPromotedToType ( <STR_LIT> , Vec64 , <STR_LIT> ) ; } setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; if ( Subtarget -> hasFlatAddressSpace ( ) ) { setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; } setOperationAction ( <STR_LIT> , <STR_LIT> , Legal ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Legal ) ; setOperationAction ( <STR_LIT> , <STR_LIT> ,


<s> return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: LOCAL_<ISA_LIT>ZE_Y , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: LOCAL_<ISA_LIT>ZE_Z , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , getImplicitParameterOffset ( MFI , GRID_DIM ) , false ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TGID_X ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TGID_Y ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TGID_Z ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TIDIG_X ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TIDIG_Y ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TIDIG_Z ) , VT ) ; case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant , VT . getStoreSize ( ) , <NUM_LIT> ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; } case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) ) ; case <STR_LIT> : case <STR_LIT> : return DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) , DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) ) ) ; case <STR_LIT> : { SDValue M0 = copyToM0 ( DAG , DAG . getEntryNode ( ) , DL , Op . getOperand ( <NUM_LIT> ) ) ; SDValue Glue = M0 . getValue (


<s> bool AlignedBy4 = ( Align % <NUM_LIT> == <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS || AddrSpace == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) ) { return false ; } if ( Subtarget -> hasUnalignedBufferAccess ( ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AddrSpace == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) ? ( Align % <NUM_LIT> == <NUM_LIT> ) : true ; } return true ; } if ( VT . bitsLT ( <STR_LIT> ) ) return false ; if ( IsFast ) * IsFast = true ; return VT . bitsGT ( <STR_LIT> ) && Align %


<s> if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned MaxPrivateBits = <NUM_LIT> * getSubtarget ( ) -> getMaxPrivateElementSize ( ) ; return ( MemVT . getSizeInBits ( ) <= MaxPrivateBits ) ; } else if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS


<s> if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ; } else if ( AS

== AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) {


<s> if ( Hi . getOpcode ( ) == <STR_LIT> && Hi . getOperand ( <NUM_LIT> ) . getValueType ( ) == <STR_LIT> && ( isa < ConstantSDNode > ( Lo ) || Lo . isUndef ( ) ) ) { Lo = DAG . getNode ( <STR_LIT> , SDLoc ( Lo ) , <STR_LIT> , Lo ) ; Hi = Hi . getOperand ( <NUM_LIT>


<s> static bool convertBuildVectorCastElt ( SelectionDAG & DAG , SDValue & Lo , SDValue & Hi ) { if ( Hi . getOpcode ( ) == <STR_LIT> && Hi . getOperand ( <NUM_LIT> ) . getValueType ( ) == <STR_LIT> && ( isa < ConstantSDNode > ( Lo ) || Lo . isUndef ( ) ) ) { Lo = DAG . getNode ( <STR_LIT> , SDLoc ( Lo ) , <STR_LIT> , Lo ) ; Hi = Hi . getOperand ( <NUM_LIT> ) ; return true


<s> EVT ScalarVT = VT . getScalarType ( ) ; unsigned Size = ScalarVT . getSizeInBits ( ) ; if ( Size == <NUM_LIT> ) return NumElts ; if ( Size == <NUM_LIT> ) return <NUM_LIT>


<s> MVT <ISA_LIT>TargetLowering :: getRegisterTypeForCallingConv ( LLVMContext & Context , CallingConv :: ID CC , EVT VT ) const { if ( CC != CallingConv :: <ISA_LIT>_KERNEL && VT . isVector ( ) ) { EVT ScalarVT = VT . getScalarType ( ) ; unsigned Size = ScalarVT . getSizeInBits


<s> unsigned <ISA_LIT>TargetLowering :: getVectorTypeBreakdownForCallingConv ( LLVMContext & Context , CallingConv :: ID CC , EVT VT , EVT & IntermediateVT , unsigned & NumIntermediates , MVT & RegisterVT ) const { if ( CC != CallingConv :: <ISA_LIT>_KERNEL && VT . isVector ( ) ) { unsigned NumElts = VT . getVectorNumElements ( ) ; EVT ScalarVT = VT . getScalarType ( ) ; unsigned Size = ScalarVT . getSizeInBits ( ) ; if ( Size == <NUM_LIT> ) { RegisterVT = ScalarVT .


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return Op . getValueType ( ) . getScalarType ( ) != <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { bool IsIEEEMode = Subtarget -> enableIEEEBit ( DAG . getMachineFunction ( ) ) ; if ( IsIEEEMode ) { if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( Op . getValueType ( ) ) ) return true ; return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( Op . getValueType ( ) ) ) { return DAG . isKnownNeverSNaN ( Op . getOperand ( <NUM_LIT> ) ) && DAG . isKnownNeverSNaN ( Op . getOperand ( <NUM_LIT> ) ) ; } return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { for ( unsigned i = <NUM_LIT> , e = Op . getNumOperands ( ) ; i != e ; ++ i ) { SDValue SrcOp = Op . getOperand ( i ) ; if ( ! isCanonicalized ( DAG , SrcOp , MaxDepth - <NUM_LIT> ) ) return false ; } return true ; } case <STR_LIT> : case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : return false ; case <STR_LIT> : { unsigned IntrinsicID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IntrinsicID ) { case <STR_LIT>


<s> bool <ISA_LIT>TargetLowering :: isCheapAddrSpaceCast ( unsigned SrcAS , unsigned DestAS ) const { if ( SrcAS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) return true ; return isNoopAddrSpaceCast ( SrcAS ,


<s> bool <ISA_LIT>TargetLowering :: isCheapAddrSpaceCast ( unsigned SrcAS , unsigned DestAS ) const { if ( SrcAS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) return true ; return isNoopAddrSpaceCast ( SrcAS , DestAS


<s> if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { return isLegalMUBUFAddressingMode ( AM ) ; } else if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . REGION_ADDRESS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS || AS == AMDGPUA<ISA_LIT> . UNKNOWN_ADDRESS_SPACE ) { return isLegalFlatAddressingMode ( AM ) ; } else { llvm_unreachable ( <STR_LIT>


<s> bool <ISA_LIT>TargetLowering :: isNoopAddrSpaceCast ( unsigned SrcAS , unsigned DestAS


<s> return ( GA -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || GA -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || GA -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT

) && ! shouldEmitGOTReloc ( GA -> getGlobal ( ) ) ;


<s> switch ( N -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : { const RegisterSDNode * R = nullptr ; if ( N -> getOpcode ( ) == <STR_LIT> ) { R = dyn_cast < RegisterSDNode > ( N ) ; } else { R = dyn_cast < RegisterSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; } if ( R ) { const MachineFunction * MF = FLI -> MF ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; unsigned Reg = R -> getReg ( ) ; if ( TRI . isPhysicalRegister ( Reg ) ) return TRI . isVGPR ( MRI , Reg ) ; if ( MRI . isLiveIn ( Reg ) ) { if ( TRI . isVGPR ( MRI , Reg ) ) return true ; else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; } return ! DA || DA -> isDivergent ( FLI -> getValueFromVirtualReg ( Reg ) ) ; } } break ; case <STR_LIT> : { const LoadSDNode * L = dyn_cast < LoadSDNode > ( N ) ; if ( L -> getMemOperand ( ) -> getAddrSpace ( ) == Subtarget -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ) return true ; } break ; case <STR_LIT> : return true ; break ; case <STR_LIT> : { } return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT>


<s> SDValue <ISA_LIT>TargetLowering :: lowerADDRSPACECAST ( SDValue Op , SelectionDAG & DAG ) const { SDLoc SL ( Op ) ; const AddrSpaceCastSDNode * ASC = cast < AddrSpaceCastSDNode > ( Op ) ; SDValue Src = ASC -> getOperand ( <NUM_LIT> ) ; SDValue FlatNullPtr = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( getTargetMachine ( ) ) ; if ( ASC -> getSrcAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { unsigned DestAS = ASC -> getDestAddressSpace ( ) ; if ( DestAS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || DestAS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , FlatNullPtr , <STR_LIT> ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , Ptr , SegmentNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { unsigned SrcAS = ASC -> getSrcAddressSpace ( ) ; if ( SrcAS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || SrcAS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( SrcAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , SegmentNullPtr ,


<s> SDValue Src = ASC -> getOperand ( <NUM_LIT> ) ; SDValue FlatNullPtr = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( getTargetMachine ( ) ) ; if ( ASC -> getSrcAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { unsigned DestAS = ASC -> getDestAddressSpace ( ) ; if ( DestAS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || DestAS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , FlatNullPtr , <STR_LIT> ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , Ptr , SegmentNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { unsigned SrcAS = ASC ->


<s> const GlobalValue * GV = GSD -> getGlobal ( ) ; if ( GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS && GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT && GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS && ! GV -> getType ( ) -> getElementType ( ) -> isFunctionTy ( ) ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; EVT PtrVT = Op . getValueType ( ) ; if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout .


<s> const GlobalValue * GV = GSD -> getGlobal ( ) ; if ( GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS && GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT && GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS && ! GV -> getType ( ) -> getElementType ( ) -> isFunctionTy ( ) ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; EVT PtrVT = Op . getValueType ( ) ; if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( PtrTy ) ; MachinePointerInfo PtrInfo ( UndefValue :: get (


<s> Type * Ty = MemVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; if ( MemVT . getStoreSize ( ) < <NUM_LIT> && Align < <NUM_LIT> ) { int64_t AlignDownOffset = alignDown ( Offset , <NUM_LIT> ) ; int64_t OffsetDiff = Offset - AlignDownOffset ; EVT IntVT = MemVT . changeTypeToInteger ( ) ; SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , AlignDownOffset ) ; SDValue Load = DAG . getLoad ( <STR_LIT> , SL , Chain , Ptr , PtrInfo , <NUM_LIT> , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; SDValue ShiftAmt = DAG . getConstant ( OffsetDiff * <NUM_LIT> , SL , <STR_LIT> ) ; SDValue Extract = DAG . getNode ( <STR_LIT> , SL ,


<s> MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; if ( MemVT . getStoreSize ( ) < <NUM_LIT> && Align < <NUM_LIT> ) { int64_t AlignDownOffset = alignDown ( Offset , <NUM_LIT> ) ; int64_t OffsetDiff = Offset - AlignDownOffset ; EVT IntVT = MemVT . changeTypeToInteger ( ) ; SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , AlignDownOffset ) ; SDValue Load = DAG . getLoad ( <STR_LIT> , SL , Chain , Ptr , PtrInfo , <NUM_LIT> , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; SDValue ShiftAmt = DAG . getConstant ( OffsetDiff


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * InputPtrReg ; const TargetRegisterClass * RC ; std :: tie ( InputPtrReg , RC ) = Info -> getPreloadedValue ( <STR_LIT> ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction (


<s> assert ( ( ! Result . getNode ( ) || Result . getNode ( ) -> getNumValues ( ) == <NUM_LIT> ) && <STR_LIT> ) ; return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID (


<s> return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue


<s> SDLoc DL ( Op ) ; StoreSDNode * Store = cast < StoreSDNode > ( Op ) ; EVT VT = Store -> getMemoryVT ( ) ; if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT>


<s> auto & ArgUsageInfo = DAG . getPass ( ) -> getAnalysis < <ISA_LIT>ArgumentUsageInfo > ( ) ; const <ISA_LIT>FunctionArgInfo & CalleeArgInfo = ArgUsageInfo . lookupFuncArgInfo ( * CalleeFunc ) ; const <ISA_LIT>FunctionArgInfo & CallerArgInfo = Info . getArgInfo ( ) ; <STR_LIT> InputRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; for ( auto InputID : InputRegs ) { const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo . getPreloadedValue ( InputID ) ; if ( ! OutgoingArg ) continue ; const ArgDescriptor * IncomingArg ; const TargetRegisterClass * IncomingArgRC ; std :: tie ( IncomingArg , IncomingArgRC ) = CallerArgInfo . getPreloadedValue ( InputID ) ; assert ( IncomingArgRC == ArgRC ) ; EVT ArgVT = TRI -> getSpillSize ( * ArgRC ) == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue InputReg


<s> return DAG . getNode ( <STR_LIT> , SL , VT , NewVec ) ; } if ( convertBuildVectorCastElt ( DAG , Hi , Lo ) ) { SDValue NewVec = DAG . getBuildVector ( <STR_LIT> , SL , { Hi , Lo } ) ; return DAG . getNode ( <STR_LIT> , SL ,


<s> case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : return performRcpCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> : return performFMed3Combine ( N , DCI ) ; case <STR_LIT> : return performCvtPkRTZCombine ( N , DCI ) ; case <STR_LIT> : return performClampCombine ( N , DCI ) ; case <STR_LIT> : { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> || VT == <STR_LIT> ) { SDLoc SL ( N ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ; EVT EltVT = Src . getValueType ( )


<s> } if ( VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> hasMed3_16 ( ) ) ) { SDValue Var = Op0 . getOperand ( <NUM_LIT> ) ; if ( ! DAG . isKnownNeverSNaN ( Var ) ) return SDValue ( ) ; return DAG . getNode ( <STR_LIT> , SL , K0 -> getValueType ( <NUM_LIT> ) ,

Var , SDValue ( K0 , <NUM_LIT> ) , SDValue ( K1 , <NUM_LIT> ) ) ;


<s> Info . setScratchRSrcReg ( PrivateSegmentBufferReg ) ; if ( MFI . hasCalls ( ) ) { unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } else { unsigned PrivateSegmentWaveByteOffsetReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchWaveOffsetReg ( PrivateSegmentWaveByteOffsetReg ) ; } } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; if ( HasStackObjects && ! MFI . hasCalls ( ) ) { unsigned ScratchWaveOffsetReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchWaveOffsetReg ( ScratchWaveOffsetReg ) ; }


<s> static void reservePrivateMemoryRegs ( const TargetMachine & TM , MachineFunction & MF , const <ISA_LIT>RegisterInfo & TRI , <ISA_LIT>MachineFunctionInfo & Info ) { MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; bool HasStackObjects = MFI . hasStackObjects ( ) ; if ( HasStackObjects ) Info . setHasNonSpillStackObjects ( true ) ; if ( TM . getOptLevel ( ) == CodeGenOpt :: None ) HasStackObjects = true ; bool RequiresStackAccess = HasStackObjects || MFI . hasCalls ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ST . isAmdCodeObjectV2 ( MF . getFunction ( ) ) ) { if ( RequiresStackAccess ) { unsigned PrivateSegmentBufferReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchRSrcReg ( PrivateSegmentBufferReg ) ; if ( MFI . hasCalls ( ) ) { unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } else


<s> bool <ISA_LIT>TargetLowering :: shouldEmitFixup ( const GlobalValue * GV ) const { const Triple & TT = getTargetMachine ( ) . getTargetTriple ( ) ; return ( GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || GV -> getType (

) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) && <STR_LIT> ( TT ) ;


<s> if ( ( MemVT . isSimple ( ) && ! DCI . isAfterLegalizeDAG ( ) ) || MemVT . getSizeInBits ( ) >= <NUM_LIT> ) return SDValue ( ) ; SDLoc SL ( Ld ) ; assert ( ( ! MemVT . isVector ( ) || Ld -> getExtensionType ( ) == <STR_LIT> ) && <STR_LIT> ) ; SDValue Ptr = Ld -> getBasePtr ( ) ; SDValue NewLoad = DAG . getLoad ( <STR_LIT> , <STR_LIT> , <STR_LIT> , SL , Ld -> getChain ( ) , Ptr , Ld -> getOffset ( ) , Ld -> getPointerInfo ( ) , <STR_LIT> , Ld -> getAlignment ( ) , Ld -> getMemOperand ( ) -> getFlags ( ) , Ld -> getAAInfo ( ) , nullptr ) ; EVT TruncVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , MemVT . getSizeInBits ( ) ) ; if ( MemVT . isFloatingPoint ( ) ) { assert ( Ld -> getExtensionType ( ) == <STR_LIT> && <STR_LIT> ) ; TruncVT = MemVT . changeTypeToInteger ( )


<s> for ( MachineInstr * Instr : EarlyTermInstrs ) { if ( MF . getFunction ( ) . getCallingConv ( ) != CallingConv :: <ISA_LIT>_GS ) earlyTerm ( * Instr , EarlyExitBlock ) ; Instr -> eraseFromParent ( ) ; } EarlyTermInstrs . clear ( ) ; MadeChange = true ; } if ( ! EpilogInstrs . empty ( ) ) { MachineBasicBlock * EmptyMBBAtEnd = nullptr ; assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( EpilogInstrs . size ( ) > <NUM_LIT> ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } for ( auto MI : EpilogInstrs ) { auto MBB = MI -> getParent ( ) ; if ( MBB == & MF . back ( ) && MI == & MBB -> back ( ) ) continue ; if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB -> addSuccessor ( EmptyMBBAtEnd ) ; MDT -> getBase ( ) . insertEdge ( MBB , EmptyMBBAtEnd ) ; BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI -> eraseFromParent ( ) ; MadeChange =


<s> } EarlyTermInstrs . clear ( ) ; MadeChange = true ; } if ( ! EpilogInstrs . empty ( ) ) { MachineBasicBlock * EmptyMBBAtEnd = nullptr ; assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( EpilogInstrs . size ( ) > <NUM_LIT> ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } for ( auto MI : EpilogInstrs ) { auto MBB = MI -> getParent ( ) ; if ( MBB == & MF . back ( ) && MI == & MBB -> back ( ) ) continue ; if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB -> addSuccessor ( EmptyMBBAtEnd ) ; MDT -> getBase ( ) . insertEdge ( MBB , EmptyMBBAtEnd ) ; BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI -> eraseFromParent ( ) ; MadeChange = true ; } EpilogInstrs . clear (


<s> void <ISA_LIT>LateBranchLowering :: earlyTerm ( MachineInstr & MI , MachineBasicBlock * EarlyExitBlock ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc DL = MI . getDebugLoc ( ) ; auto BranchMI = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ; auto Next = std :: next ( MI . getIterator ( ) ) ; if ( Next != MBB . end ( ) && ! Next -> isTerminator ( ) ) splitBlock ( MBB , * BranchMI , MDT ) ; MBB . addSuccessor ( EarlyExitBlock )


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> } if ( ! EarlyTermInstrs . empty ( ) ) { MachineBasicBlock * EarlyExitBlock = MF . CreateMachineBasicBlock ( ) ; DebugLoc DL ; MF . insert ( MF . end ( ) , EarlyExitBlock ) ; BuildMI ( * EarlyExitBlock , EarlyExitBlock -> end ( ) , DL , TII -> get ( MovOpc ) , ExecReg ) . addImm ( <NUM_LIT> ) ; generateEndPgm ( * EarlyExitBlock , EarlyExitBlock -> end ( ) , DL , TII , MF ) ; for ( MachineInstr * Instr : EarlyTermInstrs ) { if ( MF . getFunction ( ) . getCallingConv ( ) != CallingConv :: <ISA_LIT>_GS ) earlyTerm ( * Instr , EarlyExitBlock ) ; Instr -> eraseFromParent ( ) ; } EarlyTermInstrs . clear ( ) ; MadeChange = true ; } if ( ! EpilogInstrs . empty ( ) ) { MachineBasicBlock * EmptyMBBAtEnd = nullptr ; assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( EpilogInstrs . size ( ) > <NUM_LIT> ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } for ( auto MI : EpilogInstrs ) { auto MBB = MI -> getParent ( ) ; if ( MBB == & MF . back ( ) && MI == & MBB -> back ( ) ) continue ; if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB -> addSuccessor ( EmptyMBBAtEnd ) ; MDT -> getBase ( ) . insertEdge ( MBB , EmptyMBBAtEnd ) ; BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI -> eraseFromParent


<s> for ( std :: list < CombineInfo > & AddrList : MergeableInsts ) { if ( AddrList . front ( ) . InstClass == CI . InstClass && AddrList . front ( ) . hasSameBaseAddress ( * CI . I ) ) { AddrList . emplace_back ( CI ) ; return ; } } MergeableInsts . emplace_back ( <NUM_LIT> ,


<s> if ( ! CI . hasMergeableAddress ( * MRI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; addInstToMergeableList ( CI , MergeableInsts ) ; } for ( std :: list < std :: list < CombineInfo >> :: iterator I = MergeableInsts . begin ( ) , E = MergeableInsts . end ( ) ; I != E ; ) { std :: list < CombineInfo > & MergeList = * I ; if ( MergeList . size ( ) <= <NUM_LIT> ) { I = MergeableInsts . erase ( I ) ; continue ; } MergeList . sort ( [ ] ( const CombineInfo & A , const CombineInfo & B ) { return A . Offset < B . Offset ; }


<s> Register DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; Register DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ; MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ; Register FullDestReg = MRI -> createVirtualRegister ( TRI -> getVGPR64Class ( ) ) ; MachineInstr * FullBase = BuildMI ( * MBB , MBBI , DL , TII -> get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) .

addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ;


<s> MachineInstr * Mov = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Val ) ; ( void ) Mov ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Mov -> dump ( )


<s> unsigned OperandsToMatch [ ] = { <STR_LIT> :: cpol , <STR_LIT> :: d16 , <STR_LIT> :: unorm , <STR_LIT> :: da , <STR_LIT> :: r128 , <STR_LIT> :: a16 } ; for ( auto op : OperandsToMatch ) { int Idx = <STR_LIT> ( CI . I -> getOpcode ( ) , op ) ; if ( <STR_LIT> ( Paired . I -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ; unsigned MinMask = std :: min ( CI . DMask , Paired . DMask ) ; unsigned AllowedBitsForMin = llvm ::


<s> if ( ! Op . isReg ( ) ) return None ; MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || ! Def -> getOperand ( <NUM_LIT> ) .


<s> const llvm :: <STR_LIT> * NewFormatInfo = llvm :: <STR_LIT> ( OldFormatInfo -> BitsPerComp , ComponentCount , OldFormatInfo -> NumFormat , STI ) ; if ( ! NewFormatInfo ) return <NUM_LIT> ; assert ( NewFormatInfo -> NumFormat == OldFormatInfo -> NumFormat &&

NewFormatInfo -> BitsPerComp == OldFormatInfo -> BitsPerComp ) ;


<s> static unsigned getBufferFormatWithCompCount ( unsigned OldFormat , unsigned ComponentCount , const GCNSubtarget & STI ) { if ( ComponentCount > <NUM_LIT> ) return <NUM_LIT> ; const llvm :: <STR_LIT> * OldFormatInfo = llvm :: <STR_LIT> ( OldFormat , STI ) ; if ( ! OldFormatInfo ) return <NUM_LIT> ; const llvm :: <STR_LIT> * NewFormatInfo = llvm :: <STR_LIT> ( OldFormatInfo -> BitsPerComp , ComponentCount , OldFormatInfo -> NumFormat , STI ) ; if ( ! NewFormatInfo ) return <NUM_LIT> ; assert ( NewFormatInfo -> NumFormat == OldFormatInfo -> NumFormat && NewFormatInfo -> BitsPerComp == OldFormatInfo -> BitsPerComp ) ; return NewFormatInfo


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( <STR_LIT> ( Opc ) -> BVH ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DS_READ ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return


<s> case TBUFFER_LOAD : case TBUFFER_STORE : return <STR_LIT> ( <STR_LIT> ( CI . I -> getOpcode ( ) ) , Width ) ; case UNKNOWN : llvm_unreachable ( <STR_LIT> ) ; case S_BUFFER_LOAD_IMM : switch ( Width ) { default : return <NUM_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; } case MIMG : assert ( ( countPopulation ( CI . DMask | Paired . DMask ) == Width ) && <STR_LIT> ) ; return <STR_LIT> ( CI . I -> getOpcode


<s> static unsigned getOpcodeWidth ( const MachineInstr & MI , const <ISA_LIT>InstrInfo & TII ) { const unsigned Opc = MI . getOpcode ( ) ; if ( TII . isMUBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } if ( TII . isMIMG ( MI ) ) { uint64_t DMaskImm = TII . getNamedOperand ( MI , <STR_LIT> :: dmask ) -> getImm ( ) ; return countPopulation ( DMaskImm ) ; } if ( TII . isMTBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } switch ( Opc ) { case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> :


<s> if ( Info && <STR_LIT> ( Info -> BaseOpcode ) -> Sampler ) Result . SSamp = true ; return Result ; } if ( TII . isMTBUF ( Opc ) ) { if ( <STR_LIT> ( Opc ) ) Result . VAddr = true ; if ( <STR_LIT> ( Opc ) ) Result . SRsrc = true ; if ( <STR_LIT> ( Opc ) ) Result . SOffset = true ; return Result ; } switch ( Opc ) { default : return Result ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result . SBase = true ; return Result ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result . Addr =


<s> static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> , <NUM_LIT> } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } , } ; assert ( CI . Width >= <NUM_LIT> && CI . Width <= <NUM_LIT> ) ; assert ( Paired . Width >= <NUM_LIT> && Paired . Width <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ Paired . Width - <NUM_LIT> ] ; Idx0 = Idxs [ Paired . Width ] [ CI . Width - <NUM_LIT> ] ; } else { Idx0 = Idxs [ <NUM_LIT> ] [ CI


<s> unsigned BitWidth = <NUM_LIT> * ( CI . Width + Paired . Width ) ; return TRI -> hasAGPRs ( getDataRegClass ( * CI . I ) ) ? TRI -> getAGPRClassForBitWidth ( BitWidth )

: TRI -> getVGPRClassForBitWidth ( BitWidth ) ;


<s> unsigned BitWidth = <NUM_LIT> * ( CI . Width + Paired . Width ) ; return TRI -> hasAGPRs ( getDataRegClass ( * CI . I ) ) ? TRI -> getAGPRClassForBitWidth ( BitWidth ) : TRI

-> getVGPRClassForBitWidth ( BitWidth ) ;


<s> MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT>


<s> unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg


<s> const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired


<s> unsigned DMaskIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: dmask ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; for ( unsigned I = <NUM_LIT> , E = ( * CI . I ) . getNumOperands ( ) ; I != E ; ++ I ) { if ( I == DMaskIdx ) MIB . addImm ( MergedDMask ) ; else MIB . add ( ( * CI . I ) . getOperand ( I ) ) ; } assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; unsigned SubRegIdx0 , SubRegIdx1 ; std :: tie ( SubRegIdx0 , SubRegIdx1 ) = getSubRegIdxs ( CI , Paired ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent


<s> unsigned NewOffset0 = CI . Offset ; unsigned NewOffset1 = Paired . Offset ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , Paired . I , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I


<s> unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; CI . I -> eraseFromParent (


<s> MachineInstr * New = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; CI . I -> eraseFromParent


<s> const unsigned Opcode = getNewOpcode ( CI , Paired ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STM ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . CPol ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) .

addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ;


<s> std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; CI . I ->


<s> MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI , Paired ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII

-> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ;


<s> static uint32_t mostAlignedValueInRange ( uint32_t Lo , uint32_t


<s> for ( std :: list < std :: list < CombineInfo >> :: iterator I = MergeableInsts . begin ( ) , E = MergeableInsts . end ( ) ; I != E ; ) { std :: list < CombineInfo > & MergeList = * I ; bool OptimizeListAgain = false ; if ( ! optimizeInstsWithSameBaseAddr ( MergeList , OptimizeListAgain ) ) { I = MergeableInsts . erase ( I ) ; continue ; } Modified =


<s> bool <ISA_LIT>LoadStoreOptimizer :: optimizeInstsWithSameBaseAddr ( std :: list < CombineInfo > & MergeList , bool & OptimizeListAgain ) { if ( MergeList . empty ( ) ) return false ; bool Modified = false ; for ( auto I = MergeList . begin ( ) , Next = std :: next ( I ) ; Next != MergeList . end ( ) ; Next = std :: next ( I ) ) { auto First = I ; auto Second = Next ; if ( ( * First ) . Order > ( * Second ) . Order ) std :: swap ( First , Second ) ; CombineInfo & CI = * First ; CombineInfo & Paired = * Second ; SmallVector < MachineInstr * , <NUM_LIT> > InstsToMove ; if ( ! checkAndPrepareMerge ( CI , Paired , InstsToMove ) ) { ++ I ; continue ; } Modified = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CI . I << <STR_LIT> << * Paired . I ) ; switch ( CI . InstClass ) { default : llvm_unreachable ( <STR_LIT> ) ; break ; case DS_READ : { MachineBasicBlock :: iterator NewMI = mergeRead2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case DS_WRITE : { MachineBasicBlock :: iterator NewMI = mergeWrite2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case S_BUFFER_LOAD_IMM : { MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII


<s> MachineBasicBlock :: iterator NewMI = mergeRead2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case DS_WRITE : { MachineBasicBlock :: iterator NewMI = mergeWrite2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case S_BUFFER_LOAD_IMM : { MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG :


<s> TII = STM -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; SmallPtrSet < MachineInstr * , <NUM_LIT> > AnchorList ; MemInfoMap Visited ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator SectionEnd ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = SectionEnd ) { bool CollectModified ; std :: list < std :: list < CombineInfo >> MergeableInsts ; std :: tie ( SectionEnd , CollectModified ) = collectMergeableInsts ( I , E , Visited , AnchorList , MergeableInsts ) ; Modified |= CollectModified ; do { OptimizeAgain = false ; Modified |= optimizeBlock ( MergeableInsts ) ; } while ( OptimizeAgain ) ; } Visited . clear (


<s> Base -> setIsKill ( false ) ; TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ->


<s> void <ISA_LIT>LoadStoreOptimizer :: updateBaseAndOffset ( MachineInstr & MI , Register NewBase , int32_t NewOffset ) const { auto Base = TII -> getNamedOperand (


<s> assert ( MemOp . mayLoadOrStore ( ) ) ; for ( MachineInstr * InstToMove : InstsToMove ) { if ( ! InstToMove -> mayLoadOrStore ( ) ) continue ; if ( ! memAccessesCanBeReordered ( MemOp , * InstToMove , TII , AA ) ) return false ; } return true


<s> MachineBasicBlock * MBB = MI . getParent ( ) ; MachineBasicBlock :: iterator MBBI = MI . getIterator ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; assert ( ( TRI -> getRegSizeInBits ( Addr . Base . LoReg , * MRI ) == <NUM_LIT> || Addr . Base . LoSubReg ) && <STR_LIT> ) ; assert ( ( TRI -> getRegSizeInBits ( Addr . Base . HiReg , * MRI ) == <NUM_LIT> || Addr . Base . HiSubReg ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineOperand OffsetLo = createRegOrImm ( static_cast < int32_t > ( Addr . Offset ) , MI ) ; MachineOperand OffsetHi = createRegOrImm ( static_cast < int32_t > ( Addr . Offset >> <NUM_LIT> ) , MI ) ; unsigned CarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DeadCarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub1 = MRI -> createVirtualRegister ( &


<s> return ! ( A -> mayStore ( ) || B -> mayStore ( ) ) ||

TII -> areMemAccessesTriviallyDisjoint ( * A , * B , AA ) ;


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeBufferLoadPair ( CombineInfo & CI ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get


<s> DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg (


<s> const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; return Next


<s> unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent (


<s> const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent (


<s> unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1

) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ;


<s> BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent


<s> for ( ; BlockI != End ; ++ BlockI ) { MachineInstr & MI = * BlockI ; if ( promoteConstantOffsetToImm ( MI , Visited , AnchorList ) ) Modified = true ; if ( MI . hasOrderedMemoryRef ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; ++ BlockI ; break ; } const InstClassEnum InstClass = getInstClass ( MI . getOpcode ( ) , * TII ) ; if ( InstClass == UNKNOWN ) continue ; CombineInfo CI ; CI . setMI ( MI , * TII , * STM ) ; CI . Order = Order ++ ; if ( ! CI . hasMergeableAddress ( * MRI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> if ( promoteConstantOffsetToImm ( MI , Visited , AnchorList ) ) Modified = true ; if ( MI . hasOrderedMemoryRef ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; ++ BlockI ; break ; } const InstClassEnum InstClass = getInstClass ( MI . getOpcode ( ) , * TII ) ; if ( InstClass == UNKNOWN ) continue ; CombineInfo CI ; CI . setMI ( MI , * TII , * STM ) ; CI . Order = Order ++ ; if ( ! CI . hasMergeableAddress ( *


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case


<s> return Info -> BaseOpcode ; } if ( TII . isMTBUF ( Opc ) ) return <STR_LIT> ( Opc ) ; return - <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return Opc ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <STR_LIT>


<s> if ( TII . isMIMG ( MI ) ) { uint64_t DMaskImm = TII . getNamedOperand ( MI , <STR_LIT> :: dmask ) -> getImm ( ) ; return countPopulation ( DMaskImm ) ; } if ( TII . isMTBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } switch ( Opc ) { case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT>


<s> static void addDefsToList ( const MachineInstr & MI , SmallVectorImpl < const MachineOperand * > & Defs ) { for ( const MachineOperand & Def : MI . defs


<s> continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ) continue ; int AddrIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: addr ) ; const MachineOperand & AddrReg0 = CI . I -> getOperand ( AddrIdx ) ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) { int OffsetIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: offset ) ; CI . Offset0 = CI . I -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ; CI . Offset1 = MBBI -> getOperand ( OffsetIdx ) .


<s> ++ MBBI ; SmallVector < const MachineOperand * , <NUM_LIT> > DefsToMove ; addDefsToList ( * CI . I , DefsToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( MBBI -> getOpcode ( ) != CI . I -> getOpcode ( ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) return false ; if ( MBBI -> mayLoadOrStore ( ) && ! memAccessesCanBeReordered ( * CI . I , * MBBI , TII , AA ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsToList ( * MBBI , DefsToMove ) ; continue ; } addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , DefsToMove , CI


<s> static bool memAccessesCanBeReordered ( MachineBasicBlock :: iterator A , MachineBasicBlock :: iterator B , const <ISA_LIT>InstrInfo *


<s> static bool memAccessesCanBeReordered ( MachineBasicBlock :: iterator A , MachineBasicBlock :: iterator B

, const <ISA_LIT>InstrInfo * TII , AliasAnalysis * AA ) {


<s> const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addImm ( CI . BaseOff ) . addReg ( AddrReg -> getReg ( ) ) ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent (


<s> if ( CI . BaseOff ) { BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addImm ( CI . BaseOff ) . addReg ( Addr -> getReg ( ) ) ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ; return Next


<s> if ( ( CI . Offset0 % CI . EltSize != <NUM_LIT> ) || ( CI . Offset1 % CI . EltSize != <NUM_LIT> ) ) return false ; unsigned EltOffset0 = CI . Offset0 / CI . EltSize ; unsigned EltOffset1 = CI . Offset1 / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { CI . Offset0 = EltOffset0 / <NUM_LIT> ; CI . Offset1 = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset0 = EltOffset0 ; CI . Offset1 = EltOffset1 ; return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset0 , CI . Offset1 ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { CI . Offset0 = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . Offset1 = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( OffsetDiff ) ) { CI . Offset0 = EltOffset0 - CI . BaseOff / CI . EltSize ; CI . Offset1 = EltOffset1 - CI . BaseOff / CI . EltSize ; return true ; } return


<s> if ( ! STM . loadStoreOptEnabled ( ) ) return false ; TII = STM . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ; return Modified


<s> FunctionPass * llvm :: create<ISA_LIT>LoadStoreOptimizerPass ( TargetMachine & TM ) { return new <ISA_LIT>LoadStoreOptimizer ( TM


<s> MachineBasicBlock :: iterator MBBI = I ; ++ MBBI ; SmallVector < const MachineOperand * , <NUM_LIT> > DefsToMove ; addDefsToList ( * I , DefsToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( MBBI -> getOpcode ( ) != I -> getOpcode ( ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) return E ; if ( MBBI -> mayLoadOrStore ( ) && ! memAccessesCanBeReordered ( * I , * MBBI , TII , AA ) ) { InstsToMove . push_back ( & * MBBI ) ; addDefsToList ( * MBBI , DefsToMove ) ; continue ; } addToListsIfDependent ( * MBBI , DefsToMove , InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return E ; if ( addToListsIfDependent ( * MBBI , DefsToMove , InstsToMove ) ) continue ; int AddrIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: addr ) ; const MachineOperand & AddrReg0 = I -> getOperand ( AddrIdx ) ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) { int OffsetIdx = <STR_LIT> ( I -> getOpcode


<s> return ( TII -> areMemAccessesTriviallyDisjoint ( * A , * B , AA ) || ! ( A -> mayStore (


<s> static bool memAccessesCanBeReordered ( MachineBasicBlock :: iterator A , MachineBasicBlock :: iterator B , const

<ISA_LIT>InstrInfo * TII , llvm :: AliasAnalysis * AA ) {


<s> unsigned Offset1 = TII -> getNamedOperand ( * Paired , <STR_LIT> :: offset ) -> getImm ( ) & <NUM_LIT> ; unsigned NewOffset0 = Offset0 / EltSize ; unsigned NewOffset1 = Offset1 / EltSize ; unsigned Opc = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; bool UseST64 = ( NewOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( NewOffset1 % <NUM_LIT> == <NUM_LIT> ) ; if ( UseST64 ) { NewOffset0 /= <NUM_LIT> ; NewOffset1 /= <NUM_LIT> ; Opc = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; } unsigned SubRegIdx0 = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = I -> getDebugLoc ( ) ; MachineInstrBuilder Read2 = BuildMI ( * MBB , Paired , DL , Read2Desc , DestReg ) . addOperand ( * AddrReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , Paired , DL , CopyDesc ) . addOperand ( * Dest0 ) . addReg ( DestReg


<s> std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = I -> getDebugLoc ( ) ; MachineInstrBuilder Write2 = BuildMI ( * MBB , Paired , DL , Write2Desc ) . addOperand ( * Addr ) . addOperand ( * Data0 ) . addOperand ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; moveInstsAfter ( Write2 , InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( I ) ; I -> eraseFromParent ( ) ; Paired -> eraseFromParent


<s> unsigned Opc = MI . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { unsigned Size = ( Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; MachineBasicBlock :: iterator Match = findMatchingD<ISA_LIT>nst ( I , Size , InstsToMove ) ; if ( Match != E ) { Modified = true ; I = mergeRead2Pair ( I , Match , Size , InstsToMove ) ; } else { ++ I ; } continue ; } else if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { unsigned Size = ( Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; MachineBasicBlock :: iterator Match = findMatchingD<ISA_LIT>nst ( I , Size , InstsToMove ) ; if ( Match != E ) { Modified = true ; I = mergeWrite2Pair ( I , Match , Size , InstsToMove ) ; } else { ++ I ; } continue ; } ++ I ; } return


<s> default : return nullptr ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; } } unsigned BitWidth = <NUM_LIT> * ( CI . Width + Paired . Width ) ; return TRI -> isAGPRClass ( getDataRegClass ( * CI . I ) ) ? TRI ->

getAGPRClassForBitWidth ( BitWidth ) : TRI -> getVGPRClassForBitWidth ( BitWidth ) ;


<s> MachineBasicBlock * MBB = CI . I -> getParent ( ) ; MachineBasicBlock :: iterator E = MBB -> end ( ) ; MachineBasicBlock :: iterator MBBI = CI . I ; int AddrIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: addr ) ; const MachineOperand & AddrReg0 = CI . I -> getOperand ( AddrIdx ) ; if ( TargetRegisterInfo :: isPhysicalRegister ( AddrReg0 . getReg ( ) ) || MRI -> hasOneNonDBGUse ( AddrReg0 . getReg ( ) ) ) return false ; ++ MBBI ; DenseSet < unsigned > DefsToMove ; addDefsToList ( * CI . I , DefsToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( MBBI -> getOpcode ( ) != CI . I -> getOpcode ( ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ! memAccessesCanBeReordered ( * CI . I , * MBBI , TII , AA ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsToList ( * MBBI , DefsToMove ) ; continue ; } addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ) continue ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) { int OffsetIdx = <STR_LIT> ( CI . I -> getOpcode (


<s> TII = STM . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; assert ( MRI -> isSSA ( )


<s> break ; } for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { AddrIdx [ i ] = <STR_LIT> ( CI . I -> getOpcode ( ) , AddrOpName [ i ] ) ; AddrReg [ i ] = & CI . I -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isReg ( ) && ( TargetRegisterInfo :: isPhysicalRegister ( AddrReg [ i ] -> getReg ( ) ) || MRI -> hasOneNonDBGUse ( AddrReg [ i ] -> getReg ( ) ) ) ) return false ; } ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( MBBI -> getOpcode ( ) != CI . I -> getOpcode ( ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , TII , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , TII , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = true ; for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { const MachineOperand & AddrRegNext = MBBI -> getOperand


<s> if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) ) ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI


<s> const MachineOperand * AddrReg = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: addr ) ; const MachineOperand * Data0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: data0 ) ; const MachineOperand * Data1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: data0 ) ; unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? write2ST64Opcode ( CI . EltSize ) : write2Opcode ( CI . EltSize ) ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState ::


<s> TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : MF ) { CreatedX2 = <NUM_LIT>


<s> ++ MBBI ; if ( MBBI -> getOpcode ( ) != I -> getOpcode ( ) ) return E ; if ( MBBI -> hasOrderedMemoryRef ( ) ) return E ; int AddrIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: addr ) ; const MachineOperand & AddrReg0 = I -> getOperand ( AddrIdx ) ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) { int OffsetIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: offset ) ; unsigned Offset0 = I -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ; unsigned Offset1 = MBBI -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ; if ( offsetsCanBeCombined ( Offset0 , Offset1 , EltSize ) ) return MBBI ; } return


<s> int AddrIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: addr ) ; const MachineOperand & AddrReg0 = I -> getOperand ( AddrIdx ) ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) { int OffsetIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: offset ) ; unsigned Offset0 = I -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ; unsigned Offset1 = MBBI -> getOperand ( OffsetIdx )


<s> AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . addPreserved < LiveVariables > (


<s> AU . addPreserved < LiveVariables > ( ) ; AU . addRequired < LiveIntervals > ( ) ; MachineFunctionPass :: getAnalysisUsage (


<s> const char * getPassName ( ) const override { return <STR_LIT>


<s> unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = I -> getDebugLoc ( ) ; MachineInstrBuilder Read2 = BuildMI ( * MBB , I , DL , Read2Desc , DestReg ) . addOperand ( * AddrReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; unsigned SubRegIdx0 = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; MachineInstr * Copy0 = BuildMI ( * MBB , I , DL , CopyDesc ) . addOperand ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , I , DL , CopyDesc ) . addOperand ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; LIS -> InsertMachineInstrInMaps ( Read2 ) ; SlotIndex PairedIndex = LIS -> getInstructionIndex ( Paired ) ; LiveRange & M0Range = LIS -> getRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ; LiveRange :: Segment * M0Segment = M0Range . getSegmentContaining ( PairedIndex ) ; bool UpdateM0Range = M0Segment -> end == PairedIndex . getRegSlot ( ) ; LIS -> ReplaceMachineInstrInMaps ( I , Copy0 ) ; LIS -> ReplaceMachineInstrInMaps ( Paired , Copy1 ) ; I -> eraseFromParent ( ) ; Paired -> eraseFromParent (


<s> const TargetRegisterClass * SuperRC = ( EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = I -> getDebugLoc ( ) ; MachineInstrBuilder Read2 = BuildMI ( * MBB , I , DL , Read2Desc , DestReg ) . addOperand ( * AddrReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; unsigned SubRegIdx0 = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; MachineInstr * Copy0 = BuildMI ( * MBB , I , DL , CopyDesc ) . addOperand ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , I , DL , CopyDesc ) . addOperand ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; LIS -> InsertMachineInstrInMaps ( Read2 ) ; SlotIndex PairedIndex = LIS -> getInstructionIndex ( Paired ) ; LiveRange & M0Range = LIS -> getRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ; LiveRange :: Segment * M0Segment = M0Range .


<s> if ( MI . hasOrderedMemoryRef ( ) ) { ++ I ; continue ; } unsigned Opc = MI . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { unsigned Size = ( Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; MachineBasicBlock :: iterator Match = findMatchingD<ISA_LIT>nst ( I , Size ) ; if ( Match != E ) { Modified = true ; I = mergeRead2Pair ( I , Match , Size ) ; } else { ++ I ; } continue ; } else if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { unsigned Size = ( Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; MachineBasicBlock :: iterator Match = findMatchingD<ISA_LIT>nst ( I ,


<s> const TargetSubtargetInfo & STM = MF . getSubtarget ( ) ; TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( STM . getRegisterInfo ( ) ) ; TII = static_cast < const <ISA_LIT>InstrInfo * > ( STM . getInstrInfo


<s> TII = static_cast < const <ISA_LIT>InstrInfo * > ( STM . getInstrInfo ( ) ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; assert ( ! MRI -> isSSA ( ) ) ; bool Modified =


<s> static void addDefsToList ( const MachineInstr & MI , DenseSet < unsigned


<s> for ( const MachineOperand & Def : MI . defs ( ) ) Defs .


<s> for ( MachineOperand & Use : MI . operands ( ) ) { if ( Use . isReg ( ) && Use . readsReg ( ) && Defs . count ( Use . getReg ( ) ) ) { Insts . push_back ( & MI


<s> DenseSet < unsigned > DefsToMove ; addDefsToList ( * CI . I , DefsToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( MBBI -> getOpcode ( ) != CI . I -> getOpcode ( ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , TII , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , TII , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsToList ( * MBBI , DefsToMove ) ; continue ; } addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ) continue ; bool Match = true ; for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { const MachineOperand & AddrRegNext = MBBI -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isImm ( ) || AddrRegNext . isImm ( ) ) { if ( AddrReg [ i ] -> isImm ( ) != AddrRegNext . isImm ( ) || AddrReg [ i ] -> getImm ( ) != AddrRegNext . getImm ( ) ) { Match = false ; break ; } continue ; } if ( AddrReg [ i ] -> getReg ( ) != AddrRegNext . getReg ( )

|| AddrReg [ i ] -> getSubReg ( ) != AddrRegNext . getSubReg ( ) ) {


<s> static bool memAccessesCanBeReordered ( MachineBasicBlock :: iterator A , MachineBasicBlock :: iterator B , const <ISA_LIT>InstrInfo