<s> return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: LOCAL_<ISA_LIT>ZE_Y , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: LOCAL_<ISA_LIT>ZE_Z , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , getImplicitParameterOffset ( MFI , GRID_DIM ) , false ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TGID_X ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TGID_Y ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TGID_Z ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TIDIG_X ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TIDIG_Y ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: TIDIG_Z ) , VT ) ; case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant , VT . getStoreSize ( ) , <NUM_LIT> ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; } case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) ) ; case <STR_LIT> : case <STR_LIT> : return DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) , DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) ) ) ; case <STR_LIT> : { SDValue M0 = copyToM0 ( DAG , DAG . getEntryNode ( ) , DL , Op . getOperand ( <NUM_LIT> ) ) ;

SDValue Glue = M0 . getValue ( <NUM_LIT> ) ;


<s> bool AlignedBy4 = ( Align % <NUM_LIT> == <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS || AddrSpace == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) ) { return false ; } if ( Subtarget -> hasUnalignedBufferAccess ( ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AddrSpace == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) ? ( Align % <NUM_LIT> == <NUM_LIT> ) : true ; } return true ; } if ( VT . bitsLT ( <STR_LIT> ) ) return false ; if ( IsFast ) * IsFast = true ;

return VT . bitsGT ( <STR_LIT> ) && Align % <NUM_LIT> == <NUM_LIT> ;


<s> if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned MaxPrivateBits = <NUM_LIT> * getSubtarget ( ) -> getMaxPrivateElementSize ( ) ; return ( MemVT . getSizeInBits ( ) <= MaxPrivateBits ) ;

} else if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) {


<s> if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ;

} else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) {


<s> if ( Hi . getOpcode ( ) == <STR_LIT> && Hi . getOperand ( <NUM_LIT> ) . getValueType ( ) == <STR_LIT> && ( isa < ConstantSDNode > ( Lo ) || Lo . isUndef ( ) ) ) { Lo = DAG . getNode ( <STR_LIT> , SDLoc ( Lo ) , <STR_LIT> , Lo ) ;


<s> static bool convertBuildVectorCastElt ( SelectionDAG & DAG , SDValue & Lo , SDValue & Hi ) { if ( Hi . getOpcode ( ) == <STR_LIT> && Hi . getOperand ( <NUM_LIT> ) . getValueType ( ) == <STR_LIT> && ( isa < ConstantSDNode > ( Lo ) || Lo . isUndef ( ) ) ) { Lo = DAG . getNode ( <STR_LIT> , SDLoc ( Lo ) , <STR_LIT> , Lo ) ; Hi = Hi . getOperand ( <NUM_LIT> ) ;


<s> EVT ScalarVT = VT . getScalarType ( ) ; unsigned Size = ScalarVT . getSizeInBits ( ) ; if ( Size == <NUM_LIT> ) return NumElts ;

if ( Size == <NUM_LIT> ) return <NUM_LIT> * NumElts ;


<s> MVT <ISA_LIT>TargetLowering :: getRegisterTypeForCallingConv ( LLVMContext & Context , CallingConv :: ID CC , EVT VT ) const { if ( CC != CallingConv :: <ISA_LIT>_KERNEL && VT . isVector ( ) ) { EVT ScalarVT = VT . getScalarType ( ) ;

unsigned Size = ScalarVT . getSizeInBits ( ) ;


<s> if ( Size == <NUM_LIT> ) return <STR_LIT> ; if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) && isPowerOf2_32 ( VT . getVectorNumElements ( ) ) ) return VT . isInteger ( ) ? <STR_LIT> : <STR_LIT> ; } return TargetLowering :: getRegisterTypeForCallingConv ( Context , CC , VT ) ;


<s> IntermediateVT = RegisterVT ; NumIntermediates = <NUM_LIT> * NumElts ; return NumIntermediates ; } if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) && isPowerOf2_32 ( NumElts ) ) { RegisterVT = VT . isInteger ( ) ? <STR_LIT> : <STR_LIT> ; IntermediateVT = RegisterVT ; NumIntermediates = NumElts / <NUM_LIT> ; return NumIntermediates ; }


<s> unsigned <ISA_LIT>TargetLowering :: getVectorTypeBreakdownForCallingConv ( LLVMContext & Context , CallingConv :: ID CC , EVT VT , EVT & IntermediateVT , unsigned & NumIntermediates , MVT & RegisterVT ) const { if ( CC != CallingConv :: <ISA_LIT>_KERNEL && VT . isVector ( ) ) { unsigned NumElts = VT . getVectorNumElements ( ) ; EVT ScalarVT = VT . getScalarType ( ) ; unsigned Size = ScalarVT . getSizeInBits ( ) ; if ( Size == <NUM_LIT> ) {

RegisterVT = ScalarVT . getSimpleVT ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return Op . getValueType ( ) . getScalarType ( ) != <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { bool IsIEEEMode = Subtarget -> enableIEEEBit ( DAG . getMachineFunction ( ) ) ; if ( IsIEEEMode ) { if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( Op . getValueType ( ) ) ) return true ; return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } if ( Subtarget -> supportsMinMaxDenormModes ( ) || denormalsEnabledForType ( Op . getValueType ( ) ) ) { return DAG . isKnownNeverSNaN ( Op . getOperand ( <NUM_LIT> ) ) && DAG . isKnownNeverSNaN ( Op . getOperand ( <NUM_LIT> ) ) ; } return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { for ( unsigned i = <NUM_LIT> , e = Op . getNumOperands ( ) ; i != e ; ++ i ) { SDValue SrcOp = Op . getOperand ( i ) ; if ( ! isCanonicalized ( DAG , SrcOp , MaxDepth - <NUM_LIT> ) ) return false ; } return true ; } case <STR_LIT> : case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : { return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) && isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; } case <STR_LIT> : return false ; case <STR_LIT> : { unsigned IntrinsicID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IntrinsicID ) {


<s> bool <ISA_LIT>TargetLowering :: isCheapAddrSpaceCast ( unsigned SrcAS , unsigned DestAS ) const { if ( SrcAS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) return true ;

return isNoopAddrSpaceCast ( SrcAS , DestAS ) ;


<s> return AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ;


<s> static bool isFlatGlobalAddrSpace ( unsigned AS , <ISA_LIT>AS AMDGPUA<ISA_LIT> ) { return AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ;


<s> if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { return isLegalMUBUFAddressingMode ( AM ) ; } else if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . REGION_ADDRESS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS || AS == AMDGPUA<ISA_LIT> . UNKNOWN_ADDRESS_SPACE ) { return isLegalFlatAddressingMode ( AM ) ; } else {


<s> return isFlatGlobalAddrSpace ( SrcAS , AMDGPUA<ISA_LIT> ) && isFlatGlobalAddrSpace ( DestAS , AMDGPUA<ISA_LIT> ) ;


<s> bool <ISA_LIT>TargetLowering :: isOffsetFoldingLegal ( const GlobalAddressSDNode * GA ) const { return ( GA -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || GA -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || GA -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) && ! shouldEmitGOTReloc ( GA -> getGlobal ( ) ) ;


<s> switch ( N -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : { const RegisterSDNode * R = nullptr ; if ( N -> getOpcode ( ) == <STR_LIT> ) { R = dyn_cast < RegisterSDNode > ( N ) ; } else { R = dyn_cast < RegisterSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; } if ( R ) { const MachineFunction * MF = FLI -> MF ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; unsigned Reg = R -> getReg ( ) ; if ( TRI . isPhysicalRegister ( Reg ) ) return TRI . isVGPR ( MRI , Reg ) ; if ( MRI . isLiveIn ( Reg ) ) { if ( TRI . isVGPR ( MRI , Reg ) ) return true ; else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; } return ! DA || DA -> isDivergent ( FLI -> getValueFromVirtualReg ( Reg ) ) ; } } break ; case <STR_LIT> : { const LoadSDNode * L = dyn_cast < LoadSDNode > ( N ) ; if ( L -> getMemOperand ( ) -> getAddrSpace ( ) == Subtarget -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ) return true ; } break ; case <STR_LIT> : return true ; break ; case <STR_LIT> : { } return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ;


<s> SDValue <ISA_LIT>TargetLowering :: lowerADDRSPACECAST ( SDValue Op , SelectionDAG & DAG ) const { SDLoc SL ( Op ) ; const AddrSpaceCastSDNode * ASC = cast < AddrSpaceCastSDNode > ( Op ) ; SDValue Src = ASC -> getOperand ( <NUM_LIT> ) ; SDValue FlatNullPtr = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( getTargetMachine ( ) ) ; if ( ASC -> getSrcAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { unsigned DestAS = ASC -> getDestAddressSpace ( ) ; if ( DestAS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || DestAS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , FlatNullPtr , <STR_LIT> ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , Ptr , SegmentNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { unsigned SrcAS = ASC -> getSrcAddressSpace ( ) ; if ( SrcAS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || SrcAS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( SrcAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ;

SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , SegmentNullPtr , <STR_LIT> ) ;


<s> SDValue Src = ASC -> getOperand ( <NUM_LIT> ) ; SDValue FlatNullPtr = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( getTargetMachine ( ) ) ; if ( ASC -> getSrcAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { unsigned DestAS = ASC -> getDestAddressSpace ( ) ; if ( DestAS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || DestAS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , FlatNullPtr , <STR_LIT> ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , Ptr , SegmentNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) {

unsigned SrcAS = ASC -> getSrcAddressSpace ( ) ;


<s> MVT SimpleVT = VT . getSimpleVT ( ) ; MVT VecType = <STR_LIT> ( SimpleVT , <NUM_LIT> ) ; SDValue NewOld = DAG . getBuildVector ( VecType , DL , { New , Old } ) ; SDValue Ops [ ] = { ChainIn , Addr , NewOld } ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , AtomicNode -> getMemOperand ( ) ) ;


<s> SDValue CastLo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Lo ) ; SDValue CastHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi ) ; SDValue Blend = DAG . getBuildVector ( <STR_LIT> , SL , { CastLo , CastHi } ) ; return DAG . getNode ( <STR_LIT> , SL , VT , Blend ) ; } assert ( VT == <STR_LIT> || VT == <STR_LIT> ) ; SDValue Lo = Op . getOperand ( <NUM_LIT> ) ; SDValue Hi = Op . getOperand ( <NUM_LIT> ) ; Lo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Lo ) ; Hi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi ) ; Lo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Lo ) ; Hi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi ) ; SDValue ShlHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ; SDValue Or = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Lo , ShlHi ) ; return DAG . getNode ( <STR_LIT> , SL , VT , Or ) ;


<s> const GlobalValue * GV = GSD -> getGlobal ( ) ; if ( GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS && GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT && GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS && ! GV -> getType ( ) -> getElementType ( ) -> isFunctionTy ( ) ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; EVT PtrVT = Op . getValueType ( ) ; if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ;

unsigned Align = DataLayout . getABITypeAlignment ( PtrTy ) ;


<s> const GlobalValue * GV = GSD -> getGlobal ( ) ; if ( GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS && GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT && GSD -> getAddressSpace ( ) != AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS && ! GV -> getType ( ) -> getElementType ( ) -> isFunctionTy ( ) ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; EVT PtrVT = Op . getValueType ( ) ; if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( PtrTy ) ;

MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ;


<s> Type * Ty = MemVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; if ( MemVT . getStoreSize ( ) < <NUM_LIT> && Align < <NUM_LIT> ) { int64_t AlignDownOffset = alignDown ( Offset , <NUM_LIT> ) ; int64_t OffsetDiff = Offset - AlignDownOffset ; EVT IntVT = MemVT . changeTypeToInteger ( ) ; SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , AlignDownOffset ) ; SDValue Load = DAG . getLoad ( <STR_LIT> , SL , Chain , Ptr , PtrInfo , <NUM_LIT> , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; SDValue ShiftAmt = DAG . getConstant ( OffsetDiff * <NUM_LIT> , SL , <STR_LIT> ) ;

SDValue Extract = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Load , ShiftAmt ) ;


<s> MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; if ( MemVT . getStoreSize ( ) < <NUM_LIT> && Align < <NUM_LIT> ) { int64_t AlignDownOffset = alignDown ( Offset , <NUM_LIT> ) ; int64_t OffsetDiff = Offset - AlignDownOffset ; EVT IntVT = MemVT . changeTypeToInteger ( ) ; SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , AlignDownOffset ) ; SDValue Load = DAG . getLoad ( <STR_LIT> , SL , Chain , Ptr , PtrInfo , <NUM_LIT> , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;

SDValue ShiftAmt = DAG . getConstant ( OffsetDiff * <NUM_LIT> , SL , <STR_LIT> ) ;


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * InputPtrReg ; const TargetRegisterClass * RC ; std :: tie ( InputPtrReg , RC ) = Info -> getPreloadedValue ( <STR_LIT> ) ;

MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ;


<s> MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg -> getRegister ( ) ) , PtrVT ) ; return DAG . getObjectPtrOffset ( SL , BasePtr , Offset ) ;


<s> if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ;


<s> assert ( ( ! Result . getNode ( ) || Result . getNode ( ) -> getNumValues ( ) == <NUM_LIT> ) && <STR_LIT> ) ; return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ;

case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ;


<s> return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ;


<s> SDLoc DL ( Op ) ; StoreSDNode * Store = cast < StoreSDNode > ( Op ) ; EVT VT = Store -> getMemoryVT ( ) ; if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ;


<s> auto & ArgUsageInfo = DAG . getPass ( ) -> getAnalysis < <ISA_LIT>ArgumentUsageInfo > ( ) ; const <ISA_LIT>FunctionArgInfo & CalleeArgInfo = ArgUsageInfo . lookupFuncArgInfo ( * CalleeFunc ) ; const <ISA_LIT>FunctionArgInfo & CallerArgInfo = Info . getArgInfo ( ) ; <STR_LIT> InputRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; for ( auto InputID : InputRegs ) { const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo . getPreloadedValue ( InputID ) ; if ( ! OutgoingArg ) continue ; const ArgDescriptor * IncomingArg ; const TargetRegisterClass * IncomingArgRC ; std :: tie ( IncomingArg , IncomingArgRC ) = CallerArgInfo . getPreloadedValue ( InputID ) ; assert ( IncomingArgRC == ArgRC ) ; EVT ArgVT = TRI -> getSpillSize ( * ArgRC ) == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ;


<s> return DAG . getNode ( <STR_LIT> , SL , VT , NewVec ) ; } if ( convertBuildVectorCastElt ( DAG , Hi , Lo ) ) { SDValue NewVec = DAG . getBuildVector ( <STR_LIT> , SL , { Hi , Lo } ) ;


<s> SDValue <ISA_LIT>TargetLowering :: performBuildVectorCombine ( SDNode * N , DAGCombinerInfo & DCI ) const { SDLoc SL ( N ) ; if ( ! isTypeLegal ( <STR_LIT> ) ) return SDValue ( ) ; SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> ) { SDValue Lo = N -> getOperand ( <NUM_LIT> ) ; SDValue Hi = N -> getOperand ( <NUM_LIT> ) ; if ( convertBuildVectorCastElt ( DAG , Lo , Hi ) ) { SDValue NewVec = DAG . getBuildVector ( <STR_LIT> , SL , { Lo , Hi } ) ; return DAG . getNode ( <STR_LIT> , SL , VT , NewVec ) ; } if ( convertBuildVectorCastElt ( DAG , Hi , Lo ) ) { SDValue NewVec = DAG . getBuildVector ( <STR_LIT> , SL , { Hi , Lo } ) ; return DAG . getNode ( <STR_LIT> , SL , VT , NewVec ) ; } } return SDValue ( ) ;


<s> case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : return performRcpCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> : return performFMed3Combine ( N , DCI ) ; case <STR_LIT> : return performCvtPkRTZCombine ( N , DCI ) ; case <STR_LIT> : return performClampCombine ( N , DCI ) ; case <STR_LIT> : { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> || VT == <STR_LIT> ) { SDLoc SL ( N ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DAG . getNode ( Opc , SL , EltVT , DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) , DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ) ; } } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ; unsigned LeftoverBitIdx = BitIndex % <NUM_LIT> ; SDLoc SL ( N ) ; SDValue Cast = DAG . getNode ( <STR_LIT> , SL , NewVT , Vec ) ; DCI . AddToWorklist ( Cast . getNode ( ) ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cast , DAG . getConstant ( EltIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Elt . getNode ( ) ) ; SDValue Srl = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Elt , DAG . getConstant ( LeftoverBitIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Srl . getNode ( ) ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , SL , EltVT . changeTypeToInteger ( ) , Srl ) ; DCI . AddToWorklist ( Trunc . getNode ( ) ) ; return DAG . getNode ( <STR_LIT> , SL , EltVT , Trunc ) ; } return SDValue ( ) ;


<s> } if ( VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> hasMed3_16 ( ) ) ) { SDValue Var = Op0 . getOperand ( <NUM_LIT> ) ; if ( ! DAG . isKnownNeverSNaN ( Var ) ) return SDValue ( ) ;

return DAG . getNode ( <STR_LIT> , SL , K0 -> getValueType ( <NUM_LIT> ) , Var , SDValue ( K0 , <NUM_LIT> ) , SDValue ( K1 , <NUM_LIT> ) ) ;


<s> if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , true ) ) return Med3 ; } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , false ) ) return Med3 ; } if ( ( ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) || ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) ) && ( VT == <STR_LIT> || VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> has16BitInsts ( ) ) || ( VT == <STR_LIT> && Subtarget -> hasVOP3PInsts ( ) ) ) && Op0 . hasOneUse ( ) ) { if ( SDValue Res = performFPMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 ) ) return Res ;


<s> Info . setScratchRSrcReg ( PrivateSegmentBufferReg ) ; if ( MFI . hasCalls ( ) ) { unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } else { unsigned PrivateSegmentWaveByteOffsetReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchWaveOffsetReg ( PrivateSegmentWaveByteOffsetReg ) ; } } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; if ( HasStackObjects && ! MFI . hasCalls ( ) ) { unsigned ScratchWaveOffsetReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchWaveOffsetReg ( ScratchWaveOffsetReg ) ;


<s> static void reservePrivateMemoryRegs ( const TargetMachine & TM , MachineFunction & MF , const <ISA_LIT>RegisterInfo & TRI , <ISA_LIT>MachineFunctionInfo & Info ) { MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; bool HasStackObjects = MFI . hasStackObjects ( ) ; if ( HasStackObjects ) Info . setHasNonSpillStackObjects ( true ) ; if ( TM . getOptLevel ( ) == CodeGenOpt :: None ) HasStackObjects = true ; bool RequiresStackAccess = HasStackObjects || MFI . hasCalls ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ST . isAmdCodeObjectV2 ( MF . getFunction ( ) ) ) { if ( RequiresStackAccess ) { unsigned PrivateSegmentBufferReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchRSrcReg ( PrivateSegmentBufferReg ) ; if ( MFI . hasCalls ( ) ) { unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ;


<s> return ( GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) && <STR_LIT> ( TT ) ;


<s> bool <ISA_LIT>TargetLowering :: shouldEmitFixup ( const GlobalValue * GV ) const { const Triple & TT = getTargetMachine ( ) . getTargetTriple ( ) ;

return ( GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) && <STR_LIT> ( TT ) ;


<s> bool <ISA_LIT>TargetLowering :: shouldEmitGOTReloc ( const GlobalValue * GV ) const { return ( GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) && ! shouldEmitFixup ( GV ) && ! getTargetMachine ( ) . shouldAssumeDSOLocal ( * GV -> getParent ( ) , GV ) ;


<s> if ( ( MemVT . isSimple ( ) && ! DCI . isAfterLegalizeDAG ( ) ) || MemVT . getSizeInBits ( ) >= <NUM_LIT> ) return SDValue ( ) ; SDLoc SL ( Ld ) ; assert ( ( ! MemVT . isVector ( ) || Ld -> getExtensionType ( ) == <STR_LIT> ) && <STR_LIT> ) ; SDValue Ptr = Ld -> getBasePtr ( ) ; SDValue NewLoad = DAG . getLoad ( <STR_LIT> , <STR_LIT> , <STR_LIT> , SL , Ld -> getChain ( ) , Ptr , Ld -> getOffset ( ) , Ld -> getPointerInfo ( ) , <STR_LIT> , Ld -> getAlignment ( ) , Ld -> getMemOperand ( ) -> getFlags ( ) , Ld -> getAAInfo ( ) , nullptr ) ; EVT TruncVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , MemVT . getSizeInBits ( ) ) ; if ( MemVT . isFloatingPoint ( ) ) { assert ( Ld -> getExtensionType ( ) == <STR_LIT> && <STR_LIT> ) ;

TruncVT = MemVT . changeTypeToInteger ( ) ;


<s> for ( MachineInstr * Instr : EarlyTermInstrs ) { if ( MF . getFunction ( ) . getCallingConv ( ) != CallingConv :: <ISA_LIT>_GS ) earlyTerm ( * Instr , EarlyExitBlock ) ; Instr -> eraseFromParent ( ) ; } EarlyTermInstrs . clear ( ) ; MadeChange = true ; } if ( ! EpilogInstrs . empty ( ) ) { MachineBasicBlock * EmptyMBBAtEnd = nullptr ; assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( EpilogInstrs . size ( ) > <NUM_LIT> ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } for ( auto MI : EpilogInstrs ) { auto MBB = MI -> getParent ( ) ; if ( MBB == & MF . back ( ) && MI == & MBB -> back ( ) ) continue ; if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB -> addSuccessor ( EmptyMBBAtEnd ) ; MDT -> getBase ( ) . insertEdge ( MBB , EmptyMBBAtEnd ) ; BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI -> eraseFromParent ( ) ;


<s> } EarlyTermInstrs . clear ( ) ; MadeChange = true ; } if ( ! EpilogInstrs . empty ( ) ) { MachineBasicBlock * EmptyMBBAtEnd = nullptr ; assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( EpilogInstrs . size ( ) > <NUM_LIT> ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } for ( auto MI : EpilogInstrs ) { auto MBB = MI -> getParent ( ) ; if ( MBB == & MF . back ( ) && MI == & MBB -> back ( ) ) continue ; if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB -> addSuccessor ( EmptyMBBAtEnd ) ; MDT -> getBase ( ) . insertEdge ( MBB , EmptyMBBAtEnd ) ; BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI -> eraseFromParent ( ) ; MadeChange = true ; }


<s> void <ISA_LIT>LateBranchLowering :: earlyTerm ( MachineInstr & MI , MachineBasicBlock * EarlyExitBlock ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc DL = MI . getDebugLoc ( ) ; auto BranchMI = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ; auto Next = std :: next ( MI . getIterator ( ) ) ; if ( Next != MBB . end ( ) && ! Next -> isTerminator ( ) ) splitBlock ( MBB , * BranchMI , MDT ) ;


<s> if ( IsPS && ( HasExports || MustExport ) ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: ET_NULL ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; } BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> StringRef getPassName ( ) const override {


<s> } if ( ! EarlyTermInstrs . empty ( ) ) { MachineBasicBlock * EarlyExitBlock = MF . CreateMachineBasicBlock ( ) ; DebugLoc DL ; MF . insert ( MF . end ( ) , EarlyExitBlock ) ; BuildMI ( * EarlyExitBlock , EarlyExitBlock -> end ( ) , DL , TII -> get ( MovOpc ) , ExecReg ) . addImm ( <NUM_LIT> ) ; generateEndPgm ( * EarlyExitBlock , EarlyExitBlock -> end ( ) , DL , TII , MF ) ; for ( MachineInstr * Instr : EarlyTermInstrs ) { if ( MF . getFunction ( ) . getCallingConv ( ) != CallingConv :: <ISA_LIT>_GS ) earlyTerm ( * Instr , EarlyExitBlock ) ; Instr -> eraseFromParent ( ) ; } EarlyTermInstrs . clear ( ) ; MadeChange = true ; } if ( ! EpilogInstrs . empty ( ) ) { MachineBasicBlock * EmptyMBBAtEnd = nullptr ; assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( EpilogInstrs . size ( ) > <NUM_LIT> ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } for ( auto MI : EpilogInstrs ) { auto MBB = MI -> getParent ( ) ; if ( MBB == & MF . back ( ) && MI == & MBB -> back ( ) ) continue ; if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB -> addSuccessor ( EmptyMBBAtEnd ) ; MDT -> getBase ( ) . insertEdge ( MBB , EmptyMBBAtEnd ) ; BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ;


<s> <ISA_LIT>LateBranchLowering ( ) : MachineFunctionPass ( ID ) {


<s> else if ( Op . readsReg ( ) && Op . getReg ( ) . isPhysical ( ) ) PhysRegUses . insert ( Op . getReg ( ) ) ; } }


<s> for ( std :: list < CombineInfo > & AddrList : MergeableInsts ) { if ( AddrList . front ( ) . InstClass == CI . InstClass && AddrList . front ( ) . hasSameBaseAddress ( * CI . I ) ) { AddrList . emplace_back ( CI ) ; return ; } }

MergeableInsts . emplace_back ( <NUM_LIT> , CI ) ;


<s> if ( Use . isReg ( ) && ( ( Use . readsReg ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && Use . getReg ( ) . isPhysical ( ) && PhysRegUses . count ( Use . getReg ( ) ) ) ) ) { Insts . push_back ( & MI ) ; addDefsUsesToList ( MI , RegDefs , PhysRegUses ) ; return true ;


<s> for ( MachineOperand & Use : MI . operands ( ) ) { if ( Use . isReg ( ) && ( ( Use . readsReg ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && Use . getReg ( ) . isPhysical ( ) && PhysRegUses . count ( Use . getReg ( ) ) ) ) ) { Insts . push_back ( & MI ) ; addDefsUsesToList ( MI , RegDefs , PhysRegUses ) ; return true ; } } return false ;


<s> addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; int Swizzled = <STR_LIT> ( MBBI -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && MBBI -> getOperand ( Swizzled ) . getImm ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , InstsToMove ) ) continue ; if ( & * MBBI == & * Paired . I ) { if ( TRI -> hasAGPRs ( getDataRegClass ( * MBBI ) ) != IsAGPR ) return false ; if ( IsAGPR && CI . InstClass == DS_WRITE ) return false ; if ( canMoveInstsAcrossMemOp ( * MBBI , InstsToMove , AA ) ) { if ( CI . InstClass != MIMG ) offsetsCanBeCombined ( CI , * STM , Paired , true ) ; return true ; } return false ; } if ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , InstsToMove , AA ) ) break ; } return false ;


<s> if ( ! CI . hasMergeableAddress ( * MRI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; addInstToMergeableList ( CI , MergeableInsts ) ; } for ( std :: list < std :: list < CombineInfo >> :: iterator I = MergeableInsts . begin ( ) , E = MergeableInsts . end ( ) ; I != E ; ) { std :: list < CombineInfo > & MergeList = * I ; if ( MergeList . size ( ) <= <NUM_LIT> ) { I = MergeableInsts . erase ( I ) ; continue ; } MergeList . sort ( [ ] ( const CombineInfo & A , const CombineInfo & B ) { return A . Offset < B . Offset ;


<s> MachineMemOperand * MMO = MF . getMachineMemOperand ( A , <NUM_LIT> , Size ) ; MMO -> setOffset ( MinOffset ) ; return MMO ;


<s> Register DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; Register DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ; MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ; Register FullDestReg = MRI -> createVirtualRegister ( TRI -> getVGPR64Class ( ) ) ;

MachineInstr * FullBase = BuildMI ( * MBB , MBBI , DL , TII -> get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ;


<s> MachineInstr * Mov = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Val ) ; ( void ) Mov ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Mov -> dump ( ) ) ;


<s> APInt V ( <NUM_LIT> , Val , true ) ; if ( TII -> isInlineConstant ( V ) ) return MachineOperand :: CreateImm ( Val ) ; Register Reg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * Mov = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Val ) ; ( void ) Mov ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Mov -> dump ( ) ) ; return MachineOperand :: CreateReg ( Reg , false ) ;


<s> unsigned OperandsToMatch [ ] = { <STR_LIT> :: cpol , <STR_LIT> :: d16 , <STR_LIT> :: unorm , <STR_LIT> :: da , <STR_LIT> :: r128 , <STR_LIT> :: a16 } ; for ( auto op : OperandsToMatch ) { int Idx = <STR_LIT> ( CI . I -> getOpcode ( ) , op ) ; if ( <STR_LIT> ( Paired . I -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ; unsigned MinMask = std :: min ( CI . DMask , Paired . DMask ) ;

unsigned AllowedBitsForMin = llvm :: countTrailingZeros ( MaxMask ) ;


<s> int Idx = <STR_LIT> ( CI . I -> getOpcode ( ) , op ) ; if ( <STR_LIT> ( Paired . I -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ; unsigned MinMask = std :: min ( CI . DMask , Paired . DMask ) ; unsigned AllowedBitsForMin = llvm :: countTrailingZeros ( MaxMask ) ; if ( ( <NUM_LIT> << AllowedBitsForMin ) <= MinMask ) return false ; return true ;


<s> if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || ! Def -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return None ; return Def -> getOperand ( <NUM_LIT> ) . getImm ( ) ;


<s> if ( ! Op . isReg ( ) ) return None ; MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ;

if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || ! Def -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return None ;


<s> const llvm :: <STR_LIT> * NewFormatInfo = llvm :: <STR_LIT> ( OldFormatInfo -> BitsPerComp , ComponentCount , OldFormatInfo -> NumFormat , STI ) ; if ( ! NewFormatInfo ) return <NUM_LIT> ;

assert ( NewFormatInfo -> NumFormat == OldFormatInfo -> NumFormat && NewFormatInfo -> BitsPerComp == OldFormatInfo -> BitsPerComp ) ;


<s> static unsigned getBufferFormatWithCompCount ( unsigned OldFormat , unsigned ComponentCount , const GCNSubtarget & STI ) { if ( ComponentCount > <NUM_LIT> ) return <NUM_LIT> ; const llvm :: <STR_LIT> * OldFormatInfo = llvm :: <STR_LIT> ( OldFormat , STI ) ; if ( ! OldFormatInfo ) return <NUM_LIT> ; const llvm :: <STR_LIT> * NewFormatInfo = llvm :: <STR_LIT> ( OldFormatInfo -> BitsPerComp , ComponentCount , OldFormatInfo -> NumFormat , STI ) ; if ( ! NewFormatInfo ) return <NUM_LIT> ; assert ( NewFormatInfo -> NumFormat == OldFormatInfo -> NumFormat && NewFormatInfo -> BitsPerComp == OldFormatInfo -> BitsPerComp ) ;


<s> } if ( const auto * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) { return TRI -> getRegClassForReg ( * MRI , Dst -> getReg ( ) ) ; } if ( const auto * Src = TII -> getNamedOperand ( MI , <STR_LIT> :: sdata ) ) { return TRI -> getRegClassForReg ( * MRI , Src -> getReg ( ) ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( <STR_LIT> ( Opc ) -> BVH ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DS_READ ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> switch ( Opc ) { default : if ( TII . isMUBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( <STR_LIT> ( Opc ) -> BVH ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ;


<s> case TBUFFER_LOAD : case TBUFFER_STORE : return <STR_LIT> ( <STR_LIT> ( CI . I -> getOpcode ( ) ) , Width ) ; case UNKNOWN : llvm_unreachable ( <STR_LIT> ) ; case S_BUFFER_LOAD_IMM : switch ( Width ) { default : return <NUM_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; } case MIMG : assert ( ( countPopulation ( CI . DMask | Paired . DMask ) == Width ) && <STR_LIT> ) ;

return <STR_LIT> ( CI . I -> getOpcode ( ) , Width ) ;


<s> static unsigned getOpcodeWidth ( const MachineInstr & MI , const <ISA_LIT>InstrInfo & TII ) { const unsigned Opc = MI . getOpcode ( ) ; if ( TII . isMUBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } if ( TII . isMIMG ( MI ) ) { uint64_t DMaskImm = TII . getNamedOperand ( MI , <STR_LIT> :: dmask ) -> getImm ( ) ; return countPopulation ( DMaskImm ) ; } if ( TII . isMTBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } switch ( Opc ) { case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : LLVM_FALLTHROUGH ;


<s> if ( Info && <STR_LIT> ( Info -> BaseOpcode ) -> Sampler ) Result . SSamp = true ; return Result ; } if ( TII . isMTBUF ( Opc ) ) { if ( <STR_LIT> ( Opc ) ) Result . VAddr = true ; if ( <STR_LIT> ( Opc ) ) Result . SRsrc = true ; if ( <STR_LIT> ( Opc ) ) Result . SOffset = true ; return Result ; } switch ( Opc ) { default : return Result ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result . SBase = true ; return Result ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> , <NUM_LIT> } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } , } ; assert ( CI . Width >= <NUM_LIT> && CI . Width <= <NUM_LIT> ) ; assert ( Paired . Width >= <NUM_LIT> && Paired . Width <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ Paired . Width - <NUM_LIT> ] ; Idx0 = Idxs [ Paired . Width ] [ CI . Width - <NUM_LIT> ] ; } else {

Idx0 = Idxs [ <NUM_LIT> ] [ CI . Width - <NUM_LIT> ] ;


<s> unsigned BitWidth = <NUM_LIT> * ( CI . Width + Paired . Width ) ;

return TRI -> hasAGPRs ( getDataRegClass ( * CI . I ) ) ? TRI -> getAGPRClassForBitWidth ( BitWidth ) : TRI -> getVGPRClassForBitWidth ( BitWidth ) ;


<s> if ( ! AddrOp -> isReg ( ) ) return false ; if ( AddrOp -> getReg ( ) . isPhysical ( ) ) return false ; if ( MRI . hasOneNonDBGUse ( AddrOp -> getReg ( ) ) ) return false ;


<s> const MachineOperand * AddrOp = AddrReg [ i ] ; if ( AddrOp -> isImm ( ) ) continue ; if ( ! AddrOp -> isReg ( ) ) return false ; if ( AddrOp -> getReg ( ) . isPhysical ( ) ) return false ; if ( MRI . hasOneNonDBGUse ( AddrOp -> getReg ( ) ) ) return false ; } return true ;


<s> if ( AddrReg [ i ] -> isImm ( ) || AddrRegNext . isImm ( ) ) { if ( AddrReg [ i ] -> isImm ( ) != AddrRegNext . isImm ( ) || AddrReg [ i ] -> getImm ( ) != AddrRegNext . getImm ( ) ) { return false ;


<s> MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ;

const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ;


<s> unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ;

MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ;


<s> const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ;

assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ;


<s> unsigned DMaskIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: dmask ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; for ( unsigned I = <NUM_LIT> , E = ( * CI . I ) . getNumOperands ( ) ; I != E ; ++ I ) { if ( I == DMaskIdx ) MIB . addImm ( MergedDMask ) ; else MIB . add ( ( * CI . I ) . getOperand ( I ) ) ; } assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; unsigned SubRegIdx0 , SubRegIdx1 ; std :: tie ( SubRegIdx0 , SubRegIdx1 ) = getSubRegIdxs ( CI , Paired ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; CI . I -> eraseFromParent ( ) ;


<s> const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdst ) ; unsigned NewOffset0 = CI . Offset ; unsigned NewOffset1 = Paired . Offset ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ;


<s> unsigned NewOffset0 = CI . Offset ; unsigned NewOffset1 = Paired . Offset ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; }

MachineInstrBuilder Read2 = BuildMI ( * MBB , Paired . I , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ;


<s> unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ;


<s> MachineInstr * New = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ;


<s> const unsigned Opcode = getNewOpcode ( CI , Paired ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STM ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . CPol ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ;


<s> std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ;


<s> MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI , Paired ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ;

auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ;


<s> if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , Paired . I , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ; moveInstsAfter ( Write2 , InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ; return Write2 ;


<s> static uint32_t mostAlignedValueInRange ( uint32_t Lo , uint32_t Hi ) { return Hi & maskLeadingOnes < uint32_t > ( countLeadingZeros ( ( Lo - <NUM_LIT> ) ^ Hi ) + <NUM_LIT> ) ;


<s> if ( ! Info0 ) return false ; const llvm :: <STR_LIT> * Info1 = llvm :: <STR_LIT> ( Paired . Format , STI ) ; if ( ! Info1 ) return false ; if ( Info0 -> BitsPerComp != Info1 -> BitsPerComp || Info0 -> NumFormat != Info1 -> NumFormat ) return false ; if ( Info0 -> BitsPerComp != <NUM_LIT> ) return false ; if ( getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , STI ) == <NUM_LIT> ) return false ; } uint32_t EltOffset0 = CI . Offset / CI . EltSize ; uint32_t EltOffset1 = Paired . Offset / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width == EltOffset1 || EltOffset1 + Paired . Width == EltOffset0 ) && CI . CPol == Paired . CPol && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . CPol == Paired . CPol ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { if ( Modify ) { CI . Offset = EltOffset0 / <NUM_LIT> ; Paired . Offset = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; } return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { if ( Modify ) { CI . Offset = EltOffset0 ; Paired . Offset = EltOffset1 ; } return true ; } uint32_t Min = std :: min ( EltOffset0 , EltOffset1 ) ; uint32_t Max = std :: max ( EltOffset0 , EltOffset1 ) ; const uint32_t Mask = maskTrailingOnes < uint32_t > ( <NUM_LIT> ) * <NUM_LIT> ; if ( ( ( Max - Min ) & ~ Mask ) == <NUM_LIT> ) { if ( Modify ) { uint32_t BaseOff = mostAlignedValueInRange ( Max - <NUM_LIT> * <NUM_LIT> , Min ) ; BaseOff |= Min & maskTrailingOnes < uint32_t > ( <NUM_LIT> ) ; CI . BaseOff = BaseOff * CI . EltSize ; CI . Offset = ( EltOffset0 - BaseOff ) / <NUM_LIT> ; Paired . Offset = ( EltOffset1 - BaseOff ) / <NUM_LIT> ; CI . UseST64 = true ; } return true ; } if ( isUInt < <NUM_LIT> > ( Max - Min ) ) { if ( Modify ) { uint32_t BaseOff = mostAlignedValueInRange ( Max - <NUM_LIT> , Min ) ; CI . BaseOff = BaseOff * CI . EltSize ; CI . Offset = EltOffset0 - BaseOff ; Paired . Offset = EltOffset1 - BaseOff ; } return true ; } return false ;


<s> for ( std :: list < std :: list < CombineInfo >> :: iterator I = MergeableInsts . begin ( ) , E = MergeableInsts . end ( ) ; I != E ; ) { std :: list < CombineInfo > & MergeList = * I ; bool OptimizeListAgain = false ; if ( ! optimizeInstsWithSameBaseAddr ( MergeList , OptimizeListAgain ) ) { I = MergeableInsts . erase ( I ) ; continue ; }


<s> bool <ISA_LIT>LoadStoreOptimizer :: optimizeInstsWithSameBaseAddr ( std :: list < CombineInfo > & MergeList , bool & OptimizeListAgain ) { if ( MergeList . empty ( ) ) return false ; bool Modified = false ; for ( auto I = MergeList . begin ( ) , Next = std :: next ( I ) ; Next != MergeList . end ( ) ; Next = std :: next ( I ) ) { auto First = I ; auto Second = Next ; if ( ( * First ) . Order > ( * Second ) . Order ) std :: swap ( First , Second ) ; CombineInfo & CI = * First ; CombineInfo & Paired = * Second ; SmallVector < MachineInstr * , <NUM_LIT> > InstsToMove ; if ( ! checkAndPrepareMerge ( CI , Paired , InstsToMove ) ) { ++ I ; continue ; } Modified = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CI . I << <STR_LIT> << * Paired . I ) ; switch ( CI . InstClass ) { default : llvm_unreachable ( <STR_LIT> ) ; break ; case DS_READ : { MachineBasicBlock :: iterator NewMI = mergeRead2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case DS_WRITE : { MachineBasicBlock :: iterator NewMI = mergeWrite2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case S_BUFFER_LOAD_IMM : { MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired , InstsToMove ) ;


<s> MachineBasicBlock :: iterator NewMI = mergeRead2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case DS_WRITE : { MachineBasicBlock :: iterator NewMI = mergeWrite2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case S_BUFFER_LOAD_IMM : { MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; }


<s> TII = STM -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; SmallPtrSet < MachineInstr * , <NUM_LIT> > AnchorList ; MemInfoMap Visited ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator SectionEnd ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = SectionEnd ) { bool CollectModified ; std :: list < std :: list < CombineInfo >> MergeableInsts ; std :: tie ( SectionEnd , CollectModified ) = collectMergeableInsts ( I , E , Visited , AnchorList , MergeableInsts ) ; Modified |= CollectModified ; do { OptimizeAgain = false ; Modified |= optimizeBlock ( MergeableInsts ) ; } while ( OptimizeAgain ) ; }


TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> setImm ( NewOffset ) ;


<s> void <ISA_LIT>LoadStoreOptimizer :: updateBaseAndOffset ( MachineInstr & MI , Register NewBase , int32_t NewOffset ) const {

auto Base = TII -> getNamedOperand ( MI , <STR_LIT> :: vaddr ) ;


<s> switch ( CI . InstClass ) { default : return ( Width <= <NUM_LIT> ) && ( STM . hasDwordx3LoadStores ( ) || ( Width != <NUM_LIT> ) ) ; case S_BUFFER_LOAD_IMM : switch ( Width ) { default : return false ; case <NUM_LIT> : case <NUM_LIT> : case <NUM_LIT> : return true ; }


<s> if ( Use . isReg ( ) && ( ( Use . readsReg ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && TargetRegisterInfo :: isPhysicalRegister ( Use . getReg ( ) ) && PhysRegUses . count ( Use . getReg ( ) ) ) ) ) { Insts . push_back ( & MI ) ; addDefsUsesToList ( MI , RegDefs , PhysRegUses ) ; return true ; } } return false ;


<s> assert ( MemOp . mayLoadOrStore ( ) ) ; for ( MachineInstr * InstToMove : InstsToMove ) { if ( ! InstToMove -> mayLoadOrStore ( ) ) continue ; if ( ! memAccessesCanBeReordered ( MemOp , * InstToMove , TII , AA ) ) return false ; }


<s> MachineBasicBlock * MBB = MI . getParent ( ) ; MachineBasicBlock :: iterator MBBI = MI . getIterator ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; assert ( ( TRI -> getRegSizeInBits ( Addr . Base . LoReg , * MRI ) == <NUM_LIT> || Addr . Base . LoSubReg ) && <STR_LIT> ) ; assert ( ( TRI -> getRegSizeInBits ( Addr . Base . HiReg , * MRI ) == <NUM_LIT> || Addr . Base . HiSubReg ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineOperand OffsetLo = createRegOrImm ( static_cast < int32_t > ( Addr . Offset ) , MI ) ; MachineOperand OffsetHi = createRegOrImm ( static_cast < int32_t > ( Addr . Offset >> <NUM_LIT> ) , MI ) ; unsigned CarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DeadCarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ; MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ; unsigned FullDestReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * FullBase = BuildMI ( * MBB , MBBI , DL , TII -> get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ; ( void ) FullBase ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; FullBase -> dump ( ) ; dbgs ( ) << <STR_LIT> ; ) ; return FullDestReg ;


<s> MachineBasicBlock * MBB = MI . getParent ( ) ; MachineBasicBlock :: iterator MBBI = MI . getIterator ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; assert ( ( TRI -> getRegSizeInBits ( Addr . Base . LoReg , * MRI ) == <NUM_LIT> || Addr . Base . LoSubReg ) && <STR_LIT> ) ; assert ( ( TRI -> getRegSizeInBits ( Addr . Base . HiReg , * MRI ) == <NUM_LIT> || Addr . Base . HiSubReg ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineOperand OffsetLo = createRegOrImm ( static_cast < int32_t > ( Addr . Offset ) , MI ) ; MachineOperand OffsetHi = createRegOrImm ( static_cast < int32_t > ( Addr . Offset >> <NUM_LIT> ) , MI ) ; unsigned CarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DeadCarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ;

unsigned DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ;


<s> } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , TII , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , TII , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = true ; for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { const MachineOperand & AddrRegNext = MBBI -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isImm ( ) || AddrRegNext . isImm ( ) ) { if ( AddrReg [ i ] -> isImm ( ) != AddrRegNext . isImm ( ) || AddrReg [ i ] -> getImm ( ) != AddrRegNext . getImm ( ) ) { Match = false ; break ; } continue ; } if ( AddrReg [ i ] -> getReg ( ) != AddrRegNext . getReg ( ) || AddrReg [ i ] -> getSubReg ( ) != AddrRegNext . getSubReg ( ) ) { Match = false ; break ; } } if ( Match ) { int OffsetIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: offset ) ; CI . Offset0 = CI . I -> getOperand ( OffsetIdx ) . getImm ( ) ; CI . Width0 = getOpcodeWidth ( * CI . I ) ; CI . Offset1 = MBBI -> getOperand ( OffsetIdx ) . getImm ( ) ; CI . Width1 = getOpcodeWidth ( * MBBI ) ; CI . Paired = MBBI ; if ( ( CI . InstClass == DS_READ ) || ( CI . InstClass == DS_WRITE ) ) { CI . Offset0 &= <NUM_LIT> ; CI . Offset1 &= <NUM_LIT> ; } else { CI . GLC0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: glc ) -> getImm ( ) ; CI . GLC1 = TII -> getNamedOperand ( * MBBI , <STR_LIT> :: glc ) -> getImm ( ) ; if ( CI . InstClass != S_BUFFER_LOAD_IMM ) { CI . SLC0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: slc ) -> getImm ( ) ; CI . SLC1 = TII -> getNamedOperand ( * MBBI , <STR_LIT> :: slc ) -> getImm ( ) ; } } if ( widthsFit ( * STM , CI ) && offsetsCanBeCombined ( CI ) ) if ( canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , TII , AA ) ) return true ;


<s> static bool memAccessesCanBeReordered ( MachineBasicBlock :: iterator A , MachineBasicBlock :: iterator B , const <ISA_LIT>InstrInfo * TII , AliasAnalysis * AA ) { return ! ( A -> mayStore ( ) || B -> mayStore ( ) ) || TII -> areMemAccessesTriviallyDisjoint ( * A , * B , AA ) ;


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeBufferLoadPair ( CombineInfo & CI ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ;

const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ;


<s> DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ;

BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ;


<s> const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ;


<s> unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ;


<s> const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ;


<s> const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; return Next ;


<s> unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; }

MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ;


<s> BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ;


<s> unsigned EltOffset0 = CI . Offset0 / CI . EltSize ; unsigned EltOffset1 = CI . Offset1 / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width0 == EltOffset1 || EltOffset1 + CI . Width1 == EltOffset0 ) && CI . GLC0 == CI . GLC1 && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC0 == CI . SLC1 ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { CI . Offset0 = EltOffset0 / <NUM_LIT> ; CI . Offset1 = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset0 = EltOffset0 ; CI . Offset1 = EltOffset1 ; return true ;


<s> for ( ; BlockI != End ; ++ BlockI ) { MachineInstr & MI = * BlockI ; if ( promoteConstantOffsetToImm ( MI , Visited , AnchorList ) ) Modified = true ; if ( MI . hasOrderedMemoryRef ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; ++ BlockI ; break ; } const InstClassEnum InstClass = getInstClass ( MI . getOpcode ( ) , * TII ) ; if ( InstClass == UNKNOWN ) continue ; CombineInfo CI ; CI . setMI ( MI , * TII , * STM ) ; CI . Order = Order ++ ; if ( ! CI . hasMergeableAddress ( * MRI ) ) continue ;


<s> if ( promoteConstantOffsetToImm ( MI , Visited , AnchorList ) ) Modified = true ; if ( MI . hasOrderedMemoryRef ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; ++ BlockI ; break ; } const InstClassEnum InstClass = getInstClass ( MI . getOpcode ( ) , * TII ) ; if ( InstClass == UNKNOWN ) continue ; CombineInfo CI ; CI . setMI ( MI , * TII , * STM ) ; CI . Order = Order ++ ;

if ( ! CI . hasMergeableAddress ( * MRI ) ) continue ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> :


<s> return Info -> BaseOpcode ; } if ( TII . isMTBUF ( Opc ) ) return <STR_LIT> ( Opc ) ; return - <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return Opc ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> if ( TII . isMIMG ( MI ) ) { uint64_t DMaskImm = TII . getNamedOperand ( MI , <STR_LIT> :: dmask ) -> getImm ( ) ; return countPopulation ( DMaskImm ) ; } if ( TII . isMTBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } switch ( Opc ) { case <STR_LIT> : return <NUM_LIT> ;


<s> assert ( CI . Width >= <NUM_LIT> && CI . Width <= <NUM_LIT> ) ; assert ( Paired . Width >= <NUM_LIT> && Paired . Width <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ Paired . Width - <NUM_LIT> ] ; Idx0 = Idxs [ Paired . Width ] [ CI . Width - <NUM_LIT> ] ; } else { Idx0 = Idxs [ <NUM_LIT> ] [ CI . Width - <NUM_LIT> ] ; Idx1 = Idxs [ CI . Width ] [ Paired . Width - <NUM_LIT> ] ; } return std :: make_pair ( Idx0 , Idx1 ) ;


<s> Next = std :: next ( I ) ) { auto First = I ; auto Second = Next ; if ( ( * First ) . Order > ( * Second ) . Order ) std :: swap ( First , Second ) ; CombineInfo & CI = * First ; CombineInfo & Paired = * Second ; SmallVector < MachineInstr * , <NUM_LIT> > InstsToMove ; if ( ! checkAndPrepareMerge ( CI , Paired , InstsToMove ) ) { ++ I ; continue ; } Modified = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CI . I << <STR_LIT> << * Paired . I ) ; switch ( CI . InstClass ) { default : llvm_unreachable ( <STR_LIT> ) ; break ; case DS_READ : { MachineBasicBlock :: iterator NewMI = mergeRead2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case DS_WRITE : { MachineBasicBlock :: iterator NewMI = mergeWrite2Pair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; break ; } case S_BUFFER_LOAD_IMM : { MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ;


<s> static void addDefsToList ( const MachineInstr & MI , SmallVectorImpl < const MachineOperand * > & Defs ) {

for ( const MachineOperand & Def : MI . defs ( ) ) {


<s> bool ReadDef = MI . readsVirtualRegister ( Def -> getReg ( ) ) ; if ( ReadDef ) { Insts . push_back ( & MI ) ; addDefsToList ( MI , Defs ) ; return true ; } } return false ;


<s> continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ) continue ; int AddrIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: addr ) ; const MachineOperand & AddrReg0 = CI . I -> getOperand ( AddrIdx ) ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) { int OffsetIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: offset ) ; CI . Offset0 = CI . I -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ;

CI . Offset1 = MBBI -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ;


<s> ++ MBBI ; SmallVector < const MachineOperand * , <NUM_LIT> > DefsToMove ; addDefsToList ( * CI . I , DefsToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( MBBI -> getOpcode ( ) != CI . I -> getOpcode ( ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) return false ; if ( MBBI -> mayLoadOrStore ( ) && ! memAccessesCanBeReordered ( * CI . I , * MBBI , TII , AA ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsToList ( * MBBI , DefsToMove ) ; continue ; } addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ;

if ( addToListsIfDependent ( * MBBI , DefsToMove , CI . InstsToMove ) ) continue ;


<s> const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addImm ( CI . BaseOff ) . addReg ( AddrReg -> getReg ( ) ) ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ;


<s> if ( CI . BaseOff ) { BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addImm ( CI . BaseOff ) . addReg ( Addr -> getReg ( ) ) ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ;


<s> if ( ( CI . Offset0 % CI . EltSize != <NUM_LIT> ) || ( CI . Offset1 % CI . EltSize != <NUM_LIT> ) ) return false ; unsigned EltOffset0 = CI . Offset0 / CI . EltSize ; unsigned EltOffset1 = CI . Offset1 / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { CI . Offset0 = EltOffset0 / <NUM_LIT> ; CI . Offset1 = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset0 = EltOffset0 ; CI . Offset1 = EltOffset1 ; return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset0 , CI . Offset1 ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { CI . Offset0 = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . Offset1 = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( OffsetDiff ) ) { CI . Offset0 = EltOffset0 - CI . BaseOff / CI . EltSize ; CI . Offset1 = EltOffset1 - CI . BaseOff / CI . EltSize ; return true ; }


<s> bool <ISA_LIT>LoadStoreOptimizer :: offsetsCanBeCombined ( CombineInfo & CI ) { if ( CI . Offset0 == CI . Offset1 ) return false ; if ( ( CI . Offset0 % CI . EltSize != <NUM_LIT> ) || ( CI . Offset1 % CI . EltSize != <NUM_LIT> ) ) return false ; unsigned EltOffset0 = CI . Offset0 / CI . EltSize ; unsigned EltOffset1 = CI . Offset1 / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { CI . Offset0 = EltOffset0 / <NUM_LIT> ; CI . Offset1 = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset0 = EltOffset0 ; CI . Offset1 = EltOffset1 ; return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset0 , CI . Offset1 ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { CI . Offset0 = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . Offset1 = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64 = true ; return true ;


<s> if ( findMatchingD<ISA_LIT>nst ( CI ) ) { Modified = true ; I = mergeRead2Pair ( CI ) ; } else { ++ I ; } continue ; } else if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { CI . EltSize = ( Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; if ( findMatchingD<ISA_LIT>nst ( CI ) ) { Modified = true ; I = mergeWrite2Pair ( CI ) ; } else { ++ I ; } continue ; } ++ I ; } return Modified ;


<s> AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ; return Modified ;


<s> if ( ! STM . loadStoreOptEnabled ( ) ) return false ; TII = STM . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>LoadStoreOptimizerPass ( TargetMachine & TM ) {

return new <ISA_LIT>LoadStoreOptimizer ( TM ) ;


<s> return new <ISA_LIT>LoadStoreOptimizer ( TM ) ;


<s> MachineBasicBlock :: iterator MBBI = I ; ++ MBBI ; SmallVector < const MachineOperand * , <NUM_LIT> > DefsToMove ; addDefsToList ( * I , DefsToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( MBBI -> getOpcode ( ) != I -> getOpcode ( ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) return E ; if ( MBBI -> mayLoadOrStore ( ) && ! memAccessesCanBeReordered ( * I , * MBBI , TII , AA ) ) { InstsToMove . push_back ( & * MBBI ) ; addDefsToList ( * MBBI , DefsToMove ) ; continue ; } addToListsIfDependent ( * MBBI , DefsToMove , InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return E ; if ( addToListsIfDependent ( * MBBI , DefsToMove , InstsToMove ) ) continue ; int AddrIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: addr ) ; const MachineOperand & AddrReg0 = I -> getOperand ( AddrIdx ) ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) {

int OffsetIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: offset ) ;