<s> SmallVector < Register , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < Register , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping Observer ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( LoadTy . isVector ( ) ) { if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; } else { if ( Helper . narrowScalar ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; } MRI . setRegBank ( DstReg , <STR_LIT> ) ;


<s> Register SOffset ; Register VOffset ; int64_t ImmOffset = <NUM_LIT> ; unsigned MMOOffset = setBufferOffsets ( B , * this , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , VOffset , SOffset , ImmOffset , Alignment ) ; const unsigned MemSize = ( Ty . getSizeInBits ( ) + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * BaseMMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; if ( MMOOffset != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset , MemSize ) ; Register RSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VIndex , <STR_LIT> ) ; SmallVector < Register , <NUM_LIT> > LoadParts ( NumLoads ) ; MachineBasicBlock :: iterator MII = MI . getIterator ( ) ; MachineInstrSpan Span ( MII , & B . getMBB ( ) ) ; for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ] = Dst ; } else { LoadParts [ i ] = MRI . createGenericVirtualRegister ( Ty ) ; MRI . setRegBank ( LoadParts [ i ] , <STR_LIT> ) ; } MachineMemOperand * MMO = BaseMMO ; if ( i != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset + <NUM_LIT> * i , MemSize ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoadParts [ i ] ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset + <NUM_LIT> * i ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; } if ( RSrcBank != & <STR_LIT> ) { B . setInstr ( * Span . begin ( ) ) ; MI . eraseFromParent ( ) ; SmallSet < Register , <NUM_LIT> > OpsToWaterfall ; OpsToWaterfall . insert ( RSrc ) ; executeInWaterfallLoop ( B , make_range ( Span . begin ( ) , Span . end ( ) ) , OpsToWaterfall , MRI ) ; } if ( NumLoads != <NUM_LIT> ) { if ( Ty . isVector ( ) ) B . buildConcatVectors ( Dst , LoadParts ) ; else B . buildMerge ( Dst , LoadParts ) ;


<s> NumLoads = LoadSize / <NUM_LIT> ; Ty = Ty . divide ( NumLoads ) ; } const Align Alignment = NumLoads > <NUM_LIT> ? Align ( <NUM_LIT> * NumLoads ) : Align ( <NUM_LIT> ) ; MachineIRBuilder B ( MI ) ; MachineFunction & MF = B . getMF ( ) ; Register SOffset ; Register VOffset ; int64_t ImmOffset = <NUM_LIT> ; unsigned MMOOffset = setBufferOffsets ( B , * this , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , VOffset , SOffset , ImmOffset , Alignment ) ; const unsigned MemSize = ( Ty . getSizeInBits ( ) + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * BaseMMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; if ( MMOOffset != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset , MemSize ) ; Register RSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VIndex , <STR_LIT> ) ; SmallVector < Register , <NUM_LIT> > LoadParts ( NumLoads ) ; MachineBasicBlock :: iterator MII = MI . getIterator ( ) ; MachineInstrSpan Span ( MII , & B . getMBB ( ) ) ; for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ] = Dst ; } else { LoadParts [ i ] = MRI . createGenericVirtualRegister ( Ty ) ; MRI . setRegBank ( LoadParts [ i ] , <STR_LIT> ) ; } MachineMemOperand * MMO = BaseMMO ; if ( i != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset + <NUM_LIT> * i , MemSize ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoadParts [ i ] ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset + <NUM_LIT> * i ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; }


<s> void changedInstr ( MachineInstr & MI ) override {


<s> Register Reg = MI . getOperand ( Op ) . getReg ( ) ; const RegisterBank * OpBank = getRegBank ( Reg , MRI , * TRI ) ; if ( OpBank -> getID ( ) != <STR_LIT> ) SGPROperandRegs . insert ( Reg ) ; }


<s> Reg = B . buildCopy ( Ty , Reg ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Reg , <STR_LIT> ) ; } Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ;


<s> LLT Ty = MRI . getType ( Reg ) ; MachineIRBuilder B ( MI ) ; if ( Bank != & <STR_LIT> ) { Reg = B . buildCopy ( Ty , Reg ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Reg , <STR_LIT> ) ; } Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ; MRI . setType ( SGPR , Ty ) ;

const TargetRegisterClass * Constrained = constrainGenericRegister ( Reg , <STR_LIT> , MRI ) ;


<s> if ( CurrBank && * CurrBank != Bank ) { Register Copy = B . buildCopy ( MRI . getType ( Reg ) , Reg ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Copy , Bank ) ; return Copy ;


<s> Register Copy = B . buildCopy ( MRI . getType ( Reg ) , Reg ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Copy , Bank ) ;


<s> } else { auto ShiftAmt = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( ShiftAmt . getReg ( <NUM_LIT> ) , RegBank ) ; B . buildAShr ( Hi32Reg , Lo32Reg , ShiftAmt ) ; }


<s> return ( CachePolicy >> <NUM_LIT> ) & <NUM_LIT> ;


<s> static unsigned extractSWZ ( unsigned CachePolicy ) { return ( CachePolicy >> <NUM_LIT> ) & <NUM_LIT> ;


<s> const RegisterBank & CCBank = ( DstBank == <STR_LIT> && SrcBank == <STR_LIT> && IdxBank == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; LLT CCTy = ( CCBank == <STR_LIT> ) ? S32 : LLT :: scalar ( <NUM_LIT> ) ; if ( CCBank == <STR_LIT> && IdxBank == <STR_LIT> ) { Idx = B . buildCopy ( S32 , Idx ) -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MRI . setRegBank ( Idx , <STR_LIT> ) ; } LLT EltTy = VecTy . getScalarType ( ) ; SmallVector < Register , <NUM_LIT> > DstRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; unsigned NumLanes = DstRegs . size ( ) ; if ( ! NumLanes ) NumLanes = <NUM_LIT> ; else EltTy = MRI . getType ( DstRegs [ <NUM_LIT> ] ) ; auto UnmergeToEltTy = B . buildUnmerge ( EltTy , VecReg ) ; SmallVector < Register , <NUM_LIT> > Res ( NumLanes ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) Res [ L ] = UnmergeToEltTy . getReg ( L ) ; for ( unsigned I = <NUM_LIT> ; I < NumElem ; ++ I ) { auto IC = B . buildConstant ( S32 , I ) ; MRI . setRegBank ( IC -> getOperand ( <NUM_LIT> ) . getReg ( ) , <STR_LIT> ) ; auto Cmp = B . buildICmp ( CmpInst :: ICMP_EQ , CCTy , Idx , IC ) ; MRI . setRegBank ( Cmp -> getOperand ( <NUM_LIT> ) . getReg ( ) , CCBank ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) { auto S = B . buildSelect ( EltTy , Cmp , UnmergeToEltTy . getReg ( I * NumLanes + L ) , Res [ L ] ) ; for ( unsigned N : { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ) MRI . setRegBank ( S -> getOperand ( N ) . getReg ( ) , DstBank ) ; Res [ L ] = S -> getOperand ( <NUM_LIT> ) . getReg ( ) ; }


<s> case TargetOpcode :: G_UMAX : return TargetOpcode :: G_ZEXT ; default : return TargetOpcode :: G_ANYEXT ; }


<s> if ( Ty . isVector ( ) ) { assert ( Ty . getElementCount ( ) . isKnownMultipleOf ( <NUM_LIT> ) ) ; return LLT :: scalarOrVector ( Ty . getElementCount ( ) . divideCoefficientBy ( <NUM_LIT> ) , Ty . getElementType ( ) ) ; }

assert ( Ty . getScalarSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> ) ;


<s> const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + <NUM_LIT> ; if ( MustBeSGPR ) { unsigned NewBank = getRegBankID ( OpReg , MRI , <STR_LIT> ) ; OpdsMapping [ I ] = <STR_LIT> ( NewBank , Size ) ; } else { OpdsMapping [ I ] = <STR_LIT> ( <STR_LIT> , Size ) ; } } return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , NumOps ) ;


<s> const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> && <STR_LIT> ( AS ) ) { if ( isScalarLoadLegal ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; unsigned PtrBankID = Subtarget . useFlatForGlobal ( ) ? <STR_LIT> : <STR_LIT> ; PtrMapping = <STR_LIT> ( PtrBankID , PtrSize ) ; } } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; const RegisterBankInfo :: InstructionMapping & Mapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ; return Mapping ;


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getInstrMappingForLoad ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( <NUM_LIT> ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT PtrTy = MRI . getType ( PtrReg ) ; unsigned AS = PtrTy . getAddressSpace ( ) ; unsigned PtrSize = PtrTy . getSizeInBits ( ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> && <STR_LIT> ( AS ) ) { if ( isScalarLoadLegal ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; unsigned PtrBankID = Subtarget . useFlatForGlobal ( ) ? <STR_LIT> : <STR_LIT> ; PtrMapping = <STR_LIT> ( PtrBankID , PtrSize ) ; } } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; const RegisterBankInfo :: InstructionMapping & Mapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ;


<s> Register Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { RegBank = regBankUnion ( RegBank , Bank -> getID ( ) ) ; if ( RegBank == <STR_LIT> ) break ;


<s> for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; Register Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { RegBank = regBankUnion ( RegBank , Bank -> getID ( ) ) ; if ( RegBank == <STR_LIT> ) break ; }


<s> const RegisterBankInfo :: ValueMapping * <ISA_LIT>RegisterBankInfo :: getSGPROpMapping ( Register Reg , const MachineRegisterInfo & MRI , const TargetRegisterInfo & TRI ) const { unsigned Bank = getRegBankID ( Reg , MRI , <STR_LIT> ) ; unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ;


<s> unsigned Size = PtrTy . getSizeInBits ( ) ; if ( Subtarget . useFlatForGlobal ( ) || ! <STR_LIT> ( PtrTy . getAddressSpace ( ) ) ) return <STR_LIT> ( <STR_LIT> , Size ) ;

const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ;


<s> LLT PtrTy = MRI . getType ( PtrReg ) ; unsigned Size = PtrTy . getSizeInBits ( ) ; if ( Subtarget . useFlatForGlobal ( ) || ! <STR_LIT> ( PtrTy . getAddressSpace ( ) ) ) return <STR_LIT> ( <STR_LIT> , Size ) ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; return <STR_LIT> ( PtrBank -> getID ( ) , Size ) ;


<s> if ( ! StoreVT . isVector ( ) || StoreVT . getElementType ( ) != S16 ) return Reg ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; SmallVector < Register , <NUM_LIT> > WideRegs ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) WideRegs . push_back ( Unmerge . getReg ( I ) ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; int NumElts = StoreVT . getNumElements ( ) ; return B . buildMerge ( LLT :: fixed_vector ( NumElts , S32 ) , WideRegs ) . getReg ( <NUM_LIT> ) ;


<s> if ( ! MI . hasOneMemOperand ( ) ) return false ; const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; const unsigned AS = MMO -> getAddrSpace ( ) ; const bool IsConst = AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ; return MMO -> getAlign ( ) >= Align ( <NUM_LIT> ) && ! MMO -> isAtomic ( ) && ( IsConst || ! MMO -> isVolatile ( ) ) && ( IsConst || MMO -> isInvariant ( ) || memOpHasNoClobbered ( MMO ) ) && <ISA_LIT>InstrInfo :: isUniformMMO ( MMO ) ;


<s> static bool isScalarLoadLegal ( const MachineInstr & MI ) { if ( ! MI . hasOneMemOperand ( ) ) return false ; const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ;

const unsigned AS = MMO -> getAddrSpace ( ) ;


<s> static unsigned regBankBoolUnion ( unsigned RB0 , unsigned RB1 ) { if ( RB0 == <STR_LIT> ) return RB1 ; if ( RB1 == <STR_LIT> ) return RB0 ;


<s> Register VOffset = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned CachePolicy = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned ImmOffset ; std :: tie ( VOffset , ImmOffset ) = splitBufferOffsets ( B , VOffset ) ; const bool Offen = ! isZero ( VOffset , MRI ) ; unsigned Opc = <STR_LIT> ; switch ( <NUM_LIT> * MemSize ) { case <NUM_LIT> : Opc = Offen ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : Opc = Offen ? <STR_LIT> : <STR_LIT> ; break ; default : Opc = Offen ? <STR_LIT> : <STR_LIT> ;

if ( Size > <NUM_LIT> ) Opc = <STR_LIT> ( Opc , Size / <NUM_LIT> ) ;


<s> SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ; if ( Add && ( int ) Offset >= <NUM_LIT> ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ;


<s> SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; InstOffsetVal = ImmOffset ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; return SOffset + ImmOffset ; } } Register Base ; unsigned Offset ; std :: tie ( Base , Offset ) = <STR_LIT> ( * MRI , CombinedOffset ) ; uint32_t SOffset , ImmOffset ; if ( ( int ) Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ; if ( Add && ( int ) Offset >= <NUM_LIT> ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } }

if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) {


<s> LLT EltTy = Ty . getElementType ( ) ; unsigned EltSize = EltTy . getSizeInBits ( ) ; assert ( FirstSize % EltSize == <NUM_LIT> ) ; unsigned FirstPartNumElts = FirstSize / EltSize ;

unsigned RemainderElts = ( TotalSize - FirstSize ) / EltSize ;


<s> LLT EltTy = Ty . getElementType ( ) ; assert ( <NUM_LIT> % EltTy . getSizeInBits ( ) == <NUM_LIT> ) ; return LLT :: fixed_vector ( <NUM_LIT> / EltTy . getSizeInBits ( ) , EltTy ) ;


<s> if ( ! Ty . isVector ( ) ) return LLT :: scalar ( <NUM_LIT> ) ;


<s> auto ShiftOffset = Signed ? B . buildAShr ( S64 , SrcReg , OffsetReg ) : B . buildLShr ( S64 , SrcReg , OffsetReg ) ; auto UnmergeSOffset = B . buildUnmerge ( { S32 , S32 } , ShiftOffset ) ; if ( auto ConstWidth = getIConstantVRegValWithLookThrough ( WidthReg , MRI ) ) { auto Zero = B . buildConstant ( S32 , <NUM_LIT> ) ; auto WidthImm = ConstWidth -> Value . getZExtValue ( ) ; if ( WidthImm <= <NUM_LIT> ) { auto Extract = Signed ? B . buildSbfx ( S32 , UnmergeSOffset . getReg ( <NUM_LIT> ) , Zero , WidthReg ) : B . buildUbfx ( S32 , UnmergeSOffset . getReg ( <NUM_LIT> ) , Zero , WidthReg ) ; auto Extend = Signed ? B . buildAShr ( S32 , Extract , B . buildConstant ( S32 , <NUM_LIT> ) ) : Zero ; B . buildMerge ( DstReg , { Extract , Extend } ) ; } else { auto UpperWidth = B . buildConstant ( S32 , WidthImm - <NUM_LIT> ) ; auto Extract = Signed ? B . buildSbfx ( S32 , UnmergeSOffset . getReg ( <NUM_LIT> ) , Zero , UpperWidth ) : B . buildUbfx ( S32 , UnmergeSOffset . getReg ( <NUM_LIT> ) , Zero , UpperWidth ) ; B . buildMerge ( DstReg , { UnmergeSOffset . getReg ( <NUM_LIT> ) , Extract } ) ; } MI . eraseFromParent ( ) ; return true ; } auto ExtShift = B . buildSub ( S32 , B . buildConstant ( S32 , <NUM_LIT> ) , WidthReg ) ; auto SignBit = B . buildShl ( S64 , ShiftOffset , ExtShift ) ; if ( Signed ) B . buildAShr ( S64 , SignBit , ExtShift ) ; else B . buildLShr ( S64 , SignBit , ExtShift ) ; MI . eraseFromParent ( ) ; return true ;


<s> if ( ( int ) Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ; if ( Add && ( int ) Offset >= <NUM_LIT> ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = CombinedOffset ; } else { VOffsetReg = B . buildCopy ( S32 , CombinedOffset ) . getReg ( <NUM_LIT> ) ;


<s> unsigned <ISA_LIT>RegisterBankInfo :: getMappingType ( const MachineRegisterInfo & MRI , const MachineInstr & MI ) const { unsigned RegBank = <STR_LIT> ;

for ( const MachineOperand & MO : MI . operands ( ) ) {


<s> for ( const MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ;


<s> bool <ISA_LIT>RegisterBankInfo :: isSALUMapping ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( const MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ;

if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) {


<s> if ( Size == <NUM_LIT> && Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return std :: numeric_limits < unsigned > :: max ( ) ; return RegisterBankInfo :: copyCost ( Dst , Src , Size ) ;


<s> for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) {

unsigned Size = getSizeInBits ( MI . getOperand ( i ) . getReg ( ) , MRI , * TRI ) ;


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getDefaultMappingSOP ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ;

const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ;


<s> unsigned Size1 = getSizeInBits ( Reg1 , MRI , * TRI ) ; unsigned Bank1 = getRegBankID ( Reg1 , MRI , * TRI ) ; OpdsMapping [ OpdIdx ++ ] = <STR_LIT> ( Bank1 , Size1 ) ; for ( unsigned e = MI . getNumOperands ( ) ; OpdIdx != e ; ++ OpdIdx ) { unsigned Size = getSizeInBits ( MI . getOperand ( OpdIdx ) . getReg ( ) , MRI , * TRI ) ; OpdsMapping [ OpdIdx ] = <STR_LIT> ( <STR_LIT> , Size ) ; }

return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ;


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank && Bank -> getID ( ) != <STR_LIT> ) return false ; } return true ;


<s> for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ] = Dst ; } else { LoadParts [ i ] = MRI . createGenericVirtualRegister ( Ty ) ; MRI . setRegBank ( LoadParts [ i ] , <STR_LIT> ) ; } MachineMemOperand * MMO = BaseMMO ; if ( i != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset + <NUM_LIT> * i , MemSize ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoadParts [ i ] ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset + <NUM_LIT> * i ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; } if ( RSrcBank != & <STR_LIT> ) { B . setInstr ( * Span . begin ( ) ) ; MI . eraseFromParent ( ) ; SmallSet < Register , <NUM_LIT> > OpsToWaterfall ; OpsToWaterfall . insert ( RSrc ) ;

executeInWaterfallLoop ( B , make_range ( Span . begin ( ) , Span . end ( ) ) , OpsToWaterfall , MRI ) ;


<s> } const unsigned Alignment = NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ; MachineIRBuilder B ( MI ) ; MachineFunction & MF = B . getMF ( ) ; Register SOffset ; Register VOffset ; int64_t ImmOffset = <NUM_LIT> ; unsigned MMOOffset = setBufferOffsets ( B , * this , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , VOffset , SOffset , ImmOffset , Alignment ) ; const unsigned MemSize = ( Ty . getSizeInBits ( ) + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * BaseMMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; if ( MMOOffset != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset , MemSize ) ; Register RSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VIndex , <STR_LIT> ) ; SmallVector < Register , <NUM_LIT> > LoadParts ( NumLoads ) ; MachineBasicBlock :: iterator MII = MI . getIterator ( ) ; MachineInstrSpan Span ( MII , & B . getMBB ( ) ) ; for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ] = Dst ; } else { LoadParts [ i ] = MRI . createGenericVirtualRegister ( Ty ) ; MRI . setRegBank ( LoadParts [ i ] , <STR_LIT> ) ; } MachineMemOperand * MMO = BaseMMO ; if ( i != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset + <NUM_LIT> * i , MemSize ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoadParts [ i ] ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset + <NUM_LIT> * i ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; } if ( RSrcBank != & <STR_LIT> ) { B . setInstr ( * Span . begin ( ) ) ; MI . eraseFromParent ( ) ; SmallSet < Register , <NUM_LIT> > OpsToWaterfall ; OpsToWaterfall . insert ( RSrc ) ; executeInWaterfallLoop ( B , make_range ( Span . begin ( ) , Span . end ( ) ) , OpsToWaterfall , MRI ) ; } if ( NumLoads != <NUM_LIT> ) { if ( Ty . isVector ( ) ) B . buildConcatVectors ( Dst , LoadParts ) ; else B . buildMerge ( Dst , LoadParts ) ;


<s> const unsigned MaxNonSmrdLoadSize = <NUM_LIT> ; if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) { const RegisterBank * PtrBank = OpdMapper . getInstrMapping ( ) . getOperandMapping ( <NUM_LIT> ) . BreakDown [ <NUM_LIT> ] . RegBank ; if ( PtrBank == & <STR_LIT> ) return false ; SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ;

LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ;


<s> const RegisterBank * PtrBank = OpdMapper . getInstrMapping ( ) . getOperandMapping ( <NUM_LIT> ) . BreakDown [ <NUM_LIT> ] . RegBank ; if ( PtrBank == & <STR_LIT> ) return false ; SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ;


<s> const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( <NUM_LIT> ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT PtrTy = MRI . getType ( PtrReg ) ; unsigned AS = PtrTy . getAddressSpace ( ) ; unsigned PtrSize = PtrTy . getSizeInBits ( ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> && <ISA_LIT>TargetLowering :: isFlatGlobalAddrSpace ( AS ) ) { if ( isScalarLoadLegal ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else {


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getInstrMappingForLoad ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; LLT LoadTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } default : return RegisterBankInfo :: getInstrAlternativeMappings ( MI ) ;


<s> static llvm :: once_flag InitializeRegisterBankFlag ; static auto InitializeRegisterBankOnce = [ this ] ( ) { assert ( & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> ) ; ( void ) this ; } ; llvm :: call_once ( InitializeRegisterBankFlag , InitializeRegisterBankOnce ) ;


<s> <ISA_LIT>RegisterBankInfo :: <ISA_LIT>RegisterBankInfo ( const GCNSubtarget & ST ) : Subtarget ( ST ) , TRI ( Subtarget . getRegisterInfo ( ) ) , TII ( Subtarget . getInstrInfo ( ) ) { static llvm :: once_flag InitializeRegisterBankFlag ; static auto InitializeRegisterBankOnce = [ this ] ( ) { assert ( & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> ) ;


<s> return MMO -> getAlign ( ) >= Align ( <NUM_LIT> ) && ! MMO -> isAtomic ( ) && ( IsConst || ! MMO -> isVolatile ( ) ) && ( IsConst || MMO -> isInvariant ( ) || ( MMO -> getFlags ( ) & MONoClobber ) ) && <ISA_LIT>InstrInfo :: isUniformMMO ( MMO ) ;


<s> Src = B . buildCopy ( Ty , Src ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Src , <STR_LIT> ) ; } LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; unsigned NumParts = Bits / <NUM_LIT> ; SmallVector < Register , <NUM_LIT> > SrcParts ; SmallVector < Register , <NUM_LIT> > DstParts ; if ( Bits == <NUM_LIT> ) { SrcParts . push_back ( Src ) ; } else { auto Unmerge = B . buildUnmerge ( S32 , Src ) ; for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) SrcParts . push_back ( Unmerge . getReg ( i ) ) ; } for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) { Register SrcPart = SrcParts [ i ] ; Register DstPart = MRI . createVirtualRegister ( & <STR_LIT> ) ; MRI . setType ( DstPart , NumParts == <NUM_LIT> ? Ty : S32 ) ; const TargetRegisterClass * Constrained = constrainGenericRegister ( SrcPart , <STR_LIT> , MRI ) ; ( void ) Constrained ; assert ( Constrained && <STR_LIT> ) ;

B . buildInstr ( <STR_LIT> , { DstPart } , { SrcPart } ) ;


<s> if ( Bank == & <STR_LIT> ) return ; MachineIRBuilder B ( MI ) ; Reg = buildReadFirstLane ( B , MRI , Reg ) ;

MI . getOperand ( OpIdx ) . setReg ( Reg ) ;


<s> return false ; } Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register OffsetReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register WidthReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; ApplyRegBankMapping ApplyBank ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , ApplyBank ) ; auto OffsetMask = B . buildConstant ( S32 , maskTrailingOnes < unsigned > ( <NUM_LIT> ) ) ; auto ClampOffset = B . buildAnd ( S32 , OffsetReg , OffsetMask ) ; auto ShiftWidth = B . buildShl ( S32 , WidthReg , B . buildConstant ( S32 , <NUM_LIT> ) ) ; auto MergedInputs = B . buildOr ( S32 , ClampOffset , ShiftWidth ) ; unsigned Opc = Ty == S32 ? ( Signed ? <STR_LIT> : <STR_LIT> ) : ( Signed ? <STR_LIT> : <STR_LIT> ) ; auto MIB = B . buildInstr ( Opc , { DstReg } , { SrcReg , MergedInputs } ) ; if ( ! constrainSelectedInstRegOperands ( * MIB , * TII , * TRI , * this ) ) llvm_unreachable ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; return true ;


<s> auto Cmp = B . buildICmp ( CmpInst :: ICMP_EQ , CCTy , Idx , IC ) ; MRI . setRegBank ( Cmp -> getOperand ( <NUM_LIT> ) . getReg ( ) , CCBank ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) { Register Op0 = constrainRegToBank ( MRI , B , InsRegs [ L ] , DstBank ) ; Register Op1 = UnmergeToEltTy . getReg ( I * NumLanes + L ) ; Op1 = constrainRegToBank ( MRI , B , Op1 , DstBank ) ; Register Select = B . buildSelect ( EltTy , Cmp , Op0 , Op1 ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Select , DstBank ) ; Ops [ I * NumLanes + L ] = Select ; } } LLT MergeTy = LLT :: vector ( Ops . size ( ) , EltTy ) ; if ( MergeTy == MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { B . buildBuildVector ( MI . getOperand ( <NUM_LIT> ) , Ops ) ; } else { auto Vec = B . buildBuildVector ( MergeTy , Ops ) ;

MRI . setRegBank ( Vec -> getOperand ( <NUM_LIT> ) . getReg ( ) , DstBank ) ;


<s> static std :: pair < LLT , LLT > splitUnequalType ( LLT Ty , unsigned FirstSize ) { unsigned TotalSize = Ty . getSizeInBits ( ) ; if ( ! Ty . isVector ( ) ) return { LLT :: scalar ( FirstSize ) , LLT :: scalar ( TotalSize - FirstSize ) } ; LLT EltTy = Ty . getElementType ( ) ; unsigned EltSize = EltTy . getSizeInBits ( ) ; assert ( FirstSize % EltSize == <NUM_LIT> ) ; unsigned FirstPartNumElts = FirstSize / EltSize ;


<s> unsigned EltSize = EltTy . getSizeInBits ( ) ; assert ( FirstSize % EltSize == <NUM_LIT> ) ; unsigned FirstPartNumElts = FirstSize / EltSize ; unsigned RemainderElts = ( TotalSize - FirstSize ) / EltSize ; return { LLT :: scalarOrVector ( FirstPartNumElts , EltTy ) , LLT :: scalarOrVector ( RemainderElts , EltTy ) } ;


<s> assert ( <NUM_LIT> % EltTy . getSizeInBits ( ) == <NUM_LIT> ) ;

return LLT :: vector ( <NUM_LIT> / EltTy . getSizeInBits ( ) , EltTy ) ;


<s> Sizes [ I ] = getSizeInBits ( Reg , MRI , * TRI ) ; } for ( unsigned I = <NUM_LIT> , E = MI . getNumExplicitDefs ( ) ; I != E ; ++ I ) { unsigned SizeI = getSizeInBits ( MI . getOperand ( I ) . getReg ( ) , MRI , * TRI ) ; Operands [ I ] = <STR_LIT> ( <STR_LIT> , SizeI ) ; } unsigned MappingID = <NUM_LIT> ; for ( const auto & Entry : Table ) { for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { int OpIdx = RegSrcOpIdx [ I ] ; Operands [ OpIdx ] = <STR_LIT> ( Entry . RegBanks [ I ] , Sizes [ I ] ) ;


<s> unsigned MappingID = <NUM_LIT> ; for ( const auto & Entry : Table ) { for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { int OpIdx = RegSrcOpIdx [ I ] ; Operands [ OpIdx ] = <STR_LIT> ( Entry . RegBanks [ I ] , Sizes [ I ] ) ; } AltMappings . push_back ( & getInstructionMapping ( MappingID ++ , Entry . Cost , getOperandsMapping ( Operands ) , Operands . size ( ) ) ) ; }


<s> assert ( ValMapping . NumBreakDowns == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . RegBank == ValMapping . BreakDown [ <NUM_LIT> ] . RegBank ) ; return <NUM_LIT> ;


<s> unsigned <ISA_LIT>RegisterBankInfo :: getBreakDownCost ( const ValueMapping & ValMapping , const RegisterBank * CurBank ) const { assert ( ValMapping . NumBreakDowns == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . RegBank == ValMapping . BreakDown [ <NUM_LIT> ] . RegBank ) ;


<s> SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; unsigned OpdIdx = <NUM_LIT> ; unsigned Size0 = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; OpdsMapping [ OpdIdx ++ ] = <STR_LIT> ( <STR_LIT> , Size0 ) ; if ( MI . getOperand ( OpdIdx ) . isIntrinsicID ( ) ) OpdsMapping [ OpdIdx ++ ] = nullptr ; unsigned Reg1 = MI . getOperand ( OpdIdx ) . getReg ( ) ; unsigned Size1 = getSizeInBits ( Reg1 , MRI , * TRI ) ; unsigned DefaultBankID = Size1 == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; unsigned Bank1 = getRegBankID ( Reg1 , MRI , * TRI , DefaultBankID ) ; OpdsMapping [ OpdIdx ++ ] = <STR_LIT> ( Bank1 , Size1 ) ; for ( unsigned e = MI . getNumOperands ( ) ; OpdIdx != e ; ++ OpdIdx ) { const MachineOperand & MO = MI . getOperand ( OpdIdx ) ; if ( ! MO . isReg ( ) ) continue ; unsigned Size = getSizeInBits ( MO . getReg ( ) , MRI , * TRI ) ; unsigned BankID = Size == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; OpdsMapping [ OpdIdx ] = <STR_LIT> ( BankID , Size ) ;


<s> switch ( MI . getOperand ( MI . getNumExplicitDefs ( ) ) . getIntrinsicID ( ) ) { case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; }


assert ( MRI . getType ( Reg ) . getSizeInBits ( ) == NewTy . getSizeInBits ( ) ) ;


<s> assert ( MRI . getType ( Reg ) . getSizeInBits ( ) == NewTy . getSizeInBits ( ) ) ;


<s> void <ISA_LIT>RegisterBankInfo :: split64BitValueForMapping ( MachineIRBuilder & B , SmallVector < Register , <NUM_LIT> > & Regs , LLT HalfTy , unsigned Reg ) const { assert ( HalfTy . getSizeInBits ( ) == <NUM_LIT> ) ; MachineRegisterInfo * MRI = B . getMRI ( ) ; unsigned LoLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; unsigned HiLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ; Regs . push_back ( HiLHS ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoLHS ) . addDef ( HiLHS ) . addUse ( Reg ) ;


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ;

if ( Bank && Bank -> getID ( ) != <STR_LIT> ) return false ;


<s> const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank && Bank -> getID ( ) != <STR_LIT> ) return false ; } return true ;


<s> static bool AlreadyInit = false ; if ( AlreadyInit ) return ; AlreadyInit = true ; const RegisterBank & RBSGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBSGPR ; assert ( & RBSGPR == & <STR_LIT> ) ;

const RegisterBank & RBVGPR = getRegBank ( <STR_LIT> ) ;


<s> if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return <NUM_LIT> ; return RegisterBankInfo :: copyCost ( Dst , Src , Size ) ;


<s> const RegisterBank & <ISA_LIT>RegisterBankInfo :: getRegBankFromRegClass ( const TargetRegisterClass & RC ) const { if ( & RC == & <STR_LIT> ) return <STR_LIT> ; if ( TRI -> isSGPRClass ( & RC ) ) return <STR_LIT> ; if ( TRI -> isAGPRClass ( & RC ) ) return <STR_LIT> ;


<s> if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( isVectorRegisterBank ( * Bank ) ) return false ; assert ( Bank -> getID ( ) == <STR_LIT> || Bank -> getID ( ) == <STR_LIT> ) ;


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; InstructionMappings AltMappings ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_LOAD : { InstructionMapping SSMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . emplace_back ( std :: move ( SSMapping ) ) ; InstructionMapping VVMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ;

AltMappings . emplace_back ( std :: move ( VVMapping ) ) ;


<s> InstructionMapping SSMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . emplace_back ( std :: move ( SSMapping ) ) ; InstructionMapping VVMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ;


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Mapping = InstructionMapping { <NUM_LIT> , <NUM_LIT> , nullptr , MI . getNumOperands ( ) } ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; switch ( MI . getOpcode ( ) ) { default : break ; case <STR_LIT> : { unsigned Size = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . getSizeInBits ( ) ; OpdsMapping [ <NUM_LIT> ] = <STR_LIT> ( <STR_LIT> , Size ) ; Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ; return Mapping ; } case <STR_LIT> : { for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; unsigned Size = MRI . getType ( MI . getOperand ( i ) . getReg ( ) ) . getSizeInBits ( ) ; OpdsMapping [ i ] = <STR_LIT> ( <STR_LIT> , Size ) ; } Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ; return Mapping ; } case <STR_LIT> : { assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; unsigned Size = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . getSizeInBits ( ) ; const ValueMapping * ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; const ValueMapping * PtrMapping = <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) ; OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ; return Mapping ; } case <STR_LIT> : return getInstrMappingForLoad ( MI ) ; } unsigned BankID = <STR_LIT> ; Mapping = InstructionMapping { <NUM_LIT> , <NUM_LIT> , nullptr , MI . getNumOperands ( ) } ; unsigned Size = <NUM_LIT> ; for ( unsigned Idx = <NUM_LIT> ; Idx < MI . getNumOperands ( ) ; ++ Idx ) {

if ( MI . getOperand ( Idx ) . isReg ( ) ) Size = getSizeInBits ( MI . getOperand ( Idx ) . getReg ( ) , MRI , * TRI ) ;


<s> unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; unsigned PtrSize = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; if ( isInstrUniform ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ;


<s> } LLT EltTy = VecTy . getScalarType ( ) ; SmallVector < Register , <NUM_LIT> > InsRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; unsigned NumLanes = InsRegs . size ( ) ; if ( ! NumLanes ) { NumLanes = <NUM_LIT> ; InsRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } else { EltTy = MRI . getType ( InsRegs [ <NUM_LIT> ] ) ; } auto UnmergeToEltTy = B . buildUnmerge ( EltTy , VecReg ) ; SmallVector < Register , <NUM_LIT> > Ops ( NumElem * NumLanes ) ; for ( unsigned I = <NUM_LIT> ; I < NumElem ; ++ I ) { auto IC = B . buildConstant ( S32 , I ) ; MRI . setRegBank ( IC -> getOperand ( <NUM_LIT> ) . getReg ( ) , <STR_LIT> ) ; auto Cmp = B . buildICmp ( CmpInst :: ICMP_EQ , CCTy , Idx , IC ) ; MRI . setRegBank ( Cmp -> getOperand ( <NUM_LIT> ) . getReg ( ) , CCBank ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) { auto S = B . buildSelect ( EltTy , Cmp , InsRegs [ L ] , UnmergeToEltTy . getReg ( I * NumLanes + L ) ) ; for ( unsigned N : { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ) MRI . setRegBank ( S -> getOperand ( N ) . getReg ( ) , DstBank ) ; Ops [ I * NumLanes + L ] = S -> getOperand ( <NUM_LIT> ) . getReg ( ) ; } } LLT MergeTy = LLT :: vector ( Ops . size ( ) , EltTy ) ;

if ( MergeTy == MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) {


<s> if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = CombinedOffset ; } else { VOffsetReg = B . buildCopy ( S32 , CombinedOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; }

SOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ;


<s> if ( <STR_LIT> ( * Imm , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; InstOffsetVal = ImmOffset ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; return SOffset + ImmOffset ; } } Register Base ; unsigned Offset ; MachineInstr * Unused ; std :: tie ( Base , Offset , Unused ) = <STR_LIT> ( * MRI , CombinedOffset ) ; uint32_t SOffset , ImmOffset ; if ( Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ;


<s> if ( <STR_LIT> ( * Imm , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; InstOffsetVal = ImmOffset ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; return SOffset + ImmOffset ; } } Register Base ; unsigned Offset ; MachineInstr * Unused ; std :: tie ( Base , Offset , Unused ) = <STR_LIT> ( * MRI , CombinedOffset ) ; uint32_t SOffset , ImmOffset ; if ( Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ;


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const LLT LoadTy = MRI . getType ( DstReg ) ; unsigned LoadSize = LoadTy . getSizeInBits ( ) ; const unsigned MaxNonSmrdLoadSize = <NUM_LIT> ; if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) { Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> ) return false ; SrcRegs . push_back ( PtrReg ) ; } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy . getScalarType ( ) . getSizeInBits ( ) ; const LLT LoadSplitTy = LLT :: vector ( SplitElts , LoadTy . getScalarType ( ) ) ; ApplyRegBankMapping O ( MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; MachineInstr * LegalizedInst = getOtherVRegDef ( MRI , DstReg , * RepairInst ) ; Register TmpReg = MRI . createGenericVirtualRegister ( MRI . getType ( DstReg ) ) ; LegalizedInst -> getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; B . setInsertPt ( * RepairInst -> getParent ( ) , RepairInst ) ; for ( unsigned DefIdx = <NUM_LIT> , e = DefRegs . size ( ) ; DefIdx != e ; ++ DefIdx ) { Register IdxReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildConstant ( IdxReg , DefIdx ) ;

MRI . setRegBank ( IdxReg , <STR_LIT> ) ;


<s> const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ;


<s> MachineMemOperand * BaseMMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; if ( MMOOffset != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset , MemSize ) ; Register RSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VIndex , <STR_LIT> ) ; SmallVector < Register , <NUM_LIT> > LoadParts ( NumLoads ) ; MachineBasicBlock :: iterator MII = MI . getIterator ( ) ; MachineInstrSpan Span ( MII , & B . getMBB ( ) ) ; for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ] = Dst ; } else { LoadParts [ i ] = MRI . createGenericVirtualRegister ( Ty ) ; MRI . setRegBank ( LoadParts [ i ] , <STR_LIT> ) ; } MachineMemOperand * MMO = BaseMMO ; if ( i != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset + <NUM_LIT> * i , MemSize ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoadParts [ i ] ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset + <NUM_LIT> * i ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; } if ( RSrcBank != & <STR_LIT> ) { B . setInstr ( * Span . begin ( ) ) ; MI . eraseFromParent ( ) ; SmallSet < Register , <NUM_LIT> > OpsToWaterfall ; OpsToWaterfall . insert ( RSrc ) ; executeInWaterfallLoop ( B , make_range ( Span . begin ( ) , Span . end ( ) ) , OpsToWaterfall , MRI ) ; } if ( NumLoads != <NUM_LIT> ) { if ( Ty . isVector ( ) ) B . buildConcatVectors ( Dst , LoadParts ) ; else B . buildMerge ( Dst , LoadParts ) ;


<s> if ( LoadSize != <NUM_LIT> ) return false ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineIRBuilder B ( MI ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; if ( MMO -> getAlign ( ) < Align ( <NUM_LIT> ) ) { LLT Part64 , Part32 ; std :: tie ( Part64 , Part32 ) = splitUnequalType ( LoadTy , <NUM_LIT> ) ; auto Load0 = B . buildLoadFromOffset ( Part64 , PtrReg , * MMO , <NUM_LIT> ) ; auto Load1 = B . buildLoadFromOffset ( Part32 , PtrReg , * MMO , <NUM_LIT> ) ; auto Undef = B . buildUndef ( LoadTy ) ; auto Ins0 = B . buildInsert ( LoadTy , Undef , Load0 , <NUM_LIT> ) ; B . buildInsert ( MI . getOperand ( <NUM_LIT> ) , Ins0 , Load1 , <NUM_LIT> ) ; } else { LLT WiderTy = widen96To128 ( LoadTy ) ; auto WideLoad = B . buildLoadFromOffset ( WiderTy , PtrReg , * MMO , <NUM_LIT> ) ; B . buildExtract ( MI . getOperand ( <NUM_LIT> ) , WideLoad , <NUM_LIT> ) ; } MI . eraseFromParent ( ) ;


<s> std :: tie ( Part64 , Part32 ) = splitUnequalType ( LoadTy , <NUM_LIT> ) ; auto Load0 = B . buildLoadFromOffset ( Part64 , PtrReg , * MMO , <NUM_LIT> ) ; auto Load1 = B . buildLoadFromOffset ( Part32 , PtrReg , * MMO , <NUM_LIT> ) ; auto Undef = B . buildUndef ( LoadTy ) ; auto Ins0 = B . buildInsert ( LoadTy , Undef , Load0 , <NUM_LIT> ) ; B . buildInsert ( MI . getOperand ( <NUM_LIT> ) , Ins0 , Load1 , <NUM_LIT> ) ; } else { LLT WiderTy = widen96To128 ( LoadTy ) ; auto WideLoad = B . buildLoadFromOffset ( WiderTy , PtrReg , * MMO , <NUM_LIT> ) ; B . buildExtract ( MI . getOperand ( <NUM_LIT> ) , WideLoad , <NUM_LIT> ) ; } MI . eraseFromParent ( ) ; return true ; } if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < Register , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < Register , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ;

ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ;


<s> else EltTy = MRI . getType ( DstRegs [ <NUM_LIT> ] ) ; auto UnmergeToEltTy = B . buildUnmerge ( EltTy , VecReg ) ; SmallVector < Register , <NUM_LIT> > Res ( NumLanes ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) Res [ L ] = UnmergeToEltTy . getReg ( L ) ; for ( unsigned I = <NUM_LIT> ; I < NumElem ; ++ I ) { auto IC = B . buildConstant ( S32 , I ) ; MRI . setRegBank ( IC -> getOperand ( <NUM_LIT> ) . getReg ( ) , <STR_LIT> ) ; auto Cmp = B . buildICmp ( CmpInst :: ICMP_EQ , CCTy , Idx , IC ) ; MRI . setRegBank ( Cmp -> getOperand ( <NUM_LIT> ) . getReg ( ) , CCBank ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) { auto S = B . buildSelect ( EltTy , Cmp , UnmergeToEltTy . getReg ( I * NumLanes + L ) , Res [ L ] ) ; for ( unsigned N : { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ) MRI . setRegBank ( S -> getOperand ( N ) . getReg ( ) , DstBank ) ; Res [ L ] = S -> getOperand ( <NUM_LIT> ) . getReg ( ) ; } }


<s> } assert ( DefRegs . size ( ) == <NUM_LIT> ) ; assert ( Src0Regs . size ( ) == Src1Regs . size ( ) && ( Src0Regs . empty ( ) || Src0Regs . size ( ) == <NUM_LIT> ) ) ; MachineIRBuilder B ( MI ) ; if ( Src0Regs . empty ( ) ) split64BitValueForMapping ( B , Src0Regs , HalfTy , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; else setRegsToType ( MRI , Src0Regs , HalfTy ) ; if ( Src1Regs . empty ( ) ) split64BitValueForMapping ( B , Src1Regs , HalfTy , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; else setRegsToType ( MRI , Src1Regs , HalfTy ) ; setRegsToType ( MRI , DefRegs , HalfTy ) ; B . buildInstr ( Opc ) . addDef ( DefRegs [ <NUM_LIT> ] ) . addUse ( Src0Regs [ <NUM_LIT> ] ) . addUse ( Src1Regs [ <NUM_LIT> ] ) ; B . buildInstr ( Opc ) . addDef ( DefRegs [ <NUM_LIT> ] ) . addUse ( Src0Regs [ <NUM_LIT> ] ) . addUse ( Src1Regs [ <NUM_LIT> ] ) ; MRI . setRegBank ( DstReg , getRegBank ( <STR_LIT> ) ) ; MI . eraseFromParent ( ) ; return ; } case <STR_LIT> : applyDefaultMapping ( OpdMapper ) ; executeInWaterfallLoop ( MI , MRI , { <NUM_LIT> } ) ; return ; case <STR_LIT> : { switch ( MI . getOperand ( MI . getNumExplicitDefs ( ) ) . getIntrinsicID ( ) ) { case <STR_LIT> : { executeInWaterfallLoop ( MI , MRI , { <NUM_LIT> } ) ; return ; } case <STR_LIT> : { executeInWaterfallLoop ( MI , MRI , { <NUM_LIT> , <NUM_LIT> } ) ; return ; } default : break ; } break ; }


<s> assert ( MRI . getType ( Reg ) . getSizeInBits ( ) == NewTy . getSizeInBits ( ) ) ; MRI . setType ( Reg , NewTy ) ; }


<s> unsigned LoLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; unsigned HiLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ;


<s> unsigned HiLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ; Regs . push_back ( HiLHS ) ;

B . buildInstr ( <STR_LIT> ) . addDef ( LoLHS ) . addDef ( HiLHS ) . addUse ( Reg ) ;


<s> if ( Src1Regs . empty ( ) ) split64BitValueForMapping ( B , Src1Regs , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; B . buildInstr ( Opc ) . addDef ( DefRegs [ <NUM_LIT> ] ) . addUse ( Src0Regs [ <NUM_LIT> ] ) . addUse ( Src1Regs [ <NUM_LIT> ] ) ; B . buildInstr ( Opc ) . addDef ( DefRegs [ <NUM_LIT> ] ) . addUse ( Src0Regs [ <NUM_LIT> ] ) . addUse ( Src1Regs [ <NUM_LIT> ] ) ; MRI . setRegBank ( DstReg , getRegBank ( <STR_LIT> ) ) ; MI . eraseFromParent ( ) ; return ; } default :


<s> break ; } assert ( DefRegs . size ( ) == <NUM_LIT> ) ; assert ( Src0Regs . size ( ) == Src1Regs . size ( ) && ( Src0Regs . empty ( ) || Src0Regs . size ( ) == <NUM_LIT> ) ) ; MachineIRBuilder B ( MI ) ; if ( Src0Regs . empty ( ) ) split64BitValueForMapping ( B , Src0Regs , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Src1Regs . empty ( ) ) split64BitValueForMapping ( B , Src1Regs , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; B . buildInstr ( Opc ) . addDef ( DefRegs [ <NUM_LIT> ] ) . addUse ( Src0Regs [ <NUM_LIT> ] ) . addUse ( Src1Regs [ <NUM_LIT> ] ) ; B . buildInstr ( Opc ) . addDef ( DefRegs [ <NUM_LIT> ] ) . addUse ( Src0Regs [ <NUM_LIT> ] ) . addUse ( Src1Regs [ <NUM_LIT> ] ) ;

MRI . setRegBank ( DstReg , getRegBank ( <STR_LIT> ) ) ;


<s> assert ( ValMapping . NumBreakDowns == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . RegBank == ValMapping . BreakDown [ <NUM_LIT> ] . RegBank ) ; return <NUM_LIT> ;


<s> const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; const InstructionMapping & SVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; break ; } case TargetOpcode :: G_LOAD : { unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; return AltMappings ; } case TargetOpcode :: G_ICMP : { unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & SVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ;


<s> } if ( Src1Regs . empty ( ) ) split64BitValueForMapping ( B , Src1Regs , HalfTy , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; else { setRegsToType ( MRI , Src1Regs , HalfTy ) ; } if ( Src2Regs . empty ( ) ) split64BitValueForMapping ( B , Src2Regs , HalfTy , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; else setRegsToType ( MRI , Src2Regs , HalfTy ) ; setRegsToType ( MRI , DefRegs , HalfTy ) ; B . buildSelect ( DefRegs [ <NUM_LIT> ] , Src0Regs [ <NUM_LIT> ] , Src1Regs [ <NUM_LIT> ] , Src2Regs [ <NUM_LIT> ] ) ; B . buildSelect ( DefRegs [ <NUM_LIT> ] , Src0Regs [ <NUM_LIT> ] , Src1Regs [ <NUM_LIT> ] , Src2Regs [ <NUM_LIT> ] ) ; MRI . setRegBank ( DstReg , getRegBank ( <STR_LIT> ) ) ; MI . eraseFromParent ( ) ; return ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( DstReg ) ; if ( DstTy . getSizeInBits ( ) != <NUM_LIT> ) break ; LLT HalfTy = getHalfSizedType ( DstTy ) ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > Src0Regs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > Src1Regs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( DefRegs . empty ( ) ) {

assert ( Src0Regs . empty ( ) && Src1Regs . empty ( ) ) ;


<s> MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , O ) ; if ( MMO -> getAlign ( ) < Align ( <NUM_LIT> ) ) { LLT Part64 , Part32 ; std :: tie ( Part64 , Part32 ) = splitUnequalType ( LoadTy , <NUM_LIT> ) ; auto Load0 = B . buildLoadFromOffset ( Part64 , PtrReg , * MMO , <NUM_LIT> ) ; auto Load1 = B . buildLoadFromOffset ( Part32 , PtrReg , * MMO , <NUM_LIT> ) ; auto Undef = B . buildUndef ( LoadTy ) ; auto Ins0 = B . buildInsert ( LoadTy , Undef , Load0 , <NUM_LIT> ) ; B . buildInsert ( MI . getOperand ( <NUM_LIT> ) , Ins0 , Load1 , <NUM_LIT> ) ; } else { LLT WiderTy = widen96To128 ( LoadTy ) ; auto WideLoad = B . buildLoadFromOffset ( WiderTy , PtrReg , * MMO , <NUM_LIT> ) ; B . buildExtract ( MI . getOperand ( <NUM_LIT> ) , WideLoad , <NUM_LIT> ) ; } MI . eraseFromParent ( ) ; return true ; } if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < Register , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < Register , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping Observer ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( LoadTy . isVector ( ) ) {

if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ;


<s> if ( LoadSize != <NUM_LIT> ) return false ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , O ) ; if ( MMO -> getAlign ( ) < Align ( <NUM_LIT> ) ) { LLT Part64 , Part32 ; std :: tie ( Part64 , Part32 ) = splitUnequalType ( LoadTy , <NUM_LIT> ) ; auto Load0 = B . buildLoadFromOffset ( Part64 , PtrReg , * MMO , <NUM_LIT> ) ; auto Load1 = B . buildLoadFromOffset ( Part32 , PtrReg , * MMO , <NUM_LIT> ) ; auto Undef = B . buildUndef ( LoadTy ) ; auto Ins0 = B . buildInsert ( LoadTy , Undef , Load0 , <NUM_LIT> ) ; B . buildInsert ( MI . getOperand ( <NUM_LIT> ) , Ins0 , Load1 , <NUM_LIT> ) ; } else { LLT WiderTy = widen96To128 ( LoadTy ) ; auto WideLoad = B . buildLoadFromOffset ( WiderTy , PtrReg , * MMO , <NUM_LIT> ) ; B . buildExtract ( MI . getOperand ( <NUM_LIT> ) , WideLoad , <NUM_LIT> ) ; } MI . eraseFromParent ( ) ; return true ; } if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < Register , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < Register , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping Observer ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( LoadTy . isVector ( ) ) { if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; } else { if ( Helper . narrowScalar ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; } MRI . setRegBank ( DstReg , <STR_LIT> ) ; return true ;


<s> if ( Optional < int64_t > Imm = getConstantVRegSExtVal ( CombinedOffset , * MRI ) ) { uint32_t SOffset , ImmOffset ; if ( <STR_LIT> ( * Imm , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; InstOffsetVal = ImmOffset ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; return SOffset + ImmOffset ; } } Register Base ; unsigned Offset ; std :: tie ( Base , Offset ) = <STR_LIT> ( * MRI , CombinedOffset ) ; uint32_t SOffset , ImmOffset ; if ( Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;

const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ;


<s> VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = CombinedOffset ; } else { VOffsetReg = B . buildCopy ( S32 , CombinedOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; }


<s> void <ISA_LIT>RegisterBankInfo :: constrainOpWithReadfirstlane ( MachineInstr & MI , MachineRegisterInfo & MRI , unsigned OpIdx ) const { Register Reg = MI . getOperand ( OpIdx ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank != & <STR_LIT> ) return ; MachineIRBuilder B ( MI ) ; Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ;

const TargetRegisterClass * Constrained = constrainGenericRegister ( Reg , <STR_LIT> , MRI ) ;


<s> <ISA_LIT>RegisterBankInfo :: <ISA_LIT>RegisterBankInfo ( const TargetRegisterInfo & TRI ) : <ISA_LIT>GenRegisterBankInfo ( ) , TRI ( static_cast < const <ISA_LIT>RegisterInfo * > ( & TRI ) ) { static bool AlreadyInit = false ; if ( AlreadyInit ) return ; AlreadyInit = true ; const RegisterBank & RBSGPR = getRegBank ( <STR_LIT> ) ;


<s> if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) { return std :: numeric_limits < unsigned > :: max ( ) ; } if ( Size == <NUM_LIT> && Dst . getID ( ) == <STR_LIT> && ( Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ;

return RegisterBankInfo :: copyCost ( Dst , Src , Size ) ;


<s> if ( Size == <NUM_LIT> && Dst . getID ( ) == <STR_LIT> && ( Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ; return RegisterBankInfo :: copyCost ( Dst , Src , Size ) ;


<s> unsigned Size = getSizeInBits ( MI . getOperand ( I ) . getReg ( ) , MRI , * TRI ) ; OpdsMapping [ I ] = <STR_LIT> ( <STR_LIT> , Size ) ; }

return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ;


<s> return AltMappings ; } case TargetOpcode :: G_ICMP : { unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & SVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; return AltMappings ; } case TargetOpcode :: G_SELECT : { unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ;

const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ;


<s> const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; unsigned PtrSize = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; if ( isInstrUniform ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ;


<s> const RegisterBank & <ISA_LIT>RegisterBankInfo :: getRegBankFromRegClass ( const TargetRegisterClass & RC ) const {

if ( TRI -> isSGPRClass ( & RC ) ) return getRegBank ( <STR_LIT> ) ;


<s> if ( TRI -> isSGPRClass ( & RC ) ) return getRegBank ( <STR_LIT> ) ; return getRegBank ( <STR_LIT> ) ;


<s> const RegisterBank * Bank = getRegBank ( Reg , MRI , TRI ) ; return Bank ? Bank -> getID ( ) : Default ;


<s> const RegisterBank * Bank = getRegBank ( Reg , MRI , TRI ) ;

return Bank ? Bank -> getID ( ) : Default ;


<s> const MachineFunction * MF = MO . getParent ( ) -> getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; const MachineInstr * Def = MRI . getVRegDef ( MO . getReg ( ) ) ; if ( ! Def ) return false ; if ( Def -> getOpcode ( ) == <STR_LIT> ) { C = Def -> getOperand ( <NUM_LIT> ) . getCImm ( ) -> getSExtValue ( ) ; return true ;


<s> const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; return <ISA_LIT>InstrInfo :: isUniformMMO ( MMO ) ;


<s> if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( Bank -> getID ( ) == <STR_LIT> ) return false ; assert ( Bank -> getID ( ) == <STR_LIT> || Bank -> getID ( ) == <STR_LIT> ) ; } }


<s> if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( Bank -> getID ( ) == <STR_LIT> ) return false ; assert ( Bank -> getID ( ) == <STR_LIT> || Bank -> getID ( ) == <STR_LIT> ) ; } } return true ;


<s> unsigned <ISA_LIT>RegisterInfo :: getFrameRegister ( const MachineFunction & MF ) const { return <STR_LIT> ;


assert ( Channel < array_lengthof ( SubRegs ) ) ;


<s> unsigned <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( unsigned Channel ) { static const unsigned SubRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; assert ( Channel < array_lengthof ( SubRegs ) ) ; return SubRegs [ Channel ] ;


<s> static const unsigned SubRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; assert ( Channel < array_lengthof ( SubRegs ) ) ; return SubRegs [ Channel ] ;


<s> const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

return FuncInfo -> getFrameOffsetReg ( ) ;


<s> CallingConv :: ID CC = MF -> getFunction ( ) -> getCallingConv ( ) ; switch ( CC ) { case CallingConv :: C : case CallingConv :: Fast : return CSR_<ISA_LIT>_HighRegs_SaveList ; default : { static const MCPhysReg NoCalleeSavedReg = <STR_LIT> ;


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getCallPreservedMask ( const MachineFunction & MF , CallingConv :: ID CC ) const { switch ( CC ) { case CallingConv :: C :


<s> const MCPhysReg * <ISA_LIT>RegisterInfo :: getCalleeSavedRegs ( const MachineFunction * ) const {


<s> <ISA_LIT>RegisterInfo :: <ISA_LIT>RegisterInfo ( ) : <ISA_LIT>GenRegisterInfo ( <NUM_LIT> ) {


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getAllAllocatableSRegMask ( ) const {


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getAllVGPRRegMask ( ) const { return CSR_<ISA_LIT>_AllVGPRs_RegMask ;


<s> case CallingConv :: Fast : case CallingConv :: Cold : return CSR_<ISA_LIT>_HighRegs_SaveList ; default : { static const MCPhysReg NoCalleeSavedReg = <STR_LIT> ; return & NoCalleeSavedReg ; }


<s> const MCPhysReg * <ISA_LIT>RegisterInfo :: getCalleeSavedRegsViaCopy ( const MachineFunction * MF ) const {


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getCallPreservedMask ( const MachineFunction & MF , CallingConv :: ID CC ) const { switch ( CC ) { case CallingConv :: C : case CallingConv :: Fast : case CallingConv :: Cold : return CSR_<ISA_LIT>_HighRegs_RegMask ; default :


<s> switch ( CC ) { case CallingConv :: C : case CallingConv :: Fast : case CallingConv :: Cold : return CSR_<ISA_LIT>_HighRegs_RegMask ; default : return nullptr ;


<s> const <ISA_LIT>FrameLowering * TFI = MF . getSubtarget < GCNSubtarget > ( ) . getFrameLowering ( ) ; const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : FuncInfo -> getStackPtrOffsetReg ( ) ;


<s> return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : FuncInfo -> getStackPtrOffsetReg ( ) ;


<s> const unsigned NumRegIndex = NumRegs - <NUM_LIT> ; assert ( NumRegIndex < array_lengthof ( SubRegFromChannelTable ) && <STR_LIT> ) ;

assert ( Channel < array_lengthof ( SubRegFromChannelTable [ <NUM_LIT> ] ) ) ;


<s> for ( ; R . isValid ( ) ; ++ R ) Reserved . set ( * R ) ;


<s> assert ( ! <STR_LIT> ) ; return nullptr ;


<s> const <ISA_LIT>FrameLowering * TFI = MF . getSubtarget < GCNSubtarget > ( ) . getFrameLowering ( ) ; const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : FuncInfo -> getStackPtrOffsetReg ( ) ;


<s> const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : FuncInfo -> getStackPtrOffsetReg ( ) ;


<s> case CallingConv :: Cold : return CSR_<ISA_LIT>_HighRegs_SaveList ; default : { static const MCPhysReg NoCalleeSavedReg = <STR_LIT> ; return & NoCalleeSavedReg ;


<s> void <ISA_LIT>RegisterInfo :: eliminateFrameIndex ( MachineBasicBlock :: iterator MI , int SPAdj , unsigned FIOperandNum , RegScavenger * RS ) const {


<s> void <ISA_LIT>RegisterInfo :: eliminateFrameIndex ( MachineBasicBlock :: iterator MI , int SPAdj , unsigned FIOperandNum , RegScavenger * RS ) const { llvm_unreachable ( <STR_LIT> ) ;


<s> const MCPhysReg * <ISA_LIT>RegisterInfo :: getCalleeSavedRegs ( const MachineFunction * MF ) const {


<s> unsigned <ISA_LIT>RegisterInfo :: getIndirectSubReg ( unsigned IndirectIndex ) const { return getSubRegFromChannel ( IndirectIndex ) ;


<s> LDSGlobals . erase ( std :: remove_if ( LDSGlobals . begin ( ) , LDSGlobals . end ( ) , [ & ] ( GlobalVariable * GV ) { return shouldIgnorePointerReplacement ( GV ) ; } ) , LDSGlobals . end ( ) ) ; return LDSGlobals ;


<s> std :: vector < GlobalVariable * > LDSGlobals = <STR_LIT> ( M ) ; LDSGlobals . erase ( std :: remove_if ( LDSGlobals . begin ( ) , LDSGlobals . end ( ) , [ & ] ( GlobalVariable * GV ) {

return shouldIgnorePointerReplacement ( GV ) ;


<s> for ( const auto & GI : * ECNode ) { auto * CGN = GI . second ; auto * F = CGN -> getFunction ( ) ;

if ( ! F || F -> isDeclaration ( ) || llvm :: <STR_LIT> ( F ) ) continue ;


<s> auto * ECNode = CG . getExternalCallingNode ( ) ; for ( const auto & GI : * ECNode ) { auto * CGN = GI . second ; auto * F = CGN -> getFunction ( ) ;


<s> std :: vector < GlobalVariable * > collectLDSRequiringPointerReplace ( ) { std :: vector < GlobalVariable * > LDSGlobals = llvm :: <STR_LIT> ( M ) ; llvm :: erase_if ( LDSGlobals , [ & ] ( GlobalVariable * GV ) { return shouldIgnorePointerReplacement ( GV ) ;


<s> SmallVector < User * , <NUM_LIT> > UserStack ( GV -> users ( ) ) ; SmallPtrSet < User * , <NUM_LIT> > VisitedUsers ; while ( ! UserStack . empty ( ) ) { auto * U = UserStack . pop_back_val ( ) ; if ( ! VisitedUsers . insert ( U ) . second ) continue ; if ( isa < GlobalValue > ( U ) ) return SmallPtrSet < Function * , <NUM_LIT> > ( ) ; if ( isa < Constant > ( U ) ) { append_range ( UserStack , U -> users ( ) ) ; continue ; } Function * F = cast < Instruction > ( U ) -> getFunction ( ) ; if ( ! llvm :: <STR_LIT> ( F ) ) LDSAccessors . insert ( F ) ; } return LDSAccessors ;


<s> for ( Function & F : M . functions ( ) ) { if ( ! llvm :: <STR_LIT> ( & F ) ) continue ; Function * K = & F ; KernelToCallees [ K ] = collectReachableCallees ( K ) ; }


<s> void collectReachableCallees ( DenseMap < Function * , SmallPtrSet < Function * , <NUM_LIT> >> & KernelToCallees ) { for ( Function & F : M . functions ( ) ) {

if ( ! llvm :: <STR_LIT> ( & F ) ) continue ;


<s> auto * I16Ty = Type :: getInt16Ty ( Ctx ) ; GlobalVariable * LDSPointer = new GlobalVariable ( M , I16Ty , false , GlobalValue :: InternalLinkage , UndefValue :: get ( I16Ty ) , GV -> getName ( ) + Twine ( <STR_LIT> ) , nullptr , GlobalVariable :: NotThreadLocal , <ISA_LIT>AS :: LOCAL_ADDRESS ) ;

LDSPointer -> setUnnamedAddr ( GlobalValue :: UnnamedAddr :: Global ) ;


<s> UserStack . push_back ( U ) ; while ( ! UserStack . empty ( ) ) { auto * UU = UserStack . pop_back_val ( ) ; if ( ! VisitedUsers . insert ( UU ) . second ) continue ; if ( isa < GlobalValue > ( UU ) ) continue ; if ( isa < Constant > ( UU ) ) { append_range ( UserStack , UU -> users ( ) ) ; continue ; } auto * I = cast < Instruction > ( UU ) ; Function * F = I -> getFunction ( ) ; if ( CollectKernelInsts ) { if ( ! llvm :: <STR_LIT> ( F ) ) { continue ; } } else { if ( llvm :: <STR_LIT> ( F ) ) { continue ; } }

FunctionToInsts . insert ( std :: make_pair ( F , SmallPtrSet < Instruction * , <NUM_LIT> > ( ) ) ) ;


<s> set_intersect ( Callees , LDSAccessors ) ; if ( Callees . empty ( ) ) continue ; LDSPointer = createLDSPointer ( GV ) ; initializeLDSPointer ( K , GV , LDSPointer ) ; } if ( ! LDSPointer ) return false ; replaceLDSUseByPointer ( GV , LDSPointer ) ;


<s> } else { auto * CE = cast < ConstantExpr > ( U ) ; convertConstantExprsToInstructions ( I , CE , & UserInsts ) ; } for ( auto * II : UserInsts ) { auto * ReplaceInst = getReplacementInst ( F , GV , LDSPointer ) ;

II -> replaceUsesOfWith ( GV , ReplaceInst ) ;


<s> SmallVector < User * , <NUM_LIT> > LDSUsers ( GV -> users ( ) ) ; for ( auto * U : LDSUsers ) { auto FunctionToInsts = <STR_LIT> ( U , false ) ; for ( const auto & FunctionToInst : FunctionToInsts ) {


<s> auto BasicBlockEntry = KernelToInitBB . insert ( std :: make_pair ( K , nullptr ) ) ; if ( ! BasicBlockEntry . second ) return BasicBlockEntry . first -> second ; auto & EBB = K -> getEntryBlock ( ) ; auto * EI = & ( * ( EBB . getFirstInsertionPt ( ) ) ) ; BasicBlock :: reverse_iterator RIT ( EBB . getTerminator ( ) ) ; while ( ! isa < AllocaInst > ( * RIT ) && ( & * RIT != EI ) ) ++ RIT ;

if ( isa < AllocaInst > ( * RIT ) ) -- RIT ;


<s> if ( ! BasicBlockEntry . second ) return BasicBlockEntry . first -> second ; auto * EI = & ( * ( K -> getEntryBlock ( ) . getFirstInsertionPt ( ) ) ) ; IRBuilder < > Builder ( EI ) ; Value * Mbcnt = Builder . CreateIntrinsic ( <STR_LIT> , { } , { Builder . getInt32 ( - <NUM_LIT> ) , Builder . getInt32 ( <NUM_LIT> ) } ) ; Value * Cond = Builder . CreateICmpEQ ( Mbcnt , Builder . getInt32 ( <NUM_LIT> ) ) ; Instruction * WB = cast < Instruction > ( Builder . CreateIntrinsic ( <STR_LIT> , { } , { } ) ) ; BasicBlock * NBB = SplitBlockAndInsertIfThen ( Cond , WB , false ) -> getParent ( ) ; KernelToInitBB [ K ] = NBB ; return NBB ;


<s> Value * Cond = Builder . CreateICmpEQ ( Mbcnt , Builder . getInt32 ( <NUM_LIT> ) ) ; Instruction * WB = cast < Instruction > ( Builder . CreateIntrinsic ( <STR_LIT> , { } , { } ) ) ;

BasicBlock * NBB = SplitBlockAndInsertIfThen ( Cond , WB , false ) -> getParent ( ) ;


<s> std :: vector < GlobalVariable * > LDSGlobals = <STR_LIT> ( M ) ; llvm :: erase_if ( LDSGlobals , [ & ] ( GlobalVariable * GV ) { return shouldIgnorePointerReplacement ( GV ) ;


<s> return new <ISA_LIT>ReplaceLDSUseWithPointer ( ) ;


return new <ISA_LIT>ReplaceLDSUseWithPointer ( ) ;


<s> if ( ! PointerEntry . second ) return PointerEntry . first -> second ; auto * I16Ty = Type :: getInt16Ty ( Ctx ) ; GlobalVariable * LDSPointer = new GlobalVariable ( M , I16Ty , false , GlobalValue :: InternalLinkage , UndefValue :: get ( I16Ty ) , GV -> getName ( ) + Twine ( <STR_LIT> ) , nullptr , GlobalVariable :: NotThreadLocal , <ISA_LIT>AS :: LOCAL_ADDRESS ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {

AU . addRequired < TargetPassConfig > ( ) ;


<s> auto * EI = & ( * ( F -> getEntryBlock ( ) . getFirstInsertionPt ( ) ) ) ; IRBuilder < > Builder ( EI ) ;

auto * V = Builder . CreateBitCast ( Builder . CreateGEP ( Builder . getInt8Ty ( ) , LDSMemBaseAddr , Builder . CreateLoad ( LDSPointer -> getValueType ( ) , LDSPointer ) ) , GV -> getType ( ) ) ;


<s> if ( ! LDSEntry . second ) { auto ReplaceInstEntry = LDSEntry . first -> second . insert ( std :: make_pair ( GV , nullptr ) ) ; if ( ! ReplaceInstEntry . second ) return ReplaceInstEntry . first -> second ; } auto * EI = & ( * ( F -> getEntryBlock ( ) . getFirstInsertionPt ( ) ) ) ;


<s> auto PointerEntry = KernelToLDSPointers . insert ( std :: make_pair ( K , SmallPtrSet < GlobalVariable * , <NUM_LIT> > ( ) ) ) ; if ( ! PointerEntry . second ) if ( PointerEntry . first -> second . contains ( LDSPointer ) ) return ; auto * EI = & ( * ( activateLaneZero ( K ) -> getFirstInsertionPt ( ) ) ) ; IRBuilder < > Builder ( EI ) ; Builder . CreateStore ( Builder . CreatePtrToInt ( GV , Type :: getInt16Ty ( Ctx ) ) , LDSPointer ) ; KernelToLDSPointers [ K ] . insert ( LDSPointer ) ;


<s> void initializeLDSPointer ( Function * K , GlobalVariable * GV , GlobalVariable * LDSPointer ) { auto PointerEntry = KernelToLDSPointers . insert ( std :: make_pair ( K , SmallPtrSet < GlobalVariable * , <NUM_LIT> > ( ) ) ) ; if ( ! PointerEntry . second ) if ( PointerEntry . first -> second . contains ( LDSPointer ) ) return ;

auto * EI = & ( * ( activateLaneZero ( K ) -> getFirstInsertionPt ( ) ) ) ;


<s> for ( auto * I : Insts ) { SmallPtrSet < Instruction * , <NUM_LIT> > UserInsts ; if ( U == I ) { UserInsts . insert ( I ) ; } else { auto * CE = cast < ConstantExpr > ( U ) ; convertConstantExprsToInstructions ( I , CE , & UserInsts ) ; } for ( auto * II : UserInsts ) { auto * ReplaceInst = getReplacementInst ( F , GV , LDSPointer ) ; II -> replaceUsesOfWith ( GV , ReplaceInst ) ; } } }


<s> SmallVector < User * , <NUM_LIT> > LDSUsers ( GV -> users ( ) ) ; for ( auto * U : LDSUsers ) { auto FunctionToInsts = <STR_LIT> ( U , false ) ; for ( auto FI = FunctionToInsts . begin ( ) , FE = FunctionToInsts . end ( ) ; FI != FE ; ++ FI ) { Function * F = FI -> first ; auto & Insts = FI -> second ; for ( auto * I : Insts ) { SmallPtrSet < Instruction * , <NUM_LIT> > UserInsts ; if ( U == I ) { UserInsts . insert ( I ) ; } else { auto * CE = cast < ConstantExpr > ( U ) ; convertConstantExprsToInstructions ( I , CE , & UserInsts ) ; } for ( auto * II : UserInsts ) { auto * ReplaceInst = getReplacementInst ( F , GV , LDSPointer ) ; II -> replaceUsesOfWith ( GV , ReplaceInst ) ;


<s> LDSUseReplacer . replaceLDSUse ( ) ; return PreservedAnalyses :: all ( ) ;


<s> ReplaceLDSUseImpl LDSUseReplacer { M } ;

return LDSUseReplacer . replaceLDSUse ( ) ;


<s> ReplaceLDSUseImpl LDSUseReplacer { M } ; return LDSUseReplacer . replaceLDSUse ( ) ;


<s> <ISA_LIT>ResourceUsageAnalysis ( ) : CallGraphSCCPass ( ID ) {


<s> bool doInitialization ( CallGraph & CG ) override {


<s> return CallGraphSCCPass :: doInitialization ( CG ) ;


<s> assert ( MF && <STR_LIT> ) ; auto CI = CallGraphResourceInfo . insert ( std :: make_pair ( & F , <ISA_LIT>FunctionResourceInfo ( ) ) ) ; <ISA_LIT>FunctionResourceInfo & Info = CI . first -> second ; assert ( CI . second && <STR_LIT> ) ; Info = analyzeResourceUsage ( * MF , TM ) ; HasIndirectCall |= Info . HasIndirectCall ; } if ( HasIndirectCall ) propagateIndirectCallRegisterUsage ( ) ; return false ;


<s> <ISA_LIT>ResourceUsageAnalysis ( ) : ModulePass ( ID ) {


<s> CallGraphResourceInfo . clear ( ) ; return ModulePass :: doInitialization ( M ) ;


<s> assert ( Op . getImm ( ) == <NUM_LIT> ) ; return nullptr ; } if ( auto * GA = dyn_cast < GlobalAlias > ( Op . getGlobal ( ) ) ) return cast < Function > ( GA -> getOperand ( <NUM_LIT> ) ) ;

return cast < Function > ( Op . getGlobal ( ) ) ;


<s> } if ( auto * GA = dyn_cast < GlobalAlias > ( Op . getGlobal ( ) ) ) return cast < Function > ( GA -> getOperand ( <NUM_LIT> ) ) ;


<s> } } for ( auto & I : CallGraphResourceInfo ) { auto & Info = I . getSecond ( ) ; if ( Info . HasIndirectCall ) { Info . NumExplicitSGPR = std :: max ( Info . NumExplicitSGPR , NonKernelMaxSGPRs ) ; Info . NumVGPR = std :: max ( Info . NumVGPR , NonKernelMaxVGPRs ) ; Info . NumAGPR = std :: max ( Info . NumAGPR , NonKernelMaxAGPRs ) ; } }


<s> Function * F = I -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) ) continue ; MachineModuleInfo & MMI = getAnalysis < MachineModuleInfoWrapperPass > ( ) . getMMI ( ) ; MachineFunction & MF = MMI . getOrCreateMachineFunction ( * F ) ; auto CI = CallGraphResourceInfo . insert ( std :: make_pair ( & MF . getFunction ( ) , <ISA_LIT>FunctionResourceInfo ( ) ) ) ; <ISA_LIT>FunctionResourceInfo & Info = CI . first -> second ; assert ( CI . second && <STR_LIT> ) ; Info = analyzeResourceUsage ( MF , TM ) ; HasIndirectCall |= Info . HasIndirectCall ; } if ( HasIndirectCall ) propagateIndirectCallRegisterUsage ( ) ; return false ;


<s> bool <ISA_LIT>ResourceUsageAnalysis :: runOnSCC ( CallGraphSCC & SCC ) { auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; if ( ! TPC ) return false ; const TargetMachine & TM = TPC -> getTM < TargetMachine > ( ) ; bool HasIndirectCall = false ; for ( CallGraphNode * I : SCC ) { Function * F = I -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) ) continue ; MachineModuleInfo & MMI = getAnalysis < MachineModuleInfoWrapperPass > ( ) . getMMI ( ) ; MachineFunction & MF = MMI . getOrCreateMachineFunction ( * F ) ; auto CI = CallGraphResourceInfo . insert ( std :: make_pair ( & MF . getFunction ( ) , <ISA_LIT>FunctionResourceInfo ( ) ) ) ; <ISA_LIT>FunctionResourceInfo & Info = CI . first -> second ; assert ( CI . second && <STR_LIT> ) ;

Info = analyzeResourceUsage ( MF , TM ) ;


<s> <ISA_LIT>RewriteOutArguments ( ) : FunctionPass ( ID ) {


<s> StoreInst * <ISA_LIT> = dyn_cast < StoreInst > ( U . getUser ( ) ) ; if ( UseCount > MaxUses ) return false ; if ( ! <ISA_LIT> ) { auto * BCI = dyn_cast < BitCastInst > ( U . getUser ( ) ) ; if ( ! BCI || ! BCI -> hasOneUse ( ) ) return false ; Type * DestEltTy = BCI -> getType ( ) -> getPointerElementType ( ) ; if ( DestEltTy -> isAggregateType ( ) ) return false ; Type * SrcEltTy = Arg . getType ( ) -> getPointerElementType ( ) ; if ( SrcEltTy -> isArrayTy ( ) ) return false ; if ( ( SrcEltTy -> isStructTy ( ) && ( SrcEltTy -> getStructNumElements ( ) != <NUM_LIT> ) ) ) return false ; if ( DL -> getTypeAllocSize ( SrcEltTy ) != DL -> getTypeAllocSize ( DestEltTy ) ) return false ;


<s> return new <ISA_LIT>RewriteOutArguments ( ) ;


<s> bool <ISA_LIT>RewriteOutArguments :: doInitialization ( Module & M ) { DL = & M . getDataLayout ( ) ; return false ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {

AU . addRequired < MemoryDependenceWrapperPass > ( ) ;


<s> VectorType * VT0 = dyn_cast < VectorType > ( Ty0 ) ; VectorType * VT1 = dyn_cast < VectorType > ( Ty1 ) ; if ( ! VT0 || ! VT1 ) return false ; if ( VT0 -> getNumElements ( ) != <NUM_LIT> || VT1 -> getNumElements ( ) != <NUM_LIT> ) return false ;

return DL -> getTypeSizeInBits ( VT0 -> getElementType ( ) ) == DL -> getTypeSizeInBits ( VT1 -> getElementType ( ) ) ;


<s> if ( ! VT0 || ! VT1 ) return false ; if ( VT0 -> getNumElements ( ) != <NUM_LIT> || VT1 -> getNumElements ( ) != <NUM_LIT> ) return false ; return DL -> getTypeSizeInBits ( VT0 -> getElementType ( ) ) == DL -> getTypeSizeInBits ( VT1 -> getElementType ( ) ) ;


<s> bool <ISA_LIT>RewriteOutArguments :: isVec3ToVec4Shuffle ( Type * Ty0 , Type * Ty1 ) const { auto * VT0 = dyn_cast < FixedVectorType > ( Ty0 ) ; auto * VT1 = dyn_cast < FixedVectorType > ( Ty1 ) ; if ( ! VT0 || ! VT1 ) return false ;

if ( VT0 -> getNumElements ( ) != <NUM_LIT> || VT1 -> getNumElements ( ) != <NUM_LIT> ) return false ;


<s> NewFunc -> setComdat ( F . getComdat ( ) ) ; NewFunc -> stealArgumentListFrom ( F ) ; AttrBuilder RetAttrs ; RetAttrs . addAttribute ( Attribute :: SExt ) ; RetAttrs . addAttribute ( Attribute :: ZExt ) ; RetAttrs . addAttribute ( Attribute :: NoAlias ) ; NewFunc -> removeAttributes ( AttributeList :: ReturnIndex , RetAttrs ) ; NewFunc -> getBasicBlockList ( ) . splice ( NewFunc -> begin ( ) , F . getBasicBlockList ( ) ) ; for ( std :: pair < ReturnInst * , ReplacementVec > & Replacement : Replacements ) { ReturnInst * RI = Replacement . first ; IRBuilder < > B ( RI ) ; B . SetCurrentDebugLocation ( RI -> getDebugLoc ( ) ) ; int RetIdx = <NUM_LIT> ; Value * NewRetVal = UndefValue :: get ( NewRetTy ) ; Value * RetVal = RI -> getReturnValue ( ) ; if ( RetVal ) NewRetVal = B . CreateInsertValue ( NewRetVal , RetVal , RetIdx ++ ) ; for ( std :: pair < Argument * , Value * > ReturnPoint : Replacement . second ) { Argument * Arg = ReturnPoint . first ; Value * Val = ReturnPoint . second ; Type * EltTy = Arg -> getType ( ) -> getPointerElementType ( ) ; if ( Val -> getType ( ) != EltTy ) { Type * EffectiveEltTy = EltTy ; if ( StructType * CT = dyn_cast < StructType > ( EltTy ) ) { assert ( CT -> getNumElements ( ) == <NUM_LIT> ) ; EffectiveEltTy = CT -> getElementType ( <NUM_LIT> ) ; } if ( DL -> getTypeSizeInBits ( EffectiveEltTy ) != DL -> getTypeSizeInBits ( Val -> getType ( ) ) ) { assert ( isVec3ToVec4Shuffle ( EffectiveEltTy , Val -> getType ( ) ) ) ; Val = B . CreateShuffleVector ( Val , ArrayRef < int > { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ) ; } Val = B . CreateBitCast ( Val , EffectiveEltTy ) ; if ( EltTy != EffectiveEltTy ) Val = B . CreateInsertValue ( UndefValue :: get ( EltTy ) , Val , <NUM_LIT> ) ; } NewRetVal = B . CreateInsertValue ( NewRetVal , Val , RetIdx ++ ) ; } if ( RetVal ) RI -> setOperand ( <NUM_LIT> , NewRetVal ) ; else { B . CreateRet ( NewRetVal ) ; RI -> eraseFromParent ( ) ; } } SmallVector < Value * , <NUM_LIT> > StubCallArgs ; for ( Argument & Arg : F . args ( ) ) { if ( OutArgIndexes . count ( Arg . getArgNo ( ) ) ) { StubCallArgs . push_back ( UndefValue :: get ( Arg . getType ( ) ) ) ; } else { StubCallArgs . push_back ( & Arg ) ; } } BasicBlock * StubBB = BasicBlock :: Create ( Ctx , <STR_LIT> , & F ) ; IRBuilder < > B ( StubBB ) ;

CallInst * StubCall = B . CreateCall ( NewFunc , StubCallArgs ) ;