<s> switch ( MI . getOperand ( MI . getNumExplicitDefs ( ) ) . getIntrinsicID ( ) ) { case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT>


<s> for ( unsigned Reg : Regs ) { assert ( MRI . getType ( Reg ) . getSizeInBits ( ) == NewTy


<s> assert ( MRI . getType ( Reg ) . getSizeInBits ( ) == NewTy . getSizeInBits ( ) ) ; MRI . setType ( Reg , NewTy


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank && Bank -> getID ( ) != <STR_LIT>


<s> static bool AlreadyInit = false ; if ( AlreadyInit ) return ; AlreadyInit = true ; const RegisterBank & RBSGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBSGPR ; assert ( & RBSGPR == & <STR_LIT> ) ; const RegisterBank & RBVGPR = getRegBank ( <STR_LIT>


<s> const RegisterBank & <ISA_LIT>RegisterBankInfo :: getRegBankFromRegClass ( const TargetRegisterClass & RC ) const { if ( & RC == & <STR_LIT> ) return <STR_LIT> ; if ( TRI -> isSGPRClass ( & RC ) ) return <STR_LIT> ; if ( TRI -> isAGPRClass ( & RC ) ) return <STR_LIT> ; return <STR_LIT>


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; InstructionMappings AltMappings ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_LOAD : { InstructionMapping SSMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . emplace_back ( std :: move ( SSMapping ) ) ; InstructionMapping VVMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . emplace_back ( std :: move


<s> InstructionMapping SSMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . emplace_back ( std :: move ( SSMapping ) ) ; InstructionMapping VVMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . emplace_back ( std :: move


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Mapping = InstructionMapping { <NUM_LIT> , <NUM_LIT> , nullptr , MI . getNumOperands ( ) } ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; switch ( MI . getOpcode ( ) ) { default : break ; case <STR_LIT> : { unsigned Size = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . getSizeInBits ( ) ; OpdsMapping [ <NUM_LIT> ] = <STR_LIT> ( <STR_LIT> , Size ) ; Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ; return Mapping ; } case <STR_LIT> : { for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; unsigned Size = MRI . getType ( MI . getOperand ( i ) . getReg ( ) ) . getSizeInBits ( ) ; OpdsMapping [ i ] = <STR_LIT> ( <STR_LIT> , Size ) ; } Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ; return Mapping ; } case <STR_LIT> : { assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; unsigned Size = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . getSizeInBits ( ) ; const ValueMapping * ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; const ValueMapping * PtrMapping = <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) ; OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ; return Mapping ; } case <STR_LIT> : return getInstrMappingForLoad ( MI ) ; } unsigned BankID = <STR_LIT> ; Mapping = InstructionMapping { <NUM_LIT> , <NUM_LIT> , nullptr , MI . getNumOperands ( ) } ; unsigned Size = <NUM_LIT> ; for ( unsigned Idx = <NUM_LIT> ; Idx < MI . getNumOperands ( ) ; ++ Idx ) { if ( MI . getOperand ( Idx ) . isReg ( ) ) Size = getSizeInBits ( MI . getOperand ( Idx ) .


<s> unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; unsigned PtrSize = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; if ( isInstrUniform ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ; return Mapping


<s> unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; unsigned PtrSize = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; if ( isInstrUniform ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; Mapping . setOperandsMapping ( getOperandsMapping ( OpdsMapping ) ) ; return


<s> } LLT EltTy = VecTy . getScalarType ( ) ; SmallVector < Register , <NUM_LIT> > InsRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; unsigned NumLanes = InsRegs . size ( ) ; if ( ! NumLanes ) { NumLanes = <NUM_LIT> ; InsRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } else { EltTy = MRI . getType ( InsRegs [ <NUM_LIT> ] ) ; } auto UnmergeToEltTy = B . buildUnmerge ( EltTy , VecReg ) ; SmallVector < Register , <NUM_LIT> > Ops ( NumElem * NumLanes ) ; for ( unsigned I = <NUM_LIT> ; I < NumElem ; ++ I ) { auto IC = B . buildConstant ( S32 , I ) ; MRI . setRegBank ( IC -> getOperand ( <NUM_LIT> ) . getReg ( ) , <STR_LIT> ) ; auto Cmp = B . buildICmp ( CmpInst :: ICMP_EQ , CCTy , Idx , IC ) ; MRI . setRegBank ( Cmp -> getOperand ( <NUM_LIT> ) . getReg ( ) , CCBank ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) { auto S = B . buildSelect ( EltTy , Cmp , InsRegs [ L ] , UnmergeToEltTy . getReg ( I * NumLanes + L ) ) ; for ( unsigned N : { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ) MRI . setRegBank ( S -> getOperand ( N ) . getReg ( ) , DstBank ) ; Ops [ I * NumLanes + L ] = S -> getOperand ( <NUM_LIT> ) . getReg ( ) ; } } LLT MergeTy = LLT :: vector ( Ops . size ( ) , EltTy ) ; if ( MergeTy == MRI . getType ( MI . getOperand ( <NUM_LIT> ) .


<s> if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = CombinedOffset ; } else { VOffsetReg = B . buildCopy ( S32 , CombinedOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; } SOffsetReg = B . buildConstant ( S32 ,


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const LLT LoadTy = MRI . getType ( DstReg ) ; unsigned LoadSize = LoadTy . getSizeInBits ( ) ; const unsigned MaxNonSmrdLoadSize = <NUM_LIT> ; if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) { Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> ) return false ; SrcRegs . push_back ( PtrReg ) ; } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy . getScalarType ( ) . getSizeInBits ( ) ; const LLT LoadSplitTy = LLT :: vector ( SplitElts , LoadTy . getScalarType ( ) ) ; ApplyRegBankMapping O ( MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; MachineInstr * LegalizedInst = getOtherVRegDef ( MRI , DstReg , * RepairInst ) ; Register TmpReg = MRI . createGenericVirtualRegister ( MRI . getType ( DstReg ) ) ; LegalizedInst -> getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; B . setInsertPt ( * RepairInst -> getParent ( ) , RepairInst ) ; for ( unsigned DefIdx = <NUM_LIT> , e = DefRegs . size ( ) ; DefIdx != e ; ++ DefIdx ) { Register IdxReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildConstant ( IdxReg , DefIdx ) ; MRI . setRegBank ( IdxReg


<s> const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ; Regs . push_back


<s> if ( LoadSize != <NUM_LIT> ) return false ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineIRBuilder B ( MI ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; if ( MMO -> getAlign ( ) < Align ( <NUM_LIT> ) ) { LLT Part64 , Part32 ; std :: tie ( Part64 , Part32 ) = splitUnequalType ( LoadTy , <NUM_LIT> ) ; auto Load0 = B . buildLoadFromOffset ( Part64 , PtrReg , * MMO , <NUM_LIT> ) ; auto Load1 = B . buildLoadFromOffset ( Part32 , PtrReg , * MMO , <NUM_LIT> ) ; auto Undef = B . buildUndef ( LoadTy ) ; auto Ins0 = B . buildInsert ( LoadTy , Undef , Load0 , <NUM_LIT> ) ; B . buildInsert ( MI . getOperand ( <NUM_LIT> ) , Ins0 , Load1 , <NUM_LIT> ) ; } else { LLT WiderTy = widen96To128 ( LoadTy ) ; auto WideLoad = B . buildLoadFromOffset ( WiderTy , PtrReg , * MMO , <NUM_LIT> ) ; B . buildExtract ( MI . getOperand ( <NUM_LIT> ) , WideLoad , <NUM_LIT> ) ; } MI . eraseFromParent ( ) ; return


<s> std :: tie ( Part64 , Part32 ) = splitUnequalType ( LoadTy , <NUM_LIT> ) ; auto Load0 = B . buildLoadFromOffset ( Part64 , PtrReg , * MMO , <NUM_LIT> ) ; auto Load1 = B . buildLoadFromOffset ( Part32 , PtrReg , * MMO , <NUM_LIT> ) ; auto Undef = B . buildUndef ( LoadTy ) ; auto Ins0 = B . buildInsert ( LoadTy , Undef , Load0 , <NUM_LIT> ) ; B . buildInsert ( MI . getOperand ( <NUM_LIT> ) , Ins0 , Load1 , <NUM_LIT> ) ; } else { LLT WiderTy = widen96To128 ( LoadTy ) ; auto WideLoad = B . buildLoadFromOffset ( WiderTy , PtrReg , * MMO , <NUM_LIT> ) ; B . buildExtract ( MI . getOperand ( <NUM_LIT> ) , WideLoad , <NUM_LIT> ) ; } MI . eraseFromParent ( ) ; return true ; } if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < Register , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < Register , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping O ( * this , MRI ,


<s> else EltTy = MRI . getType ( DstRegs [ <NUM_LIT> ] ) ; auto UnmergeToEltTy = B . buildUnmerge ( EltTy , VecReg ) ; SmallVector < Register , <NUM_LIT> > Res ( NumLanes ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) Res [ L ] = UnmergeToEltTy . getReg ( L ) ; for ( unsigned I = <NUM_LIT> ; I < NumElem ; ++ I ) { auto IC = B . buildConstant ( S32 , I ) ; MRI . setRegBank ( IC -> getOperand ( <NUM_LIT> ) . getReg ( ) , <STR_LIT> ) ; auto Cmp = B . buildICmp ( CmpInst :: ICMP_EQ , CCTy , Idx , IC ) ; MRI . setRegBank ( Cmp -> getOperand ( <NUM_LIT> ) . getReg ( ) , CCBank ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) { auto S = B . buildSelect ( EltTy , Cmp , UnmergeToEltTy . getReg ( I * NumLanes + L ) , Res [ L ] ) ; for ( unsigned N : { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ) MRI . setRegBank ( S -> getOperand ( N ) . getReg ( ) , DstBank ) ; Res [ L ] = S -> getOperand ( <NUM_LIT> ) . getReg ( ) ; } } for ( unsigned L = <NUM_LIT> ; L <


<s> static void setRegsToType ( MachineRegisterInfo & MRI , ArrayRef < unsigned > Regs


<s> unsigned LoLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; unsigned HiLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS ,


<s> unsigned HiLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ; Regs . push_back ( HiLHS ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoLHS ) . addDef (


<s> break ; } assert ( DefRegs . size ( ) == <NUM_LIT> ) ; assert ( Src0Regs . size ( ) == Src1Regs . size ( ) && ( Src0Regs . empty ( ) || Src0Regs . size ( ) == <NUM_LIT> ) ) ; MachineIRBuilder B ( MI ) ; if ( Src0Regs . empty ( ) ) split64BitValueForMapping ( B , Src0Regs , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Src1Regs . empty ( ) ) split64BitValueForMapping ( B , Src1Regs , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; B . buildInstr ( Opc ) . addDef ( DefRegs [ <NUM_LIT> ] ) . addUse ( Src0Regs [ <NUM_LIT> ] ) . addUse ( Src1Regs [ <NUM_LIT> ] ) ; B . buildInstr ( Opc ) . addDef ( DefRegs [ <NUM_LIT> ] ) . addUse ( Src0Regs [ <NUM_LIT> ] ) . addUse ( Src1Regs [ <NUM_LIT> ] ) ; MRI . setRegBank ( DstReg , getRegBank (


<s> const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; const InstructionMapping & SVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; break ; } case TargetOpcode :: G_LOAD : { unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; return AltMappings ; } case TargetOpcode :: G_ICMP : { unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & SVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SVMapping )


<s> } if ( Src1Regs . empty ( ) ) split64BitValueForMapping ( B , Src1Regs , HalfTy , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; else { setRegsToType ( MRI , Src1Regs , HalfTy ) ; } if ( Src2Regs . empty ( ) ) split64BitValueForMapping ( B , Src2Regs , HalfTy , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; else setRegsToType ( MRI , Src2Regs , HalfTy ) ; setRegsToType ( MRI , DefRegs , HalfTy ) ; B . buildSelect ( DefRegs [ <NUM_LIT> ] , Src0Regs [ <NUM_LIT> ] , Src1Regs [ <NUM_LIT> ] , Src2Regs [ <NUM_LIT> ] ) ; B . buildSelect ( DefRegs [ <NUM_LIT> ] , Src0Regs [ <NUM_LIT> ] , Src1Regs [ <NUM_LIT> ] , Src2Regs [ <NUM_LIT> ] ) ; MRI . setRegBank ( DstReg , getRegBank ( <STR_LIT> ) ) ; MI . eraseFromParent ( ) ; return ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( DstReg ) ; if ( DstTy . getSizeInBits ( ) != <NUM_LIT> ) break ; LLT HalfTy = getHalfSizedType ( DstTy ) ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > Src0Regs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > Src1Regs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( DefRegs . empty ( ) ) { assert ( Src0Regs . empty ( ) && Src1Regs .


<s> MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , O ) ; if ( MMO -> getAlign ( ) < Align ( <NUM_LIT> ) ) { LLT Part64 , Part32 ; std :: tie ( Part64 , Part32 ) = splitUnequalType ( LoadTy , <NUM_LIT> ) ; auto Load0 = B . buildLoadFromOffset ( Part64 , PtrReg , * MMO , <NUM_LIT> ) ; auto Load1 = B . buildLoadFromOffset ( Part32 , PtrReg , * MMO , <NUM_LIT> ) ; auto Undef = B . buildUndef ( LoadTy ) ; auto Ins0 = B . buildInsert ( LoadTy , Undef , Load0 , <NUM_LIT> ) ; B . buildInsert ( MI . getOperand ( <NUM_LIT> ) , Ins0 , Load1 , <NUM_LIT> ) ; } else { LLT WiderTy = widen96To128 ( LoadTy ) ; auto WideLoad = B . buildLoadFromOffset ( WiderTy , PtrReg , * MMO , <NUM_LIT> ) ; B . buildExtract ( MI . getOperand ( <NUM_LIT> ) , WideLoad , <NUM_LIT> ) ; } MI . eraseFromParent ( ) ; return true ; } if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < Register , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < Register , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping Observer ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( LoadTy . isVector ( ) ) { if ( Helper . fewerElementsVector ( MI , <NUM_LIT> ,

LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ;


<s> if ( Optional < int64_t > Imm = getConstantVRegSExtVal ( CombinedOffset , * MRI ) ) { uint32_t SOffset , ImmOffset ; if ( <STR_LIT> ( * Imm , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; InstOffsetVal = ImmOffset ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; return SOffset + ImmOffset ; } } Register Base ; unsigned Offset ; std :: tie ( Base , Offset ) = <STR_LIT> ( * MRI , CombinedOffset ) ; uint32_t SOffset , ImmOffset ; if ( Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 ,


<s> VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = CombinedOffset ; } else { VOffsetReg = B . buildCopy ( S32 , CombinedOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; } SOffsetReg = B . buildConstant ( S32 , <NUM_LIT> )


<s> void <ISA_LIT>RegisterBankInfo :: constrainOpWithReadfirstlane ( MachineInstr & MI , MachineRegisterInfo & MRI , unsigned OpIdx ) const { Register Reg = MI . getOperand ( OpIdx ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank != & <STR_LIT> ) return ; MachineIRBuilder B ( MI ) ; Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ; const TargetRegisterClass * Constrained = constrainGenericRegister ( Reg ,


<s> <ISA_LIT>RegisterBankInfo :: <ISA_LIT>RegisterBankInfo ( const TargetRegisterInfo & TRI ) : <ISA_LIT>GenRegisterBankInfo ( ) , TRI ( static_cast < const <ISA_LIT>RegisterInfo * > ( & TRI ) ) { static bool AlreadyInit = false ; if ( AlreadyInit ) return ; AlreadyInit = true ; const RegisterBank & RBSGPR = getRegBank ( <STR_LIT> ) ; ( void )


<s> void <ISA_LIT>RegisterBankInfo :: applyMappingImpl ( const OperandsMapper


<s> return applyDefaultMapping ( OpdMapper


<s> if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) { return std :: numeric_limits < unsigned > :: max ( ) ; } if ( Size == <NUM_LIT> && Dst . getID ( ) == <STR_LIT> && ( Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ; return RegisterBankInfo :: copyCost ( Dst , Src


<s> unsigned Size = getSizeInBits ( MI . getOperand ( I ) . getReg ( ) , MRI , * TRI ) ; OpdsMapping [ I ] = <STR_LIT> ( <STR_LIT> , Size ) ; } return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI .


<s> return AltMappings ; } case TargetOpcode :: G_ICMP : { unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & SVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , nullptr , <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , Size ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; return AltMappings ; } case TargetOpcode :: G_SELECT : { unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) , <STR_LIT> ( <STR_LIT> , Size


<s> const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; unsigned PtrSize = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; if ( isInstrUniform ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> (


<s> const RegisterBank & <ISA_LIT>RegisterBankInfo :: getRegBankFromRegClass ( const TargetRegisterClass & RC ) const { if ( TRI -> isSGPRClass ( & RC )


<s> const RegisterBank * Bank = getRegBank ( Reg , MRI , TRI ) ; return Bank ? Bank -> getID ( ) :


<s> if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( Bank -> getID ( ) == <STR_LIT> ) return false ; assert ( Bank -> getID ( ) == <STR_LIT> || Bank -> getID ( ) == <STR_LIT> ) ; } } return true


<s> virtual unsigned getHWRegIndex ( unsigned


<s> assert ( ! <STR_LIT> ) ; return BitVector


<s> BitVector getReservedRegs ( const MachineFunction & MF )


<s> unsigned <ISA_LIT>RegisterInfo :: getFrameRegister ( const MachineFunction & MF )


<s> const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return FuncInfo -> getFrameOffsetReg


<s> bool enableMultipleCopyHints ( ) const override


<s> CallingConv :: ID CC = MF -> getFunction ( ) -> getCallingConv ( ) ; switch ( CC ) { case CallingConv :: C : case CallingConv :: Fast : return CSR_<ISA_LIT>_HighRegs_SaveList ; default : { static const MCPhysReg NoCalleeSavedReg = <STR_LIT> ; return & NoCalleeSavedReg


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getCallPreservedMask ( const MachineFunction & MF , CallingConv :: ID CC ) const { switch ( CC ) { case CallingConv :: C : case CallingConv ::


<s> const MCPhysReg * <ISA_LIT>RegisterInfo :: getCalleeSavedRegs ( const MachineFunction * ) const { return & CalleeSavedReg


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getAllVGPRRegMask


<s> const MCPhysReg * <ISA_LIT>RegisterInfo :: getCalleeSavedRegsViaCopy ( const MachineFunction * MF


<s> const MCPhysReg * <ISA_LIT>RegisterInfo :: getCalleeSavedRegsViaCopy ( const MachineFunction * MF ) const { return


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getCallPreservedMask ( const MachineFunction & MF , CallingConv :: ID CC ) const { switch ( CC ) { case CallingConv :: C : case CallingConv :: Fast : case CallingConv :: Cold : return CSR_<ISA_LIT>_HighRegs_RegMask ; default : return nullptr


<s> const unsigned NumRegIndex = NumRegs - <NUM_LIT> ; assert ( NumRegIndex < array_lengthof ( SubRegFromChannelTable ) && <STR_LIT> ) ; assert ( Channel < array_lengthof (

SubRegFromChannelTable [ <NUM_LIT> ] ) ) ;


<s> for ( ; R . isValid ( ) ; ++ R ) Reserved


<s> virtual const TargetRegisterClass * getCFGStructurizerRegClass (


<s> const <ISA_LIT>FrameLowering * TFI = MF . getSubtarget < GCNSubtarget > ( ) . getFrameLowering ( ) ; const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : FuncInfo ->


<s> void <ISA_LIT>RegisterInfo :: eliminateFrameIndex ( MachineBasicBlock :: iterator MI , int SPAdj , unsigned FIOperandNum , RegScavenger * RS ) const { llvm_unreachable ( <STR_LIT>


<s> const MCPhysReg * <ISA_LIT>RegisterInfo :: getCalleeSavedRegs ( const MachineFunction * MF ) const { return &


<s> unsigned <ISA_LIT>RegisterInfo :: getIndirectSubReg ( unsigned


<s> std :: vector < GlobalVariable * > LDSGlobals = <STR_LIT> ( M ) ; LDSGlobals . erase ( std :: remove_if ( LDSGlobals . begin ( ) , LDSGlobals . end ( ) , [ & ] ( GlobalVariable * GV ) { return shouldIgnorePointerReplacement ( GV


<s> for ( const auto & GI : * ECNode ) { auto * CGN = GI . second ; auto * F = CGN -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) || llvm ::


<s> auto * ECNode = CG . getExternalCallingNode ( ) ; for ( const auto & GI : * ECNode ) { auto * CGN = GI . second ; auto * F = CGN -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) || llvm :: <STR_LIT> (


<s> std :: vector < GlobalVariable * > collectLDSRequiringPointerReplace ( ) { std :: vector < GlobalVariable * > LDSGlobals = llvm :: <STR_LIT> ( M ) ; llvm :: erase_if ( LDSGlobals , [ & ] ( GlobalVariable * GV ) { return shouldIgnorePointerReplacement ( GV ) ; } )


<s> void collectReachableCallees ( DenseMap < Function * , SmallPtrSet < Function * , <NUM_LIT> >> & KernelToCallees ) { for ( Function & F : M . functions ( ) ) { if ( ! llvm :: <STR_LIT> ( &


<s> auto * I16Ty = Type :: getInt16Ty ( Ctx ) ; GlobalVariable * LDSPointer = new GlobalVariable ( M , I16Ty , false , GlobalValue :: InternalLinkage , UndefValue :: get ( I16Ty ) , GV -> getName ( ) + Twine ( <STR_LIT> ) , nullptr , GlobalVariable :: NotThreadLocal , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; LDSPointer -> setUnnamedAddr ( GlobalValue :: UnnamedAddr


<s> UserStack . push_back ( U ) ; while ( ! UserStack . empty ( ) ) { auto * UU = UserStack . pop_back_val ( ) ; if ( ! VisitedUsers . insert ( UU ) . second ) continue ; if ( isa < GlobalValue > ( UU ) ) continue ; if ( isa < Constant > ( UU ) ) { append_range ( UserStack , UU -> users ( ) ) ; continue ; } auto * I = cast < Instruction > ( UU ) ; Function * F = I -> getFunction ( ) ; if ( CollectKernelInsts ) { if ( ! llvm :: <STR_LIT> ( F ) ) { continue ; } } else { if ( llvm :: <STR_LIT> ( F ) ) { continue ; } } FunctionToInsts . insert ( std :: make_pair ( F , SmallPtrSet < Instruction * , <NUM_LIT> >


<s> set_intersect ( Callees , LDSAccessors ) ; if ( Callees . empty ( ) ) continue ; LDSPointer = createLDSPointer ( GV ) ; initializeLDSPointer ( K , GV , LDSPointer ) ; } if ( ! LDSPointer ) return false ; replaceLDSUseByPointer ( GV , LDSPointer ) ; return true


<s> } else { auto * CE = cast < ConstantExpr > ( U ) ; convertConstantExprsToInstructions ( I , CE , & UserInsts ) ; } for ( auto * II : UserInsts ) { auto * ReplaceInst = getReplacementInst ( F , GV , LDSPointer ) ; II -> replaceUsesOfWith ( GV ,


<s> SmallVector < User * , <NUM_LIT> > LDSUsers ( GV -> users ( ) ) ; for ( auto * U : LDSUsers ) { auto FunctionToInsts = <STR_LIT> ( U , false ) ; for ( const auto & FunctionToInst : FunctionToInsts ) { Function * F = FunctionToInst


<s> auto BasicBlockEntry = KernelToInitBB . insert ( std :: make_pair ( K , nullptr ) ) ; if ( ! BasicBlockEntry . second ) return BasicBlockEntry . first -> second ; auto & EBB = K -> getEntryBlock ( ) ; auto * EI = & ( * ( EBB . getFirstInsertionPt ( ) ) ) ; BasicBlock :: reverse_iterator RIT ( EBB . getTerminator ( ) ) ; while ( ! isa < AllocaInst > ( * RIT ) && ( & * RIT != EI ) ) ++ RIT ; if ( isa < AllocaInst > ( * RIT )


<s> Value * Cond = Builder . CreateICmpEQ ( Mbcnt , Builder . getInt32 ( <NUM_LIT> ) ) ; Instruction * WB = cast < Instruction > ( Builder . CreateIntrinsic ( <STR_LIT> , { } , { } ) ) ; BasicBlock * NBB = SplitBlockAndInsertIfThen ( Cond , WB , false )


<s> std :: vector < GlobalVariable * > LDSGlobals = <STR_LIT> ( M ) ; llvm :: erase_if ( LDSGlobals , [ & ] ( GlobalVariable * GV ) { return shouldIgnorePointerReplacement ( GV ) ; }


<s> if ( ! PointerEntry . second ) return PointerEntry . first -> second ; auto * I16Ty = Type :: getInt16Ty ( Ctx ) ; GlobalVariable * LDSPointer = new GlobalVariable ( M , I16Ty , false , GlobalValue :: InternalLinkage , UndefValue :: get ( I16Ty ) , GV -> getName ( ) + Twine ( <STR_LIT> ) , nullptr , GlobalVariable :: NotThreadLocal , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; LDSPointer -> setUnnamedAddr ( GlobalValue ::


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < TargetPassConfig >


<s> AU . addRequired < TargetPassConfig > (


<s> auto * EI = & ( * ( F -> getEntryBlock ( ) . getFirstInsertionPt ( ) ) ) ; IRBuilder < > Builder ( EI ) ; auto * V = Builder . CreateBitCast ( Builder . CreateGEP ( Builder . getInt8Ty ( ) , LDSMemBaseAddr , Builder . CreateLoad

( LDSPointer -> getValueType ( ) , LDSPointer ) ) , GV -> getType ( ) ) ;


<s> if ( ! LDSEntry . second ) { auto ReplaceInstEntry = LDSEntry . first -> second . insert ( std :: make_pair ( GV , nullptr ) ) ; if ( ! ReplaceInstEntry . second ) return ReplaceInstEntry . first -> second ; } auto * EI = & ( * ( F -> getEntryBlock ( ) . getFirstInsertionPt ( ) ) ) ; IRBuilder < > Builder


<s> void initializeLDSPointer ( Function * K , GlobalVariable * GV , GlobalVariable * LDSPointer ) { auto PointerEntry = KernelToLDSPointers . insert ( std :: make_pair ( K , SmallPtrSet < GlobalVariable * , <NUM_LIT> > ( ) ) ) ; if ( ! PointerEntry . second ) if ( PointerEntry . first -> second . contains ( LDSPointer ) ) return ; auto * EI = & ( * ( activateLaneZero ( K


<s> ReplaceLDSUseImpl ( Module & M ) : M ( M ) , Ctx ( M . getContext ( ) ) , DL ( M


<s> LDSMemBaseAddr = Constant :: getIntegerValue ( PointerType :: get ( Type :: getInt8Ty ( M . getContext ( ) ) , <ISA_LIT>AS

:: LOCAL_ADDRESS ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ;


<s> ReplaceLDSUseImpl LDSUseReplacer { M } ; return LDSUseReplacer . replaceLDSUse (


<s> if ( DL . getTypeAllocSize ( GV -> getValueType ( ) )


<s> bool doInitialization ( CallGraph & CG ) override { CallGraphResourceInfo . clear ( )


<s> return ModulePass :: doInitialization


<s> AU . addRequired < MachineModuleInfoWrapperPass > (


<s> assert ( Op . getImm ( ) == <NUM_LIT> ) ; return nullptr ; } if ( auto * GA = dyn_cast < GlobalAlias > ( Op . getGlobal ( ) ) ) return cast < Function > ( GA -> getOperand ( <NUM_LIT> ) ) ; return cast < Function > ( Op . getGlobal (


<s> } if ( auto * GA = dyn_cast < GlobalAlias > ( Op . getGlobal ( ) ) ) return cast < Function > ( GA -> getOperand ( <NUM_LIT> ) ) ; return cast < Function > (


<s> assert ( Info != CallGraphResourceInfo . end ( ) && <STR_LIT>


<s> bool <ISA_LIT>ResourceUsageAnalysis :: runOnSCC ( CallGraphSCC & SCC ) { auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; if ( ! TPC ) return false ; const TargetMachine & TM = TPC -> getTM < TargetMachine > ( ) ; bool HasIndirectCall = false ; for ( CallGraphNode * I : SCC ) { Function * F = I -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) ) continue ; MachineModuleInfo & MMI = getAnalysis < MachineModuleInfoWrapperPass > ( ) . getMMI ( ) ; MachineFunction & MF = MMI . getOrCreateMachineFunction ( * F ) ; auto CI = CallGraphResourceInfo . insert ( std :: make_pair ( & MF . getFunction ( ) , <ISA_LIT>FunctionResourceInfo ( ) ) ) ; <ISA_LIT>FunctionResourceInfo & Info = CI . first -> second ; assert ( CI . second && <STR_LIT> ) ; Info = analyzeResourceUsage (


<s> StoreInst * <ISA_LIT> = dyn_cast < StoreInst > ( U . getUser ( ) ) ; if ( UseCount > MaxUses ) return false ; if ( ! <ISA_LIT> ) { auto * BCI = dyn_cast < BitCastInst > ( U . getUser ( ) ) ; if ( ! BCI || ! BCI -> hasOneUse ( ) ) return false ; Type * DestEltTy = BCI -> getType ( ) -> getPointerElementType ( ) ; if ( DestEltTy -> isAggregateType ( ) ) return false ; Type * SrcEltTy = Arg . getType ( ) -> getPointerElementType ( ) ; if ( SrcEltTy -> isArrayTy ( ) ) return false ; if ( ( SrcEltTy -> isStructTy ( ) && ( SrcEltTy -> getStructNumElements ( ) != <NUM_LIT> ) ) ) return false ; if ( DL -> getTypeAllocSize ( SrcEltTy ) != DL -> getTypeAllocSize ( DestEltTy ) ) return false ; return checkArgumentUses ( *


<s> return new <ISA_LIT>RewriteOutArguments (


<s> bool <ISA_LIT>RewriteOutArguments :: doInitialization ( Module


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MemoryDependenceWrapperPass >


<s> AU . addRequired < MemoryDependenceWrapperPass > (


<s> if ( ! ArgTy || ( ArgTy -> getAddressSpace ( ) != DL -> getAllocaAddrSpace ( ) && ! AnyAddressSpace ) || Arg . hasByValAttr ( )

|| Arg . hasStructRetAttr ( ) || DL -> getTypeStoreSize ( ArgTy -> getPointerElementType ( ) ) > MaxOutArgSizeBytes ) {


<s> VectorType * VT0 = dyn_cast < VectorType > ( Ty0 ) ; VectorType * VT1 = dyn_cast < VectorType > ( Ty1 ) ; if ( ! VT0 || ! VT1 ) return false ; if ( VT0 -> getNumElements ( ) != <NUM_LIT> || VT1 -> getNumElements ( ) != <NUM_LIT> ) return false ; return DL -> getTypeSizeInBits ( VT0 -> getElementType ( ) )

== DL -> getTypeSizeInBits ( VT1 -> getElementType ( ) ) ;


<s> bool <ISA_LIT>RewriteOutArguments :: isVec3ToVec4Shuffle ( Type * Ty0 , Type * Ty1 ) const { auto * VT0 = dyn_cast < FixedVectorType > ( Ty0 ) ; auto * VT1 = dyn_cast < FixedVectorType > ( Ty1 ) ; if ( ! VT0 || ! VT1 ) return false ; if ( VT0 -> getNumElements ( ) != <NUM_LIT> || VT1

-> getNumElements ( ) != <NUM_LIT> ) return false ;


<s> NewFunc -> setComdat ( F . getComdat ( ) ) ; NewFunc -> stealArgumentListFrom ( F ) ; AttrBuilder RetAttrs ; RetAttrs . addAttribute ( Attribute :: SExt ) ; RetAttrs . addAttribute ( Attribute :: ZExt ) ; RetAttrs . addAttribute ( Attribute :: NoAlias ) ; NewFunc -> removeAttributes ( AttributeList :: ReturnIndex , RetAttrs ) ; NewFunc -> getBasicBlockList ( ) . splice ( NewFunc -> begin ( ) , F . getBasicBlockList ( ) ) ; for ( std :: pair < ReturnInst * , ReplacementVec > & Replacement : Replacements ) { ReturnInst * RI = Replacement . first ; IRBuilder < > B ( RI ) ; B . SetCurrentDebugLocation ( RI -> getDebugLoc ( ) ) ; int RetIdx = <NUM_LIT> ; Value * NewRetVal = UndefValue :: get ( NewRetTy ) ; Value * RetVal = RI -> getReturnValue ( ) ; if ( RetVal ) NewRetVal = B . CreateInsertValue ( NewRetVal , RetVal , RetIdx ++ ) ; for ( std :: pair < Argument * , Value * > ReturnPoint : Replacement . second ) { Argument * Arg = ReturnPoint . first ; Value * Val = ReturnPoint . second ; Type * EltTy = Arg -> getType ( ) -> getPointerElementType ( ) ; if ( Val -> getType ( ) != EltTy ) { Type * EffectiveEltTy = EltTy ; if ( StructType * CT = dyn_cast < StructType > ( EltTy ) ) { assert ( CT -> getNumElements ( ) == <NUM_LIT> ) ; EffectiveEltTy = CT -> getElementType ( <NUM_LIT> ) ; } if ( DL -> getTypeSizeInBits ( EffectiveEltTy ) != DL -> getTypeSizeInBits ( Val -> getType ( ) ) ) { assert ( isVec3ToVec4Shuffle ( EffectiveEltTy , Val -> getType ( ) ) ) ; Val = B . CreateShuffleVector ( Val , ArrayRef < int > { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ) ; } Val = B . CreateBitCast ( Val , EffectiveEltTy ) ; if ( EltTy != EffectiveEltTy ) Val = B . CreateInsertValue ( UndefValue :: get ( EltTy ) , Val , <NUM_LIT> ) ; } NewRetVal = B . CreateInsertValue ( NewRetVal , Val , RetIdx ++ ) ; } if ( RetVal ) RI -> setOperand ( <NUM_LIT> , NewRetVal ) ; else { B . CreateRet ( NewRetVal ) ; RI -> eraseFromParent ( ) ; } } SmallVector < Value * , <NUM_LIT> > StubCallArgs ; for ( Argument & Arg : F . args ( ) ) { if ( OutArgIndexes . count ( Arg . getArgNo ( ) ) ) { StubCallArgs . push_back ( UndefValue :: get ( Arg . getType ( ) ) ) ; } else { StubCallArgs . push_back ( & Arg ) ; } } BasicBlock * StubBB = BasicBlock :: Create ( Ctx , <STR_LIT> , & F ) ; IRBuilder < > B ( StubBB ) ; CallInst * StubCall = B .


<s> if ( ! ThisReplaceable ) continue ; for ( std :: pair < ReturnInst * , StoreInst * > Store : ReplaceableStores ) { Value * ReplVal = Store . second -> getValueOperand ( ) ; auto & ValVec = Replacements [ Store . first ] ; if ( llvm :: find_if ( ValVec , [ OutArg ] ( const std :: pair < Argument * , Value * > & Entry ) { return Entry . first == OutArg ; } ) != ValVec . end ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * OutArg << '\n' ) ; ThisReplaceable = false ; break ; } ValVec . emplace_back ( OutArg , ReplVal ) ; Store . second -> eraseFromParent ( ) ; } if ( ThisReplaceable ) { ReturnTypes . push_back ( ArgTy ) ; OutArgIndexes . insert ( OutArg -> getArgNo ( ) ) ; ++ NumOutArgumentsReplaced ; Changing = true ; } } } while ( Changing ) ; if ( Replacements . empty ( ) ) return false ; LLVMContext & Ctx = F . getParent ( ) -> getContext ( ) ; StructType * NewRetTy = StructType :: create ( Ctx , ReturnTypes , F . getName ( ) ) ; FunctionType * NewFuncTy = FunctionType :: get ( NewRetTy , F . getFunctionType ( ) -> params ( ) , F . isVarArg ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * NewRetTy << '\n' ) ; Function * NewFunc = Function :: Create ( NewFuncTy , Function :: PrivateLinkage , F . getName ( ) + <STR_LIT> ) ; F . getParent ( ) -> getFunctionList ( ) . insert ( F . getIterator ( ) , NewFunc ) ; NewFunc -> copyAttributesFrom ( & F ) ; NewFunc -> setComdat ( F . getComdat ( ) ) ; NewFunc -> stealArgumentListFrom ( F ) ; AttrBuilder RetAttrs ; RetAttrs . addAttribute ( Attribute :: SExt ) ; RetAttrs . addAttribute ( Attribute :: ZExt ) ; RetAttrs . addAttribute ( Attribute :: NoAlias ) ; NewFunc -> removeAttributes ( AttributeList :: ReturnIndex , RetAttrs ) ; NewFunc -> getBasicBlockList ( ) . splice ( NewFunc -> begin ( ) , F . getBasicBlockList ( ) ) ; for ( std :: pair < ReturnInst * , ReplacementVec > & Replacement : Replacements ) { ReturnInst * RI = Replacement . first ; IRBuilder < > B ( RI ) ; B . SetCurrentDebugLocation ( RI -> getDebugLoc ( ) ) ; int RetIdx = <NUM_LIT> ; Value * NewRetVal = UndefValue :: get ( NewRetTy ) ; Value * RetVal = RI -> getReturnValue ( ) ; if ( RetVal ) NewRetVal = B . CreateInsertValue ( NewRetVal , RetVal , RetIdx


<s> Store . second -> eraseFromParent ( ) ; } if ( ThisReplaceable ) { ReturnTypes . push_back ( ArgTy ) ; OutArgIndexes . insert ( { OutArg -> getArgNo ( ) , ArgTy } ) ; ++ NumOutArgumentsReplaced ; Changing = true ; } } } while ( Changing ) ; if ( Replacements . empty ( ) ) return false ; LLVMContext & Ctx = F . getParent ( ) -> getContext ( ) ; StructType * NewRetTy = StructType :: create ( Ctx , ReturnTypes , F . getName ( ) ) ; FunctionType * NewFuncTy = FunctionType :: get ( NewRetTy , F . getFunctionType ( ) -> params ( ) , F . isVarArg ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * NewRetTy << '\n' ) ; Function * NewFunc = Function :: Create ( NewFuncTy , Function :: PrivateLinkage , F . getName ( ) + <STR_LIT> ) ; F . getParent ( ) -> getFunctionList ( ) . insert ( F . getIterator ( ) , NewFunc ) ; NewFunc -> copyAttributesFrom ( & F ) ; NewFunc -> setComdat ( F . getComdat ( ) ) ; NewFunc -> stealArgumentListFrom ( F ) ; AttributeMask RetAttrs ; RetAttrs . addAttribute ( Attribute :: SExt ) ; RetAttrs . addAttribute ( Attribute :: ZExt ) ; RetAttrs . addAttribute ( Attribute :: NoAlias ) ; NewFunc -> removeRetAttrs ( RetAttrs ) ; NewFunc -> getBasicBlockList ( ) . splice ( NewFunc -> begin ( ) , F . getBasicBlockList ( ) ) ; for ( std :: pair < ReturnInst * , ReplacementVec > & Replacement : Replacements ) { ReturnInst * RI = Replacement . first ; IRBuilder < > B ( RI ) ; B . SetCurrentDebugLocation ( RI -> getDebugLoc ( ) ) ; int RetIdx = <NUM_LIT> ; Value * NewRetVal = UndefValue :: get ( NewRetTy ) ; Value * RetVal = RI -> getReturnValue ( ) ; if ( RetVal ) NewRetVal = B . CreateInsertValue ( NewRetVal , RetVal , RetIdx ++ ) ; for ( std :: pair < Argument * , Value * > ReturnPoint : Replacement . second ) NewRetVal = B . CreateInsertValue ( NewRetVal , ReturnPoint . second , RetIdx ++ ) ; if ( RetVal ) RI -> setOperand ( <NUM_LIT> , NewRetVal ) ; else { B . CreateRet ( NewRetVal


<s> for ( Use & U : Arg . uses ( ) ) { StoreInst * <ISA_LIT> = dyn_cast < StoreInst > ( U . getUser ( ) ) ; if ( UseCount > MaxUses ) return false ; if ( ! <ISA_LIT> ) { auto * BCI = dyn_cast < BitCastInst > ( U . getUser ( ) ) ; if ( ! BCI || ! BCI -> hasOneUse ( ) ) return false ; Type * DestEltTy = BCI -> getType


<s> unsigned ArgNumRegs = DL -> getTypeStoreSize ( ArgTy ) / <NUM_LIT> ; if ( ArgNumRegs + ReturnNumRegs > MaxNumRetRegs ) continue ; for ( ReturnInst * RI : Returns ) { BasicBlock * BB = RI -> getParent ( ) ; MemDepResult Q = MDA -> getPointerDependencyFrom ( MemoryLocation ( OutArg ) , true , BB -> end ( ) , BB , RI ) ; StoreInst * <ISA_LIT> = nullptr ; if ( Q . isDef ( ) ) <ISA_LIT> = dyn_cast < StoreInst > ( Q . getInst ( ) ) ; if ( <ISA_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * <ISA_LIT> << '\n' ) ; ReplaceableStores . emplace_back ( RI , <ISA_LIT> ) ; } else { ThisReplaceable = false ; break ; } } if ( ! ThisReplaceable ) continue ; for ( std :: pair < ReturnInst * , StoreInst * > Store : ReplaceableStores ) { Value * ReplVal = Store . second -> getValueOperand ( ) ; auto & ValVec = Replacements [ Store . first ] ; if ( llvm :: find_if ( ValVec , [ OutArg ] ( const std :: pair < Argument * , Value * > & Entry ) { return Entry . first == OutArg ; } ) != ValVec . end ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * OutArg << '\n' ) ; ThisReplaceable = false ; break ; } ValVec . emplace_back ( OutArg , ReplVal ) ; Store . second -> eraseFromParent ( ) ; } if ( ThisReplaceable ) { ReturnTypes . push_back ( ArgTy ) ; OutArgIndexes . insert ( OutArg -> getArgNo ( ) ) ; ++ NumOutArgumentsReplaced ; Changing = true ; } } } while ( Changing ) ; if ( Replacements . empty ( ) ) return false ; LLVMContext & Ctx = F . getParent ( ) -> getContext ( ) ; StructType * NewRetTy = StructType :: create ( Ctx , ReturnTypes , F . getName ( ) ) ; FunctionType * NewFuncTy = FunctionType :: get ( NewRetTy , F . getFunctionType ( ) -> params ( ) , F . isVarArg ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * NewRetTy << '\n' ) ; Function * NewFunc = Function :: Create ( NewFuncTy , Function :: PrivateLinkage , F . getName ( ) + <STR_LIT> ) ; F . getParent ( ) -> getFunctionList ( ) . insert ( F . getIterator ( ) , NewFunc ) ; NewFunc -> copyAttributesFrom ( & F ) ; NewFunc -> setComdat ( F . getComdat ( ) ) ; NewFunc -> stealArgumentListFrom ( F ) ; AttrBuilder RetAttrs


<s> Prog . MDVersionSeq . push_back ( MDRevision ) ; if ( auto MD = M . getNamedMetadata ( <STR_LIT> ) ) { for ( unsigned I = <NUM_LIT> ; I < MD -> getNumOperands ( ) ; ++ I ) { auto Node = MD -> getOperand ( I ) ; if ( Node -> getNumOperands ( ) > <NUM_LIT> ) Prog . PrintfInfo . push_back ( cast < MDString > ( Node -> getOperand ( <NUM_LIT> ) ) -> getString ( ) ) ; } } for ( auto & F : M . functions ( ) ) { if ( ! F . getMetadata ( <STR_LIT>


<s> Prog . MDVersionSeq . push_back ( MDVersion ) ; Prog . MDVersionSeq . push_back ( MDRevision ) ; IsaInfo :: Metadata & IIM = Prog . IsaInfo ; IIM . WavefrontSize = <STR_LIT> :: getWavefrontSize ( Features ) ; IIM . LocalMemorySize = <STR_LIT> :: getLocalMemorySize ( Features ) ; IIM . EUsPerCU = <STR_LIT> :: getEUsPerCU ( Features ) ; IIM . MaxWavesPerEU = <STR_LIT> :: getMaxWavesPerEU ( Features ) ; IIM . MaxFlatWorkGroupSize = <STR_LIT> :: getMaxFlatWorkGroupSize ( Features ) ; IIM . SGPRAllocGranule = <STR_LIT> :: getSGPRAllocGranule ( Features ) ; IIM . TotalNumSGPRs = <STR_LIT> :: getTotalNumSGPRs


<s> YamlIO . mapRequired ( KeyName :: MDVersion , Prog . MDVersionSeq ) ; YamlIO . mapRequired ( KeyName :: IsaInfo , Prog . IsaInfo ) ; YamlIO . mapOptional ( KeyName :: PrintfInfo


<s> auto P = Program :: Metadata :: fromYAML ( YAML ) ; auto S = P . toYAML ( ) ; llvm :: errs ( ) << <STR_LIT> << ( YAML == S ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> ; if ( YAML != S


<s> } } const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; for ( auto & Arg : F . args ( ) ) { unsigned I = Arg . getArgNo ( ) ; Type * T = Arg . getType ( ) ; auto TypeName = dyn_cast < MDString > ( F . getMetadata ( <STR_LIT> ) -> getOperand ( I ) ) -> getString ( ) ; auto BaseTypeName = cast < MDString > ( F . getMetadata ( <STR_LIT> ) -> getOperand ( I ) ) -> getString ( ) ; StringRef ArgName ; if ( auto ArgNameMD = F . getMetadata ( <STR_LIT> ) ) ArgName = cast < MDString > ( ArgNameMD -> getOperand ( I ) ) -> getString ( ) ; auto TypeQual = cast < MDString > ( F . getMetadata ( <STR_LIT> ) -> getOperand ( I ) ) -> getString ( ) ; auto AccQual = cast < MDString > ( F . getMetadata ( <STR_LIT> ) -> getOperand ( I ) ) -> getString ( ) ; KernelArg :: Kind Kind ; if ( TypeQual . find ( <STR_LIT> ) != StringRef :: npos ) Kind = KernelArg :: Pipe ; else Kind = StringSwitch < KernelArg :: Kind > ( BaseTypeName ) . Case ( <STR_LIT> , KernelArg :: Sampler ) . Case ( <STR_LIT> , KernelArg :: Queue ) . Cases ( <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , KernelArg :: Image ) . Cases ( <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , KernelArg :: Image ) . Cases ( <STR_LIT> , <STR_LIT> , KernelArg :: Image ) . Default ( isa < PointerType > ( T ) ? ( T -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ? KernelArg :: DynamicSharedPointer : KernelArg :: GlobalBuffer ) : KernelArg :: ByValue ) ; Kernel . Args . emplace_back ( getRuntimeMDForKernelArg ( DL , T , Kind , BaseTypeName , TypeName , ArgName , TypeQual , AccQual ) ) ; } if ( F . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) { auto Int64T = Type :: getInt64Ty ( F . getContext ( ) ) ; Kernel . Args . emplace_back ( getRuntimeMDForKernelArg ( DL , Int64T , KernelArg :: HiddenGlobalOffsetX ) ) ; Kernel . Args . emplace_back ( getRuntimeMDForKernelArg ( DL , Int64T , KernelArg :: HiddenGlobalOffsetY ) ) ; Kernel . Args . emplace_back ( getRuntimeMDForKernelArg ( DL , Int64T , KernelArg :: HiddenGlobalOffsetZ ) ) ; if ( F . getParent ( ) -> getNamedMetadata ( <STR_LIT>


<s> return KernelArg :: F64 ; case Type :: IntegerTyID : { bool Signed = ! TypeName . startswith ( <STR_LIT> ) ; switch ( Ty -> getIntegerBitWidth ( ) ) { case <NUM_LIT> : return Signed ? KernelArg :: I8 : KernelArg :: U8 ; case <NUM_LIT> : return Signed ? KernelArg :: I16 : KernelArg :: U16 ; case <NUM_LIT>


<s> std :: string llvm :: getRuntimeMDYAMLString ( Module & M ) { Program :: Metadata Prog ; Prog . MDVersionSeq . push_back ( MDVersion ) ; Prog . MDVersionSeq . push_back ( MDRevision ) ; if ( auto MD = M . getNamedMetadata ( <STR_LIT> ) ) { for ( unsigned I = <NUM_LIT> ; I < MD -> getNumOperands ( ) ; ++ I ) { auto Node = MD ->


<s> static std :: vector < uint32_t > getThreeInt32 ( MDNode * Node ) { assert ( Node -> getNumOperands ( ) == <NUM_LIT> ) ; std :: vector < uint32_t > V ; for ( const MDOperand & Op : Node -> operands ( ) ) { const ConstantInt * CI = mdconst :: extract < ConstantInt > ( Op


<s> for ( const MDOperand & Op : Node -> operands ( ) ) { const ConstantInt * CI = mdconst :: extract < ConstantInt > ( Op ) ; V . push_back ( CI -> getZExtValue ( )


<s> static void mapping ( IO & YamlIO , Program :: Metadata & Prog ) { YamlIO . mapRequired ( KeyName :: MDVersion , Prog . MDVersionSeq


<s> static bool CanLowerPriorityDirectlyInPredecessors ( const MachineBasicBlock & MBB , MBBInfoSet & MBBInfos ) { for ( const MachineBasicBlock * Pred : MBB . predecessors ( ) ) { if ( ! MBBInfos [ Pred ] . MayReachVMEMLoad ) continue ; for ( const MachineBasicBlock * Succ : Pred -> successors ( ) ) { if ( MBBInfos [ Succ ] . MayReachVMEMLoad ) return false ; } } return


<s> for ( const MachineBasicBlock * Pred : MBB . predecessors ( ) ) { if ( ! MBBInfos [ Pred ] . MayReachVMEMLoad ) continue ; for ( const MachineBasicBlock * Succ : Pred


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> if ( any_of ( MBB , isVMEMLoad ) ) Worklist . push_back ( & MBB ) ; } while ( ! Worklist . empty ( ) ) { const MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; MBBInfo & Info = MBBInfos [ MBB ] ; if ( ! Info . MayReachVMEMLoad ) { Info . MayReachVMEMLoad = true ; Worklist . append ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; } } MachineBasicBlock & Entry = MF . front ( ) ; if ( ! MBBInfos [ & Entry ] . MayReachVMEMLoad ) return false ; MachineBasicBlock :: iterator I = Entry . begin ( ) , E = Entry . end ( ) ; while ( I != E && ! <ISA_LIT>InstrInfo :: isVALU ( * I ) && ! I -> isTerminator ( ) ) ++ I ; Entry . insert ( I , BuildSetprioMI ( MF , HighPriority ) ) ; SmallSet < MachineBasicBlock * , <NUM_LIT> > PriorityLoweringBlocks ; for ( MachineBasicBlock & MBB : MF ) { if ( MBBInfos [ & MBB ] . MayReachVMEMLoad ) { if ( MBB . succ_empty ( ) ) PriorityLoweringBlocks . insert ( & MBB ) ; continue ; } if ( CanLowerPriorityDirectlyInPredecessors ( MBB


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , DevName ( GPU ) , Is64bit ( false ) , DumpCode ( false ) , <ISA_LIT>ALUInst ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT>Subtarget :: <ISA_LIT> ) , FP64 ( false ) , FP64Denormals ( false ) , FP32Denormals ( false ) , FastFMAF32 ( false ) , CaymanISA ( false ) , FlatAddressSpace ( false ) , FlatForGlobal ( false ) , EnableIRStructurizer ( true ) , EnablePromoteAlloca ( false ) , EnableIfCvt ( true ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , WavefrontSize ( <NUM_LIT> ) , CFALUBug ( false ) , LocalMemorySize ( <NUM_LIT> ) , EnableVGPRSpilling ( false ) , SGPRInitBug ( false ) , IsGCN ( false ) , GCN1Encoding ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , LDSBankCount ( <NUM_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , EnableHugeScratchBuffer ( false ) , FrameLowering ( nullptr ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) , TargetTriple ( TT ) { initializeSubtargetDependencies ( TT , GPU , FS ) ; const unsigned MaxStackAlign = <NUM_LIT> * <NUM_LIT> ; if ( getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) { InstrInfo . reset ( new <ISA_LIT>InstrInfo ( * this ) ) ; TLInfo . reset ( new <ISA_LIT>TargetLowering ( TM , * this ) ) ; FrameLowering . reset ( new <ISA_LIT>FrameLowering ( TargetFrameLowering :: StackGrowsUp , MaxStackAlign , <NUM_LIT> ) ) ; } else { InstrInfo . reset ( new <ISA_LIT>InstrInfo ( * this ) ) ; TLInfo . reset ( new <ISA_LIT>TargetLowering (


<s> bool dumpCode ( ) const { return DumpCode


<s> bool enableSubRegLiveness ( ) const override { return true


<s> unsigned getExplicitKernelArgOffset ( )


<s> unsigned getExplicitKernelArgOffset ( ) const { return isAmdHsaOS ( ) ? <NUM_LIT>


<s> const <ISA_LIT>FrameLowering * getFrameLowering ( )


<s> const <ISA_LIT>InstrInfo * getInstrInfo


<s> const <ISA_LIT>InstrInfo * getInstrInfo ( ) const


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return <NUM_LIT> ; llvm_unreachable (


<s> unsigned getMaxWavesPerCU ( ) const { if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return <NUM_LIT> ; llvm_unreachable ( <STR_LIT>


<s> assert ( getGeneration ( ) <= NORTHERN_ISLANDS ) ; switch ( getWavefrontSize ( ) ) { case <NUM_LIT> : return <NUM_LIT>


<s> unsigned getWavefrontSize ( ) const { return WavefrontSize


<s> if ( Size == <NUM_LIT> ) return ( getGeneration ( ) >= EVERGREEN ) ; if ( Size == <NUM_LIT> ) return ( getGeneration (


<s> if ( Size == <NUM_LIT> ) return ( getGeneration ( ) >=


<s> bool hasBFE ( ) const { return ( getGeneration ( ) >= EVERGREEN


<s> bool hasBFI ( ) const { return ( getGeneration ( )


<s> bool hasCaymanISA ( ) const { return CaymanISA


<s> assert ( getGeneration ( ) <= NORTHERN_ISLANDS ) ; return


SOUTHERN_ISLANDS || hasCaymanISA ( ) ) ;


<s> bool hasMulI24 ( ) const { return ( getGeneration ( ) >= SOUTHERN_ISLANDS || hasCaymanISA


<s> if ( GPU == <STR_LIT> && TT . getArch ( ) == Triple :: amdgcn ) GPU = <STR_LIT> ; ParseSubtargetFeatures ( GPU , FullFS ) ; if ( getGeneration ( )

<= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) {


<s> bool is64bit ( ) const { return Is64bit


<s> bool isIfCvtEnabled ( ) const { return EnableIfCvt


<s> bool IsIRStructurizerEnabled ( ) const { return EnableIRStructurizer


<s> bool <ISA_LIT>Subtarget :: isVGPRSpillingEnabled ( const <ISA_LIT>MachineFunctionInfo * MFI ) const { return MFI -> getShaderType ( ) ==

ShaderType :: COMPUTE || EnableVGPRSpilling ;


<s> bool r600ALUEncoding ( ) const { return <ISA_LIT>ALUInst


<s> if ( getGeneration ( ) >= SOUTHERN_ISLANDS ) return static_cast < const <ISA_LIT>Subtarget * > ( this ) -> getFrameLowering


<s> if ( getGeneration ( ) >= SOUTHERN_ISLANDS ) return static_cast < const

<ISA_LIT>Subtarget * > ( this ) -> getFrameLowering ( ) ;


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS )


<s> if ( getGeneration ( ) >= SOUTHERN_ISLANDS ) return static_cast < const <ISA_LIT>Subtarget * > ( this


<s> return <NUM_LIT> * <NUM_LIT> / getWavefrontSize


<s> inline const <ISA_LIT>TargetLowering * <ISA_LIT>Subtarget :: getTargetLowering ( ) const { if ( getGeneration ( ) >= SOUTHERN_ISLANDS ) return static_cast < const <ISA_LIT>Subtarget * > ( this ) -> getTargetLowering


<s> TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , FP32Denormals ( false ) , FPExceptions ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) ,

HasTrigReducedRange ( false ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSize ( <NUM_LIT> ) {


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , TargetTriple ( TT ) , Gen ( SOUTHERN_ISLANDS ) , IsaVersion ( ISAVersion0_0_0 ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP64FP16Denormals ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , VIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasR128A16 ( false ) , HasDLInsts ( false ) , D16PreservesUnusedBits ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) { CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering ( ) ) ) ; Legalizer . reset ( new <ISA_LIT>LegalizerInfo ( * this , TM ) ) ; RegBankInfo . reset ( new <ISA_LIT>RegisterBankInfo ( *


<s> unsigned getAddressableNumSGPRs ( ) const { return <STR_LIT> :: getAddressableNumSGPRs ( this


<s> return <STR_LIT> :: getAddressableNumVGPRs


<s> return <STR_LIT> :: getMaxFlatWorkGroupSize ( this


<s> return <STR_LIT> :: getMaxFlatWorkGroupSize


<s> unsigned getMaxNumSGPRs ( unsigned WavesPerEU , bool Addressable ) const { return <STR_LIT> :: getMaxNumSGPRs ( this , WavesPerEU ,


<s> unsigned getMaxNumVGPRs ( unsigned WavesPerEU


<s> return <STR_LIT> :: getMaxWavesPerCU (


<s> return <STR_LIT> :: getMaxWavesPerCU ( this ,


<s> return <STR_LIT> :: getMaxWavesPerEU ( this


<s> return <STR_LIT> :: getMaxWorkGroupsPerCU (


<s> return <STR_LIT> :: getMinFlatWorkGroupSize


<s> unsigned getMinFlatWorkGroupSize ( ) const override { return <STR_LIT> :: getMinFlatWorkGroupSize ( this


<s> return <STR_LIT> :: getMinNumVGPRs ( this , WavesPerEU


<s> return <STR_LIT> :: getMinNumVGPRs (


<s> return <STR_LIT> :: getMinWavesPerEU ( this


<s> return <STR_LIT> :: getSGPRAllocGranule


<s> unsigned getSGPREncodingGranule ( ) const { return <STR_LIT> :: getSGPREncodingGranule


<s> return <STR_LIT> :: getTotalNumVGPRs (


<s> unsigned getTotalNumVGPRs ( ) const { return <STR_LIT> :: getTotalNumVGPRs (


<s> unsigned getWavesPerWorkGroup ( unsigned FlatWorkGroupSize


<s> return <STR_LIT> :: getWavesPerWorkGroup ( this ,


<s> if ( Dep . getKind ( ) != SDep :: Kind :: Data || ! Dep . getReg ( ) || ! Def -> isInstr ( ) || ! Use -> isInstr ( ) ) return ; MachineInstr * DefI = Def -> getInstr ( ) ; MachineInstr * UseI = Use -> getInstr ( ) ; if ( DefI -> isBundle ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getRegisterInfo ( ) ; auto Reg = Dep . getReg ( ) ; MachineBasicBlock :: const_instr_iterator I ( DefI -> getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator E ( DefI -> getParent ( ) -> instr_end ( ) ) ; unsigned Lat = <NUM_LIT> ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { if ( I -> modifiesRegister ( Reg , TRI ) ) Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * I ) ; else if ( Lat ) -- Lat ; } Dep . setLatency ( Lat ) ; } else if ( UseI -> isBundle ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getRegisterInfo ( ) ; auto Reg = Dep . getReg ( ) ; MachineBasicBlock :: const_instr_iterator I ( UseI -> getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator E ( UseI -> getParent ( ) -> instr_end ( ) ) ; unsigned Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * DefI ) ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) && Lat ; ++ I ) { if ( I -> readsRegister ( Reg ,


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT ) : TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , HasMadMacF32Insts ( false ) , HasDsSrc2Insts ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24

( true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange ( false ) , MaxWavesPerEU ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSizeLog2 ( <NUM_LIT> ) {


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT ) : TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , HasMadMacF32Insts ( false ) , HasDsSrc2Insts ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 (

true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange ( false ) , MaxWavesPerEU ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSizeLog2 ( <NUM_LIT> ) {


<s> bool GCNSubtarget :: enableFlatScratch


<s> unsigned GCNSubtarget :: getConstantBusLimit ( unsigned Opcode ) const { if ( getGeneration ( ) < GFX10 ) return <NUM_LIT> ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; } return


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) return <NUM_LIT> ; if ( MFI . hasFlatScratchInit ( ) ) { if ( getGeneration ( ) >= <ISA_LIT>Subtarget

:: VOLCANIC_ISLANDS ) return <NUM_LIT> ;


<s> bool RequestedFlatWorkGroupSize = F . hasFnAttribute ( <STR_LIT> ) ; std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default , true ) ; if ( Requested . second && Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinWavesPerEU ( ) || Requested . second > getMaxWavesPerEU ( ) ) return Default ; if ( RequestedFlatWorkGroupSize && Requested . first <

MinImpliedByFlatWorkGroupSize ) return Default ;


<s> std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default , true ) ; if ( Requested . second && Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinWavesPerEU ( ) || Requested . second > getMaxWavesPerEU ( ) ) return Default ; if ( RequestedFlatWorkGroupSize && Requested . first < MinImpliedByFlatWorkGroupSize ) return Default ; return Requested


<s> bool GCNSubtarget :: hasMadF16 ( ) const { return InstrInfo . pseudoToMCOpcode ( <STR_LIT> ) != -


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU ,

FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> unsigned <ISA_LIT>Subtarget :: getImplicitArgNumBytes ( const Function & F ) const { assert ( <STR_LIT> ( F . getCallingConv ( ) ) ) ; if ( F . hasFnAttribute ( <STR_LIT>


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) return <NUM_LIT> ; if ( isMesaKernel ( F


<s> unsigned ExplicitOffset = getExplicitKernelArgOffset ( F ) ; uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes ; unsigned ImplicitBytes = getImplicitArgNumBytes ( F ) ; if ( ImplicitBytes != <NUM_LIT> ) { const Align Alignment = getAlignmentForImplicitArgPtr ( ) ; TotalSize = alignTo ( ExplicitArgBytes , Alignment ) + ImplicitBytes ; MaxAlign = std :: max ( MaxAlign ,


<s> TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , FP32Denormals ( false ) , FPExceptions ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy

( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange ( false ) , MaxWavesPerEU ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSize ( <NUM_LIT> ) {


<s> continue ; } MachineInstr & MI1 = * SUa -> getInstr ( ) ; if ( ( TII -> isVMEM ( MI1 ) && TII -> isVMEM ( MI2 ) ) || ( TII -> isFLAT ( MI1 ) && TII -> isFLAT ( MI2 ) ) || ( TII -> isSMRD ( MI1 ) && TII -> isSMRD ( MI2 ) ) || ( TII -> isDS ( MI1 ) && TII -> isDS ( MI2 ) ) ) { SU . addPredBarrier ( SUa ) ; for ( const SDep & <ISA_LIT> : SU . Preds ) { if ( <ISA_LIT> . getSUnit ( ) != SUa ) SUa -> addPred ( SDep ( <ISA_LIT> . getSUnit ( ) , SDep :: Artificial ) ) ; } if ( & SU != & DAG -> ExitSU ) { for ( const SDep & <ISA_LIT> : SUa -> Succs ) { if ( <ISA_LIT> . getSUnit ( ) != & SU ) <ISA_LIT> . getSUnit ( ) -> addPred ( SDep ( & SU , SDep :: Artificial ) ) ; } } } SUa =


<s> unsigned Occupancy = std :: min ( getMaxWavesPerEU ( ) , getOccupancyWithLocalMemSize ( LDSSize , MF . getFunction ( ) ) ) ; if ( NumSGPRs ) Occupancy = std :: min ( Occupancy , getOccupancyWithNumSGPRs ( NumSGPRs ) ) ; if ( NumVGPRs ) Occupancy = std :: min ( Occupancy ,

getOccupancyWithNumVGPRs ( NumVGPRs ) ) ;


<s> bool d16PreservesUnusedBits ( ) const


<s> return getRegisterInfo ( ) -> getBoolRC


<s> return countLeadingZeros ( MaxWaveScratchSize )


<s> unsigned MaxWaves = getMaxWavesPerEU ( ) ; unsigned Limit = getLocalMemorySize ( ) * MaxWaves / WorkGroupsPerCu ; unsigned NumWaves = Limit / ( Bytes ? Bytes : <NUM_LIT> ) ; NumWaves = std :: min ( NumWaves , MaxWaves ) ; NumWaves = std :: max ( NumWaves , <NUM_LIT>


<s> void GCNSubtarget :: getPostRAMutations ( std :: vector < std :: unique_ptr < ScheduleDAGMutation >> & Mutations ) const { Mutations . push_back ( std :: make_unique < MemOpClusterMutation > ( & InstrInfo ) ) ; Mutations . push_back ( std :: make_unique < FillMFMAShadowMutation > (


<s> if ( Requested . second && Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinWavesPerEU ( ) || Requested . first > getMaxWavesPerEU ( ) ) return Default ; if ( Requested . second > getMaxWavesPerEU ( ) ) return Default ; if ( RequestedFlatWorkGroupSize && Requested . first < MinImpliedByFlatWorkGroupSize ) return Default ; return Requested


<s> bool hasCodeObjectV3 ( ) const { return isAmdHsaOS ( ) ? CodeObjectV3


<s> bool hasCodeObjectV3 ( ) const { return isAmdHsaOS ( ) ?


<s> bool hasDot1Insts ( ) const { return HasDot1Insts


<s> bool hasDwordx3LoadStores ( ) const { return


<s> bool hasFlatSegmentOffsetBug ( ) const { return


<s> bool hasNoDataDepHazard ( ) const { return HasNoDataDepHazard


<s> bool hasNoSdstCMPX ( ) const { return HasNoSdstCMPX


<s> bool hasNSAtoVMEMBug ( ) const { return HasNSAtoVMEMBug


<s> return getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS && getGeneration


<s> bool hasReadM0SendMsgHazard ( ) const { return getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS

&& getGeneration ( ) <= <ISA_LIT>Subtarget :: GFX9 ;


<s> bool hasRegisterBanking ( ) const { return HasRegisterBanking


<s> bool hasVMEMtoScalarWriteHazard ( ) const { return


<s> assert ( ! hasFP64 ( ) || ( getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) ) ; if ( ! hasAddr64 ( ) && ! FS . contains ( <STR_LIT> ) ) { FlatForGlobal = true ; } if ( MaxPrivateElementSize == <NUM_LIT> ) MaxPrivateElementSize = <NUM_LIT> ; if ( LDSBankCount == <NUM_LIT> ) LDSBankCount = <NUM_LIT> ; if ( TT . getArch ( ) == Triple :: amdgcn ) { if ( LocalMemorySize == <NUM_LIT> ) LocalMemorySize = <NUM_LIT> ; if ( ! HasMovrel && ! HasVGPRIndexMode ) HasMovrel = true ; } if ( WavefrontSize == <NUM_LIT> ) WavefrontSize = <NUM_LIT> ; HasFminFmaxLegacy = getGeneration ( ) < <ISA_LIT>Subtarget


<s> bool isSRAMECCEnabled ( ) const { return EnableSRAMECC


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) ,

HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> for ( ++ I ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { if ( I -> modifiesRegister ( Reg , TRI ) ) Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * I ) ; else if ( Lat ) -- Lat ; } Dep . setLatency ( Lat ) ; } else if ( DstI -> isBundle ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getRegisterInfo ( ) ; auto Reg = Dep . getReg ( ) ; MachineBasicBlock :: const_instr_iterator I ( DstI -> getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator E ( DstI -> getParent ( ) -> instr_end ( ) ) ; unsigned Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * SrcI ) ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) && Lat ; ++ I ) { if ( I -> readsRegister ( Reg , TRI ) ) break ; -- Lat


<s> <ISA_LIT>DwarfFlavour get<ISA_LIT>DwarfFlavour ( ) const { return getWavefrontSize ( ) == <NUM_LIT> ?


<s> MaxAlign = Align ( <NUM_LIT> ) ; for ( const Argument & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; const Align Alignment ( DL . getABITypeAlignment (


<s> const InlineAsmLowering * getInlineAsmLowering ( )


<s> InstructionSelector * getInstructionSelector ( ) const


<s> InstructionSelector * getInstructionSelector ( ) const override { return InstSelector . get


<s> unsigned getMaxPrivateElementSize ( bool ForBufferRSrc = false )


<s> return getGeneration ( ) <= SEA_ISLANDS


<s> Align getStackAlignment ( ) const { return Align ( <NUM_LIT>


<s> bool hasD16Images ( ) const { return getGeneration ( ) >=


<s> bool hasDot4Insts ( ) const { return HasDot4Insts


<s> bool hasFlat ( ) const { return ( getGeneration ( ) > <ISA_LIT>Subtarget ::


<s> bool hasFlatScratchSTMode ( ) const { return hasFlatScratchInsts ( )


<s> bool hasFractBug ( ) const { return getGeneration ( ) == SOUTHERN_ISLANDS


<s> bool hasFractBug ( ) const { return getGeneration (


<s> bool hasGFX10_3Insts ( ) const { return


<s> bool hasGWSSemaReleaseAll ( ) const { return


<s> bool hasMAIInsts ( ) const { return HasMAIInsts


<s> bool hasOffset3fBug ( ) const { return HasOffset3fBug


<s> bool hasPkFmacF16Inst ( ) const { return


<s> bool hasSMRDReadVALUDefHazard ( ) const { return getGeneration ( ) ==


<s> bool hasUnalignedBufferAccessEnabled ( )


<s> bool hasUnalignedDSAccess ( ) const { return UnalignedDSAccess


<s> bool hasUnalignedDSAccess ( ) const { return


<s> return UnalignedDSAccess && UnalignedAccessMode


<s> bool hasUnalignedDSAccessEnabled ( ) const


<s> bool hasUsableDSOffset ( ) const { return getGeneration ( )


<s> bool isWave32 ( ) const { return getWavefrontSize ( ) ==


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , TargetTriple ( TT ) , Gen ( SOUTHERN_ISLANDS ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP64FP16Denormals ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , EnablePRTStrictNull ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , VIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasR128A16 ( false ) , HasDLInsts ( false ) , HasDot1Insts ( false ) , HasDot2Insts ( false ) , EnableSRAMECC ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) { CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering (


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64Denormals ( false ) , FPExceptions ( false ) , FlatForGlobal ( false ) , UnalignedBufferAccess ( false ) , EnableXNACK ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , DumpCode ( false ) , FP64 ( false ) , IsGCN ( false ) , GCN1Encoding ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , FlatAddressSpace ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) { initializeSubtargetDependencies ( TT , GPU ,


<s> case SEA_ISLANDS : return <NUM_LIT> ; default : llvm_unreachable (


<s> return <NUM_LIT> ; default : llvm_unreachable ( <STR_LIT> )


<s> return <STR_LIT> ( getFeatureBits ( )


<s> case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return


<s> if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return


<s> bool hasFP16Denormals ( const Function & F ) const { return FP64FP16Denormals


<s> bool hasFP32Denormals ( const Function &


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64FP16Denormals ( false ) , FPExceptions ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , DumpCode ( false ) , FP64 ( false ) , IsGCN ( false ) , GCN1Encoding ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , HasVOP3PInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasInv2PiInlineImm ( false ) , HasSDWA ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAClampVOPC ( false ) , HasDPP ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) { AS = <STR_LIT> ( TT ) ; initializeSubtargetDependencies ( TT , GPU


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64FP16Denormals ( false ) , FPExceptions ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , DumpCode ( false ) , FP64 ( false ) , IsGCN ( false ) , GCN1Encoding ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , HasVOP3PInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasInv2PiInlineImm ( false ) , HasSDWA ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAClampVOPC ( false ) , HasDPP ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) { AS = <STR_LIT> ( TT )


<s> return debuggerInsertNops ( ) && debuggerReserveRegs ( ) &&


<s> unsigned getAddressableNumSGPRs ( ) const { return <STR_LIT> :: getAddressableNumSGPRs (


<s> unsigned getExplicitKernelArgOffset ( const MachineFunction & MF ) const { return isAmdCodeObjectV2 ( MF ) ?


<s> unsigned getExplicitKernelArgOffset ( const MachineFunction & MF ) const { return isAmdCodeObjectV2 ( MF ) ? <NUM_LIT>


<s> Default . second = <STR_LIT> ( F , <STR_LIT> , Default . second ) ; Default . first = std :: min ( Default . first , Default . second ) ; std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default ) ; if ( Requested . first > Requested .


<s> std :: pair < unsigned , unsigned > <ISA_LIT>Subtarget :: getFlatWorkGroupSizes ( const Function & F ) const { std :: pair < unsigned , unsigned > Default = <STR_LIT> ( F . getCallingConv ( ) ) ? std :: pair < unsigned , unsigned > ( getWavefrontSize ( ) * <NUM_LIT> , getWavefrontSize ( ) * <NUM_LIT> ) : std :: pair < unsigned , unsigned > ( <NUM_LIT> , getWavefrontSize ( ) ) ; Default . second = <STR_LIT> ( F , <STR_LIT> , Default . second ) ; Default . first = std :: min ( Default . first , Default . second ) ; std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default ) ; if ( Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinFlatWorkGroupSize ( ) ) return Default ; if ( Requested . second > getMaxFlatWorkGroupSize ( ) ) return Default ; return Requested


<s> if ( isMesaKernel ( MF ) ) return <NUM_LIT> ; if ( isAmdHsaOS ( ) && isOpenCLEnv ( ) ) return <NUM_LIT> ; return <NUM_LIT>


<s> if ( isMesaKernel ( MF ) ) return <NUM_LIT> ; if ( isAmdHsaOS ( ) && isOpenCLEnv ( ) ) return <NUM_LIT> ; return


<s> const InstructionSelector * getInstructionSelector ( ) const override { assert ( GISel && <STR_LIT> ) ; return GISel -> getInstructionSelector ( )


<s> const InstructionSelector * getInstructionSelector ( ) const override { assert ( GISel && <STR_LIT> ) ; return GISel -> getInstructionSelector


<s> if ( ImplicitBytes == <NUM_LIT> ) return ExplicitArgBytes ; unsigned Alignment = getAlignmentForImplicitArgPtr ( )


<s> unsigned Alignment = getAlignmentForImplicitArgPtr ( ) ; return alignTo ( ExplicitArgBytes , Alignment


<s> const LegalizerInfo * getLegalizerInfo ( ) const override { assert ( GISel && <STR_LIT>


<s> assert ( GISel && <STR_LIT> ) ; return GISel -> getLegalizerInfo (


<s> unsigned getMaxFlatWorkGroupSize ( ) const { return <STR_LIT> :: getMaxFlatWorkGroupSize ( getFeatureBits


<s> return <STR_LIT> :: getMaxNumSGPRs ( getFeatureBits ( ) , WavesPerEU , Addressable


<s> unsigned getMaxNumVGPRs ( unsigned WavesPerEU ) const


<s> unsigned getMaxWavesPerCU ( unsigned FlatWorkGroupSize ) const { return <STR_LIT> :: getMaxWavesPerCU ( getFeatureBits


<s> return <STR_LIT> :: getMaxWavesPerEU ( getFeatureBits


<s> return <STR_LIT> :: getMaxWorkGroupsPerCU ( getFeatureBits (


<s> return <STR_LIT> :: getMinFlatWorkGroupSize ( getFeatureBits


<s> return <STR_LIT> :: getMinNumSGPRs ( getFeatureBits ( )


<s> return <STR_LIT> :: getMinWavesPerEU ( getFeatureBits (


<s> return <STR_LIT> :: getMinWavesPerEU ( getFeatureBits


<s> unsigned getOccupancyWithLocalMemSize ( const MachineFunction & MF ) const { const auto * MFI = MF . getInfo


<s> const auto * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return getOccupancyWithLocalMemSize ( MFI -> getLDSSize ( ) , * MF . getFunction (


<s> unsigned <ISA_LIT>Subtarget :: getOccupancyWithNumSGPRs ( unsigned SGPRs ) const { if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return


<s> unsigned <ISA_LIT>Subtarget :: getOccupancyWithNumVGPRs ( unsigned VGPRs ) const { if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> )


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; if ( getGeneration ( ) == <ISA_LIT>Subtarget ::


<s> unsigned getReservedNumVGPRs ( const MachineFunction & MF ) const { return debuggerReserveRegs ( ) ?


<s> return <STR_LIT> :: getSGPRAllocGranule (


<s> return <STR_LIT> :: getSGPREncodingGranule (


<s> unsigned getSGPREncodingGranule ( ) const { return <STR_LIT> :: getSGPREncodingGranule ( getFeatureBits (


<s> unsigned getTotalNumSGPRs ( ) const { return <STR_LIT> :: getTotalNumSGPRs ( getFeatureBits ( )


<s> return <STR_LIT> :: getTotalNumSGPRs ( getFeatureBits ( )


<s> return <STR_LIT> :: getTotalNumVGPRs (


<s> return <STR_LIT> :: getVGPRAllocGranule ( getFeatureBits


<s> return <STR_LIT> :: getVGPREncodingGranule (


<s> return <STR_LIT> :: getVGPREncodingGranule ( getFeatureBits ( )


<s> unsigned getWavesPerWorkGroup ( unsigned FlatWorkGroupSize ) const { return <STR_LIT> :: getWavesPerWorkGroup ( getFeatureBits


<s> return <STR_LIT> :: getWavesPerWorkGroup ( getFeatureBits (


<s> bool hasAddr64 ( ) const { return ( getGeneration ( ) < VOLCANIC_ISLANDS


<s> return getGeneration ( ) < <ISA_LIT>Subtarget ::


<s> bool hasMed3_16 ( ) const { return getGeneration ( ) >=


<s> bool hasMed3_16 ( ) const { return getGeneration (


<s> bool hasSDWAClampVOPC ( ) const { return


<s> return isAmdHsaOS ( ) || isMesaKernel ( MF


<s> return isMesa3DOS ( ) && <STR_LIT> ( MF .

getFunction ( ) -> getCallingConv ( ) ) ;


<s> bool isMesaKernel ( const MachineFunction & MF ) const { return isMesa3DOS ( ) && ! <STR_LIT> ( MF . getFunction ( ) ->


<s> return isMesa3DOS ( ) && ! <STR_LIT> ( MF .


<s> bool isOpenCLEnv ( ) const { return TargetTriple . getEnvironment ( ) == Triple :: OpenCL || TargetTriple .


<s> bool isOpenCLEnv ( ) const { return TargetTriple . getEnvironment ( ) == Triple :: OpenCL || TargetTriple . getEnvironmentName (


<s> return EnableVGPRSpilling || ! <STR_LIT> ( F . getCallingConv (


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>Subtarget ( TT , GPU , FS , TM ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , TLInfo


<s> void setGISelAccessor ( GISelAccessor & GISel


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>Subtarget ( TT , GPU , FS , TM ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp ,

getStackAlignment ( ) , <NUM_LIT> ) , TLInfo ( TM , * this ) {


<s> std :: pair < unsigned , unsigned > WavesPerEU = MFI . getWavesPerEU ( ) ; unsigned MaxNumSGPRs = getMaxNumSGPRs ( WavesPerEU . first , false ) ; unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs ( WavesPerEU . first , true ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) { unsigned Requested = <STR_LIT> ( F , <STR_LIT> , MaxNumSGPRs ) ; if ( Requested && ( Requested <= getReservedNumSGPRs ( MF ) ) ) Requested = <NUM_LIT> ; unsigned InputNumSGPRs = MFI . getNumPreloadedSGPRs ( ) ; if ( Requested && Requested < InputNumSGPRs ) Requested = InputNumSGPRs ; if ( Requested && Requested > getMaxNumSGPRs ( WavesPerEU . first , false ) ) Requested = <NUM_LIT> ; if ( WavesPerEU . second && Requested && Requested < getMinNumSGPRs ( WavesPerEU . second ) ) Requested = <NUM_LIT> ; if ( Requested ) MaxNumSGPRs


<s> if ( Requested && Requested <= getReservedNumVGPRs ( MF ) ) Requested = <NUM_LIT> ; if ( Requested && Requested > getMaxNumVGPRs ( WavesPerEU . first ) ) Requested = <NUM_LIT> ; if ( WavesPerEU . second && Requested && Requested < getMinNumVGPRs ( WavesPerEU . second )


<s> Mutations . push_back ( llvm :: make_unique < MemOpClusterMutation > (


<s> void <ISA_LIT>Subtarget :: getPostRAMutations ( std :: vector < std :: unique_ptr < ScheduleDAGMutation >> & Mutations ) const { Mutations . push_back ( llvm :: make_unique <

MemOpClusterMutation > ( & InstrInfo ) ) ;


<s> unsigned getVGPRAllocGranule ( ) const { return <STR_LIT> :: getVGPRAllocGranule ( getFeatureBits ( )


<s> return TargetTriple . getEnvironment ( ) ==


<s> unsigned getOccupancyWithLocalMemSize ( const MachineFunction & MF ) const { const auto * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return getOccupancyWithLocalMemSize ( MFI -> getLDSSize ( ) , MF . getFunction


<s> return getOccupancyWithLocalMemSize ( MFI -> getLDSSize ( ) , MF . getFunction (


<s> bool isMesaGfxShader ( const MachineFunction & MF ) const { return isMesa3DOS ( ) && <STR_LIT> ( MF . getFunction (


<s> return isMesa3DOS ( ) && ! <STR_LIT> ( MF . getFunction (


<s> TargetTriple ( TT ) , SubtargetFeatureBits ( FeatureBits ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , FP32Denormals ( false ) , FPExceptions ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 (

true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSize ( <NUM_LIT> ) {


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT , getFeatureBits ( ) ) , TargetTriple ( TT ) , Gen ( SOUTHERN_ISLANDS ) , IsaVersion ( ISAVersion0_0_0 ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP64FP16Denormals ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableVGPRSpilling ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , VIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasDLInsts ( false ) , D16PreservesUnusedBits ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) { AS = <STR_LIT> ( TT ) ; CallLoweringInfo . reset ( new <ISA_LIT>CallLowering (


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64FP16Denormals ( false ) , FPExceptions ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , DumpCode ( false ) , FP64 ( false ) , FMA ( false ) , MIMG_R128 ( false ) , IsGCN ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , HasIntClamp ( false ) , HasVOP3PInsts ( false ) , HasMadMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasInv2PiInlineImm ( false ) , HasSDWA ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) { AS = <STR_LIT> ( TT ) ; initializeSubtargetDependencies ( TT ,


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const GCNTargetMachine & TM ) : <ISA_LIT>Subtarget ( TT , GPU , FS , TM ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , TLInfo ( TM , * this ) { CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering ( ) ) ) ; Legalizer . reset ( new <ISA_LIT>LegalizerInfo ( * this , TM ) ) ; RegBankInfo . reset ( new <ISA_LIT>RegisterBankInfo ( * getRegisterInfo ( ) ) ) ; InstSelector . reset ( new <ISA_LIT>InstructionSelector ( * this , * static_cast < <ISA_LIT>RegisterBankInfo * > (


<s> TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , FPExceptions ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasInv2PiInlineImm

( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange ( false ) , MaxWavesPerEU ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSize ( <NUM_LIT> ) {


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT ) : TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , FPExceptions ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange (

false ) , MaxWavesPerEU ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSize ( <NUM_LIT> ) {