<s> if ( ! ThisReplaceable ) continue ; for ( std :: pair < ReturnInst * , StoreInst * > Store : ReplaceableStores ) { Value * ReplVal = Store . second -> getValueOperand ( ) ; auto & ValVec = Replacements [ Store . first ] ; if ( llvm :: find_if ( ValVec , [ OutArg ] ( const std :: pair < Argument * , Value * > & Entry ) { return Entry . first == OutArg ; } ) != ValVec . end ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * OutArg << '\n' ) ; ThisReplaceable = false ; break ; } ValVec . emplace_back ( OutArg , ReplVal ) ; Store . second -> eraseFromParent ( ) ; } if ( ThisReplaceable ) { ReturnTypes . push_back ( ArgTy ) ; OutArgIndexes . insert ( OutArg -> getArgNo ( ) ) ; ++ NumOutArgumentsReplaced ; Changing = true ; } } } while ( Changing ) ; if ( Replacements . empty ( ) ) return false ; LLVMContext & Ctx = F . getParent ( ) -> getContext ( ) ; StructType * NewRetTy = StructType :: create ( Ctx , ReturnTypes , F . getName ( ) ) ; FunctionType * NewFuncTy = FunctionType :: get ( NewRetTy , F . getFunctionType ( ) -> params ( ) , F . isVarArg ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * NewRetTy << '\n' ) ; Function * NewFunc = Function :: Create ( NewFuncTy , Function :: PrivateLinkage , F . getName ( ) + <STR_LIT> ) ; F . getParent ( ) -> getFunctionList ( ) . insert ( F . getIterator ( ) , NewFunc ) ; NewFunc -> copyAttributesFrom ( & F ) ; NewFunc -> setComdat ( F . getComdat ( ) ) ; NewFunc -> stealArgumentListFrom ( F ) ; AttrBuilder RetAttrs ; RetAttrs . addAttribute ( Attribute :: SExt ) ; RetAttrs . addAttribute ( Attribute :: ZExt ) ; RetAttrs . addAttribute ( Attribute :: NoAlias ) ; NewFunc -> removeAttributes ( AttributeList :: ReturnIndex , RetAttrs ) ; NewFunc -> getBasicBlockList ( ) . splice ( NewFunc -> begin ( ) , F . getBasicBlockList ( ) ) ; for ( std :: pair < ReturnInst * , ReplacementVec > & Replacement : Replacements ) { ReturnInst * RI = Replacement . first ; IRBuilder < > B ( RI ) ; B . SetCurrentDebugLocation ( RI -> getDebugLoc ( ) ) ; int RetIdx = <NUM_LIT> ; Value * NewRetVal = UndefValue :: get ( NewRetTy ) ; Value * RetVal = RI -> getReturnValue ( ) ;

if ( RetVal ) NewRetVal = B . CreateInsertValue ( NewRetVal , RetVal , RetIdx ++ ) ;


<s> Store . second -> eraseFromParent ( ) ; } if ( ThisReplaceable ) { ReturnTypes . push_back ( ArgTy ) ; OutArgIndexes . insert ( { OutArg -> getArgNo ( ) , ArgTy } ) ; ++ NumOutArgumentsReplaced ; Changing = true ; } } } while ( Changing ) ; if ( Replacements . empty ( ) ) return false ; LLVMContext & Ctx = F . getParent ( ) -> getContext ( ) ; StructType * NewRetTy = StructType :: create ( Ctx , ReturnTypes , F . getName ( ) ) ; FunctionType * NewFuncTy = FunctionType :: get ( NewRetTy , F . getFunctionType ( ) -> params ( ) , F . isVarArg ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * NewRetTy << '\n' ) ; Function * NewFunc = Function :: Create ( NewFuncTy , Function :: PrivateLinkage , F . getName ( ) + <STR_LIT> ) ; F . getParent ( ) -> getFunctionList ( ) . insert ( F . getIterator ( ) , NewFunc ) ; NewFunc -> copyAttributesFrom ( & F ) ; NewFunc -> setComdat ( F . getComdat ( ) ) ; NewFunc -> stealArgumentListFrom ( F ) ; AttributeMask RetAttrs ; RetAttrs . addAttribute ( Attribute :: SExt ) ; RetAttrs . addAttribute ( Attribute :: ZExt ) ; RetAttrs . addAttribute ( Attribute :: NoAlias ) ; NewFunc -> removeRetAttrs ( RetAttrs ) ; NewFunc -> getBasicBlockList ( ) . splice ( NewFunc -> begin ( ) , F . getBasicBlockList ( ) ) ; for ( std :: pair < ReturnInst * , ReplacementVec > & Replacement : Replacements ) { ReturnInst * RI = Replacement . first ; IRBuilder < > B ( RI ) ; B . SetCurrentDebugLocation ( RI -> getDebugLoc ( ) ) ; int RetIdx = <NUM_LIT> ; Value * NewRetVal = UndefValue :: get ( NewRetTy ) ; Value * RetVal = RI -> getReturnValue ( ) ; if ( RetVal ) NewRetVal = B . CreateInsertValue ( NewRetVal , RetVal , RetIdx ++ ) ; for ( std :: pair < Argument * , Value * > ReturnPoint : Replacement . second ) NewRetVal = B . CreateInsertValue ( NewRetVal , ReturnPoint . second , RetIdx ++ ) ; if ( RetVal ) RI -> setOperand ( <NUM_LIT> , NewRetVal ) ; else {


<s> for ( Use & U : Arg . uses ( ) ) { StoreInst * <ISA_LIT> = dyn_cast < StoreInst > ( U . getUser ( ) ) ; if ( UseCount > MaxUses ) return false ; if ( ! <ISA_LIT> ) { auto * BCI = dyn_cast < BitCastInst > ( U . getUser ( ) ) ; if ( ! BCI || ! BCI -> hasOneUse ( ) ) return false ;

Type * DestEltTy = BCI -> getType ( ) -> getPointerElementType ( ) ;


<s> if ( UseCount > MaxUses ) return false ; if ( ! <ISA_LIT> ) { auto * BCI = dyn_cast < BitCastInst > ( U . getUser ( ) ) ; if ( ! BCI || ! BCI -> hasOneUse ( ) ) return false ; Type * DestEltTy = BCI -> getType ( ) -> getPointerElementType ( ) ; if ( DestEltTy -> isAggregateType ( ) ) return false ; Type * SrcEltTy = Arg . getType ( ) -> getPointerElementType ( ) ; if ( SrcEltTy -> isArrayTy ( ) ) return false ; if ( ( SrcEltTy -> isStructTy ( ) && ( SrcEltTy -> getNumContainedTypes ( ) != <NUM_LIT> ) ) ) return false ; if ( DL -> getTypeAllocSize ( SrcEltTy ) != DL -> getTypeAllocSize ( DestEltTy ) ) return false ; return checkArgumentUses ( * BCI ) ;


<s> unsigned ArgNumRegs = DL -> getTypeStoreSize ( ArgTy ) / <NUM_LIT> ; if ( ArgNumRegs + ReturnNumRegs > MaxNumRetRegs ) continue ; for ( ReturnInst * RI : Returns ) { BasicBlock * BB = RI -> getParent ( ) ; MemDepResult Q = MDA -> getPointerDependencyFrom ( MemoryLocation ( OutArg ) , true , BB -> end ( ) , BB , RI ) ; StoreInst * <ISA_LIT> = nullptr ; if ( Q . isDef ( ) ) <ISA_LIT> = dyn_cast < StoreInst > ( Q . getInst ( ) ) ; if ( <ISA_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * <ISA_LIT> << '\n' ) ; ReplaceableStores . emplace_back ( RI , <ISA_LIT> ) ; } else { ThisReplaceable = false ; break ; } } if ( ! ThisReplaceable ) continue ; for ( std :: pair < ReturnInst * , StoreInst * > Store : ReplaceableStores ) { Value * ReplVal = Store . second -> getValueOperand ( ) ; auto & ValVec = Replacements [ Store . first ] ; if ( llvm :: find_if ( ValVec , [ OutArg ] ( const std :: pair < Argument * , Value * > & Entry ) { return Entry . first == OutArg ; } ) != ValVec . end ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * OutArg << '\n' ) ; ThisReplaceable = false ; break ; } ValVec . emplace_back ( OutArg , ReplVal ) ; Store . second -> eraseFromParent ( ) ; } if ( ThisReplaceable ) { ReturnTypes . push_back ( ArgTy ) ; OutArgIndexes . insert ( OutArg -> getArgNo ( ) ) ; ++ NumOutArgumentsReplaced ; Changing = true ; } } } while ( Changing ) ; if ( Replacements . empty ( ) ) return false ; LLVMContext & Ctx = F . getParent ( ) -> getContext ( ) ; StructType * NewRetTy = StructType :: create ( Ctx , ReturnTypes , F . getName ( ) ) ; FunctionType * NewFuncTy = FunctionType :: get ( NewRetTy , F . getFunctionType ( ) -> params ( ) , F . isVarArg ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * NewRetTy << '\n' ) ; Function * NewFunc = Function :: Create ( NewFuncTy , Function :: PrivateLinkage , F . getName ( ) + <STR_LIT> ) ; F . getParent ( ) -> getFunctionList ( ) . insert ( F . getIterator ( ) , NewFunc ) ; NewFunc -> copyAttributesFrom ( & F ) ; NewFunc -> setComdat ( F . getComdat ( ) ) ; NewFunc -> stealArgumentListFrom ( F ) ;


<s> auto P = Program :: Metadata :: fromYAML ( YAML ) ; auto S = P . toYAML ( ) ; errs ( ) << <STR_LIT> << ( YAML == S ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> ; if ( YAML != S ) { errs ( ) << <STR_LIT> << YAML << '\n' << <STR_LIT> << S << '\n' ; }


<s> Prog . MDVersionSeq . push_back ( MDRevision ) ; if ( auto MD = M . getNamedMetadata ( <STR_LIT> ) ) { for ( unsigned I = <NUM_LIT> ; I < MD -> getNumOperands ( ) ; ++ I ) { auto Node = MD -> getOperand ( I ) ; if ( Node -> getNumOperands ( ) > <NUM_LIT> ) Prog . PrintfInfo . push_back ( cast < MDString > ( Node -> getOperand ( <NUM_LIT> ) ) -> getString ( ) ) ; } } for ( auto & F : M . functions ( ) ) {

if ( ! F . getMetadata ( <STR_LIT> ) ) continue ;


<s> Prog . MDVersionSeq . push_back ( MDRevision ) ; if ( auto MD = M . getNamedMetadata ( <STR_LIT> ) ) { for ( unsigned I = <NUM_LIT> ; I < MD -> getNumOperands ( ) ; ++ I ) { auto Node = MD -> getOperand ( I ) ; if ( Node -> getNumOperands ( ) > <NUM_LIT> ) Prog . PrintfInfo . push_back ( cast < MDString > ( Node -> getOperand ( <NUM_LIT> ) ) -> getString ( ) ) ; }


<s> Prog . MDVersionSeq . push_back ( MDVersion ) ; Prog . MDVersionSeq . push_back ( MDRevision ) ; IsaInfo :: Metadata & IIM = Prog . IsaInfo ; IIM . WavefrontSize = <STR_LIT> :: getWavefrontSize ( Features ) ; IIM . LocalMemorySize = <STR_LIT> :: getLocalMemorySize ( Features ) ; IIM . EUsPerCU = <STR_LIT> :: getEUsPerCU ( Features ) ; IIM . MaxWavesPerEU = <STR_LIT> :: getMaxWavesPerEU ( Features ) ; IIM . MaxFlatWorkGroupSize = <STR_LIT> :: getMaxFlatWorkGroupSize ( Features ) ; IIM . SGPRAllocGranule = <STR_LIT> :: getSGPRAllocGranule ( Features ) ;

IIM . TotalNumSGPRs = <STR_LIT> :: getTotalNumSGPRs ( Features ) ;


<s> YamlIO . mapRequired ( KeyName :: MDVersion , Prog . MDVersionSeq ) ; YamlIO . mapRequired ( KeyName :: IsaInfo , Prog . IsaInfo ) ;

YamlIO . mapOptional ( KeyName :: PrintfInfo , Prog . PrintfInfo ) ;


<s> auto P = Program :: Metadata :: fromYAML ( YAML ) ; auto S = P . toYAML ( ) ; llvm :: errs ( ) << <STR_LIT> << ( YAML == S ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> ;


<s> static void checkRuntimeMDYAMLString ( const std :: string & YAML ) { auto P = Program :: Metadata :: fromYAML ( YAML ) ; auto S = P . toYAML ( ) ; llvm :: errs ( ) << <STR_LIT> << ( YAML == S ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> ; if ( YAML != S ) { llvm :: errs ( ) << <STR_LIT> << YAML << '\n' << <STR_LIT> << S << '\n' ; }


<s> case <ISA_LIT>AS :: LOCAL_ADDRESS : return KernelArg :: Local ; case <ISA_LIT>AS :: FLAT_ADDRESS : return KernelArg :: Generic ; case <ISA_LIT>AS :: REGION_ADDRESS : return KernelArg :: Region ; default : return KernelArg :: Private ;


<s> } } const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; for ( auto & Arg : F . args ( ) ) { unsigned I = Arg . getArgNo ( ) ; Type * T = Arg . getType ( ) ; auto TypeName = dyn_cast < MDString > ( F . getMetadata ( <STR_LIT> ) -> getOperand ( I ) ) -> getString ( ) ; auto BaseTypeName = cast < MDString > ( F . getMetadata ( <STR_LIT> ) -> getOperand ( I ) ) -> getString ( ) ; StringRef ArgName ; if ( auto ArgNameMD = F . getMetadata ( <STR_LIT> ) ) ArgName = cast < MDString > ( ArgNameMD -> getOperand ( I ) ) -> getString ( ) ; auto TypeQual = cast < MDString > ( F . getMetadata ( <STR_LIT> ) -> getOperand ( I ) ) -> getString ( ) ; auto AccQual = cast < MDString > ( F . getMetadata ( <STR_LIT> ) -> getOperand ( I ) ) -> getString ( ) ; KernelArg :: Kind Kind ; if ( TypeQual . find ( <STR_LIT> ) != StringRef :: npos ) Kind = KernelArg :: Pipe ; else Kind = StringSwitch < KernelArg :: Kind > ( BaseTypeName ) . Case ( <STR_LIT> , KernelArg :: Sampler ) . Case ( <STR_LIT> , KernelArg :: Queue ) . Cases ( <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , KernelArg :: Image ) . Cases ( <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , KernelArg :: Image ) . Cases ( <STR_LIT> , <STR_LIT> , KernelArg :: Image ) . Default ( isa < PointerType > ( T ) ? ( T -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ? KernelArg :: DynamicSharedPointer : KernelArg :: GlobalBuffer ) : KernelArg :: ByValue ) ; Kernel . Args . emplace_back ( getRuntimeMDForKernelArg ( DL , T , Kind , BaseTypeName , TypeName , ArgName , TypeQual , AccQual ) ) ; } if ( F . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) { auto Int64T = Type :: getInt64Ty ( F . getContext ( ) ) ; Kernel . Args . emplace_back ( getRuntimeMDForKernelArg ( DL , Int64T , KernelArg :: HiddenGlobalOffsetX ) ) ; Kernel . Args . emplace_back ( getRuntimeMDForKernelArg ( DL , Int64T , KernelArg :: HiddenGlobalOffsetY ) ) ; Kernel . Args . emplace_back ( getRuntimeMDForKernelArg ( DL , Int64T , KernelArg :: HiddenGlobalOffsetZ ) ) ;


<s> Arg . Kind = Kind ; Arg . ValueType = getRuntimeMDValueType ( T , BaseTypeName ) ; if ( ! AccQual . empty ( ) ) { Arg . AccQual = StringSwitch < KernelArg :: AccessQualifer > ( AccQual ) . Case ( <STR_LIT> , KernelArg :: ReadOnly ) . Case ( <STR_LIT> , KernelArg :: WriteOnly ) . Case ( <STR_LIT> , KernelArg :: ReadWrite ) . Default ( KernelArg :: AccNone ) ; } if ( auto * PT = dyn_cast < PointerType > ( T ) ) { Arg . AddrQual = getRuntimeAddrSpace ( static_cast < <ISA_LIT>AS :: AddressSpaces > ( PT -> getAddressSpace ( ) ) ) ;


<s> return KernelArg :: F64 ; case Type :: IntegerTyID : { bool Signed = ! TypeName . startswith ( <STR_LIT> ) ; switch ( Ty -> getIntegerBitWidth ( ) ) { case <NUM_LIT> : return Signed ? KernelArg :: I8 : KernelArg :: U8 ; case <NUM_LIT> : return Signed ? KernelArg :: I16 : KernelArg :: U16 ;


<s> return KernelArg :: F16 ; case Type :: FloatTyID : return KernelArg :: F32 ; case Type :: DoubleTyID : return KernelArg :: F64 ; case Type :: IntegerTyID : { bool Signed = ! TypeName . startswith ( <STR_LIT> ) ; switch ( Ty -> getIntegerBitWidth ( ) ) { case <NUM_LIT> : return Signed ? KernelArg :: I8 : KernelArg :: U8 ; case <NUM_LIT> : return Signed ? KernelArg :: I16 : KernelArg :: U16 ; case <NUM_LIT> : return Signed ? KernelArg :: I32 : KernelArg :: U32 ; case <NUM_LIT> : return Signed ? KernelArg :: I64 : KernelArg :: U64 ;


<s> std :: string llvm :: getRuntimeMDYAMLString ( Module & M ) { Program :: Metadata Prog ; Prog . MDVersionSeq . push_back ( MDVersion ) ; Prog . MDVersionSeq . push_back ( MDRevision ) ; if ( auto MD = M . getNamedMetadata ( <STR_LIT> ) ) { for ( unsigned I = <NUM_LIT> ; I < MD -> getNumOperands ( ) ; ++ I ) {


<s> static std :: vector < uint32_t > getThreeInt32 ( MDNode * Node ) { assert ( Node -> getNumOperands ( ) == <NUM_LIT> ) ; std :: vector < uint32_t > V ; for ( const MDOperand & Op : Node -> operands ( ) ) {

const ConstantInt * CI = mdconst :: extract < ConstantInt > ( Op ) ;


<s> for ( const MDOperand & Op : Node -> operands ( ) ) { const ConstantInt * CI = mdconst :: extract < ConstantInt > ( Op ) ;

V . push_back ( CI -> getZExtValue ( ) ) ;


<s> static void mapping ( IO & YamlIO , Program :: Metadata & Prog ) {

YamlIO . mapRequired ( KeyName :: MDVersion , Prog . MDVersionSeq ) ;


<s> YamlIO . mapRequired ( KeyName :: MDVersion , Prog . MDVersionSeq ) ; YamlIO . mapOptional ( KeyName :: PrintfInfo , Prog . PrintfInfo ) ; YamlIO . mapOptional ( KeyName :: Kernels , Prog . Kernels ) ;


<s> <ISA_LIT>SetWavePriority ( ) : MachineFunctionPass ( ID ) {


<s> return BuildMI ( MF , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( priority ) ;


<s> static bool CanLowerPriorityDirectlyInPredecessors ( const MachineBasicBlock & MBB , MBBInfoSet & MBBInfos ) { for ( const MachineBasicBlock * Pred : MBB . predecessors ( ) ) { if ( ! MBBInfos [ Pred ] . MayReachVMEMLoad ) continue ; for ( const MachineBasicBlock * Succ : Pred -> successors ( ) ) { if ( MBBInfos [ Succ ] . MayReachVMEMLoad ) return false ; } }


<s> for ( const MachineBasicBlock * Pred : MBB . predecessors ( ) ) { if ( ! MBBInfos [ Pred ] . MayReachVMEMLoad ) continue ;

for ( const MachineBasicBlock * Succ : Pred -> successors ( ) ) {


<s> return new <ISA_LIT>SetWavePriority ( ) ;


<s> StringRef getPassName ( ) const override {


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> return <ISA_LIT>InstrInfo :: isVMEM ( MI ) && MI . mayLoad ( ) ;


<s> if ( any_of ( MBB , isVMEMLoad ) ) Worklist . push_back ( & MBB ) ; } while ( ! Worklist . empty ( ) ) { const MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; MBBInfo & Info = MBBInfos [ MBB ] ; if ( ! Info . MayReachVMEMLoad ) { Info . MayReachVMEMLoad = true ; Worklist . append ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; } } MachineBasicBlock & Entry = MF . front ( ) ; if ( ! MBBInfos [ & Entry ] . MayReachVMEMLoad ) return false ; MachineBasicBlock :: iterator I = Entry . begin ( ) , E = Entry . end ( ) ; while ( I != E && ! <ISA_LIT>InstrInfo :: isVALU ( * I ) && ! I -> isTerminator ( ) ) ++ I ; Entry . insert ( I , BuildSetprioMI ( MF , HighPriority ) ) ; SmallSet < MachineBasicBlock * , <NUM_LIT> > PriorityLoweringBlocks ; for ( MachineBasicBlock & MBB : MF ) { if ( MBBInfos [ & MBB ] . MayReachVMEMLoad ) { if ( MBB . succ_empty ( ) ) PriorityLoweringBlocks . insert ( & MBB ) ; continue ; }

if ( CanLowerPriorityDirectlyInPredecessors ( MBB , MBBInfos ) ) {


<s> bool isXNACKEnabled ( ) const { return EnableXNACK ;


<s> initializeSubtargetDependencies ( TT , GPU , FS ) ; const unsigned MaxStackAlign = <NUM_LIT> * <NUM_LIT> ; if ( getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) { InstrInfo . reset ( new <ISA_LIT>InstrInfo ( * this ) ) ; TLInfo . reset ( new <ISA_LIT>TargetLowering ( TM , * this ) ) ; FrameLowering . reset ( new <ISA_LIT>FrameLowering ( TargetFrameLowering :: StackGrowsUp , MaxStackAlign , <NUM_LIT> ) ) ; } else { InstrInfo . reset ( new <ISA_LIT>InstrInfo ( * this ) ) ; TLInfo . reset ( new <ISA_LIT>TargetLowering ( TM , * this ) ) ; FrameLowering . reset ( new <ISA_LIT>FrameLowering ( TargetFrameLowering :: StackGrowsUp , MaxStackAlign , <NUM_LIT> ) ) ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , DevName ( GPU ) , Is64bit ( false ) , DumpCode ( false ) , <ISA_LIT>ALUInst ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT>Subtarget :: <ISA_LIT> ) , FP64 ( false ) , FP64Denormals ( false ) , FP32Denormals ( false ) , FastFMAF32 ( false ) , CaymanISA ( false ) , FlatAddressSpace ( false ) , FlatForGlobal ( false ) , EnableIRStructurizer ( true ) , EnablePromoteAlloca ( false ) , EnableIfCvt ( true ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , WavefrontSize ( <NUM_LIT> ) , CFALUBug ( false ) , LocalMemorySize ( <NUM_LIT> ) , EnableVGPRSpilling ( false ) , SGPRInitBug ( false ) , IsGCN ( false ) , GCN1Encoding ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , LDSBankCount ( <NUM_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , EnableHugeScratchBuffer ( false ) , FrameLowering ( nullptr ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) , TargetTriple ( TT ) { initializeSubtargetDependencies ( TT , GPU , FS ) ; const unsigned MaxStackAlign = <NUM_LIT> * <NUM_LIT> ; if ( getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) { InstrInfo . reset ( new <ISA_LIT>InstrInfo ( * this ) ) ; TLInfo . reset ( new <ISA_LIT>TargetLowering ( TM , * this ) ) ; FrameLowering . reset ( new <ISA_LIT>FrameLowering ( TargetFrameLowering :: StackGrowsUp , MaxStackAlign , <NUM_LIT> ) ) ; } else { InstrInfo . reset ( new <ISA_LIT>InstrInfo ( * this ) ) ;


<s> bool enableSubRegLiveness ( ) const override {


<s> default : llvm_unreachable ( <STR_LIT> ) ; case SEA_ISLANDS : return <NUM_LIT> ; }


<s> unsigned getExplicitKernelArgOffset ( ) const {

return isAmdHsaOS ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> Generation getGeneration ( ) const { return Gen ;


<s> const InstrItineraryData * getInstrItineraryData ( ) const override { return & InstrItins ;


<s> <STR_LIT> <ISA_LIT>Subtarget :: getIsaVersion ( ) const { return <STR_LIT> ( getFeatureBits ( ) ) ;


<s> int getLDSBankCount ( ) const { return LDSBankCount ;


<s> int getLocalMemorySize ( ) const { return LocalMemorySize ;


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return <NUM_LIT> ;


<s> unsigned getMaxWavesPerCU ( ) const { if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return <NUM_LIT> ;


<s> const <ISA_LIT>RegisterInfo * getRegisterInfo ( ) const override { return & InstrInfo -> getRegisterInfo ( ) ;


<s> assert ( getGeneration ( ) <= NORTHERN_ISLANDS ) ; switch ( getWavefrontSize ( ) ) { case <NUM_LIT> :


<s> short getTexVTXClauseSize ( ) const { return TexVTXClauseSize ;


<s> unsigned getWavefrontSize ( ) const {


<s> if ( Size == <NUM_LIT> ) return ( getGeneration ( ) >= EVERGREEN ) ;

if ( Size == <NUM_LIT> ) return ( getGeneration ( ) >= SOUTHERN_ISLANDS ) ;


<s> return ( getGeneration ( ) >= EVERGREEN ) ;


return ( getGeneration ( ) >= EVERGREEN ) ;


<s> bool hasCARRY ( ) const { return ( getGeneration ( ) >= EVERGREEN ) ;


<s> bool hasCaymanISA ( ) const { return CaymanISA ;


<s> assert ( getGeneration ( ) <= NORTHERN_ISLANDS ) ; return CFALUBug ;


<s> assert ( getGeneration ( ) <= NORTHERN_ISLANDS ) ;


<s> bool hasFFBH ( ) const { return ( getGeneration ( ) >= EVERGREEN ) ;


<s> bool hasFlatAddressSpace ( ) const { return FlatAddressSpace ;


<s> bool hasHWFP64 ( ) const { return FP64 ;


return ( getGeneration ( ) >= SOUTHERN_ISLANDS || hasCaymanISA ( ) ) ;


<s> bool hasSGPRInitBug ( ) const { return SGPRInitBug ;


<s> bool hasVertexCache ( ) const { return HasVertexCache ;


<s> if ( GPU == <STR_LIT> && TT . getArch ( ) == Triple :: amdgcn ) GPU = <STR_LIT> ; ParseSubtargetFeatures ( GPU , FullFS ) ;

if ( getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) {


<s> return TargetTriple . getOS ( ) == Triple :: AMDHSA ;


<s> bool isIfCvtEnabled ( ) const { return EnableIfCvt ;


<s> bool IsIRStructurizerEnabled ( ) const {


<s> bool isTargetELF ( ) const { return false ;


<s> bool <ISA_LIT>Subtarget :: isVGPRSpillingEnabled ( const <ISA_LIT>MachineFunctionInfo * MFI ) const { return MFI -> getShaderType ( ) == ShaderType :: COMPUTE || EnableVGPRSpilling ;


<s> bool <ISA_LIT>Subtarget :: isVGPRSpillingEnabled ( const <ISA_LIT>MachineFunctionInfo * MFI ) const {

return MFI -> getShaderType ( ) == ShaderType :: COMPUTE || EnableVGPRSpilling ;


<s> bool loadStoreOptEnabled ( ) const { return EnableLoadStoreOpt ;


<s> void <ISA_LIT>Subtarget :: overrideSchedPolicy ( MachineSchedPolicy & Policy , MachineInstr * begin , MachineInstr * end , unsigned NumRegionInstrs ) const { if ( getGeneration ( ) >= SOUTHERN_ISLANDS ) { Policy . ShouldTrackPressure = true ; Policy . OnlyTopDown = false ; Policy . OnlyBottomUp = false ;


<s> return EnableUnsafeDSOffsetFolding ;


<s> return static_cast < const <ISA_LIT>Subtarget * > ( this ) -> getInstrInfo ( ) ;


<s> if ( getGeneration ( ) >= SOUTHERN_ISLANDS ) return static_cast < const <ISA_LIT>Subtarget * > ( this ) -> getInstrInfo ( ) ; return static_cast < const <ISA_LIT>Subtarget * > ( this ) -> getInstrInfo ( ) ;


<s> return <NUM_LIT> * <NUM_LIT> / getWavefrontSize ( ) ;


<s> inline const <ISA_LIT>TargetLowering * <ISA_LIT>Subtarget :: getTargetLowering ( ) const {

if ( getGeneration ( ) >= SOUTHERN_ISLANDS ) return static_cast < const <ISA_LIT>Subtarget * > ( this ) -> getTargetLowering ( ) ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT ) : TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , FP32Denormals ( false ) , FPExceptions ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange ( false ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSize ( <NUM_LIT> ) {


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , TargetTriple ( TT ) , Gen ( SOUTHERN_ISLANDS ) , IsaVersion ( ISAVersion0_0_0 ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP64FP16Denormals ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , VIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasR128A16 ( false ) , HasDLInsts ( false ) , D16PreservesUnusedBits ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) { CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering ( ) ) ) ; Legalizer . reset ( new <ISA_LIT>LegalizerInfo ( * this , TM ) ) ;

RegBankInfo . reset ( new <ISA_LIT>RegisterBankInfo ( * getRegisterInfo ( ) ) ) ;


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , TargetTriple ( TT ) , Gen ( SOUTHERN_ISLANDS ) , IsaVersion ( ISAVersion0_0_0 ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP64FP16Denormals ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , VIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasR128A16 ( false ) , HasDLInsts ( false ) , D16PreservesUnusedBits ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) { CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering ( ) ) ) ; Legalizer . reset ( new <ISA_LIT>LegalizerInfo ( * this , TM ) ) ; RegBankInfo . reset ( new <ISA_LIT>RegisterBankInfo ( * getRegisterInfo ( ) ) ) ; InstSelector . reset ( new <ISA_LIT>InstructionSelector ( * this , * static_cast < <ISA_LIT>RegisterBankInfo * > ( RegBankInfo . get ( ) ) , TM ) ) ;


<s> unsigned getAddressableNumSGPRs ( ) const {

return <STR_LIT> :: getAddressableNumSGPRs ( this ) ;


<s> unsigned getMaxNumSGPRs ( unsigned WavesPerEU , bool Addressable ) const {

return <STR_LIT> :: getMaxNumSGPRs ( this , WavesPerEU , Addressable ) ;


<s> return <STR_LIT> :: getMaxNumSGPRs ( this , WavesPerEU , Addressable ) ;


<s> unsigned getMaxNumVGPRs ( unsigned WavesPerEU ) const { return <STR_LIT> :: getMaxNumVGPRs ( this , WavesPerEU ) ;


<s> unsigned getMaxWavesPerEU ( unsigned FlatWorkGroupSize ) const override { return <STR_LIT> :: getMaxWavesPerEU ( this , FlatWorkGroupSize ) ;


<s> unsigned getMaxWorkGroupsPerCU ( unsigned FlatWorkGroupSize ) const override { return <STR_LIT> :: getMaxWorkGroupsPerCU ( this , FlatWorkGroupSize ) ;


<s> unsigned getMinFlatWorkGroupSize ( ) const override {

return <STR_LIT> :: getMinFlatWorkGroupSize ( this ) ;


<s> return <STR_LIT> :: getMinNumSGPRs ( this , WavesPerEU ) ;


<s> return <STR_LIT> :: getSGPRAllocGranule ( this ) ;


<s> unsigned getSGPREncodingGranule ( ) const { return <STR_LIT> :: getSGPREncodingGranule ( this ) ;


<s> unsigned getSGPREncodingGranule ( ) const {

return <STR_LIT> :: getSGPREncodingGranule ( this ) ;


<s> unsigned getTotalNumSGPRs ( ) const { return <STR_LIT> :: getTotalNumSGPRs ( this ) ;


<s> return <STR_LIT> :: getTotalNumSGPRs ( this ) ;


<s> unsigned getTotalNumVGPRs ( ) const {

return <STR_LIT> :: getTotalNumVGPRs ( this ) ;


<s> return <STR_LIT> :: getVGPRAllocGranule ( this ) ;


<s> unsigned getVGPREncodingGranule ( ) const { return <STR_LIT> :: getVGPREncodingGranule ( this ) ;


<s> bool hasR128A16 ( ) const { return HasR128A16 ;


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , DX10Clamp ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> if ( Dep . getKind ( ) != SDep :: Kind :: Data || ! Dep . getReg ( ) || ! Def -> isInstr ( ) || ! Use -> isInstr ( ) ) return ; MachineInstr * DefI = Def -> getInstr ( ) ; MachineInstr * UseI = Use -> getInstr ( ) ; if ( DefI -> isBundle ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getRegisterInfo ( ) ; auto Reg = Dep . getReg ( ) ; MachineBasicBlock :: const_instr_iterator I ( DefI -> getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator E ( DefI -> getParent ( ) -> instr_end ( ) ) ; unsigned Lat = <NUM_LIT> ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { if ( I -> modifiesRegister ( Reg , TRI ) ) Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * I ) ; else if ( Lat ) -- Lat ; } Dep . setLatency ( Lat ) ; } else if ( UseI -> isBundle ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getRegisterInfo ( ) ; auto Reg = Dep . getReg ( ) ; MachineBasicBlock :: const_instr_iterator I ( UseI -> getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator E ( UseI -> getParent ( ) -> instr_end ( ) ) ; unsigned Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * DefI ) ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) && Lat ; ++ I ) {

if ( I -> readsRegister ( Reg , TRI ) ) break ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT ) :

TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , HasMadMacF32Insts ( false ) , HasDsSrc2Insts ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange ( false ) , MaxWavesPerEU ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSizeLog2 ( <NUM_LIT> ) {


<s> return EnableFlatScratch && hasFlatScratchInsts ( ) ;


<s> unsigned GCNSubtarget :: getConstantBusLimit ( unsigned Opcode ) const { if ( getGeneration ( ) < GFX10 ) return <NUM_LIT> ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; }


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) return <NUM_LIT> ; if ( MFI . hasFlatScratchInit ( ) ) {

if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ;


<s> bool RequestedFlatWorkGroupSize = F . hasFnAttribute ( <STR_LIT> ) ; std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default , true ) ; if ( Requested . second && Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinWavesPerEU ( ) || Requested . second > getMaxWavesPerEU ( ) ) return Default ;

if ( RequestedFlatWorkGroupSize && Requested . first < MinImpliedByFlatWorkGroupSize ) return Default ;


<s> std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default , true ) ; if ( Requested . second && Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinWavesPerEU ( ) || Requested . second > getMaxWavesPerEU ( ) ) return Default ; if ( RequestedFlatWorkGroupSize && Requested . first < MinImpliedByFlatWorkGroupSize ) return Default ;


<s> bool GCNSubtarget :: hasMadF16 ( ) const {

return InstrInfo . pseudoToMCOpcode ( <STR_LIT> ) != - <NUM_LIT> ;


<s> return InstrInfo . pseudoToMCOpcode ( <STR_LIT> ) != - <NUM_LIT> ;


<s> ParseSubtargetFeatures ( GPU , GPU , FullFS ) ; if ( Gen == <ISA_LIT>Subtarget :: INVALID ) { Gen = TT . getOS ( ) == Triple :: AMDHSA ? <ISA_LIT>Subtarget :: SEA_ISLANDS : <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ; } assert ( ! hasFP64 ( ) || ( getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) ) ; assert ( hasAddr64 ( ) || hasFlat ( ) ) ; if ( ! hasAddr64 ( ) && ! FS . contains ( <STR_LIT> ) && ! FlatForGlobal ) { ToggleFeature ( <STR_LIT> ) ; FlatForGlobal = true ; } if ( ! hasFlat ( ) && ! FS . contains ( <STR_LIT> ) && FlatForGlobal ) { ToggleFeature ( <STR_LIT> ) ; FlatForGlobal = false ; } if ( MaxPrivateElementSize == <NUM_LIT> ) MaxPrivateElementSize = <NUM_LIT> ; if ( LDSBankCount == <NUM_LIT> ) LDSBankCount = <NUM_LIT> ; if ( TT . getArch ( ) == Triple :: amdgcn ) { if ( LocalMemorySize == <NUM_LIT> ) LocalMemorySize = <NUM_LIT> ; if ( ! HasMovrel && ! HasVGPRIndexMode ) HasMovrel = true ;


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) :

<ISA_LIT>GenSubtargetInfo ( TT , GPU , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> unsigned <ISA_LIT>Subtarget :: getImplicitArgNumBytes ( const Function & F ) const { assert ( <STR_LIT> ( F . getCallingConv ( ) ) ) ;

if ( F . hasFnAttribute ( <STR_LIT> ) ) return <NUM_LIT> ;


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) return <NUM_LIT> ;

if ( isMesaKernel ( F ) ) return <NUM_LIT> ;


<s> unsigned ExplicitOffset = getExplicitKernelArgOffset ( F ) ; uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes ; unsigned ImplicitBytes = getImplicitArgNumBytes ( F ) ; if ( ImplicitBytes != <NUM_LIT> ) { const Align Alignment = getAlignmentForImplicitArgPtr ( ) ; TotalSize = alignTo ( ExplicitArgBytes , Alignment ) + ImplicitBytes ;

MaxAlign = std :: max ( MaxAlign , Alignment ) ;


<s> continue ; } MachineInstr & MI1 = * SUa -> getInstr ( ) ; if ( ( TII -> isVMEM ( MI1 ) && TII -> isVMEM ( MI2 ) ) || ( TII -> isFLAT ( MI1 ) && TII -> isFLAT ( MI2 ) ) || ( TII -> isSMRD ( MI1 ) && TII -> isSMRD ( MI2 ) ) || ( TII -> isDS ( MI1 ) && TII -> isDS ( MI2 ) ) ) { SU . addPredBarrier ( SUa ) ; for ( const SDep & <ISA_LIT> : SU . Preds ) { if ( <ISA_LIT> . getSUnit ( ) != SUa ) SUa -> addPred ( SDep ( <ISA_LIT> . getSUnit ( ) , SDep :: Artificial ) ) ; } if ( & SU != & DAG -> ExitSU ) { for ( const SDep & <ISA_LIT> : SUa -> Succs ) { if ( <ISA_LIT> . getSUnit ( ) != & SU ) <ISA_LIT> . getSUnit ( ) -> addPred ( SDep ( & SU , SDep :: Artificial ) ) ; } } }


<s> } } SmallPtrSet < const SUnit * , <NUM_LIT> > Visited ; while ( ! Preds . empty ( ) ) { const SUnit * SU = Preds . pop_back_val ( ) ; if ( llvm :: find ( Succs , SU ) != Succs . end ( ) ) return false ; Visited . insert ( SU ) ; for ( const SDep & <ISA_LIT> : SU -> Preds ) if ( <ISA_LIT> . getSUnit ( ) != SU && ! Visited . count ( <ISA_LIT> . getSUnit ( ) ) ) Preds . push_back ( <ISA_LIT> . getSUnit ( ) ) ;


<s> unsigned Occupancy = std :: min ( getMaxWavesPerEU ( ) , getOccupancyWithLocalMemSize ( LDSSize , MF . getFunction ( ) ) ) ; if ( NumSGPRs ) Occupancy = std :: min ( Occupancy , getOccupancyWithNumSGPRs ( NumSGPRs ) ) ;

if ( NumVGPRs ) Occupancy = std :: min ( Occupancy , getOccupancyWithNumVGPRs ( NumVGPRs ) ) ;


<s> assert ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL || F . getCallingConv ( ) == CallingConv :: SPIR_KERNEL ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; uint64_t ExplicitArgBytes = <NUM_LIT> ; MaxAlign = Align :: None ( ) ; for ( const Argument & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; const Align Alignment ( DL . getABITypeAlignment ( ArgTy ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( ArgTy ) ; ExplicitArgBytes = alignTo ( ExplicitArgBytes , Alignment ) + AllocSize ; MaxAlign = std :: max ( MaxAlign , Alignment ) ; } return ExplicitArgBytes ;


<s> return countLeadingZeros ( MaxWaveScratchSize ) + getWavefrontSizeLog2 ( ) ;


<s> unsigned MaxWaves = getMaxWavesPerEU ( ) ; unsigned Limit = getLocalMemorySize ( ) * MaxWaves / WorkGroupsPerCu ; unsigned NumWaves = Limit / ( Bytes ? Bytes : <NUM_LIT> ) ; NumWaves = std :: min ( NumWaves , MaxWaves ) ;

NumWaves = std :: max ( NumWaves , <NUM_LIT> ) ;


<s> Mutations . push_back ( std :: make_unique < MemOpClusterMutation > ( & InstrInfo ) ) ; Mutations . push_back ( std :: make_unique < FillMFMAShadowMutation > ( & InstrInfo ) ) ;


<s> void GCNSubtarget :: getPostRAMutations ( std :: vector < std :: unique_ptr < ScheduleDAGMutation >> & Mutations ) const { Mutations . push_back ( std :: make_unique < MemOpClusterMutation > ( & InstrInfo ) ) ;

Mutations . push_back ( std :: make_unique < FillMFMAShadowMutation > ( & InstrInfo ) ) ;


<s> if ( Requested . second && Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinWavesPerEU ( ) || Requested . first > getMaxWavesPerEU ( ) ) return Default ; if ( Requested . second > getMaxWavesPerEU ( ) ) return Default ; if ( RequestedFlatWorkGroupSize && Requested . first < MinImpliedByFlatWorkGroupSize ) return Default ;


return isAmdHsaOS ( ) ? CodeObjectV3 : false ;


<s> bool hasDot1Insts ( ) const { return HasDot1Insts ;


<s> bool hasDot6Insts ( ) const { return HasDot6Insts ;


<s> bool hasDwordx3LoadStores ( ) const { return CIInsts ;


<s> bool hasDwordx3LoadStores ( ) const {


<s> bool hasFlatSegmentOffsetBug ( ) const {


<s> return LDSMisalignedBug && ! EnableCuMode ;


<s> bool hasLDSMisalignedBug ( ) const { return LDSMisalignedBug && ! EnableCuMode ;


<s> bool hasNSAEncoding ( ) const { return HasNSAEncoding ;


<s> return getGeneration ( ) == <ISA_LIT>Subtarget :: GFX9 ;


<s> bool hasReadM0SendMsgHazard ( ) const {

return getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS && getGeneration ( ) <= <ISA_LIT>Subtarget :: GFX9 ;


<s> bool hasRegisterBanking ( ) const { return HasRegisterBanking ;


<s> bool hasScalarFlatScratchInsts ( ) const { return ScalarFlatScratchInsts ;


<s> bool hasSMEMtoVectorWriteHazard ( ) const { return HasSMEMtoVectorWriteHazard ;


<s> bool hasSMovFedHazard ( ) const { return getGeneration ( ) == <ISA_LIT>Subtarget :: GFX9 ;


<s> bool hasVcmpxExecWARHazard ( ) const { return HasVcmpxExecWARHazard ;


<s> bool hasVMEMtoScalarWriteHazard ( ) const {


<s> bool hasVOP3Literal ( ) const { return HasVOP3Literal ;


<s> bool hasVscnt ( ) const { return HasVscnt ;


<s> assert ( ! hasFP64 ( ) || ( getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) ) ; if ( ! hasAddr64 ( ) && ! FS . contains ( <STR_LIT> ) ) { FlatForGlobal = true ; } if ( MaxPrivateElementSize == <NUM_LIT> ) MaxPrivateElementSize = <NUM_LIT> ; if ( LDSBankCount == <NUM_LIT> ) LDSBankCount = <NUM_LIT> ; if ( TT . getArch ( ) == Triple :: amdgcn ) { if ( LocalMemorySize == <NUM_LIT> ) LocalMemorySize = <NUM_LIT> ; if ( ! HasMovrel && ! HasVGPRIndexMode ) HasMovrel = true ; } if ( WavefrontSize == <NUM_LIT> ) WavefrontSize = <NUM_LIT> ;

HasFminFmaxLegacy = getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ;


<s> bool isCuModeEnabled ( ) const { return EnableCuMode ;


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) :

<ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> for ( ++ I ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { if ( I -> modifiesRegister ( Reg , TRI ) ) Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * I ) ; else if ( Lat ) -- Lat ; } Dep . setLatency ( Lat ) ; } else if ( DstI -> isBundle ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getRegisterInfo ( ) ; auto Reg = Dep . getReg ( ) ; MachineBasicBlock :: const_instr_iterator I ( DstI -> getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator E ( DstI -> getParent ( ) -> instr_end ( ) ) ; unsigned Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * SrcI ) ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) && Lat ; ++ I ) { if ( I -> readsRegister ( Reg , TRI ) ) break ;


<s> <ISA_LIT>DwarfFlavour get<ISA_LIT>DwarfFlavour ( ) const {

return getWavefrontSize ( ) == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ;


<s> const Align Alignment ( DL . getABITypeAlignment ( ArgTy ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( ArgTy ) ; ExplicitArgBytes = alignTo ( ExplicitArgBytes , Alignment ) + AllocSize ; MaxAlign = std :: max ( MaxAlign , Alignment ) ;


<s> MaxAlign = Align ( <NUM_LIT> ) ; for ( const Argument & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ;

const Align Alignment ( DL . getABITypeAlignment ( ArgTy ) ) ;


<s> return InlineAsmLoweringInfo . get ( ) ;


<s> InstructionSelector * getInstructionSelector ( ) const override {


<s> return ( ForBufferRSrc || ! enableFlatScratch ( ) ) ? MaxPrivateElementSize : <NUM_LIT> ;


<s> unsigned getSetRegWaitStates ( ) const { return getGeneration ( ) <= SEA_ISLANDS ? <NUM_LIT> : <NUM_LIT> ;


<s> unsigned getWavesPerEUForWorkGroup ( unsigned FlatWorkGroupSize ) const override { return <STR_LIT> :: getWavesPerEUForWorkGroup ( this , FlatWorkGroupSize ) ;


<s> return <STR_LIT> :: getWavesPerEUForWorkGroup ( this , FlatWorkGroupSize ) ;


<s> bool hasA16 ( ) const { return hasR128A16 ( ) || hasGFX10A16 ( ) ;


<s> return getGeneration ( ) >= VOLCANIC_ISLANDS ;


return getGeneration ( ) >= VOLCANIC_ISLANDS ;


<s> bool hasDenormModeInst ( ) const { return getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ;


<s> return HasDPP && getGeneration ( ) < GFX10 ;


<s> return ( getGeneration ( ) > <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) ;


return ( getGeneration ( ) > <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) ;


<s> bool hasFlatScratchSTMode ( ) const {

return hasFlatScratchInsts ( ) && hasGFX10_3Insts ( ) ;


<s> return hasFlatScratchInsts ( ) && hasGFX10_3Insts ( ) ;


return getGeneration ( ) == SOUTHERN_ISLANDS ;


<s> bool hasGWSSemaReleaseAll ( ) const {


<s> bool hasImageGather4D16Bug ( ) const { return HasImageGather4D16Bug ;


<s> bool hasImageStoreD16Bug ( ) const { return HasImageStoreD16Bug ;


<s> bool hasReadVCCZBug ( ) const { return getGeneration ( ) <= SEA_ISLANDS ;


<s> return getGeneration ( ) <= SEA_ISLANDS ;


<s> bool hasScalarMulHiInsts ( ) const { return GFX9Insts ;


<s> bool hasScalarPackInsts ( ) const { return GFX9Insts ;


<s> bool hasSMRDReadVALUDefHazard ( ) const {


<s> bool hasUnalignedDSAccess ( ) const {


<s> return getGeneration ( ) != SOUTHERN_ISLANDS ;


<s> bool hasUsableDivScaleConditionOutput ( ) const { return getGeneration ( ) != SOUTHERN_ISLANDS ;


<s> bool hasUsableDSOffset ( ) const { return getGeneration ( ) >= SEA_ISLANDS ;


return getGeneration ( ) >= SEA_ISLANDS ;


<s> return getGeneration ( ) >= VOLCANIC_ISLANDS ;


<s> return getWavefrontSize ( ) == <NUM_LIT> ;


return getWavefrontSize ( ) == <NUM_LIT> ;


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , TargetTriple ( TT ) , Gen ( SOUTHERN_ISLANDS ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP64FP16Denormals ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , EnablePRTStrictNull ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , VIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasR128A16 ( false ) , HasDLInsts ( false ) , HasDot1Insts ( false ) , HasDot2Insts ( false ) , EnableSRAMECC ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) {

CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering ( ) ) ) ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64Denormals ( false ) , FPExceptions ( false ) , FlatForGlobal ( false ) , UnalignedBufferAccess ( false ) , EnableXNACK ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , DumpCode ( false ) , FP64 ( false ) , IsGCN ( false ) , GCN1Encoding ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , FlatAddressSpace ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {

initializeSubtargetDependencies ( TT , GPU , FS ) ;


<s> case SEA_ISLANDS : return <NUM_LIT> ; default :


<s> <STR_LIT> <ISA_LIT>Subtarget :: getIsaVersion ( ) const { return <STR_LIT> ( getFeatureBits ( ) ) ;


<s> case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> :


<s> if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ; if ( Bytes <= <NUM_LIT> ) return <NUM_LIT> ;


<s> return <NUM_LIT> ; case <NUM_LIT> : return hasCaymanISA ( ) ? <NUM_LIT> : <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> bool hasFP16Denormals ( const Function & F ) const {


<s> bool hasFP64Denormals ( const Function & F ) const { return FP64FP16Denormals ;


<s> bool hasFP64FP16Denormals ( const Function & F ) const { return FP64FP16Denormals ;


<s> <ISA_LIT>Subtarget ( TT , GPU , FS , TM ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , TLInfo ( TM , * this ) , GISel ( ) {


<s> <ISA_LIT>Subtarget :: ~ <ISA_LIT>Subtarget ( ) {


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64FP16Denormals ( false ) , FPExceptions ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , DumpCode ( false ) , FP64 ( false ) , IsGCN ( false ) , GCN1Encoding ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , HasVOP3PInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasInv2PiInlineImm ( false ) , HasSDWA ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAClampVOPC ( false ) , HasDPP ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) { AS = <STR_LIT> ( TT ) ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64FP16Denormals ( false ) , FPExceptions ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , DumpCode ( false ) , FP64 ( false ) , IsGCN ( false ) , GCN1Encoding ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , HasVOP3PInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasInv2PiInlineImm ( false ) , HasSDWA ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAClampVOPC ( false ) , HasDPP ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> bool debuggerReserveRegs ( ) const { return DebuggerReserveRegs ;


<s> bool debuggerSupported ( ) const { return debuggerInsertNops ( ) && debuggerReserveRegs ( ) && debuggerEmitPrologue ( ) ;


<s> return <STR_LIT> ( MF . getFunction ( ) -> getCallingConv ( ) ) ;


<s> bool enableIEEEBit ( const MachineFunction & MF ) const { return <STR_LIT> ( MF . getFunction ( ) -> getCallingConv ( ) ) ;


<s> bool flatScratchIsPointer ( ) const { return getGeneration ( ) >= GFX9 ;


<s> unsigned getAddressableNumSGPRs ( ) const {

return <STR_LIT> :: getAddressableNumSGPRs ( getFeatureBits ( ) ) ;


<s> unsigned getAddressableNumSGPRs ( ) const { return <STR_LIT> :: getAddressableNumSGPRs ( getFeatureBits ( ) ) ;


<s> return <STR_LIT> :: getAddressableNumVGPRs ( getFeatureBits ( ) ) ;


<s> return GISel -> getCallLowering ( ) ;


<s> return <STR_LIT> :: getEUsPerCU ( getFeatureBits ( ) ) ;


<s> unsigned getExplicitKernelArgOffset ( const MachineFunction & MF ) const {

return isAmdCodeObjectV2 ( MF ) ? <NUM_LIT> : <NUM_LIT> ;


<s> Default . second = <STR_LIT> ( F , <STR_LIT> , Default . second ) ; Default . first = std :: min ( Default . first , Default . second ) ; std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default ) ;

if ( Requested . first > Requested . second ) return Default ;


<s> std :: pair < unsigned , unsigned > <ISA_LIT>Subtarget :: getFlatWorkGroupSizes ( const Function & F ) const { std :: pair < unsigned , unsigned > Default = <STR_LIT> ( F . getCallingConv ( ) ) ? std :: pair < unsigned , unsigned > ( getWavefrontSize ( ) * <NUM_LIT> , getWavefrontSize ( ) * <NUM_LIT> ) : std :: pair < unsigned , unsigned > ( <NUM_LIT> , getWavefrontSize ( ) ) ; Default . second = <STR_LIT> ( F , <STR_LIT> , Default . second ) ; Default . first = std :: min ( Default . first , Default . second ) ; std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default ) ; if ( Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinFlatWorkGroupSize ( ) ) return Default ; if ( Requested . second > getMaxFlatWorkGroupSize ( ) ) return Default ;


<s> if ( isMesaKernel ( MF ) ) return <NUM_LIT> ; if ( isAmdHsaOS ( ) && isOpenCLEnv ( ) ) return <NUM_LIT> ;


<s> const InstructionSelector * getInstructionSelector ( ) const override { assert ( GISel && <STR_LIT> ) ;

return GISel -> getInstructionSelector ( ) ;


<s> if ( ImplicitBytes == <NUM_LIT> ) return ExplicitArgBytes ;

unsigned Alignment = getAlignmentForImplicitArgPtr ( ) ;


<s> unsigned Alignment = getAlignmentForImplicitArgPtr ( ) ;

return alignTo ( ExplicitArgBytes , Alignment ) + ImplicitBytes ;


<s> const LegalizerInfo * getLegalizerInfo ( ) const override {


<s> unsigned getMaxFlatWorkGroupSize ( ) const {

return <STR_LIT> :: getMaxFlatWorkGroupSize ( getFeatureBits ( ) ) ;


<s> return <STR_LIT> :: getMaxNumSGPRs ( getFeatureBits ( ) , WavesPerEU , Addressable ) ;


<s> unsigned getMaxNumVGPRs ( unsigned WavesPerEU ) const { return <STR_LIT> :: getMaxNumVGPRs ( getFeatureBits ( ) , WavesPerEU ) ;


<s> unsigned getMaxWavesPerCU ( unsigned FlatWorkGroupSize ) const {

return <STR_LIT> :: getMaxWavesPerCU ( getFeatureBits ( ) , FlatWorkGroupSize ) ;


<s> unsigned getMaxWavesPerCU ( unsigned FlatWorkGroupSize ) const { return <STR_LIT> :: getMaxWavesPerCU ( getFeatureBits ( ) , FlatWorkGroupSize ) ;


<s> return <STR_LIT> :: getMaxWavesPerEU ( getFeatureBits ( ) , FlatWorkGroupSize ) ;


<s> return <STR_LIT> :: getMaxWorkGroupsPerCU ( getFeatureBits ( ) , FlatWorkGroupSize ) ;


<s> unsigned getMinFlatWorkGroupSize ( ) const { return <STR_LIT> :: getMinFlatWorkGroupSize ( getFeatureBits ( ) ) ;


<s> return <STR_LIT> :: getMinNumSGPRs ( getFeatureBits ( ) , WavesPerEU ) ;


<s> return <STR_LIT> :: getMinNumVGPRs ( getFeatureBits ( ) , WavesPerEU ) ;


<s> unsigned getOccupancyWithLocalMemSize ( const MachineFunction & MF ) const {

const auto * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> const auto * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

return getOccupancyWithLocalMemSize ( MFI -> getLDSSize ( ) , * MF . getFunction ( ) ) ;


<s> unsigned <ISA_LIT>Subtarget :: getOccupancyWithNumSGPRs ( unsigned SGPRs ) const { if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ;


<s> if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> unsigned <ISA_LIT>Subtarget :: getOccupancyWithNumVGPRs ( unsigned VGPRs ) const { if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( VGPRs <= <NUM_LIT> ) return <NUM_LIT> ;


<s> assert ( GISel && <STR_LIT> ) ; return GISel -> getRegBankInfo ( ) ;


<s> return GISel -> getRegBankInfo ( ) ;


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ;

if ( getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) return <NUM_LIT> ;


<s> return debuggerReserveRegs ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> unsigned getReservedNumVGPRs ( const MachineFunction & MF ) const {

return debuggerReserveRegs ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> unsigned getSGPRAllocGranule ( ) const { return <STR_LIT> :: getSGPRAllocGranule ( getFeatureBits ( ) ) ;


<s> unsigned getSGPREncodingGranule ( ) const {

return <STR_LIT> :: getSGPREncodingGranule ( getFeatureBits ( ) ) ;


<s> unsigned getTotalNumSGPRs ( ) const {

return <STR_LIT> :: getTotalNumSGPRs ( getFeatureBits ( ) ) ;


<s> return <STR_LIT> :: getVGPRAllocGranule ( getFeatureBits ( ) ) ;


<s> std :: pair < unsigned , unsigned > <ISA_LIT>Subtarget :: getWavesPerEU ( const Function & F ) const { std :: pair < unsigned , unsigned > Default ( <NUM_LIT> , getMaxWavesPerEU ( ) ) ; std :: pair < unsigned , unsigned > FlatWorkGroupSizes = getFlatWorkGroupSizes ( F ) ; unsigned MinImpliedByFlatWorkGroupSize = getMaxWavesPerEU ( FlatWorkGroupSizes . second ) ; bool RequestedFlatWorkGroupSize = false ; if ( F . hasFnAttribute ( <STR_LIT> ) || F . hasFnAttribute ( <STR_LIT> ) ) { Default . first = MinImpliedByFlatWorkGroupSize ; RequestedFlatWorkGroupSize = true ;


<s> unsigned getWavesPerWorkGroup ( unsigned FlatWorkGroupSize ) const {

return <STR_LIT> :: getWavesPerWorkGroup ( getFeatureBits ( ) , FlatWorkGroupSize ) ;


return ( getGeneration ( ) < VOLCANIC_ISLANDS ) ;


<s> return ( getGeneration ( ) < VOLCANIC_ISLANDS ) ;


<s> return getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ;


<s> bool hasMin3Max3_16 ( ) const { return getGeneration ( ) >= GFX9 ;


<s> return getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 ;


<s> bool isAmdCodeObjectV2 ( const MachineFunction & MF ) const { return isAmdHsaOS ( ) || isMesaKernel ( MF ) ;


<s> bool isMesaGfxShader ( const MachineFunction & MF ) const { return isMesa3DOS ( ) && <STR_LIT> ( MF . getFunction ( ) -> getCallingConv ( ) ) ;


<s> bool isMesaKernel ( const MachineFunction & MF ) const {

return isMesa3DOS ( ) && ! <STR_LIT> ( MF . getFunction ( ) -> getCallingConv ( ) ) ;


return TargetTriple . getEnvironment ( ) == Triple :: OpenCL || TargetTriple . getEnvironmentName ( ) == <STR_LIT> ;


<s> return EnableVGPRSpilling || ! <STR_LIT> ( F . getCallingConv ( ) ) ;


<s> case <STR_LIT> : case <STR_LIT> : IdQuery = true ; case <STR_LIT> : Dim = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : IdQuery = true ; case <STR_LIT> : Dim = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : IdQuery = true ; case <STR_LIT> : Dim = <NUM_LIT> ; break ; default : break ; } if ( Dim <= <NUM_LIT> ) { if ( auto Node = Kernel -> getMetadata ( <STR_LIT> ) ) if ( Node -> getNumOperands ( ) == <NUM_LIT> ) MinSize = MaxSize = mdconst :: extract < ConstantInt > ( Node -> getOperand ( Dim ) ) -> getZExtValue ( ) ; } }


<s> Policy . OnlyTopDown = false ; Policy . OnlyBottomUp = false ; if ( ! enable<ISA_LIT>Scheduler ( ) ) Policy . ShouldTrackLaneMasks = true ;


<s> Policy . OnlyBottomUp = false ; if ( ! enable<ISA_LIT>Scheduler ( ) ) Policy . ShouldTrackLaneMasks = true ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>Subtarget ( TT , GPU , FS , TM ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , TLInfo ( TM , * this ) {


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) :

<ISA_LIT>Subtarget ( TT , GPU , FS , TM ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , TLInfo ( TM , * this ) {


<s> void setGISelAccessor ( GISelAccessor & GISel ) { this -> GISel . reset ( & GISel ) ;


<s> } if ( getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) { FP64FP16Denormals = false ; FP32Denormals = false ; } if ( MaxPrivateElementSize == <NUM_LIT> ) MaxPrivateElementSize = <NUM_LIT> ; return * this ;


<s> std :: pair < unsigned , unsigned > WavesPerEU = MFI . getWavesPerEU ( ) ; unsigned MaxNumSGPRs = getMaxNumSGPRs ( WavesPerEU . first , false ) ; unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs ( WavesPerEU . first , true ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) { unsigned Requested = <STR_LIT> ( F , <STR_LIT> , MaxNumSGPRs ) ; if ( Requested && ( Requested <= getReservedNumSGPRs ( MF ) ) ) Requested = <NUM_LIT> ; unsigned InputNumSGPRs = MFI . getNumPreloadedSGPRs ( ) ; if ( Requested && Requested < InputNumSGPRs ) Requested = InputNumSGPRs ; if ( Requested && Requested > getMaxNumSGPRs ( WavesPerEU . first , false ) ) Requested = <NUM_LIT> ; if ( WavesPerEU . second && Requested && Requested < getMinNumSGPRs ( WavesPerEU . second ) ) Requested = <NUM_LIT> ;

if ( Requested ) MaxNumSGPRs = Requested ;


<s> if ( Requested && Requested <= getReservedNumVGPRs ( MF ) ) Requested = <NUM_LIT> ; if ( Requested && Requested > getMaxNumVGPRs ( WavesPerEU . first ) ) Requested = <NUM_LIT> ;

if ( WavesPerEU . second && Requested && Requested < getMinNumVGPRs ( WavesPerEU . second ) ) Requested = <NUM_LIT> ;


<s> void <ISA_LIT>Subtarget :: getPostRAMutations ( std :: vector < std :: unique_ptr < ScheduleDAGMutation >> & Mutations ) const {

Mutations . push_back ( llvm :: make_unique < MemOpClusterMutation > ( & InstrInfo ) ) ;


<s> if ( MaxPrivateElementSize == <NUM_LIT> ) MaxPrivateElementSize = <NUM_LIT> ; if ( LDSBankCount == <NUM_LIT> ) LDSBankCount = <NUM_LIT> ; if ( TT . getArch ( ) == Triple :: amdgcn ) { if ( LocalMemorySize == <NUM_LIT> ) LocalMemorySize = <NUM_LIT> ; if ( ! HasMovrel && ! HasVGPRIndexMode ) HasMovrel = true ; } return * this ;


<s> RegBankInfo . reset ( new <ISA_LIT>RegisterBankInfo ( * getRegisterInfo ( ) ) ) ; InstSelector . reset ( new <ISA_LIT>InstructionSelector ( * this , * static_cast < <ISA_LIT>RegisterBankInfo * > ( RegBankInfo . get ( ) ) ) ) ;


<s> return <STR_LIT> :: getVGPRAllocGranule ( getFeatureBits ( ) ) ; ;


<s> unsigned getVGPRAllocGranule ( ) const {

return <STR_LIT> :: getVGPRAllocGranule ( getFeatureBits ( ) ) ; ;


<s> bool needWaitcntBeforeBarrier ( ) const { return true ;


<s> unsigned getOccupancyWithLocalMemSize ( const MachineFunction & MF ) const { const auto * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

return getOccupancyWithLocalMemSize ( MFI -> getLDSSize ( ) , MF . getFunction ( ) ) ;


<s> return isMesa3DOS ( ) && <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ;


<s> bool isMesaGfxShader ( const MachineFunction & MF ) const {

return isMesa3DOS ( ) && <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ;


<s> return isMesa3DOS ( ) && ! <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ;


<s> TargetTriple ( TT ) , SubtargetFeatureBits ( FeatureBits ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , FP32Denormals ( false ) , FPExceptions ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSize ( <NUM_LIT> ) {


<s> GCNSubtarget :: GCNSubtarget ( const Triple & TT , StringRef GPU , StringRef FS , const GCNTargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT , getFeatureBits ( ) ) , TargetTriple ( TT ) , Gen ( SOUTHERN_ISLANDS ) , IsaVersion ( ISAVersion0_0_0 ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP64FP16Denormals ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableVGPRSpilling ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , VIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasR128A16 ( false ) , HasDLInsts ( false ) , D16PreservesUnusedBits ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) { CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering ( ) ) ) ; Legalizer . reset ( new <ISA_LIT>LegalizerInfo ( * this , TM ) ) ; RegBankInfo . reset ( new <ISA_LIT>RegisterBankInfo ( * getRegisterInfo ( ) ) ) ; InstSelector . reset ( new <ISA_LIT>InstructionSelector ( * this , * static_cast < <ISA_LIT>RegisterBankInfo * > ( RegBankInfo . get ( ) ) , TM ) ) ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT , getFeatureBits ( ) ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , DX10Clamp ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT , getFeatureBits ( ) ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , DX10Clamp ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT , getFeatureBits ( ) ) , TargetTriple ( TT ) , Gen ( SOUTHERN_ISLANDS ) , IsaVersion ( ISAVersion0_0_0 ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP64FP16Denormals ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableVGPRSpilling ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , VIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasDLInsts ( false ) , D16PreservesUnusedBits ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) { AS = <STR_LIT> ( TT ) ;

CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering ( ) ) ) ;


<s> FullFS += FS ; ParseSubtargetFeatures ( GPU , FullFS ) ; if ( getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) { FP32Denormals = false ; FP64Denormals = false ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64FP16Denormals ( false ) , FPExceptions ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , DumpCode ( false ) , FP64 ( false ) , FMA ( false ) , MIMG_R128 ( false ) , IsGCN ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , HasIntClamp ( false ) , HasVOP3PInsts ( false ) , HasMadMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasInv2PiInlineImm ( false ) , HasSDWA ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) { AS = <STR_LIT> ( TT ) ;

initializeSubtargetDependencies ( TT , GPU , FS ) ;


<s> return <STR_LIT> ( MF . getFunction ( ) , <STR_LIT> , <NUM_LIT> ) ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const GCNTargetMachine & TM ) : <ISA_LIT>Subtarget ( TT , GPU , FS , TM ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , TLInfo ( TM , * this ) { CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * getTargetLowering ( ) ) ) ; Legalizer . reset ( new <ISA_LIT>LegalizerInfo ( * this , TM ) ) ; RegBankInfo . reset ( new <ISA_LIT>RegisterBankInfo ( * getRegisterInfo ( ) ) ) ;

InstSelector . reset ( new <ISA_LIT>InstructionSelector ( * this , * static_cast < <ISA_LIT>RegisterBankInfo * > ( RegBankInfo . get ( ) ) ) ) ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT ) :

TargetTriple ( TT ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , FPExceptions ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange ( false ) , MaxWavesPerEU ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSize ( <NUM_LIT> ) {


<s> <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , TargetTriple ( TT ) , Gen ( TT . getOS ( ) == Triple :: AMDHSA ? SEA_ISLANDS : SOUTHERN_ISLANDS ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , FastDenormalF32 ( false ) , HalfRate64Ops ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , DoesNotSupportXNACK ( false ) , EnableCuMode ( false ) , TrapHandler ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , EnableDS128 ( false ) , EnablePRTStrictNull ( false ) , DumpCode ( false ) , FP64 ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , GFX8Insts ( false ) , GFX9Insts ( false ) , GFX10Insts ( false ) , GFX7GFX8GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , HasIntClamp ( false ) , HasFmaMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasScalarAtomics ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , HasDPP8 ( false ) , HasR128A16 ( false ) , HasGFX10A16 ( false ) , HasNSAEncoding ( false ) , HasDLInsts ( false ) , HasDot1Insts ( false ) , HasDot2Insts ( false ) , HasDot3Insts ( false ) , HasDot4Insts ( false ) , HasDot5Insts ( false ) , HasDot6Insts ( false ) , HasMAIInsts ( false ) , HasPkFmacF16Inst ( false ) , HasAtomicFaddInsts ( false ) , EnableSRAMECC ( false ) , DoesNotSupportSRAMECC ( false ) , HasNoSdstCMPX ( false ) , HasVscnt ( false ) , HasRegisterBanking ( false ) , HasVOP3Literal ( false ) , HasNoDataDepHazard ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , ScalarFlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , LDSMisalignedBug ( false ) , HasMFMAInlineLiteralBug ( false ) , ScalarizeGlobal ( false ) , HasVcmpxPermlaneHazard ( false ) , HasVMEMtoScalarWriteHazard ( false ) , HasSMEMtoVectorWriteHazard ( false ) , HasInstFwdPrefetchBug ( false ) , HasVcmpxExecWARHazard ( false ) , HasLdsBranchVmemWARHazard ( false ) , HasNSAtoVMEMBug ( false ) , HasOffset3fBug ( false ) , HasFlatSegmentOffsetBug ( false ) , FeatureDisable ( false ) , InstrInfo ( initializeSubtargetDependencies ( TT , GPU , FS ) ) , TLInfo ( TM , * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) {

MaxWavesPerEU = <STR_LIT> :: getMaxWavesPerEU ( this ) ;


<s> bool RequestedFlatWorkGroupSize = false ; if ( F . hasFnAttribute ( <STR_LIT> ) ) { Default . first = MinImpliedByFlatWorkGroupSize ; RequestedFlatWorkGroupSize = true ; } std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default , true ) ; if ( Requested . second && Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinWavesPerEU ( ) || Requested . first > getMaxWavesPerEU ( ) ) return Default ; if ( Requested . second > getMaxWavesPerEU ( ) ) return Default ;

if ( RequestedFlatWorkGroupSize && Requested . first < MinImpliedByFlatWorkGroupSize ) return Default ;


<s> if ( MaxPrivateElementSize == <NUM_LIT> ) MaxPrivateElementSize = <NUM_LIT> ; if ( LDSBankCount == <NUM_LIT> ) LDSBankCount = <NUM_LIT> ; if ( TT . getArch ( ) == Triple :: amdgcn ) { if ( LocalMemorySize == <NUM_LIT> ) LocalMemorySize = <NUM_LIT> ; if ( ! HasMovrel && ! HasVGPRIndexMode ) HasMovrel = true ; } if ( WavefrontSize == <NUM_LIT> ) WavefrontSize = <NUM_LIT> ; HasFminFmaxLegacy = getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ; if ( ! FS . contains ( <STR_LIT> ) && DoesNotSupportXNACK && EnableXNACK ) {


<s> SmallString < <NUM_LIT> > FullFS ( <STR_LIT> ) ; if ( isAmdHsaOS ( ) ) FullFS += <STR_LIT> ; FullFS += <STR_LIT> ; if ( FS . find_lower ( <STR_LIT> ) != StringRef :: npos ) { if ( FS . find_lower ( <STR_LIT> ) == StringRef :: npos ) FullFS += <STR_LIT> ; if ( FS . find_lower ( <STR_LIT> ) == StringRef :: npos ) FullFS += <STR_LIT> ;


<s> MachineInstr * UseI = Use -> getInstr ( ) ; if ( DefI -> isBundle ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getRegisterInfo ( ) ; auto Reg = Dep . getReg ( ) ; MachineBasicBlock :: const_instr_iterator I ( DefI -> getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator E ( DefI -> getParent ( ) -> instr_end ( ) ) ; unsigned Lat = <NUM_LIT> ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { if ( I -> modifiesRegister ( Reg , TRI ) ) Lat = InstrInfo . getInstrLatency ( getInstrItineraryData ( ) , * I ) ; else if ( Lat ) -- Lat ; } Dep . setLatency ( Lat ) ; } else if ( UseI -> isBundle ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getRegisterInfo ( ) ; auto Reg = Dep . getReg ( ) ; MachineBasicBlock :: const_instr_iterator I ( UseI -> getIterator ( ) ) ;

MachineBasicBlock :: const_instr_iterator E ( UseI -> getParent ( ) -> instr_end ( ) ) ;


<s> TargetTriple ( TT ) , GCN3Encoding ( false ) , Has16BitInsts ( false ) , HasMadMixInsts ( false ) , HasMadMacF32Insts ( false ) , HasDsSrc2Insts ( false ) , HasSDWA ( false ) , HasVOP3PInsts ( false ) , HasMulI24 ( true ) , HasMulU24 ( true ) , HasSMulHi ( false ) , HasInv2PiInlineImm ( false ) , HasFminFmaxLegacy ( true ) , EnablePromoteAlloca ( false ) , HasTrigReducedRange ( false ) , MaxWavesPerEU ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , WavefrontSizeLog2 ( <NUM_LIT> ) {


<s> DAG = static_cast < ScheduleDAGMI * > ( DAGInstrs ) ; const TargetSchedModel * TSchedModel = DAGInstrs -> getSchedModel ( ) ; if ( ! TSchedModel || DAG -> SUnits . empty ( ) ) return ; auto LastSALU = DAG -> SUnits . begin ( ) ; auto E = DAG -> SUnits . end ( ) ; SmallPtrSet < SUnit * , <NUM_LIT> > Visited ; for ( SUnit & SU : DAG -> SUnits ) { MachineInstr & MAI = * SU . getInstr ( ) ; if ( ! TII -> isMAI ( MAI ) || MAI . getOpcode ( ) == <STR_LIT> || MAI . getOpcode ( ) == <STR_LIT> ) continue ; unsigned Lat = TSchedModel -> computeInstrLatency ( & MAI ) - <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( SU ) ;

dbgs ( ) << <STR_LIT> << Lat << <STR_LIT> ) ;


<s> } } SmallPtrSet < const SUnit * , <NUM_LIT> > Visited ; while ( ! Preds . empty ( ) ) { const SUnit * SU = Preds . pop_back_val ( ) ; if ( llvm :: is_contained ( Succs , SU ) ) return false ; Visited . insert ( SU ) ; for ( const SDep & <ISA_LIT> : SU -> Preds ) if ( <ISA_LIT> . getSUnit ( ) != SU && ! Visited . count ( <ISA_LIT> . getSUnit ( ) ) ) Preds . push_back ( <ISA_LIT> . getSUnit ( ) ) ; } return true ;


<s> if ( NumSGPRs ) Occupancy = std :: min ( Occupancy , getOccupancyWithNumSGPRs ( NumSGPRs ) ) ; if ( NumVGPRs ) Occupancy = std :: min ( Occupancy , getOccupancyWithNumVGPRs ( NumVGPRs ) ) ; return Occupancy ;


<s> return std :: make_unique < FillMFMAShadowMutation > ( & InstrInfo ) ;


<s> bool GCNSubtarget :: enableFlatScratch ( ) const {

return flatScratchIsArchitected ( ) || ( EnableFlatScratch && hasFlatScratchInsts ( ) ) ;


<s> FillMFMAShadowMutation ( const <ISA_LIT>InstrInfo * tii ) : TII ( tii ) {


<s> return isAmdHsaOS ( ) ? Align ( <NUM_LIT> ) : Align ( <NUM_LIT> ) ;


<s> Align getAlignmentForImplicitArgPtr ( ) const { return isAmdHsaOS ( ) ? Align ( <NUM_LIT> ) : Align ( <NUM_LIT> ) ;


<s> unsigned Requested = <STR_LIT> ( F , <STR_LIT> , MaxNumSGPRs ) ; if ( Requested && ( Requested <= ReservedNumSGPRs ) ) Requested = <NUM_LIT> ; unsigned InputNumSGPRs = PreloadedSGPRs ; if ( Requested && Requested < InputNumSGPRs ) Requested = InputNumSGPRs ; if ( Requested && Requested > getMaxNumSGPRs ( WavesPerEU . first , false ) ) Requested = <NUM_LIT> ; if ( WavesPerEU . second && Requested && Requested < getMinNumSGPRs ( WavesPerEU . second ) ) Requested = <NUM_LIT> ; if ( Requested ) MaxNumSGPRs = Requested ; } if ( hasSGPRInitBug ( ) ) MaxNumSGPRs = <STR_LIT> :: FIXED_NUM_SGPRS_FOR_INIT_BUG ; return std :: min ( MaxNumSGPRs - ReservedNumSGPRs , MaxAddressableNumSGPRs ) ;


<s> unsigned GCNSubtarget :: getBaseMaxNumVGPRs ( const Function & F , std :: pair < unsigned , unsigned > WavesPerEU ) const { unsigned MaxNumVGPRs = getMaxNumVGPRs ( WavesPerEU . first ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) { unsigned Requested = <STR_LIT> ( F , <STR_LIT> , MaxNumVGPRs ) ; if ( hasGFX90AInsts ( ) ) Requested *= <NUM_LIT> ; if ( Requested && Requested > getMaxNumVGPRs ( WavesPerEU . first ) ) Requested = <NUM_LIT> ;


<s> if ( Requested && Requested > getMaxNumVGPRs ( WavesPerEU . first ) ) Requested = <NUM_LIT> ; if ( WavesPerEU . second && Requested && Requested < getMinNumVGPRs ( WavesPerEU . second ) ) Requested = <NUM_LIT> ;

if ( Requested ) MaxNumVGPRs = Requested ;


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) return <NUM_LIT> ; if ( HasFlatScratchInit || HasArchitectedFlatScratch ) { if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; if ( getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) return <NUM_LIT> ; } if ( isXNACKEnabled ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; if ( getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) return <NUM_LIT> ; }

if ( isXNACKEnabled ( ) ) return <NUM_LIT> ;


<s> std :: pair < unsigned , unsigned > <ISA_LIT>Subtarget :: getDefaultFlatWorkGroupSize ( CallingConv :: ID CC ) const { switch ( CC ) { case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : return std :: make_pair ( <NUM_LIT> , getWavefrontSize ( ) ) ; default :

return std :: make_pair ( <NUM_LIT> , getMaxFlatWorkGroupSize ( ) ) ;


<s> std :: pair < unsigned , unsigned > <ISA_LIT>Subtarget :: getDefaultFlatWorkGroupSize ( CallingConv :: ID CC ) const { switch ( CC ) { case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS :


<s> for ( const Argument & Arg : F . args ( ) ) { const bool IsByRef = Arg . hasByRefAttr ( ) ; Type * ArgTy = IsByRef ? Arg . getParamByRefType ( ) : Arg . getType ( ) ;

MaybeAlign Alignment = IsByRef ? Arg . getParamAlign ( ) : None ;


<s> if ( Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinFlatWorkGroupSize ( ) ) return Default ; if ( Requested . second > getMaxFlatWorkGroupSize ( ) ) return Default ;


<s> std :: pair < unsigned , unsigned > Default = getDefaultFlatWorkGroupSize ( F . getCallingConv ( ) ) ; std :: pair < unsigned , unsigned > Requested = <STR_LIT> ( F , <STR_LIT> , Default ) ; if ( Requested . first > Requested . second ) return Default ; if ( Requested . first < getMinFlatWorkGroupSize ( ) ) return Default ; if ( Requested . second > getMaxFlatWorkGroupSize ( ) ) return Default ;


<s> return <STR_LIT> ( F , <STR_LIT> , <NUM_LIT> ) ;


<s> unsigned getLocalMemorySize ( ) const { return LocalMemorySize ;


<s> unsigned <ISA_LIT>Subtarget :: getMaxLocalMemSizeWithWaveCount ( unsigned NWaves , const Function & F ) const { if ( NWaves == <NUM_LIT> ) return getLocalMemorySize ( ) ;

unsigned WorkGroupSize = getFlatWorkGroupSizes ( F ) . second ;


<s> unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU ( WorkGroupSize ) ; if ( ! WorkGroupsPerCu ) return <NUM_LIT> ;

unsigned MaxWaves = getMaxWavesPerEU ( ) ;


<s> unsigned MaxUserSGPRs = <NUM_LIT> + <NUM_LIT> + <NUM_LIT> + <NUM_LIT> + <NUM_LIT> + <NUM_LIT> + <NUM_LIT> ; unsigned MaxSystemSGPRs = <NUM_LIT> + <NUM_LIT> + <NUM_LIT> + <NUM_LIT> + <NUM_LIT> ; return MaxUserSGPRs + MaxSystemSGPRs ;


<s> unsigned GCNSubtarget :: getMaxNumSGPRs ( const Function & F ) const { return getBaseMaxNumSGPRs ( F , getWavesPerEU ( F ) , getMaxNumPreloadedSGPRs ( ) , getReservedNumSGPRs ( F ) ) ;


<s> const Function & F = MF . getFunction ( ) ; const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return getBaseMaxNumVGPRs ( F , MFI . getWavesPerEU ( ) ) ;


<s> const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return getBaseMaxNumVGPRs ( F , MFI . getWavesPerEU ( ) ) ;


<s> unsigned getMaxWavesPerEU ( ) const { return MaxWavesPerEU ;


<s> unsigned ReqdSize = getReqdWorkGroupSize ( Kernel , Dimension ) ; if ( ReqdSize != std :: numeric_limits < unsigned > :: max ( ) ) return ReqdSize - <NUM_LIT> ; return getFlatWorkGroupSizes ( Kernel ) . second - <NUM_LIT> ;


<s> if ( ReqdSize != std :: numeric_limits < unsigned > :: max ( ) ) return ReqdSize - <NUM_LIT> ; return getFlatWorkGroupSizes ( Kernel ) . second - <NUM_LIT> ;


<s> unsigned NumGroups = getLocalMemorySize ( ) / ( Bytes ? Bytes : <NUM_LIT> ) ; if ( NumGroups == <NUM_LIT> ) return <NUM_LIT> ; NumGroups = std :: min ( MaxWorkGroupsPerCu , NumGroups ) ; const unsigned MaxGroupNumWaves = ( MaxWorkGroupSize + WaveSize - <NUM_LIT> ) / WaveSize ; unsigned MaxWaves = NumGroups * MaxGroupNumWaves ; MaxWaves = std :: min ( MaxWaves , getMaxWavesPerEU ( ) ) ; assert ( MaxWaves > <NUM_LIT> && MaxWaves <= getMaxWavesPerEU ( ) && <STR_LIT> ) ; return MaxWaves ;


<s> const unsigned MaxWorkGroupsPerCu = getMaxWorkGroupsPerCU ( MaxWorkGroupSize ) ; if ( ! MaxWorkGroupsPerCu ) return <NUM_LIT> ; const unsigned WaveSize = getWavefrontSize ( ) ; unsigned NumGroups = getLocalMemorySize ( ) / ( Bytes ? Bytes : <NUM_LIT> ) ; if ( NumGroups == <NUM_LIT> ) return <NUM_LIT> ; NumGroups = std :: min ( MaxWorkGroupsPerCu , NumGroups ) ; const unsigned MaxGroupNumWaves = ( MaxWorkGroupSize + WaveSize - <NUM_LIT> ) / WaveSize ; unsigned MaxWaves = NumGroups * MaxGroupNumWaves ; MaxWaves = std :: min ( MaxWaves , getMaxWavesPerEU ( ) ) ; assert ( MaxWaves > <NUM_LIT> && MaxWaves <= getMaxWavesPerEU ( ) && <STR_LIT> ) ; return MaxWaves ;


<s> if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ;


<s> unsigned GCNSubtarget :: getOccupancyWithNumVGPRs ( unsigned VGPRs ) const { unsigned MaxWaves = getMaxWavesPerEU ( ) ; unsigned Granule = getVGPRAllocGranule ( ) ;

if ( VGPRs < Granule ) return MaxWaves ;


<s> unsigned MaxWaves = getMaxWavesPerEU ( ) ; unsigned Granule = getVGPRAllocGranule ( ) ; if ( VGPRs < Granule ) return MaxWaves ; unsigned RoundedRegs = ( ( VGPRs + Granule - <NUM_LIT> ) / Granule ) * Granule ; return std :: min ( std :: max ( getTotalNumVGPRs ( ) / RoundedRegs , <NUM_LIT> ) , MaxWaves ) ;


<s> Mutations . push_back ( std :: make_unique < FillMFMAShadowMutation > ( & InstrInfo ) ) ;


<s> void GCNSubtarget :: getPostRAMutations ( std :: vector < std :: unique_ptr < ScheduleDAGMutation >> & Mutations ) const {

Mutations . push_back ( std :: make_unique < FillMFMAShadowMutation > ( & InstrInfo ) ) ;


<s> auto Node = Kernel . getMetadata ( <STR_LIT> ) ; if ( Node && Node -> getNumOperands ( ) == <NUM_LIT> ) return mdconst :: extract < ConstantInt > ( Node -> getOperand ( Dim ) ) -> getZExtValue ( ) ; return std :: numeric_limits < unsigned > :: max ( ) ;


<s> auto Node = Kernel . getMetadata ( <STR_LIT> ) ; if ( Node && Node -> getNumOperands ( ) == <NUM_LIT> ) return mdconst :: extract < ConstantInt > ( Node -> getOperand ( Dim ) ) -> getZExtValue ( ) ;

return std :: numeric_limits < unsigned > :: max ( ) ;


<s> if ( hasFlatAddressSpace ( ) && isEntry && ! flatScratchIsArchitected ( ) && enableFlatScratch ( ) ) { FunctionHasFlatScratchInit = true ; } return getBaseReservedNumSGPRs ( FunctionHasFlatScratchInit ) ;


<s> return <NUM_LIT> << WavefrontSizeLog2 ;


<s> std :: pair < unsigned , unsigned > getWavesPerEU ( const Function & F ) const {

std :: pair < unsigned , unsigned > FlatWorkGroupSizes = getFlatWorkGroupSizes ( F ) ;


<s> bool GCNSubtarget :: hasMadF16 ( ) const { return InstrInfo . pseudoToMCOpcode ( <STR_LIT> ) != - <NUM_LIT> ;


<s> return InstrInfo . pseudoToMCOpcode ( <STR_LIT> ) != - <NUM_LIT> ;


return HasMadMacF32Insts || ! isGCN ( ) ;


<s> return HasMadMacF32Insts || ! isGCN ( ) ;


<s> } if ( MaxPrivateElementSize == <NUM_LIT> ) MaxPrivateElementSize = <NUM_LIT> ; if ( LDSBankCount == <NUM_LIT> ) LDSBankCount = <NUM_LIT> ; if ( TT . getArch ( ) == Triple :: amdgcn ) { if ( LocalMemorySize == <NUM_LIT> ) LocalMemorySize = <NUM_LIT> ; if ( ! HasMovrel && ! HasVGPRIndexMode ) HasMovrel = true ; } if ( WavefrontSizeLog2 == <NUM_LIT> ) WavefrontSizeLog2 = <NUM_LIT> ; HasFminFmaxLegacy = getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ; HasSMulHi = getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 ; TargetID . setTargetIDFromFeaturesString ( FS ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TargetID . getXnackSetting ( ) << '\n' ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TargetID . getSramEccSetting ( ) << '\n' ) ;


<s> return isAmdHsaOS ( ) || isMesaKernel ( F ) ;


<s> bool <ISA_LIT>Subtarget :: isMesaKernel ( const Function & F ) const { return isMesa3DOS ( ) && ! <STR_LIT> ( F . getCallingConv ( ) ) ;


<s> const MachineInstr * MI = SU -> getInstr ( ) ;

return MI && TII -> isSALU ( * MI ) && ! MI -> isTerminator ( ) ;


<s> const MachineInstr * MI = SU -> getInstr ( ) ; return MI && TII -> isVALU ( * MI ) ;


<s> bool isVALU ( const SUnit * SU ) const { const MachineInstr * MI = SU -> getInstr ( ) ; return MI && TII -> isVALU ( * MI ) ;


<s> SUnit * SU = Worklist . pop_back_val ( ) ; if ( ! Visited . insert ( SU ) . second ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * From ) ; dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SU ) ; dbgs ( ) << '\n' ) ; if ( SU -> addPred ( SDep ( From , SDep :: Artificial ) , false ) ) ++ Linked ; for ( SDep & <ISA_LIT> : From -> Succs ) { SUnit * SUv = <ISA_LIT> . getSUnit ( ) ; if ( SUv != From && isVALU ( SUv ) && canAddEdge ( SUv , SU ) ) SUv -> addPred ( SDep ( SU , SDep :: Artificial ) , false ) ; } for ( SDep & <ISA_LIT> : SU -> Succs ) { SUnit * Succ = <ISA_LIT> . getSUnit ( ) ; if ( Succ != SU && isSALU ( Succ ) && canAddEdge ( From , Succ ) ) Worklist . push_back ( Succ ) ; }


<s> case <STR_LIT> : IdQuery = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : Dim = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : IdQuery = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : Dim = <NUM_LIT> ; break ; default : break ; } if ( Dim <= <NUM_LIT> ) { unsigned ReqdSize = getReqdWorkGroupSize ( * Kernel , Dim ) ; if ( ReqdSize != std :: numeric_limits < unsigned > :: max ( ) ) MinSize = MaxSize = ReqdSize ; } } } if ( ! MaxSize ) return false ; if ( IdQuery ) MinSize = <NUM_LIT> ; else ++ MaxSize ; MDBuilder MDB ( I -> getContext ( ) ) ; MDNode * MaxWorkGroupSizeRange = MDB . createRange ( APInt ( <NUM_LIT> , MinSize ) , APInt ( <NUM_LIT> , MaxSize ) ) ;

I -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ;


<s> case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : return std :: make_pair ( <NUM_LIT> , getWavefrontSize ( ) ) ; default : return std :: make_pair ( <NUM_LIT> , <NUM_LIT> * getWavefrontSize ( ) ) ;


<s> if ( getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ;


<s> if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; if ( SGPRs <= <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , FS ) , TargetTriple ( TT ) , Gen ( TT . getArch ( ) == Triple :: amdgcn ? SOUTHERN_ISLANDS : <ISA_LIT> ) , IsaVersion ( ISAVersion0_0_0 ) , WavefrontSize ( <NUM_LIT> ) , LocalMemorySize ( <NUM_LIT> ) , LDSBankCount ( <NUM_LIT> ) , MaxPrivateElementSize ( <NUM_LIT> ) , FastFMAF32 ( false ) , HalfRate64Ops ( false ) , FP32Denormals ( false ) , FP64FP16Denormals ( false ) , FPExceptions ( false ) , DX10Clamp ( false ) , FlatForGlobal ( false ) , AutoWaitcntBeforeBarrier ( false ) , CodeObjectV3 ( false ) , UnalignedScratchAccess ( false ) , UnalignedBufferAccess ( false ) , HasApertureRegs ( false ) , EnableXNACK ( false ) , TrapHandler ( false ) , DebuggerInsertNops ( false ) , DebuggerReserveRegs ( false ) , DebuggerEmitPrologue ( false ) , EnableHugePrivateBuffer ( false ) , EnableVGPRSpilling ( false ) , EnablePromoteAlloca ( false ) , EnableLoadStoreOpt ( false ) , EnableUnsafeDSOffsetFolding ( false ) , Enable<ISA_LIT>Scheduler ( false ) , DumpCode ( false ) , FP64 ( false ) , FMA ( false ) , IsGCN ( false ) , GCN3Encoding ( false ) , CIInsts ( false ) , GFX9Insts ( false ) , SGPRInitBug ( false ) , HasSMemRealTime ( false ) , Has16BitInsts ( false ) , HasIntClamp ( false ) , HasVOP3PInsts ( false ) , HasMadMixInsts ( false ) , HasMovrel ( false ) , HasVGPRIndexMode ( false ) , HasScalarStores ( false ) , HasInv2PiInlineImm ( false ) , HasSDWA ( false ) , HasSDWAOmod ( false ) , HasSDWAScalar ( false ) , HasSDWASdst ( false ) , HasSDWAMac ( false ) , HasSDWAOutModsVOPC ( false ) , HasDPP ( false ) , FlatAddressSpace ( false ) , FlatInstOffsets ( false ) , FlatGlobalInsts ( false ) , FlatScratchInsts ( false ) , AddNoCarryInsts ( false ) , HasUnpackedD16VMem ( false ) , <ISA_LIT>ALUInst ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , ScalarizeGlobal ( false ) , FeatureDisable ( false ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> switch ( TargetTriple . getOS ( ) ) { case Triple :: AMDHSA : case Triple :: AMDPAL :