<s> <ISA_LIT>AlwaysInlinePass ( bool GlobalOpt = true ) : GlobalOpt ( GlobalOpt ) {


<s> <ISA_LIT>PromoteAllocaPass ( TargetMachine & TM ) : TM ( TM ) {


<s> <ISA_LIT>PromoteAllocaToVectorPass ( TargetMachine & TM ) : TM ( TM ) {


<s> <ISA_LIT>PropagateAttributesEarlyPass ( TargetMachine & TM ) : TM ( TM ) {


<s> <ISA_LIT>PropagateAttributesLatePass ( TargetMachine & TM ) : TM ( TM ) {


<s> <ISA_LIT>SimplifyLibCallsPass ( TargetMachine & TM ) : TM ( TM ) {


<s> inline bool isFlatGlobalAddrSpace ( unsigned AS ) { return AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS > <ISA_LIT>AS :: MAX_<ISA_LIT>_ADDRESS ;


<s> inline bool isFlatGlobalAddrSpace ( unsigned AS ) {

return AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS > <ISA_LIT>AS :: MAX_<ISA_LIT>_ADDRESS ;


<s> <ISA_LIT>AAResult ( <ISA_LIT>AAResult && Arg ) : AAResultBase ( std :: move ( Arg ) ) , DL ( Arg . DL ) {


<s> if ( auto * WrapperPass = P . getAnalysisIfAvailable < <ISA_LIT>AAWrapperPass > ( ) ) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ; } ) {


<s> bool doFinalization ( Module & M ) override { Result . reset ( ) ;


<s> Result . reset ( new <ISA_LIT>AAResult ( M . getDataLayout ( ) ) ) ;


<s> <ISA_LIT>AAResult run ( Function & F , AnalysisManager < Function > & AM ) {

return <ISA_LIT>AAResult ( F . getParent ( ) -> getDataLayout ( ) ) ;


<s> <ISA_LIT>AAResult ( <ISA_LIT>AAResult && Arg ) : AAResultBase ( std :: move ( Arg ) ) , DL ( Arg . DL ) , AS ( Arg . AS ) , ASAliasRules ( Arg . ASAliasRules ) {


<s> AliasResult Result = ASAliasRules . getAliasResult ( asA , asB ) ; if ( Result == NoAlias ) return Result ;

return AAResultBase :: alias ( LocA , LocB ) ;


<s> unsigned asA = LocA . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ;

unsigned asB = LocB . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ;


<s> case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : break ; } unsigned ArgNo = Arg -> getArgNo ( ) ; if ( F -> hasParamAttribute ( ArgNo , Attribute :: NoAlias ) && ( F -> hasParamAttribute ( ArgNo , Attribute :: ReadNone ) || F -> hasParamAttribute ( ArgNo , Attribute :: ReadOnly ) ) ) { return true ; } } return AAResultBase :: pointsToConstantMemory ( Loc , OrLocal ) ;


<s> case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : break ; } unsigned ArgNo = Arg -> getArgNo ( ) ; if ( F -> hasParamAttribute ( ArgNo , Attribute :: NoAlias ) && ( F -> hasParamAttribute ( ArgNo , Attribute :: ReadNone ) || F -> hasParamAttribute ( ArgNo , Attribute :: ReadOnly ) ) ) { return true ; }


<s> explicit <ISA_LIT>AAResult ( const DataLayout & DL ) : AAResultBase ( ) , DL ( DL ) {


<s> case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : break ; } unsigned ArgNo = Arg -> getArgNo ( ) ; if ( F -> hasParamAttribute ( ArgNo , Attribute :: NoAlias ) && ( F -> hasParamAttribute ( ArgNo , Attribute :: ReadNone ) || F -> hasParamAttribute ( ArgNo , Attribute :: ReadOnly ) ) ) { return true ; } }

return AAResultBase :: pointsToConstantMemory ( Loc , OrLocal ) ;


<s> initialize<ISA_LIT>AAWrapperPassPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> { ASMay , ASMay , ASNo , ASMay , ASMay , ASMay , ASMay , ASMay } , { ASMay , ASMay , ASNo , ASNo , ASMay , ASNo , ASMay , ASMay } , { ASNo , ASNo , ASMay , ASNo , ASNo , ASNo , ASNo , ASNo } , { ASMay , ASNo , ASNo , ASMay , ASNo , ASNo , ASNo , ASNo } , { ASMay , ASMay , ASNo , ASNo , ASNo , ASNo , ASMay , ASMay } , { ASMay , ASNo , ASNo , ASNo , ASNo , ASMay , ASNo , ASNo } , { ASMay , ASMay , ASNo , ASNo , ASMay , ASNo , ASNo , ASMay } , { ASMay , ASMay , ASNo , ASNo , ASMay , ASNo , ASMay , ASMay


<s> { ASMay , ASMay , ASNo , ASMay , ASMay , ASMay , ASMay , ASMay } , { ASMay , ASMay , ASNo , ASNo , ASMay , ASNo , ASMay , ASMay } , { ASNo , ASNo , ASMay , ASNo , ASNo , ASNo , ASNo , ASNo } , { ASMay , ASNo , ASNo , ASMay , ASNo , ASNo , ASNo , ASNo } , { ASMay , ASMay , ASNo , ASNo , ASNo , ASNo , ASMay , ASMay } , { ASMay , ASNo , ASNo , ASNo , ASNo , ASMay , ASNo , ASNo } , { ASMay , ASMay , ASNo , ASNo , ASMay , ASNo , ASNo , ASMay } , { ASMay , ASMay , ASNo , ASNo , ASMay , ASNo , ASMay , ASMay } } ; return ASAliasRules [ AS1 ] [ AS2 ] ;


<s> unsigned asB = LocB . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ; AliasResult Result = getAliasResult ( asA , asB ) ;

if ( Result == NoAlias ) return Result ;


<s> default : return AAResultBase :: pointsToConstantMemory ( Loc , AAQI , OrLocal ) ; case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : break ; } unsigned ArgNo = Arg -> getArgNo ( ) ; if ( F -> hasParamAttribute ( ArgNo , Attribute :: NoAlias ) && ( F -> hasParamAttribute ( ArgNo , Attribute :: ReadNone ) || F -> hasParamAttribute ( ArgNo , Attribute :: ReadOnly ) ) ) {


<s> if ( const GlobalVariable * GV = dyn_cast < GlobalVariable > ( Base ) ) { if ( GV -> isConstant ( ) ) return true ; } else if ( const Argument * Arg = dyn_cast < Argument > ( Base ) ) { const Function * F = Arg -> getParent ( ) ; switch ( F -> getCallingConv ( ) ) { default : return AAResultBase :: pointsToConstantMemory ( Loc , AAQI , OrLocal ) ; case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL :


<s> break ; } unsigned ArgNo = Arg -> getArgNo ( ) ; if ( F -> getAttributes ( ) . hasAttribute ( ArgNo + <NUM_LIT> , Attribute :: NoAlias ) && ( F -> getAttributes ( ) . hasAttribute ( ArgNo + <NUM_LIT> , Attribute :: ReadNone ) || F -> getAttributes ( ) . hasAttribute ( ArgNo + <NUM_LIT> , Attribute :: ReadOnly ) ) ) { return true ; } }


<s> bool <ISA_LIT>AAResult :: pointsToConstantMemory ( const MemoryLocation & Loc , AAQueryInfo & AAQI , bool OrLocal ) { unsigned AS = Loc . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) return true ; const Value * Base = GetUnderlyingObject ( Loc . Ptr , DL ) ; AS = Base -> getType ( ) -> getPointerAddressSpace ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) return true ; if ( const GlobalVariable * GV = dyn_cast < GlobalVariable > ( Base ) ) {

if ( GV -> isConstant ( ) ) return true ;


<s> std :: swap ( asA , asB ) ; std :: swap ( A , B ) ; } if ( asA == <ISA_LIT>AS :: FLAT_ADDRESS && ( asB == <ISA_LIT>AS :: LOCAL_ADDRESS || asB == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ) { const auto * ObjA = getUnderlyingObject ( A . Ptr -> stripPointerCastsAndInvariantGroups ( ) ) ; if ( const LoadInst * LI = dyn_cast < LoadInst > ( ObjA ) ) { if ( LI -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS ) return NoAlias ;

} else if ( const Argument * Arg = dyn_cast < Argument > ( ObjA ) ) {


<s> AliasResult <ISA_LIT>AAResult :: alias ( const MemoryLocation & LocA , const MemoryLocation & LocB , AAQueryInfo & AAQI ) { unsigned asA = LocA . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ; unsigned asB = LocB . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ; AliasResult Result = getAliasResult ( asA , asB ) ; if ( Result == NoAlias ) return Result ; MemoryLocation A = LocA ; MemoryLocation B = LocB ; if ( asA != <ISA_LIT>AS :: FLAT_ADDRESS ) { std :: swap ( asA , asB ) ; std :: swap ( A , B ) ; } if ( asA == <ISA_LIT>AS :: FLAT_ADDRESS && ( asB == <ISA_LIT>AS :: LOCAL_ADDRESS || asB == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ) { const auto * ObjA = getUnderlyingObject ( A . Ptr -> stripPointerCastsAndInvariantGroups ( ) ) ;

if ( const LoadInst * LI = dyn_cast < LoadInst > ( ObjA ) ) {


<s> explicit <ISA_LIT>AAResult ( const DataLayout & DL , Triple T ) : AAResultBase ( ) , DL ( DL ) {


<s> static AliasResult getAliasResult ( unsigned AS1 , unsigned AS2 ) { static_assert ( <ISA_LIT>AS :: MAX_<ISA_LIT>_ADDRESS <= <NUM_LIT> , <STR_LIT> ) ;


<s> unsigned AS = Loc . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) return true ; const Value * Base = getUnderlyingObject ( Loc . Ptr ) ; AS = Base -> getType ( ) -> getPointerAddressSpace ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) return true ; if ( const GlobalVariable * GV = dyn_cast < GlobalVariable > ( Base ) ) { if ( GV -> isConstant ( ) ) return true ; } else if ( const Argument * Arg = dyn_cast < Argument > ( Base ) ) { const Function * F = Arg -> getParent ( ) ; switch ( F -> getCallingConv ( ) ) { default : return AAResultBase :: pointsToConstantMemory ( Loc , AAQI , OrLocal ) ; case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES :


<s> if ( GV -> isConstant ( ) ) return true ; } else if ( const Argument * Arg = dyn_cast < Argument > ( Base ) ) { const Function * F = Arg -> getParent ( ) ; switch ( F -> getCallingConv ( ) ) { default : return AAResultBase :: pointsToConstantMemory ( Loc , AAQI , OrLocal ) ; case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : break ; } unsigned ArgNo = Arg -> getArgNo ( ) ; if ( F -> hasParamAttribute ( ArgNo , Attribute :: NoAlias ) && ( F -> hasParamAttribute ( ArgNo , Attribute :: ReadNone ) || F -> hasParamAttribute ( ArgNo , Attribute :: ReadOnly ) ) ) { return true ; }


<s> unsigned asA = LocA . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ; unsigned asB = LocB . Ptr -> getType ( ) -> getPointerAddressSpace ( ) ; AliasResult Result = getAliasResult ( asA , asB ) ; if ( Result == NoAlias ) return Result ; return AAResultBase :: alias ( LocA , LocB ) ;


<s> <ISA_LIT>AAResult ( <ISA_LIT>AAResult && Arg ) : AAResultBase ( std :: move ( Arg ) ) , DL ( Arg . DL ) , ASAliasRules ( Arg . ASAliasRules ) {


<s> <ISA_LIT>AAWrapperPass ( ) : ImmutablePass ( ID ) { initialize<ISA_LIT>AAWrapperPassPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> return new <ISA_LIT>AAWrapperPass ( ) ;


<s> return new <ISA_LIT>ExternalAAWrapper ( ) ;


<s> ImmutablePass * llvm :: create<ISA_LIT>ExternalAAWrapperPass ( ) {

return new <ISA_LIT>ExternalAAWrapper ( ) ;


<s> bool doInitialization ( Module & M ) override { Result . reset ( new <ISA_LIT>AAResult ( M . getDataLayout ( ) , Triple ( M . getTargetTriple ( ) ) ) ) ; return false ;


<s> bool invalidate ( Function & , const PreservedAnalyses & ) {


<s> if ( const GlobalVariable * GV = dyn_cast < GlobalVariable > ( Base ) ) { if ( GV -> isConstant ( ) ) return true ; } else if ( const Argument * Arg = dyn_cast < Argument > ( Base ) ) { const Function * F = Arg -> getParent ( ) ; switch ( F -> getCallingConv ( ) ) { default : return AAResultBase :: pointsToConstantMemory ( Loc , OrLocal ) ; case CallingConv :: <ISA_LIT>_LS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : break ; } unsigned ArgNo = Arg -> getArgNo ( ) ; if ( F -> hasParamAttribute ( ArgNo , Attribute :: NoAlias ) && ( F -> hasParamAttribute ( ArgNo , Attribute :: ReadNone ) || F -> hasParamAttribute ( ArgNo , Attribute :: ReadOnly ) ) ) { return true ;


<s> ModulePass ( ID ) , GlobalOpt ( GlobalOpt ) {


<s> <ISA_LIT>AlwaysInline ( bool GlobalOpt = false ) :

ModulePass ( ID ) , GlobalOpt ( GlobalOpt ) {


<s> ModulePass * llvm :: create<ISA_LIT>AlwaysInlinePass ( bool GlobalOpt ) { return new <ISA_LIT>AlwaysInline ( GlobalOpt ) ;


<s> StringRef getPassName ( ) const override {


<s> std :: vector < GlobalAlias * > AliasesToRemove ; std :: vector < Function * > FuncsToClone ; for ( GlobalAlias & A : M . aliases ( ) ) { if ( Function * F = dyn_cast < Function > ( A . getAliasee ( ) ) ) { A . replaceAllUsesWith ( F ) ; AliasesToRemove . push_back ( & A ) ; } } if ( GlobalOpt ) { for ( GlobalAlias * A : AliasesToRemove ) { A -> eraseFromParent ( ) ; } } for ( Function & F : M ) { if ( ! F . hasLocalLinkage ( ) && ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( Attribute :: NoInline ) ) FuncsToClone . push_back ( & F ) ; }


<s> static bool alwaysInlineImpl ( Module & M , bool GlobalOpt ) { std :: vector < GlobalAlias * > AliasesToRemove ; SmallPtrSet < Function * , <NUM_LIT> > FuncsToAlwaysInline ; SmallPtrSet < Function * , <NUM_LIT> > FuncsToNoInline ; for ( GlobalAlias & A : M . aliases ( ) ) { if ( Function * F = dyn_cast < Function > ( A . getAliasee ( ) ) ) { A . replaceAllUsesWith ( F ) ; AliasesToRemove . push_back ( & A ) ; } } if ( GlobalOpt ) { for ( GlobalAlias * A : AliasesToRemove ) { A -> eraseFromParent ( ) ; } } for ( GlobalVariable & GV : M . globals ( ) ) { unsigned AS = GV . getAddressSpace ( ) ; if ( ( AS == <ISA_LIT>AS :: REGION_ADDRESS ) || ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && ! <ISA_LIT>TargetMachine :: EnableLowerModuleLDS ) ) recursivelyVisitUsers ( GV , FuncsToAlwaysInline ) ; } if ( ! <ISA_LIT>TargetMachine :: EnableFunctionCalls || StressCalls ) { auto IncompatAttr = StressCalls ? Attribute :: AlwaysInline : Attribute :: NoInline ;


<s> } } for ( GlobalVariable & GV : M . globals ( ) ) { unsigned AS = GV . getAddressSpace ( ) ; if ( ( AS == <ISA_LIT>AS :: REGION_ADDRESS ) || ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && ! <ISA_LIT>TargetMachine :: EnableLowerModuleLDS ) ) recursivelyVisitUsers ( GV , FuncsToAlwaysInline ) ; } if ( ! <ISA_LIT>TargetMachine :: EnableFunctionCalls || StressCalls ) { auto IncompatAttr = StressCalls ? Attribute :: AlwaysInline : Attribute :: NoInline ; for ( Function & F : M ) { if ( ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( IncompatAttr ) ) { if ( StressCalls ) { if ( ! FuncsToAlwaysInline . count ( & F ) ) FuncsToNoInline . insert ( & F ) ; } else FuncsToAlwaysInline . insert ( & F ) ; } }


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesAll ( ) ;


<s> User * U = Stack . pop_back_val ( ) ; if ( ! Visited . insert ( U ) . second ) continue ; if ( Instruction * I = dyn_cast < Instruction > ( U ) ) { Function * F = I -> getParent ( ) -> getParent ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { F -> removeFnAttr ( Attribute :: NoInline ) ; FuncsToAlwaysInline . insert ( F ) ; Stack . push_back ( F ) ; } continue ; }

append_range ( Stack , U -> users ( ) ) ;


<s> PreservedAnalyses <ISA_LIT>AlwaysInlinePass :: run ( Module & M , ModuleAnalysisManager & AM ) { alwaysInlineImpl ( M , GlobalOpt ) ;


<s> bool <ISA_LIT>AlwaysInline :: runOnModule ( Module & M ) {

return alwaysInlineImpl ( M , GlobalOpt ) ;


<s> if ( Instruction * I = dyn_cast < Instruction > ( U ) ) { Function * F = I -> getParent ( ) -> getParent ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { FuncsToAlwaysInline . insert ( F ) ;


<s> for ( User * U : GV . users ( ) ) Stack . push_back ( U ) ; while ( ! Stack . empty ( ) ) { User * U = Stack . pop_back_val ( ) ; if ( ! Visited . insert ( U ) . second ) continue ;

if ( Instruction * I = dyn_cast < Instruction > ( U ) ) {


<s> unsigned AS = GV . getAddressSpace ( ) ; if ( AS != <ISA_LIT>AS :: LOCAL_ADDRESS && AS != <ISA_LIT>AS :: REGION_ADDRESS ) continue ; recursivelyVisitUsers ( GV , FuncsToAlwaysInline ) ; } if ( ! <ISA_LIT>TargetMachine :: EnableFunctionCalls || StressCalls ) { auto IncompatAttr = StressCalls ? Attribute :: AlwaysInline : Attribute :: NoInline ; for ( Function & F : M ) {

if ( ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( IncompatAttr ) ) {


<s> std :: vector < GlobalAlias * > AliasesToRemove ; SmallPtrSet < Function * , <NUM_LIT> > FuncsToAlwaysInline ; SmallPtrSet < Function * , <NUM_LIT> > FuncsToNoInline ; for ( GlobalAlias & A : M . aliases ( ) ) { if ( Function * F = dyn_cast < Function > ( A . getAliasee ( ) ) ) { A . replaceAllUsesWith ( F ) ; AliasesToRemove . push_back ( & A ) ; } } if ( GlobalOpt ) { for ( GlobalAlias * A : AliasesToRemove ) { A -> eraseFromParent ( ) ; } } for ( GlobalVariable & GV : M . globals ( ) ) { unsigned AS = GV . getAddressSpace ( ) ; if ( AS != <ISA_LIT>AS :: LOCAL_ADDRESS && AS != <ISA_LIT>AS :: REGION_ADDRESS ) continue ; recursivelyVisitUsers ( GV , FuncsToAlwaysInline ) ; } if ( ! <ISA_LIT>TargetMachine :: EnableFunctionCalls || StressCalls ) { auto IncompatAttr = StressCalls ? Attribute :: AlwaysInline : Attribute :: NoInline ; for ( Function & F : M ) { if ( ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( IncompatAttr ) ) { if ( StressCalls ) { if ( ! FuncsToAlwaysInline . count ( & F ) ) FuncsToNoInline . insert ( & F ) ; } else FuncsToAlwaysInline . insert ( & F ) ; } } } for ( Function * F : FuncsToAlwaysInline ) F -> addFnAttr ( Attribute :: AlwaysInline ) ; for ( Function * F : FuncsToNoInline ) F -> addFnAttr ( Attribute :: NoInline ) ;

return ! FuncsToAlwaysInline . empty ( ) || ! FuncsToNoInline . empty ( ) ;


<s> } for ( Function * F : FuncsToClone ) { ValueToValueMapTy VMap ; Function * NewFunc = CloneFunction ( F , VMap ) ; NewFunc -> setLinkage ( GlobalValue :: InternalLinkage ) ; F -> replaceAllUsesWith ( NewFunc ) ; } for ( Function & F : M ) { if ( F . hasLocalLinkage ( ) && ! F . hasFnAttribute ( Attribute :: NoInline ) ) { F . addFnAttr ( Attribute :: AlwaysInline ) ;


<s> std :: vector < GlobalAlias * > AliasesToRemove ; SmallPtrSet < Function * , <NUM_LIT> > FuncsToAlwaysInline ; SmallPtrSet < Function * , <NUM_LIT> > FuncsToNoInline ; Triple TT ( M . getTargetTriple ( ) ) ; for ( GlobalAlias & A : M . aliases ( ) ) { if ( Function * F = dyn_cast < Function > ( A . getAliasee ( ) ) ) { if ( TT . getArch ( ) == Triple :: amdgcn && A . getLinkage ( ) != GlobalValue :: InternalLinkage ) continue ; A . replaceAllUsesWith ( F ) ;


<s> for ( GlobalVariable & GV : M . globals ( ) ) { unsigned AS = GV . getAddressSpace ( ) ; if ( ( AS == <ISA_LIT>AS :: REGION_ADDRESS ) || ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && ( ! <ISA_LIT>TargetMachine :: EnableLowerModuleLDS || ! GV . hasInitializer ( ) ) ) ) recursivelyVisitUsers ( GV , FuncsToAlwaysInline ) ; } if ( ! <ISA_LIT>TargetMachine :: EnableFunctionCalls || StressCalls ) { auto IncompatAttr = StressCalls ? Attribute :: AlwaysInline : Attribute :: NoInline ; for ( Function & F : M ) { if ( ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( IncompatAttr ) ) { if ( StressCalls ) {

if ( ! FuncsToAlwaysInline . count ( & F ) ) FuncsToNoInline . insert ( & F ) ;


<s> <ISA_LIT>AlwaysInline ( bool GlobalOpt ) : ModulePass ( ID ) , GlobalOpt ( GlobalOpt ) {


<s> std :: vector < GlobalAlias * > AliasesToRemove ; SmallPtrSet < Function * , <NUM_LIT> > FuncsToAlwaysInline ; SmallPtrSet < Function * , <NUM_LIT> > FuncsToNoInline ; for ( GlobalAlias & A : M . aliases ( ) ) { if ( Function * F = dyn_cast < Function > ( A . getAliasee ( ) ) ) { A . replaceAllUsesWith ( F ) ; AliasesToRemove . push_back ( & A ) ; } } if ( GlobalOpt ) { for ( GlobalAlias * A : AliasesToRemove ) { A -> eraseFromParent ( ) ; } } for ( GlobalVariable & GV : M . globals ( ) ) { unsigned AS = GV . getAddressSpace ( ) ;

if ( ( AS == <ISA_LIT>AS :: REGION_ADDRESS ) || ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && ( ! <ISA_LIT>TargetMachine :: EnableLowerModuleLDS || ! GV . hasInitializer ( ) ) ) ) recursivelyVisitUsers ( GV , FuncsToAlwaysInline ) ;


<s> SmallPtrSet < Function * , <NUM_LIT> > FuncsToNoInline ; for ( GlobalAlias & A : M . aliases ( ) ) { if ( Function * F = dyn_cast < Function > ( A . getAliasee ( ) ) ) { A . replaceAllUsesWith ( F ) ; AliasesToRemove . push_back ( & A ) ; } } if ( GlobalOpt ) { for ( GlobalAlias * A : AliasesToRemove ) { A -> eraseFromParent ( ) ; } } for ( GlobalVariable & GV : M . globals ( ) ) { unsigned AS = GV . getAddressSpace ( ) ; if ( ( AS == <ISA_LIT>AS :: REGION_ADDRESS ) || ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && ( ! <ISA_LIT>TargetMachine :: EnableLowerModuleLDS || ! GV . hasInitializer ( ) ) ) ) recursivelyVisitUsers ( GV , FuncsToAlwaysInline ) ; } if ( ! <ISA_LIT>TargetMachine :: EnableFunctionCalls || StressCalls ) { auto IncompatAttr = StressCalls ? Attribute :: AlwaysInline : Attribute :: NoInline ; for ( Function & F : M ) { if ( ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( IncompatAttr ) ) { if ( StressCalls ) { if ( ! FuncsToAlwaysInline . count ( & F ) ) FuncsToNoInline . insert ( & F ) ; } else FuncsToAlwaysInline . insert ( & F ) ; } } } for ( Function * F : FuncsToAlwaysInline ) F -> addFnAttr ( Attribute :: AlwaysInline ) ; for ( Function * F : FuncsToNoInline ) F -> addFnAttr ( Attribute :: NoInline ) ;


<s> User * U = Stack . pop_back_val ( ) ; if ( ! Visited . insert ( U ) . second ) continue ; if ( Instruction * I = dyn_cast < Instruction > ( U ) ) { Function * F = I -> getParent ( ) -> getParent ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { F -> removeFnAttr ( Attribute :: NoInline ) ; FuncsToAlwaysInline . insert ( F ) ; Stack . push_back ( F ) ; } continue ; } for ( User * UU : U -> users ( ) ) Stack . push_back ( UU ) ; }


<s> } if ( ! <ISA_LIT>TargetMachine :: EnableFunctionCalls || StressCalls ) { auto IncompatAttr = StressCalls ? Attribute :: AlwaysInline : Attribute :: NoInline ; for ( Function & F : M ) { if ( ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( IncompatAttr ) ) { if ( StressCalls ) { if ( ! FuncsToAlwaysInline . count ( & F ) ) FuncsToNoInline . insert ( & F ) ; } else FuncsToAlwaysInline . insert ( & F ) ; } } } for ( Function * F : FuncsToAlwaysInline ) F -> addFnAttr ( Attribute :: AlwaysInline ) ; for ( Function * F : FuncsToNoInline ) F -> addFnAttr ( Attribute :: NoInline ) ; return ! FuncsToAlwaysInline . empty ( ) || ! FuncsToNoInline . empty ( ) ;


<s> A . replaceAllUsesWith ( F ) ; AliasesToRemove . push_back ( & A ) ; } } if ( GlobalOpt ) { for ( GlobalAlias * A : AliasesToRemove ) { A -> eraseFromParent ( ) ; } } for ( GlobalVariable & GV : M . globals ( ) ) { unsigned AS = GV . getType ( ) -> getAddressSpace ( ) ; if ( AS != <ISA_LIT>AS :: LOCAL_ADDRESS && AS != <ISA_LIT>AS . REGION_ADDRESS ) continue ; recursivelyVisitUsers ( GV , FuncsToAlwaysInline ) ; } if ( ! <ISA_LIT>TargetMachine :: EnableFunctionCalls || StressCalls ) { auto IncompatAttr = StressCalls ? Attribute :: AlwaysInline : Attribute :: NoInline ; for ( Function & F : M ) { if ( ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( IncompatAttr ) ) { if ( StressCalls ) { if ( ! FuncsToAlwaysInline . count ( & F ) ) FuncsToNoInline . insert ( & F ) ; } else FuncsToAlwaysInline . insert ( & F ) ; } } } for ( Function * F : FuncsToAlwaysInline ) F -> addFnAttr ( Attribute :: AlwaysInline ) ; for ( Function * F : FuncsToNoInline ) F -> addFnAttr ( Attribute :: NoInline ) ; return ! FuncsToAlwaysInline . empty ( ) || ! FuncsToNoInline . empty ( ) ;


<s> std :: vector < GlobalAlias * > AliasesToRemove ; std :: vector < Function * > FuncsToClone ; for ( GlobalAlias & A : M . aliases ( ) ) { if ( Function * F = dyn_cast < Function > ( A . getAliasee ( ) ) ) { A . replaceAllUsesWith ( F ) ; AliasesToRemove . push_back ( & A ) ; } } for ( GlobalAlias * A : AliasesToRemove ) { A -> eraseFromParent ( ) ; } for ( Function & F : M ) { if ( ! F . hasLocalLinkage ( ) && ! F . isDeclaration ( ) && ! F . use_empty ( ) && ! F . hasFnAttribute ( Attribute :: NoInline ) ) FuncsToClone . push_back ( & F ) ; }


<s> <ISA_LIT>AlwaysInline ( ) : ModulePass ( ID ) {


<s> ModulePass * llvm :: create<ISA_LIT>AlwaysInlinePass ( ) { return new <ISA_LIT>AlwaysInline ( ) ;


<s> return new <ISA_LIT>AlwaysInline ( ) ;


<s> M . getFunctionList ( ) . push_back ( NewFunc ) ; F -> replaceAllUsesWith ( NewFunc ) ; } for ( Function & F : M ) { if ( F . hasLocalLinkage ( ) && ! F . hasFnAttribute ( Attribute :: NoInline ) ) { F . addFnAttr ( Attribute :: AlwaysInline ) ; } } return false ;


<s> ValueToValueMapTy VMap ; Function * NewFunc = CloneFunction ( F , VMap , false ) ; NewFunc -> setLinkage ( GlobalValue :: InternalLinkage ) ; M . getFunctionList ( ) . push_back ( NewFunc ) ; F -> replaceAllUsesWith ( NewFunc ) ; } for ( Function & F : M ) { if ( F . hasLocalLinkage ( ) && ! F . hasFnAttribute ( Attribute :: NoInline ) ) { F . addFnAttr ( Attribute :: AlwaysInline ) ;


<s> bool <ISA_LIT>AnnotateKernelFeatures :: addAttrsForIntrinsics ( Module & M , ArrayRef < StringRef [ <NUM_LIT> ] > IntrinsicToAttr ) { bool Changed = false ; for ( const StringRef * Arr : IntrinsicToAttr ) { if ( Function * Fn = M . getFunction ( Arr [ <NUM_LIT> ] ) ) { addAttrToCallers ( Fn , Arr [ <NUM_LIT> ] ) ; Changed = true ; } } return Changed ;


<s> void <ISA_LIT>AnnotateKernelFeatures :: addAttrToCallers ( Function * Intrin , StringRef AttrName ) { SmallPtrSet < Function * , <NUM_LIT> > SeenFuncs ; for ( User * U : Intrin -> users ( ) ) { CallInst * CI = cast < CallInst > ( U ) ; Function * CallingFunction = CI -> getParent ( ) -> getParent ( ) ;

if ( SeenFuncs . insert ( CallingFunction ) . second ) CallingFunction -> addFnAttr ( AttrName ) ;


<s> for ( User * U : Intrin -> users ( ) ) { CallInst * CI = cast < CallInst > ( U ) ; Function * CallingFunction = CI -> getParent ( ) -> getParent ( ) ; if ( SeenFuncs . insert ( CallingFunction ) . second ) CallingFunction -> addFnAttr ( AttrName ) ; }


<s> <ISA_LIT>AnnotateKernelFeatures ( ) : ModulePass ( ID ) {


return new <ISA_LIT>AnnotateKernelFeatures ( ) ;


<s> return new <ISA_LIT>AnnotateKernelFeatures ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {


<s> for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited ) ) return true ;


<s> static const StringRef HSAIntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } } ; bool Changed = addAttrsForIntrinsics ( M , IntrinsicToAttr ) ; if ( TT . getOS ( ) == Triple :: AMDHSA || TT . getOS ( ) == Triple :: Mesa3D ) { Changed |= addAttrsForIntrinsics ( M , HSAIntrinsicToAttr ) ; for ( Function & F : M ) { if ( F . hasFnAttribute ( <STR_LIT> ) ) continue ; if ( hasAddrSpaceCast ( F ) ) F . addFnAttr ( <STR_LIT> ) ; }


<s> bool <ISA_LIT>AnnotateKernelFeatures :: runOnModule ( Module & M ) { Triple TT ( M . getTargetTriple ( ) ) ; static const StringRef IntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT>


<s> for ( CallGraphNode * I : SCC ) { Function * F = I -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ) continue ; Changed |= addFeatureAttributes ( * F ) ; }


<s> if ( ! F || F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ) continue ; Changed |= addFeatureAttributes ( * F ) ; }


<s> bool HaveCall = false ; bool HasIndirectCall = false ; bool IsFunc = ! <STR_LIT> ( F . getCallingConv ( ) ) ; CallingConv :: ID CC = F . getCallingConv ( ) ; bool CallingConvSupportsAllImplicits = ( CC != CallingConv :: <ISA_LIT>_Gfx ) ; if ( CallingConvSupportsAllImplicits && F . hasAddressTaken ( nullptr , true , true , true ) ) { for ( StringRef AttrName : ImplicitAttrNames ) { F . addFnAttr ( AttrName ) ; } Changed = true ; } for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { if ( isa < AllocaInst > ( I ) ) { HaveStackObjects = true ; continue ; } if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { const Function * Callee = dyn_cast < Function > ( CB -> getCalledOperand ( ) -> stripPointerCasts ( ) ) ; if ( ! Callee ) { if ( ! CB -> isInlineAsm ( ) ) { HasIndirectCall = true ; HaveCall = true ; } continue ; } <STR_LIT> IID = Callee -> getIntrinsicID ( ) ; if ( IID == <STR_LIT> ) { HaveCall = true ;

copyFeaturesToFunction ( F , * Callee , NeedQueuePtr ) ;


<s> NonKernelOnly = true ; return <STR_LIT> ; case <STR_LIT> : NonKernelOnly = true ; return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> <ISA_LIT>AnnotateKernelFeatures ( const TargetMachine * TM_ = nullptr ) : ModulePass ( ID ) , TM ( TM_ ) {


<s> } ; static const StringRef HSAIntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } } ; bool Changed = addAttrsForIntrinsics ( M , IntrinsicToAttr ) ; if ( TT . getOS ( ) == Triple :: AMDHSA || TT . getOS ( ) == Triple :: Mesa3D ) {

Changed |= addAttrsForIntrinsics ( M , HSAIntrinsicToAttr ) ;


<s> if ( IID == <STR_LIT> ) { HaveCall = true ; copyFeaturesToFunction ( F , * Callee , NeedQueuePtr ) ; Changed = true ; } else { bool NonKernelOnly = false ; if ( ! IsFunc && IID == <STR_LIT> ) { F . addFnAttr ( <STR_LIT> ) ; } else { StringRef AttrName = intrinsicToAttrName ( IID , NonKernelOnly , NeedQueuePtr ) ; if ( ! AttrName . empty ( ) && ( IsFunc || ! NonKernelOnly ) ) { F . addFnAttr ( AttrName ) ; Changed = true ; } } } } if ( NeedQueuePtr || ( ! IsFunc && HasApertureRegs ) ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( ! HasApertureRegs && castRequiresQueuePtr ( ASC ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ;


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( handleAttr ( Parent , Callee , <STR_LIT> ) ) NeedQueuePtr = true ;

for ( StringRef AttrName : AttrNames ) handleAttr ( Parent , Callee , AttrName ) ;


<s> static void copyFeaturesToFunction ( Function & Parent , const Function & Callee , bool & NeedQueuePtr ) {

static constexpr StringLiteral AttrNames [ ] = {


<s> } Function * F = I -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) ) continue ; Changed |= addFeatureAttributes ( * F ) ;


<s> bool Changed = false ; for ( CallGraphNode * I : SCC ) { if ( I -> getNumReferences ( ) ) NodeList . push_back ( I ) ;


<s> const GCNSubtarget & ST = TM -> getSubtarget < GCNSubtarget > ( F ) ; bool HasApertureRegs = ST . hasApertureRegs ( ) ; SmallPtrSet < const Constant * , <NUM_LIT> > ConstantExprVisited ; bool Changed = false ; bool NeedQueuePtr = false ; bool HaveCall = false ; bool IsFunc = ! <STR_LIT> ( F . getCallingConv ( ) ) ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { const Function * Callee = dyn_cast < Function > ( CB -> getCalledOperand ( ) -> stripPointerCasts ( ) ) ; if ( ! Callee ) { if ( ! CB -> isInlineAsm ( ) ) HaveCall = true ; continue ;


<s> continue ; } if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { const Function * Callee = dyn_cast < Function > ( CB -> getCalledOperand ( ) -> stripPointerCasts ( ) ) ; if ( ! Callee ) { if ( ! CB -> isInlineAsm ( ) ) { HasIndirectCall = true ; HaveCall = true ; } continue ; } <STR_LIT> IID = Callee -> getIntrinsicID ( ) ; if ( IID == <STR_LIT> ) { HaveCall = true ; copyFeaturesToFunction ( F , * Callee , NeedQueuePtr ) ; Changed = true ; } else { bool NonKernelOnly = false ; StringRef AttrName = intrinsicToAttrName ( IID , NonKernelOnly , NeedQueuePtr ) ; if ( ! AttrName . empty ( ) && ( IsFunc || ! NonKernelOnly ) ) { F . addFnAttr ( AttrName ) ; Changed = true ; } } } if ( NeedQueuePtr || ( ! IsFunc && HasApertureRegs ) ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( ! HasApertureRegs && castRequiresQueuePtr ( ASC ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , IsFunc , HasApertureRegs ) ) { NeedQueuePtr = true ; break ; } } }


<s> if ( handleAttr ( Parent , Callee , <STR_LIT> ) ) NeedQueuePtr = true ;

for ( StringRef AttrName : ImplicitAttrNames ) handleAttr ( Parent , Callee , AttrName ) ;


<s> return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> static bool isDSAddress ( const Constant * C ) { const GlobalValue * GV = dyn_cast < GlobalValue > ( C ) ;


<s> unsigned AS = GV -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ;


<s> bool Changed = false ; for ( auto * Node : reverse ( NodeList ) ) { Function * Caller = Node -> getFunction ( ) ; for ( auto I : * Node ) {

Function * Callee = std :: get < <NUM_LIT> > ( I ) -> getFunction ( ) ;


<s> bool <ISA_LIT>AnnotateKernelFeatures :: propagateUniformWorkGroupAttribute ( Function & Caller , Function & Callee ) { if ( ! Callee . hasExactDefinition ( ) ) { Callee . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; if ( ! Caller . hasFnAttribute ( <STR_LIT> ) ) Caller . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; return true ;


<s> } Function * F = I -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ) continue ;

Changed |= addFeatureAttributes ( * F ) ;


<s> SmallVector < const Constant * , <NUM_LIT> > Stack ; Stack . push_back ( EntryC ) ; while ( ! Stack . empty ( ) ) { const Constant * C = Stack . pop_back_val ( ) ; if ( IsFunc && isDSAddress ( C ) ) return true ; if ( const auto * CE = dyn_cast < ConstantExpr > ( C ) ) { if ( ! HasApertureRegs && visitConstantExpr ( CE ) ) return true ; } for ( const Use & U : C -> operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( ! ConstantExprVisited . insert ( OpC ) . second ) continue ; Stack . push_back ( OpC ) ; } } return false ;


<s> } if ( NeedQueuePtr || HasApertureRegs ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( castRequiresQueuePtr ( ASC , AS ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , AS ) ) { NeedQueuePtr = true ; break ; } } } } if ( NeedQueuePtr ) { F . addFnAttr ( <STR_LIT> ) ; Changed = true ; } if ( HasFlat && ! IsFunc && HaveCall ) { F . addFnAttr ( <STR_LIT> ) ; Changed = true ; } return Changed ;


<s> if ( NeedQueuePtr || HasApertureRegs ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( castRequiresQueuePtr ( ASC , AS ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , AS ) ) { NeedQueuePtr = true ; break ; } } } } if ( NeedQueuePtr ) { F . addFnAttr ( <STR_LIT> ) ; Changed = true ;


<s> static bool castRequiresQueuePtr ( const AddrSpaceCastInst * ASC , const <ISA_LIT>AS & AS ) { return castRequiresQueuePtr ( ASC -> getSrcAddressSpace ( ) , AS ) ;


<s> static void copyFeaturesToFunction ( Function & Parent , const Function & Callee , bool & NeedQueuePtr ) { static const StringRef AttrNames [ ] = { { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } } ;

if ( handleAttr ( Parent , Callee , <STR_LIT> ) ) NeedQueuePtr = true ;


<s> bool <ISA_LIT>AnnotateKernelFeatures :: doInitialization ( CallGraph & CG ) {

auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ;


<s> auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ;

if ( ! TPC ) report_fatal_error ( <STR_LIT> ) ;


<s> static StringRef intrinsicToAttrName ( <STR_LIT> ID , bool & NonKernelOnly , bool & IsQueuePtr ) { switch ( ID ) { case <STR_LIT> : NonKernelOnly = true ; return <STR_LIT> ; case <STR_LIT> : NonKernelOnly = true ; return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IsQueuePtr = true ; return <STR_LIT> ;


<s> NonKernelOnly = true ; return <STR_LIT> ; case <STR_LIT> : NonKernelOnly = true ; return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> unsigned SrcAS = CE -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerAddressSpace ( ) ; return castRequiresQueuePtr ( SrcAS , AS ) ; }


<s> if ( visitConstantExpr ( CE , AS ) ) return true ; } for ( const Use & U : C -> operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( ! ConstantExprVisited . insert ( OpC ) . second ) continue ; Stack . push_back ( OpC ) ; } } return false ;


<s> static bool castRequiresQueuePtr ( const AddrSpaceCastInst * ASC ) {

unsigned SrcAS = ASC -> getSrcAddressSpace ( ) ;


<s> unsigned SrcAS = ASC -> getSrcAddressSpace ( ) ; return SrcAS == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ;


<s> bool <ISA_LIT>AnnotateKernelFeatures :: hasAddrSpaceCast ( const Function & F ) { for ( const BasicBlock & BB : F ) { for ( const Instruction & I : BB ) { if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) {

if ( castRequiresQueuePtr ( ASC ) ) return true ;


<s> for ( const BasicBlock & BB : F ) { for ( const Instruction & I : BB ) { if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( castRequiresQueuePtr ( ASC ) ) return true ;


<s> } } ; static const StringRef HSAIntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } } ; bool Changed = addAttrsForIntrinsics ( M , IntrinsicToAttr ) ; if ( TT . getOS ( ) == Triple :: AMDHSA ) { Changed |= addAttrsForIntrinsics ( M , HSAIntrinsicToAttr ) ; for ( Function & F : M ) {

if ( F . hasFnAttribute ( <STR_LIT> ) ) continue ;


<s> if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( castRequiresQueuePtr ( ASC , AS ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , AS ) ) { NeedQueuePtr = true ; break ; } } } } if ( NeedQueuePtr ) { F . addFnAttr ( <STR_LIT> ) ; Changed = true ;


<s> if ( ! Callee ) continue ; <STR_LIT> IID = Callee -> getIntrinsicID ( ) ; if ( IID == <STR_LIT> ) { copyFeaturesToFunction ( F , * Callee , NeedQueuePtr ) ; Changed = true ; } else { StringRef AttrName = intrinsicToAttrName ( IID , NeedQueuePtr ) ; if ( ! AttrName . empty ( ) ) { F . addFnAttr ( AttrName ) ; Changed = true ; } } } if ( NeedQueuePtr || HasApertureRegs ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( castRequiresQueuePtr ( ASC , AS ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , AS ) ) { NeedQueuePtr = true ; break ; } } } }


<s> { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT> } , { <STR_LIT>


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IsQueuePtr = true ; return <STR_LIT> ; default : return <STR_LIT> ; }


<s> static const StringRef HSAIntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , {


<s> } if ( NeedQueuePtr || ( ! IsFunc && HasApertureRegs ) ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( ! HasApertureRegs && castRequiresQueuePtr ( ASC ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , IsFunc , HasApertureRegs ) ) { NeedQueuePtr = true ; break ; } } } } if ( NeedQueuePtr ) { F . addFnAttr ( <STR_LIT> ) ; Changed = true ; } if ( ! IsFunc && HaveCall ) { F . addFnAttr ( <STR_LIT> ) ; Changed = true ;


<s> const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , AS ) ) return true ; } } } return false ;


<s> for ( const Instruction & I : BB ) { if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( castRequiresQueuePtr ( ASC , AS ) ) return true ; } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , AS ) ) return true ; } } }


<s> } ; static const StringRef HSAIntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } } ; bool Changed = addAttrsForIntrinsics ( M , IntrinsicToAttr ) ; if ( TT . getOS ( ) == Triple :: AMDHSA || TT . getOS ( ) == Triple :: Mesa3D ) { Changed |= addAttrsForIntrinsics ( M , HSAIntrinsicToAttr ) ; for ( Function & F : M ) { if ( F . hasFnAttribute ( <STR_LIT> ) ) continue ; bool HasApertureRegs = TM && TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) . hasApertureRegs ( ) ; if ( ! HasApertureRegs && hasAddrSpaceCast ( F , AS ) ) F . addFnAttr ( <STR_LIT> ) ;


<s> { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } } ; bool Changed = addAttrsForIntrinsics ( M , IntrinsicToAttr ) ; if ( TT . getOS ( ) == Triple :: AMDHSA ) { Changed |= addAttrsForIntrinsics ( M , HSAIntrinsicToAttr ) ; for ( Function & F : M ) {


<s> if ( handleAttr ( Parent , Callee , <STR_LIT> ) ) NeedQueuePtr = true ; for ( StringRef AttrName : AttrNames ) handleAttr ( Parent , Callee , AttrName ) ;


<s> StringRef AttrName = intrinsicToAttrName ( IID , NonKernelOnly , NeedQueuePtr ) ; if ( ! AttrName . empty ( ) && ( IsFunc || ! NonKernelOnly ) ) { F . addFnAttr ( AttrName ) ; Changed = true ; } } } if ( NeedQueuePtr || HasApertureRegs ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( castRequiresQueuePtr ( ASC , AS ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , AS ) ) {


<s> Triple TT ( M . getTargetTriple ( ) ) ; static const StringRef IntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT>


<s> } ; static const StringRef HSAIntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } } ; bool Changed = addAttrsForIntrinsics ( M , IntrinsicToAttr ) ;

if ( TT . getOS ( ) == Triple :: AMDHSA ) Changed |= addAttrsForIntrinsics ( M , HSAIntrinsicToAttr ) ;


<s> if ( IID == <STR_LIT> ) { HaveCall = true ; copyFeaturesToFunction ( F , * Callee , NeedQueuePtr ) ; Changed = true ; } else { bool NonKernelOnly = false ; if ( ! IsFunc && IID == <STR_LIT> ) { F . addFnAttr ( <STR_LIT> ) ; } else { StringRef AttrName = intrinsicToAttrName ( IID , NonKernelOnly , NeedQueuePtr ) ; if ( ! AttrName . empty ( ) && ( IsFunc || ! NonKernelOnly ) ) { F . addFnAttr ( AttrName ) ; Changed = true ; } } } } if ( NeedQueuePtr || ( ! IsFunc && HasApertureRegs ) ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( ! HasApertureRegs && castRequiresQueuePtr ( ASC ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , IsFunc , HasApertureRegs ) ) { NeedQueuePtr = true ; break ;


<s> } } } } if ( NeedQueuePtr || ( ! IsFunc && HasApertureRegs ) ) continue ; if ( const AddrSpaceCastInst * ASC = dyn_cast < AddrSpaceCastInst > ( & I ) ) { if ( ! HasApertureRegs && castRequiresQueuePtr ( ASC ) ) { NeedQueuePtr = true ; continue ; } } for ( const Use & U : I . operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( visitConstantExprsRecursively ( OpC , ConstantExprVisited , IsFunc , HasApertureRegs ) ) { NeedQueuePtr = true ; break ; } } } } if ( NeedQueuePtr ) { F . addFnAttr ( <STR_LIT> ) ; Changed = true ; } if ( ! IsFunc && HaveCall ) { F . addFnAttr ( <STR_LIT> ) ;


<s> for ( Function & F : M ) { if ( F . hasFnAttribute ( <STR_LIT> ) ) continue ; auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; bool HasApertureRegs = TPC && TPC -> getTM < TargetMachine > ( ) . getSubtarget < <ISA_LIT>Subtarget > ( F ) . hasApertureRegs ( ) ; if ( ! HasApertureRegs && hasAddrSpaceCast ( F , AS ) ) F . addFnAttr ( <STR_LIT> ) ; } } return Changed ;


<s> Triple TT ( M . getTargetTriple ( ) ) ; AS = <STR_LIT> ( M ) ; static const StringRef IntrinsicToAttr [ ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } } ;

static const StringRef HSAIntrinsicToAttr [ ] [ <NUM_LIT> ] = {


<s> <STR_LIT> , <STR_LIT> } } ; bool Changed = addAttrsForIntrinsics ( M , IntrinsicToAttr ) ; if ( TT . getOS ( ) == Triple :: AMDHSA || TT . getOS ( ) == Triple :: Mesa3D ) { Changed |= addAttrsForIntrinsics ( M , HSAIntrinsicToAttr ) ; for ( Function & F : M ) {


<s> bool NeedQueuePtr = false ; bool HaveCall = false ; bool IsFunc = ! <STR_LIT> ( F . getCallingConv ( ) ) ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { CallSite CS ( & I ) ; if ( CS ) { Function * Callee = CS . getCalledFunction ( ) ; if ( ! Callee ) { if ( ! CS . isInlineAsm ( ) ) HaveCall = true ; continue ; } <STR_LIT> IID = Callee -> getIntrinsicID ( ) ; if ( IID == <STR_LIT> ) { HaveCall = true ; copyFeaturesToFunction ( F , * Callee , NeedQueuePtr ) ; Changed = true ; } else { bool NonKernelOnly = false ;

StringRef AttrName = intrinsicToAttrName ( IID , NonKernelOnly , NeedQueuePtr ) ;


<s> <ISA_LIT>AnnotateKernelFeatures ( ) : CallGraphSCCPass ( ID ) {


<s> static bool castRequiresQueuePtr ( const AddrSpaceCastInst * ASC ) { return castRequiresQueuePtr ( ASC -> getSrcAddressSpace ( ) ) ;


<s> return new <ISA_LIT>AnnotateKernelFeatures ( ) ;


<s> if ( ! TPC ) report_fatal_error ( <STR_LIT> ) ; TM = & TPC -> getTM < TargetMachine > ( ) ;


<s> if ( ! TPC ) report_fatal_error ( <STR_LIT> ) ;

TM = & TPC -> getTM < TargetMachine > ( ) ;


<s> CallGraphSCCPass :: getAnalysisUsage ( AU ) ;


CallGraphSCCPass :: getAnalysisUsage ( AU ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> static bool handleAttr ( Function & Parent , const Function & Callee , StringRef Name ) { if ( Callee . hasFnAttribute ( Name ) ) { Parent . addFnAttr ( Name ) ; return true ;


<s> switch ( ID ) { case <STR_LIT> : NonKernelOnly = true ; return <STR_LIT> ; case <STR_LIT> : NonKernelOnly = true ; return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ;


<s> for ( CallGraphNode * I : SCC ) { Function * F = I -> getFunction ( ) ;

if ( ! F || F -> isDeclaration ( ) ) continue ;


<s> bool <ISA_LIT>AnnotateKernelFeatures :: visitConstantExpr ( const ConstantExpr * CE ) { if ( CE -> getOpcode ( ) == Instruction :: AddrSpaceCast ) {

unsigned SrcAS = CE -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerAddressSpace ( ) ;


<s> const Constant * C = Stack . pop_back_val ( ) ; if ( const auto * CE = dyn_cast < ConstantExpr > ( C ) ) { if ( visitConstantExpr ( CE ) ) return true ; } for ( const Use & U : C -> operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ; if ( ! OpC ) continue ; if ( ! ConstantExprVisited . insert ( OpC ) . second ) continue ; Stack . push_back ( OpC ) ;


<s> while ( ! Stack . empty ( ) ) { const Constant * C = Stack . pop_back_val ( ) ; if ( const auto * CE = dyn_cast < ConstantExpr > ( C ) ) { if ( visitConstantExpr ( CE ) ) return true ; } for ( const Use & U : C -> operands ( ) ) { const auto * OpC = dyn_cast < Constant > ( U ) ;


<s> bool NotClobbered = false ; if ( PtrI ) NotClobbered = ! isClobberedInFunction ( & I ) ; else if ( isa < Argument > ( Ptr ) || isa < GlobalValue > ( Ptr ) ) { if ( isGlobalLoad ( I ) && ! isClobberedInFunction ( & I ) ) { NotClobbered = true ; if ( noClobberClones . count ( Ptr ) ) { PtrI = noClobberClones [ Ptr ] ; } else { Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( Ptr -> getType ( ) -> getPointerElementType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ; } } if ( PtrI ) {


<s> AU . addRequired < MemorySSAWrapperPass > ( ) ; AU . setPreservesAll ( ) ;


<s> bool <ISA_LIT>AnnotateUniformValues :: isClobberedInFunction ( LoadInst * Load ) {

const MemoryAccess * MA = MSSA -> getWalker ( ) -> getClobberingMemoryAccess ( Load ) ;


<s> MSSA = & getAnalysis < MemorySSAWrapperPass > ( ) . getMSSA ( ) ; isEntryFunc = <STR_LIT> ( F . getCallingConv ( ) ) ; visit ( F ) ; noClobberClones . clear ( ) ;


<s> bool <ISA_LIT>AnnotateUniformValues :: runOnFunction ( Function & F ) { if ( skipFunction ( F ) ) return false ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; MSSA = & getAnalysis < MemorySSAWrapperPass > ( ) . getMSSA ( ) ; isEntryFunc = <STR_LIT> ( F . getCallingConv ( ) ) ; visit ( F ) ; noClobberClones . clear ( ) ; return true ;


<s> if ( DA -> isUniform ( & I ) ) setUniformMetadata ( & I ) ;


<s> void <ISA_LIT>AnnotateUniformValues :: visitLoadInst ( LoadInst & I ) { Value * Ptr = I . getPointerOperand ( ) ; if ( ! DA -> isUniform ( Ptr ) ) return ; auto isGlobalLoad = [ & ] ( LoadInst & Load ) -> bool { return Load . getPointerAddressSpace ( ) == <ISA_LIT>AS :: GLOBAL_ADDRESS ; } ; Instruction * PtrI = dyn_cast < Instruction > ( Ptr ) ; if ( ! isEntryFunc ) { if ( PtrI ) setUniformMetadata ( PtrI ) ; return ; } bool NotClobbered = false ; bool GlobalLoad = isGlobalLoad ( I ) ; if ( PtrI ) NotClobbered = GlobalLoad && ! isClobberedInFunction ( & I ) ; else if ( isa < Argument > ( Ptr ) || isa < GlobalValue > ( Ptr ) ) { if ( GlobalLoad && ! isClobberedInFunction ( & I ) ) { NotClobbered = true ; if ( noClobberClones . count ( Ptr ) ) { PtrI = noClobberClones [ Ptr ] ; } else { Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( I . getType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ;


<s> AU . addRequired < DivergenceAnalysis > ( ) ; AU . setPreservesAll ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < DivergenceAnalysis > ( ) ;


<s> DA = & getAnalysis < DivergenceAnalysis > ( ) ;


<s> if ( skipFunction ( F ) ) return false ;

DA = & getAnalysis < DivergenceAnalysis > ( ) ;


<s> void <ISA_LIT>AnnotateUniformValues :: visitLoadInst ( LoadInst & I ) { Value * Ptr = I . getPointerOperand ( ) ;

if ( ! DA -> isUniform ( Ptr ) ) return ;


<s> Value * Ptr = I . getPointerOperand ( ) ; if ( ! DA -> isUniform ( Ptr ) ) return ; if ( Instruction * PtrI = dyn_cast < Instruction > ( Ptr ) ) setUniformMetadata ( PtrI ) ;


<s> AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . setPreservesAll ( ) ;


<s> Checklist . insert ( L -> block_begin ( ) , L -> block_end ( ) ) ; Start = L -> getHeader ( ) ; } DFS ( Start , Checklist ) ; for ( auto & BB : Checklist ) { BasicBlock :: iterator StartIt = ( BB == Load -> getParent ( ) ) ? BasicBlock :: iterator ( Load ) : BB -> end ( ) ;

if ( MDR -> getPointerDependencyFrom ( MemoryLocation ( Ptr ) , true , StartIt , BB , Load ) . isClobber ( ) ) return true ;


<s> DA = & getAnalysis < DivergenceAnalysis > ( ) ; MDR = & getAnalysis < MemoryDependenceWrapperPass > ( ) . getMemDep ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; isKernelFunc = F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL ;


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; isKernelFunc = F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL ; visit ( F ) ; noClobberClones . clear ( ) ; return true ;


<s> Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( Ptr -> getType ( ) -> getPointerElementType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ; } }


<s> if ( isa < Argument > ( Ptr ) || isa < GlobalValue > ( Ptr ) ) { if ( noClobberClones . count ( Ptr ) ) { PtrI = noClobberClones [ Ptr ] ; } else { Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( Ptr -> getType ( ) -> getPointerElementType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ;


<s> AMDGPUA<ISA_LIT> = <STR_LIT> ( M ) ;


<s> bool <ISA_LIT>AnnotateUniformValues :: doInitialization ( Module & M ) { AMDGPUA<ISA_LIT> = <STR_LIT> ( M ) ; return false ;


<s> bool NotClobbered = isKernelFunc && ! isClobberedInFunction ( & I ) ; Instruction * PtrI = dyn_cast < Instruction > ( Ptr ) ; if ( ! PtrI && NotClobbered && isGlobalLoad ( I ) ) { if ( isa < Argument > ( Ptr ) || isa < GlobalValue > ( Ptr ) ) { if ( noClobberClones . count ( Ptr ) ) { PtrI = noClobberClones [ Ptr ] ; } else { Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( Ptr -> getType ( ) -> getPointerElementType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ; } } if ( PtrI ) { setUniformMetadata ( PtrI ) ; if ( NotClobbered ) setNoClobberMetadata ( PtrI ) ;


<s> MDR = & getAnalysis < MemoryDependenceWrapperPass > ( ) . getMemDep ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; isEntryFunc = <STR_LIT> ( F . getCallingConv ( ) ) ; visit ( F ) ;


<s> MDR = & getAnalysis < MemoryDependenceWrapperPass > ( ) . getMemDep ( ) ;

LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ;


<s> if ( DA -> isUniform ( & I ) ) setUniformMetadata ( I . getParent ( ) -> getTerminator ( ) ) ;


<s> auto isGlobalLoad = [ & ] ( LoadInst & Load ) -> bool { return Load . getPointerAddressSpace ( ) == <ISA_LIT>AS :: GLOBAL_ADDRESS ; } ; bool NotClobbered = isEntryFunc && ! isClobberedInFunction ( & I ) ; Instruction * PtrI = dyn_cast < Instruction > ( Ptr ) ; if ( ! PtrI && NotClobbered && isGlobalLoad ( I ) ) { if ( isa < Argument > ( Ptr ) || isa < GlobalValue > ( Ptr ) ) { if ( noClobberClones . count ( Ptr ) ) { PtrI = noClobberClones [ Ptr ] ; } else { Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( Ptr -> getType ( ) -> getPointerElementType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ;


<s> } else { Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( Ptr -> getType ( ) -> getPointerElementType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ; } } if ( PtrI ) { setUniformMetadata ( PtrI ) ;

if ( NotClobbered ) setNoClobberMetadata ( PtrI ) ;


<s> AU . addRequired < AAResultsWrapperPass > ( ) ; AU . setPreservesAll ( ) ;


<s> AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addRequired < MemorySSAWrapperPass > ( ) ;

AU . addRequired < AAResultsWrapperPass > ( ) ;


<s> if ( ! Visited . insert ( MA ) . second ) continue ; if ( MSSA -> isLiveOnEntryDef ( MA ) ) continue ; if ( MemoryDef * Def = dyn_cast < MemoryDef > ( MA ) ) { if ( isReallyAClobber ( Def ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return true ; } WorkList . push_back ( Walker -> getClobberingMemoryAccess ( Def -> getDefiningAccess ( ) , Loc ) ) ; continue ; } const MemoryPhi * Phi = cast < MemoryPhi > ( MA ) ; for ( auto & Use : Phi -> incoming_values ( ) ) WorkList . push_back ( cast < MemoryAccess > ( & Use ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ;


<s> case <STR_LIT> : case <STR_LIT> : return false ; default : break ; } } const auto checkNoAlias = [ this , Load ] ( auto I ) -> bool { return I && AA -> isNoAlias ( I -> getPointerOperand ( ) , Load -> getPointerOperand ( ) ) ; } ; if ( checkNoAlias ( dyn_cast < AtomicCmpXchgInst > ( DefInst ) ) || checkNoAlias ( dyn_cast < AtomicRMWInst > ( DefInst ) ) ) return false ; return true ; } ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Load << '\n' ) ; while ( ! WorkList . empty ( ) ) { MemoryAccess * MA = WorkList . pop_back_val ( ) ;

if ( ! Visited . insert ( MA ) . second ) continue ;


<s> AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; isEntryFunc = <STR_LIT> ( F . getCallingConv ( ) ) ;


<s> PtrI = noClobberClones [ Ptr ] ; } else { Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( I . getType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ; } } if ( PtrI ) { setUniformMetadata ( PtrI ) ; if ( NotClobbered ) setNoClobberMetadata ( PtrI ) ;


<s> bool <ISA_LIT>AnnotateUniformValues :: runOnFunction ( Function & F ) { DA = & getAnalysis < DivergenceAnalysis > ( ) ; visit ( F ) ; return true ;


<s> Value * Ptr = I . getPointerOperand ( ) ; if ( ! DA -> isUniform ( Ptr ) ) return ;

if ( Instruction * PtrI = dyn_cast < Instruction > ( Ptr ) ) PtrI -> setMetadata ( <STR_LIT> , MDNode :: get ( I . getContext ( ) , { } ) ) ;


<s> void <ISA_LIT>AnnotateUniformValues :: visitLoadInst ( LoadInst & I ) { Value * Ptr = I . getPointerOperand ( ) ;


<s> bool <ISA_LIT>AnnotateUniformValues :: isClobberedInFunction ( LoadInst * Load ) { SetVector < BasicBlock * > Checklist ; BasicBlock * Start = Load -> getParent ( ) ; Checklist . insert ( Start ) ; const Value * Ptr = Load -> getPointerOperand ( ) ; const Loop * L = LI -> getLoopFor ( Start ) ; if ( L ) { const Loop * P = L ; do { L = P ; P = P -> getParentLoop ( ) ; } while ( P ) ; Checklist . insert ( L -> block_begin ( ) , L -> block_end ( ) ) ; Start = L -> getHeader ( ) ; } DFS ( Start , Checklist ) ; for ( auto & BB : Checklist ) { BasicBlock :: iterator StartIt = ( ! L && ( BB == Load -> getParent ( ) ) ) ? BasicBlock :: iterator ( Load ) : BB -> end ( ) ; if ( MDR -> getPointerDependencyFrom ( MemoryLocation ( Ptr ) , true , StartIt , BB , Load ) . isClobber ( ) ) return true ; } return false ;


<s> const Loop * L = LI -> getLoopFor ( Start ) ; if ( L ) { const Loop * P = L ; do { L = P ; P = P -> getParentLoop ( ) ; } while ( P ) ;

Checklist . insert ( L -> block_begin ( ) , L -> block_end ( ) ) ;


<s> DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; MSSA = & getAnalysis < MemorySSAWrapperPass > ( ) . getMSSA ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; isEntryFunc = <STR_LIT> ( F . getCallingConv ( ) ) ; Changed = false ; visit ( F ) ; return Changed ;


<s> void setNoClobberMetadata ( Instruction * I ) {

I -> setMetadata ( <STR_LIT> , MDNode :: get ( I -> getContext ( ) , { } ) ) ;


<s> void setUniformMetadata ( Instruction * I ) {


<s> Instruction * PtrI = dyn_cast < Instruction > ( Ptr ) ; if ( PtrI ) setUniformMetadata ( PtrI ) ; if ( ! isEntryFunc ) return ; bool GlobalLoad = I . getPointerAddressSpace ( ) == <ISA_LIT>AS :: GLOBAL_ADDRESS ;

if ( GlobalLoad && ! <STR_LIT> ( & I , MSSA , AA ) ) setNoClobberMetadata ( & I ) ;


<s> <ISA_LIT>AnnotateUniformValues ( ) : FunctionPass ( ID ) {


<s> FunctionPass * llvm :: create<ISA_LIT>AnnotateUniformValues ( ) {

return new <ISA_LIT>AnnotateUniformValues ( ) ;


<s> for ( auto I : predecessors ( Root ) ) if ( Set . insert ( I ) ) DFS ( I , Set ) ;


<s> bool <ISA_LIT>AnnotateUniformValues :: doInitialization ( Module & M ) {


<s> AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addRequired < MemoryDependenceWrapperPass > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addRequired < MemoryDependenceWrapperPass > ( ) ;

AU . addRequired < LoopInfoWrapperPass > ( ) ;


<s> if ( L ) { const Loop * P = L ; do { L = P ; P = P -> getParentLoop ( ) ; } while ( P ) ; Checklist . insert ( L -> block_begin ( ) , L -> block_end ( ) ) ; Start = L -> getHeader ( ) ; }


<s> DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; MDR = & getAnalysis < MemoryDependenceWrapperPass > ( ) . getMemDep ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ;

isKernelFunc = F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL ;


<s> MDR = & getAnalysis < MemoryDependenceWrapperPass > ( ) . getMemDep ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; isKernelFunc = F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL ; visit ( F ) ; noClobberClones . clear ( ) ;


<s> I -> setMetadata ( <STR_LIT> , MDNode :: get ( I -> getContext ( ) , { } ) ) ;


<s> static void setNoClobberMetadata ( Instruction * I ) {


<s> static void setUniformMetadata ( Instruction * I ) { I -> setMetadata ( <STR_LIT> , MDNode :: get ( I -> getContext ( ) , { } ) ) ;


<s> void <ISA_LIT>AnnotateUniformValues :: visitBranchInst ( BranchInst & I ) { if ( I . isUnconditional ( ) ) return ;


<s> if ( I . isUnconditional ( ) ) return ; Value * Cond = I . getCondition ( ) ;

if ( ! DA -> isUniform ( Cond ) ) return ;


<s> Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( Ptr -> getType ( ) -> getPointerElementType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ; } }


<s> if ( isa < Argument > ( Ptr ) || isa < GlobalValue > ( Ptr ) ) { if ( noClobberClones . count ( Ptr ) ) { PtrI = noClobberClones [ Ptr ] ; } else { Function * F = I . getParent ( ) -> getParent ( ) ; Value * Idx = Constant :: getIntegerValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; PtrI = GetElementPtrInst :: Create ( Ptr -> getType ( ) -> getPointerElementType ( ) , Ptr , ArrayRef < Value * > ( Idx ) , Twine ( <STR_LIT> ) , F -> getEntryBlock ( ) . getFirstNonPHI ( ) ) ; } I . replaceUsesOfWith ( Ptr , PtrI ) ; } } if ( PtrI ) { setUniformMetadata ( PtrI ) ;


<s> for ( const auto & FI : ArgInfoMap ) { OS << <STR_LIT> << FI . first -> getName ( ) << '\n' << <STR_LIT> << FI . second . PrivateSegmentBuffer << <STR_LIT> << FI . second . DispatchPtr << <STR_LIT> << FI . second . QueuePtr << <STR_LIT> << FI . second . KernargSegmentPtr << <STR_LIT> << FI . second . DispatchID << <STR_LIT> << FI . second . FlatScratchInit << <STR_LIT> << FI . second . PrivateSegmentSize << <STR_LIT> << FI . second . GridWorkGroupCountX << <STR_LIT> << FI . second . GridWorkGroupCountY << <STR_LIT> << FI . second . GridWorkGroupCountZ << <STR_LIT> << FI . second . WorkGroupIDX << <STR_LIT> << FI . second . WorkGroupIDY << <STR_LIT> << FI . second . WorkGroupIDZ << <STR_LIT> << FI . second . WorkGroupInfo << <STR_LIT> << FI . second . PrivateSegmentWaveByteOffset << <STR_LIT> << FI . second . ImplicitBufferPtr << <STR_LIT> << FI . second . ImplicitArgPtr << <STR_LIT> << FI . second . WorkItemIDX << <STR_LIT> << FI . second . WorkItemIDY << <STR_LIT> << FI . second . WorkItemIDZ << '\n' ;


<s> OS << <STR_LIT> << FI . first -> getName ( ) << '\n' << <STR_LIT> << FI . second . PrivateSegmentBuffer << <STR_LIT> << FI . second . DispatchPtr << <STR_LIT> << FI . second . QueuePtr << <STR_LIT> << FI . second . KernargSegmentPtr << <STR_LIT> << FI . second . DispatchID << <STR_LIT> << FI . second . FlatScratchInit << <STR_LIT> << FI . second . PrivateSegmentSize << <STR_LIT> << FI . second . GridWorkGroupCountX << <STR_LIT> << FI . second . GridWorkGroupCountY << <STR_LIT> << FI . second . GridWorkGroupCountZ << <STR_LIT> << FI . second . WorkGroupIDX << <STR_LIT> << FI . second . WorkGroupIDY << <STR_LIT> << FI . second . WorkGroupIDZ << <STR_LIT> << FI . second . WorkGroupInfo << <STR_LIT> << FI . second . PrivateSegmentWaveByteOffset << <STR_LIT> << FI . second . ImplicitBufferPtr << <STR_LIT> << FI . second . ImplicitArgPtr << <STR_LIT> << FI . second . WorkItemIDX << <STR_LIT> << FI . second . WorkItemIDY << <STR_LIT> << FI . second . WorkItemIDZ << '\n' ;


<s> ArgDescriptor ( unsigned Val = <NUM_LIT> , bool IsStack = false , bool IsSet = false ) : Register ( Val ) , IsStack ( IsStack ) , IsSet ( IsSet ) {


<s> static ArgDescriptor createRegister ( unsigned Reg ) { return ArgDescriptor ( Reg , false , true ) ;


<s> return ArgDescriptor ( Reg , false , true ) ;


<s> return ArgDescriptor ( Reg , true , true ) ;


<s> const <ISA_LIT>FunctionArgInfo & lookupFuncArgInfo ( const Function & F ) const { auto I = ArgInfoMap . find ( & F ) ; if ( I == ArgInfoMap . end ( ) ) { assert ( F . isDeclaration ( ) ) ;


<s> const <ISA_LIT>FunctionArgInfo & lookupFuncArgInfo ( const Function & F ) const {


<s> ArgDescriptor ( unsigned Val = <NUM_LIT> , unsigned Mask = ~ <NUM_LIT> , bool IsStack = false , bool IsSet = false ) : Reg ( Val ) , Mask ( Mask ) , IsStack ( IsStack ) , IsSet ( IsSet ) {


<s> static ArgDescriptor createArg ( const ArgDescriptor & Arg , unsigned Mask ) {

return ArgDescriptor ( Arg . Reg , Mask , Arg . IsStack , Arg . IsSet ) ;


<s> return ArgDescriptor ( Reg , Mask , false , true ) ;


<s> static ArgDescriptor createStack ( Register Reg , unsigned Mask = ~ <NUM_LIT> ) {

return ArgDescriptor ( Reg , Mask , true , true ) ;


<s> return ArgDescriptor ( Reg , Mask , true , true ) ;


<s> void ArgDescriptor :: print ( raw_ostream & OS , const TargetRegisterInfo * TRI ) const { if ( ! isSet ( ) ) {


<s> if ( isRegister ( ) ) OS << <STR_LIT> << printReg ( getRegister ( ) , TRI ) << '\n' ;

else OS << <STR_LIT> << getStackOffset ( ) << '\n' ;


<s> if ( isRegister ( ) ) OS << <STR_LIT> << PrintReg ( getRegister ( ) , TRI ) << '\n' ; else OS << <STR_LIT> << getStackOffset ( ) << '\n' ;


<s> return std :: make_tuple ( ImplicitBufferPtr ? & ImplicitBufferPtr : nullptr , & <STR_LIT> , LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( WorkGroupIDX ? & WorkGroupIDX : nullptr , & <STR_LIT> , LLT :: scalar ( <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( WorkGroupIDY ? & WorkGroupIDY : nullptr , & <STR_LIT> , LLT :: scalar ( <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( WorkGroupIDZ ? & WorkGroupIDZ : nullptr , & <STR_LIT> , LLT :: scalar ( <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( PrivateSegmentWaveByteOffset ? & PrivateSegmentWaveByteOffset : nullptr , & <STR_LIT> , LLT :: scalar ( <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( KernargSegmentPtr ? & KernargSegmentPtr : nullptr , & <STR_LIT> , LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( ImplicitArgPtr ? & ImplicitArgPtr : nullptr , & <STR_LIT> , LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( DispatchID ? & DispatchID : nullptr , & <STR_LIT> , LLT :: scalar ( <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( FlatScratchInit ? & FlatScratchInit : nullptr , & <STR_LIT> , LLT :: scalar ( <NUM_LIT> ) ) ; case <STR_LIT> : return std :: make_tuple ( DispatchPtr ? & DispatchPtr : nullptr , & <STR_LIT> , LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ;


<s> if ( I == ArgInfoMap . end ( ) ) return FixedABIFunctionInfo ;


<s> const <ISA_LIT>FunctionArgInfo & <ISA_LIT>ArgumentUsageInfo :: lookupFuncArgInfo ( const Function & F ) const {


<s> AI . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . DispatchPtr = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . QueuePtr = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . ImplicitArgPtr = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . DispatchID = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . WorkGroupIDX = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . WorkGroupIDY = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . WorkGroupIDZ = ArgDescriptor :: createRegister ( <STR_LIT> ) ; const unsigned Mask = <NUM_LIT> ;

AI . WorkItemIDX = ArgDescriptor :: createRegister ( <STR_LIT> , Mask ) ;


<s> AI . ImplicitArgPtr = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . DispatchID = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . WorkGroupIDX = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . WorkGroupIDY = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . WorkGroupIDZ = ArgDescriptor :: createRegister ( <STR_LIT> ) ;


<s> AI . ImplicitArgPtr = <STR_LIT> ; AI . DispatchID = <STR_LIT> ; AI . WorkGroupIDX = <STR_LIT> ; AI . WorkGroupIDY = <STR_LIT> ; AI . WorkGroupIDZ = <STR_LIT> ; const unsigned Mask = <NUM_LIT> ; AI . WorkItemIDX = ArgDescriptor :: createRegister ( <STR_LIT> , Mask ) ; AI . WorkItemIDY = ArgDescriptor :: createRegister ( <STR_LIT> , Mask << <NUM_LIT> ) ; AI . WorkItemIDZ = ArgDescriptor :: createRegister ( <STR_LIT> , Mask << <NUM_LIT> ) ; return AI ;


<s> <ISA_LIT>ArgumentUsageInfo ( ) : ImmutablePass ( ID ) {


<s> constexpr ArgDescriptor ( unsigned Val = <NUM_LIT> , unsigned Mask = ~ <NUM_LIT> , bool IsStack = false , bool IsSet = false ) : Reg ( Val ) , Mask ( Mask ) , IsStack ( IsStack ) , IsSet ( IsSet ) {


<s> static constexpr ArgDescriptor createArg ( const ArgDescriptor & Arg , unsigned Mask ) { return ArgDescriptor ( Arg . Reg , Mask , Arg . IsStack , Arg . IsSet ) ;


<s> ArgInfoMap . clear ( ) ; return false ;


<s> bool <ISA_LIT>ArgumentUsageInfo :: doInitialization ( Module & M ) {


<s> <ISA_LIT>FunctionArgInfo AI ; AI . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . DispatchPtr = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . QueuePtr = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . ImplicitArgPtr = ArgDescriptor :: createRegister ( <STR_LIT> ) ; AI . DispatchID = ArgDescriptor :: createRegister ( <STR_LIT> ) ;

AI . WorkGroupIDX = ArgDescriptor :: createRegister ( <STR_LIT> ) ;


<s> assert ( IsStack ) ; return StackOffset ;


<s> bool isSet ( ) const { return IsSet ;


<s> const <ISA_LIT>FunctionArgInfo & <ISA_LIT>ArgumentUsageInfo :: lookupFuncArgInfo ( const Function & F ) const { auto I = ArgInfoMap . find ( & F ) ; if ( I == ArgInfoMap . end ( ) ) { if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) return FixedABIFunctionInfo ; assert ( F . isDeclaration ( ) ) ; return ExternFunctionInfo ; } return I -> second ;


<s> OS << <STR_LIT> << FI . first -> getName ( ) << '\n' << <STR_LIT> << FI . second . PrivateSegmentBuffer << <STR_LIT> << FI . second . DispatchPtr << <STR_LIT> << FI . second . QueuePtr << <STR_LIT> << FI . second . KernargSegmentPtr << <STR_LIT> << FI . second . DispatchID << <STR_LIT> << FI . second . FlatScratchInit << <STR_LIT> << FI . second . PrivateSegmentSize << <STR_LIT> << FI . second . WorkGroupIDX << <STR_LIT> << FI . second . WorkGroupIDY << <STR_LIT> << FI . second . WorkGroupIDZ << <STR_LIT> << FI . second . WorkGroupInfo << <STR_LIT> << FI . second . PrivateSegmentWaveByteOffset << <STR_LIT> << FI . second . ImplicitBufferPtr << <STR_LIT> << FI . second . ImplicitArgPtr << <STR_LIT> << FI . second . WorkItemIDX << <STR_LIT> << FI . second . WorkItemIDY << <STR_LIT> << FI . second . WorkItemIDZ << '\n' ; }


<s> for ( const auto & FI : ArgInfoMap ) { OS << <STR_LIT> << FI . first -> getName ( ) << '\n' << <STR_LIT> << FI . second . PrivateSegmentBuffer << <STR_LIT> << FI . second . DispatchPtr << <STR_LIT> << FI . second . QueuePtr << <STR_LIT> << FI . second . KernargSegmentPtr << <STR_LIT> << FI . second . DispatchID << <STR_LIT> << FI . second . FlatScratchInit << <STR_LIT> << FI . second . PrivateSegmentSize << <STR_LIT> << FI . second . WorkGroupIDX << <STR_LIT> << FI . second . WorkGroupIDY << <STR_LIT> << FI . second . WorkGroupIDZ << <STR_LIT> << FI . second . WorkGroupInfo << <STR_LIT> << FI . second . PrivateSegmentWaveByteOffset << <STR_LIT> << FI . second . ImplicitBufferPtr << <STR_LIT> << FI . second . ImplicitArgPtr << <STR_LIT> << FI . second . WorkItemIDX << <STR_LIT> << FI . second . WorkItemIDY << <STR_LIT> << FI . second . WorkItemIDZ << '\n' ;


<s> void setFuncArgInfo ( const Function & F , const <ISA_LIT>FunctionArgInfo & ArgInfo ) { ArgInfoMap [ & F ] = ArgInfo ;


<s> <ISA_LIT>AsmBackend ( const Target & T ) : MCAsmBackend ( <STR_LIT> ) {


<s> Value = adjustFixupValue ( Fixup , Value , & Asm . getContext ( ) ) ; if ( ! Value ) return ; MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ; unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ;

uint32_t Offset = Fixup . getOffset ( ) ;


<s> Value = adjustFixupValue ( Fixup , Value , & Asm . getContext ( ) ) ; if ( ! Value ) return ; MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ; unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; uint32_t Offset = Fixup . getOffset ( ) ;

assert ( Offset + NumBytes <= Data . size ( ) && <STR_LIT> ) ;


<s> return new ELF<ISA_LIT>AsmBackend ( T , STI . getTargetTriple ( ) , getHsaAbiVersion ( & STI ) . getValueOr ( <NUM_LIT> ) ) ;


<s> std :: unique_ptr < MCObjectTargetWriter > createObjectTargetWriter ( ) const override { return create<ISA_LIT>ELFObjectWriter ( Is64Bit , OSABI , HasRelocationAddend , ABIVersion ) ;


<s> const static MCFixupKindInfo Infos [ <STR_LIT> ] = {

{ <STR_LIT> , <NUM_LIT> , <NUM_LIT> , MCFixupKindInfo :: FKF_IsPCRel } , } ;


<s> { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , MCFixupKindInfo :: FKF_IsPCRel } , } ;

if ( Kind < FirstTargetFixupKind ) return MCAsmBackend :: getFixupKindInfo ( Kind ) ;


<s> return <NUM_LIT> ; case FK_SecRel_1 : case FK_Data_1 : return <NUM_LIT> ; case FK_SecRel_2 : case FK_Data_2 : return <NUM_LIT> ; case FK_SecRel_4 : case FK_Data_4 : case FK_PCRel_4 : return <NUM_LIT> ; case FK_SecRel_8 : case FK_Data_8 :


<s> unsigned <ISA_LIT>AsmBackend :: getMinimumNopSize ( ) const {


<s> if ( ! STI . getFeatureBits ( ) [ <STR_LIT> ] ) return false ; if ( <STR_LIT> ( Inst . getOpcode ( ) ) >= <NUM_LIT> ) return true ;


<s> unsigned RelaxedOpcode = <STR_LIT> ( Inst . getOpcode ( ) ) ;


<s> Res . addOperand ( Inst . getOperand ( <NUM_LIT> ) ) ; Inst = std :: move ( Res ) ;


<s> bool <ISA_LIT>AsmBackend :: writeNopData ( raw_ostream & OS , uint64_t Count , const MCSubtargetInfo * STI ) const { OS . write_zeros ( Count % <NUM_LIT> ) ; Count /= <NUM_LIT> ; const uint32_t Encoded_S_NOP_0 = <NUM_LIT> ;

for ( uint64_t I = <NUM_LIT> ; I != Count ; ++ I ) <STR_LIT> :: write < uint32_t > ( OS , Encoded_S_NOP_0 , Endian ) ;


<s> Count /= <NUM_LIT> ; const uint32_t Encoded_S_NOP_0 = <NUM_LIT> ; for ( uint64_t I = <NUM_LIT> ; I != Count ; ++ I ) <STR_LIT> :: write < uint32_t > ( OS , Encoded_S_NOP_0 , Endian ) ; return true ;


<s> <ISA_LIT>AsmBackend ( const Target & T ) : MCAsmBackend ( ) {


<s> <ISA_LIT>MCObjectWriter ( raw_pwrite_stream & OS ) : MCObjectWriter ( OS , true ) {


<s> MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ; unsigned Offset = Fixup . getOffset ( ) ; assert ( Offset + NumBytes <= DataSize && <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i != NumBytes ; ++ i ) Data [ Offset + i ] |= uint8_t ( ( Value >> ( i * <NUM_LIT> ) ) & <NUM_LIT> ) ; }


<s> uint32_t * Dst = ( uint32_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = Value + <NUM_LIT> ; break ; } default : { unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; if ( ! Value ) return ; MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ; unsigned Offset = Fixup . getOffset ( ) ; assert ( Offset + NumBytes <= DataSize && <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i != NumBytes ; ++ i ) Data [ Offset + i ] |= uint8_t ( ( Value >> ( i * <NUM_LIT> ) ) & <NUM_LIT> ) ; } }


<s> return new ELF<ISA_LIT>AsmBackend ( T , TargetTriple . getArch ( ) == Triple :: amdgcn ) ;


<s> return create<ISA_LIT>ELFObjectWriter ( Is64Bit , OS ) ;


<s> MCObjectWriter * createObjectWriter ( raw_pwrite_stream & OS ) const override {

return create<ISA_LIT>ELFObjectWriter ( Is64Bit , OS ) ;


<s> ELF<ISA_LIT>AsmBackend ( const Target & T , bool Is64Bit ) : <ISA_LIT>AsmBackend ( T ) , Is64Bit ( Is64Bit ) {


<s> <ISA_LIT>AsmBackend ( T ) , Is64Bit ( Is64Bit ) {


<s> void executePostLayoutBinding ( MCAssembler & Asm , const MCAsmLayout & Layout ) override {


<s> bool fixupNeedsRelaxation ( const MCFixup & Fixup , uint64_t Value , const MCRelaxableFragment * DF , const MCAsmLayout & Layout ) const override { return false ;


<s> const static MCFixupKindInfo Infos [ <STR_LIT> ] = { { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , MCFixupKindInfo :: FKF_IsPCRel } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , MCFixupKindInfo :: FKF_IsPCRel } } ; if ( Kind < FirstTargetFixupKind ) return MCAsmBackend :: getFixupKindInfo ( Kind ) ;

return Infos [ Kind - FirstTargetFixupKind ] ;


<s> case FK_Data_4 : return <NUM_LIT> ; case FK_Data_8 : return <NUM_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> bool mayNeedRelaxation ( const MCInst & Inst ) const override { return false ;


<s> bool <ISA_LIT>AsmBackend :: writeNopData ( uint64_t Count , MCObjectWriter * OW ) const { OW -> WriteZeros ( Count ) ;


<s> void <ISA_LIT>MCObjectWriter :: writeObject ( MCAssembler & Asm , const MCAsmLayout & Layout ) { for ( MCAssembler :: iterator I = Asm . begin ( ) , E = Asm . end ( ) ; I != E ; ++ I ) { Asm . writeSectionData ( & * I , Layout ) ; }


<s> case <STR_LIT> : { int64_t BrImm = ( SignedValue - <NUM_LIT> ) / <NUM_LIT> ; if ( Ctx && ! isInt < <NUM_LIT> > ( BrImm ) ) Ctx -> reportError ( Fixup . getLoc ( ) , <STR_LIT> ) ; return BrImm ; } case FK_Data_1 : case FK_Data_2 :


<s> unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; uint32_t Offset = Fixup . getOffset ( ) ; assert ( Offset + NumBytes <= Data . size ( ) && <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i != NumBytes ; ++ i ) Data [ Offset + i ] |= static_cast < uint8_t > ( ( Value >> ( i * <NUM_LIT> ) ) & <NUM_LIT> ) ;


<s> void <ISA_LIT>AsmBackend :: applyFixup ( const MCAssembler & Asm , const MCFixup & Fixup , const MCValue & Target , MutableArrayRef < char > Data , uint64_t Value , bool IsResolved ) const {

Value = adjustFixupValue ( Fixup , Value , & Asm . getContext ( ) ) ;


<s> MCAsmBackend * llvm :: create<ISA_LIT>AsmBackend ( const Target & T , const MCRegisterInfo & MRI , const Triple & TT , StringRef CPU , const MCTargetOptions & Options ) { return new ELF<ISA_LIT>AsmBackend ( T , TT ) ;


return create<ISA_LIT>ELFObjectWriter ( Is64Bit , HasRelocationAddend , OS ) ;


<s> ELF<ISA_LIT>AsmBackend ( const Target & T , const Triple & TT ) : <ISA_LIT>AsmBackend ( T ) , Is64Bit ( TT . getArch ( ) == Triple :: amdgcn ) , HasRelocationAddend ( TT . getOS ( ) == Triple :: AMDHSA ) {


<s> ELF<ISA_LIT>AsmBackend ( const Target & T , const Triple & TT ) :

<ISA_LIT>AsmBackend ( T ) , Is64Bit ( TT . getArch ( ) == Triple :: amdgcn ) , HasRelocationAddend ( TT . getOS ( ) == Triple :: AMDHSA ) {


<s> Count /= <NUM_LIT> ; const uint32_t Encoded_S_NOP_0 = <NUM_LIT> ; for ( uint64_t I = <NUM_LIT> ; I != Count ; ++ I ) OW -> write32 ( Encoded_S_NOP_0 ) ;


<s> bool <ISA_LIT>AsmBackend :: writeNopData ( uint64_t Count , MCObjectWriter * OW ) const { OW -> WriteZeros ( Count % <NUM_LIT> ) ; Count /= <NUM_LIT> ; const uint32_t Encoded_S_NOP_0 = <NUM_LIT> ;

for ( uint64_t I = <NUM_LIT> ; I != Count ; ++ I ) OW -> write32 ( Encoded_S_NOP_0 ) ;


<s> return new ELF<ISA_LIT>AsmBackend ( T , STI . getTargetTriple ( ) ) ;


<s> MCAsmBackend * llvm :: create<ISA_LIT>AsmBackend ( const Target & T , const MCSubtargetInfo & STI , const MCRegisterInfo & MRI , const MCTargetOptions & Options ) {

return new ELF<ISA_LIT>AsmBackend ( T , STI . getTargetTriple ( ) ) ;


<s> case Triple :: AMDPAL : OSABI = ELF :: ELFOSABI_<ISA_LIT>_PAL ; break ; case Triple :: Mesa3D : OSABI = ELF :: ELFOSABI_<ISA_LIT>_MESA3D ; break ; default :


<s> if ( ! Value ) return ; MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ; unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ;


<s> unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; uint32_t Offset = Fixup . getOffset ( ) ; assert ( Offset + NumBytes <= DataSize && <STR_LIT> ) ;

for ( unsigned i = <NUM_LIT> ; i != NumBytes ; ++ i ) Data [ Offset + i ] |= static_cast < uint8_t > ( ( Value >> ( i * <NUM_LIT> ) ) & <NUM_LIT> ) ;


<s> return new ELF<ISA_LIT>AsmBackend ( T , STI . getTargetTriple ( ) , IsaInfo :: hasCodeObjectV3 ( & STI ) ? <NUM_LIT> : <NUM_LIT> ) ;


<s> MCInst Res ; unsigned RelaxedOpcode = <STR_LIT> ( Inst . getOpcode ( ) ) ; Res . setOpcode ( RelaxedOpcode ) ;

Res . addOperand ( Inst . getOperand ( <NUM_LIT> ) ) ;


<s> std :: unique_ptr < MCObjectWriter > createObjectWriter ( raw_pwrite_stream & OS ) const override { return create<ISA_LIT>ELFObjectWriter ( Is64Bit , OSABI , HasRelocationAddend , OS ) ;


<s> std :: unique_ptr < MCObjectWriter > createObjectWriter ( raw_pwrite_stream & OS ) const override {

return create<ISA_LIT>ELFObjectWriter ( Is64Bit , OSABI , HasRelocationAddend , OS ) ;


<s> if ( ! isInt < <NUM_LIT> > ( BrImm ) ) report_fatal_error ( <STR_LIT> ) ; uint16_t * Dst = ( uint16_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = BrImm ; break ; } case <STR_LIT> : { uint32_t * Dst = ( uint32_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = Value + <NUM_LIT> ; break ; } default : { unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; if ( ! Value ) return ; MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ;

unsigned Offset = Fixup . getOffset ( ) ;


<s> int64_t SignedValue = static_cast < int64_t > ( Value ) ; switch ( Fixup . getKind ( ) ) { case <STR_LIT> : { int64_t BrImm = ( SignedValue - <NUM_LIT> ) / <NUM_LIT> ; if ( Ctx && ! isInt < <NUM_LIT> > ( BrImm ) ) Ctx -> reportError ( Fixup . getLoc ( ) , <STR_LIT> ) ; return BrImm ; } case FK_Data_1 : case FK_Data_2 : case FK_Data_4 : case FK_Data_8 : case FK_PCRel_4 : case FK_SecRel_4 :


<s> int64_t BrImm = ( SignedValue - <NUM_LIT> ) / <NUM_LIT> ;

if ( Ctx && ! isInt < <NUM_LIT> > ( BrImm ) ) Ctx -> reportError ( Fixup . getLoc ( ) , <STR_LIT> ) ;


<s> if ( ! Value ) return ; MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ; unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; uint32_t Offset = Fixup . getOffset ( ) ; assert ( Offset + NumBytes <= Data . size ( ) && <STR_LIT> ) ;


<s> if ( ! IsResolved && Fixup . getValue ( ) -> evaluateAsValue ( Res , Layout ) && Res . isAbsolute ( ) ) { Value = Res . getConstant ( ) ;


<s> bool mayNeedRelaxation ( const MCInst & Inst , const MCSubtargetInfo & STI ) const override { return false ;


<s> if ( ! isInt < <NUM_LIT> > ( BrImm ) ) report_fatal_error ( <STR_LIT> ) ; uint16_t * Dst = ( uint16_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = BrImm ; break ; } default : { unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; if ( ! Value ) return ; MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ;


<s> return <NUM_LIT> ; case FK_SecRel_4 : case FK_Data_4 : case FK_PCRel_4 : return <NUM_LIT> ; case FK_SecRel_8 : case FK_Data_8 :


<s> void relaxInstruction ( const MCInst & Inst , const MCSubtargetInfo & STI , MCInst & Res ) const override {


<s> switch ( ( unsigned ) Fixup . getKind ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : { uint16_t * Dst = ( uint16_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = ( Value - <NUM_LIT> ) / <NUM_LIT> ; break ; } case <STR_LIT> : { uint32_t * Dst = ( uint32_t * ) ( Data + Fixup . getOffset ( ) ) ;


<s> Res . setOpcode ( RelaxedOpcode ) ; Res . addOperand ( Inst . getOperand ( <NUM_LIT> ) ) ; return ;


<s> Value <<= Info . TargetOffset ; unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; uint32_t Offset = Fixup . getOffset ( ) ; assert ( Offset + NumBytes <= Data . size ( ) && <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i != NumBytes ; ++ i ) Data [ Offset + i ] |= static_cast < uint8_t > ( ( Value >> ( i * <NUM_LIT> ) ) & <NUM_LIT> ) ;


<s> void <ISA_LIT>AsmBackend :: applyFixup ( const MCFixup & Fixup , char * Data , unsigned DataSize , uint64_t Value , bool IsPCRel ) const { switch ( ( unsigned ) Fixup . getKind ( ) ) { case <STR_LIT> : { uint16_t * Dst = ( uint16_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = ( Value - <NUM_LIT> ) / <NUM_LIT> ; break ; }


<s> uint16_t * Dst = ( uint16_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = ( Value - <NUM_LIT> ) / <NUM_LIT> ; break ; } case <STR_LIT> : { uint32_t * Dst = ( uint32_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = Value ; break ; } case <STR_LIT> : { uint32_t * Dst = ( uint32_t * ) ( Data + Fixup . getOffset ( ) ) ; * Dst = Value + <NUM_LIT> ; break ; } default : { unsigned NumBytes = getFixupKindNumBytes ( Fixup . getKind ( ) ) ; if ( ! Value ) return ; MCFixupKindInfo Info = getFixupKindInfo ( Fixup . getKind ( ) ) ; Value <<= Info . TargetOffset ; unsigned Offset = Fixup . getOffset ( ) ;

assert ( Offset + NumBytes <= DataSize && <STR_LIT> ) ;


<s> <STR_LIT> , <NUM_LIT> , <NUM_LIT> , MCFixupKindInfo :: FKF_IsPCRel } } ; if ( Kind < FirstTargetFixupKind ) return MCAsmBackend :: getFixupKindInfo ( Kind ) ;


<s> <STR_LIT> , <NUM_LIT> , <NUM_LIT> , MCFixupKindInfo :: FKF_IsPCRel } } ;


<s> Inst . addOperand ( MCOperand :: createReg ( <STR_LIT> ( getReg ( ) , * Reg . STI ) ) ) ;


<s> void addRegOrImmOperands ( MCInst & Inst , unsigned N ) const {

if ( isReg ( ) ) addRegOperands ( Inst , N ) ;


<s> void addRegWithInputModsOperands ( MCInst & Inst , unsigned N ) const {

Inst . addOperand ( MCOperand :: createImm ( Reg . Modifiers == - <NUM_LIT> ? <NUM_LIT> : Reg . Modifiers ) ) ;


<s> if ( isImm ( ) ) addImmOperands ( Inst , N ) ; else {


<s> <ISA_LIT>Operand ( enum KindTy K ) : MCParsedAsmOperand ( ) , Kind ( K ) {


<s> uint64_t TSFlags = MII . get ( Inst . getOpcode ( ) ) . TSFlags ;

if ( ( getForcedEncodingSize ( ) == <NUM_LIT> && ( TSFlags & <STR_LIT> ) ) || ( getForcedEncodingSize ( ) == <NUM_LIT> && ! ( TSFlags & <STR_LIT> ) ) ) return Match_InvalidOperand ;


<s> } if ( Div == <NUM_LIT> ) { Div = <NUM_LIT> ; return true ;


<s> if ( Div == <NUM_LIT> ) { Div = <NUM_LIT> ; return true ;


<s> if ( Mul != <NUM_LIT> && Mul != <NUM_LIT> && Mul != <NUM_LIT> ) return false ;


<s> static bool ConvertOmodMul ( int64_t & Mul ) {

if ( Mul != <NUM_LIT> && Mul != <NUM_LIT> && Mul != <NUM_LIT> ) return false ;


<s> static std :: unique_ptr < <ISA_LIT>Operand > CreateExpr ( const class MCExpr * Expr , SMLoc S ) { auto Op = llvm :: make_unique < <ISA_LIT>Operand > ( Expression ) ; Op -> Expr = Expr ; Op -> StartLoc = S ;


<s> Op -> Expr = Expr ; Op -> StartLoc = S ; Op -> EndLoc = S ;


<s> Op -> Imm . Val = Val ; Op -> Imm . IsFPImm = IsFPImm ; Op -> Imm . Type = Type ;


<s> Op -> Reg . STI = STI ; Op -> Reg . Modifiers = - <NUM_LIT> ; Op -> Reg . IsForcedVOP3 = ForceVOP3 ; Op -> StartLoc = S ;


<s> auto Res = llvm :: make_unique < <ISA_LIT>Operand > ( Token ) ; Res -> Tok . Data = Str . data ( ) ; Res -> Tok . Length = Str . size ( ) ;


<s> static std :: unique_ptr < <ISA_LIT>Operand > CreateToken ( StringRef Str , SMLoc Loc , bool HasExplicitEncodingSize = true ) {

auto Res = llvm :: make_unique < <ISA_LIT>Operand > ( Token ) ;


<s> if ( Op . isReg ( ) ) { Op . addRegOperands ( Inst , <NUM_LIT> ) ; continue ; } if ( Op . isToken ( ) && Op . getToken ( ) == <STR_LIT> ) { GDSOnly = true ; continue ; } OptionalIdx [ Op . getImmTy ( ) ] = i ; } unsigned OffsetIdx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyOffset ] ; ( ( <ISA_LIT>Operand & ) * Operands [ OffsetIdx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ; if ( ! GDSOnly ) { unsigned GD<ISA_LIT>dx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyGDS ] ;

( ( <ISA_LIT>Operand & ) * Operands [ GD<ISA_LIT>dx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ;


<s> void <ISA_LIT>AsmParser :: cvtDSOffset01 ( MCInst & Inst , const OperandVector & Operands ) { std :: map < enum <ISA_LIT>Operand :: ImmTy , unsigned > OptionalIdx ; for ( unsigned i = <NUM_LIT> , e = Operands . size ( ) ; i != e ; ++ i ) { <ISA_LIT>Operand & Op = ( ( <ISA_LIT>Operand & ) * Operands [ i ] ) ; if ( Op . isReg ( ) ) { Op . addRegOperands ( Inst , <NUM_LIT> ) ; continue ; } OptionalIdx [ Op . getImmTy ( ) ] = i ; } unsigned Offset0Idx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyDSOffset0 ] ; unsigned Offset1Idx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyDSOffset1 ] ; unsigned GD<ISA_LIT>dx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyGDS ] ; ( ( <ISA_LIT>Operand & ) * Operands [ Offset0Idx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ; ( ( <ISA_LIT>Operand & ) * Operands [ Offset1Idx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ; ( ( <ISA_LIT>Operand & ) * Operands [ GD<ISA_LIT>dx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ;

Inst . addOperand ( MCOperand :: createReg ( <STR_LIT> ) ) ;


<s> if ( OptionalIdx . count ( <ISA_LIT>Operand :: ImmTyGLC ) ) { unsigned GLCIdx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyGLC ] ; ( ( <ISA_LIT>Operand & ) * Operands [ GLCIdx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ; } unsigned SLCIdx = OptionalIdx [ <ISA_LIT>Operand :: ImmTySLC ] ; unsigned TFEIdx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyTFE ] ;

( ( <ISA_LIT>Operand & ) * Operands [ SLCIdx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ;


<s> Op . addRegOperands ( Inst , <NUM_LIT> ) ; continue ; } if ( Op . isImm ( ) && Op . getImmTy ( ) == <ISA_LIT>Operand :: ImmTyNone ) { Op . addImmOperands ( Inst , <NUM_LIT> ) ; continue ; } if ( Op . isToken ( ) ) { continue ; } assert ( Op . isImm ( ) ) ; OptionalIdx [ Op . getImmTy ( ) ] = i ; } assert ( OptionalIdx . size ( ) == <NUM_LIT> ) ; unsigned OffsetIdx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyOffset ] ; unsigned GLCIdx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyGLC ] ; unsigned SLCIdx = OptionalIdx [ <ISA_LIT>Operand :: ImmTySLC ] ; unsigned TFEIdx = OptionalIdx [ <ISA_LIT>Operand :: ImmTyTFE ] ; ( ( <ISA_LIT>Operand & ) * Operands [ OffsetIdx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ;

( ( <ISA_LIT>Operand & ) * Operands [ GLCIdx ] ) . addImmOperands ( Inst , <NUM_LIT> ) ;


<s> bool defaultTokenHasSuffix ( ) const { StringRef Token ( Tok . Data , Tok . Length ) ; return Token . endswith ( <STR_LIT> ) || Token . endswith ( <STR_LIT> ) ;


<s> unsigned getForcedEncodingSize ( ) const {


<s> assert ( isImm ( ) ) ; return Imm . Type ;


<s> switch ( RegWidth ) { default : return - <NUM_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> return StringSwitch < unsigned > ( RegName ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Default ( <NUM_LIT> ) ;


<s> MCTargetStreamer & TS = * getParser ( ) . getStreamer ( ) . getTargetStreamer ( ) ; return static_cast < <ISA_LIT>TargetStreamer & > ( TS ) ;


<s> MCTargetStreamer & TS = * getParser ( ) . getStreamer ( ) . getTargetStreamer ( ) ;

return static_cast < <ISA_LIT>TargetStreamer & > ( TS ) ;


<s> return StringRef ( Tok . Data , Tok . Length ) ;


<s> return Reg . Modifiers != - <NUM_LIT> ;


<s> bool hasSGPR102_SGPR103 ( ) const { return ! isVI ( ) ;


<s> bool <ISA_LIT>Operand :: isDSOffset ( ) const { return isImm ( ) && isUInt < <NUM_LIT> > ( getImm ( ) ) ;


<s> bool isDSOffset0 ( ) const { assert ( isImm ( ) ) ;


<s> return isImm ( ) && isUInt < <NUM_LIT> > ( getImm ( ) ) ;


<s> bool isDSOffset1 ( ) const { assert ( isImm ( ) ) ; return Imm . Type == ImmTyDSOffset1 ;


<s> bool isForcedVOP3 ( ) const { return ForcedEncodingSize == <NUM_LIT> ;


<s> return ForcedEncodingSize == <NUM_LIT> ;


<s> return isImm ( ) && ( ( Imm . Val <= <NUM_LIT> && Imm . Val >= - <NUM_LIT> ) || ( F == <NUM_LIT> || F == <NUM_LIT> || F == - <NUM_LIT> || F == <NUM_LIT> || F == - <NUM_LIT> || F == <NUM_LIT> || F == - <NUM_LIT> || F == <NUM_LIT> || F == - <NUM_LIT> ) ) ;


<s> bool <ISA_LIT>Operand :: isMubufOffset ( ) const {

return isImm ( ) && isUInt < <NUM_LIT> > ( getImm ( ) ) ;


<s> bool <ISA_LIT>Operand :: isMubufOffset ( ) const { return isImm ( ) && isUInt < <NUM_LIT> > ( getImm ( ) ) ;


return Kind == Register && Reg . Modifiers == - <NUM_LIT> ;


<s> return Reg . TRI -> getRegClass ( RCID ) . contains ( getReg ( ) ) ;


<s> bool isSCSrc32 ( ) const { return isInlineImm ( ) || ( isReg ( ) && isRegClass ( <STR_LIT> ) ) ;


return ( isReg ( ) && isRegClass ( <STR_LIT> ) ) || isInlineImm ( ) ;


<s> return ( isReg ( ) && isRegClass ( <STR_LIT> ) ) || isInlineImm ( ) ;


<s> bool is<ISA_LIT> ( ) const { return <STR_LIT> ( getSTI ( ) ) ;


<s> return isImm ( ) && ! isUInt < <NUM_LIT> > ( getImm ( ) ) && isUInt < <NUM_LIT> > ( getImm ( ) ) ;


<s> bool <ISA_LIT>Operand :: isSMRDLiteralOffset ( ) const { return isImm ( ) && ! isUInt < <NUM_LIT> > ( getImm ( ) ) && isUInt < <NUM_LIT> > ( getImm ( ) ) ;


<s> bool <ISA_LIT>Operand :: isSMRDOffset ( ) const {


<s> bool isSoppBrTarget ( ) const { return isExpr ( ) || isImm ( ) ;


<s> return isImm ( ) || ( isReg ( ) && isRegClass ( <STR_LIT> ) ) ;


<s> bool isSSrc64 ( ) const { return isImm ( ) || isInlineImm ( ) || ( isReg ( ) && isRegClass ( <STR_LIT> ) ) ;


<s> bool isToken ( ) const override { return Kind == Token ;


return isInlineImm ( ) || ( isReg ( ) && isRegClass ( <STR_LIT> ) ) ;


<s> bool isVCSrc64 ( ) const { return isInlineImm ( ) || ( isReg ( ) && isRegClass ( <STR_LIT> ) ) ;


<s> if ( DstOp . isReg ( ) && DstOp . isRegClass ( <STR_LIT> ) ) return true ; if ( Operands . size ( ) >= <NUM_LIT> ) return true ; if ( Operands . size ( ) > <NUM_LIT> ) { <ISA_LIT>Operand & Src1Op = ( ( <ISA_LIT>Operand & ) * Operands [ <NUM_LIT> ] ) ; if ( Src1Op . getReg ( ) && ( Src1Op . isRegClass ( <STR_LIT> ) || Src1Op . isRegClass ( <STR_LIT> ) ) ) return true ; } return false ;


return isImm ( ) || ( isReg ( ) && isRegClass ( <STR_LIT> ) ) ;


<s> void LLVMInitialize<ISA_LIT>AsmParser ( ) { RegisterMCAsmParser < <ISA_LIT>AsmParser > A ( The<ISA_LIT>Target ) ; RegisterMCAsmParser < <ISA_LIT>AsmParser > B ( TheGCNTarget ) ;