<s> std :: pair < Value * , Value * > <ISA_LIT>PromoteAllocaImpl :: getLocalSizeYZ ( IRBuilder < > & Builder ) { Function & F = * Builder . GetInsertBlock ( ) -> getParent ( ) ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( TM , F ) ; if ( ! IsAMDHSA ) { Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; F . removeFnAttr


<s> Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; F . removeFnAttr ( <STR_LIT> ) ; DispatchPtr -> addDereferenceableRetAttr ( <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; ST . makeLIDRangeMetadata (


<s> if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast_or_null < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast_or_null < AtomicCmpXchgInst > ( UseInst ) ) { if ( CAS -> isVolatile ( ) ) return false ; } if ( ICmpInst * ICmp = dyn_cast < ICmpInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , ICmp , <NUM_LIT> , <NUM_LIT> ) ) return false ; WorkList . push_back ( ICmp ) ; } if ( ! User -> getType ( ) -> isPointerTy ( ) ) continue ; if ( GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( UseInst ) ) { if ( ! GEP -> isInBounds ( ) ) return false ; } if ( SelectInst * <ISA_LIT> = dyn_cast < SelectInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , <ISA_LIT> , <NUM_LIT> , <NUM_LIT> ) ) return false ; } if ( PHINode * Phi = dyn_cast < PHINode > ( UseInst ) ) { switch ( Phi -> getNumIncomingValues ( ) ) { case <NUM_LIT> : break ; case <NUM_LIT> : if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , Phi , <NUM_LIT> , <NUM_LIT> ) ) return false ; break ; default : return


<s> if ( isa < ConstantPointerNull > ( Phi -> getIncomingValue ( I ) ) ) Phi -> setIncomingValue ( I , ConstantPointerNull :: get ( NewTy ) ) ; } } continue ; } IntrinsicInst * Intr = dyn_cast < IntrinsicInst > ( Call ) ; if ( ! Intr ) { std :: vector < Type * > ArgTypes ; for ( unsigned ArgIdx = <NUM_LIT> , ArgEnd = Call -> getNumArgOperands ( ) ; ArgIdx != ArgEnd ; ++ ArgIdx ) { ArgTypes . push_back ( Call -> getArgOperand ( ArgIdx ) -> getType ( ) ) ; } Function * F = Call -> getCalledFunction ( ) ; FunctionType * NewType = FunctionType :: get ( Call -> getType ( ) , ArgTypes , F -> isVarArg ( ) ) ; Constant * C = Mod -> getOrInsertFunction ( ( F -> getName ( ) + <STR_LIT> ) . str ( ) , NewType , F -> getAttributes ( ) ) ; Function * NewF = cast < Function > ( C ) ; Call -> setCalledFunction ( NewF ) ; continue ; } Builder . SetInsertPoint ( Intr ) ; switch ( Intr -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : Intr -> eraseFromParent ( ) ; continue ; case <STR_LIT> : { MemCpyInst * MemCpy = cast < MemCpyInst > ( Intr ) ; Builder . CreateMemCpy ( MemCpy -> getRawDest ( ) , MemCpy -> getRawSource ( ) , MemCpy -> getLength ( ) , MemCpy -> getAlignment ( ) , MemCpy -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : { MemMoveInst * MemMove = cast < MemMoveInst > ( Intr ) ; Builder . CreateMemMove ( MemMove -> getRawDest ( ) , MemMove -> getRawSource ( ) , MemMove -> getLength ( ) , MemMove -> getAlignment ( ) , MemMove -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : { MemSetInst * MemSet = cast < MemSetInst > ( Intr ) ; Builder . CreateMemSet ( MemSet -> getRawDest ( ) , MemSet -> getValue ( ) , MemSet -> getLength ( ) , MemSet -> getAlignment ( ) , MemSet -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Intr -> eraseFromParent ( ) ; continue ; case <STR_LIT> : { Value * Src = Intr


<s> bool <ISA_LIT>PromoteAlloca :: runOnFunction ( Function & F ) { FunctionType * FTy = F . getFunctionType ( ) ; LocalMemAvailable = ST . getLocalMemorySize ( ) ; for ( unsigned i = <NUM_LIT> , e = FTy -> getNumParams ( ) ; i != e ; ++ i ) { Type * ParamTy = FTy -> getParamType ( i ) ; if ( ParamTy -> isPointerTy ( ) && ParamTy -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalMemAvailable = <NUM_LIT> ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; break ; } } if ( LocalMemAvailable > <NUM_LIT> ) { for ( Module :: global_iterator I = Mod -> global_begin ( ) , E = Mod -> global_end ( ) ; I != E ; ++ I ) { GlobalVariable * GV = & * I ; if ( GV -> getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) { Instruction * Use = dyn_cast < Instruction > ( * U ) ; if ( ! Use ) continue


<s> Value * TIdY = Builder . CreateCall ( ReadTIDIGY , { } ) ; Value * TIdZ = Builder . CreateCall ( ReadTIDIGZ , { } ) ; Value * Tmp0 = Builder . CreateMul ( TCntY , TCntZ ) ; Tmp0 = Builder . CreateMul ( Tmp0 , TIdX ) ; Value * Tmp1 = Builder . CreateMul ( TIdY , TCntZ ) ; Value * TID = Builder . CreateAdd ( Tmp0 , Tmp1 ) ; TID = Builder . CreateAdd ( TID , TIdZ ) ; std :: vector < Value * > Indices ; Indices . push_back ( Constant :: getNullValue ( Type :: getInt32Ty ( Mod -> getContext ( ) ) ) ) ; Indices . push_back ( TID ) ; Value * Offset = Builder . CreateGEP ( GVTy , GV , Indices ) ; I . mutateType ( Offset -> getType ( ) ) ; I . replaceAllUsesWith ( Offset ) ; I . eraseFromParent ( ) ; for ( std :: vector < Value * > :: iterator i = WorkList . begin ( ) , e = WorkList . end ( ) ; i != e ; ++ i ) { Value * V = * i ; CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { Type * EltTy = V -> getType ( ) -> getPointerElementType ( ) ; PointerType * NewTy = PointerType :: get ( EltTy , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; if ( isa < AddrSpaceCastInst > ( V ) ) continue ; V -> mutateType ( NewTy ) ; continue ; } IntrinsicInst * Intr = dyn_cast < IntrinsicInst > ( Call ) ; if ( ! Intr ) { std :: vector < Type * > ArgTypes ; for ( unsigned ArgIdx = <NUM_LIT> , ArgEnd = Call -> getNumArgOperands ( ) ; ArgIdx != ArgEnd ; ++ ArgIdx ) { ArgTypes . push_back ( Call -> getArgOperand ( ArgIdx ) -> getType ( ) ) ; } Function * F = Call -> getCalledFunction ( ) ; FunctionType * NewType = FunctionType :: get ( Call -> getType ( ) , ArgTypes , F -> isVarArg ( ) ) ; Constant * C = Mod -> getOrInsertFunction ( ( F -> getName ( ) + <STR_LIT> ) . str ( ) , NewType , F -> getAttributes ( ) ) ; Function * NewF = cast < Function > ( C ) ; Call -> setCalledFunction ( NewF ) ; continue ; } Builder . SetInsertPoint ( Intr ) ; switch ( Intr -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : Intr -> eraseFromParent ( ) ; continue ; case <STR_LIT> : { MemCpyInst * MemCpy = cast < MemCpyInst >


<s> return VectorType :: get ( ArrayTy -> getArrayElementType ( ) , ArrayTy ->


<s> } } if ( LocalMemAvailable > <NUM_LIT> ) { for ( Module :: global_iterator I = Mod -> global_begin ( ) , E = Mod -> global_end ( ) ; I != E ; ++ I ) { GlobalVariable * GV = I ; PointerType * GVTy = GV -> getType ( ) ; if ( GVTy -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV


<s> if ( GVTy -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) { Instruction * Use = dyn_cast < Instruction > ( * U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) .

getTypeAllocSize ( GVTy -> getElementType ( ) ) ;


<s> Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext (


<s> const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( * TM , F ) ; if ( ! IsAMDHSA ) { Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 (


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = dyn_cast < VectorType > ( AllocaTy ) ; if ( ! VectorTy ) VectorTy = arrayTypeToVecType ( cast < ArrayType > ( AllocaTy ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AT ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction ::


<s> return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; ST . makeLIDRangeMetadata ( LoadZU )


<s> <STR_LIT> IntrID = <STR_LIT> :: not_intrinsic ; switch ( N ) { case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ; CallInst * CI = Builder


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } LocalMemLimit = ST . getLocalMemorySize ( ) ; if ( LocalMemLimit == <NUM_LIT> ) return false ; const DataLayout & DL = Mod -> getDataLayout ( ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != AS . LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment


<s> IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ; IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ; const <ISA_LIT>CommonSubtarget & ST = <ISA_LIT>CommonSubtarget :: get ( * TM , F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; AS = <STR_LIT> ( * F . getParent ( ) ) ; bool SufficientLDS = hasSufficientLocalMem ( F ) ; bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ;


<s> if ( CallInst * CI = dyn_cast < CallInst > ( User ) ) { if ( ! isCallPromotable ( CI ) ) return false ; WorkList . push_back ( User ) ; continue ; } Instruction * UseInst = dyn_cast < Instruction > ( User ) ; if ( UseInst && UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ; if ( StoreInst * <ISA_LIT> = dyn_cast_or_null < StoreInst > ( UseInst ) ) { if ( <ISA_LIT> -> getPointerOperand ( ) != Val )


<s> MaxWorkGroupSizeRange = MDB . createRange ( APInt ( <NUM_LIT> , <NUM_LIT> ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; const Triple & TT = TM -> getTargetTriple ( ) ; IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ; IsAMDHSA = TT . getOS ( ) == Triple


<s> if ( ! CI -> getCalledFunction ( ) ) return false ; IntrinsicInst * II = dyn_cast < IntrinsicInst > ( CI ) ; if ( ! II ) return false ; switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT>


<s> IntrinsicInst * II = dyn_cast < IntrinsicInst > ( CI ) ; if ( ! II ) return false ; switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> return false ; } } const <ISA_LIT>Subtarget & ST = TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) ; LocalMemAvailable = ST . getLocalMemorySize ( ) ; if ( LocalMemAvailable == <NUM_LIT> ) return false ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Use & U : GV . uses ( ) ) { Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) . getTypeAllocSize ( GV . getValueType ( ) ) ; } } LocalMemAvailable = std :: max ( <NUM_LIT> , LocalMemAvailable ) ; DEBUG ( dbgs ( ) << LocalMemAvailable <<


<s> return false ; } } const <ISA_LIT>Subtarget & ST = TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) ; LocalMemAvailable = ST . getLocalMemorySize ( ) ; if ( LocalMemAvailable == <NUM_LIT> ) return false ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Use & U : GV . uses ( ) ) { Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) . getTypeAllocSize ( GV . getValueType ( ) ) ; } } LocalMemAvailable = std :: max ( <NUM_LIT> , LocalMemAvailable ) ; DEBUG ( dbgs ( ) << LocalMemAvailable << <STR_LIT> ) ; BasicBlock & EntryBB = * F .


<s> std :: map < GetElementPtrInst * , Value * > GEPVectorIdx ; std :: vector < Value * > WorkList ; for ( User * AllocaUser : Alloca -> users ( ) ) { GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( std :: vector < Value * > :: iterator I = WorkList . begin ( ) , E = WorkList . end ( ) ; I != E ; ++ I ) { Instruction * Inst = cast < Instruction > ( * I ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VectorTy -> getPointerTo ( <NUM_LIT> ) ) ; Value * VecValue = Builder . CreateLoad ( BitCast


<s> if ( UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ; if ( LoadInst * LI = dyn_cast < LoadInst > ( UseInst ) ) { if ( LI -> isVolatile ( ) ) return false ; continue ; } if ( StoreInst * <ISA_LIT> = dyn_cast < StoreInst > ( UseInst ) ) { if ( <ISA_LIT> -> isVolatile ( ) ) return false ; if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast < AtomicCmpXchgInst > ( UseInst ) ) { if ( CAS -> isVolatile ( ) ) return false ; } if ( ICmpInst * ICmp = dyn_cast < ICmpInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , ICmp , <NUM_LIT> , <NUM_LIT> )


<s> if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast < AtomicCmpXchgInst > ( UseInst ) ) { if ( CAS -> isVolatile ( ) ) return false ; } if ( ICmpInst * ICmp = dyn_cast < ICmpInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , ICmp , <NUM_LIT> , <NUM_LIT> ) ) return false ; WorkList . push_back ( ICmp ) ; } if ( ! User -> getType ( ) -> isPointerTy ( ) ) continue ; if ( GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( UseInst ) ) { if ( ! GEP -> isInBounds ( ) ) return false ; } if ( SelectInst * <ISA_LIT> = dyn_cast < SelectInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , <ISA_LIT> , <NUM_LIT> , <NUM_LIT> ) ) return false ; } if ( PHINode * Phi = dyn_cast < PHINode > ( UseInst )


<s> CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( )


<s> DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext ::


<s> Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; ST . makeLIDRangeMetadata ( LoadZU


<s> IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( * TM , F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; AS = <STR_LIT> ( * F . getParent ( ) ) ; bool SufficientLDS = hasSufficientLocalMem ( F ) ; bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ; if ( AI ) Changed |= handleAlloca ( * AI ,


<s> if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , <ISA_LIT> -> getValueOperand ( ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : break ; default : llvm_unreachable ( <STR_LIT> )


<s> <ISA_LIT>PromoteAlloca ( const TargetMachine * TM_ = nullptr ) : FunctionPass ( ID ) , TM ( TM_ ) ,

Mod ( nullptr ) , LocalMemAvailable ( <NUM_LIT> ) {


<s> } Instruction * UseInst = dyn_cast < Instruction > ( User ) ; if ( UseInst && UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ; if ( StoreInst * <ISA_LIT> = dyn_cast_or_null < StoreInst > ( UseInst ) ) { if ( <ISA_LIT> -> isVolatile ( ) ) return false ; if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( LoadInst * LI = dyn_cast_or_null < LoadInst > ( UseInst ) ) { if ( LI -> isVolatile ( ) ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast_or_null < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast_or_null <


<s> bool <ISA_LIT>PromoteAlloca :: doInitialization ( Module & M ) { if ( ! TM ) return


<s> bool <ISA_LIT>PromoteAlloca :: doInitialization ( Module


<s> DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; break ; } } if ( LocalMemAvailable > <NUM_LIT> ) { for ( Module :: global_iterator I = Mod -> global_begin ( ) , E = Mod -> global_end ( ) ; I != E ; ++ I ) { GlobalVariable * GV = & * I ; PointerType * GVTy = GV -> getType ( ) ; if ( GVTy -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) { Instruction * Use = dyn_cast < Instruction > ( * U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod ->

getDataLayout ( ) . getTypeAllocSize ( GVTy -> getElementType ( ) ) ;


<s> for ( User * AllocaUser : Alloca -> users ( ) ) { GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MemorySSAWrapperPass > ( )


<s> bool IsArg = isa < Argument > ( Ptr ) ; IRBuilder < > B ( IsArg ? ArgCastInsertPt : & * std :: next ( cast < Instruction > ( Ptr ) -> getIterator ( ) ) ) ; PointerType * NewPT = PointerType :: getWithSamePointeeType ( PT , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; Value * Cast = B . CreateAddrSpaceCast ( Ptr , NewPT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Value * CastBack = B . CreateAddrSpaceCast ( Cast , PT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Ptr -> replaceUsesWithIf ( CastBack , [ Cast ] ( Use & U ) { return U . getUser ( ) != Cast ; } ) ; return


<s> bool <ISA_LIT>PromoteKernelArguments :: promotePointer ( Value * Ptr ) { enqueueUsers ( Ptr ) ; PointerType * PT = cast < PointerType > ( Ptr -> getType ( ) ) ; if ( PT -> getAddressSpace ( ) != <ISA_LIT>AS :: FLAT_ADDRESS ) return false ; bool IsArg = isa < Argument > ( Ptr ) ; IRBuilder < > B ( IsArg ? ArgCastInsertPt : & * std :: next ( cast < Instruction > ( Ptr ) -> getIterator ( ) ) ) ; PointerType * NewPT = PointerType :: getWithSamePointeeType ( PT , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; Value * Cast = B . CreateAddrSpaceCast ( Ptr , NewPT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Value * CastBack = B . CreateAddrSpaceCast ( Cast , PT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Ptr -> replaceUsesWithIf ( CastBack , [ Cast ] ( Use & U ) {


<s> bool <ISA_LIT>PromoteKernelArguments :: runOnFunction ( Function & F ) { MemorySSA & MSSA = getAnalysis < MemorySSAWrapperPass > ( ) . getMSSA


<s> while ( ! PtrUsers . empty ( ) ) { Instruction * U = dyn_cast < Instruction > ( PtrUsers . pop_back_val ( ) ) ; if ( ! U ) continue ; switch ( U -> getOpcode ( ) ) { default : break ; case Instruction :: Load : { LoadInst * LD = cast < LoadInst > ( U ) ; if ( LD -> getPointerOperand ( ) -> stripInBoundsOffsets ( ) == Ptr && ! <STR_LIT> ( LD , MSSA , AA ) ) Ptrs . push_back ( LD ) ; break ; } case Instruction ::


<s> AU . addRequired < MemorySSAWrapperPass > ( ) ; AU . setPreservesAll (


<s> IRBuilder < > B ( LI ? & * std :: next ( cast < Instruction > ( Ptr ) -> getIterator ( ) ) : ArgCastInsertPt ) ; PointerType * NewPT = PointerType :: getWithSamePointeeType ( PT , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; Value * Cast = B . CreateAddrSpaceCast ( Ptr , NewPT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Value * CastBack = B . CreateAddrSpaceCast ( Cast , PT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Ptr -> replaceUsesWithIf ( CastBack , [ Cast ] ( Use & U ) { return U . getUser ( ) != Cast ; } ) ; return true


<s> bool <ISA_LIT>PromoteKernelArguments :: runOnFunction ( Function & F ) { MemorySSA & MSSA = getAnalysis < MemorySSAWrapperPass > ( ) .


<s> NewRoots . clear ( ) ; for ( auto & F : M . functions ( ) ) { if ( F . isDeclaration ( ) ) continue ; const FnProperties CalleeProps ( * TM , F ) ; SmallVector < std :: pair < CallBase * , Function * > , <NUM_LIT> > ToReplace ; SmallSet < CallBase * , <NUM_LIT> > Visited ; for ( User * U : F . users ( ) ) { Instruction * I = dyn_cast < Instruction > ( U ) ; if ( ! I ) continue ; CallBase * CI = dyn_cast < CallBase > ( I ) ; if ( ! CI || CI -> getCalledOperand ( ) != & F ) continue ; Function * Caller = CI -> getCaller ( ) ; if ( ! Caller || ! Visited . insert ( CI ) . second ) continue ; if ( ! Roots . count ( Caller ) && ! NewRoots . count ( Caller ) ) continue ; const FnProperties CallerProps ( * TM , * Caller ) ; if ( CalleeProps == CallerProps ) { if ( ! Roots . count ( & F ) ) NewRoots . insert ( & F ) ; continue ; } Function * NewF = findFunction ( CallerProps , & F ) ; if ( ! NewF ) { const FnProperties NewProps = CalleeProps . adjustToCaller ( CallerProps ) ; if ( ! AllowClone ) { setFeatures ( F , NewProps . Features ) ; setAttributes ( F , NewProps . Attributes ) ; NewRoots . insert ( & F ) ; Changed = true ; break ; } NewF = cloneWithProperties ( F , NewProps ) ; Clones . push_back ( Clone ( CallerProps , & F , NewF ) ) ; NewRoots . insert ( NewF ) ; } ToReplace . push_back ( std :: make_pair ( CI , NewF ) ) ; Replaced . insert ( & F ) ; Changed =


<s> TM = & TPC -> getTM < TargetMachine > ( ) ; } if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ; return <ISA_LIT>PropagateAttributes ( TM , false )


<s> bool <ISA_LIT>PropagateAttributesEarly :: runOnFunction ( Function & F ) { if ( ! TM ) { auto * TPC = getAnalysisIfAvailable < TargetPassConfig > (


<s> Clone ( FeatureBitset FeatureMask , Function * OrigF ,


<s> setFeatures ( * NewF , NewFeatures ) ; if ( F . hasName ( ) ) { std :: string NewName = NewF -> getName ( ) ; NewF -> takeName ( & F


<s> Function * <ISA_LIT>PropagateAttributes :: findFunction ( const FeatureBitset & FeaturesNeeded , Function * OrigF ) { for ( Clone & C : Clones ) if ( C . OrigF == OrigF && FeaturesNeeded == C . FeatureMask ) return C . NewF ; return


<s> } Function * NewF = findFunction ( CallerBits , & F ) ; if ( ! NewF ) { FeatureBitset NewFeatures ( ( CalleeBits & ~ TargetFeatures ) | CallerBits ) ; if ( ! AllowClone ) { setFeatures ( F , NewFeatures ) ; NewRoots . insert ( & F ) ; Changed = true ; break ; } NewF = cloneWithFeatures ( F , NewFeatures ) ; Clones . push_back ( Clone ( CallerBits , & F , NewF ) ) ; NewRoots . insert ( NewF ) ; } ToReplace . push_back ( std :: make_pair ( CI , NewF ) ) ; Replaced . insert ( & F ) ; Changed = true ; } while ( ! ToReplace . empty ( ) ) { auto R = ToReplace . pop_back_val ( ) ; R . first -> setCalledFunction ( R . second ) ; } } } while ( ! NewRoots . empty ( ) ) ; for ( Function * F : Replaced )


<s> bool <ISA_LIT>PropagateAttributesEarly :: runOnFunction ( Function & F ) { if ( ! TM || ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ; return <ISA_LIT>PropagateAttributes ( TM , false )


<s> bool <ISA_LIT>PropagateAttributesEarly :: runOnFunction ( Function & F ) { if ( ! TM || ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ; return <ISA_LIT>PropagateAttributes ( TM , false ) . process ( F


<s> if ( ! TM ) return false ; return <ISA_LIT>PropagateAttributes ( TM , true ) . process (


<s> Roots . insert ( NewRoots . begin ( ) , NewRoots . end ( ) ) ; NewRoots . clear ( ) ; for ( auto & F : M . functions ( ) ) { if ( F . isDeclaration ( ) ) continue ; const FnProperties CalleeProps ( * TM , F ) ; SmallVector < std :: pair < CallBase * , Function * > , <NUM_LIT> > ToReplace ; SmallSet < CallBase * , <NUM_LIT> > Visited ; for ( User * U : F . users ( ) ) { Instruction * I = dyn_cast < Instruction > ( U ) ; if ( ! I ) continue ; CallBase * CI = dyn_cast < CallBase > ( I ) ; if ( ! CI || CI -> getCalledOperand ( ) != & F ) continue ; Function * Caller = CI -> getCaller ( ) ; if ( ! Caller || ! Visited . insert ( CI ) . second ) continue ; if ( ! Roots . count ( Caller ) && ! NewRoots . count ( Caller ) ) continue ; const FnProperties CallerProps ( * TM , * Caller ) ; if ( CalleeProps == CallerProps ) { if ( ! Roots . count ( & F ) ) NewRoots . insert ( & F ) ; continue ; } Function * NewF = findFunction ( CallerProps , & F ) ; if ( ! NewF ) { const FnProperties NewProps = CalleeProps . adjustToCaller ( CallerProps ) ; if ( ! AllowClone ) { setFeatures ( F , NewProps . Features ) ; setAttributes ( F , NewProps . Attributes ) ; NewRoots . insert ( & F ) ; Changed = true ; break ; } NewF = cloneWithProperties ( F , NewProps ) ; Clones . push_back ( Clone ( CallerProps , & F , NewF ) ) ; NewRoots . insert ( NewF ) ; } ToReplace . push_back ( std :: make_pair ( CI , NewF ) ) ; Replaced . insert ( & F ) ; Changed = true ; } while ( ! ToReplace .


<s> setFeatures ( * NewF , NewFeatures ) ; if ( F . hasName ( ) ) { std :: string NewName = std :: string ( NewF -> getName ( ) ) ; NewF -> takeName ( & F ) ; F . setName ( NewName


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << '\n' ) ; ValueToValueMapTy dummy ; Function * NewF = CloneFunction ( & F , dummy ) ; setFeatures ( * NewF , NewFeatures ) ; if ( F . hasName (


<s> FnProperties adjustToCaller ( const FnProperties & CallerProps ) const { FnProperties New ( ( Features & ~ TargetFeatures ) | CallerProps .


<s> FnProperties adjustToCaller ( const FnProperties & CallerProps ) const { FnProperties New ( ( Features & ~ TargetFeatures ) |


<s> <ISA_LIT>PropagateAttributes ( const TargetMachine * TM , bool AllowClone ) : TM ( TM ) , AllowClone


<s> <ISA_LIT>PropagateAttributes ( const TargetMachine * TM


<s> <ISA_LIT>PropagateAttributesEarly ( const TargetMachine * TM = nullptr ) : FunctionPass ( ID ) , TM (


<s> <ISA_LIT>PropagateAttributesLate ( const TargetMachine * TM = nullptr


<s> Properties ( Props ) , OrigF ( OrigF )


<s> Clone ( const FnProperties & Props , Function * OrigF , Function * NewF ) : Properties ( Props ) , OrigF ( OrigF ) ,


<s> NewF -> setLinkage ( GlobalValue :: InternalLinkage ) ; if ( F . hasName ( ) && F . hasLocalLinkage ( ) ) { std :: string NewName = std :: string ( NewF -> getName ( ) ) ; NewF -> takeName ( & F


<s> setAttributes ( * NewF , NewProps . Attributes ) ; NewF -> setVisibility ( GlobalValue :: DefaultVisibility ) ; NewF -> setLinkage ( GlobalValue :: InternalLinkage ) ; if ( F . hasName ( ) && F . hasLocalLinkage ( ) ) { std :: string NewName = std :: string


<s> for ( Clone & C : Clones ) if ( C . OrigF == OrigF && PropsNeeded == C .


<s> Features = TM . getSubtargetImpl ( F ) -> getFeatureBits ( ) ; for ( unsigned I = <NUM_LIT> ; I < NumAttr ; ++ I ) if ( F . hasFnAttribute ( AttributeNames [ I ] ) ) Attributes [ I ]

= F . getFnAttribute ( AttributeNames [ I ] ) ;


<s> } if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ; return <ISA_LIT>PropagateAttributes ( TM , false ) .


<s> auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; if ( ! TPC


<s> for ( unsigned I = <NUM_LIT> ; I < NumAttr ; ++ I ) { F . removeFnAttr ( AttributeNames [ I ] ) ; if ( NewAttrs [ I ] ) { LLVM_DEBUG ( dbgs ( ) << '\t' << NewAttrs [ I ]


<s> void <ISA_LIT>PropagateAttributes :: setFeatures ( Function & F , const FeatureBitset & NewFeatures ) { std :: string NewFeatureStr = getFeatureString ( NewFeatures ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getFeatureString ( NewFeatures & TargetFeatures ) << <STR_LIT> << F . getName ( ) << '\n' ) ; F . removeFnAttr (


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getFeatureString ( NewFeatures & TargetFeatures )


<s> <ISA_LIT>RegBankCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , const <ISA_LIT>LegalizerInfo * LI , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( false , true , LI , EnableOpt , OptSize , MinSize ) , KB ( KB ) , MDT ( MDT ) { if ( ! Generated . parseCommandLineOption ( ) ) report_fatal_error (


<s> <ISA_LIT>RegBankCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( ) ) , RBI ( * MF . getSubtarget ( ) . getRegBankInfo ( ) ) , TRI ( * MF . getSubtarget (

) . getRegisterInfo ( ) ) , Helper ( Helper ) { }


<s> <ISA_LIT>RegBankCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( ) ) , RBI ( * MF . getSubtarget (

) . getRegBankInfo ( ) ) , TRI ( * MF . getSubtarget ( ) . getRegisterInfo ( ) ) , Helper ( Helper ) { }


<s> B . setInstrAndDebugLoc ( MI ) ; B . buildInstr ( MatchInfo . Opc , { MI . getOperand ( <NUM_LIT> ) } , { MatchInfo . Val0 , MatchInfo . Val1 , MatchInfo . Val2 } , MI . getFlags ( ) ) ; MI . eraseFromParent


<s> void <ISA_LIT>RegBankCombinerHelper :: applyMed3 ( MachineInstr & MI , Med3MatchInfo & MatchInfo ) { B . setInstrAndDebugLoc ( MI ) ; B . buildInstr ( MatchInfo . Opc , { MI . getOperand ( <NUM_LIT> ) } , { MatchInfo . Val0 , MatchInfo . Val1 , MatchInfo . Val2 } , MI .


<s> bool <ISA_LIT>RegBankCombinerHelper :: matchIntMinMaxToMed3 ( MachineInstr & MI , Med3MatchInfo & MatchInfo ) { Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! isVgprRegBank ( Dst ) ) return false ; if ( MRI . getType ( Dst ) . isVector ( ) ) return false ; MinMaxMedOpc OpcodeTriple = getMinMaxPair ( MI . getOpcode ( ) ) ; Register Val ; Optional < ValueAndVReg > K0 ,


<s> bool <ISA_LIT>RegBankCombinerHelper :: matchMed ( MachineInstr & MI , MachineRegisterInfo & MRI , MinMaxMedOpc MMMOpc , Register & Val , CstTy & K0 , CstTy & K1 ) { return mi_match ( MI , MRI , m_any_of ( m_CommutativeBinOp ( MMMOpc . Min , m_CommutativeBinOp ( MMMOpc . Max , m_Reg ( Val ) , m_Cst ( K0 ) ) , m_Cst ( K1 ) ) , m_CommutativeBinOp ( MMMOpc . Max , m_CommutativeBinOp ( MMMOpc . Min , m_Reg (


<s> return mi_match ( MI , MRI , m_any_of ( m_CommutativeBinOp ( MMMOpc . Min , m_CommutativeBinOp ( MMMOpc . Max , m_Reg ( Val ) , m_Cst ( K0 ) ) , m_Cst ( K1 ) ) , m_CommutativeBinOp ( MMMOpc . Max , m_CommutativeBinOp ( MMMOpc . Min , m_Reg ( Val ) , m_Cst ( K1 ) ) ,


<s> <ISA_LIT>RegBankCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( ) ) , Subtarget ( MF . getSubtarget < GCNSubtarget > ( ) ) , RBI ( * Subtarget . getRegBankInfo ( ) )

, TRI ( * Subtarget . getRegisterInfo ( ) ) , TII ( * Subtarget . getInstrInfo ( ) ) , Helper ( Helper ) { }


<s> <ISA_LIT>RegBankCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( ) ) , Subtarget ( MF . getSubtarget < GCNSubtarget > ( ) ) , RBI ( * Subtarget . getRegBankInfo ( ) ) ,

TRI ( * Subtarget . getRegisterInfo ( ) ) , TII ( * Subtarget . getInstrInfo ( ) ) , Helper ( Helper ) { }


<s> B . buildInstr ( <STR_LIT> , { MI . getOperand ( <NUM_LIT> )


<s> B . buildInstr ( <STR_LIT> , { MI . getOperand ( <NUM_LIT> ) } , { Reg } , MI . getFlags ( ) ) ; MI . eraseFromParent (


<s> B . setInstrAndDebugLoc ( MI ) ; B . buildInstr ( MatchInfo . Opc , { MI . getOperand ( <NUM_LIT> ) } , { getAsVgpr ( MatchInfo . Val0 ) , getAsVgpr ( MatchInfo . Val1 ) , getAsVgpr ( MatchInfo . Val2 ) } , MI . getFlags ( ) ) ; MI . eraseFromParent


<s> Register Def = Use . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Use . getOpcode ( ) == <STR_LIT> && isVgprRegBank ( Def ) ) return Def ; } Register VgprReg = B . buildCopy ( MRI . getType ( Reg


<s> if ( Use . getOpcode ( ) == <STR_LIT> && isVgprRegBank ( Def ) ) return Def ; } Register VgprReg = B . buildCopy ( MRI . getType ( Reg ) , Reg ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( VgprReg , RBI


<s> bool <ISA_LIT>RegBankCombinerHelper :: getIEEE ( ) { return getMode ( ) .


<s> <ISA_LIT>RegBankCombinerHelper :: MinMaxMedOpc <ISA_LIT>RegBankCombinerHelper :: getMinMaxPair ( unsigned Opc ) { switch ( Opc ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : return { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; case <STR_LIT> : case <STR_LIT> : return { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; case <STR_LIT> : case <STR_LIT> : return { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; case


<s> <STR_LIT> <ISA_LIT>RegBankCombinerHelper :: getMode ( ) { return MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( )


<s> return MF . getInfo < <ISA_LIT>MachineFunctionInfo >


<s> if ( ( getIEEE ( ) && getDX10Clamp ( ) && isFminnumIeee ( MI ) && isKnownNeverSNaN ( Val , MRI ) ) || isKnownNeverNaN ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ) { Reg = Val ; return true ; } return


<s> if ( ! matchMed < GFCstAndRegMatch > ( MI , MRI , OpcodeTriple , Val , K0 , K1 ) ) return false ; if ( K0 -> Value > K1 -> Value ) return false ; if ( ( getIEEE ( ) && isFminnumIeee ( MI ) ) || isKnownNeverNaN ( Dst , MRI ) ) { if ( ( ! MRI . hasOneNonDBGUse ( K0 -> VReg ) || TII . isInlineConstant ( K0 -> Value ) ) && ( ! MRI . hasOneNonDBGUse ( K1 -> VReg ) || TII . isInlineConstant ( K1 -> Value ) ) ) { MatchInfo = { OpcodeTriple . Med , Val , K0 -> VReg , K1 -> VReg } ; return true


<s> Register Val ; Optional < FPValueAndVReg > K0 , K1 ; if ( ! matchMed < GFCstAndRegMatch > ( MI , MRI , OpcodeTriple , Val , K0 , K1 ) ) return false ; if ( K0 -> Value > K1 -> Value ) return false ; if ( ( getIEEE ( ) && isFminnumIeee ( MI ) ) || isKnownNeverNaN ( Dst , MRI ) ) { if ( ( ! MRI . hasOneNonDBGUse ( K0 -> VReg ) || TII . isInlineConstant ( K0 -> Value ) ) && ( ! MRI . hasOneNonDBGUse ( K1 -> VReg ) || TII . isInlineConstant ( K1 -> Value ) ) ) { MatchInfo = { OpcodeTriple . Med , Val , K0 -> VReg , K1 -> VReg } ; return true ; } } return


<s> <ISA_LIT>RegBankCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( ) ) , RBI ( * MF . getSubtarget ( ) . getRegBankInfo ( ) ) , TRI ( * MF . getSubtarget ( ) . getRegisterInfo (

) ) , TII ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ) , Helper ( Helper ) { }


<s> <ISA_LIT>RegBankCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( ) ) , RBI ( * MF . getSubtarget ( ) . getRegBankInfo ( ) ) , TRI ( * MF . getSubtarget ( ) . getRegisterInfo ( ) ) , TII ( * MF .

getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ) , Helper ( Helper ) { }


<s> <ISA_LIT>RegBankCombiner :: <ISA_LIT>RegBankCombiner ( bool IsOptNone ) : MachineFunctionPass ( ID ) , IsOptNone ( IsOptNone ) { initialize<ISA_LIT>RegBankCombinerPass ( * PassRegistry :: getPassRegistry


<s> <ISA_LIT>RegBankCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , const <ISA_LIT>LegalizerInfo * LI , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( false , true , LI , EnableOpt , OptSize , MinSize ) , KB ( KB


<s> <ISA_LIT>RegBankCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , const <ISA_LIT>LegalizerInfo * LI , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( false , true , LI , EnableOpt , OptSize , MinSize ) , KB ( KB ) , MDT ( MDT ) { if ( ! GeneratedRuleCfg . parseCommandLineOption (


<s> bool <ISA_LIT>RegBankCombinerInfo :: combine ( GISelChangeObserver & Observer , MachineInstr & MI , MachineIRBuilder & B ) const { CombinerHelper Helper ( Observer , B , KB , MDT ) ; <ISA_LIT>GenRegBankCombinerHelper Generated ( GeneratedRuleCfg


<s> bool <ISA_LIT>RegBankCombinerInfo :: combine ( GISelChangeObserver & Observer , MachineInstr & MI , MachineIRBuilder &


<s> AU . addRequired < TargetPassConfig > ( ) ; AU . setPreservesCFG ( ) ; getSelectionDAGFallbackAnalysisUsage ( AU ) ; AU . addRequired < GISelKnownBitsAnalysis > ( ) ; AU . addPreserved < GISelKnownBitsAnalysis > ( ) ; if ( ! IsOptNone ) { AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved <


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>LegalizerInfo * LI = static_cast < const <ISA_LIT>LegalizerInfo * > ( ST . getLegalizerInfo ( ) ) ; GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>RegBankCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , LI , KB , MDT ) ; Combiner C ( PCInfo


<s> GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>RegBankCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( )


<s> <ISA_LIT>RegisterBankInfo :: <ISA_LIT>RegisterBankInfo ( const GCNSubtarget & ST ) : <ISA_LIT>GenRegisterBankInfo ( ) , Subtarget ( ST ) , TRI ( Subtarget . getRegisterInfo ( ) ) , TII ( Subtarget . getInstrInfo ( ) ) { static bool AlreadyInit = false ; if ( AlreadyInit ) return ; AlreadyInit = true ; const RegisterBank & RBSGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBSGPR ; assert ( & RBSGPR == & <STR_LIT> ) ; const RegisterBank & RBVGPR = getRegBank ( <STR_LIT> ) ; ( void )


<s> void applyBank ( MachineInstr & MI ) { for ( MachineOperand & Op : MI . operands ( ) ) { if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg ( ) ; if ( MRI . getRegClassOrRegBank (


<s> if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg ( ) ; if ( MRI . getRegClassOrRegBank ( Reg ) ) continue ; const RegisterBank * RB = NewBank ; if ( MRI . getType ( Reg ) == LLT :: scalar ( <NUM_LIT> ) ) RB = ( NewBank == & <STR_LIT> ?


<s> SmallVector < unsigned , <NUM_LIT> > SGPRIndexes ; for ( int I = NumDefs , NumOps = MI . getNumOperands ( ) ; I != NumOps ; ++ I ) { if ( ! MI . getOperand ( I ) . isReg ( ) ) continue ; if ( I == RsrcIdx || I == RsrcIdx

+ <NUM_LIT> ) SGPRIndexes . push_back ( I ) ;


<s> assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy . getScalarType ( ) . getSizeInBits ( ) ; const LLT LoadSplitTy = LLT :: vector ( SplitElts , LoadTy . getScalarType ( ) ) ; ApplyRegBankMapping O ( MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; MachineInstr * LegalizedInst = getOtherVRegDef ( MRI , DstReg ,


<s> void <ISA_LIT>RegisterBankInfo :: constrainOpWithReadfirstlane ( MachineInstr & MI , MachineRegisterInfo & MRI , unsigned OpIdx ) const { Register Reg = MI . getOperand ( OpIdx ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank != & <STR_LIT> ) return ; MachineIRBuilder B ( MI ) ; Register SGPR = MRI . createVirtualRegister (


<s> B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ; const TargetRegisterClass * Constrained = constrainGenericRegister ( Reg , <STR_LIT> , MRI ) ; ( void ) Constrained ; assert ( Constrained && <STR_LIT> ) ; MI . getOperand ( OpIdx ) .


<s> void createdInstr ( MachineInstr & MI ) override { NewInsts . push_back ( &


<s> bool <ISA_LIT>RegisterBankInfo :: executeInWaterfallLoop ( MachineInstr & MI , MachineRegisterInfo & MRI , ArrayRef < unsigned > OpIndices ) const { MachineIRBuilder B (


<s> if ( mi_match ( Reg , MRI , m_GAdd ( m_Reg ( Base ) , m_ICst (

Const ) ) ) ) return std :: make_pair ( Base , Const ) ;


<s> if ( mi_match ( Reg , MRI , m_ICst ( Const ) ) ) return std :: make_pair ( Register ( ) , Const ) ; Register Base ; if ( mi_match ( Reg , MRI , m_GAdd ( m_Reg ( Base ) , m_ICst ( Const ) ) ) ) return std :: make_pair


<s> if ( ValMapping . NumBreakDowns >= <NUM_LIT> || ValMapping . BreakDown [ <NUM_LIT> ] . Length >= <NUM_LIT> ) return <NUM_LIT> ; assert ( ValMapping . NumBreakDowns == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ]

. StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . RegBank == ValMapping . BreakDown [ <NUM_LIT> ] . RegBank ) ;


<s> if ( Ty . isVector ( ) ) { assert ( Ty . getNumElements ( ) % <NUM_LIT> == <NUM_LIT> ) ; return LLT :: scalarOrVector ( Ty . getNumElements ( ) / <NUM_LIT> , Ty . getElementType ( ) ) ; } assert ( Ty . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT>


<s> for ( MachineInstr & Other : MRI . def_instructions ( Reg ) ) { if ( & Other != & MI


<s> if ( & RC == & <STR_LIT> ) return <STR_LIT> ; return TRI -> isSGPRClass ( & RC ) ? <STR_LIT> :


<s> unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ; return <STR_LIT> ( Bank , Size


<s> const RegisterBankInfo :: ValueMapping * <ISA_LIT>RegisterBankInfo :: getSGPROpMapping ( Register Reg , const MachineRegisterInfo & MRI , const TargetRegisterInfo & TRI ) const { unsigned Bank = getRegBankID ( Reg , MRI , TRI , <STR_LIT> ) ; unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ; return <STR_LIT> ( Bank


<s> LLT StoreVT = MRI . getType ( Reg ) ; if ( ! StoreVT . isVector ( ) || StoreVT . getElementType ( ) != S16 ) return Reg ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; SmallVector < Register , <NUM_LIT> > WideRegs ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) WideRegs . push_back


<s> const LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; LLT StoreVT = MRI . getType ( Reg ) ; if ( ! StoreVT . isVector ( ) || StoreVT . getElementType ( ) != S16 ) return Reg ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; SmallVector < Register ,


<s> if ( ! MI . hasOneMemOperand ( ) ) return false ; const MachineMemOperand * MMO = * MI .


<s> Register Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( Bank -> getID ( ) == <STR_LIT> ) return false ; assert ( Bank -> getID ( ) == <STR_LIT> || Bank -> getID ( ) == <STR_LIT> ) ; } } return true


<s> bool <ISA_LIT>RegisterBankInfo :: isSALUMapping ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; Register Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( Bank -> getID (


<s> static bool isZero ( Register Reg , MachineRegisterInfo & MRI ) { int64_t C ; return mi_match ( Reg , MRI , m_ICst ( C


<s> static unsigned regBankUnion ( unsigned RB0 , unsigned RB1 ) { return ( RB0 == <STR_LIT> && RB1 == <STR_LIT> ) ? <STR_LIT>


<s> for ( Register Reg : Regs ) { assert ( MRI . getType ( Reg ) . getSizeInBits ( ) == NewTy . getSizeInBits


<s> const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ; Regs . push_back ( HiLHS ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoLHS ) . addDef ( HiLHS )


<s> MachineRegisterInfo * MRI = B . getMRI ( ) ; Register LoLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; Register HiLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ; Regs . push_back ( HiLHS ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoLHS ) .


<s> if ( ( int32_t ) Overflow < <NUM_LIT> ) { Overflow += ImmOffset ; ImmOffset = <NUM_LIT> ; } C1 = ImmOffset ; if ( Overflow != <NUM_LIT> ) { if ( ! BaseReg ) BaseReg = B . buildConstant ( S32 , Overflow ) . getReg ( <NUM_LIT> ) ; else { auto OverflowVal = B . buildConstant ( S32 , Overflow ) ; BaseReg = B . buildAdd ( S32 , BaseReg , OverflowVal ) . getReg ( <NUM_LIT> ) ; } } } if ( ! BaseReg ) BaseReg = B . buildConstant (

S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ;


<s> SmallVector < unsigned , <NUM_LIT> > SrcReg ( OpdMapper . getVRegs ( OpIdx ) ) ; if ( ! SrcReg . empty ( ) ) { assert ( SrcReg . size (


<s> for ( MachineInstr * MI : NewInsts ) applyBank (


<s> static auto InitializeRegisterBankOnce = [ this ] ( ) { assert ( & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> ) ; ( void )


<s> MI . eraseFromParent ( ) ; } assert ( ! MRI . getRegClassOrRegBank ( DstReg ) ) ; MRI . setRegBank ( DstReg , * NewBank ) ; return ; } if ( Opc == <STR_LIT> ) { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const RegisterBank * DstBank = RBI . getRegBank ( DstReg , MRI , * RBI . TRI ) ; assert ( DstBank != & <STR_LIT> ) ; } for ( MachineOperand & Op : MI . operands ( ) ) { if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg ( ) ; if ( Reg . isPhysical ( ) || MRI . getRegClassOrRegBank ( Reg ) ) continue ; const RegisterBank * RB = NewBank ; if ( MRI . getType ( Reg ) == LLT :: scalar ( <NUM_LIT> ) ) { assert ( NewBank == & <STR_LIT> && <STR_LIT>


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const RegisterBank * DstBank = OpdMapper . getInstrMapping ( ) . getOperandMapping ( <NUM_LIT> ) . BreakDown [ <NUM_LIT> ] . RegBank ; if ( DstBank == & <STR_LIT> ) { if ( Ty == S32 ) return true ; return false ; } Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register OffsetReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register WidthReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; ApplyRegBankMapping ApplyBank ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & ApplyBank ) ; MachineIRBuilder B ( MI ) ; B . setChangeObserver ( Observer ) ; auto OffsetMask = B . buildConstant ( S32 , maskTrailingOnes < unsigned > ( <NUM_LIT> ) ) ; auto ClampOffset = B . buildAnd ( S32 ,


<s> NumLoads = LoadSize / <NUM_LIT> ; Ty = divideLLT ( Ty , NumLoads ) ; } const unsigned Alignment = NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ; MachineIRBuilder B ( MI ) ; MachineFunction & MF = B . getMF ( ) ; Register SOffset ; Register VOffset ; int64_t ImmOffset = <NUM_LIT> ; unsigned MMOOffset = setBufferOffsets ( B , * this , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , VOffset , SOffset , ImmOffset , Alignment ) ; const unsigned MemSize = ( Ty . getSizeInBits ( ) + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * BaseMMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; if ( MMOOffset != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset , MemSize ) ; Register RSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VIndex , <STR_LIT> ) ; SmallVector < Register , <NUM_LIT> > LoadParts ( NumLoads ) ; MachineBasicBlock :: iterator MII = MI . getIterator ( ) ; MachineInstrSpan Span ( MII , & B . getMBB ( ) ) ; for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ]


<s> B . buildInstr ( <STR_LIT> ) . addDef ( TmpReg0 ) . addUse ( SrcReg , <NUM_LIT> , <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( TmpReg1 ) . addUse ( SrcReg , <NUM_LIT> , <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( TmpReg0 ) . addImm ( <STR_LIT> ) . addUse (


<s> if ( SrcTy . getSizeInBits ( ) == <NUM_LIT> ) { B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( SrcReg ) ; return constrainGenericRegister ( DstReg , <STR_LIT> , MRI ) && constrainGenericRegister ( SrcReg , <STR_LIT> , MRI ) ; } Register TmpReg0 = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register TmpReg1 = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( TmpReg0 ) . addUse ( SrcReg , <NUM_LIT>


<s> void <ISA_LIT>RegisterBankInfo :: constrainOpWithReadfirstlane ( MachineInstr & MI , MachineRegisterInfo & MRI , unsigned OpIdx ) const { Register Reg = MI . getOperand ( OpIdx ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank != & <STR_LIT> ) return ; MachineIRBuilder B ( MI ) ; Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ; MRI . setType ( SGPR , MRI .


<s> if ( Size == <NUM_LIT> && ( Dst . getID ( ) == <STR_LIT> ) && ( isVectorRegisterBank ( Src ) || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Dst . getID ( ) == <STR_LIT> && Src . getID


<s> static LLT divideLLT ( LLT Ty , int Factor ) { if ( Ty . isVector ( ) ) return LLT :: vector ( Ty . getNumElements ( ) / Factor , Ty . getElementType ( ) ) ; return LLT :: scalar ( Ty . getSizeInBits ( ) / Factor


<s> const RegisterBankInfo :: ValueMapping * <ISA_LIT>RegisterBankInfo :: getAGPROpMapping ( Register Reg , const MachineRegisterInfo

& MRI , const TargetRegisterInfo & TRI ) const {


<s> unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ; return <STR_LIT> ( <STR_LIT> ,


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & SrcOp = MI . getOperand ( i ) ; if ( ! SrcOp . isReg ( ) ) continue ; unsigned Size = getSizeInBits ( SrcOp . getReg ( ) , MRI , * TRI ) ; OpdsMapping [ i ] = <STR_LIT> ( <STR_LIT> , Size ) ; } return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getDefaultMappingSOP ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & SrcOp = MI


<s> const int NumOps = MI . getNumOperands ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( NumOps ) ; for ( int I = <NUM_LIT> ; I != NumOps ; ++ I ) { if ( ! MI . getOperand ( I ) . isReg ( ) ) continue ; Register OpReg = MI . getOperand ( I ) . getReg ( ) ; if ( ! OpReg ) continue ; unsigned Size = getSizeInBits ( OpReg , MRI , * TRI ) ; const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + <NUM_LIT> ; if ( MustBeSGPR ) { unsigned NewBank = getRegBankID ( OpReg , MRI , * TRI , <STR_LIT> ) ; OpdsMapping [ I ] = <STR_LIT> ( NewBank , Size ) ; } else { OpdsMapping [ I ] = <STR_LIT>


<s> const int NumOps = MI . getNumOperands ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( NumOps ) ; for ( int I = <NUM_LIT> ; I != NumOps ; ++ I ) { if ( ! MI . getOperand ( I ) . isReg ( ) ) continue ; Register OpReg = MI . getOperand ( I


<s> unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; LLT LoadTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT PtrTy = MRI . getType ( PtrReg ) ; unsigned AS = PtrTy . getAddressSpace ( ) ; unsigned PtrSize = PtrTy . getSizeInBits ( ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> && <ISA_LIT>TargetLowering :: isFlatGlobalAddrSpace ( AS ) ) { if ( isScalarLoadLegal ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; unsigned PtrBankID = Subtarget . useFlatForGlobal ( ) ? <STR_LIT> : <STR_LIT> ; PtrMapping = <STR_LIT> ( PtrBankID , PtrSize ) ; ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy ) ; } } else { ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; const RegisterBankInfo :: InstructionMapping & Mapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ; return Mapping


<s> const RegisterBank & <ISA_LIT>RegisterBankInfo :: getRegBankFromRegClass ( const TargetRegisterClass & RC , LLT Ty ) const { if ( & RC == & <STR_LIT> ) return <STR_LIT> ; if ( TRI -> isSGPRClass ( & RC ) ) { if ( ! Ty . isValid ( ) ) return


<s> if ( ! Def ) return Reg ; return Def -> getOperand ( <NUM_LIT> ) .


<s> static bool isVectorRegisterBank ( const RegisterBank


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const CmpInst :: Predicate Pred = minMaxToCompare ( MI . getOpcode ( ) ) ; MachineInstr * Sel = buildExpandedScalarMinMax ( B , Pred , Dst , Src0 , Src1 ) ; Register CmpReg = Sel -> getOperand ( <NUM_LIT> ) . getReg (


<s> return CmpInst :: ICMP_SLT ; case TargetOpcode :: G_SMAX : return CmpInst :: ICMP_SGT ; case TargetOpcode :: G_UMIN : return CmpInst ::


<s> switch ( Opc ) { case TargetOpcode :: G_SMIN : case TargetOpcode :: G_SMAX : return TargetOpcode :: G_SEXT ; case TargetOpcode :: G_UMIN : case TargetOpcode


<s> if ( RB0 == - <NUM_LIT> ) return RB1 ; if ( RB1 == - <NUM_LIT> ) return RB0 ; if ( RB0 == <STR_LIT> || RB1 == <STR_LIT> ) return <STR_LIT> ; return regBankUnion ( RB0 , RB1 )


<s> if ( RB0 == - <NUM_LIT> ) return RB1 ; if ( RB1 == -


<s> MachineRegisterInfo & MRI = * B . getMRI ( ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register WaterfallIdx = IdxUseInstr . getOperand ( OpIdx ) . getReg ( ) ; B . setInsertPt ( * IdxUseInstr . getParent ( ) , IdxUseInstr . getIterator ( ) ) ; auto MaterializedOffset = B . buildConstant ( S32


<s> OpdMapper . getMI ( ) . getOperand ( OpIdx ) . setReg ( SrcReg [ <NUM_LIT> ] ) ; return true ; } return false


<s> SmallVector < unsigned , <NUM_LIT> > SrcReg ( OpdMapper . getVRegs ( OpIdx ) ) ; if ( ! SrcReg . empty ( ) ) { assert ( SrcReg . size ( ) ==


<s> return std :: make_pair ( ExtLo . getReg ( <NUM_LIT> ) , ShiftHi . getReg ( <NUM_LIT> ) ) ; } auto ShiftHi = B . buildLShr ( S32 , Bitcast , B . buildConstant ( S32 , <NUM_LIT> ) ) ; if ( ExtOpcode == TargetOpcode :: G_ZEXT ) { auto ExtLo = B . buildAnd ( S32 , Bitcast , B . buildConstant ( S32 , <NUM_LIT> ) ) ; return std :: make_pair ( ExtLo . getReg ( <NUM_LIT> ) , ShiftHi . getReg ( <NUM_LIT> ) ) ; } assert ( ExtOpcode == TargetOpcode ::


<s> <ISA_LIT>RegisterBankInfo :: <ISA_LIT>RegisterBankInfo ( const GCNSubtarget & ST ) : <ISA_LIT>GenRegisterBankInfo ( ) , Subtarget ( ST ) , TRI ( Subtarget . getRegisterInfo ( ) ) , TII ( Subtarget


<s> if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) { const RegisterBank * PtrBank = OpdMapper . getInstrMapping ( ) . getOperandMapping ( <NUM_LIT> ) . BreakDown [ <NUM_LIT> ] . RegBank ; if ( PtrBank == & <STR_LIT> ) return false ; SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> )


<s> if ( Dst . getID ( ) == <STR_LIT> && isVectorRegisterBank ( Src ) ) { return std :: numeric_limits < unsigned > :: max ( ) ; } if ( Size == <NUM_LIT> && ( Dst . getID ( ) == <STR_LIT> ) && ( isVectorRegisterBank ( Src ) || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Src . getID ( ) == <STR_LIT> ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return <NUM_LIT> ; return RegisterBankInfo :: copyCost ( Dst , Src


<s> if ( ! MI . hasOneMemOperand ( ) ) return false ; const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; const unsigned AS = MMO -> getAddrSpace ( ) ; const bool IsConst = AS == <ISA_LIT>AS :: CONSTANT_ADDRESS ||

AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ;


<s> Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const CmpInst :: Predicate Pred = minMaxToCompare ( MI . getOpcode ( ) ) ; LLT CmpType = LLT :: scalar ( <NUM_LIT> ) ; auto Cmp = B . buildICmp ( Pred , CmpType , Src0 , Src1 ) ; B . buildSelect ( Dst , Cmp , Src0 , Src1 ) ; B . getMRI ( ) -> setRegBank ( Cmp . getReg ( <NUM_LIT> )


<s> unsigned <ISA_LIT>RegisterBankInfo :: copyCost ( const RegisterBank & A , const RegisterBank & B , unsigned Size ) const { return RegisterBankInfo :: copyCost ( A


<s> RegisterBankInfo :: InstructionMappings <ISA_LIT>RegisterBankInfo :: getInstrAlternativeMappings ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; InstructionMappings AltMappings ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_LOAD : { const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; return AltMappings ; } default : break ; } return RegisterBankInfo :: getInstrAlternativeMappings ( MI )


<s> static bool isInstrUniform ( const MachineInstr & MI ) { if ( ! MI . hasOneMemOperand (


<s> RegisterBankInfo :: InstructionMappings <ISA_LIT>RegisterBankInfo :: addMappingFromTable ( const MachineInstr & MI , const MachineRegisterInfo & MRI , const std :: array < unsigned , NumOps > RegSrcOpIdx , ArrayRef < OpRegBankEntry < NumOps >> Table ) const { InstructionMappings AltMappings ; SmallVector < const ValueMapping * , <NUM_LIT> > Operands ( MI . getNumOperands ( ) ) ; unsigned Sizes [ NumOps ] ; for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { Register Reg = MI . getOperand ( RegSrcOpIdx [ I ] ) . getReg ( ) ; Sizes [ I ] = getSizeInBits ( Reg , MRI , * TRI ) ; } for ( unsigned I = <NUM_LIT> , E = MI . getNumExplicitDefs ( ) ; I != E ; ++ I ) { unsigned SizeI = getSizeInBits ( MI . getOperand ( I ) . getReg ( ) , MRI , * TRI ) ; Operands [ I ] = <STR_LIT> ( <STR_LIT> , SizeI ) ; } unsigned MappingID = <NUM_LIT> ; for ( const auto & Entry : Table ) { for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { int OpIdx = RegSrcOpIdx [ I ] ; Operands [ OpIdx ] = <STR_LIT> ( Entry . RegBanks [ I ] , Sizes [ I ] ) ; } AltMappings . push_back ( & getInstructionMapping ( MappingID ++ , Entry . Cost , getOperandsMapping ( Operands ) , Operands . size ( ) ) ) ; } return AltMappings


<s> const LLT LoadTy = MRI . getType ( DstReg ) ; unsigned LoadSize = LoadTy . getSizeInBits ( ) ; const unsigned MaxNonSmrdLoadSize = <NUM_LIT> ; if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) return false ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy . getScalarType ( ) . getSizeInBits ( ) ; const LLT LoadSplitTy = LLT :: vector ( SplitElts , LoadTy . getScalarType ( ) ) ; ApplyRegBankMapping O ( MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized )


<s> return std :: numeric_limits < unsigned > :: max ( ) ; } if ( Size == <NUM_LIT> && Dst . getID ( ) == <STR_LIT> && ( Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) ==

<STR_LIT> ) return std :: numeric_limits < unsigned > :: max ( ) ;


<s> LLT LoadTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; unsigned PtrSize = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; if ( isInstrUniform ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy


<s> auto Extract = Signed ? B . buildSbfx ( S32 , UnmergeSOffset . getReg ( <NUM_LIT> ) , Zero , UpperWidth ) : B . buildUbfx ( S32 , UnmergeSOffset . getReg ( <NUM_LIT> ) , Zero , UpperWidth ) ; B . buildMerge ( DstReg , { UnmergeSOffset . getReg ( <NUM_LIT> ) , Extract } ) ; } MI . eraseFromParent ( ) ; return true ; } auto ExtShift = B . buildSub ( S32 , B . buildConstant ( S32 , <NUM_LIT> ) , WidthReg ) ; auto SignBit = B . buildShl ( S64 , ShiftOffset , ExtShift ) ; if ( Signed ) B . buildAShr ( S64 , SignBit , ExtShift ) ; else B . buildLShr ( S64 , SignBit , ExtShift ) ; MI . eraseFromParent ( ) ; return true ; } ApplyRegBankMapping ApplyBank ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , ApplyBank ) ; auto OffsetMask = B . buildConstant ( S32 , maskTrailingOnes < unsigned > ( <NUM_LIT> ) ) ; auto ClampOffset = B . buildAnd ( S32 , OffsetReg , OffsetMask ) ; auto ShiftWidth = B . buildShl ( S32 , WidthReg , B . buildConstant ( S32 , <NUM_LIT> ) ) ; auto MergedInputs = B . buildOr ( S32 , ClampOffset , ShiftWidth ) ; unsigned Opc = Ty == S32 ? ( Signed ? <STR_LIT> : <STR_LIT> ) : ( Signed ? <STR_LIT> : <STR_LIT> ) ; auto MIB = B . buildInstr ( Opc , { DstReg } , { SrcReg , MergedInputs } ) ; if ( ! constrainSelectedInstRegOperands ( * MIB , * TII , * TRI , * this ) ) llvm_unreachable ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; return


<s> Align Alignment = assumeAligned ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) ) ; const RegisterBank * SizeBank = getRegBank ( AllocSize , MRI , * TRI ) ; if ( SizeBank != & <STR_LIT> ) return false ; LLT PtrTy = MRI . getType ( Dst ) ; LLT IntPtrTy = LLT :: scalar ( PtrTy . getSizeInBits ( ) ) ; const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register SPReg = Info -> getStackPtrOffsetReg ( ) ; ApplyRegBankMapping ApplyBank ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , ApplyBank ) ; auto WaveSize = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , ST . getWavefrontSizeLog2 ( ) ) ; auto ScaledSize = B . buildShl ( IntPtrTy , AllocSize , WaveSize ) ; auto SPCopy = B . buildCopy ( PtrTy , SPReg ) ; if ( Alignment > TFI . getStackAlign ( ) ) { auto PtrAdd = B . buildPtrAdd (


<s> SmallVector < Register , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < Register , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping Observer ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( LoadTy . isVector ( ) ) { if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; } else { if ( Helper . narrowScalar ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; } MRI . setRegBank ( DstReg , <STR_LIT> ) ; return


<s> NumLoads = LoadSize / <NUM_LIT> ; Ty = Ty . divide ( NumLoads ) ; } const Align Alignment = NumLoads > <NUM_LIT> ? Align ( <NUM_LIT> * NumLoads ) : Align ( <NUM_LIT> ) ; MachineIRBuilder B ( MI ) ; MachineFunction & MF = B . getMF ( ) ; Register SOffset ; Register VOffset ; int64_t ImmOffset = <NUM_LIT> ; unsigned MMOOffset = setBufferOffsets ( B , * this , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , VOffset , SOffset , ImmOffset , Alignment ) ; const unsigned MemSize = ( Ty . getSizeInBits ( ) + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * BaseMMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; if ( MMOOffset != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset , MemSize ) ; Register RSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VIndex , <STR_LIT> ) ; SmallVector < Register , <NUM_LIT> > LoadParts ( NumLoads ) ; MachineBasicBlock :: iterator MII = MI . getIterator ( ) ; MachineInstrSpan Span ( MII , & B . getMBB ( ) ) ; for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ] = Dst ; } else { LoadParts [ i ] = MRI . createGenericVirtualRegister ( Ty ) ; MRI . setRegBank ( LoadParts [ i ] , <STR_LIT> ) ; } MachineMemOperand * MMO = BaseMMO ; if ( i != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset + <NUM_LIT> * i , MemSize ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoadParts [ i ] ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset + <NUM_LIT> * i ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; } if ( RSrcBank != & <STR_LIT>


<s> Register Reg = MI . getOperand ( Op ) . getReg ( ) ; const RegisterBank * OpBank = getRegBank ( Reg , MRI , * TRI ) ; if ( OpBank -> getID ( ) != <STR_LIT> ) SGPROperandRegs . insert ( Reg ) ; } return ! SGPROperandRegs . empty (


<s> Reg = B . buildCopy ( Ty , Reg ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Reg , <STR_LIT> ) ; } Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ; MRI . setType (


<s> LLT Ty = MRI . getType ( Reg ) ; MachineIRBuilder B ( MI ) ; if ( Bank != & <STR_LIT> ) { Reg = B . buildCopy ( Ty , Reg ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Reg , <STR_LIT> ) ; } Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ; MRI . setType ( SGPR , Ty ) ; const TargetRegisterClass * Constrained = constrainGenericRegister ( Reg


<s> Register Copy = B . buildCopy ( MRI . getType ( Reg ) , Reg ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Copy , Bank ) ; return


<s> } else { auto ShiftAmt = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( ShiftAmt . getReg ( <NUM_LIT> ) , RegBank ) ; B . buildAShr ( Hi32Reg , Lo32Reg , ShiftAmt ) ; } } else


<s> if ( Ty . isVector ( ) ) { assert ( Ty . getElementCount ( ) . isKnownMultipleOf ( <NUM_LIT> ) ) ; return LLT :: scalarOrVector ( Ty . getElementCount ( ) . divideCoefficientBy ( <NUM_LIT> ) , Ty . getElementType ( ) ) ; } assert ( Ty . getScalarSizeInBits ( ) % <NUM_LIT> == <NUM_LIT>


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getInstrMappingForLoad ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( <NUM_LIT> ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT PtrTy = MRI . getType ( PtrReg ) ; unsigned AS = PtrTy . getAddressSpace ( ) ; unsigned PtrSize = PtrTy . getSizeInBits ( ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> && <STR_LIT> ( AS ) ) { if ( isScalarLoadLegal ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; unsigned PtrBankID = Subtarget . useFlatForGlobal ( ) ? <STR_LIT> : <STR_LIT> ; PtrMapping = <STR_LIT> ( PtrBankID , PtrSize ) ; } } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; const RegisterBankInfo :: InstructionMapping & Mapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ; return


<s> unsigned <ISA_LIT>RegisterBankInfo :: getRegBankID ( Register Reg , const MachineRegisterInfo & MRI ,


<s> const RegisterBank * Bank = getRegBank ( Reg , MRI , *


<s> const RegisterBankInfo :: ValueMapping * <ISA_LIT>RegisterBankInfo :: getSGPROpMapping ( Register Reg , const MachineRegisterInfo & MRI , const TargetRegisterInfo & TRI ) const { unsigned Bank = getRegBankID ( Reg , MRI , <STR_LIT> ) ; unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ; return <STR_LIT> ( Bank , Size


<s> unsigned Size = PtrTy . getSizeInBits ( ) ; if ( Subtarget . useFlatForGlobal ( ) || ! <STR_LIT> ( PtrTy . getAddressSpace ( ) ) ) return <STR_LIT> ( <STR_LIT> , Size ) ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI


<s> static bool isScalarLoadLegal ( const MachineInstr & MI ) { if ( ! MI . hasOneMemOperand ( ) ) return false ; const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; const unsigned AS = MMO -> getAddrSpace


<s> static unsigned regBankBoolUnion ( unsigned RB0 , unsigned RB1 ) { if ( RB0 == <STR_LIT> ) return RB1 ; if ( RB1 == <STR_LIT> ) return RB0 ; if ( RB0 == <STR_LIT> || RB1 == <STR_LIT> ) return


<s> if ( RB0 == <STR_LIT> || RB1 == <STR_LIT> ) return <STR_LIT> ; return regBankUnion ( RB0 ,


<s> Register VOffset = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned CachePolicy = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned ImmOffset ; std :: tie ( VOffset , ImmOffset ) = splitBufferOffsets ( B , VOffset ) ; const bool Offen = ! isZero ( VOffset , MRI ) ; unsigned Opc = <STR_LIT> ; switch ( <NUM_LIT> * MemSize ) { case <NUM_LIT> : Opc = Offen ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : Opc = Offen ? <STR_LIT> : <STR_LIT> ; break ; default : Opc = Offen ? <STR_LIT> : <STR_LIT> ; if ( Size > <NUM_LIT> ) Opc = <STR_LIT> ( Opc ,


<s> SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ; if ( Add && ( int ) Offset >= <NUM_LIT> ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT>


<s> SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; InstOffsetVal = ImmOffset ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; return SOffset + ImmOffset ; } } Register Base ; unsigned Offset ; std :: tie ( Base , Offset ) = <STR_LIT> ( * MRI , CombinedOffset ) ; uint32_t SOffset , ImmOffset ; if ( ( int ) Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ; if ( Add && ( int ) Offset >= <NUM_LIT> ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI )


<s> LLT EltTy = Ty . getElementType ( ) ; unsigned EltSize = EltTy . getSizeInBits ( ) ; assert ( FirstSize % EltSize == <NUM_LIT> ) ; unsigned FirstPartNumElts = FirstSize / EltSize ; unsigned RemainderElts = ( TotalSize - FirstSize )


<s> if ( ! Ty . isVector ( ) ) return LLT :: scalar ( <NUM_LIT> ) ; LLT EltTy = Ty . getElementType (


<s> if ( ( int ) Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Alignment ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ; if ( Add && ( int ) Offset >= <NUM_LIT> ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = CombinedOffset ; } else { VOffsetReg = B . buildCopy ( S32 , CombinedOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank


<s> unsigned <ISA_LIT>RegisterBankInfo :: getMappingType ( const MachineRegisterInfo & MRI , const MachineInstr & MI ) const { unsigned RegBank = <STR_LIT> ; for ( const MachineOperand & MO : MI . operands


<s> for ( const MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg (


<s> bool <ISA_LIT>RegisterBankInfo :: isSALUMapping ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( const MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , *


<s> for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { unsigned Size = getSizeInBits ( MI . getOperand ( i ) .


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getDefaultMappingSOP ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF .


<s> unsigned Size1 = getSizeInBits ( Reg1 , MRI , * TRI ) ; unsigned Bank1 = getRegBankID ( Reg1 , MRI , * TRI ) ; OpdsMapping [ OpdIdx ++ ] = <STR_LIT> ( Bank1 , Size1 ) ; for ( unsigned e = MI . getNumOperands ( ) ; OpdIdx != e ; ++ OpdIdx ) { unsigned Size = getSizeInBits ( MI . getOperand ( OpdIdx ) . getReg ( ) , MRI , * TRI ) ; OpdsMapping [ OpdIdx ] = <STR_LIT> ( <STR_LIT> , Size ) ; } return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands (


<s> for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ] = Dst ; } else { LoadParts [ i ] = MRI . createGenericVirtualRegister ( Ty ) ; MRI . setRegBank ( LoadParts [ i ] , <STR_LIT> ) ; } MachineMemOperand * MMO = BaseMMO ; if ( i != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset + <NUM_LIT> * i , MemSize ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoadParts [ i ] ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset + <NUM_LIT> * i ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; } if ( RSrcBank != & <STR_LIT> ) { B . setInstr ( * Span . begin ( ) ) ; MI . eraseFromParent ( ) ; SmallSet < Register , <NUM_LIT> > OpsToWaterfall ; OpsToWaterfall . insert ( RSrc ) ; executeInWaterfallLoop ( B , make_range ( Span . begin ( ) , Span .


<s> const unsigned MaxNonSmrdLoadSize = <NUM_LIT> ; if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) { const RegisterBank * PtrBank = OpdMapper . getInstrMapping ( ) . getOperandMapping ( <NUM_LIT> ) . BreakDown [ <NUM_LIT> ] . RegBank ; if ( PtrBank == & <STR_LIT> ) return false ; SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer


<s> const RegisterBank * PtrBank = OpdMapper . getInstrMapping ( ) . getOperandMapping ( <NUM_LIT> ) . BreakDown [ <NUM_LIT> ] . RegBank ; if ( PtrBank == & <STR_LIT> ) return false ; SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned NumSplitParts = LoadTy . getSizeInBits ( ) / MaxNonSmrdLoadSize ; const LLT LoadSplitTy = LoadTy . divide ( NumSplitParts ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( LoadTy . isVector


<s> const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( <NUM_LIT> ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT PtrTy = MRI . getType ( PtrReg ) ; unsigned AS = PtrTy . getAddressSpace ( ) ; unsigned PtrSize = PtrTy . getSizeInBits ( ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> && <ISA_LIT>TargetLowering :: isFlatGlobalAddrSpace ( AS ) ) { if ( isScalarLoadLegal ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT>


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getInstrMappingForLoad ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; LLT LoadTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT PtrTy = MRI .


<s> <ISA_LIT>RegisterBankInfo :: <ISA_LIT>RegisterBankInfo ( const GCNSubtarget & ST ) : Subtarget ( ST ) , TRI ( Subtarget . getRegisterInfo ( ) ) , TII ( Subtarget . getInstrInfo ( ) ) { static llvm :: once_flag InitializeRegisterBankFlag ; static auto InitializeRegisterBankOnce = [ this ] ( ) { assert ( & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> ) ; ( void )


<s> Src = B . buildCopy ( Ty , Src ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Src , <STR_LIT> ) ; } LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; unsigned NumParts = Bits / <NUM_LIT> ; SmallVector < Register , <NUM_LIT> > SrcParts ; SmallVector < Register , <NUM_LIT> > DstParts ; if ( Bits == <NUM_LIT> ) { SrcParts . push_back ( Src ) ; } else { auto Unmerge = B . buildUnmerge ( S32 , Src ) ; for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) SrcParts . push_back ( Unmerge . getReg ( i ) ) ; } for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) { Register SrcPart = SrcParts [ i ] ; Register DstPart = MRI . createVirtualRegister ( & <STR_LIT> ) ; MRI . setType ( DstPart , NumParts == <NUM_LIT> ? Ty : S32 ) ; const TargetRegisterClass * Constrained = constrainGenericRegister ( SrcPart , <STR_LIT> , MRI ) ; ( void ) Constrained ; assert ( Constrained && <STR_LIT> ) ; B . buildInstr ( <STR_LIT> , { DstPart } , { SrcPart


<s> if ( Bank == & <STR_LIT> ) return ; MachineIRBuilder B ( MI ) ; Reg = buildReadFirstLane ( B , MRI , Reg ) ; MI . getOperand ( OpIdx ) . setReg (


<s> auto Cmp = B . buildICmp ( CmpInst :: ICMP_EQ , CCTy , Idx , IC ) ; MRI . setRegBank ( Cmp -> getOperand ( <NUM_LIT> ) . getReg ( ) , CCBank ) ; for ( unsigned L = <NUM_LIT> ; L < NumLanes ; ++ L ) { Register Op0 = constrainRegToBank ( MRI , B , InsRegs [ L ] , DstBank ) ; Register Op1 = UnmergeToEltTy . getReg ( I * NumLanes + L ) ; Op1 = constrainRegToBank ( MRI , B , Op1 , DstBank ) ; Register Select = B . buildSelect ( EltTy , Cmp , Op0 , Op1 ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( Select , DstBank ) ; Ops [ I * NumLanes + L ] = Select ; } } LLT MergeTy = LLT :: vector ( Ops . size ( ) , EltTy ) ; if ( MergeTy == MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { B . buildBuildVector ( MI . getOperand ( <NUM_LIT> ) , Ops ) ; } else { auto Vec = B . buildBuildVector ( MergeTy , Ops ) ; MRI . setRegBank ( Vec -> getOperand ( <NUM_LIT>


<s> static std :: pair < LLT , LLT > splitUnequalType ( LLT Ty , unsigned FirstSize ) { unsigned TotalSize = Ty . getSizeInBits ( ) ; if ( ! Ty . isVector ( ) ) return { LLT :: scalar ( FirstSize ) , LLT :: scalar ( TotalSize - FirstSize ) } ; LLT EltTy = Ty . getElementType ( ) ; unsigned EltSize = EltTy . getSizeInBits ( ) ; assert ( FirstSize % EltSize == <NUM_LIT> ) ; unsigned FirstPartNumElts = FirstSize / EltSize ; unsigned RemainderElts = ( TotalSize


<s> assert ( <NUM_LIT> % EltTy . getSizeInBits ( ) == <NUM_LIT> ) ; return LLT :: vector ( <NUM_LIT> / EltTy .


<s> assert ( <NUM_LIT> % EltTy . getSizeInBits ( ) == <NUM_LIT> ) ; return LLT :: vector ( <NUM_LIT> / EltTy . getSizeInBits (


<s> unsigned MappingID = <NUM_LIT> ; for ( const auto & Entry : Table ) { for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { int OpIdx = RegSrcOpIdx [ I ] ; Operands [ OpIdx ] = <STR_LIT> ( Entry . RegBanks [ I ] , Sizes [ I ] ) ; } AltMappings . push_back ( & getInstructionMapping ( MappingID ++ , Entry . Cost , getOperandsMapping ( Operands ) , Operands . size ( ) ) ) ; } return AltMappings


<s> unsigned <ISA_LIT>RegisterBankInfo :: getBreakDownCost ( const ValueMapping & ValMapping , const RegisterBank * CurBank ) const { assert ( ValMapping . NumBreakDowns == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . RegBank == ValMapping . BreakDown [ <NUM_LIT> ] . RegBank ) ; return <NUM_LIT>