<s> if ( CallInst * CI = dyn_cast < CallInst > ( User ) ) { if ( ! isCallPromotable ( CI ) ) return false ; WorkList . push_back ( User ) ; continue ; } Instruction * UseInst = dyn_cast < Instruction > ( User ) ; if ( UseInst && UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ; if ( StoreInst * <ISA_LIT> = dyn_cast_or_null < StoreInst > ( UseInst ) ) {

if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ;


<s> MaxWorkGroupSizeRange = MDB . createRange ( APInt ( <NUM_LIT> , <NUM_LIT> ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; const Triple & TT = TM -> getTargetTriple ( ) ; IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ;

IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ;


<s> for ( Value * V : WorkList ) { CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { Type * EltTy = V -> getType ( ) -> getPointerElementType ( ) ; PointerType * NewTy = PointerType :: get ( EltTy , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; if ( isa < AddrSpaceCastInst > ( V ) ) continue ; V -> mutateType ( NewTy ) ; continue ; } IntrinsicInst * Intr = dyn_cast < IntrinsicInst > ( Call ) ; if ( ! Intr ) { std :: vector < Type * > ArgTypes ; for ( unsigned ArgIdx = <NUM_LIT> , ArgEnd = Call -> getNumArgOperands ( ) ; ArgIdx != ArgEnd ; ++ ArgIdx ) { ArgTypes . push_back ( Call -> getArgOperand ( ArgIdx ) -> getType ( ) ) ; } Function * F = Call -> getCalledFunction ( ) ; FunctionType * NewType = FunctionType :: get ( Call -> getType ( ) , ArgTypes , F -> isVarArg ( ) ) ; Constant * C = Mod -> getOrInsertFunction ( ( F -> getName ( ) + <STR_LIT> ) . str ( ) , NewType , F -> getAttributes ( ) ) ; Function * NewF = cast < Function > ( C ) ; Call -> setCalledFunction ( NewF ) ; continue ; } Builder . SetInsertPoint ( Intr ) ; switch ( Intr -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : Intr -> eraseFromParent ( ) ; continue ; case <STR_LIT> : { MemCpyInst * MemCpy = cast < MemCpyInst > ( Intr ) ; Builder . CreateMemCpy ( MemCpy -> getRawDest ( ) , MemCpy -> getRawSource ( ) , MemCpy -> getLength ( ) , MemCpy -> getAlignment ( ) , MemCpy -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : { MemMoveInst * MemMove = cast < MemMoveInst > ( Intr ) ; Builder . CreateMemMove ( MemMove -> getRawDest ( ) , MemMove -> getRawSource ( ) , MemMove -> getLength ( ) , MemMove -> getAlignment ( ) , MemMove -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ;


<s> if ( ! CI -> getCalledFunction ( ) ) return false ; IntrinsicInst * II = dyn_cast < IntrinsicInst > ( CI ) ; if ( ! II ) return false ; switch ( II -> getIntrinsicID ( ) ) {


<s> IntrinsicInst * II = dyn_cast < IntrinsicInst > ( CI ) ; if ( ! II ) return false ; switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> :


<s> return false ; } } const <ISA_LIT>Subtarget & ST = TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) ; LocalMemAvailable = ST . getLocalMemorySize ( ) ; if ( LocalMemAvailable == <NUM_LIT> ) return false ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Use & U : GV . uses ( ) ) { Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) . getTypeAllocSize ( GV . getValueType ( ) ) ; } } LocalMemAvailable = std :: max ( <NUM_LIT> , LocalMemAvailable ) ;

DEBUG ( dbgs ( ) << LocalMemAvailable << <STR_LIT> ) ;


<s> return false ; } } const <ISA_LIT>Subtarget & ST = TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) ; LocalMemAvailable = ST . getLocalMemorySize ( ) ; if ( LocalMemAvailable == <NUM_LIT> ) return false ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Use & U : GV . uses ( ) ) { Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) . getTypeAllocSize ( GV . getValueType ( ) ) ; } } LocalMemAvailable = std :: max ( <NUM_LIT> , LocalMemAvailable ) ; DEBUG ( dbgs ( ) << LocalMemAvailable << <STR_LIT> ) ;

BasicBlock & EntryBB = * F . begin ( ) ;


<s> if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VectorTy -> getPointerTo ( <NUM_LIT> ) ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VectorTy -> getPointerTo ( <NUM_LIT> ) ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Inst -> getOperand ( <NUM_LIT> ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ;


<s> if ( std :: find ( WorkList . begin ( ) , WorkList . end ( ) , User ) != WorkList . end ( ) ) continue ; if ( isa < CallInst > ( User ) ) { WorkList . push_back ( User ) ; continue ;


<s> std :: map < GetElementPtrInst * , Value * > GEPVectorIdx ; std :: vector < Value * > WorkList ; for ( User * AllocaUser : Alloca -> users ( ) ) { GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( std :: vector < Value * > :: iterator I = WorkList . begin ( ) , E = WorkList . end ( ) ; I != E ; ++ I ) { Instruction * Inst = cast < Instruction > ( * I ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VectorTy -> getPointerTo ( <NUM_LIT> ) ) ;

Value * VecValue = Builder . CreateLoad ( BitCast ) ;


<s> if ( UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ; if ( LoadInst * LI = dyn_cast < LoadInst > ( UseInst ) ) { if ( LI -> isVolatile ( ) ) return false ; continue ; } if ( StoreInst * <ISA_LIT> = dyn_cast < StoreInst > ( UseInst ) ) { if ( <ISA_LIT> -> isVolatile ( ) ) return false ; if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast < AtomicCmpXchgInst > ( UseInst ) ) { if ( CAS -> isVolatile ( ) ) return false ; } if ( ICmpInst * ICmp = dyn_cast < ICmpInst > ( UseInst ) ) {

if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , ICmp , <NUM_LIT> , <NUM_LIT> ) ) return false ;


<s> switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Type * VecPtrTy = VectorTy -> getPointerTo ( AS . PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { Type * VecPtrTy = VectorTy -> getPointerTo ( AS . PRIVATE_ADDRESS ) ; StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , <ISA_LIT> -> getValueOperand ( ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : break ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> for ( unsigned i = <NUM_LIT> , e = FTy -> getNumParams ( ) ; i != e ; ++ i ) { Type * ParamTy = FTy -> getParamType ( i ) ; if ( ParamTy -> isPointerTy ( ) && ParamTy -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalMemAvailable = <NUM_LIT> ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; break ; } } if ( LocalMemAvailable > <NUM_LIT> ) { for ( Module :: global_iterator I = Mod -> global_begin ( ) , E = Mod -> global_end ( ) ; I != E ; ++ I ) { GlobalVariable * GV = I ; PointerType * GVTy = GV -> getType ( ) ; if ( GVTy -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) { Instruction * Use = dyn_cast < Instruction > ( * U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) . getTypeAllocSize ( GVTy -> getElementType ( ) ) ;


<s> if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast < AtomicCmpXchgInst > ( UseInst ) ) { if ( CAS -> isVolatile ( ) ) return false ; } if ( ICmpInst * ICmp = dyn_cast < ICmpInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , ICmp , <NUM_LIT> , <NUM_LIT> ) ) return false ; WorkList . push_back ( ICmp ) ; } if ( ! User -> getType ( ) -> isPointerTy ( ) ) continue ; if ( GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( UseInst ) ) { if ( ! GEP -> isInBounds ( ) ) return false ; } if ( SelectInst * <ISA_LIT> = dyn_cast < SelectInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , <ISA_LIT> , <NUM_LIT> , <NUM_LIT> ) ) return false ; }

if ( PHINode * Phi = dyn_cast < PHINode > ( UseInst ) ) {


<s> CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ;

MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ;


<s> DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ;

LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ;


<s> Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ;


<s> IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( * TM , F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; AS = <STR_LIT> ( * F . getParent ( ) ) ; bool SufficientLDS = hasSufficientLocalMem ( F ) ; bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ;

if ( AI ) Changed |= handleAlloca ( * AI , SufficientLDS ) ;


<s> if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , <ISA_LIT> -> getValueOperand ( ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : break ; default :


<s> <ISA_LIT>PromoteAlloca ( const TargetMachine * TM_ = nullptr ) :

FunctionPass ( ID ) , TM ( TM_ ) , Mod ( nullptr ) , LocalMemAvailable ( <NUM_LIT> ) {


<s> FunctionPass ( ID ) , TM ( TM_ ) , Mod ( nullptr ) , LocalMemAvailable ( <NUM_LIT> ) {


<s> } Instruction * UseInst = dyn_cast < Instruction > ( User ) ; if ( UseInst && UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ; if ( StoreInst * <ISA_LIT> = dyn_cast_or_null < StoreInst > ( UseInst ) ) { if ( <ISA_LIT> -> isVolatile ( ) ) return false ; if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( LoadInst * LI = dyn_cast_or_null < LoadInst > ( UseInst ) ) { if ( LI -> isVolatile ( ) ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast_or_null < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ;

} else if ( AtomicCmpXchgInst * CAS = dyn_cast_or_null < AtomicCmpXchgInst > ( UseInst ) ) {


<s> WorkList . push_back ( User ) ; continue ; } Instruction * UseInst = dyn_cast < Instruction > ( User ) ; if ( UseInst && UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ; if ( StoreInst * <ISA_LIT> = dyn_cast_or_null < StoreInst > ( UseInst ) ) { if ( <ISA_LIT> -> isVolatile ( ) ) return false ; if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( LoadInst * LI = dyn_cast_or_null < LoadInst > ( UseInst ) ) { if ( LI -> isVolatile ( ) ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast_or_null < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast_or_null < AtomicCmpXchgInst > ( UseInst ) ) { if ( CAS -> isVolatile ( ) ) return false ; } if ( ! User -> getType ( ) -> isPointerTy ( ) ) continue ; WorkList . push_back ( User ) ; if ( ! collectUsesWithPtrTypes ( User , WorkList ) ) return false ; } return true ;


<s> bool <ISA_LIT>PromoteAlloca :: doInitialization ( Module & M ) {


<s> DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; break ; } } if ( LocalMemAvailable > <NUM_LIT> ) { for ( Module :: global_iterator I = Mod -> global_begin ( ) , E = Mod -> global_end ( ) ; I != E ; ++ I ) { GlobalVariable * GV = & * I ; PointerType * GVTy = GV -> getType ( ) ; if ( GVTy -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) { Instruction * Use = dyn_cast < Instruction > ( * U ) ; if ( ! Use ) continue ;

if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) . getTypeAllocSize ( GVTy -> getElementType ( ) ) ;


<s> for ( User * AllocaUser : Alloca -> users ( ) ) { GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) {

Instruction * Inst = cast < Instruction > ( V ) ;


<s> while ( ! PtrUsers . empty ( ) ) { Instruction * U = dyn_cast < Instruction > ( PtrUsers . pop_back_val ( ) ) ; if ( ! U ) continue ; switch ( U -> getOpcode ( ) ) { default : break ; case Instruction :: Load : { LoadInst * LD = cast < LoadInst > ( U ) ; PointerType * PT = dyn_cast < PointerType > ( LD -> getType ( ) ) ; if ( ! PT || ( PT -> getAddressSpace ( ) != <ISA_LIT>AS :: FLAT_ADDRESS && PT -> getAddressSpace ( ) != <ISA_LIT>AS :: GLOBAL_ADDRESS && PT -> getAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) || LD -> getPointerOperand ( ) -> stripInBoundsOffsets ( ) != Ptr ) break ; const MemoryAccess * MA = MSSA -> getWalker ( ) -> getClobberingMemoryAccess ( LD ) ; if ( MSSA -> isLiveOnEntryDef ( MA ) ) Ptrs . push_back ( LD ) ; break ; } case Instruction :: GetElementPtr : case Instruction :: AddrSpaceCast : case Instruction :: BitCast : if ( U -> getOperand ( <NUM_LIT> ) -> stripInBoundsOffsets ( ) == Ptr ) PtrUsers . append ( U -> user_begin ( ) , U -> user_end ( ) ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {

AU . addRequired < MemorySSAWrapperPass > ( ) ;


<s> AU . addRequired < MemorySSAWrapperPass > ( ) ; AU . setPreservesAll ( ) ;


<s> bool IsArg = isa < Argument > ( Ptr ) ; IRBuilder < > B ( IsArg ? ArgCastInsertPt : & * std :: next ( cast < Instruction > ( Ptr ) -> getIterator ( ) ) ) ; PointerType * NewPT = PointerType :: getWithSamePointeeType ( PT , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; Value * Cast = B . CreateAddrSpaceCast ( Ptr , NewPT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Value * CastBack = B . CreateAddrSpaceCast ( Cast , PT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Ptr -> replaceUsesWithIf ( CastBack , [ Cast ] ( Use & U ) { return U . getUser ( ) != Cast ; } ) ;


<s> bool <ISA_LIT>PromoteKernelArguments :: promotePointer ( Value * Ptr ) { enqueueUsers ( Ptr ) ; PointerType * PT = cast < PointerType > ( Ptr -> getType ( ) ) ; if ( PT -> getAddressSpace ( ) != <ISA_LIT>AS :: FLAT_ADDRESS ) return false ; bool IsArg = isa < Argument > ( Ptr ) ; IRBuilder < > B ( IsArg ? ArgCastInsertPt : & * std :: next ( cast < Instruction > ( Ptr ) -> getIterator ( ) ) ) ; PointerType * NewPT = PointerType :: getWithSamePointeeType ( PT , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; Value * Cast = B . CreateAddrSpaceCast ( Ptr , NewPT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Value * CastBack = B . CreateAddrSpaceCast ( Cast , PT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ;

Ptr -> replaceUsesWithIf ( CastBack , [ Cast ] ( Use & U ) { return U . getUser ( ) != Cast ; } ) ;


<s> PreservedAnalyses PA ; PA . preserveSet < CFGAnalyses > ( ) ; PA . preserve < MemorySSAAnalysis > ( ) ; return PA ;


<s> bool <ISA_LIT>PromoteKernelArguments :: runOnFunction ( Function & F ) {

MemorySSA & MSSA = getAnalysis < MemorySSAWrapperPass > ( ) . getMSSA ( ) ;


<s> <ISA_LIT>PromoteKernelArguments ( ) : FunctionPass ( ID ) {


return new <ISA_LIT>PromoteKernelArguments ( ) ;


<s> if ( LD -> getPointerOperand ( ) -> stripInBoundsOffsets ( ) == Ptr && ! <STR_LIT> ( LD , MSSA , AA ) ) Ptrs . push_back ( LD ) ; break ; } case Instruction :: GetElementPtr : case Instruction :: AddrSpaceCast : case Instruction :: BitCast : if ( U -> getOperand ( <NUM_LIT> ) -> stripInBoundsOffsets ( ) == Ptr ) PtrUsers . append ( U -> user_begin ( ) , U -> user_end ( ) ) ; break ;


<s> while ( ! PtrUsers . empty ( ) ) { Instruction * U = dyn_cast < Instruction > ( PtrUsers . pop_back_val ( ) ) ; if ( ! U ) continue ; switch ( U -> getOpcode ( ) ) { default : break ; case Instruction :: Load : { LoadInst * LD = cast < LoadInst > ( U ) ; if ( LD -> getPointerOperand ( ) -> stripInBoundsOffsets ( ) == Ptr && ! <STR_LIT> ( LD , MSSA , AA ) ) Ptrs . push_back ( LD ) ; break ; }


<s> AU . addRequired < MemorySSAWrapperPass > ( ) ;


<s> LI -> setMetadata ( <STR_LIT> , MDNode :: get ( LI -> getContext ( ) , { } ) ) ; return true ;


<s> IRBuilder < > B ( LI ? & * std :: next ( cast < Instruction > ( Ptr ) -> getIterator ( ) ) : ArgCastInsertPt ) ; PointerType * NewPT = PointerType :: getWithSamePointeeType ( PT , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; Value * Cast = B . CreateAddrSpaceCast ( Ptr , NewPT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Value * CastBack = B . CreateAddrSpaceCast ( Cast , PT , Twine ( Ptr -> getName ( ) , <STR_LIT> ) ) ; Ptr -> replaceUsesWithIf ( CastBack , [ Cast ] ( Use & U ) { return U . getUser ( ) != Cast ; } ) ;


<s> MemorySSA & MSSA = AM . getResult < MemorySSAAnalysis > ( F ) . getMSSA ( ) ; AliasAnalysis & AA = AM . getResult < AAManager > ( F ) ; if ( <ISA_LIT>PromoteKernelArguments ( ) . run ( F , MSSA , AA ) ) { PreservedAnalyses PA ; PA . preserveSet < CFGAnalyses > ( ) ; PA . preserve < MemorySSAAnalysis > ( ) ; return PA ;


<s> MemorySSA & MSSA = getAnalysis < MemorySSAWrapperPass > ( ) . getMSSA ( ) ; AliasAnalysis & AA = getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; return run ( F , MSSA , AA ) ;


<s> NewRoots . clear ( ) ; for ( auto & F : M . functions ( ) ) { if ( F . isDeclaration ( ) ) continue ; const FnProperties CalleeProps ( * TM , F ) ; SmallVector < std :: pair < CallBase * , Function * > , <NUM_LIT> > ToReplace ; SmallSet < CallBase * , <NUM_LIT> > Visited ; for ( User * U : F . users ( ) ) { Instruction * I = dyn_cast < Instruction > ( U ) ; if ( ! I ) continue ; CallBase * CI = dyn_cast < CallBase > ( I ) ; if ( ! CI || CI -> getCalledOperand ( ) != & F ) continue ; Function * Caller = CI -> getCaller ( ) ; if ( ! Caller || ! Visited . insert ( CI ) . second ) continue ; if ( ! Roots . count ( Caller ) && ! NewRoots . count ( Caller ) ) continue ; const FnProperties CallerProps ( * TM , * Caller ) ; if ( CalleeProps == CallerProps ) { if ( ! Roots . count ( & F ) ) NewRoots . insert ( & F ) ; continue ; } Function * NewF = findFunction ( CallerProps , & F ) ; if ( ! NewF ) { const FnProperties NewProps = CalleeProps . adjustToCaller ( CallerProps ) ; if ( ! AllowClone ) { setFeatures ( F , NewProps . Features ) ; setAttributes ( F , NewProps . Attributes ) ; NewRoots . insert ( & F ) ; Changed = true ; break ; } NewF = cloneWithProperties ( F , NewProps ) ; Clones . push_back ( Clone ( CallerProps , & F , NewF ) ) ; NewRoots . insert ( NewF ) ; } ToReplace . push_back ( std :: make_pair ( CI , NewF ) ) ; Replaced . insert ( & F ) ;


<s> TM = & TPC -> getTM < TargetMachine > ( ) ; } if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ;

return <ISA_LIT>PropagateAttributes ( TM , false ) . process ( F ) ;


<s> bool <ISA_LIT>PropagateAttributesEarly :: runOnFunction ( Function & F ) { if ( ! TM ) {

auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ;


<s> Function * NewF = CloneFunction ( & F , dummy ) ; setFeatures ( * NewF , NewFeatures ) ; if ( F . hasName ( ) ) { std :: string NewName = NewF -> getName ( ) ; NewF -> takeName ( & F ) ; F . setName ( NewName ) ; F . setVisibility ( GlobalValue :: DefaultVisibility ) ; F . setLinkage ( GlobalValue :: InternalLinkage ) ;


<s> setFeatures ( * NewF , NewFeatures ) ; if ( F . hasName ( ) ) { std :: string NewName = NewF -> getName ( ) ;


<s> Function * <ISA_LIT>PropagateAttributes :: findFunction ( const FeatureBitset & FeaturesNeeded , Function * OrigF ) { for ( Clone & C : Clones ) if ( C . OrigF == OrigF && FeaturesNeeded == C . FeatureMask ) return C . NewF ;


<s> for ( Clone & C : Clones ) if ( C . OrigF == OrigF && FeaturesNeeded == C . FeatureMask ) return C . NewF ; return nullptr ;


<s> } Function * NewF = findFunction ( CallerBits , & F ) ; if ( ! NewF ) { FeatureBitset NewFeatures ( ( CalleeBits & ~ TargetFeatures ) | CallerBits ) ; if ( ! AllowClone ) { setFeatures ( F , NewFeatures ) ; NewRoots . insert ( & F ) ; Changed = true ; break ; } NewF = cloneWithFeatures ( F , NewFeatures ) ; Clones . push_back ( Clone ( CallerBits , & F , NewF ) ) ; NewRoots . insert ( NewF ) ; } ToReplace . push_back ( std :: make_pair ( CI , NewF ) ) ; Replaced . insert ( & F ) ; Changed = true ; } while ( ! ToReplace . empty ( ) ) { auto R = ToReplace . pop_back_val ( ) ; R . first -> setCalledFunction ( R . second ) ; } } } while ( ! NewRoots . empty ( ) ) ;


<s> if ( CallerBits == ( CalleeBits & TargetFeatures ) ) { NewRoots . insert ( & F ) ; continue ; } Function * NewF = findFunction ( CallerBits , & F ) ; if ( ! NewF ) { FeatureBitset NewFeatures ( ( CalleeBits & ~ TargetFeatures ) | CallerBits ) ; if ( ! AllowClone ) { setFeatures ( F , NewFeatures ) ; NewRoots . insert ( & F ) ; Changed = true ; break ; } NewF = cloneWithFeatures ( F , NewFeatures ) ; Clones . push_back ( Clone ( CallerBits , & F , NewF ) ) ; NewRoots . insert ( NewF ) ; } ToReplace . push_back ( std :: make_pair ( CI , NewF ) ) ; Replaced . insert ( & F ) ; Changed = true ; } while ( ! ToReplace . empty ( ) ) { auto R = ToReplace . pop_back_val ( ) ; R . first -> setCalledFunction ( R . second ) ; }


<s> bool <ISA_LIT>PropagateAttributesEarly :: runOnFunction ( Function & F ) { if ( ! TM || ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ;


return <ISA_LIT>PropagateAttributes ( TM , true ) . process ( M ) ;


<s> return <ISA_LIT>PropagateAttributes ( TM , true ) . process ( M ) ;


<s> return <ISA_LIT>PropagateAttributes ( & TM , true ) . process ( M ) ? PreservedAnalyses :: none ( ) : PreservedAnalyses :: all ( ) ;


<s> Roots . insert ( NewRoots . begin ( ) , NewRoots . end ( ) ) ; NewRoots . clear ( ) ; for ( auto & F : M . functions ( ) ) { if ( F . isDeclaration ( ) ) continue ; const FnProperties CalleeProps ( * TM , F ) ; SmallVector < std :: pair < CallBase * , Function * > , <NUM_LIT> > ToReplace ; SmallSet < CallBase * , <NUM_LIT> > Visited ; for ( User * U : F . users ( ) ) { Instruction * I = dyn_cast < Instruction > ( U ) ; if ( ! I ) continue ; CallBase * CI = dyn_cast < CallBase > ( I ) ; if ( ! CI || CI -> getCalledOperand ( ) != & F ) continue ; Function * Caller = CI -> getCaller ( ) ; if ( ! Caller || ! Visited . insert ( CI ) . second ) continue ; if ( ! Roots . count ( Caller ) && ! NewRoots . count ( Caller ) ) continue ; const FnProperties CallerProps ( * TM , * Caller ) ; if ( CalleeProps == CallerProps ) { if ( ! Roots . count ( & F ) ) NewRoots . insert ( & F ) ; continue ; } Function * NewF = findFunction ( CallerProps , & F ) ; if ( ! NewF ) { const FnProperties NewProps = CalleeProps . adjustToCaller ( CallerProps ) ; if ( ! AllowClone ) { setFeatures ( F , NewProps . Features ) ; setAttributes ( F , NewProps . Attributes ) ; NewRoots . insert ( & F ) ; Changed = true ; break ; } NewF = cloneWithProperties ( F , NewProps ) ; Clones . push_back ( Clone ( CallerProps , & F , NewF ) ) ; NewRoots . insert ( NewF ) ; } ToReplace . push_back ( std :: make_pair ( CI , NewF ) ) ; Replaced . insert ( & F ) ; Changed = true ; }


<s> setFeatures ( * NewF , NewFeatures ) ; if ( F . hasName ( ) ) { std :: string NewName = std :: string ( NewF -> getName ( ) ) ; NewF -> takeName ( & F ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << '\n' ) ; ValueToValueMapTy dummy ; Function * NewF = CloneFunction ( & F , dummy ) ; setFeatures ( * NewF , NewFeatures ) ;


<s> FnProperties adjustToCaller ( const FnProperties & CallerProps ) const {

FnProperties New ( ( Features & ~ TargetFeatures ) | CallerProps . Features ) ;


<s> <ISA_LIT>PropagateAttributes ( const TargetMachine * TM , bool AllowClone ) :


<s> <ISA_LIT>PropagateAttributesEarly ( const TargetMachine * TM = nullptr ) :


<s> Clone ( const FnProperties & Props , Function * OrigF , Function * NewF ) :

Properties ( Props ) , OrigF ( OrigF ) , NewF ( NewF ) {


<s> NewF -> setLinkage ( GlobalValue :: InternalLinkage ) ; if ( F . hasName ( ) && F . hasLocalLinkage ( ) ) { std :: string NewName = std :: string ( NewF -> getName ( ) ) ;


<s> setAttributes ( * NewF , NewProps . Attributes ) ; NewF -> setVisibility ( GlobalValue :: DefaultVisibility ) ; NewF -> setLinkage ( GlobalValue :: InternalLinkage ) ; if ( F . hasName ( ) && F . hasLocalLinkage ( ) ) {

std :: string NewName = std :: string ( NewF -> getName ( ) ) ;


<s> Function * <ISA_LIT>PropagateAttributes :: findFunction ( const FnProperties & PropsNeeded , Function * OrigF ) { for ( Clone & C : Clones ) if ( C . OrigF == OrigF && PropsNeeded == C . Properties ) return C . NewF ; return nullptr ;


<s> Features = TM . getSubtargetImpl ( F ) -> getFeatureBits ( ) ;

for ( unsigned I = <NUM_LIT> ; I < NumAttr ; ++ I ) if ( F . hasFnAttribute ( AttributeNames [ I ] ) ) Attributes [ I ] = F . getFnAttribute ( AttributeNames [ I ] ) ;


<s> Features = TM . getSubtargetImpl ( F ) -> getFeatureBits ( ) ; for ( unsigned I = <NUM_LIT> ; I < NumAttr ; ++ I ) if ( F . hasFnAttribute ( AttributeNames [ I ] ) ) Attributes [ I ] = F . getFnAttribute ( AttributeNames [ I ] ) ;


<s> if ( Features [ KV . Value ] ) Ret += ( StringRef ( <STR_LIT> ) + KV . Key + <STR_LIT> ) . str ( ) ; else if ( TargetFeatures [ KV . Value ] ) Ret += ( StringRef ( <STR_LIT> ) + KV . Key + <STR_LIT> ) . str ( ) ;


<s> const FnProperties CalleeProps ( * TM , F ) ; SmallVector < std :: pair < CallBase * , Function * > , <NUM_LIT> > ToReplace ; SmallSet < CallBase * , <NUM_LIT> > Visited ; for ( User * U : F . users ( ) ) { Instruction * I = dyn_cast < Instruction > ( U ) ; if ( ! I ) continue ; CallBase * CI = dyn_cast < CallBase > ( I ) ; if ( ! CI ) continue ; Function * Caller = CI -> getCaller ( ) ; if ( ! Caller || ! Visited . insert ( CI ) . second ) continue ; if ( ! Roots . count ( Caller ) && ! NewRoots . count ( Caller ) ) continue ; const FnProperties CallerProps ( * TM , * Caller ) ; if ( CalleeProps == CallerProps ) { if ( ! Roots . count ( & F ) ) NewRoots . insert ( & F ) ; continue ; } Function * NewF = findFunction ( CallerProps , & F ) ; if ( ! NewF ) { const FnProperties NewProps = CalleeProps . adjustToCaller ( CallerProps ) ; if ( ! AllowClone ) { setFeatures ( F , NewProps . Features ) ; setAttributes ( F , NewProps . Attributes ) ; NewRoots . insert ( & F ) ; Changed = true ; break ;


<s> SmallVector < std :: pair < CallBase * , Function * > , <NUM_LIT> > ToReplace ; SmallSet < CallBase * , <NUM_LIT> > Visited ; for ( User * U : F . users ( ) ) { Instruction * I = dyn_cast < Instruction > ( U ) ; if ( ! I ) continue ; CallBase * CI = dyn_cast < CallBase > ( I ) ; if ( ! CI ) continue ; Function * Caller = CI -> getCaller ( ) ; if ( ! Caller || ! Visited . insert ( CI ) . second ) continue ; if ( ! Roots . count ( Caller ) && ! NewRoots . count ( Caller ) ) continue ; const FnProperties CallerProps ( * TM , * Caller ) ; if ( CalleeProps == CallerProps ) { if ( ! Roots . count ( & F ) ) NewRoots . insert ( & F ) ; continue ; } Function * NewF = findFunction ( CallerProps , & F ) ; if ( ! NewF ) { const FnProperties NewProps = CalleeProps . adjustToCaller ( CallerProps ) ; if ( ! AllowClone ) { setFeatures ( F , NewProps . Features ) ; setAttributes ( F , NewProps . Attributes ) ; NewRoots . insert ( & F ) ; Changed = true ; break ; } NewF = cloneWithProperties ( F , NewProps ) ; Clones . push_back ( Clone ( CallerProps , & F , NewF ) ) ; NewRoots . insert ( NewF ) ; } ToReplace . push_back ( std :: make_pair ( CI , NewF ) ) ; Replaced . insert ( & F ) ; Changed = true ; } while ( ! ToReplace . empty ( ) ) { auto R = ToReplace . pop_back_val ( ) ; R . first -> setCalledFunction ( R . second ) ; } } } while ( ! NewRoots . empty ( ) ) ; for ( Function * F : Replaced ) { if ( F -> use_empty ( ) ) F -> eraseFromParent ( ) ;


<s> } if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ;


<s> auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ;


<s> for ( unsigned I = <NUM_LIT> ; I < NumAttr ; ++ I ) { F . removeFnAttr ( AttributeNames [ I ] ) ; if ( NewAttrs [ I ] ) {

LLVM_DEBUG ( dbgs ( ) << '\t' << NewAttrs [ I ] -> getAsString ( ) << '\n' ) ;


<s> void <ISA_LIT>PropagateAttributes :: setFeatures ( Function & F , const FeatureBitset & NewFeatures ) { std :: string NewFeatureStr = getFeatureString ( NewFeatures ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getFeatureString ( NewFeatures & TargetFeatures ) << <STR_LIT> << F . getName ( ) << '\n' ) ;


<s> <ISA_LIT>RegBankCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , const <ISA_LIT>LegalizerInfo * LI , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( false , true , LI , EnableOpt , OptSize , MinSize ) , KB ( KB ) , MDT ( MDT ) { if ( ! Generated . parseCommandLineOption ( ) ) report_fatal_error ( <STR_LIT> ) ;


<s> <ISA_LIT>RegBankCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , const <ISA_LIT>LegalizerInfo * LI , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( false , true , LI , EnableOpt , OptSize , MinSize ) , KB ( KB ) , MDT ( MDT ) {

if ( ! Generated . parseCommandLineOption ( ) ) report_fatal_error ( <STR_LIT> ) ;


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; return false ;


<s> <ISA_LIT>RegBankCombinerHelperState ( CombinerHelper & Helper , <ISA_LIT>RegBankCombinerHelper & RegBankHelper ) : Helper ( Helper ) , RegBankHelper ( RegBankHelper ) {


<s> B . setInstrAndDebugLoc ( MI ) ; B . buildInstr ( MatchInfo . Opc , { MI . getOperand ( <NUM_LIT> ) } , { MatchInfo . Val0 , MatchInfo . Val1 , MatchInfo . Val2 } , MI . getFlags ( ) ) ;


<s> void <ISA_LIT>RegBankCombinerHelper :: applyMed3 ( MachineInstr & MI , Med3MatchInfo & MatchInfo ) { B . setInstrAndDebugLoc ( MI ) ;

B . buildInstr ( MatchInfo . Opc , { MI . getOperand ( <NUM_LIT> ) } , { MatchInfo . Val0 , MatchInfo . Val1 , MatchInfo . Val2 } , MI . getFlags ( ) ) ;


<s> return RBI . getRegBank ( Reg , MRI , TRI ) -> getID ( ) == <STR_LIT> ;


<s> bool <ISA_LIT>RegBankCombinerHelper :: matchIntMinMaxToMed3 ( MachineInstr & MI , Med3MatchInfo & MatchInfo ) { Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! isVgprRegBank ( Dst ) ) return false ; if ( MRI . getType ( Dst ) . isVector ( ) ) return false ; MinMaxMedOpc OpcodeTriple = getMinMaxPair ( MI . getOpcode ( ) ) ; Register Val ;


<s> bool <ISA_LIT>RegBankCombinerHelper :: matchMed ( MachineInstr & MI , MachineRegisterInfo & MRI , MinMaxMedOpc MMMOpc , Register & Val , CstTy & K0 , CstTy & K1 ) {

return mi_match ( MI , MRI , m_any_of ( m_CommutativeBinOp ( MMMOpc . Min , m_CommutativeBinOp ( MMMOpc . Max , m_Reg ( Val ) , m_Cst ( K0 ) ) , m_Cst ( K1 ) ) , m_CommutativeBinOp ( MMMOpc . Max , m_CommutativeBinOp ( MMMOpc . Min , m_Reg ( Val ) , m_Cst ( K1 ) ) , m_Cst ( K0 ) ) ) ) ;


<s> if ( ! isVgprRegBank ( Dst ) ) return false ; if ( MRI . getType ( Dst ) . isVector ( ) ) return false ; MinMaxMedOpc OpcodeTriple = getMinMaxPair ( MI . getOpcode ( ) ) ; Register Val , K0 , K1 ; if ( ! matchMed < ICstRegMatch > ( MI , MRI , OpcodeTriple , Val , K0 , K1 ) ) return false ; const APInt & K0_Imm = getConstantIntVRegVal ( K0 , MRI ) -> getValue ( ) ; const APInt & K1_Imm = getConstantIntVRegVal ( K1 , MRI ) -> getValue ( ) ; if ( OpcodeTriple . Med == <STR_LIT> && K0_Imm . sgt ( K1_Imm ) ) return false ; if ( OpcodeTriple . Med == <STR_LIT> && K0_Imm . ugt ( K1_Imm ) ) return false ; MatchInfo = { OpcodeTriple . Med , Val , K0 , K1 } ; return true ;


<s> if ( ! matchMed < ICstRegMatch > ( MI , MRI , OpcodeTriple , Val , K0 , K1 ) ) return false ; const APInt & K0_Imm = getConstantIntVRegVal ( K0 , MRI ) -> getValue ( ) ; const APInt & K1_Imm = getConstantIntVRegVal ( K1 , MRI ) -> getValue ( ) ; if ( OpcodeTriple . Med == <STR_LIT> && K0_Imm . sgt ( K1_Imm ) ) return false ; if ( OpcodeTriple . Med == <STR_LIT> && K0_Imm . ugt ( K1_Imm ) ) return false ; MatchInfo = { OpcodeTriple . Med , Val , K0 , K1 } ; return true ;


<s> bool <ISA_LIT>RegBankCombinerHelper :: matchMed ( MachineInstr & MI , MachineRegisterInfo & MRI , MinMaxMedOpc MMMOpc , Register & Val , Register & K0 , Register & K1 ) { return mi_match ( MI , MRI , m_any_of ( m_CommutativeBinOp ( MMMOpc . Min , m_CommutativeBinOp ( MMMOpc . Max , m_Reg ( Val ) , m_Cst ( K0 ) ) , m_Cst ( K1 ) ) , m_CommutativeBinOp ( MMMOpc . Max , m_CommutativeBinOp ( MMMOpc . Min , m_Reg ( Val ) , m_Cst ( K1 ) ) , m_Cst ( K0 ) ) ) ) ;


<s> B . buildInstr ( <STR_LIT> , { MI . getOperand ( <NUM_LIT> ) } , { Reg } , MI . getFlags ( ) ) ;


<s> B . setInstrAndDebugLoc ( MI ) ; B . buildInstr ( MatchInfo . Opc , { MI . getOperand ( <NUM_LIT> ) } , { getAsVgpr ( MatchInfo . Val0 ) , getAsVgpr ( MatchInfo . Val1 ) , getAsVgpr ( MatchInfo . Val2 ) } , MI . getFlags ( ) ) ;


<s> Register Def = Use . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Use . getOpcode ( ) == <STR_LIT> && isVgprRegBank ( Def ) ) return Def ; }

Register VgprReg = B . buildCopy ( MRI . getType ( Reg ) , Reg ) . getReg ( <NUM_LIT> ) ;


<s> if ( Use . getOpcode ( ) == <STR_LIT> && isVgprRegBank ( Def ) ) return Def ; } Register VgprReg = B . buildCopy ( MRI . getType ( Reg ) , Reg ) . getReg ( <NUM_LIT> ) ;

MRI . setRegBank ( VgprReg , RBI . getRegBank ( <STR_LIT> ) ) ;


<s> bool <ISA_LIT>RegBankCombinerHelper :: getIEEE ( ) {


<s> <ISA_LIT>RegBankCombinerHelper :: MinMaxMedOpc <ISA_LIT>RegBankCombinerHelper :: getMinMaxPair ( unsigned Opc ) { switch ( Opc ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : return { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; case <STR_LIT> : case <STR_LIT> : return { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; case <STR_LIT> : case <STR_LIT> : return { <STR_LIT> , <STR_LIT> , <STR_LIT> } ;


<s> <STR_LIT> <ISA_LIT>RegBankCombinerHelper :: getMode ( ) {

return MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getMode ( ) ;


<s> return ( KO_FPImm -> isExactlyValue ( <NUM_LIT> ) && K1_FPImm -> isExactlyValue ( <NUM_LIT> ) ) || ( KO_FPImm -> isExactlyValue ( <NUM_LIT> ) && K1_FPImm -> isExactlyValue ( <NUM_LIT> ) ) ;


<s> return MI -> getOpcode ( ) == <STR_LIT> ;


<s> return MI . getOpcode ( ) == <STR_LIT> ;


<s> bool <ISA_LIT>RegBankCombinerHelper :: isFminnumIeee ( const MachineInstr & MI ) { return MI . getOpcode ( ) == <STR_LIT> ;


<s> MachineInstr * Op3 = getDefIgnoringCopies ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ; if ( Op3 -> getOpcode ( ) == TargetOpcode :: G_FCONSTANT ) return Op3 -> getOperand ( <NUM_LIT> ) . getFPImm ( ) -> isExactlyValue ( <NUM_LIT> ) ; return false ; } ; if ( isKnownNeverNaN ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) || ( getIEEE ( ) && getDX10Clamp ( ) && ( isKnownNeverSNaN ( Val , MRI ) || isOp3Zero ( ) ) ) ) { Reg = Val ; return true ; } return false ;


<s> if ( ( getIEEE ( ) && getDX10Clamp ( ) && isFminnumIeee ( MI ) && isKnownNeverSNaN ( Val , MRI ) ) || isKnownNeverNaN ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ) { Reg = Val ; return true ; }


<s> auto OpcodeTriple = getMinMaxPair ( MI . getOpcode ( ) ) ; Register Val ; Optional < FPValueAndVReg > K0 , K1 ; if ( ! matchMed < GFCstOrSplatGFCstMatch > ( MI , MRI , OpcodeTriple , Val , K0 , K1 ) ) return false ; if ( ! K0 -> Value . isExactlyValue ( <NUM_LIT> ) || ! K1 -> Value . isExactlyValue ( <NUM_LIT> ) ) return false ; if ( ( getIEEE ( ) && getDX10Clamp ( ) && isFminnumIeee ( MI ) && isKnownNeverSNaN ( Val , MRI ) ) || isKnownNeverNaN ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ) { Reg = Val ; return true ;


<s> if ( ! matchMed < GFCstAndRegMatch > ( MI , MRI , OpcodeTriple , Val , K0 , K1 ) ) return false ; if ( K0 -> Value > K1 -> Value ) return false ; if ( ( getIEEE ( ) && isFminnumIeee ( MI ) ) || isKnownNeverNaN ( Dst , MRI ) ) { if ( ( ! MRI . hasOneNonDBGUse ( K0 -> VReg ) || TII . isInlineConstant ( K0 -> Value ) ) && ( ! MRI . hasOneNonDBGUse ( K1 -> VReg ) || TII . isInlineConstant ( K1 -> Value ) ) ) { MatchInfo = { OpcodeTriple . Med , Val , K0 -> VReg , K1 -> VReg } ;


<s> Register Val ; Optional < FPValueAndVReg > K0 , K1 ; if ( ! matchMed < GFCstAndRegMatch > ( MI , MRI , OpcodeTriple , Val , K0 , K1 ) ) return false ; if ( K0 -> Value > K1 -> Value ) return false ; if ( ( getIEEE ( ) && isFminnumIeee ( MI ) ) || isKnownNeverNaN ( Dst , MRI ) ) { if ( ( ! MRI . hasOneNonDBGUse ( K0 -> VReg ) || TII . isInlineConstant ( K0 -> Value ) ) && ( ! MRI . hasOneNonDBGUse ( K1 -> VReg ) || TII . isInlineConstant ( K1 -> Value ) ) ) { MatchInfo = { OpcodeTriple . Med , Val , K0 -> VReg , K1 -> VReg } ; return true ; } }


<s> if ( ( getIEEE ( ) && isFminnumIeee ( MI ) ) || isKnownNeverNaN ( Dst , MRI ) ) { if ( ( ! MRI . hasOneNonDBGUse ( K0 -> VReg ) || TII . isInlineConstant ( K0 -> Value ) ) && ( ! MRI . hasOneNonDBGUse ( K1 -> VReg ) || TII . isInlineConstant ( K1 -> Value ) ) ) { MatchInfo = { OpcodeTriple . Med , Val , K0 -> VReg , K1 -> VReg } ; return true ; }


<s> <ISA_LIT>RegBankCombiner :: <ISA_LIT>RegBankCombiner ( bool IsOptNone ) : MachineFunctionPass ( ID ) , IsOptNone ( IsOptNone ) {

initialize<ISA_LIT>RegBankCombinerPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> <ISA_LIT>RegBankCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , const <ISA_LIT>LegalizerInfo * LI , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( false , true , LI , EnableOpt , OptSize , MinSize ) , KB ( KB ) , MDT ( MDT ) {

if ( ! GeneratedRuleCfg . parseCommandLineOption ( ) ) report_fatal_error ( <STR_LIT> ) ;


<s> bool <ISA_LIT>RegBankCombinerInfo :: combine ( GISelChangeObserver & Observer , MachineInstr & MI , MachineIRBuilder & B ) const { CombinerHelper Helper ( Observer , B , KB , MDT ) ;

<ISA_LIT>GenRegBankCombinerHelper Generated ( GeneratedRuleCfg ) ;


<s> FunctionPass * create<ISA_LIT>RegBankCombiner ( bool IsOptNone ) { return new <ISA_LIT>RegBankCombiner ( IsOptNone ) ;


<s> return new <ISA_LIT>RegBankCombiner ( IsOptNone ) ;


<s> AU . addPreserved < GISelKnownBitsAnalysis > ( ) ; if ( ! IsOptNone ) { AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; } MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> AU . addRequired < TargetPassConfig > ( ) ; AU . setPreservesCFG ( ) ; getSelectionDAGFallbackAnalysisUsage ( AU ) ; AU . addRequired < GISelKnownBitsAnalysis > ( ) ; AU . addPreserved < GISelKnownBitsAnalysis > ( ) ; if ( ! IsOptNone ) { AU . addRequired < MachineDominatorTree > ( ) ;

AU . addPreserved < MachineDominatorTree > ( ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>LegalizerInfo * LI = static_cast < const <ISA_LIT>LegalizerInfo * > ( ST . getLegalizerInfo ( ) ) ; GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>RegBankCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , LI , KB , MDT ) ;


<s> GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ;

<ISA_LIT>RegBankCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , LI , KB , MDT ) ;


<s> unsigned Sizes [ NumOps ] ; for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { Register Reg = MI . getOperand ( RegSrcOpIdx [ I ] ) . getReg ( ) ; Sizes [ I ] = getSizeInBits ( Reg , MRI , * TRI ) ; } for ( unsigned I = <NUM_LIT> , E = MI . getNumExplicitDefs ( ) ; I != E ; ++ I ) { unsigned SizeI = getSizeInBits ( MI . getOperand ( I ) . getReg ( ) , MRI , * TRI ) ; Operands [ I ] = <STR_LIT> ( <STR_LIT> , SizeI ) ;


<s> <ISA_LIT>RegisterBankInfo :: <ISA_LIT>RegisterBankInfo ( const GCNSubtarget & ST ) : <ISA_LIT>GenRegisterBankInfo ( ) , Subtarget ( ST ) , TRI ( Subtarget . getRegisterInfo ( ) ) , TII ( Subtarget . getInstrInfo ( ) ) { static bool AlreadyInit = false ; if ( AlreadyInit ) return ; AlreadyInit = true ; const RegisterBank & RBSGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBSGPR ; assert ( & RBSGPR == & <STR_LIT> ) ; const RegisterBank & RBVGPR = getRegBank ( <STR_LIT> ) ;


<s> if ( AlreadyInit ) return ; AlreadyInit = true ; const RegisterBank & RBSGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBSGPR ; assert ( & RBSGPR == & <STR_LIT> ) ; const RegisterBank & RBVGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBVGPR ; assert ( & RBVGPR == & <STR_LIT> ) ;


<s> void applyBank ( MachineInstr & MI ) { for ( MachineOperand & Op : MI . operands ( ) ) { if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg ( ) ;

if ( MRI . getRegClassOrRegBank ( Reg ) ) continue ;


<s> if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg ( ) ; if ( MRI . getRegClassOrRegBank ( Reg ) ) continue ; const RegisterBank * RB = NewBank ;

if ( MRI . getType ( Reg ) == LLT :: scalar ( <NUM_LIT> ) ) RB = ( NewBank == & <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ) ;


<s> SmallVector < unsigned , <NUM_LIT> > SGPRIndexes ; for ( int I = NumDefs , NumOps = MI . getNumOperands ( ) ; I != NumOps ; ++ I ) { if ( ! MI . getOperand ( I ) . isReg ( ) ) continue ;

if ( I == RsrcIdx || I == RsrcIdx + <NUM_LIT> ) SGPRIndexes . push_back ( I ) ;


<s> assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy . getScalarType ( ) . getSizeInBits ( ) ; const LLT LoadSplitTy = LLT :: vector ( SplitElts , LoadTy . getScalarType ( ) ) ; ApplyRegBankMapping O ( MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ;

MachineInstr * LegalizedInst = getOtherVRegDef ( MRI , DstReg , * RepairInst ) ;


<s> ApplyRegBankMapping ( MachineRegisterInfo & MRI_ , const RegisterBank * RB ) : MRI ( MRI_ ) , NewBank ( RB ) {


<s> void changedInstr ( MachineInstr & MI ) override {


<s> void changingInstr ( MachineInstr & MI ) override {


<s> const RegisterBank * OpBank = getRegBank ( Reg , MRI , * TRI ) ; if ( OpBank -> getID ( ) == <STR_LIT> ) SGPROperandRegs . insert ( Reg ) ; } return ! SGPROperandRegs . empty ( ) ;


<s> void <ISA_LIT>RegisterBankInfo :: constrainOpWithReadfirstlane ( MachineInstr & MI , MachineRegisterInfo & MRI , unsigned OpIdx ) const { Register Reg = MI . getOperand ( OpIdx ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank != & <STR_LIT> ) return ; MachineIRBuilder B ( MI ) ;

Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ;


<s> B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ; const TargetRegisterClass * Constrained = constrainGenericRegister ( Reg , <STR_LIT> , MRI ) ; ( void ) Constrained ; assert ( Constrained && <STR_LIT> ) ;

MI . getOperand ( OpIdx ) . setReg ( SGPR ) ;


<s> return std :: numeric_limits < unsigned > :: max ( ) ; } if ( Size == <NUM_LIT> && ( Dst . getID ( ) == <STR_LIT> || Dst . getID ( ) == <STR_LIT> ) && ( Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return std :: numeric_limits < unsigned > :: max ( ) ; return RegisterBankInfo :: copyCost ( Dst , Src , Size ) ;


<s> void createdInstr ( MachineInstr & MI ) override {


<s> void erasingInstr ( MachineInstr & MI ) override {


<s> bool <ISA_LIT>RegisterBankInfo :: executeInWaterfallLoop ( MachineInstr & MI , MachineRegisterInfo & MRI , ArrayRef < unsigned > OpIndices ) const {


<s> return executeInWaterfallLoop ( B , MI , MRI , OpIndices ) ;


<s> static unsigned extractDLC ( unsigned CachePolicy ) { return ( CachePolicy >> <NUM_LIT> ) & <NUM_LIT> ;


<s> return ( CachePolicy >> <NUM_LIT> ) & <NUM_LIT> ;


<s> if ( mi_match ( Reg , MRI , m_ICst ( Const ) ) ) return std :: make_pair ( Register ( ) , Const ) ; Register Base ;

if ( mi_match ( Reg , MRI , m_GAdd ( m_Reg ( Base ) , m_ICst ( Const ) ) ) ) return std :: make_pair ( Base , Const ) ;


<s> if ( ValMapping . NumBreakDowns >= <NUM_LIT> || ValMapping . BreakDown [ <NUM_LIT> ] . Length >= <NUM_LIT> ) return <NUM_LIT> ;

assert ( ValMapping . NumBreakDowns == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . Length == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . StartIdx == <NUM_LIT> && ValMapping . BreakDown [ <NUM_LIT> ] . RegBank == ValMapping . BreakDown [ <NUM_LIT> ] . RegBank ) ;


<s> if ( ! Op . isReg ( ) ) continue ; unsigned Size = getSizeInBits ( Op . getReg ( ) , MRI , * TRI ) ; OpdsMapping [ I ] = <STR_LIT> ( <STR_LIT> , Size ) ; } return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ;


<s> for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { unsigned Size = getSizeInBits ( MI . getOperand ( i ) . getReg ( ) , MRI , * TRI ) ; unsigned BankID = Size == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; OpdsMapping [ i ] = <STR_LIT> ( BankID , Size ) ;


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getDefaultMappingSOP ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { unsigned Size = getSizeInBits ( MI . getOperand ( i ) . getReg ( ) , MRI , * TRI ) ; unsigned BankID = Size == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; OpdsMapping [ i ] = <STR_LIT> ( BankID , Size ) ;


<s> Register Reg1 = MI . getOperand ( OpdIdx ) . getReg ( ) ; unsigned Size1 = getSizeInBits ( Reg1 , MRI , * TRI ) ; unsigned DefaultBankID = Size1 == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; unsigned Bank1 = getRegBankID ( Reg1 , MRI , * TRI , DefaultBankID ) ; OpdsMapping [ OpdIdx ++ ] = <STR_LIT> ( Bank1 , Size1 ) ; for ( unsigned e = MI . getNumOperands ( ) ; OpdIdx != e ; ++ OpdIdx ) { const MachineOperand & MO = MI . getOperand ( OpdIdx ) ; if ( ! MO . isReg ( ) ) continue ; unsigned Size = getSizeInBits ( MO . getReg ( ) , MRI , * TRI ) ; unsigned BankID = Size == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; OpdsMapping [ OpdIdx ] = <STR_LIT> ( BankID , Size ) ; } return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ;


<s> if ( Ty . isVector ( ) ) { assert ( Ty . getNumElements ( ) % <NUM_LIT> == <NUM_LIT> ) ; return LLT :: scalarOrVector ( Ty . getNumElements ( ) / <NUM_LIT> , Ty . getElementType ( ) ) ; }

assert ( Ty . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> ) ;


<s> return LLT :: scalarOrVector ( Ty . getNumElements ( ) / <NUM_LIT> , Ty . getElementType ( ) ) ; } assert ( Ty . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> ) ; return LLT :: scalar ( Ty . getSizeInBits ( ) / <NUM_LIT> ) ;


<s> return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } default : return RegisterBankInfo :: getInstrAlternativeMappings ( MI ) ;


<s> unsigned PtrSize = PtrTy . getSizeInBits ( ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> && ( AS != <ISA_LIT>AS :: LOCAL_ADDRESS && AS != <ISA_LIT>AS :: REGION_ADDRESS && AS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) && isInstrUniformNonExtLoadAlign4 ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ;


<s> for ( MachineInstr & Other : MRI . def_instructions ( Reg ) ) {


<s> for ( MachineInstr & Other : MRI . def_instructions ( Reg ) ) { if ( & Other != & MI ) return & Other ; } return nullptr ;


<s> if ( & RC == & <STR_LIT> ) return <STR_LIT> ;

return TRI -> isSGPRClass ( & RC ) ? <STR_LIT> : <STR_LIT> ;


<s> const RegisterBank & <ISA_LIT>RegisterBankInfo :: getRegBankFromRegClass ( const TargetRegisterClass & RC ) const { if ( & RC == & <STR_LIT> ) return <STR_LIT> ; return TRI -> isSGPRClass ( & RC ) ? <STR_LIT> : <STR_LIT> ;


<s> const RegisterBank * Bank = getRegBank ( Reg , MRI , TRI ) ; return Bank ? Bank -> getID ( ) : Default ;


<s> unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ;


<s> const RegisterBankInfo :: ValueMapping * <ISA_LIT>RegisterBankInfo :: getSGPROpMapping ( Register Reg , const MachineRegisterInfo & MRI , const TargetRegisterInfo & TRI ) const { unsigned Bank = getRegBankID ( Reg , MRI , TRI , <STR_LIT> ) ; unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ;


<s> unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ; return <STR_LIT> ( <STR_LIT> , Size ) ;


<s> LLT StoreVT = MRI . getType ( Reg ) ; if ( ! StoreVT . isVector ( ) || StoreVT . getElementType ( ) != S16 ) return Reg ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; SmallVector < Register , <NUM_LIT> > WideRegs ;

for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) WideRegs . push_back ( Unmerge . getReg ( I ) ) ;


<s> const LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; LLT StoreVT = MRI . getType ( Reg ) ; if ( ! StoreVT . isVector ( ) || StoreVT . getElementType ( ) != S16 ) return Reg ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ;

SmallVector < Register , <NUM_LIT> > WideRegs ;


<s> if ( ! MI . hasOneMemOperand ( ) ) return false ;

const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ;


<s> Register Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( Bank -> getID ( ) == <STR_LIT> ) return false ; assert ( Bank -> getID ( ) == <STR_LIT> || Bank -> getID ( ) == <STR_LIT> ) ; } }


<s> bool <ISA_LIT>RegisterBankInfo :: isSALUMapping ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; Register Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) {

if ( Bank -> getID ( ) == <STR_LIT> ) return false ;


<s> static bool isZero ( Register Reg , MachineRegisterInfo & MRI ) { int64_t C ;

return mi_match ( Reg , MRI , m_ICst ( C ) ) && C == <NUM_LIT> ;


<s> static unsigned regBankUnion ( unsigned RB0 , unsigned RB1 ) {


assert ( MRI . getType ( Reg ) . getSizeInBits ( ) == NewTy . getSizeInBits ( ) ) ;


<s> const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ; Regs . push_back ( HiLHS ) ;

B . buildInstr ( <STR_LIT> ) . addDef ( LoLHS ) . addDef ( HiLHS ) . addUse ( Reg ) ;


<s> MachineRegisterInfo * MRI = B . getMRI ( ) ; Register LoLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; Register HiLHS = MRI -> createGenericVirtualRegister ( HalfTy ) ; const RegisterBank * Bank = getRegBank ( Reg , * MRI , * TRI ) ; MRI -> setRegBank ( LoLHS , * Bank ) ; MRI -> setRegBank ( HiLHS , * Bank ) ; Regs . push_back ( LoLHS ) ; Regs . push_back ( HiLHS ) ;


<s> std :: tie ( BaseReg , ImmOffset ) = getBaseWithConstantOffset ( * B . getMRI ( ) , OrigOffset ) ; unsigned C1 = <NUM_LIT> ; if ( ImmOffset != <NUM_LIT> ) { unsigned Overflow = ImmOffset & ~ MaxImm ; ImmOffset -= Overflow ; if ( ( int32_t ) Overflow < <NUM_LIT> ) { Overflow += ImmOffset ; ImmOffset = <NUM_LIT> ; } C1 = ImmOffset ; if ( Overflow != <NUM_LIT> ) { if ( ! BaseReg ) BaseReg = B . buildConstant ( S32 , Overflow ) . getReg ( <NUM_LIT> ) ; else { auto OverflowVal = B . buildConstant ( S32 , Overflow ) ; BaseReg = B . buildAdd ( S32 , BaseReg , OverflowVal ) . getReg ( <NUM_LIT> ) ; } }


<s> if ( ( int32_t ) Overflow < <NUM_LIT> ) { Overflow += ImmOffset ; ImmOffset = <NUM_LIT> ; } C1 = ImmOffset ; if ( Overflow != <NUM_LIT> ) { if ( ! BaseReg ) BaseReg = B . buildConstant ( S32 , Overflow ) . getReg ( <NUM_LIT> ) ; else { auto OverflowVal = B . buildConstant ( S32 , Overflow ) ; BaseReg = B . buildAdd ( S32 , BaseReg , OverflowVal ) . getReg ( <NUM_LIT> ) ; } } }

if ( ! BaseReg ) BaseReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ;


<s> SmallVector < unsigned , <NUM_LIT> > SrcReg ( OpdMapper . getVRegs ( OpIdx ) ) ; if ( ! SrcReg . empty ( ) ) {

assert ( SrcReg . size ( ) == <NUM_LIT> ) ;


<s> for ( MachineInstr * MI : NewInsts ) applyBank ( * MI ) ;


<s> static auto InitializeRegisterBankOnce = [ this ] ( ) { assert ( & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> && & getRegBank ( <STR_LIT> ) == & <STR_LIT> ) ;


<s> MI . eraseFromParent ( ) ; } assert ( ! MRI . getRegClassOrRegBank ( DstReg ) ) ; MRI . setRegBank ( DstReg , * NewBank ) ; return ; } if ( Opc == <STR_LIT> ) { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const RegisterBank * DstBank = RBI . getRegBank ( DstReg , MRI , * RBI . TRI ) ; assert ( DstBank != & <STR_LIT> ) ; } for ( MachineOperand & Op : MI . operands ( ) ) { if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg ( ) ; if ( Reg . isPhysical ( ) || MRI . getRegClassOrRegBank ( Reg ) ) continue ; const RegisterBank * RB = NewBank ; if ( MRI . getType ( Reg ) == LLT :: scalar ( <NUM_LIT> ) ) {

assert ( NewBank == & <STR_LIT> && <STR_LIT> ) ;


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const RegisterBank * DstBank = OpdMapper . getInstrMapping ( ) . getOperandMapping ( <NUM_LIT> ) . BreakDown [ <NUM_LIT> ] . RegBank ; if ( DstBank == & <STR_LIT> ) { if ( Ty == S32 ) return true ; return false ; } Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register OffsetReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register WidthReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; ApplyRegBankMapping ApplyBank ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & ApplyBank ) ; MachineIRBuilder B ( MI ) ; B . setChangeObserver ( Observer ) ; auto OffsetMask = B . buildConstant ( S32 , maskTrailingOnes < unsigned > ( <NUM_LIT> ) ) ;

auto ClampOffset = B . buildAnd ( S32 , OffsetReg , OffsetMask ) ;


<s> const unsigned MemSize = ( Ty . getSizeInBits ( ) + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * BaseMMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; if ( MMOOffset != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset , MemSize ) ; Register RSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VIndex , <STR_LIT> ) ; SmallVector < Register , <NUM_LIT> > LoadParts ( NumLoads ) ; MachineBasicBlock :: iterator MII = MI . getIterator ( ) ; MachineInstrSpan Span ( MII , & B . getMBB ( ) ) ; for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) { LoadParts [ i ] = Dst ; } else { LoadParts [ i ] = MRI . createGenericVirtualRegister ( Ty ) ; MRI . setRegBank ( LoadParts [ i ] , <STR_LIT> ) ; } MachineMemOperand * MMO = BaseMMO ; if ( i != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset + <NUM_LIT> * i , MemSize ) ; B . buildInstr ( <STR_LIT> ) . addDef ( LoadParts [ i ] ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset + <NUM_LIT> * i ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> NumLoads = LoadSize / <NUM_LIT> ; Ty = divideLLT ( Ty , NumLoads ) ; } const unsigned Alignment = NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ; MachineIRBuilder B ( MI ) ; MachineFunction & MF = B . getMF ( ) ; Register SOffset ; Register VOffset ; int64_t ImmOffset = <NUM_LIT> ; unsigned MMOOffset = setBufferOffsets ( B , * this , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , VOffset , SOffset , ImmOffset , Alignment ) ; const unsigned MemSize = ( Ty . getSizeInBits ( ) + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * BaseMMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; if ( MMOOffset != <NUM_LIT> ) BaseMMO = MF . getMachineMemOperand ( BaseMMO , MMOOffset , MemSize ) ; Register RSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VIndex , <STR_LIT> ) ; SmallVector < Register , <NUM_LIT> > LoadParts ( NumLoads ) ; MachineBasicBlock :: iterator MII = MI . getIterator ( ) ; MachineInstrSpan Span ( MII , & B . getMBB ( ) ) ; for ( int i = <NUM_LIT> ; i < NumLoads ; ++ i ) { if ( NumLoads == <NUM_LIT> ) {


<s> } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy . getScalarType ( ) . getSizeInBits ( ) ; const LLT LoadSplitTy = LLT :: vector ( SplitElts , LoadTy . getScalarType ( ) ) ; ApplyRegBankMapping O ( * this , MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; MachineInstr * LegalizedInst = getOtherVRegDef ( MRI , DstReg , * RepairInst ) ; Register TmpReg = MRI . createGenericVirtualRegister ( MRI . getType ( DstReg ) ) ; LegalizedInst -> getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; B . setInsertPt ( * RepairInst -> getParent ( ) , RepairInst ) ; for ( unsigned DefIdx = <NUM_LIT> , e = DefRegs . size ( ) ; DefIdx != e ; ++ DefIdx ) { Register IdxReg = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , DefIdx ) . getReg ( <NUM_LIT> ) ; MRI . setRegBank ( IdxReg , <STR_LIT> ) ; B . buildExtractVectorElement ( DefRegs [ DefIdx ] , TmpReg , IdxReg ) ; } MRI . setRegBank ( DstReg , <STR_LIT> ) ; return true ;


<s> ApplyRegBankMapping ( const <ISA_LIT>RegisterBankInfo & RBI_ , MachineRegisterInfo & MRI_ , const RegisterBank * RB ) : RBI ( RBI_ ) , MRI ( MRI_ ) , NewBank ( RB ) {


<s> static MachineInstr * buildExpandedScalarMinMax ( MachineIRBuilder & B , CmpInst :: Predicate Pred , Register Dst , Register Src0 , Register Src1 ) { const LLT CmpType = LLT :: scalar ( <NUM_LIT> ) ; auto Cmp = B . buildICmp ( Pred , CmpType , Src0 , Src1 ) ; return B . buildSelect ( Dst , Cmp , Src0 , Src1 ) ;


<s> B . buildInstr ( <STR_LIT> ) . addDef ( TmpReg0 ) . addUse ( SrcReg , <NUM_LIT> , <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( TmpReg1 ) . addUse ( SrcReg , <NUM_LIT> , <STR_LIT> ) ;

B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( TmpReg0 ) . addImm ( <STR_LIT> ) . addUse ( TmpReg1 ) . addImm ( <STR_LIT> ) ;


<s> if ( SrcTy . getSizeInBits ( ) == <NUM_LIT> ) { B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( SrcReg ) ; return constrainGenericRegister ( DstReg , <STR_LIT> , MRI ) && constrainGenericRegister ( SrcReg , <STR_LIT> , MRI ) ; } Register TmpReg0 = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register TmpReg1 = MRI . createVirtualRegister ( & <STR_LIT> ) ;

B . buildInstr ( <STR_LIT> ) . addDef ( TmpReg0 ) . addUse ( SrcReg , <NUM_LIT> , <STR_LIT> ) ;


<s> void <ISA_LIT>RegisterBankInfo :: constrainOpWithReadfirstlane ( MachineInstr & MI , MachineRegisterInfo & MRI , unsigned OpIdx ) const { Register Reg = MI . getOperand ( OpIdx ) . getReg ( ) ; const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ; if ( Bank != & <STR_LIT> ) return ; MachineIRBuilder B ( MI ) ; Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( SGPR ) . addReg ( Reg ) ;

MRI . setType ( SGPR , MRI . getType ( Reg ) ) ;


<s> if ( Size == <NUM_LIT> && ( Dst . getID ( ) == <STR_LIT> ) && ( isVectorRegisterBank ( Src ) || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ;

if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return <NUM_LIT> ;


<s> static LLT divideLLT ( LLT Ty , int Factor ) { if ( Ty . isVector ( ) ) return LLT :: vector ( Ty . getNumElements ( ) / Factor , Ty . getElementType ( ) ) ;

return LLT :: scalar ( Ty . getSizeInBits ( ) / Factor ) ;


<s> unsigned Size = getSizeInBits ( Reg , MRI , TRI ) ;


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & SrcOp = MI . getOperand ( i ) ; if ( ! SrcOp . isReg ( ) ) continue ; unsigned Size = getSizeInBits ( SrcOp . getReg ( ) , MRI , * TRI ) ; OpdsMapping [ i ] = <STR_LIT> ( <STR_LIT> , Size ) ; }

return getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ;


<s> const RegisterBankInfo :: InstructionMapping & <ISA_LIT>RegisterBankInfo :: getDefaultMappingSOP ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( MI . getNumOperands ( ) ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) {

const MachineOperand & SrcOp = MI . getOperand ( i ) ;


<s> const int NumOps = MI . getNumOperands ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( NumOps ) ; for ( int I = <NUM_LIT> ; I != NumOps ; ++ I ) { if ( ! MI . getOperand ( I ) . isReg ( ) ) continue ; Register OpReg = MI . getOperand ( I ) . getReg ( ) ; if ( ! OpReg ) continue ; unsigned Size = getSizeInBits ( OpReg , MRI , * TRI ) ; const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + <NUM_LIT> ; if ( MustBeSGPR ) { unsigned NewBank = getRegBankID ( OpReg , MRI , * TRI , <STR_LIT> ) ; OpdsMapping [ I ] = <STR_LIT> ( NewBank , Size ) ; } else {

OpdsMapping [ I ] = <STR_LIT> ( <STR_LIT> , Size ) ;


<s> const int NumOps = MI . getNumOperands ( ) ; SmallVector < const ValueMapping * , <NUM_LIT> > OpdsMapping ( NumOps ) ; for ( int I = <NUM_LIT> ; I != NumOps ; ++ I ) { if ( ! MI . getOperand ( I ) . isReg ( ) ) continue ;

Register OpReg = MI . getOperand ( I ) . getReg ( ) ;


<s> case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ;


<s> ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; unsigned PtrBankID = Subtarget . useFlatForGlobal ( ) ? <STR_LIT> : <STR_LIT> ; PtrMapping = <STR_LIT> ( PtrBankID , PtrSize ) ; ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy ) ; } } else { ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; const RegisterBankInfo :: InstructionMapping & Mapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ; return Mapping ;


<s> unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; LLT LoadTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT PtrTy = MRI . getType ( PtrReg ) ; unsigned AS = PtrTy . getAddressSpace ( ) ; unsigned PtrSize = PtrTy . getSizeInBits ( ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; if ( PtrBank == & <STR_LIT> && <ISA_LIT>TargetLowering :: isFlatGlobalAddrSpace ( AS ) ) { if ( isScalarLoadLegal ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; unsigned PtrBankID = Subtarget . useFlatForGlobal ( ) ? <STR_LIT> : <STR_LIT> ; PtrMapping = <STR_LIT> ( PtrBankID , PtrSize ) ; ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy ) ; } } else { ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; const RegisterBankInfo :: InstructionMapping & Mapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( OpdsMapping ) , MI . getNumOperands ( ) ) ;


<s> if ( ! Ty . isValid ( ) ) return <STR_LIT> ; return Ty == LLT :: scalar ( <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; } return TRI -> isAGPRClass ( & RC ) ? <STR_LIT> : <STR_LIT> ;


<s> const RegisterBank & <ISA_LIT>RegisterBankInfo :: getRegBankFromRegClass ( const TargetRegisterClass & RC , LLT Ty ) const { if ( & RC == & <STR_LIT> ) return <STR_LIT> ; if ( TRI -> isSGPRClass ( & RC ) ) {

if ( ! Ty . isValid ( ) ) return <STR_LIT> ;


return Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> MachineInstr * Def = getDefIgnoringCopies ( Reg , MRI ) ; if ( ! Def ) return Reg ; return Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> unsigned Size = PtrTy . getSizeInBits ( ) ; if ( Subtarget . useFlatForGlobal ( ) || ! <ISA_LIT>TargetLowering :: isFlatGlobalAddrSpace ( PtrTy . getAddressSpace ( ) ) ) return <STR_LIT> ( <STR_LIT> , Size ) ; const RegisterBank * PtrBank = getRegBank ( PtrReg , MRI , * TRI ) ; return <STR_LIT> ( PtrBank -> getID ( ) , Size ) ;


<s> Register Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( Bank -> getID ( ) != <STR_LIT> ) return false ;


<s> if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; Register Reg = MI . getOperand ( i ) . getReg ( ) ; if ( const RegisterBank * Bank = getRegBank ( Reg , MRI , * TRI ) ) { if ( Bank -> getID ( ) != <STR_LIT> ) return false ; } } return true ;


<s> const unsigned AS = MMO -> getAddrSpace ( ) ; const bool IsConst = AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ; return MMO -> getSize ( ) >= <NUM_LIT> && MMO -> getAlign ( ) >= Align ( <NUM_LIT> ) && ! MMO -> isAtomic ( ) && ( IsConst || ! MMO -> isVolatile ( ) ) && ( IsConst || MMO -> isInvariant ( ) || memOpHasNoClobbered ( MMO ) ) && <ISA_LIT>InstrInfo :: isUniformMMO ( MMO ) ;


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const CmpInst :: Predicate Pred = minMaxToCompare ( MI . getOpcode ( ) ) ; MachineInstr * Sel = buildExpandedScalarMinMax ( B , Pred , Dst , Src0 , Src1 ) ;

Register CmpReg = Sel -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> MachineInstr * Sel = buildExpandedScalarMinMax ( B , Pred , Dst , Src0 , Src1 ) ; Register CmpReg = Sel -> getOperand ( <NUM_LIT> ) . getReg ( ) ; B . getMRI ( ) -> setRegBank ( CmpReg , <STR_LIT> ) ; MI . eraseFromParent ( ) ;


<s> static bool memOpHasNoClobbered ( const MachineMemOperand * MMO ) { const Instruction * I = dyn_cast_or_null < Instruction > ( MMO -> getValue ( ) ) ; return I && I -> getMetadata ( <STR_LIT> ) ;


<s> return CmpInst :: ICMP_SLT ; case TargetOpcode :: G_SMAX : return CmpInst :: ICMP_SGT ; case TargetOpcode :: G_UMIN :


<s> switch ( Opc ) { case TargetOpcode :: G_SMIN : case TargetOpcode :: G_SMAX : return TargetOpcode :: G_SEXT ; case TargetOpcode :: G_UMIN :


<s> if ( RB0 == - <NUM_LIT> ) return RB1 ;


<s> MachineRegisterInfo & MRI = * B . getMRI ( ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register WaterfallIdx = IdxUseInstr . getOperand ( OpIdx ) . getReg ( ) ; B . setInsertPt ( * IdxUseInstr . getParent ( ) , IdxUseInstr . getIterator ( ) ) ;

auto MaterializedOffset = B . buildConstant ( S32 , ConstOffset ) ;


<s> } } Register Base ; unsigned Offset ; MachineInstr * Unused ; std :: tie ( Base , Offset , Unused ) = <STR_LIT> ( * MRI , CombinedOffset ) ; uint32_t SOffset , ImmOffset ; if ( Offset > <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , & RBI . Subtarget , Align ) ) { if ( RBI . getRegBank ( Base , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = Base ; SOffsetReg = B . buildConstant ( S32 , SOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } if ( SOffset == <NUM_LIT> ) { VOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; SOffsetReg = Base ; InstOffsetVal = ImmOffset ; return <NUM_LIT> ; } } if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = CombinedOffset ; } else { VOffsetReg = B . buildCopy ( S32 , CombinedOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ;


<s> } } if ( MachineInstr * Add = getOpcodeDef ( <STR_LIT> , CombinedOffset , * MRI ) ) { Register Src0 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Register Src1 = getSrcRegIgnoringCopies ( * MRI , Add -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; const RegisterBank * Src0Bank = RBI . getRegBank ( Src0 , * MRI , * RBI . TRI ) ; const RegisterBank * Src1Bank = RBI . getRegBank ( Src1 , * MRI , * RBI . TRI ) ; if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src0 ; SOffsetReg = Src1 ; return <NUM_LIT> ; } if ( Src0Bank == & <STR_LIT> && Src1Bank == & <STR_LIT> ) { VOffsetReg = Src1 ; SOffsetReg = Src0 ; return <NUM_LIT> ; } } if ( RBI . getRegBank ( CombinedOffset , * MRI , * RBI . TRI ) == & <STR_LIT> ) { VOffsetReg = CombinedOffset ; } else { VOffsetReg = B . buildCopy ( S32 , CombinedOffset ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( VOffsetReg , <STR_LIT> ) ; } SOffsetReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . getMRI ( ) -> setRegBank ( SOffsetReg , <STR_LIT> ) ; return <NUM_LIT> ;


<s> OpdMapper . getMI ( ) . getOperand ( OpIdx ) . setReg ( SrcReg [ <NUM_LIT> ] ) ; return true ; }


<s> SmallVector < unsigned , <NUM_LIT> > SrcReg ( OpdMapper . getVRegs ( OpIdx ) ) ; if ( ! SrcReg . empty ( ) ) {


<s> return std :: make_pair ( ExtLo . getReg ( <NUM_LIT> ) , ShiftHi . getReg ( <NUM_LIT> ) ) ; } auto ShiftHi = B . buildLShr ( S32 , Bitcast , B . buildConstant ( S32 , <NUM_LIT> ) ) ; if ( ExtOpcode == TargetOpcode :: G_ZEXT ) { auto ExtLo = B . buildAnd ( S32 , Bitcast , B . buildConstant ( S32 , <NUM_LIT> ) ) ; return std :: make_pair ( ExtLo . getReg ( <NUM_LIT> ) , ShiftHi . getReg ( <NUM_LIT> ) ) ; }

assert ( ExtOpcode == TargetOpcode :: G_ANYEXT ) ;


<s> auto ExtLo = B . buildSExtInReg ( S32 , Bitcast , <NUM_LIT> ) ; auto ShiftHi = B . buildAShr ( S32 , Bitcast , B . buildConstant ( S32 , <NUM_LIT> ) ) ; return std :: make_pair ( ExtLo . getReg ( <NUM_LIT> ) , ShiftHi . getReg ( <NUM_LIT> ) ) ; } auto ShiftHi = B . buildLShr ( S32 , Bitcast , B . buildConstant ( S32 , <NUM_LIT> ) ) ; if ( ExtOpcode == TargetOpcode :: G_ZEXT ) { auto ExtLo = B . buildAnd ( S32 , Bitcast , B . buildConstant ( S32 , <NUM_LIT> ) ) ; return std :: make_pair ( ExtLo . getReg ( <NUM_LIT> ) , ShiftHi . getReg ( <NUM_LIT> ) ) ; } assert ( ExtOpcode == TargetOpcode :: G_ANYEXT ) ; return std :: make_pair ( Bitcast . getReg ( <NUM_LIT> ) , ShiftHi . getReg ( <NUM_LIT> ) ) ;


<s> if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) { const RegisterBank * PtrBank = OpdMapper . getInstrMapping ( ) . getOperandMapping ( <NUM_LIT> ) . BreakDown [ <NUM_LIT> ] . RegBank ; if ( PtrBank == & <STR_LIT> ) return false ; SrcRegs . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ;

LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> if ( Dst . getID ( ) == <STR_LIT> && isVectorRegisterBank ( Src ) ) { return std :: numeric_limits < unsigned > :: max ( ) ; } if ( Size == <NUM_LIT> && ( Dst . getID ( ) == <STR_LIT> ) && ( isVectorRegisterBank ( Src ) || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Src . getID ( ) == <STR_LIT> ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return <NUM_LIT> ;

return RegisterBankInfo :: copyCost ( Dst , Src , Size ) ;


<s> if ( Dst . getID ( ) == <STR_LIT> && isVectorRegisterBank ( Src ) ) { return std :: numeric_limits < unsigned > :: max ( ) ; } if ( Size == <NUM_LIT> && ( Dst . getID ( ) == <STR_LIT> ) && ( isVectorRegisterBank ( Src ) || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Src . getID ( ) == <STR_LIT> ) return std :: numeric_limits < unsigned > :: max ( ) ; if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return <NUM_LIT> ; return RegisterBankInfo :: copyCost ( Dst , Src , Size ) ;


<s> if ( ! MI . hasOneMemOperand ( ) ) return false ; const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; const unsigned AS = MMO -> getAddrSpace ( ) ; const bool IsConst = AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ; return MMO -> getSize ( ) >= <NUM_LIT> && MMO -> getAlignment ( ) >= <NUM_LIT> && ! MMO -> isAtomic ( ) && ( IsConst || ! MMO -> isVolatile ( ) ) && ( IsConst || MMO -> isInvariant ( ) || memOpHasNoClobbered ( MMO ) ) && <ISA_LIT>InstrInfo :: isUniformMMO ( MMO ) ;


<s> if ( ! MI . hasOneMemOperand ( ) ) return false ; const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; const unsigned AS = MMO -> getAddrSpace ( ) ;

const bool IsConst = AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ;


<s> Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const CmpInst :: Predicate Pred = minMaxToCompare ( MI . getOpcode ( ) ) ; LLT CmpType = LLT :: scalar ( <NUM_LIT> ) ; auto Cmp = B . buildICmp ( Pred , CmpType , Src0 , Src1 ) ; B . buildSelect ( Dst , Cmp , Src0 , Src1 ) ;

B . getMRI ( ) -> setRegBank ( Cmp . getReg ( <NUM_LIT> ) , <STR_LIT> ) ;


<s> unsigned <ISA_LIT>RegisterBankInfo :: copyCost ( const RegisterBank & A , const RegisterBank & B , unsigned Size ) const {

return RegisterBankInfo :: copyCost ( A , B , Size ) ;


<s> return RegisterBankInfo :: copyCost ( A , B , Size ) ;


<s> const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; return AltMappings ; } default : break ;


<s> RegisterBankInfo :: InstructionMappings <ISA_LIT>RegisterBankInfo :: getInstrAlternativeMappings ( const MachineInstr & MI ) const { const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned Size = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; InstructionMappings AltMappings ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_LOAD : { const InstructionMapping & SSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & SSMapping ) ; const InstructionMapping & VVMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VVMapping ) ; const InstructionMapping & VSMapping = getInstructionMapping ( <NUM_LIT> , <NUM_LIT> , getOperandsMapping ( { <STR_LIT> ( <STR_LIT> , Size ) , <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) } ) , <NUM_LIT> ) ; AltMappings . push_back ( & VSMapping ) ; return AltMappings ; } default : break ; }

return RegisterBankInfo :: getInstrAlternativeMappings ( MI ) ;


<s> break ; case <STR_LIT> : { unsigned Size = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . getSizeInBits ( ) ; OpdsMapping [ <NUM_LIT> ] = <STR_LIT> ( <STR_LIT> , Size ) ; break ; } case <STR_LIT> : { for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; unsigned Size = MRI . getType ( MI . getOperand ( i ) . getReg ( ) ) . getSizeInBits ( ) ; OpdsMapping [ i ] = <STR_LIT> ( <STR_LIT> , Size ) ; } break ; } case <STR_LIT> : { assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; unsigned Size = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . getSizeInBits ( ) ; const ValueMapping * ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; const ValueMapping * PtrMapping = <STR_LIT> ( <STR_LIT> , <NUM_LIT> ) ; OpdsMapping [ <NUM_LIT> ] = ValMapping ; OpdsMapping [ <NUM_LIT> ] = PtrMapping ; break ;


<s> static bool isInstrUniform ( const MachineInstr & MI ) {

if ( ! MI . hasOneMemOperand ( ) ) return false ;


<s> const MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; return <STR_LIT> ( MMO ) ;


<s> RegisterBankInfo :: InstructionMappings <ISA_LIT>RegisterBankInfo :: addMappingFromTable ( const MachineInstr & MI , const MachineRegisterInfo & MRI , const std :: array < unsigned , NumOps > RegSrcOpIdx , ArrayRef < OpRegBankEntry < NumOps >> Table ) const { InstructionMappings AltMappings ; SmallVector < const ValueMapping * , <NUM_LIT> > Operands ( MI . getNumOperands ( ) ) ; unsigned Sizes [ NumOps ] ; for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { Register Reg = MI . getOperand ( RegSrcOpIdx [ I ] ) . getReg ( ) ; Sizes [ I ] = getSizeInBits ( Reg , MRI , * TRI ) ; } for ( unsigned I = <NUM_LIT> , E = MI . getNumExplicitDefs ( ) ; I != E ; ++ I ) { unsigned SizeI = getSizeInBits ( MI . getOperand ( I ) . getReg ( ) , MRI , * TRI ) ; Operands [ I ] = <STR_LIT> ( <STR_LIT> , SizeI ) ; } unsigned MappingID = <NUM_LIT> ; for ( const auto & Entry : Table ) { for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { int OpIdx = RegSrcOpIdx [ I ] ; Operands [ OpIdx ] = <STR_LIT> ( Entry . RegBanks [ I ] , Sizes [ I ] ) ; } AltMappings . push_back ( & getInstructionMapping ( MappingID ++ , Entry . Cost , getOperandsMapping ( Operands ) , Operands . size ( ) ) ) ; }


<s> const LLT LoadTy = MRI . getType ( DstReg ) ; unsigned LoadSize = LoadTy . getSizeInBits ( ) ; const unsigned MaxNonSmrdLoadSize = <NUM_LIT> ; if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) return false ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy . getScalarType ( ) . getSizeInBits ( ) ; const LLT LoadSplitTy = LLT :: vector ( SplitElts , LoadTy . getScalarType ( ) ) ; ApplyRegBankMapping O ( MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ;

if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ;


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const LLT LoadTy = MRI . getType ( DstReg ) ; unsigned LoadSize = LoadTy . getSizeInBits ( ) ; const unsigned MaxNonSmrdLoadSize = <NUM_LIT> ; if ( LoadSize <= MaxNonSmrdLoadSize ) return false ; SmallVector < unsigned , <NUM_LIT> > DefRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; SmallVector < unsigned , <NUM_LIT> > SrcRegs ( OpdMapper . getVRegs ( <NUM_LIT> ) ) ; if ( SrcRegs . empty ( ) ) return false ; assert ( LoadSize % MaxNonSmrdLoadSize == <NUM_LIT> ) ; MachineInstr * RepairInst = getOtherVRegDef ( MRI , DstReg , MI ) ; Register BasePtrReg = SrcRegs [ <NUM_LIT> ] ; LLT PtrTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . setType ( BasePtrReg , PtrTy ) ; MachineIRBuilder B ( MI ) ; unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy . getScalarType ( ) . getSizeInBits ( ) ; const LLT LoadSplitTy = LLT :: vector ( SplitElts , LoadTy . getScalarType ( ) ) ; ApplyRegBankMapping O ( MRI , & <STR_LIT> ) ; GISelObserverWrapper Observer ( & O ) ; B . setChangeObserver ( Observer ) ; LegalizerHelper Helper ( B . getMF ( ) , Observer , B ) ; if ( Helper . fewerElementsVector ( MI , <NUM_LIT> , LoadSplitTy ) != LegalizerHelper :: Legalized ) return false ; MachineInstr * LegalizedInst = getOtherVRegDef ( MRI , DstReg , * RepairInst ) ; Register TmpReg = MRI . createGenericVirtualRegister ( MRI . getType ( DstReg ) ) ; LegalizedInst -> getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; B . setInsertPt ( * RepairInst -> getParent ( ) , RepairInst ) ; for ( unsigned DefIdx = <NUM_LIT> , e = DefRegs . size ( ) ; DefIdx != e ; ++ DefIdx ) { Register IdxReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildConstant ( IdxReg , DefIdx ) ; MRI . setRegBank ( IdxReg , getRegBank ( <STR_LIT> ) ) ; B . buildExtractVectorElement ( DefRegs [ DefIdx ] , TmpReg , IdxReg ) ;


<s> return std :: numeric_limits < unsigned > :: max ( ) ; } if ( Size == <NUM_LIT> && Dst . getID ( ) == <STR_LIT> && ( Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> || Src . getID ( ) == <STR_LIT> ) ) return std :: numeric_limits < unsigned > :: max ( ) ;

if ( Dst . getID ( ) == <STR_LIT> && Src . getID ( ) == <STR_LIT> ) return std :: numeric_limits < unsigned > :: max ( ) ;


<s> static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; }


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> , <STR_LIT> , <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; const std :: array < unsigned , <NUM_LIT> > RegSrcOpIdx = { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ; return addMappingFromTable < <NUM_LIT> > ( MI , MRI , RegSrcOpIdx , makeArrayRef ( Table ) ) ; } case <STR_LIT> : case <STR_LIT> : { static const OpRegBankEntry < <NUM_LIT> > Table [ <NUM_LIT> ] = { { { <STR_LIT> } , <NUM_LIT> } , { { <STR_LIT> , <NUM_LIT>


<s> LLT LoadTy = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; unsigned PtrSize = getSizeInBits ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , * TRI ) ; const ValueMapping * ValMapping ; const ValueMapping * PtrMapping ; if ( isInstrUniform ( MI ) ) { ValMapping = <STR_LIT> ( <STR_LIT> , Size ) ; PtrMapping = <STR_LIT> ( <STR_LIT> , PtrSize ) ; } else {

ValMapping = <STR_LIT> ( <STR_LIT> , LoadTy ) ;


<s> auto Extract = Signed ? B . buildSbfx ( S32 , UnmergeSOffset . getReg ( <NUM_LIT> ) , Zero , UpperWidth ) : B . buildUbfx ( S32 , UnmergeSOffset . getReg ( <NUM_LIT> ) , Zero , UpperWidth ) ; B . buildMerge ( DstReg , { UnmergeSOffset . getReg ( <NUM_LIT> ) , Extract } ) ; } MI . eraseFromParent ( ) ; return true ; } auto ExtShift = B . buildSub ( S32 , B . buildConstant ( S32 , <NUM_LIT> ) , WidthReg ) ; auto SignBit = B . buildShl ( S64 , ShiftOffset , ExtShift ) ; if ( Signed ) B . buildAShr ( S64 , SignBit , ExtShift ) ; else B . buildLShr ( S64 , SignBit , ExtShift ) ; MI . eraseFromParent ( ) ; return true ; } ApplyRegBankMapping ApplyBank ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , ApplyBank ) ; auto OffsetMask = B . buildConstant ( S32 , maskTrailingOnes < unsigned > ( <NUM_LIT> ) ) ; auto ClampOffset = B . buildAnd ( S32 , OffsetReg , OffsetMask ) ; auto ShiftWidth = B . buildShl ( S32 , WidthReg , B . buildConstant ( S32 , <NUM_LIT> ) ) ; auto MergedInputs = B . buildOr ( S32 , ClampOffset , ShiftWidth ) ; unsigned Opc = Ty == S32 ? ( Signed ? <STR_LIT> : <STR_LIT> ) : ( Signed ? <STR_LIT> : <STR_LIT> ) ; auto MIB = B . buildInstr ( Opc , { DstReg } , { SrcReg , MergedInputs } ) ; if ( ! constrainSelectedInstRegOperands ( * MIB , * TII , * TRI , * this ) ) llvm_unreachable ( <STR_LIT> ) ; MI . eraseFromParent ( ) ;


<s> Align Alignment = assumeAligned ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) ) ; const RegisterBank * SizeBank = getRegBank ( AllocSize , MRI , * TRI ) ; if ( SizeBank != & <STR_LIT> ) return false ; LLT PtrTy = MRI . getType ( Dst ) ; LLT IntPtrTy = LLT :: scalar ( PtrTy . getSizeInBits ( ) ) ; const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register SPReg = Info -> getStackPtrOffsetReg ( ) ; ApplyRegBankMapping ApplyBank ( * this , MRI , & <STR_LIT> ) ; MachineIRBuilder B ( MI , ApplyBank ) ; auto WaveSize = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , ST . getWavefrontSizeLog2 ( ) ) ; auto ScaledSize = B . buildShl ( IntPtrTy , AllocSize , WaveSize ) ; auto SPCopy = B . buildCopy ( PtrTy , SPReg ) ; if ( Alignment > TFI . getStackAlign ( ) ) {

auto PtrAdd = B . buildPtrAdd ( PtrTy , SPCopy , ScaledSize ) ;