<s> if ( LoadSize == <NUM_LIT> ) GridSizeX = Load ; break ; case GRID_<ISA_LIT>ZE_Y : if ( LoadSize == <NUM_LIT> ) GridSizeY = Load ; break ; case GRID_<ISA_LIT>ZE_Z : if ( LoadSize == <NUM_LIT> ) GridSizeZ = Load ; break ; default : break ; } } bool MadeChange = false ; Value * WorkGroupSizes [ <NUM_LIT> ] = { WorkGroupSizeX , WorkGroupSizeY , WorkGroupSizeZ } ; Value * GridSizes [ <NUM_LIT> ] = { GridSizeX , GridSizeY , GridSizeZ } ; for ( int I = <NUM_LIT> ; HasUniformWorkGroupSize && I < <NUM_LIT> ; ++ I ) { Value * GroupSize = WorkGroupSizes [ I ] ; Value * GridSize = GridSizes [ I ] ; if ( ! GroupSize || ! GridSize ) continue ; for ( User * U : GroupSize -> users ( ) ) { auto * ZextGroupSize = dyn_cast < ZExtInst > ( U ) ; if ( ! ZextGroupSize ) continue ; for ( User * ZextUser : ZextGroupSize -> users ( ) ) { auto * <ISA_LIT> = dyn_cast < SelectInst > ( ZextUser ) ; if ( ! <ISA_LIT> ) continue ; using namespace llvm :: PatternMatch ; auto GroupIDIntrin = I == <NUM_LIT> ? m_Intrinsic < <STR_LIT> > ( ) : ( I == <NUM_LIT> ? m_Intrinsic < <STR_LIT> > ( ) : m_Intrinsic < <STR_LIT> > ( ) ) ; auto SubExpr = m_Sub ( m_Specific ( GridSize ) , m_Mul ( GroupIDIntrin , m_Specific ( ZextGroupSize ) ) ) ; ICmpInst :: Predicate Pred ; if ( match ( <ISA_LIT> , m_Select ( m_ICmp ( Pred , SubExpr , m_Specific ( ZextGroupSize ) ) , SubExpr , m_Specific ( ZextGroupSize ) ) ) && Pred == ICmpInst :: ICMP_ULT ) { if ( HasReqdWorkGroupSize ) { ConstantInt * KnownSize = mdconst :: extract < ConstantInt > ( MD -> getOperand ( I ) ) ; <ISA_LIT> -> replaceAllUsesWith ( ConstantExpr :: getIntegerCast ( KnownSize , <ISA_LIT> -> getType ( ) , false ) ) ; } else {

<ISA_LIT> -> replaceAllUsesWith ( ZextGroupSize ) ;


<s> <ISA_LIT>LowerKernelAttributes ( ) : ModulePass ( ID ) {


<s> if ( ! U -> hasOneUse ( ) ) continue ; int64_t Offset = <NUM_LIT> ; if ( GetPointerBaseWithConstantOffset ( U , Offset , DL ) != CI ) continue ; auto * BCI = dyn_cast < BitCastInst > ( * U -> user_begin ( ) ) ; if ( ! BCI || ! BCI -> hasOneUse ( ) ) continue ; auto * Load = dyn_cast < LoadInst > ( * BCI -> user_begin ( ) ) ; if ( ! Load || ! Load -> isSimple ( ) ) continue ; unsigned LoadSize = DL . getTypeStoreSize ( Load -> getType ( ) ) ; switch ( Offset ) { case WORKGROUP_<ISA_LIT>ZE_X : if ( LoadSize == <NUM_LIT> ) WorkGroupSizeX = Load ; break ; case WORKGROUP_<ISA_LIT>ZE_Y : if ( LoadSize == <NUM_LIT> ) WorkGroupSizeY = Load ; break ; case WORKGROUP_<ISA_LIT>ZE_Z : if ( LoadSize == <NUM_LIT> ) WorkGroupSizeZ = Load ; break ; case GRID_<ISA_LIT>ZE_X : if ( LoadSize == <NUM_LIT> ) GridSizeX = Load ; break ; case GRID_<ISA_LIT>ZE_Y : if ( LoadSize == <NUM_LIT> ) GridSizeY = Load ; break ; case GRID_<ISA_LIT>ZE_Z : if ( LoadSize == <NUM_LIT> ) GridSizeZ = Load ; break ; default : break ; } } bool MadeChange = false ; Value * WorkGroupSizes [ <NUM_LIT> ] = { WorkGroupSizeX , WorkGroupSizeY , WorkGroupSizeZ } ; Value * GridSizes [ <NUM_LIT> ] = { GridSizeX , GridSizeY , GridSizeZ } ; for ( int I = <NUM_LIT> ; HasUniformWorkGroupSize && I < <NUM_LIT> ; ++ I ) { Value * GroupSize = WorkGroupSizes [ I ] ; Value * GridSize = GridSizes [ I ] ; if ( ! GroupSize || ! GridSize ) continue ; for ( User * U : GroupSize -> users ( ) ) { auto * ZextGroupSize = dyn_cast < ZExtInst > ( U ) ; if ( ! ZextGroupSize ) continue ; for ( User * ZextUser : ZextGroupSize -> users ( ) ) { auto * <ISA_LIT> = dyn_cast < SelectInst > ( ZextUser ) ;


<s> bool <ISA_LIT>LowerKernelAttributes :: processUse ( CallInst * CI ) { Function * F = CI -> getParent ( ) -> getParent ( ) ; auto MD = F -> getMetadata ( <STR_LIT> ) ; const bool HasReqdWorkGroupSize = MD && MD -> getNumOperands ( ) == <NUM_LIT> ; const bool HasUniformWorkGroupSize = F -> getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) == <STR_LIT> ; if ( ! HasReqdWorkGroupSize && ! HasUniformWorkGroupSize ) return false ; Value * WorkGroupSizeX = nullptr ; Value * WorkGroupSizeY = nullptr ; Value * WorkGroupSizeZ = nullptr ; Value * GridSizeX = nullptr ; Value * GridSizeY = nullptr ; Value * GridSizeZ = nullptr ; const DataLayout & DL = Mod -> getDataLayout ( ) ; for ( User * U : CI -> users ( ) ) { if ( ! U -> hasOneUse ( ) ) continue ; int64_t Offset = <NUM_LIT> ; if ( GetPointerBaseWithConstantOffset ( U , Offset , DL ) != CI ) continue ; auto * BCI = dyn_cast < BitCastInst > ( * U -> user_begin ( ) ) ; if ( ! BCI || ! BCI -> hasOneUse ( ) ) continue ; auto * Load = dyn_cast < LoadInst > ( * BCI -> user_begin ( ) ) ; if ( ! Load || ! Load -> isSimple ( ) ) continue ; unsigned LoadSize = DL . getTypeStoreSize ( Load -> getType ( ) ) ; switch ( Offset ) { case WORKGROUP_<ISA_LIT>ZE_X : if ( LoadSize == <NUM_LIT> ) WorkGroupSizeX = Load ; break ; case WORKGROUP_<ISA_LIT>ZE_Y : if ( LoadSize == <NUM_LIT> ) WorkGroupSizeY = Load ; break ; case WORKGROUP_<ISA_LIT>ZE_Z : if ( LoadSize == <NUM_LIT> ) WorkGroupSizeZ = Load ; break ; case GRID_<ISA_LIT>ZE_X : if ( LoadSize == <NUM_LIT> ) GridSizeX = Load ; break ;


<s> CallInst * CI = cast < CallInst > ( U ) ; if ( HandledUses . insert ( CI ) . second ) { if ( processUse ( CI ) ) MadeChange = true ; }


<s> initialize<ISA_LIT>LowerKernelCallsPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> ValueToValueMapTy ignored ; Function * NewF = F . empty ( ) ? Function :: Create ( F . getFunctionType ( ) , Function :: ExternalLinkage , <STR_LIT> , F . getParent ( ) ) : CloneFunction ( & F , ignored ) ; NewF -> setCallingConv ( CallingConv :: C ) ; if ( ! NewF -> isDeclaration ( ) ) { NewF -> setVisibility ( GlobalValue :: DefaultVisibility ) ;

NewF -> setLinkage ( GlobalValue :: InternalLinkage ) ;


<s> if ( ! NewF -> isDeclaration ( ) ) { NewF -> setVisibility ( GlobalValue :: DefaultVisibility ) ; NewF -> setLinkage ( GlobalValue :: InternalLinkage ) ; }


<s> ModulePass * create<ISA_LIT>LowerKernelCallsPass ( ) {

return new <ISA_LIT>LowerKernelCalls ( ) ;


<s> return new <ISA_LIT>LowerKernelCalls ( ) ;


<s> StringRef getPassName ( ) const override {


<s> CallInst * CI = dyn_cast < CallInst > ( * UI ++ ) ; if ( ! CI ) continue ; if ( ! FBody ) FBody = cloneKernel ( F ) ;


<s> UI != UE ; ) { CallInst * CI = dyn_cast < CallInst > ( * UI ++ ) ; if ( ! CI ) continue ; if ( ! FBody ) FBody = cloneKernel ( F ) ;


<s> static void setNameForBody ( Function * FBody , const Function & FKernel ) { StringRef Name = FKernel . getName ( ) ; SmallString < <NUM_LIT> > NewName ( <STR_LIT> ) ; NewName += Name ; NewName += <STR_LIT> ; FBody -> setName ( NewName . str ( ) ) ;


<s> SmallString < <NUM_LIT> > NewName ( <STR_LIT> ) ; NewName += Name ; NewName += <STR_LIT> ;

FBody -> setName ( NewName . str ( ) ) ;


<s> continue ; } if ( auto * <ISA_LIT> = dyn_cast < StoreInst > ( U ) ) { if ( <ISA_LIT> -> getPointerOperand ( ) == Ptr ) <ISA_LIT> -> setAlignment ( std :: max ( A , <ISA_LIT> -> getAlign ( ) ) ) ; continue ; } if ( auto * AI = dyn_cast < AtomicRMWInst > ( U ) ) { if ( AI -> getPointerOperand ( ) == Ptr ) AI -> setAlignment ( std :: max ( A , AI -> getAlign ( ) ) ) ; continue ; } if ( auto * AI = dyn_cast < AtomicCmpXchgInst > ( U ) ) { if ( AI -> getPointerOperand ( ) == Ptr ) AI -> setAlignment ( std :: max ( A , AI -> getAlign ( ) ) ) ; continue ; } if ( auto * GEP = dyn_cast < GetElementPtrInst > ( U ) ) { unsigned BitWidth = DL . getIndexTypeSizeInBits ( GEP -> getType ( ) ) ; APInt Off ( BitWidth , <NUM_LIT> ) ; if ( GEP -> getPointerOperand ( ) == Ptr ) { Align GA ; if ( GEP -> accumulateConstantOffset ( DL , Off ) ) GA = commonAlignment ( A , Off . getLimitedValue ( ) ) ; refineUsesAlignmentAndAA ( GEP , GA , DL , AliasScope , NoAlias , MaxDepth - <NUM_LIT> ) ; }


<s> static void removeFromUsedLists ( Module & M , const std :: vector < GlobalVariable * > & LocalVars ) { SmallPtrSet < Constant * , <NUM_LIT> > LocalVarsSet ; for ( size_t I = <NUM_LIT> ; I < LocalVars . size ( ) ; I ++ ) { if ( Constant * C = dyn_cast < Constant > ( LocalVars [ I ] -> stripPointerCasts ( ) ) ) { LocalVarsSet . insert ( C ) ;


<s> UsedList = <STR_LIT> ( M ) ; bool Changed = processUsedLDS ( M ) ; for ( Function & F : M . functions ( ) ) { if ( F . isDeclaration ( ) ) continue ; if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) continue ; Changed |= processUsedLDS ( M , & F ) ;


<s> for ( Function & F : M . functions ( ) ) { if ( F . isDeclaration ( ) ) continue ; if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) continue ; Changed |= processUsedLDS ( M , & F ) ; }


<s> for ( size_t I = <NUM_LIT> ; I < LocalVars . size ( ) ; I ++ ) { GlobalVariable * GV = LocalVars [ I ] ; Constant * GEPIdx [ ] = { ConstantInt :: get ( I32 , <NUM_LIT> ) , ConstantInt :: get ( I32 , I ) } ; Constant * GEP = ConstantExpr :: getGetElementPtr ( LDSTy , SGV , GEPIdx ) ; if ( F ) { for ( User * U : make_early_inc_range ( GV -> users ( ) ) ) { if ( ConstantExpr * C = dyn_cast < ConstantExpr > ( U ) ) <STR_LIT> ( C , F ) ; } GV -> removeDeadConstantUsers ( ) ; GV -> replaceUsesWithIf ( GEP , [ F ] ( Use & U ) { Instruction * I = dyn_cast < Instruction > ( U . getUser ( ) ) ; return I && I -> getFunction ( ) == F ; } ) ; } else { GV -> replaceAllUsesWith ( GEP ) ; } if ( GV -> use_empty ( ) ) { UsedList . erase ( GV ) ; GV -> eraseFromParent ( ) ; } uint64_t Off = DL . getStructLayout ( LDSTy ) -> getElementOffset ( I ) ; Align A = commonAlignment ( StructAlign , Off ) ; if ( I ) NoAliasList [ I - <NUM_LIT> ] = AliasScopes [ I - <NUM_LIT> ] ; MDNode * NoAlias = NoAliasList . empty ( ) ? nullptr : MDNode :: get ( Ctx , NoAliasList ) ; MDNode * AliasScope = AliasScopes . empty ( ) ? nullptr : MDNode :: get ( Ctx , { AliasScopes [ I ] } ) ; refineUsesAlignmentAndAA ( GEP , A , DL , AliasScope , NoAlias ) ; } if ( ! F ) { IRBuilder < > Builder ( Ctx ) ;

for ( Function & Func : M . functions ( ) ) {


<s> bool runOnModule ( Module & M ) override { CallGraph CG = CallGraph ( M ) ; UsedList = getUsedList ( M ) ; bool Changed = superAlignLDSGlobals ( M ) ; Changed |= processUsedLDS ( CG , M ) ; for ( Function & F : M . functions ( ) ) { if ( F . isDeclaration ( ) ) continue ; if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) continue ; Changed |= processUsedLDS ( CG , M , & F ) ; } UsedList . clear ( ) ;


<s> for ( size_t I = <NUM_LIT> ; I < FoundLocalVars . size ( ) ; I ++ ) { GlobalVariable * FGV = FoundLocalVars [ I ] ; Align DataAlign = <STR_LIT> ( DL , FGV ) ; uint64_t DataAlignV = DataAlign . value ( ) ; if ( uint64_t Rem = CurrentOffset % DataAlignV ) { uint64_t Padding = DataAlignV - Rem ; Type * ATy = ArrayType :: get ( Type :: getInt8Ty ( Ctx ) , Padding ) ; LocalVars . push_back ( new GlobalVariable ( M , ATy , false , GlobalValue :: InternalLinkage , UndefValue :: get ( ATy ) , <STR_LIT> , nullptr , GlobalValue :: NotThreadLocal , <ISA_LIT>AS :: LOCAL_ADDRESS , false ) ) ; CurrentOffset += Padding ; } LocalVars . push_back ( FGV ) ; CurrentOffset += DL . getTypeAllocSize ( FGV -> getValueType ( ) ) ; } } std :: vector < Type * > LocalVarTypes ; LocalVarTypes . reserve ( LocalVars . size ( ) ) ; std :: transform ( LocalVars . cbegin ( ) , LocalVars . cend ( ) , std :: back_inserter ( LocalVarTypes ) , [ ] ( const GlobalVariable * V ) -> Type * { return V -> getValueType ( ) ; } ) ; std :: string VarName ( F ? ( Twine ( <STR_LIT> ) + F -> getName ( ) + <STR_LIT> ) . str ( ) : <STR_LIT> ) ; StructType * LDSTy = StructType :: create ( Ctx , LocalVarTypes , VarName + <STR_LIT> ) ; Align MaxAlign = <STR_LIT> ( DL , LocalVars [ <NUM_LIT> ] ) ; GlobalVariable * SGV = new GlobalVariable ( M , LDSTy , false , GlobalValue :: InternalLinkage , UndefValue :: get ( LDSTy ) , VarName , nullptr , GlobalValue :: NotThreadLocal , <ISA_LIT>AS :: LOCAL_ADDRESS , false ) ; SGV -> setAlignment ( MaxAlign ) ; if ( ! F ) { appendToCompilerUsed ( M , { static_cast < GlobalValue * > ( ConstantExpr :: getPointerBitCastOrAddrSpaceCast ( cast < Constant > ( SGV ) , Type :: getInt8PtrTy ( Ctx ) ) ) } ) ; } removeFromUsedLists ( M , LocalVars ) ; Type * I32 = Type :: getInt32Ty ( Ctx ) ; for ( size_t I = <NUM_LIT> ; I < LocalVars . size ( ) ; I ++ ) { GlobalVariable * GV = LocalVars [ I ] ; Constant * GEPIdx [ ] = { ConstantInt :: get ( I32 , <NUM_LIT> ) , ConstantInt :: get ( I32 , I ) } ; Constant * GEP = ConstantExpr :: getGetElementPtr ( LDSTy , SGV , GEPIdx ) ; if ( F ) { GV -> replaceUsesWithIf ( GEP , [ F ] ( Use & U ) { return <STR_LIT> ( U . getUser ( ) , F ) ; } ) ; } else {


<s> UsedList = <STR_LIT> ( M ) ; bool Changed = processUsedLDS ( M ) ; for ( Function & F : M . functions ( ) ) { if ( ! <STR_LIT> ( & F ) ) continue ;


<s> if ( ! <STR_LIT> ( & F ) ) continue ; Changed |= processUsedLDS ( M , & F ) ; } UsedList . clear ( ) ; return Changed ;


<s> LocalVars . push_back ( FGV ) ; CurrentOffset += LayoutFields [ I ] . Size ; } } std :: vector < Type * > LocalVarTypes ; LocalVarTypes . reserve ( LocalVars . size ( ) ) ; std :: transform ( LocalVars . cbegin ( ) , LocalVars . cend ( ) , std :: back_inserter ( LocalVarTypes ) , [ ] ( const GlobalVariable * V ) -> Type * { return V -> getValueType ( ) ; } ) ; std :: string VarName ( F ? ( Twine ( <STR_LIT> ) + F -> getName ( ) + <STR_LIT> ) . str ( ) : <STR_LIT> ) ; StructType * LDSTy = StructType :: create ( Ctx , LocalVarTypes , VarName + <STR_LIT> ) ; Align StructAlign = <STR_LIT> ( DL , LocalVars [ <NUM_LIT> ] ) ; GlobalVariable * SGV = new GlobalVariable ( M , LDSTy , false , GlobalValue :: InternalLinkage , UndefValue :: get ( LDSTy ) , VarName , nullptr , GlobalValue :: NotThreadLocal , <ISA_LIT>AS :: LOCAL_ADDRESS , false ) ; SGV -> setAlignment ( StructAlign ) ; if ( ! F ) { appendToCompilerUsed ( M , { static_cast < GlobalValue * > ( ConstantExpr :: getPointerBitCastOrAddrSpaceCast ( cast < Constant > ( SGV ) , Type :: getInt8PtrTy ( Ctx ) ) ) } ) ; } removeFromUsedLists ( M , LocalVars ) ; SmallVector < MDNode * > AliasScopes ; SmallVector < Metadata * > NoAliasList ; if ( LocalVars . size ( ) > <NUM_LIT> ) { MDBuilder MDB ( Ctx ) ; AliasScopes . reserve ( LocalVars . size ( ) ) ; MDNode * Domain = MDB . createAnonymousAliasScopeDomain ( ) ; for ( size_t I = <NUM_LIT> ; I < LocalVars . size ( ) ; I ++ ) { MDNode * Scope = MDB . createAnonymousAliasScope ( Domain ) ; AliasScopes . push_back ( Scope ) ; } NoAliasList . append ( & AliasScopes [ <NUM_LIT> ] , AliasScopes . end ( ) ) ; } Type * I32 = Type :: getInt32Ty ( Ctx ) ; for ( size_t I = <NUM_LIT> ; I < LocalVars . size ( ) ; I ++ ) {


<s> bool Changed = processUsedLDS ( M ) ; for ( Function & F : M . functions ( ) ) {

if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) continue ;


<s> return ; } SmallVector < Constant * , <NUM_LIT> > Init ; auto * CA = cast < ConstantArray > ( GV -> getInitializer ( ) ) ; for ( auto & Op : CA -> operands ( ) ) { Constant * C = cast < Constant > ( Op ) ; if ( ! ToRemove . contains ( C -> stripPointerCasts ( ) ) ) { Init . push_back ( C ) ; }


<s> } SmallVector < Constant * , <NUM_LIT> > Init ; auto * CA = cast < ConstantArray > ( GV -> getInitializer ( ) ) ; for ( auto & Op : CA -> operands ( ) ) { Constant * C = cast < Constant > ( Op ) ; if ( ! ToRemove . contains ( C -> stripPointerCasts ( ) ) ) { Init . push_back ( C ) ; } } if ( Init . size ( ) == CA -> getNumOperands ( ) ) { return ;


<s> std :: vector < Type * > LocalVarTypes ; LocalVarTypes . reserve ( LocalVars . size ( ) ) ; std :: transform ( LocalVars . cbegin ( ) , LocalVars . cend ( ) , std :: back_inserter ( LocalVarTypes ) , [ ] ( const GlobalVariable * V ) -> Type * { return V -> getValueType ( ) ; } ) ; StructType * LDSTy = StructType :: create ( Ctx , LocalVarTypes , llvm :: StringRef ( <STR_LIT> ) ) ; Align MaxAlign = <STR_LIT> ( DL , LocalVars [ <NUM_LIT> ] ) ; Constant * InstanceAddress = Constant :: getIntegerValue ( PointerType :: get ( LDSTy , <ISA_LIT>AS :: LOCAL_ADDRESS ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; GlobalVariable * SGV = new GlobalVariable ( M , LDSTy , false , GlobalValue :: InternalLinkage , UndefValue :: get ( LDSTy ) , <STR_LIT> , nullptr , GlobalValue :: NotThreadLocal , <ISA_LIT>AS :: LOCAL_ADDRESS , false ) ; SGV -> setAlignment ( MaxAlign ) ; appendToCompilerUsed ( M , { static_cast < GlobalValue * > ( ConstantExpr :: getPointerBitCastOrAddrSpaceCast ( cast < Constant > ( SGV ) , Type :: getInt8PtrTy ( Ctx ) ) ) } ) ; removeFromUsedLists ( M , LocalVars ) ; Type * I32 = Type :: getInt32Ty ( Ctx ) ; for ( size_t I = <NUM_LIT> ; I < LocalVars . size ( ) ; I ++ ) { GlobalVariable * GV = LocalVars [ I ] ; Constant * GEPIdx [ ] = { ConstantInt :: get ( I32 , <NUM_LIT> ) , ConstantInt :: get ( I32 , I ) } ; GV -> replaceAllUsesWith ( ConstantExpr :: getGetElementPtr ( LDSTy , InstanceAddress , GEPIdx ) ) ; GV -> eraseFromParent ( ) ; } { IRBuilder < > Builder ( Ctx ) ; SmallPtrSet < Function * , <NUM_LIT> > Kernels ;


<s> <ISA_LIT>LowerModuleLDS ( ) : ModulePass ( ID ) { initialize<ISA_LIT>LowerModuleLDSPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> ModulePass * llvm :: create<ISA_LIT>LowerModuleLDSPass ( ) { return new <ISA_LIT>LowerModuleLDS ( ) ;


<s> return new <ISA_LIT>LowerModuleLDS ( ) ;


<s> FunctionType * FTy = FunctionType :: get ( Type :: getVoidTy ( Ctx ) , { } ) ; Function * Decl = <STR_LIT> ( Func -> getParent ( ) , <STR_LIT> , { } ) ; Value * UseInstance [ <NUM_LIT> ] = { Builder . CreateInBoundsGEP ( SGV -> getValueType ( ) , SGV , ConstantInt :: get ( Type :: getInt32Ty ( Ctx ) , <NUM_LIT> ) ) } ; Builder . CreateCall ( FTy , Decl , { } , { OperandBundleDefT < Value * > ( <STR_LIT> , UseInstance ) } , <STR_LIT> ) ;


<s> } } if ( auto * LI = dyn_cast < LoadInst > ( U ) ) { LI -> setAlignment ( std :: max ( A , LI -> getAlign ( ) ) ) ; continue ; } if ( auto * <ISA_LIT> = dyn_cast < StoreInst > ( U ) ) { if ( <ISA_LIT> -> getPointerOperand ( ) == Ptr ) <ISA_LIT> -> setAlignment ( std :: max ( A , <ISA_LIT> -> getAlign ( ) ) ) ; continue ; } if ( auto * AI = dyn_cast < AtomicRMWInst > ( U ) ) { if ( AI -> getPointerOperand ( ) == Ptr ) AI -> setAlignment ( std :: max ( A , AI -> getAlign ( ) ) ) ; continue ; } if ( auto * AI = dyn_cast < AtomicCmpXchgInst > ( U ) ) { if ( AI -> getPointerOperand ( ) == Ptr ) AI -> setAlignment ( std :: max ( A , AI -> getAlign ( ) ) ) ; continue ; } if ( auto * GEP = dyn_cast < GetElementPtrInst > ( U ) ) { unsigned BitWidth = DL . getIndexTypeSizeInBits ( GEP -> getType ( ) ) ; APInt Off ( BitWidth , <NUM_LIT> ) ; if ( GEP -> getPointerOperand ( ) == Ptr ) { Align GA ; if ( GEP -> accumulateConstantOffset ( DL , Off ) ) GA = commonAlignment ( A , Off . getLimitedValue ( ) ) ; refineUsesAlignmentAndAA ( GEP , GA , DL , AliasScope , NoAlias , MaxDepth - <NUM_LIT> ) ; } continue ; } if ( auto * I = dyn_cast < Instruction > ( U ) ) { if ( I -> getOpcode ( ) == Instruction :: BitCast || I -> getOpcode ( ) == Instruction :: AddrSpaceCast ) refineUsesAlignmentAndAA ( I , A , DL , AliasScope , NoAlias , MaxDepth - <NUM_LIT> ) ; } }


<s> GV -> eraseFromParent ( ) ; for ( Constant * C : ToRemove ) { C -> removeDeadConstantUsers ( ) ; } if ( ! Init . empty ( ) ) { ArrayType * ATy = ArrayType :: get ( Type :: getInt8PtrTy ( M . getContext ( ) ) , Init . size ( ) ) ; GV = new llvm :: GlobalVariable ( M , ATy , false , GlobalValue :: AppendingLinkage , ConstantArray :: get ( ATy , Init ) , Name ) ; GV -> setSection ( <STR_LIT> ) ; }


<s> return ; } SmallVector < Constant * , <NUM_LIT> > Init ; auto * CA = cast < ConstantArray > ( GV -> getInitializer ( ) ) ; for ( auto & Op : CA -> operands ( ) ) { Constant * C = cast < Constant > ( Op ) ; if ( ! ToRemove . contains ( C -> stripPointerCasts ( ) ) ) { Init . push_back ( C ) ;


<s> static void removeFromUsedLists ( Module & M , const std :: vector < GlobalVariable * > & LocalVars ) { SmallPtrSet < Constant * , <NUM_LIT> > LocalVarsSet ; for ( GlobalVariable * LocalVar : LocalVars ) if ( Constant * C = dyn_cast < Constant > ( LocalVar -> stripPointerCasts ( ) ) ) LocalVarsSet . insert ( C ) ; removeFromUsedList ( M , <STR_LIT> , LocalVarsSet ) ;


<s> PreservedAnalyses <ISA_LIT>LowerModuleLDSPass :: run ( Module & M , ModuleAnalysisManager & ) {

return <ISA_LIT>LowerModuleLDS ( ) . runOnModule ( M ) ? PreservedAnalyses :: none ( ) : PreservedAnalyses :: all ( ) ;


<s> PreservedAnalyses <ISA_LIT>LowerModuleLDSPass :: run ( Module & M , ModuleAnalysisManager & ) { return <ISA_LIT>LowerModuleLDS ( ) . runOnModule ( M ) ? PreservedAnalyses :: none ( ) : PreservedAnalyses :: all ( ) ;


<s> const DataLayout & DL = M . getDataLayout ( ) ; bool Changed = false ; if ( ! SuperAlignLDSGlobals ) { return Changed ; } for ( auto & GV : M . globals ( ) ) { if ( GV . getType ( ) -> getPointerAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) { continue ; } if ( ! GV . hasInitializer ( ) ) { continue ; } Align Alignment = <STR_LIT> ( DL , & GV ) ; TypeSize GVSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; if ( GVSize > <NUM_LIT> ) { Alignment = std :: max ( Alignment , Align ( <NUM_LIT> ) ) ; } else if ( GVSize > <NUM_LIT> ) { Alignment = std :: max ( Alignment , Align ( <NUM_LIT> ) ) ; } else if ( GVSize > <NUM_LIT> ) { Alignment = std :: max ( Alignment , Align ( <NUM_LIT> ) ) ; } else if ( GVSize > <NUM_LIT> ) { Alignment = std :: max ( Alignment , Align ( <NUM_LIT> ) ) ; } if ( Alignment != <STR_LIT> ( DL , & GV ) ) { Changed = true ; GV . setAlignment ( Alignment ) ; } } return Changed ;


<s> } if ( ! GV . hasInitializer ( ) ) { continue ; } Align Alignment = <STR_LIT> ( DL , & GV ) ; TypeSize GVSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; if ( GVSize > <NUM_LIT> ) { Alignment = std :: max ( Alignment , Align ( <NUM_LIT> ) ) ; } else if ( GVSize > <NUM_LIT> ) { Alignment = std :: max ( Alignment , Align ( <NUM_LIT> ) ) ; } else if ( GVSize > <NUM_LIT> ) { Alignment = std :: max ( Alignment , Align ( <NUM_LIT> ) ) ; } else if ( GVSize > <NUM_LIT> ) {


<s> createEntryPHIs ( CurrentRegion ) ; removeExternalCFGSuccessors ( CodeBB ) ; CodeBB -> addSuccessor ( MergeBB ) ; CurrentRegion -> addMBB ( CodeBB ) ; return nullptr ; } else { const TargetRegisterClass * RegClass = MRI -> getRegClass ( BBSelectRegIn ) ; Register CodeBBSelectReg = MRI -> createVirtualRegister ( RegClass ) ; rewriteCodeBBTerminator ( CodeBB , MergeBB , CodeBBSelectReg ) ; bool IsRegionEntryBB = CurrentRegion -> getEntry ( ) == CodeBB ; MachineBasicBlock * IfBB = createIfBlock ( MergeBB , CodeBB , CodeBB , CodeBB , BBSelectRegIn , IsRegionEntryBB ) ; CurrentRegion -> addMBB ( IfBB ) ; if ( IsRegionEntryBB ) { CurrentRegion -> setEntry ( IfBB ) ; if ( CurrentRegion -> getHasLoop ( ) ) { MachineBasicBlock * RegionExit = CurrentRegion -> getExit ( ) ; MachineBasicBlock * ETrueBB = nullptr ; MachineBasicBlock * EFalseBB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > ECond ; const DebugLoc & DL = DebugLoc ( ) ; TII -> analyzeBranch ( * RegionExit , ETrueBB , EFalseBB , ECond ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } assert ( MRI -> hasOneDef ( Reg ) && <STR_LIT> ) ; MachineOperand * Def = & ( * ( MRI -> def_begin ( Reg ) ) ) ; MachineOperand * UseOperand = & ( RI ) ; bool UseIsOutsideDefMBB = Def -> getParent ( ) -> getParent ( ) != MBB ; if ( UseIsOutsideDefMBB && UseOperand -> isKill ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ;


<s> } for ( MachineRegisterInfo :: reg_iterator I = MRI -> reg_begin ( Register ) , E = MRI -> reg_end ( ) ; I != E ; ) { MachineOperand & O = * I ; ++ I ; if ( O . isDef ( ) ) continue ; bool IsInside = contains ( O . getParent ( ) -> getParent ( ) ) ; bool IsLoopPHI = IsInside && ( O . getParent ( ) -> isPHI ( ) && O . getParent ( ) -> getParent ( ) == getEntry ( ) ) ; bool ShouldReplace = ( IsInside && ReplaceInside ) || ( ! IsInside && ReplaceOutside ) || ( IncludeLoopPHI && IsLoopPHI ) ; if ( ShouldReplace ) { if ( Register :: isPhysicalRegister ( NewRegister ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; llvm_unreachable ( <STR_LIT> ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Register , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; O . setReg ( NewRegister ) ; } } }


<s> MachineInstr * UseInstr = UI . getParent ( ) ; for ( MachineBasicBlock :: instr_iterator MII = UseInstr -> getIterator ( ) , MIE = UseInstr -> getParent ( ) -> instr_end ( ) ; MII != MIE ; ++ MII ) { if ( ( & ( * MII ) ) == DefInstr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } } } } } }


<s> if ( ! Region -> contains ( UI . getParent ( ) -> getParent ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( void * ) Region << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } }


<s> PHIInfoElementT * NewElement = new PHIInfoElementT ( ) ; NewElement -> DestReg = DestReg ; NewElement -> DL = DL ;


<s> assert ( findPHIInfoElement ( DestReg ) == nullptr && <STR_LIT> ) ; PHISourcesT EmptySet ; PHIInfoElementT * NewElement = new PHIInfoElementT ( ) ; NewElement -> DestReg = DestReg ; NewElement -> DL = DL ; NewElement -> Sources = EmptySet ;


<s> if ( Instr . isPHI ( ) ) { unsigned PHIDestReg = getPHIDestReg ( Instr ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( Instr . dump ( ) ) ;


<s> if ( Instr . isPHI ( ) ) { unsigned PHIDestReg = getPHIDestReg ( Instr ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( Instr . dump ( ) ) ; PHIs . insert ( & Instr ) ; PHIInfo . addDest ( PHIDestReg , Instr . getDebugLoc ( ) ) ; storePHILinearizationInfoDest ( PHIDestReg , Instr ) ; } } for ( auto PI : PHIs ) { PI -> eraseFromParent ( ) ; }


<s> MachineBasicBlock * result = nullptr ; for ( auto & MFI : MF ) { if ( MFI . succ_empty ( ) ) { if ( result == nullptr ) { result = & MFI ; } else { return nullptr ; } } }


<s> MachineBasicBlock * MRT :: initializeMRT ( MachineFunction & MF , const MachineRegionInfo * RegionInfo , DenseMap < MachineRegion * , RegionMRT * > & RegionMap ) { for ( auto & MFI : MF ) { MachineBasicBlock * ExitMBB = & MFI ; if ( ExitMBB -> succ_empty ( ) ) {


<s> MachineBasicBlock * MRT :: initializeMRT ( MachineFunction & MF , const MachineRegionInfo * RegionInfo , DenseMap < MachineRegion * , RegionMRT * > & RegionMap ) { for ( auto & MFI : MF ) { MachineBasicBlock * ExitMBB = & MFI ; if ( ExitMBB -> succ_empty ( ) ) { return ExitMBB ; } } llvm_unreachable ( <STR_LIT> ) ;


<s> if ( ReplaceOutside && ( isLiveOut ( Register ) || this -> getParent ( ) -> isLiveOut ( Register ) ) ) { LinearizedRegion * Current = this ; while ( Current != nullptr && Current -> getEntry ( ) != nullptr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( Current -> print ( dbgs ( ) , MRI -> getTargetRegisterInfo ( ) ) ) ; Current -> replaceLiveOut ( Register , NewRegister ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( Current -> print ( dbgs ( ) , MRI -> getTargetRegisterInfo ( ) ) ) ; Current = Current -> getParent ( ) ; } } for ( MachineRegisterInfo :: reg_iterator I = MRI -> reg_begin ( Register ) , E = MRI -> reg_end ( ) ; I != E ; ) { MachineOperand & O = * I ;


<s> RegionMRT * MRT :: buildMRT ( MachineFunction & MF , const MachineRegionInfo * RegionInfo , const <ISA_LIT>InstrInfo * TII , MachineRegisterInfo * MRI ) { SmallPtrSet < MachineRegion * , <NUM_LIT> > PlacedRegions ; DenseMap < MachineRegion * , RegionMRT * > RegionMap ; MachineRegion * TopLevelRegion = RegionInfo -> getTopLevelRegion ( ) ; RegionMRT * Result = new RegionMRT ( TopLevelRegion ) ; RegionMap [ TopLevelRegion ] = Result ; MachineBasicBlock * Exit = initializeMRT ( MF , RegionInfo , RegionMap ) ; unsigned BBSelectRegIn = createBBSelectReg ( TII , MRI ) ; MBBMRT * ExitMRT = new MBBMRT ( Exit ) ; RegionMap [ RegionInfo -> getRegionFor ( Exit ) ] -> addChild ( ExitMRT ) ; ExitMRT -> setBBSelectRegIn ( BBSelectRegIn ) ; for ( auto MBBI : post_order ( & ( MF . front ( ) ) ) ) { MachineBasicBlock * MBB = & ( * MBBI ) ; if ( MBB == Exit ) { continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ) ; MBBMRT * NewMBB = new MBBMRT ( MBB ) ; MachineRegion * Region = RegionInfo -> getRegionFor ( MBB ) ;

if ( RegionMap . count ( Region ) == <NUM_LIT> ) {


<s> if ( RegionMap . count ( Region ) == <NUM_LIT> ) { RegionMRT * NewMRTRegion = new RegionMRT ( Region ) ; RegionMap [ Region ] = NewMRTRegion ; MachineRegion * Parent = Region -> getParent ( ) ; while ( RegionMap . count ( Parent ) == <NUM_LIT> ) { RegionMRT * NewMRTParent = new RegionMRT ( Parent ) ; NewMRTParent -> addChild ( NewMRTRegion ) ; NewMRTRegion -> setParent ( NewMRTParent ) ; RegionMap [ Parent ] = NewMRTParent ; NewMRTRegion = NewMRTParent ; Parent = Parent -> getParent ( ) ; } RegionMap [ Parent ] -> addChild ( NewMRTRegion ) ; NewMRTRegion -> setParent ( RegionMap [ Parent ] ) ; } RegionMap [ Region ] -> addChild ( NewMBB ) ; NewMBB -> setParent ( RegionMap [ Region ] ) ; RegionMap [ Region ] -> setSucc ( Region -> getExit ( ) ) ; } return Result ;


<s> } else { MachineInstr * PHIDefInstr = getDefInstr ( SourceReg ) ; MachineBasicBlock * PHIDefMBB = PHIDefInstr -> getParent ( ) ; const TargetRegisterClass * RegClass = MRI -> getRegClass ( CurrentBackedgeReg ) ; unsigned NewBackedgeReg = MRI -> createVirtualRegister ( RegClass ) ; MachineInstrBuilder BackedgePHI = BuildMI ( * PHIDefMBB , PHIDefMBB -> instr_begin ( ) , DL , TII -> get ( TargetOpcode :: PHI ) , NewBackedgeReg ) ; BackedgePHI . addReg ( CurrentBackedgeReg ) ; BackedgePHI . addMBB ( getPHIPred ( * PHIDefInstr , <NUM_LIT> ) ) ; BackedgePHI . addReg ( getPHISourceReg ( * PHIDefInstr , <NUM_LIT> ) ) ; BackedgePHI . addMBB ( ( * SRI ) . second ) ; CurrentBackedgeReg = NewBackedgeReg ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( NewBackedgeReg , TRI ) << <STR_LIT> << PrintReg ( CurrentBackedgeReg , TRI ) << <STR_LIT> << getPHIPred ( * PHIDefInstr , <NUM_LIT> ) -> getNumber ( ) << <STR_LIT> << PrintReg ( getPHISourceReg ( * PHIDefInstr , <NUM_LIT> ) , TRI ) << <STR_LIT> << ( * SRI ) . second -> getNumber ( ) ) ; }


<s> for ( auto DRI = PHIInfo . dests_begin ( ) , DE = PHIInfo . dests_end ( ) ; DRI != DE ; ++ DRI ) { unsigned DestReg = * DRI ; createEntryPHI ( CurrentRegion , DestReg ) ; } PHIInfo . clear ( ) ;


<s> for ( MachineBasicBlock :: pred_iterator PI = CodeBBStart -> pred_begin ( ) , E = CodeBBStart -> pred_end ( ) ; PI != E ; ++ PI ) { if ( ( * PI ) != CodeBBEnd ) { MachineBasicBlock * Pred = ( * PI ) ; Pred -> addSuccessor ( IfBB ) ; } } } removeExternalCFGEdges ( CodeBBStart , CodeBBEnd ) ; auto CodeBBStartI = CodeBBStart -> getIterator ( ) ; auto CodeBBEndI = CodeBBEnd -> getIterator ( ) ; auto MergeIter = MergeBB -> getIterator ( ) ; MF -> insert ( MergeIter , IfBB ) ; MF -> splice ( MergeIter , CodeBBStartI , ++ CodeBBEndI ) ; IfBB -> addSuccessor ( MergeBB ) ; IfBB -> addSuccessor ( CodeBBStart ) ; DEBUG ( dbgs ( ) << <STR_LIT> << IfBB -> getNumber ( ) << <STR_LIT> ) ; if ( ! CodeBBEnd -> isSuccessor ( MergeBB ) ) CodeBBEnd -> addSuccessor ( MergeBB ) ; DEBUG ( dbgs ( ) << <STR_LIT> << CodeBBStart -> getNumber ( ) << <STR_LIT> << CodeBBEnd -> getNumber ( ) << <STR_LIT> ) ; MachineBasicBlock * SinglePred = CodeBBStart -> pred_size ( ) == <NUM_LIT> ? * ( CodeBBStart -> pred_begin ( ) ) : nullptr ; const DebugLoc & DL = SinglePred ? SinglePred -> findDebugLoc ( SinglePred -> getFirstTerminator ( ) ) : DebugLoc ( ) ; unsigned Reg = TII -> insertEQ ( IfBB , IfBB -> begin ( ) , DL , IfReg , SelectBB -> getNumber ( ) ) ;

if ( & ( * ( IfBB -> getParent ( ) -> begin ( ) ) ) == IfBB ) {


<s> } } removeExternalCFGEdges ( CodeBBStart , CodeBBEnd ) ; auto CodeBBStartI = CodeBBStart -> getIterator ( ) ; auto CodeBBEndI = CodeBBEnd -> getIterator ( ) ; auto MergeIter = MergeBB -> getIterator ( ) ; MF -> insert ( MergeIter , IfBB ) ; MF -> splice ( MergeIter , CodeBBStartI , ++ CodeBBEndI ) ; IfBB -> addSuccessor ( MergeBB ) ; IfBB -> addSuccessor ( CodeBBStart ) ; DEBUG ( dbgs ( ) << <STR_LIT> << IfBB -> getNumber ( ) << <STR_LIT> ) ; if ( ! CodeBBEnd -> isSuccessor ( MergeBB ) ) CodeBBEnd -> addSuccessor ( MergeBB ) ; DEBUG ( dbgs ( ) << <STR_LIT> << CodeBBStart -> getNumber ( ) << <STR_LIT> << CodeBBEnd -> getNumber ( ) << <STR_LIT> ) ; MachineBasicBlock * SinglePred = CodeBBStart -> pred_size ( ) == <NUM_LIT> ? * ( CodeBBStart -> pred_begin ( ) ) : nullptr ; const DebugLoc & DL = SinglePred ? SinglePred -> findDebugLoc ( SinglePred -> getFirstTerminator ( ) ) : DebugLoc ( ) ; unsigned Reg = TII -> insertEQ ( IfBB , IfBB -> begin ( ) , DL , IfReg , SelectBB -> getNumber ( ) ) ; if ( & ( * ( IfBB -> getParent ( ) -> begin ( ) ) ) == IfBB ) {

TII -> materializeImmediate ( * IfBB , IfBB -> begin ( ) , DL , IfReg , CodeBBStart -> getNumber ( ) ) ;


<s> unsigned CodeBBSelectReg = InnerRegion -> getRegionMRT ( ) -> getInnerOutputRegister ( ) ; MachineBasicBlock * CodeEntryBB = InnerRegion -> getEntry ( ) ; MachineBasicBlock * CodeExitBB = InnerRegion -> getExit ( ) ; MachineBasicBlock * IfBB = createIfBlock ( MergeBB , CodeEntryBB , CodeExitBB , SelectBB , BBSelectRegIn , true ) ; CurrentRegion -> addMBB ( IfBB ) ; bool isEntry = CurrentRegion -> getEntry ( ) == InnerRegion -> getEntry ( ) ; if ( isEntry ) { if ( CurrentRegion -> getHasLoop ( ) ) { MachineBasicBlock * RegionExit = CurrentRegion -> getExit ( ) ; MachineBasicBlock * ETrueBB = nullptr ; MachineBasicBlock * EFalseBB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > ECond ; const DebugLoc & DL = DebugLoc ( ) ; TII -> analyzeBranch ( * RegionExit , ETrueBB , EFalseBB , ECond ) ; TII -> removeBranch ( * RegionExit ) ;

unsigned Reg = TII -> insertNE ( RegionExit , RegionExit -> instr_end ( ) , DL , CurrentRegion -> getRegionMRT ( ) -> getInnerOutputRegister ( ) , CurrentRegion -> getRegionMRT ( ) -> getEntry ( ) -> getNumber ( ) ) ;


<s> if ( Exit == nullptr && hasOneExitNode ( * MF ) ) { return & ( * ( -- ( Region -> getEntry ( ) -> getParent ( ) -> end ( ) ) ) ) ; } MachineBasicBlock * LastMerge = MF -> CreateMachineBasicBlock ( ) ; if ( Exit == nullptr ) { MachineFunction :: iterator ExitIter = MF -> end ( ) ; MF -> insert ( ExitIter , LastMerge ) ; } else { MachineFunction :: iterator ExitIter = Exit -> getIterator ( ) ; MF -> insert ( ExitIter , LastMerge ) ; LastMerge -> addSuccessor ( Exit ) ; insertUnconditionalBranch ( LastMerge , Exit ) ; DEBUG ( dbgs ( ) << <STR_LIT> << LastMerge -> getNumber ( ) << <STR_LIT> ) ; }


<s> LRegion -> setRegionMRT ( Region ) ; Region -> setLinearizedRegion ( LRegion ) ;

LRegion -> setParent ( Region -> getParent ( ) ? Region -> getParent ( ) -> getLinearizedRegion ( ) : nullptr ) ;


<s> dbgs ( ) << <STR_LIT> << PrintReg ( getBBSelectRegIn ( ) , TRI ) ; dbgs ( ) << <STR_LIT> << PrintReg ( getBBSelectRegOut ( ) , TRI ) << <STR_LIT> ; dumpDepth ( depth ) ; if ( getSucc ( ) ) dbgs ( ) << <STR_LIT> << getSucc ( ) -> getNumber ( ) << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ; for ( auto MRTI : Children ) {

MRTI -> dump ( TRI , depth + <NUM_LIT> ) ;


<s> if ( MRI -> def_begin ( Reg ) == MRI -> def_end ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } else if ( ! MRI -> hasOneDef ( Reg ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ;

for ( auto DI = MRI -> def_begin ( Reg ) , DE = MRI -> def_end ( ) ; DI != DE ; ++ DI ) {


<s> DEBUG ( dbgs ( ) << <STR_LIT> << MBBI . getNumber ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ) ; }


<s> if ( IsSingleBB && DefInstr -> getParent ( ) == InnerRegion -> getEntry ( ) ) { InnerRegion -> replaceRegisterOutsideRegion ( SourceReg , DestReg , false , MRI ) ; } const TargetRegisterClass * RegClass = MRI -> getRegClass ( DestReg ) ; unsigned NextDestReg = MRI -> createVirtualRegister ( RegClass ) ; bool IsLastDef = PHIInfo . getNumSources ( DestReg ) == <NUM_LIT> ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; insertMergePHI ( IfBB , InnerRegion -> getExit ( ) , MergeBB , DestReg , NextDestReg , SourceReg , IsLastDef ) ; PHIInfo . removeSource ( DestReg , SourceReg , CodeBB ) ; if ( IsLastDef ) { const DebugLoc & DL = IfBB -> findDebugLoc ( IfBB -> getFirstTerminator ( ) ) ; TII -> materializeImmediate ( * IfBB , IfBB -> getFirstTerminator ( ) , DL , NextDestReg , <NUM_LIT> ) ; PHIInfo . deleteDef ( DestReg ) ; } else { PHIInfo . replaceDef ( DestReg , NextDestReg ) ; } }


<s> MachineInstrBuilder MIB = BuildMI ( * MergeBB , MergeBB -> instr_begin ( ) , DL , TII -> get ( TargetOpcode :: PHI ) , DestRegister ) ; if ( IsUndefIfSource && false ) { MIB . addReg ( IfSourceRegister , RegState :: Undef ) ; } else { MIB . addReg ( IfSourceRegister ) ; }


<s> } OS << <STR_LIT> << Entry -> getNumber ( ) << <STR_LIT> << ( Exit == nullptr ? - <NUM_LIT> : Exit -> getNumber ( ) ) << <STR_LIT> << PrintReg ( getBBSelectRegIn ( ) , TRI ) << <STR_LIT> << PrintReg ( getBBSelectRegOut ( ) , TRI ) << <STR_LIT> ; for ( auto & LI : LiveOuts ) {

OS << PrintReg ( LI , TRI ) << <STR_LIT> ;


<s> if ( MBBContainsPHISource ) { for ( auto SRI = PHIInfo . sources_begin ( DestReg ) ; SRI != SE ; ++ SRI ) { PHILinearize :: PHISourceT Source = * SRI ; unsigned SourceReg = Source . first ; MachineBasicBlock * SourceMBB = Source . second ; MachineOperand * Def = & ( * ( MRI -> def_begin ( SourceReg ) ) ) ; if ( Def -> getParent ( ) -> getParent ( ) != MBB ) { ElimiatedSources . push_back ( std :: make_tuple ( DestReg , SourceReg , SourceMBB ) ) ; } } } } for ( auto & SourceInfo : ElimiatedSources ) { PHIInfo . removeSource ( std :: get < <NUM_LIT> > ( SourceInfo ) , std :: get < <NUM_LIT> > ( SourceInfo ) , std :: get < <NUM_LIT> > ( SourceInfo ) ) ;


<s> unsigned SuccSize = StartMBB -> succ_size ( ) ; if ( SuccSize > <NUM_LIT> ) { MachineBasicBlock * StartMBBSucc = * ( StartMBB -> succ_begin ( ) ) ; for ( MachineBasicBlock :: succ_iterator PI = EndMBB -> succ_begin ( ) , E = EndMBB -> succ_end ( ) ; PI != E ; ++ PI ) { if ( ( * PI ) != StartMBB && ! ( ( * PI ) == StartMBBSucc && StartMBB != EndMBB && SuccSize == <NUM_LIT> ) ) { Succs . insert ( std :: pair < MachineBasicBlock * , MachineBasicBlock * > ( EndMBB , * PI ) ) ; } } } for ( MachineBasicBlock :: pred_iterator PI = StartMBB -> pred_begin ( ) , E = StartMBB -> pred_end ( ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } assert ( MRI -> hasOneDef ( Reg ) && <STR_LIT> ) ; MachineOperand * Def = & ( * ( MRI -> def_begin ( Reg ) ) ) ; MachineOperand * UseOperand = & ( RI ) ; bool UseIsOutsideDefMBB = Def -> getParent ( ) -> getParent ( ) != MBB ; if ( UseIsOutsideDefMBB && UseOperand -> isKill ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> ) ; UseOperand -> setIsKill ( false ) ; } }


<s> assert ( MRI -> hasOneDef ( Reg ) && <STR_LIT> ) ; MachineOperand * Def = & ( * ( MRI -> def_begin ( Reg ) ) ) ; MachineOperand * UseOperand = & ( RI ) ; bool UseIsOutsideDefMBB = Def -> getParent ( ) -> getParent ( ) != MBB ; if ( UseIsOutsideDefMBB && UseOperand -> isKill ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> ) ; UseOperand -> setIsKill ( false ) ; } } } }


<s> for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { if ( isPHIRegionIndex ( PHIRegionIndices , i ) ) { NumNonRegionInputs -- ; } } if ( NumNonRegionInputs == <NUM_LIT> ) { auto DestReg = getPHIDestReg ( PHI ) ; replaceRegisterWith ( DestReg , CombinedSourceReg ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( CombinedSourceReg , TRI ) << <STR_LIT> ) ; PHI . eraseFromParent ( ) ; } else { DEBUG ( dbgs ( ) << PrintReg ( getPHIDestReg ( PHI ) , TRI ) << <STR_LIT> ) ; MachineBasicBlock * MBB = PHI . getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , PHI , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , getPHIDestReg ( PHI ) ) ; MIB . addReg ( CombinedSourceReg ) ; MIB . addMBB ( IfMBB ) ; DEBUG ( dbgs ( ) << PrintReg ( CombinedSourceReg , TRI ) << <STR_LIT> << IfMBB -> getNumber ( ) ) ; unsigned NumInputs = getPHINumInputs ( PHI ) ;


<s> } DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> << ( IsDead ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> ) ; if ( IsDead ) { LRegion -> removeLiveOut ( Reg ) ; } WasLiveOut = true ; } } if ( WasLiveOut ) LRegion -> addLiveOut ( CombinedSourceReg ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( PHI . dump ( ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( getPHIDestReg ( PHI ) , TRI ) << <STR_LIT> ) ; bool HasExternalEdge = false ; unsigned NumInputs = getPHINumInputs ( PHI ) ; for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { if ( ! isPHIRegionIndex ( PHIRegionIndices , i ) ) { HasExternalEdge = true ; } } if ( HasExternalEdge ) { MachineBasicBlock * MBB = PHI . getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , PHI , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , getPHIDestReg ( PHI ) ) ; MIB . addReg ( CombinedSourceReg ) ; MIB . addMBB ( LastMerge ) ; DEBUG ( dbgs ( ) << PrintReg ( CombinedSourceReg , TRI ) << <STR_LIT> << LastMerge -> getNumber ( ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { if ( isPHIRegionIndex ( PHIRegionIndices , i ) ) { continue ; } unsigned SourceReg = getPHISourceReg ( PHI , i ) ; MachineBasicBlock * SourcePred = getPHIPred ( PHI , i ) ; MIB . addReg ( SourceReg ) ; MIB . addMBB ( SourcePred ) ; DEBUG ( dbgs ( ) << PrintReg ( SourceReg , TRI ) << <STR_LIT> << SourcePred -> getNumber ( ) ) ;


<s> Current -> replaceLiveOut ( Register , NewRegister ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( Current -> print ( dbgs ( ) , MRI -> getTargetRegisterInfo ( ) ) ) ; Current = Current -> getParent ( ) ; } } for ( MachineRegisterInfo :: reg_iterator I = MRI -> reg_begin ( Register ) , E = MRI -> reg_end ( ) ; I != E ; ) { MachineOperand & O = * I ; ++ I ; if ( O . isDef ( ) ) continue ; bool IsInside = contains ( O . getParent ( ) -> getParent ( ) ) ; bool IsLoopPHI = IsInside && ( O . getParent ( ) -> isPHI ( ) && O . getParent ( ) -> getParent ( ) == getEntry ( ) ) ; bool ShouldReplace = ( IsInside && ReplaceInside ) || ( ! IsInside && ReplaceOutside ) || ( IncludeLoopPHI && IsLoopPHI ) ; if ( ShouldReplace ) { if ( TargetRegisterInfo :: isPhysicalRegister ( NewRegister ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; llvm_unreachable ( <STR_LIT> ) ; } else {

DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Register , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> << PrintReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ;


<s> DEBUG ( Current -> print ( dbgs ( ) , MRI -> getTargetRegisterInfo ( ) ) ) ; Current -> replaceLiveOut ( Register , NewRegister ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( Current -> print ( dbgs ( ) , MRI -> getTargetRegisterInfo ( ) ) ) ; Current = Current -> getParent ( ) ; } } for ( MachineRegisterInfo :: reg_iterator I = MRI -> reg_begin ( Register ) , E = MRI -> reg_end ( ) ; I != E ; ) { MachineOperand & O = * I ; ++ I ; if ( O . isDef ( ) ) continue ; bool IsInside = contains ( O . getParent ( ) -> getParent ( ) ) ; bool IsLoopPHI = IsInside && ( O . getParent ( ) -> isPHI ( ) && O . getParent ( ) -> getParent ( ) == getEntry ( ) ) ; bool ShouldReplace = ( IsInside && ReplaceInside ) || ( ! IsInside && ReplaceOutside ) || ( IncludeLoopPHI && IsLoopPHI ) ; if ( ShouldReplace ) { if ( TargetRegisterInfo :: isPhysicalRegister ( NewRegister ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; llvm_unreachable ( <STR_LIT> ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Register , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> << PrintReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; O . setReg ( NewRegister ) ; } }


<s> ++ I ; if ( TargetRegisterInfo :: isPhysicalRegister ( NewRegister ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; llvm_unreachable ( <STR_LIT> ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Register , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> << PrintReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; O . setReg ( NewRegister ) ; } } PHIInfo . deleteDef ( Register ) ; getRegionMRT ( ) -> replaceLiveOutReg ( Register , NewRegister ) ;


<s> auto SRI = PHIInfo . sources_begin ( DestReg ) ; unsigned SourceReg = ( * SRI ) . first ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( DestReg , TRI ) << <STR_LIT> << PrintReg ( SourceReg , TRI ) << <STR_LIT> ) ;

assert ( PHIInfo . sources_end ( DestReg ) == ++ SRI && <STR_LIT> ) ;


<s> DenseSet < unsigned > * LiveOuts = InnerRegion -> getLiveOuts ( ) ; SmallVector < unsigned , <NUM_LIT> > OldLiveOuts ; bool IsSingleBB = InnerRegion -> getEntry ( ) == InnerRegion -> getExit ( ) ; for ( auto OLI : * LiveOuts ) { OldLiveOuts . push_back ( OLI ) ; } for ( auto LI : OldLiveOuts ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( LI , TRI ) ) ; if ( ! containsDef ( CodeBB , InnerRegion , LI ) || ( ! IsSingleBB && ( getDefInstr ( LI ) -> getParent ( ) == LRegion -> getExit ( ) ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; unsigned Reg = LI ;

if ( Reg != InnerRegion -> getBBSelectRegOut ( ) ) {


<s> Regions = & ( getAnalysis < MachineRegionInfoPass > ( ) . getRegionInfo ( ) ) ; DEBUG ( Regions -> dump ( ) ) ; RegionMRT * RTree = MRT :: buildMRT ( MF , Regions , TII , MRI ) ; setRegionMRT ( RTree ) ; initializeSelectRegisters ( RTree , <NUM_LIT> , MRI , TII ) ; DEBUG ( RTree -> dump ( TRI ) ) ; bool result = structurizeRegions ( RTree , true ) ; delete RTree ; DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> bool <ISA_LIT>MachineCFGStructurizer :: shrinkPHI ( MachineInstr & PHI , unsigned CombinedSourceReg , MachineBasicBlock * SourceMBB , SmallVector < unsigned , <NUM_LIT> > & PHIIndices , unsigned * ReplaceReg ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( PHI . dump ( ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( getPHIDestReg ( PHI ) , TRI ) << <STR_LIT> ) ; bool Replaced = false ; unsigned NumInputs = getPHINumInputs ( PHI ) ; int SingleExternalEntryIndex = - <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { if ( ! isPHIRegionIndex ( PHIIndices , i ) ) { if ( SingleExternalEntryIndex == - <NUM_LIT> ) { SingleExternalEntryIndex = i ; } else { SingleExternalEntryIndex = - <NUM_LIT> ; } } } if ( SingleExternalEntryIndex > - <NUM_LIT> ) { * ReplaceReg = getPHISourceReg ( PHI , SingleExternalEntryIndex ) ; Replaced = true ; } else { MachineBasicBlock * MBB = PHI . getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , PHI , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , getPHIDestReg ( PHI ) ) ; if ( SourceMBB ) { MIB . addReg ( CombinedSourceReg ) ; MIB . addMBB ( SourceMBB ) ; DEBUG ( dbgs ( ) << PrintReg ( CombinedSourceReg , TRI ) << <STR_LIT> << SourceMBB -> getNumber ( ) ) ; } for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { if ( isPHIRegionIndex ( PHIIndices , i ) ) { continue ; } unsigned SourceReg = getPHISourceReg ( PHI , i ) ; MachineBasicBlock * SourcePred = getPHIPred ( PHI , i ) ; MIB . addReg ( SourceReg ) ; MIB . addMBB ( SourcePred ) ; DEBUG ( dbgs ( ) << PrintReg ( SourceReg , TRI ) << <STR_LIT> << SourcePred -> getNumber ( ) ) ;


<s> MachineBasicBlock * EntrySucc = split ( Entry -> getFirstNonPHI ( ) ) ; MachineBasicBlock * Exit = LRegion -> getExit ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << Entry -> getNumber ( ) << <STR_LIT> << Entry -> getNumber ( ) << <STR_LIT> << EntrySucc -> getNumber ( ) << <STR_LIT> ) ; LRegion -> addMBB ( EntrySucc ) ; if ( Exit -> isSuccessor ( Entry ) ) { Exit -> removeSuccessor ( Entry ) ; } Exit -> addSuccessor ( EntrySucc ) ; MachineInstr & Branch = * ( Exit -> instr_rbegin ( ) ) ;


<s> Exit -> addSuccessor ( EntrySucc ) ; MachineInstr & Branch = * ( Exit -> instr_rbegin ( ) ) ; for ( auto & UI : Branch . uses ( ) ) { if ( UI . isMBB ( ) && UI . getMBB ( ) == Entry ) { UI . setMBB ( EntrySucc ) ;


<s> MachineInstr & Instr = II ; if ( ! Instr . isPHI ( ) ) break ; int numPreds = getPHINumInputs ( Instr ) ; for ( int i = <NUM_LIT> ; i < numPreds ; ++ i ) { auto Pred = getPHIPred ( Instr , i ) ; if ( Pred == Exit ) { setPhiPred ( Instr , i , NewExit ) ; } } } return NewExit ;


<s> } const TargetRegisterClass * RegClass = MRI -> getRegClass ( PHIDest ) ; unsigned NewDestReg = MRI -> createVirtualRegister ( RegClass ) ; LRegion -> replaceRegisterInsideRegion ( PHIDest , NewDestReg , false , MRI ) ; MachineInstrBuilder MIB = BuildMI ( * EntrySucc , EntrySucc -> instr_begin ( ) , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewDestReg ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( NewDestReg , TRI ) << <STR_LIT> ) ; MIB . addReg ( PHISource ) ; MIB . addMBB ( Entry ) ; DEBUG ( dbgs ( ) << PrintReg ( PHISource , TRI ) << <STR_LIT> << Entry -> getNumber ( ) ) ; MIB . addReg ( RegionSourceReg ) ;


<s> PHISource = ReplaceReg ; } const TargetRegisterClass * RegClass = MRI -> getRegClass ( PHIDest ) ; unsigned NewDestReg = MRI -> createVirtualRegister ( RegClass ) ; LRegion -> replaceRegisterInsideRegion ( PHIDest , NewDestReg , false , MRI ) ; MachineInstrBuilder MIB = BuildMI ( * EntrySucc , EntrySucc -> instr_begin ( ) , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewDestReg ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( NewDestReg , TRI ) << <STR_LIT> ) ; MIB . addReg ( PHISource ) ; MIB . addMBB ( Entry ) ; DEBUG ( dbgs ( ) << PrintReg ( PHISource , TRI ) << <STR_LIT> << Entry -> getNumber ( ) ) ; MIB . addReg ( RegionSourceReg ) ; MIB . addMBB ( RegionSourceMBB ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( RegionSourceReg , TRI ) << <STR_LIT> << RegionSourceMBB -> getNumber ( ) << <STR_LIT> ) ;


<s> } else { MachineInstr * UseInstr = UI . getParent ( ) ; for ( MachineBasicBlock :: instr_iterator MII = UseInstr -> getIterator ( ) , MIE = UseInstr -> getParent ( ) -> instr_end ( ) ; MII != MIE ; ++ MII ) { if ( ( & ( * MII ) ) == DefInstr ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; }


<s> for ( auto & UI : MRI -> use_operands ( Reg ) ) { if ( UI . getParent ( ) -> getParent ( ) != MBB ) { DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } else { MachineInstr * UseInstr = UI . getParent ( ) ; for ( MachineBasicBlock :: instr_iterator MII = UseInstr -> getIterator ( ) , MIE = UseInstr -> getParent ( ) -> instr_end ( ) ; MII != MIE ; ++ MII ) { if ( ( & ( * MII ) ) == DefInstr ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } }


<s> for ( auto & UI : MRI -> use_operands ( Reg ) ) { if ( ! Region -> contains ( UI . getParent ( ) -> getParent ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << ( void * ) Region << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> ) ; for ( auto & UI : MRI -> use_operands ( Reg ) ) { if ( ! Region -> contains ( UI . getParent ( ) -> getParent ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << ( void * ) Region << <STR_LIT> << PrintReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } } }


<s> MachineBasicBlock * Exit = Region -> getSucc ( ) ; RegionMRT * TopRegion = CurrentTopRegion == nullptr ? Region : CurrentTopRegion ; if ( Exit == nullptr ) return ; auto Children = Region -> getChildren ( ) ; for ( auto CI : * Children ) { if ( CI -> isMBB ( ) ) { auto MBB = CI -> getMBBMRT ( ) -> getMBB ( ) ; storeMBBLiveOuts ( MBB , MRI , TRI , PHIInfo , TopRegion ) ; } else { LinearizedRegion * SubRegion = CI -> getRegionMRT ( ) -> getLinearizedRegion ( ) ; for ( auto MBBI : SubRegion -> MBBs ) { storeMBBLiveOuts ( MBBI , MRI , TRI , PHIInfo , TopRegion ) ; } } } if ( CurrentTopRegion == nullptr ) { auto Succ = Region -> getSucc ( ) ; for ( auto & II : * Succ ) { if ( II . isPHI ( ) ) { MachineInstr & PHI = II ; int numPreds = getPHINumInputs ( PHI ) ;

for ( int i = <NUM_LIT> ; i < numPreds ; ++ i ) {


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( LRegion -> print ( dbgs ( ) , TRI ) ) ; auto CNI = CI ; ++ CNI ; MRT * Child = ( * CI ) ; if ( Child -> isRegion ( ) ) { LinearizedRegion * InnerLRegion = Child -> getRegionMRT ( ) -> getLinearizedRegion ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( InnerLRegion -> print ( dbgs ( ) , TRI ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineBasicBlock * InnerEntry = InnerLRegion -> getEntry ( ) ; if ( ( & ( * ( InnerEntry -> getParent ( ) -> begin ( ) ) ) ) == InnerEntry ) { unsigned OuterSelect = InnerLRegion -> getBBSelectRegOut ( ) ; unsigned InnerSelectReg = InnerLRegion -> getRegionMRT ( ) -> getInnerOutputRegister ( ) ; replaceRegisterWith ( InnerSelectReg , OuterSelect ) , resolvePHIInfos ( InnerEntry ) ; if ( ! InnerLRegion -> getExit ( ) -> isSuccessor ( CurrentMerge ) ) InnerLRegion -> getExit ( ) -> addSuccessor ( CurrentMerge ) ; continue ; } BBSelectRegOut = Child -> getBBSelectRegOut ( ) ; BBSelectRegIn = Child -> getBBSelectRegIn ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( BBSelectRegIn , TRI ) << <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( BBSelectRegOut , TRI ) << <STR_LIT> ) ; MachineBasicBlock * IfEnd = CurrentMerge ; CurrentMerge = createIfRegion ( CurrentMerge , InnerLRegion , LRegion , Child -> getRegionMRT ( ) -> getEntry ( ) , BBSelectRegIn , BBSelectRegOut ) ; TII -> convertNonUniformIfRegion ( CurrentMerge , IfEnd ) ; } else { MachineBasicBlock * MBB = Child -> getMBBMRT ( ) -> getMBB ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ) ; if ( MBB == getSingleExitNode ( * ( MBB -> getParent ( ) ) ) ) {


<s> auto CNI = CI ; ++ CNI ; MRT * Child = ( * CI ) ; if ( Child -> isRegion ( ) ) { LinearizedRegion * InnerLRegion = Child -> getRegionMRT ( ) -> getLinearizedRegion ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( InnerLRegion -> print ( dbgs ( ) , TRI ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineBasicBlock * InnerEntry = InnerLRegion -> getEntry ( ) ; if ( ( & ( * ( InnerEntry -> getParent ( ) -> begin ( ) ) ) ) == InnerEntry ) { unsigned OuterSelect = InnerLRegion -> getBBSelectRegOut ( ) ; unsigned InnerSelectReg = InnerLRegion -> getRegionMRT ( ) -> getInnerOutputRegister ( ) ; replaceRegisterWith ( InnerSelectReg , OuterSelect ) , resolvePHIInfos ( InnerEntry ) ; if ( ! InnerLRegion -> getExit ( ) -> isSuccessor ( CurrentMerge ) ) InnerLRegion -> getExit ( ) -> addSuccessor ( CurrentMerge ) ; continue ; } BBSelectRegOut = Child -> getBBSelectRegOut ( ) ; BBSelectRegIn = Child -> getBBSelectRegIn ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( BBSelectRegIn , TRI ) << <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( BBSelectRegOut , TRI ) << <STR_LIT> ) ; MachineBasicBlock * IfEnd = CurrentMerge ; CurrentMerge = createIfRegion ( CurrentMerge , InnerLRegion , LRegion , Child -> getRegionMRT ( ) -> getEntry ( ) , BBSelectRegIn , BBSelectRegOut ) ; TII -> convertNonUniformIfRegion ( CurrentMerge , IfEnd ) ; } else { MachineBasicBlock * MBB = Child -> getMBBMRT ( ) -> getMBB ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ) ; if ( MBB == getSingleExitNode ( * ( MBB -> getParent ( ) ) ) ) { continue ; } BBSelectRegOut = Child -> getBBSelectRegOut ( ) ; BBSelectRegIn = Child -> getBBSelectRegIn ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( BBSelectRegIn , TRI ) << <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( BBSelectRegOut , TRI ) << <STR_LIT> ) ; MachineBasicBlock * IfEnd = CurrentMerge ; CurrentMerge = createIfRegion ( CurrentMerge , MBB , LRegion , BBSelectRegIn , BBSelectRegOut ) ; if ( CurrentMerge ) { TII -> convertNonUniformIfRegion ( CurrentMerge , IfEnd ) ; } DEBUG ( PHIInfo . dump ( MRI ) ) ; } } LRegion -> removeFalseRegisterKills ( MRI ) ; if ( LRegion -> getHasLoop ( ) ) { MachineBasicBlock * NewSucc = splitEntry ( LRegion ) ;

if ( isFunctionEntryBlock ( LRegion -> getEntry ( ) ) ) {


<s> LLVM_DEBUG ( Regions -> dump ( ) ) ; RegionMRT * RTree = MRT :: buildMRT ( MF , Regions , TII , MRI ) ; setRegionMRT ( RTree ) ; initializeSelectRegisters ( RTree , <NUM_LIT> , MRI , TII ) ; LLVM_DEBUG ( RTree -> dump ( TRI ) ) ; bool result = structurizeRegions ( RTree , true ) ;


<s> MachineRegion * Parent = Region -> getParent ( ) ; while ( RegionMap . count ( Parent ) == <NUM_LIT> ) { RegionMRT * NewMRTParent = new RegionMRT ( Parent ) ; NewMRTParent -> addChild ( NewMRTRegion ) ; NewMRTRegion -> setParent ( NewMRTParent ) ; RegionMap [ Parent ] = NewMRTParent ; NewMRTRegion = NewMRTParent ; Parent = Parent -> getParent ( ) ; } RegionMap [ Parent ] -> addChild ( NewMRTRegion ) ; NewMRTRegion -> setParent ( RegionMap [ Parent ] ) ; } RegionMap [ Region ] -> addChild ( NewMBB ) ; NewMBB -> setParent ( RegionMap [ Region ] ) ; RegionMap [ Region ] -> setSucc ( Region -> getExit ( ) ) ; } return Result ;


<s> auto CodeBBStartI = CodeBBStart -> getIterator ( ) ; auto CodeBBEndI = CodeBBEnd -> getIterator ( ) ; auto MergeIter = MergeBB -> getIterator ( ) ; MF -> insert ( MergeIter , IfBB ) ; MF -> splice ( MergeIter , CodeBBStartI , ++ CodeBBEndI ) ; IfBB -> addSuccessor ( MergeBB ) ; IfBB -> addSuccessor ( CodeBBStart ) ; DEBUG ( dbgs ( ) << <STR_LIT> << IfBB -> getNumber ( ) << <STR_LIT> ) ; if ( ! CodeBBEnd -> isSuccessor ( MergeBB ) ) CodeBBEnd -> addSuccessor ( MergeBB ) ; DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * CodeBBStart ) << <STR_LIT> << printMBBReference ( * CodeBBEnd ) << <STR_LIT> ) ; MachineBasicBlock * SinglePred = CodeBBStart -> pred_size ( ) == <NUM_LIT> ? * ( CodeBBStart -> pred_begin ( ) ) : nullptr ; const DebugLoc & DL = SinglePred ? SinglePred -> findDebugLoc ( SinglePred -> getFirstTerminator ( ) ) : DebugLoc ( ) ; unsigned Reg = TII -> insertEQ ( IfBB , IfBB -> begin ( ) , DL , IfReg , SelectBB -> getNumber ( ) ) ; if ( & ( * ( IfBB -> getParent ( ) -> begin ( ) ) ) == IfBB ) { TII -> materializeImmediate ( * IfBB , IfBB -> begin ( ) , DL , IfReg , CodeBBStart -> getNumber ( ) ) ;


<s> } LRegion -> setRegionMRT ( Region ) ; Region -> setLinearizedRegion ( LRegion ) ; LRegion -> setParent ( Region -> getParent ( ) ? Region -> getParent ( ) -> getLinearizedRegion ( ) : nullptr ) ;


<s> if ( MRI -> def_begin ( Reg ) == MRI -> def_end ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } else if ( ! MRI -> hasOneDef ( Reg ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( auto DI = MRI -> def_begin ( Reg ) , DE = MRI -> def_end ( ) ; DI != DE ; ++ DI ) {

DEBUG ( DI -> getParent ( ) -> dump ( ) ) ;


<s> PI != E ; ++ PI ) { if ( ( * PI ) != EndMBB ) { Succs . insert ( std :: pair < MachineBasicBlock * , MachineBasicBlock * > ( * PI , StartMBB ) ) ; } } for ( auto <ISA_LIT> : Succs ) { std :: pair < MachineBasicBlock * , MachineBasicBlock * > Edge = <ISA_LIT> ;

DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * Edge . first ) << <STR_LIT> << printMBBReference ( * Edge . second ) << <STR_LIT> ) ;


<s> if ( MRI -> def_begin ( Reg ) == MRI -> def_end ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } else if ( ! MRI -> hasOneDef ( Reg ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } assert ( MRI -> hasOneDef ( Reg ) && <STR_LIT> ) ;

MachineOperand * Def = & ( * ( MRI -> def_begin ( Reg ) ) ) ;


<s> DEBUG ( dbgs ( ) << printReg ( Reg , TRI ) << <STR_LIT> ) ; } if ( MRI -> def_begin ( Reg ) == MRI -> def_end ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } else if ( ! MRI -> hasOneDef ( Reg ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } assert ( MRI -> hasOneDef ( Reg ) && <STR_LIT> ) ; MachineOperand * Def = & ( * ( MRI -> def_begin ( Reg ) ) ) ; MachineOperand * UseOperand = & ( RI ) ; bool UseIsOutsideDefMBB = Def -> getParent ( ) -> getParent ( ) != MBB ; if ( UseIsOutsideDefMBB && UseOperand -> isKill ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ;


<s> IsDead = false ; } } DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> << ( IsDead ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> ) ; if ( IsDead ) { LRegion -> removeLiveOut ( Reg ) ; } WasLiveOut = true ; } } if ( WasLiveOut ) LRegion -> addLiveOut ( CombinedSourceReg ) ;


<s> MachineInstrBuilder MIB = BuildMI ( * MBB , PHI , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , getPHIDestReg ( PHI ) ) ; MIB . addReg ( CombinedSourceReg ) ; MIB . addMBB ( LastMerge ) ; DEBUG ( dbgs ( ) << printReg ( CombinedSourceReg , TRI ) << <STR_LIT> << printMBBReference ( * LastMerge ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { if ( isPHIRegionIndex ( PHIRegionIndices , i ) ) { continue ; } unsigned SourceReg = getPHISourceReg ( PHI , i ) ; MachineBasicBlock * SourcePred = getPHIPred ( PHI , i ) ; MIB . addReg ( SourceReg ) ; MIB . addMBB ( SourcePred ) ; DEBUG ( dbgs ( ) << printReg ( SourceReg , TRI ) << <STR_LIT> << printMBBReference ( * SourcePred ) ) ; }


<s> MachineOperand & O = * I ; ++ I ; if ( O . isDef ( ) ) continue ; bool IsInside = contains ( O . getParent ( ) -> getParent ( ) ) ; bool IsLoopPHI = IsInside && ( O . getParent ( ) -> isPHI ( ) && O . getParent ( ) -> getParent ( ) == getEntry ( ) ) ; bool ShouldReplace = ( IsInside && ReplaceInside ) || ( ! IsInside && ReplaceOutside ) || ( IncludeLoopPHI && IsLoopPHI ) ; if ( ShouldReplace ) { if ( TargetRegisterInfo :: isPhysicalRegister ( NewRegister ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; llvm_unreachable ( <STR_LIT> ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Register , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; O . setReg ( NewRegister ) ;


<s> } else { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Register , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; O . setReg ( NewRegister ) ; } } PHIInfo . deleteDef ( Register ) ;

getRegionMRT ( ) -> replaceLiveOutReg ( Register , NewRegister ) ;


<s> I != E ; ) { MachineOperand & O = * I ; ++ I ; if ( TargetRegisterInfo :: isPhysicalRegister ( NewRegister ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; llvm_unreachable ( <STR_LIT> ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Register , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; O . setReg ( NewRegister ) ; } }


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( PHIInfo . dump ( MRI ) ) ; for ( auto DRI = PHIInfo . dests_begin ( ) , DE = PHIInfo . dests_end ( ) ; DRI != DE ; ++ DRI ) { unsigned DestReg = * DRI ; DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( DestReg , TRI ) << <STR_LIT> ) ;

auto SRI = PHIInfo . sources_begin ( DestReg ) ;


<s> DEBUG ( PHIInfo . dump ( MRI ) ) ; for ( auto DRI = PHIInfo . dests_begin ( ) , DE = PHIInfo . dests_end ( ) ; DRI != DE ; ++ DRI ) { unsigned DestReg = * DRI ; DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( DestReg , TRI ) << <STR_LIT> ) ; auto SRI = PHIInfo . sources_begin ( DestReg ) ; unsigned SourceReg = ( * SRI ) . first ; DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( DestReg , TRI ) << <STR_LIT> << printReg ( SourceReg , TRI ) << <STR_LIT> ) ; assert ( PHIInfo . sources_end ( DestReg ) == ++ SRI && <STR_LIT> ) ; replaceRegisterWith ( DestReg , SourceReg ) ; }


<s> if ( ! containsDef ( CodeBB , InnerRegion , LI ) || ( ! IsSingleBB && ( getDefInstr ( LI ) -> getParent ( ) == LRegion -> getExit ( ) ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; unsigned Reg = LI ; if ( Reg != InnerRegion -> getBBSelectRegOut ( ) ) { const TargetRegisterClass * RegClass = MRI -> getRegClass ( Reg ) ; unsigned PHIDestReg = MRI -> createVirtualRegister ( RegClass ) ; unsigned IfSourceReg = MRI -> createVirtualRegister ( RegClass ) ; DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg ) << <STR_LIT> ) ; TII -> materializeImmediate ( * IfBB , IfBB -> getFirstTerminator ( ) , DebugLoc ( ) , IfSourceReg , <NUM_LIT> ) ; InnerRegion -> replaceRegisterOutsideRegion ( Reg , PHIDestReg , true , MRI ) ;


<s> unsigned NumInputs = getPHINumInputs ( PHI ) ; int SingleExternalEntryIndex = - <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { if ( ! isPHIRegionIndex ( PHIIndices , i ) ) { if ( SingleExternalEntryIndex == - <NUM_LIT> ) { SingleExternalEntryIndex = i ; } else { SingleExternalEntryIndex = - <NUM_LIT> ; } } } if ( SingleExternalEntryIndex > - <NUM_LIT> ) { * ReplaceReg = getPHISourceReg ( PHI , SingleExternalEntryIndex ) ; Replaced = true ; } else { MachineBasicBlock * MBB = PHI . getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , PHI , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , getPHIDestReg ( PHI ) ) ; if ( SourceMBB ) { MIB . addReg ( CombinedSourceReg ) ; MIB . addMBB ( SourceMBB ) ; DEBUG ( dbgs ( ) << printReg ( CombinedSourceReg , TRI ) << <STR_LIT> << printMBBReference ( * SourceMBB ) ) ;


<s> MachineBasicBlock * <ISA_LIT>MachineCFGStructurizer :: splitEntry ( LinearizedRegion * LRegion ) { MachineBasicBlock * Entry = LRegion -> getEntry ( ) ; MachineBasicBlock * EntrySucc = split ( Entry -> getFirstNonPHI ( ) ) ; MachineBasicBlock * Exit = LRegion -> getExit ( ) ;

DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * Entry ) << <STR_LIT> << printMBBReference ( * Entry ) << <STR_LIT> << printMBBReference ( * EntrySucc ) << <STR_LIT> ) ;


<s> LRegion -> addMBB ( EntrySucc ) ; if ( Exit -> isSuccessor ( Entry ) ) { Exit -> removeSuccessor ( Entry ) ; } Exit -> addSuccessor ( EntrySucc ) ; MachineInstr & Branch = * ( Exit -> instr_rbegin ( ) ) ; for ( auto & UI : Branch . uses ( ) ) { if ( UI . isMBB ( ) && UI . getMBB ( ) == Entry ) { UI . setMBB ( EntrySucc ) ; }


<s> unsigned RegionIndex = PHIRegionIndices [ <NUM_LIT> ] ; unsigned RegionSourceReg = getPHISourceReg ( PHI , RegionIndex ) ; MachineBasicBlock * RegionSourceMBB = getPHIPred ( PHI , RegionIndex ) ; unsigned PHIDest = getPHIDestReg ( PHI ) ; unsigned PHISource = PHIDest ; unsigned ReplaceReg ; if ( shrinkPHI ( PHI , PHIRegionIndices , & ReplaceReg ) ) { PHISource = ReplaceReg ; } const TargetRegisterClass * RegClass = MRI -> getRegClass ( PHIDest ) ; unsigned NewDestReg = MRI -> createVirtualRegister ( RegClass ) ; LRegion -> replaceRegisterInsideRegion ( PHIDest , NewDestReg , false , MRI ) ; MachineInstrBuilder MIB = BuildMI ( * EntrySucc , EntrySucc -> instr_begin ( ) , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewDestReg ) ; DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( NewDestReg , TRI ) << <STR_LIT> ) ;


<s> MachineBasicBlock * RegionSourceMBB = getPHIPred ( PHI , RegionIndex ) ; unsigned PHIDest = getPHIDestReg ( PHI ) ; unsigned PHISource = PHIDest ; unsigned ReplaceReg ; if ( shrinkPHI ( PHI , PHIRegionIndices , & ReplaceReg ) ) { PHISource = ReplaceReg ; } const TargetRegisterClass * RegClass = MRI -> getRegClass ( PHIDest ) ; unsigned NewDestReg = MRI -> createVirtualRegister ( RegClass ) ; LRegion -> replaceRegisterInsideRegion ( PHIDest , NewDestReg , false , MRI ) ;

MachineInstrBuilder MIB = BuildMI ( * EntrySucc , EntrySucc -> instr_begin ( ) , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewDestReg ) ;


<s> if ( PHIInfo . isSource ( Reg ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } else { for ( auto & UI : MRI -> use_operands ( Reg ) ) { if ( UI . getParent ( ) -> getParent ( ) != MBB ) { DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } else {

MachineInstr * UseInstr = UI . getParent ( ) ;


<s> if ( ! Region -> contains ( UI . getParent ( ) -> getParent ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << ( void * ) Region << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } } }


<s> RegionMRT * TopRegion = CurrentTopRegion == nullptr ? Region : CurrentTopRegion ; if ( Exit == nullptr ) return ; auto Children = Region -> getChildren ( ) ; for ( auto CI : * Children ) { if ( CI -> isMBB ( ) ) { auto MBB = CI -> getMBBMRT ( ) -> getMBB ( ) ; storeMBBLiveOuts ( MBB , MRI , TRI , PHIInfo , TopRegion ) ; } else { LinearizedRegion * SubRegion = CI -> getRegionMRT ( ) -> getLinearizedRegion ( ) ; for ( auto MBBI : SubRegion -> MBBs ) { storeMBBLiveOuts ( MBBI , MRI , TRI , PHIInfo , TopRegion ) ; } } } if ( CurrentTopRegion == nullptr ) { auto Succ = Region -> getSucc ( ) ; for ( auto & II : * Succ ) { if ( II . isPHI ( ) ) { MachineInstr & PHI = II ; int numPreds = getPHINumInputs ( PHI ) ; for ( int i = <NUM_LIT> ; i < numPreds ; ++ i ) { if ( Region -> contains ( getPHIPred ( PHI , i ) ) ) { unsigned PHIReg = getPHISourceReg ( PHI , i ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ( void * ) Region << <STR_LIT> << printReg ( PHIReg , TRI ) << <STR_LIT> ) ; addLiveOut ( PHIReg ) ; } }


<s> TII -> convertNonUniformIfRegion ( CurrentMerge , IfEnd ) ; } DEBUG ( PHIInfo . dump ( MRI ) ) ; } } LRegion -> removeFalseRegisterKills ( MRI ) ; if ( LRegion -> getHasLoop ( ) ) { MachineBasicBlock * NewSucc = splitEntry ( LRegion ) ; if ( isFunctionEntryBlock ( LRegion -> getEntry ( ) ) ) { resolvePHIInfos ( LRegion -> getEntry ( ) ) ; } const DebugLoc & DL = NewSucc -> findDebugLoc ( NewSucc -> getFirstNonPHI ( ) ) ; unsigned InReg = LRegion -> getBBSelectRegIn ( ) ; unsigned InnerSelectReg = MRI -> createVirtualRegister ( MRI -> getRegClass ( InReg ) ) ; unsigned NewInReg = MRI -> createVirtualRegister ( MRI -> getRegClass ( InReg ) ) ; TII -> materializeImmediate ( * ( LRegion -> getEntry ( ) ) , LRegion -> getEntry ( ) -> getFirstTerminator ( ) , DL , NewInReg , Region -> getEntry ( ) -> getNumber ( ) ) ; LRegion -> replaceRegisterInsideRegion ( InReg , InnerSelectReg , false , MRI ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; insertMergePHI ( LRegion -> getEntry ( ) , LRegion -> getExit ( ) , NewSucc , InnerSelectReg , NewInReg , LRegion -> getRegionMRT ( ) -> getInnerOutputRegister ( ) ) ; splitExit ( LRegion ) ; TII -> convertNonUniformLoopRegion ( NewSucc , LastMerge ) ; } if ( Region -> isRoot ( ) ) { TII -> insertReturn ( * LastMerge ) ; } DEBUG ( Region -> getEntry ( ) -> getParent ( ) -> dump ( ) ) ; DEBUG ( LRegion -> print ( dbgs ( ) , TRI ) ) ; DEBUG ( PHIInfo . dump ( MRI ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ;

Region -> setLinearizedRegion ( LRegion ) ;


<s> } else { RegionMRT * Region = Tree -> getRegionMRT ( ) ; for ( MRT * C : llvm :: reverse ( * Region -> getChildren ( ) ) ) if ( containsNewBackedge ( C , MBBs ) ) return true ; }


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * CodeBBStart ) << <STR_LIT> << printMBBReference ( * CodeBBEnd ) << <STR_LIT> ) ; MachineBasicBlock * SinglePred = CodeBBStart -> pred_size ( ) == <NUM_LIT> ? * ( CodeBBStart -> pred_begin ( ) ) : nullptr ; const DebugLoc & DL = SinglePred ? SinglePred -> findDebugLoc ( SinglePred -> getFirstTerminator ( ) ) : DebugLoc ( ) ; Register Reg = TII -> insertEQ ( IfBB , IfBB -> begin ( ) , DL , IfReg , SelectBB -> getNumber ( ) ) ; if ( & ( * ( IfBB -> getParent ( ) -> begin ( ) ) ) == IfBB ) { TII -> materializeImmediate ( * IfBB , IfBB -> begin ( ) , DL , IfReg , CodeBBStart -> getNumber ( ) ) ; } MachineOperand RegOp = MachineOperand :: CreateReg ( Reg , false , false , true ) ; ArrayRef < MachineOperand > Cond ( RegOp ) ;

TII -> insertBranch ( * IfBB , MergeBB , CodeBBStart , Cond , DL ) ;


<s> if ( ! Cond [ <NUM_LIT> ] . isReg ( ) ) return ;

Register CondReg = Cond [ <NUM_LIT> ] . getReg ( ) ;


<s> if ( ! Cond [ <NUM_LIT> ] . isReg ( ) ) return ; Register CondReg = Cond [ <NUM_LIT> ] . getReg ( ) ; for ( MachineOperand & MO : MRI -> use_operands ( CondReg ) ) MO . setIsKill ( false ) ;


<s> SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > Successors ; for ( MachineBasicBlock * Succ : Exit -> successors ( ) ) { if ( LRegion -> contains ( Succ ) ) { assert ( InternalSucc == nullptr ) ; InternalSucc = Succ ; } else { assert ( ExternalSucc == nullptr ) ; ExternalSucc = Succ ; } } for ( auto & TI : Exit -> terminators ( ) ) { for ( auto & UI : TI . uses ( ) ) { if ( UI . isMBB ( ) ) { auto Target = UI . getMBB ( ) ;

if ( Target != InternalSucc && Target != ExternalSucc ) {


<s> ExternalSucc = Succ ; } } for ( auto & TI : Exit -> terminators ( ) ) { for ( auto & UI : TI . uses ( ) ) { if ( UI . isMBB ( ) ) { auto Target = UI . getMBB ( ) ; if ( Target != InternalSucc && Target != ExternalSucc ) { UI . setMBB ( ExternalSucc ) ; }


<s> MRT -> setBBSelectRegIn ( InnerSelectOut ) ; return InnerSelectOut ; } else { MRT -> setBBSelectRegOut ( SelectOut ) ; unsigned NewSelectIn = createBBSelectReg ( TII , MRI ) ; MRT -> setBBSelectRegIn ( NewSelectIn ) ; return NewSelectIn ;


<s> static bool mbbHasBackEdge ( MachineBasicBlock * MBB , SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > & MBBs ) { for ( MachineBasicBlock * Succ : MBB -> successors ( ) ) if ( MBBs . contains ( Succ ) ) return true ; return false ;


<s> for ( MachineBasicBlock * Succ : MBB -> successors ( ) ) if ( MBBs . contains ( Succ ) ) return true ;


<s> } else if ( ( Current -> succ_size ( ) == <NUM_LIT> ) && * ( Current -> succ_begin ( ) ) == Succ ) { FoundIf = true ; } } return FoundIf && FoundBypass ;


<s> for ( MachineBasicBlock * Pred : StartMBB -> predecessors ( ) ) if ( Pred != EndMBB ) Succs . insert ( std :: make_pair ( Pred , StartMBB ) ) ; for ( auto <ISA_LIT> : Succs ) { std :: pair < MachineBasicBlock * , MachineBasicBlock * > Edge = <ISA_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * Edge . first ) << <STR_LIT> << printMBBReference ( * Edge . second ) << <STR_LIT> ) ;

Edge . first -> removeSuccessor ( Edge . second ) ;


<s> bool IsDead = true ; MachineBasicBlock * DefMBB = getDefInstr ( Reg ) -> getParent ( ) ; for ( const MachineOperand & MO : MRI -> use_operands ( Reg ) ) if ( MO . getParent ( ) -> getParent ( ) != DefMBB ) IsDead = false ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> << ( IsDead ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> ) ;


<s> } } } for ( MachineBasicBlock * Succ : MBB -> successors ( ) ) { for ( auto & II : * Succ ) { if ( II . isPHI ( ) ) { MachineInstr & PHI = II ; int numPreds = getPHINumInputs ( PHI ) ; for ( int i = <NUM_LIT> ; i < numPreds ; ++ i ) { if ( getPHIPred ( PHI , i ) == MBB ) { unsigned PHIReg = getPHISourceReg ( PHI , i ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << printMBBReference ( * Succ ) << <STR_LIT> << printReg ( PHIReg , TRI ) << <STR_LIT> ) ; addLiveOut ( PHIReg ) ; }


<s> for ( auto MBB : InnerRegion -> MBBs ) {


<s> bool LinearizedRegion :: contains ( MachineBasicBlock * MBB ) {


<s> bool LinearizedRegion :: contains ( MachineBasicBlock * MBB ) { return MBBs . contains ( MBB ) ;


<s> NumSources ++ ; } if ( NumSources == <NUM_LIT> ) { auto SRI = PHIInfo . sources_begin ( DestReg ) ; unsigned SourceReg = ( * SRI ) . first ; replaceRegisterWith ( DestReg , SourceReg ) ; } else { const DebugLoc & DL = Entry -> findDebugLoc ( Entry -> begin ( ) ) ; MachineInstrBuilder MIB = BuildMI ( * Entry , Entry -> instr_begin ( ) , DL , TII -> get ( TargetOpcode :: PHI ) , DestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( DestReg , TRI ) << <STR_LIT> ) ; unsigned CurrentBackedgeReg = <NUM_LIT> ; for ( auto SRI = PHIInfo . sources_begin ( DestReg ) ; SRI != SE ; ++ SRI ) { unsigned SourceReg = ( * SRI ) . first ; if ( CurrentRegion -> contains ( ( * SRI ) . second ) ) { if ( CurrentBackedgeReg == <NUM_LIT> ) { CurrentBackedgeReg = SourceReg ; } else { MachineInstr * PHIDefInstr = getDefInstr ( SourceReg ) ; MachineBasicBlock * PHIDefMBB = PHIDefInstr -> getParent ( ) ; const TargetRegisterClass * RegClass = MRI -> getRegClass ( CurrentBackedgeReg ) ; Register NewBackedgeReg = MRI -> createVirtualRegister ( RegClass ) ; MachineInstrBuilder BackedgePHI = BuildMI ( * PHIDefMBB , PHIDefMBB -> instr_begin ( ) , DL , TII -> get ( TargetOpcode :: PHI ) , NewBackedgeReg ) ; BackedgePHI . addReg ( CurrentBackedgeReg ) ; BackedgePHI . addMBB ( getPHIPred ( * PHIDefInstr , <NUM_LIT> ) ) ; BackedgePHI . addReg ( getPHISourceReg ( * PHIDefInstr , <NUM_LIT> ) ) ; BackedgePHI . addMBB ( ( * SRI ) . second ) ; CurrentBackedgeReg = NewBackedgeReg ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( NewBackedgeReg , TRI ) << <STR_LIT> << printReg ( CurrentBackedgeReg , TRI ) << <STR_LIT> << printMBBReference ( * getPHIPred ( * PHIDefInstr , <NUM_LIT> ) ) << <STR_LIT> << printReg ( getPHISourceReg ( * PHIDefInstr , <NUM_LIT> ) , TRI ) << <STR_LIT> << printMBBReference ( * ( * SRI ) . second ) ) ; } } else { MIB . addReg ( SourceReg ) ;


<s> MF -> splice ( MergeIter , CodeBBStartI , ++ CodeBBEndI ) ; IfBB -> addSuccessor ( MergeBB ) ; IfBB -> addSuccessor ( CodeBBStart ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << IfBB -> getNumber ( ) << <STR_LIT> ) ; if ( ! CodeBBEnd -> isSuccessor ( MergeBB ) ) CodeBBEnd -> addSuccessor ( MergeBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * CodeBBStart ) << <STR_LIT> << printMBBReference ( * CodeBBEnd ) << <STR_LIT> ) ; MachineBasicBlock * SinglePred = CodeBBStart -> pred_size ( ) == <NUM_LIT> ? * ( CodeBBStart -> pred_begin ( ) ) : nullptr ; const DebugLoc & DL = SinglePred ? SinglePred -> findDebugLoc ( SinglePred -> getFirstTerminator ( ) ) : DebugLoc ( ) ; Register Reg = TII -> insertEQ ( IfBB , IfBB -> begin ( ) , DL , IfReg , SelectBB -> getNumber ( ) ) ; if ( & ( * ( IfBB -> getParent ( ) -> begin ( ) ) ) == IfBB ) { TII -> materializeImmediate ( * IfBB , IfBB -> begin ( ) , DL , IfReg , CodeBBStart -> getNumber ( ) ) ; }

MachineOperand RegOp = MachineOperand :: CreateReg ( Reg , false , false , true ) ;


<s> Pred -> addSuccessor ( IfBB ) ; } } } removeExternalCFGEdges ( CodeBBStart , CodeBBEnd ) ; auto CodeBBStartI = CodeBBStart -> getIterator ( ) ; auto CodeBBEndI = CodeBBEnd -> getIterator ( ) ; auto MergeIter = MergeBB -> getIterator ( ) ; MF -> insert ( MergeIter , IfBB ) ; MF -> splice ( MergeIter , CodeBBStartI , ++ CodeBBEndI ) ; IfBB -> addSuccessor ( MergeBB ) ; IfBB -> addSuccessor ( CodeBBStart ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << IfBB -> getNumber ( ) << <STR_LIT> ) ; if ( ! CodeBBEnd -> isSuccessor ( MergeBB ) ) CodeBBEnd -> addSuccessor ( MergeBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * CodeBBStart ) << <STR_LIT> << printMBBReference ( * CodeBBEnd ) << <STR_LIT> ) ; MachineBasicBlock * SinglePred = CodeBBStart -> pred_size ( ) == <NUM_LIT> ? * ( CodeBBStart -> pred_begin ( ) ) : nullptr ; const DebugLoc & DL = SinglePred ? SinglePred -> findDebugLoc ( SinglePred -> getFirstTerminator ( ) ) : DebugLoc ( ) ; Register Reg = TII -> insertEQ ( IfBB , IfBB -> begin ( ) , DL , IfReg , SelectBB -> getNumber ( ) ) ; if ( & ( * ( IfBB -> getParent ( ) -> begin ( ) ) ) == IfBB ) { TII -> materializeImmediate ( * IfBB , IfBB -> begin ( ) , DL , IfReg , CodeBBStart -> getNumber ( ) ) ;


<s> void <ISA_LIT>MachineCFGStructurizer :: ensureCondIsNotKilled ( SmallVector < MachineOperand , <NUM_LIT> > Cond ) { if ( Cond . size ( ) != <NUM_LIT> ) return ; if ( ! Cond [ <NUM_LIT> ] . isReg ( ) ) return ; Register CondReg = Cond [ <NUM_LIT> ] . getReg ( ) ; for ( auto UI = MRI -> use_begin ( CondReg ) , E = MRI -> use_end ( ) ; UI != E ; ++ UI ) {


<s> for ( auto UI = MRI -> use_begin ( CondReg ) , E = MRI -> use_end ( ) ; UI != E ; ++ UI ) { ( * UI ) . setIsKill ( false ) ; }


<s> return LiveOuts . contains ( Reg ) ;


<s> bool LinearizedRegion :: isLiveOut ( unsigned Reg ) {


<s> } OS << MBB -> getNumber ( ) ; } OS << <STR_LIT> << Entry -> getNumber ( ) << <STR_LIT> << ( Exit == nullptr ? - <NUM_LIT> : Exit -> getNumber ( ) ) << <STR_LIT> << printReg ( getBBSelectRegIn ( ) , TRI ) << <STR_LIT> << printReg ( getBBSelectRegOut ( ) , TRI ) << <STR_LIT> ; for ( auto & LI : LiveOuts ) { OS << printReg ( LI , TRI ) << <STR_LIT> ;


<s> } if ( MRI -> def_begin ( Reg ) == MRI -> def_end ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } else if ( ! MRI -> hasOneDef ( Reg ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } assert ( MRI -> hasOneDef ( Reg ) && <STR_LIT> ) ; MachineOperand * Def = & ( * ( MRI -> def_begin ( Reg ) ) ) ;

MachineOperand * UseOperand = & ( RI ) ;


<s> } else if ( ! MRI -> hasOneDef ( Reg ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; } assert ( MRI -> hasOneDef ( Reg ) && <STR_LIT> ) ; MachineOperand * Def = & ( * ( MRI -> def_begin ( Reg ) ) ) ; MachineOperand * UseOperand = & ( RI ) ; bool UseIsOutsideDefMBB = Def -> getParent ( ) -> getParent ( ) != MBB ;

if ( UseIsOutsideDefMBB && UseOperand -> isKill ( ) ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( Current -> print ( dbgs ( ) , MRI -> getTargetRegisterInfo ( ) ) ) ; Current -> replaceLiveOut ( Register , NewRegister ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( Current -> print ( dbgs ( ) , MRI -> getTargetRegisterInfo ( ) ) ) ; Current = Current -> getParent ( ) ; } } for ( MachineRegisterInfo :: reg_iterator I = MRI -> reg_begin ( Register ) , E = MRI -> reg_end ( ) ; I != E ; ) { MachineOperand & O = * I ; ++ I ; if ( O . isDef ( ) ) continue ; bool IsInside = contains ( O . getParent ( ) -> getParent ( ) ) ; bool IsLoopPHI = IsInside && ( O . getParent ( ) -> isPHI ( ) && O . getParent ( ) -> getParent ( ) == getEntry ( ) ) ; bool ShouldReplace = ( IsInside && ReplaceInside ) || ( ! IsInside && ReplaceOutside ) || ( IncludeLoopPHI && IsLoopPHI ) ; if ( ShouldReplace ) { if ( NewRegister . isPhysical ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; llvm_unreachable ( <STR_LIT> ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Register , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> << printReg ( NewRegister , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; O . setReg ( NewRegister ) ; }


<s> TII -> materializeImmediate ( * CodeBB , CodeBB -> getFirstTerminator ( ) , DL , BBSelectReg , TrueBB -> getNumber ( ) ) ; } else { const TargetRegisterClass * RegClass = MRI -> getRegClass ( BBSelectReg ) ; Register TrueBBReg = MRI -> createVirtualRegister ( RegClass ) ; Register FalseBBReg = MRI -> createVirtualRegister ( RegClass ) ; TII -> materializeImmediate ( * CodeBB , CodeBB -> getFirstTerminator ( ) , DL , TrueBBReg , TrueBB -> getNumber ( ) ) ;

TII -> materializeImmediate ( * CodeBB , CodeBB -> getFirstTerminator ( ) , DL , FalseBBReg , FalseBB -> getNumber ( ) ) ;


<s> const TargetRegisterClass * RegClass = MRI -> getRegClass ( Reg ) ; Register PHIDestReg = MRI -> createVirtualRegister ( RegClass ) ; Register IfSourceReg = MRI -> createVirtualRegister ( RegClass ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg ) << <STR_LIT> ) ; TII -> materializeImmediate ( * IfBB , IfBB -> getFirstTerminator ( ) , DebugLoc ( ) , IfSourceReg , <NUM_LIT> ) ; InnerRegion -> replaceRegisterOutsideRegion ( Reg , PHIDestReg , true , MRI ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; insertMergePHI ( IfBB , InnerRegion -> getExit ( ) , MergeBB , PHIDestReg , IfSourceReg , Reg , true ) ; } } SmallVector < unsigned , <NUM_LIT> > Sources ; if ( PHIInfo . findSourcesFromMBB ( CodeBB , Sources ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * CodeBB ) << <STR_LIT> ) ; for ( auto <ISA_LIT> : Sources ) {


<s> MachineBasicBlock * RegionSourceMBB = getPHIPred ( PHI , RegionIndex ) ; unsigned PHIDest = getPHIDestReg ( PHI ) ; unsigned PHISource = PHIDest ; unsigned ReplaceReg ; if ( shrinkPHI ( PHI , PHIRegionIndices , & ReplaceReg ) ) { PHISource = ReplaceReg ; } const TargetRegisterClass * RegClass = MRI -> getRegClass ( PHIDest ) ; Register NewDestReg = MRI -> createVirtualRegister ( RegClass ) ; LRegion -> replaceRegisterInsideRegion ( PHIDest , NewDestReg , false , MRI ) ; MachineInstrBuilder MIB = BuildMI ( * EntrySucc , EntrySucc -> instr_begin ( ) , PHI . getDebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewDestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( NewDestReg , TRI ) << <STR_LIT> ) ; MIB . addReg ( PHISource ) ; MIB . addMBB ( Entry ) ; LLVM_DEBUG ( dbgs ( ) << printReg ( PHISource , TRI ) << <STR_LIT> << printMBBReference ( * Entry ) ) ;


<s> void LinearizedRegion :: storeLiveOutReg ( MachineBasicBlock * MBB , Register Reg , MachineInstr * DefInstr , const MachineRegisterInfo * MRI , const TargetRegisterInfo * TRI , PHILinearize & PHIInfo ) { if ( Reg . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; if ( PHIInfo . isSource ( Reg ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } else { for ( auto & UI : MRI -> use_operands ( Reg ) ) { if ( UI . getParent ( ) -> getParent ( ) != MBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } else { MachineInstr * UseInstr = UI . getParent ( ) ; for ( MachineBasicBlock :: instr_iterator MII = UseInstr -> getIterator ( ) , MIE = UseInstr -> getParent ( ) -> instr_end ( ) ; MII != MIE ; ++ MII ) { if ( ( & ( * MII ) ) == DefInstr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } } } }


<s> if ( Reg . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; if ( PHIInfo . isSource ( Reg ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } else {

for ( auto & UI : MRI -> use_operands ( Reg ) ) {


<s> if ( Reg . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; for ( auto & UI : MRI -> use_operands ( Reg ) ) { if ( ! Region -> contains ( UI . getParent ( ) -> getParent ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( void * ) Region << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; addLiveOut ( Reg ) ; } } }


<s> unsigned <ISA_LIT>MachineCFGStructurizer :: storePHILinearizationInfo ( MachineInstr & PHI , SmallVector < unsigned , <NUM_LIT> > * RegionIndices ) { unsigned DestReg = getPHIDestReg ( PHI ) ;

Register LinearizeDestReg = MRI -> createVirtualRegister ( MRI -> getRegClass ( DestReg ) ) ;


<s> PHIInfo . addDest ( LinearizeDestReg , PHI . getDebugLoc ( ) ) ; storePHILinearizationInfoDest ( LinearizeDestReg , PHI , RegionIndices ) ;


<s> dbgs ( ) << <STR_LIT> << PrintReg ( getBBSelectRegOut ( ) , TRI ) << <STR_LIT> ; dumpDepth ( depth ) ;

if ( getSucc ( ) ) dbgs ( ) << <STR_LIT> << getSucc ( ) -> getNumber ( ) << <STR_LIT> ;


<s> virtual MBBMRT * getMBBMRT ( ) { return this ;


<s> RegionMRT ( MachineRegion * MachineRegion ) : Region ( MachineRegion ) , LRegion ( nullptr ) , Succ ( nullptr ) { setParent ( nullptr ) ; setBBSelectRegOut ( <NUM_LIT> ) ; setBBSelectRegIn ( <NUM_LIT> ) ;


<s> LinearizedRegion :: ~ LinearizedRegion ( ) {


<s> } for ( auto CI : Children ) { delete & ( * CI ) ; }


<s> void addChild ( MRT * Tree ) { Children . insert ( Tree ) ;


<s> NewElement -> DestReg = DestReg ; NewElement -> DL = DL ; NewElement -> Sources = EmptySet ; PHIInfo . insert ( NewElement ) ;


<s> NewElement -> DL = DL ; NewElement -> Sources = EmptySet ; PHIInfo . insert ( NewElement ) ;


<s> void LinearizedRegion :: addMBBs ( LinearizedRegion * InnerRegion ) { for ( const auto & MBB : InnerRegion -> MBBs ) { addMBB ( MBB ) ;


<s> void LinearizedRegion :: addMBBs ( LinearizedRegion * InnerRegion ) { for ( const auto & MBB : InnerRegion -> MBBs ) {


<s> phiInfoElementAddSource ( findPHIInfoElement ( DestReg ) , SourceReg , SourceMBB ) ;


<s> for ( auto MBBI : post_order ( & ( MF . front ( ) ) ) ) { MachineBasicBlock * MBB = & ( * MBBI ) ; if ( MBB == Exit ) { continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> ) ; MBBMRT * NewMBB = new MBBMRT ( MBB ) ; MachineRegion * Region = RegionInfo -> getRegionFor ( MBB ) ; if ( RegionMap . count ( Region ) == <NUM_LIT> ) { RegionMRT * NewMRTRegion = new RegionMRT ( Region ) ; RegionMap [ Region ] = NewMRTRegion ; MachineRegion * Parent = Region -> getParent ( ) ; while ( RegionMap . count ( Parent ) == <NUM_LIT> ) { RegionMRT * NewMRTParent = new RegionMRT ( Parent ) ; NewMRTParent -> addChild ( NewMRTRegion ) ; NewMRTRegion -> setParent ( NewMRTParent ) ; RegionMap [ Parent ] = NewMRTParent ; NewMRTRegion = NewMRTParent ; Parent = Parent -> getParent ( ) ; }

RegionMap [ Parent ] -> addChild ( NewMRTRegion ) ;


<s> static void checkRegOnlyPHIInputs ( MachineFunction & MF ) { for ( auto & MBBI : MF ) { for ( MachineBasicBlock :: instr_iterator I = MBBI . instr_begin ( ) , E = MBBI . instr_end ( ) ; I != E ; ++ I ) { MachineInstr & Instr = * I ; if ( Instr . isPHI ( ) ) { int numPreds = getPHINumInputs ( Instr ) ; for ( int i = <NUM_LIT> ; i < numPreds ; ++ i ) { assert ( Instr . getOperand ( i * <NUM_LIT> + <NUM_LIT> ) . isReg ( ) && <STR_LIT> ) ; } }


<s> int numPreds = getPHINumInputs ( Instr ) ; for ( int i = <NUM_LIT> ; i < numPreds ; ++ i ) { assert ( Instr . getOperand ( i * <NUM_LIT> + <NUM_LIT> ) . isReg ( ) && <STR_LIT> ) ; } } } }


<s> if ( BBI . isPHI ( ) ) { PHIs . push_back ( & BBI ) ; }


<s> bool LinearizedRegion :: contains ( MachineBasicBlock * MBB ) { return MBBs . count ( MBB ) == <NUM_LIT> ;


<s> bool <ISA_LIT>MachineCFGStructurizer :: containsDef ( MachineBasicBlock * MBB , LinearizedRegion * InnerRegion , unsigned Register ) {

return getDefInstr ( Register ) -> getParent ( ) == MBB || InnerRegion -> contains ( getDefInstr ( Register ) -> getParent ( ) ) ;


<s> return containsNewBackedge ( Region , MBBs ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Exit -> getNumber ( ) << <STR_LIT> << ( * ( Entry -> pred_begin ( ) ) ) -> getNumber ( ) << <STR_LIT> ) ; int NumSources = <NUM_LIT> ; auto SE = PHIInfo . sources_end ( DestReg ) ; for ( auto SRI = PHIInfo . sources_begin ( DestReg ) ; SRI != SE ; ++ SRI ) { NumSources ++ ; } if ( NumSources == <NUM_LIT> ) { auto SRI = PHIInfo . sources_begin ( DestReg ) ; unsigned SourceReg = ( * SRI ) . first ; replaceRegisterWith ( DestReg , SourceReg ) ; } else { const DebugLoc & DL = Entry -> findDebugLoc ( Entry -> begin ( ) ) ; MachineInstrBuilder MIB = BuildMI ( * Entry , Entry -> instr_begin ( ) , DL , TII -> get ( TargetOpcode :: PHI ) , DestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( DestReg , TRI ) << <STR_LIT> ) ; unsigned CurrentBackedgeReg = <NUM_LIT> ; for ( auto SRI = PHIInfo . sources_begin ( DestReg ) ; SRI != SE ; ++ SRI ) { unsigned SourceReg = ( * SRI ) . first ; if ( CurrentRegion -> contains ( ( * SRI ) . second ) ) { if ( CurrentBackedgeReg == <NUM_LIT> ) { CurrentBackedgeReg = SourceReg ; } else { MachineInstr * PHIDefInstr = getDefInstr ( SourceReg ) ; MachineBasicBlock * PHIDefMBB = PHIDefInstr -> getParent ( ) ; const TargetRegisterClass * RegClass = MRI -> getRegClass ( CurrentBackedgeReg ) ; unsigned NewBackedgeReg = MRI -> createVirtualRegister ( RegClass ) ; MachineInstrBuilder BackedgePHI = BuildMI ( * PHIDefMBB , PHIDefMBB -> instr_begin ( ) , DL , TII -> get ( TargetOpcode :: PHI ) , NewBackedgeReg ) ;

BackedgePHI . addReg ( CurrentBackedgeReg ) ;


<s> ++ DRI ) { unsigned DestReg = * DRI ; createEntryPHI ( CurrentRegion , DestReg ) ; } PHIInfo . clear ( ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << IfBB -> getNumber ( ) << <STR_LIT> ) ; if ( ! CodeBBEnd -> isSuccessor ( MergeBB ) ) CodeBBEnd -> addSuccessor ( MergeBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * CodeBBStart ) << <STR_LIT> << printMBBReference ( * CodeBBEnd ) << <STR_LIT> ) ; MachineBasicBlock * SinglePred = CodeBBStart -> pred_size ( ) == <NUM_LIT> ? * ( CodeBBStart -> pred_begin ( ) ) : nullptr ; const DebugLoc & DL = SinglePred ? SinglePred -> findDebugLoc ( SinglePred -> getFirstTerminator ( ) ) : DebugLoc ( ) ;

unsigned Reg = TII -> insertEQ ( IfBB , IfBB -> begin ( ) , DL , IfReg , SelectBB -> getNumber ( ) ) ;


<s> MachineFunction * MF = Region -> getEntry ( ) -> getParent ( ) ; if ( Exit == nullptr && hasOneExitNode ( * MF ) ) { return & ( * ( -- ( Region -> getEntry ( ) -> getParent ( ) -> end ( ) ) ) ) ; } MachineBasicBlock * LastMerge = MF -> CreateMachineBasicBlock ( ) ; if ( Exit == nullptr ) { MachineFunction :: iterator ExitIter = MF -> end ( ) ; MF -> insert ( ExitIter , LastMerge ) ; } else {

MachineFunction :: iterator ExitIter = Exit -> getIterator ( ) ;


<s> void <ISA_LIT>MachineCFGStructurizer :: createLinearizedRegion ( RegionMRT * Region , unsigned SelectOut ) { LinearizedRegion * LRegion = new LinearizedRegion ( ) ; if ( SelectOut ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( SelectOut , TRI ) << <STR_LIT> ) ; } LRegion -> setRegionMRT ( Region ) ; Region -> setLinearizedRegion ( LRegion ) ; LRegion -> setParent ( Region -> getParent ( ) ? Region -> getParent ( ) -> getLinearizedRegion ( ) : nullptr ) ;


<s> PHILinearize :: dest_iterator PHILinearize :: dests_begin ( ) {

return PHILinearizeDestIterator ( PHIInfo . begin ( ) ) ;


<s> return PHILinearizeDestIterator ( PHIInfo . begin ( ) ) ;


<s> PHILinearize :: dest_iterator PHILinearize :: dests_end ( ) {

return PHILinearizeDestIterator ( PHIInfo . end ( ) ) ;


<s> dbgs ( ) << <STR_LIT> << ( void * ) Region ; dbgs ( ) << <STR_LIT> << printReg ( getBBSelectRegIn ( ) , TRI ) ; dbgs ( ) << <STR_LIT> << printReg ( getBBSelectRegOut ( ) , TRI ) << <STR_LIT> ; dumpDepth ( depth ) ; if ( getSucc ( ) ) dbgs ( ) << <STR_LIT> << getSucc ( ) -> getNumber ( ) << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ; for ( auto MRTI : Children ) { MRTI -> dump ( TRI , depth + <NUM_LIT> ) ; }


<s> void dumpDepth ( int depth ) { for ( int i = depth ; i > <NUM_LIT> ; -- i ) { dbgs ( ) << <STR_LIT> ;


<s> if ( Cond . size ( ) != <NUM_LIT> ) return ; if ( ! Cond [ <NUM_LIT> ] . isReg ( ) ) return ; unsigned CondReg = Cond [ <NUM_LIT> ] . getReg ( ) ;

for ( auto UI = MRI -> use_begin ( CondReg ) , E = MRI -> use_end ( ) ; UI != E ; ++ UI ) {


<s> bool PHILinearize :: findDest ( unsigned SourceReg , MachineBasicBlock * SourceMBB , unsigned & DestReg ) { PHIInfoElementT * InfoElement = findPHIInfoElementFromSource ( SourceReg , SourceMBB ) ; if ( InfoElement != nullptr ) { DestReg = phiInfoElementGetDest ( InfoElement ) ; return true ; }


<s> for ( auto I : PHIInfo ) { if ( phiInfoElementGetDest ( I ) == DestReg ) {


<s> PHILinearize :: PHIInfoElementT * PHILinearize :: findPHIInfoElementFromSource ( unsigned SourceReg , MachineBasicBlock * SourceMBB ) { for ( auto I : PHIInfo ) { for ( auto <ISA_LIT> : phiInfoElementGetSources ( I ) ) { if ( <ISA_LIT> . first == SourceReg && ( <ISA_LIT> . second == nullptr || <ISA_LIT> . second == SourceMBB ) ) {


<s> if ( <ISA_LIT> . first == SourceReg && ( <ISA_LIT> . second == nullptr || <ISA_LIT> . second == SourceMBB ) ) { return I ; } }


<s> for ( auto I : PHIInfo ) { for ( auto <ISA_LIT> : phiInfoElementGetSources ( I ) ) { if ( <ISA_LIT> . second == SourceMBB ) { FoundSource = true ;

Sources . push_back ( <ISA_LIT> . first ) ;


<s> for ( auto & TI : MBB -> terminators ( ) ) { for ( auto & UI : TI . uses ( ) ) { if ( UI . isMBB ( ) && UI . getMBB ( ) != Succ ) {


<s> auto Exit = LRegion -> getExit ( ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > Successors ; for ( MachineBasicBlock :: const_succ_iterator <ISA_LIT> = Exit -> succ_begin ( ) , SE = Exit -> succ_end ( ) ; <ISA_LIT> != SE ; ++ <ISA_LIT> ) { MachineBasicBlock * Succ = * <ISA_LIT> ; if ( LRegion -> contains ( Succ ) ) { assert ( InternalSucc == nullptr ) ; InternalSucc = Succ ; } else { assert ( ExternalSucc == nullptr ) ;


<s> void fixupRegionExits ( RegionMRT * Region ) { auto Children = Region -> getChildren ( ) ; for ( auto CI : * Children ) { if ( ! CI -> isRegion ( ) ) { fixMBBTerminator ( CI -> getMBBMRT ( ) -> getMBB ( ) ) ; } else { fixRegionTerminator ( CI -> getRegionMRT ( ) ) ; } }


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MachineRegionInfoPass > ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> AU . addRequired < MachineRegionInfoPass > ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> return getRegionMRT ( ) -> getBBSelectRegIn ( ) ;


<s> unsigned LinearizedRegion :: getBBSelectRegOut ( ) {

return getRegionMRT ( ) -> getBBSelectRegOut ( ) ;


<s> } else if ( ! MRI -> hasOneDef ( Reg ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI -> getTargetRegisterInfo ( ) ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( auto DI = MRI -> def_begin ( Reg ) , DE = MRI -> def_end ( ) ; DI != DE ; ++ DI ) { LLVM_DEBUG ( DI -> getParent ( ) -> dump ( ) ) ;


<s> MachineBasicBlock * LinearizedRegion :: getEntry ( ) {


<s> MachineBasicBlock * LinearizedRegion :: getExit ( ) {


<s> unsigned getInnerOutputRegister ( ) {

return ( * ( Children . begin ( ) ) ) -> getBBSelectRegOut ( ) ;


<s> LinearizedRegion * getLinearizedRegion ( ) { return LRegion ;


<s> LinearizedRegion * getLinearizedRegion ( ) {


<s> DenseSet < unsigned > * LinearizedRegion :: getLiveOuts ( ) { return & LiveOuts ;


<s> MachineRegion * getMachineRegion ( ) {


<s> MachineBasicBlock * getMBB ( ) { return MBB ;


<s> unsigned PHILinearize :: getNumSources ( unsigned DestReg ) { return phiInfoElementGetSources ( findPHIInfoElement ( DestReg ) ) . size ( ) ;


<s> assert ( PHI . isPHI ( ) ) ; return PHI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> MachineBasicBlock * Pred = getPHIPred ( PHI , i ) ; if ( ! Region -> contains ( Pred ) ) { PHINonRegionIndices . push_back ( i ) ; }


<s> unsigned NumInputs = getPHINumInputs ( PHI ) ; for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { MachineBasicBlock * Pred = getPHIPred ( PHI , i ) ; if ( ! Region -> contains ( Pred ) ) { PHINonRegionIndices . push_back ( i ) ; }


<s> return ( PHI . getNumOperands ( ) - <NUM_LIT> ) / <NUM_LIT> ;


<s> assert ( PHI . isPHI ( ) ) ; return PHI . getOperand ( Index * <NUM_LIT> + <NUM_LIT> ) . getMBB ( ) ;


<s> static MachineBasicBlock * getPHIPred ( MachineInstr & PHI , unsigned Index ) {


<s> unsigned NumInputs = getPHINumInputs ( PHI ) ; for ( unsigned i = <NUM_LIT> ; i < NumInputs ; ++ i ) { MachineBasicBlock * Pred = getPHIPred ( PHI , i ) ; if ( Region -> contains ( Pred ) ) { PHIRegionIndices . push_back ( i ) ; } }


<s> static unsigned getPHISourceReg ( MachineInstr & PHI , unsigned Index ) { assert ( PHI . isPHI ( ) ) ; return PHI . getOperand ( Index * <NUM_LIT> + <NUM_LIT> ) . getReg ( ) ;


<s> RegionMRT * getRegionMRT ( ) override { return this ;


<s> MachineBasicBlock * getSucc ( ) { return Succ ;


<s> return MRI -> def_begin ( Reg ) == MRI -> def_end ( ) ;


<s> bool LinearizedRegion :: hasNoDef ( unsigned Reg , MachineRegisterInfo * MRI ) {

return MRI -> def_begin ( Reg ) == MRI -> def_end ( ) ;


<s> return getSingleExitNode ( MF ) != nullptr ;


<s> if ( MBB != nullptr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBBI . getNumber ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ) ; } FallthroughMap [ & MBBI ] = MBB ;