<s> ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned > > RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } trackR<ISA_LIT> ( R<ISA_LIT> ) ; } } return false ;


<s> if ( TII -> get ( MI -> getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , MI ) ; unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned > > RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ;


<s> DebugLoc DL = Pos -> getDebugLoc ( ) ; unsigned SrcVec = BaseR<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; DenseMap < unsigned , unsigned > UpdatedRegToChan = BaseR<ISA_LIT> -> RegToChan ; std :: vector < unsigned > UpdatedUndef = BaseR<ISA_LIT> -> UndefReg ; for ( DenseMap < unsigned , unsigned > :: iterator It = R<ISA_LIT> -> RegToChan . begin ( ) , E = R<ISA_LIT> -> RegToChan . end ( ) ; It != E ; ++ It ) { unsigned DstReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned SubReg = ( * It ) . first ; unsigned Swizzle = ( * It ) . second ; unsigned Chan = getReassignedChan ( RemapChan , Swizzle ) ; MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < unsigned > :: iterator ChanPos = llvm :: find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ;


<s> MachineInstr & MI = * MII ; if ( MI . getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

if ( ! areAllUsesSwizzeable ( Reg ) ) continue ;


<s> for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { MachineBasicBlock * MB = & * MBB ; PreviousRegSeq . clear ( ) ; PreviousRegSeqByReg . clear ( ) ; PreviousRegSeqByUndefCount . clear ( ) ; for ( MachineBasicBlock :: iterator MII = MB -> begin ( ) , MIIE = MB -> end ( ) ; MII != MIIE ; ++ MII ) { MachineInstr & MI = * MII ; if ( MI . getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } MachineInstr * NewMI = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; NewMI -> dump ( ) ; ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ( * It ) . dump ( ) ; dbgs ( ) << <STR_LIT> ) ; SwizzleInput ( * It , RemapChan ) ; DEBUG ( ( * It ) . dump ( ) ) ; } R<ISA_LIT> -> Instr -> eraseFromParent ( ) ; R<ISA_LIT> -> Instr = NewMI ; R<ISA_LIT> -> RegToChan = UpdatedRegToChan ; R<ISA_LIT> -> UndefReg = UpdatedUndef ; return NewMI ;


<s> if ( skipFunction ( * Fn . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & ST = Fn . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { MachineBasicBlock * MB = & * MBB ; PreviousRegSeq . clear ( ) ; PreviousRegSeqByReg . clear ( ) ; PreviousRegSeqByUndefCount . clear ( ) ; for ( MachineBasicBlock :: iterator MII = MB -> begin ( ) , MIIE = MB -> end ( ) ; MII != MIIE ; ++ MII ) { MachineInstr & MI = * MII ; if ( MI . getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; }


<s> std :: vector < unsigned > :: iterator ChanPos = find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } MachineInstr * NewMI = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; NewMI -> dump ( ) ; ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ( * It ) . dump ( ) ; dbgs ( ) << <STR_LIT> ) ; SwizzleInput ( * It , RemapChan ) ; DEBUG ( ( * It ) . dump ( ) ) ; } R<ISA_LIT> -> Instr -> eraseFromParent ( ) ; R<ISA_LIT> -> Instr = NewMI ; R<ISA_LIT> -> RegToChan = UpdatedRegToChan ; R<ISA_LIT> -> UndefReg = UpdatedUndef ;


<s> for ( InstructionSetMap :: iterator It = PreviousRegSeqByReg . begin ( ) , E = PreviousRegSeqByReg . end ( ) ; It != E ; ++ It ) { std :: vector < MachineInstr * > & MIs = ( * It ) . second ; MIs . erase ( find ( MIs , MI ) , MIs . end ( ) ) ;


<s> PreviousRegSeqByUndefCount . clear ( ) ; for ( MachineBasicBlock :: iterator MII = MB -> begin ( ) , MIIE = MB -> end ( ) ; MII != MIIE ; ++ MII ) { MachineInstr * MI = MII ; if ( MI -> getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI -> getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , MI ) ; unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned > > RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ;


<s> PreviousRegSeq . clear ( ) ; PreviousRegSeqByReg . clear ( ) ; PreviousRegSeqByUndefCount . clear ( ) ; for ( MachineBasicBlock :: iterator MII = MB -> begin ( ) , MIIE = MB -> end ( ) ; MII != MIIE ; ++ MII ) { MachineInstr * MI = MII ; if ( MI -> getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI -> getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , MI ) ; unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ;


<s> assert ( MRI . isSSA ( ) ) ; if ( Register :: isPhysicalRegister ( Reg ) ) return false ; const MachineInstr * MI = MRI . getUniqueVRegDef ( Reg ) ; return MI && MI -> isImplicitDef ( ) ;


<s> MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < unsigned > :: iterator ChanPos = llvm :: find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } MachineInstr * NewMI = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; NewMI -> dump ( ) ; ) ;


<s> <ISA_LIT>VectorRegMerger ( ) : MachineFunctionPass ( ID ) , TII ( nullptr ) {


<s> if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) ) return false ; const MachineInstr * MI = MRI . getUniqueVRegDef ( Reg ) ; return MI && MI -> isImplicitDef ( ) ;


<s> if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) ) return false ; const MachineInstr * MI = MRI . getUniqueVRegDef ( Reg ) ;


<s> bool <ISA_LIT>VectorRegMerger :: areAllUsesSwizzeable ( Register Reg ) const { for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { if ( ! canSwizzle ( * It ) ) return false ; } return true ;


<s> bool <ISA_LIT>VectorRegMerger :: canSwizzle ( const MachineInstr & MI ) const { if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>VectorRegMerger ( ) { return new <ISA_LIT>VectorRegMerger ( ) ;


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>VectorRegMerger ( ) {

return new <ISA_LIT>VectorRegMerger ( ) ;


<s> const MachineInstr * MI = MRI . getUniqueVRegDef ( Reg ) ;


<s> <ISA_LIT>VectorRegMerger ( ) : MachineFunctionPass ( ID ) {


<s> DenseMap < Register , unsigned > UpdatedRegToChan = BaseR<ISA_LIT> -> RegToChan ; std :: vector < Register > UpdatedUndef = BaseR<ISA_LIT> -> UndefReg ; for ( DenseMap < Register , unsigned > :: iterator It = R<ISA_LIT> -> RegToChan . begin ( ) , E = R<ISA_LIT> -> RegToChan . end ( ) ; It != E ; ++ It ) { Register DstReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned SubReg = ( * It ) . first ; unsigned Swizzle = ( * It ) . second ; unsigned Chan = getReassignedChan ( RemapChan , Swizzle ) ; MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < Register > :: iterator ChanPos = llvm :: find ( UpdatedUndef , Chan ) ;

if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ;


<s> unsigned Chan = Instr -> getOperand ( i + <NUM_LIT> ) . getImm ( ) ; if ( isImplicitlyDef ( MRI , MO . getReg ( ) ) ) UndefReg . push_back ( Chan ) ; else RegToChan [ MO . getReg ( ) ] = Chan ; }


<s> void <ISA_LIT>VectorRegMerger :: RemoveMI ( MachineInstr * MI ) { for ( InstructionSetMap :: iterator It = PreviousRegSeqByReg . begin ( ) , E = PreviousRegSeqByReg . end ( ) ; It != E ; ++ It ) { std :: vector < MachineInstr * > & MIs = ( * It ) . second ; MIs . erase ( llvm :: find ( MIs , MI ) , MIs . end ( ) ) ;


<s> void <ISA_LIT>VectorRegMerger :: RemoveMI ( MachineInstr * MI ) { for ( InstructionSetMap :: iterator It = PreviousRegSeqByReg . begin ( ) , E = PreviousRegSeqByReg . end ( ) ; It != E ; ++ It ) { std :: vector < MachineInstr * > & MIs = ( * It ) . second ; MIs . erase ( llvm :: find ( MIs , MI ) , MIs . end ( ) ) ; } for ( InstructionSetMap :: iterator It = PreviousRegSeqByUndefCount . begin ( ) , E = PreviousRegSeqByUndefCount . end ( ) ; It != E ; ++ It ) { std :: vector < MachineInstr * > & MIs = ( * It ) . second ; MIs . erase ( llvm :: find ( MIs , MI ) , MIs . end ( ) ) ;


<s> Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ;


<s> for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { unsigned Swizzle = MI . getOperand ( i + Offset ) . getImm ( ) + <NUM_LIT> ; for ( unsigned j = <NUM_LIT> , e = RemapChan . size ( ) ; j < e ; j ++ ) { if ( RemapChan [ j ] . first == Swizzle ) {

MI . getOperand ( i + Offset ) . setImm ( RemapChan [ j ] . second - <NUM_LIT> ) ;


<s> for ( DenseMap < Register , unsigned > :: const_iterator It = R<ISA_LIT> . RegToChan . begin ( ) , E = R<ISA_LIT> . RegToChan . end ( ) ; It != E ; ++ It ) { PreviousRegSeqByReg [ ( * It ) . first ] . push_back ( R<ISA_LIT> . Instr ) ;


<s> for ( MachineInstr * MI : PreviousRegSeqByReg [ MOp -> getReg ( ) ] ) { CompatibleR<ISA_LIT> = PreviousRegSeq [ MI ] ; if ( R<ISA_LIT> == CompatibleR<ISA_LIT> ) continue ; if ( tryMergeVector ( & CompatibleR<ISA_LIT> , & R<ISA_LIT> , RemapChan ) ) return true ;


<s> unsigned CurrentUndexIdx = <NUM_LIT> ; for ( DenseMap < Register , unsigned > :: iterator It = ToMerge -> RegToChan . begin ( ) , E = ToMerge -> RegToChan . end ( ) ; It != E ; ++ It ) { DenseMap < Register , unsigned > :: const_iterator PosInUntouched = Untouched -> RegToChan . find ( ( * It ) . first ) ; if ( PosInUntouched != Untouched -> RegToChan . end ( ) ) { Remap . push_back ( std :: pair < unsigned , unsigned > ( ( * It ) . second , ( * PosInUntouched ) . second ) ) ; continue ; } if ( CurrentUndexIdx >= Untouched -> UndefReg . size ( ) ) return false ; Remap . push_back ( std :: pair < unsigned , unsigned > ( ( * It ) . second , Untouched -> UndefReg [ CurrentUndexIdx ++ ] ) ) ; }


<s> unsigned Op = TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: bank_swizzle ) ; MI -> getOperand ( Op ) . setImm ( BS [ i ] ) ; } unsigned Op = TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: bank_swizzle ) ; MI . getOperand ( Op ) . setImm ( BS . back ( ) ) ; if ( ! CurrentPacketMIs . empty ( ) ) setIsLastBit ( CurrentPacketMIs . back ( ) , <NUM_LIT> ) ; substitutePV ( MI , PV ) ; MachineBasicBlock :: iterator It = VLIWPacketizerList :: addToPacket ( MI ) ; if ( isTransSlot ) { endPacket ( std :: next ( It ) -> getParent ( ) , std :: next ( It ) ) ;


<s> MachineBasicBlock :: iterator FirstInBundle = CurrentPacketMIs . empty ( ) ? & MI : CurrentPacketMIs . front ( ) ; const DenseMap < unsigned , unsigned > & PV = getPreviousVector ( FirstInBundle ) ; std :: vector < <ISA_LIT>InstrInfo :: BankSwizzle > BS ; bool isTransSlot ; if ( isBundlableWithCurrentPMI ( MI , PV , BS , isTransSlot ) ) { for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) ; i < e ; i ++ ) { MachineInstr * MI = CurrentPacketMIs [ i ] ; unsigned Op = TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: bank_swizzle ) ; MI -> getOperand ( Op ) . setImm ( BS [ i ] ) ; } unsigned Op = TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: bank_swizzle ) ; MI . getOperand ( Op ) . setImm ( BS . back ( ) ) ;

if ( ! CurrentPacketMIs . empty ( ) ) setIsLastBit ( CurrentPacketMIs . back ( ) , <NUM_LIT> ) ;


<s> return new <ISA_LIT>Packetizer ( ) ;


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>Packetizer ( ) { return new <ISA_LIT>Packetizer ( ) ;


<s> for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) - <NUM_LIT> ; i < e ; i ++ ) { CurrentPacketMIs [ i ] -> dump ( ) ; dbgs ( ) << <STR_LIT> ; }


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> bool isTrans = false ; int BISlot = getSlot ( * BI ) ; if ( LastDstChan >= BISlot ) isTrans = true ; LastDstChan = BISlot ; if ( TII -> isPredicated ( * BI ) ) continue ; int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } Register Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( * BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( Dst == <STR_LIT> ) { continue ;


<s> unsigned getSlot ( const MachineInstr & MI ) const {

return TRI . getHWRegChan ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> bool ignorePseudoInstruction ( const MachineInstr & MI , const MachineBasicBlock * MBB ) override {


<s> bool isLegalToPruneDependencies ( SUnit * SUI , SUnit * SUJ ) override { return false ;


<s> bool isLegalToPruneDependencies ( SUnit * SUI , SUnit * SUJ ) override {


<s> if ( ! TII -> isALUInstr ( MI . getOpcode ( ) ) ) return true ;

if ( MI . getOpcode ( ) == <STR_LIT> ) return true ;


<s> <ISA_LIT>Packetizer ( ) : MachineFunctionPass ( ID ) {


<s> <ISA_LIT>PacketizerList ( MachineFunction & MF , const <ISA_LIT>Subtarget & ST , MachineLoopInfo & MLI ) : VLIWPacketizerList ( MF , MLI , nullptr ) , TII ( ST . getInstrInfo ( ) ) , TRI ( TII -> getRegisterInfo ( ) ) {


<s> } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ; for ( ; I != MBB -> begin ( ) ; -- I , -- RemainingCount ) { if ( TII -> isSchedulingBoundary ( * std :: prev ( I ) , & * MBB , Fn ) ) break ; } I = MBB -> begin ( ) ; if ( I == RegionEnd ) { RegionEnd = std :: prev ( RegionEnd ) ; -- RemainingCount ; continue ; } if ( I == std :: prev ( RegionEnd ) ) { RegionEnd = std :: prev ( RegionEnd ) ; continue ; } Packetizer . PacketizeMIs ( & * MBB , & * I , RegionEnd ) ; RegionEnd = I ; } }


<s> void setIsLastBit ( MachineInstr * MI , unsigned Bit ) const { unsigned LastOp = TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: last ) ;

MI -> getOperand ( LastOp ) . setImm ( Bit ) ;


<s> const DenseMap < unsigned , unsigned > :: const_iterator It = PVs . find ( Src ) ;

if ( It != PVs . end ( ) ) MI . getOperand ( OperandIdx ) . setReg ( It -> second ) ;


<s> <STR_LIT> :: src0 , <STR_LIT> :: src1 , <STR_LIT> :: src2 } ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { int OperandIdx = TII -> getOperandIdx ( MI . getOpcode ( ) , Ops [ i ] ) ; if ( OperandIdx < <NUM_LIT> ) continue ;

Register Src = MI . getOperand ( OperandIdx ) . getReg ( ) ;


<s> MI . getOperand ( Op ) . setImm ( BS . back ( ) ) ; if ( ! CurrentPacketMIs . empty ( ) ) setIsLastBit ( CurrentPacketMIs . back ( ) , <NUM_LIT> ) ; substitutePV ( MI , PV ) ; MachineBasicBlock :: iterator It = VLIWPacketizerList :: addToPacket ( MI ) ; if ( isTransSlot ) { endPacket ( std :: next ( It ) -> getParent ( ) , std :: next ( It ) ) ; } return It ; } endPacket ( MI . getParent ( ) , MI ) ;

if ( TII -> isTransOnly ( MI ) ) return MI ;


<s> if ( ! TII -> isALUInstr ( I -> getOpcode ( ) ) && ! I -> isBundle ( ) ) return Result ; MachineBasicBlock :: instr_iterator BI = I . getInstrIterator ( ) ; if ( I -> isBundle ( ) ) BI ++ ; int LastDstChan = - <NUM_LIT> ; do { bool isTrans = false ; int BISlot = getSlot ( * BI ) ; if ( LastDstChan >= BISlot ) isTrans = true ; LastDstChan = BISlot ; if ( TII -> isPredicated ( * BI ) ) continue ; int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } unsigned Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( * BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) {


<s> if ( PredI != PredJ ) return false ; if ( SUJ -> isSucc ( SUI ) ) { for ( unsigned i = <NUM_LIT> , e = SUJ -> Succs . size ( ) ; i < e ; ++ i ) { const SDep & Dep = SUJ -> Succs [ i ] ; if ( Dep . getSUnit ( ) != SUI ) continue ; if ( Dep . getKind ( ) == SDep :: Anti ) continue ; if ( Dep . getKind ( ) == SDep :: Output ) if ( MII -> getOperand ( <NUM_LIT> ) . getReg ( ) != MIJ -> getOperand ( <NUM_LIT> ) . getReg ( ) ) continue ; return false ; } }

bool ARDef = TII -> definesAddressRegister ( * MII ) || TII -> definesAddressRegister ( * MIJ ) ;


<s> if ( MI -> isKill ( ) || MI -> getOpcode ( ) == <STR_LIT> || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) { MachineBasicBlock :: iterator DeleteMI = MI ; ++ MI ; MBB -> erase ( DeleteMI ) ; End = MBB -> end ( ) ; continue ; } ++ MI ; } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ; for ( ; I != MBB -> begin ( ) ; -- I , -- RemainingCount ) { if ( TII -> isSchedulingBoundary ( * std :: prev ( I ) , & * MBB , Fn ) ) break ; } I = MBB -> begin ( ) ; if ( I == RegionEnd ) { RegionEnd = std :: prev ( RegionEnd ) ; -- RemainingCount ;


<s> bool isTrans = false ; int BISlot = getSlot ( * BI ) ; if ( LastDstChan >= BISlot ) isTrans = true ; LastDstChan = BISlot ; if ( TII -> isPredicated ( * BI ) ) continue ; int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } unsigned Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( * BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) { Result [ Dst ] = <STR_LIT> ;


<s> if ( SUJ -> isSucc ( SUI ) ) { for ( unsigned i = <NUM_LIT> , e = SUJ -> Succs . size ( ) ; i < e ; ++ i ) { const SDep & Dep = SUJ -> Succs [ i ] ; if ( Dep . getSUnit ( ) != SUI ) continue ; if ( Dep . getKind ( ) == SDep :: Anti ) continue ; if ( Dep . getKind ( ) == SDep :: Output ) if ( MII -> getOperand ( <NUM_LIT> ) . getReg ( ) != MIJ -> getOperand ( <NUM_LIT> ) . getReg ( ) ) continue ; return false ; } } bool ARDef = TII -> definesAddressRegister ( * MII ) || TII -> definesAddressRegister ( * MIJ ) ; bool ARUse = TII -> usesAddressRegister ( * MII ) || TII -> usesAddressRegister ( * MIJ ) ; return ! ARDef || ! ARUse ;


<s> unsigned Src = MI . getOperand ( OperandIdx ) . getReg ( ) ; const DenseMap < unsigned , unsigned > :: const_iterator It = PVs . find ( Src ) ; if ( It != PVs . end ( ) ) MI . getOperand ( OperandIdx ) . setReg ( It -> second ) ; }


<s> if ( TII -> isPredicated ( & * BI ) ) continue ; int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } unsigned Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( & * BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( Dst == <STR_LIT> ) { continue ; } unsigned PVReg = <NUM_LIT> ; switch ( TRI . getHWRegChan ( Dst ) ) { case <NUM_LIT> : PVReg = <STR_LIT> ; break ; case <NUM_LIT> :


<s> unsigned getSlot ( const MachineInstr * MI ) const { return TRI . getHWRegChan ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; ) ; } else return false ; } } CurrentPacketMIs . push_back ( MI ) ; if ( ! TII -> fitsConstReadLimitations ( CurrentPacketMIs ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; dbgs ( ) << <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) - <NUM_LIT> ; i < e ; i ++ ) { CurrentPacketMIs [ i ] -> dump ( ) ; dbgs ( ) << <STR_LIT> ; } dbgs ( ) << <STR_LIT> ; ) ; CurrentPacketMIs . pop_back ( ) ; return false ; } if ( ! TII -> fitsReadPortLimitations ( CurrentPacketMIs , PV , BS , isTransSlot ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; dbgs ( ) << <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) - <NUM_LIT> ; i < e ; i ++ ) {


<s> if ( TII -> isVector ( * MI ) ) return true ;

if ( ! TII -> isALUInstr ( MI -> getOpcode ( ) ) ) return true ;


<s> <STR_LIT> :: src0 , <STR_LIT> :: src1 , <STR_LIT> :: src2 } ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { int OperandIdx = TII -> getOperandIdx ( MI -> getOpcode ( ) , Ops [ i ] ) ; if ( OperandIdx < <NUM_LIT> ) continue ;

unsigned Src = MI -> getOperand ( OperandIdx ) . getReg ( ) ;


<s> return new <ISA_LIT>Packetizer ( tm ) ;


<s> MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) - <NUM_LIT> ; i < e ; i ++ ) { CurrentPacketMIs [ i ] -> dump ( ) ;


<s> const char * getPassName ( ) const override {


<s> int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } unsigned Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( & * BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( Dst == <STR_LIT> ) { continue ;


<s> if ( MI . getOpcode ( ) == <STR_LIT> ) return true ; return TII -> isLD<ISA_LIT>nstr ( MI . getOpcode ( ) ) ;


<s> <ISA_LIT>Packetizer ( const TargetMachine & TM ) : MachineFunctionPass ( ID ) {


<s> <ISA_LIT>PacketizerList ( MachineFunction & MF , MachineLoopInfo & MLI ) : VLIWPacketizerList ( MF , MLI , nullptr ) , TII ( static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ) , TRI ( TII -> getRegisterInfo ( ) ) { VLIW5 = ! MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasCaymanISA ( ) ;


<s> MachineBasicBlock :: iterator DeleteMI = MI ; ++ MI ; MBB -> erase ( DeleteMI ) ; End = MBB -> end ( ) ; continue ; } ++ MI ; } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ; for ( ; I != MBB -> begin ( ) ; -- I , -- RemainingCount ) { if ( TII -> isSchedulingBoundary ( & * std :: prev ( I ) , & * MBB , Fn ) ) break ; } I = MBB -> begin ( ) ; if ( I == RegionEnd ) { RegionEnd = std :: prev ( RegionEnd ) ; -- RemainingCount ; continue ; } if ( I == std :: prev ( RegionEnd ) ) { RegionEnd = std :: prev ( RegionEnd ) ; continue ;


<s> const DenseMap < unsigned , unsigned > :: const_iterator It = PVs . find ( Src ) ; if ( It != PVs . end ( ) ) MI . getOperand ( OperandIdx ) . setReg ( It -> second ) ; }


<s> while ( MI != End ) { if ( MI -> isKill ( ) || MI -> getOpcode ( ) == <STR_LIT> || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) { MachineBasicBlock :: iterator DeleteMI = MI ; ++ MI ; MBB -> erase ( DeleteMI ) ; End = MBB -> end ( ) ; continue ; } ++ MI ; } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ;

for ( ; I != MBB -> begin ( ) ; -- I , -- RemainingCount ) {


<s> bool <ISA_LIT>Packetizer :: runOnMachineFunction ( MachineFunction & Fn ) { const TargetInstrInfo * TII = Fn . getSubtarget ( ) . getInstrInfo ( ) ; MachineLoopInfo & MLI = getAnalysis < MachineLoopInfo > ( ) ; <ISA_LIT>PacketizerList Packetizer ( Fn , MLI ) ; assert ( Packetizer . getResourceTracker ( ) && <STR_LIT> ) ; if ( Packetizer . getResourceTracker ( ) -> getInstrItins ( ) -> isEmpty ( ) ) return false ; for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { MachineBasicBlock :: iterator End = MBB -> end ( ) ; MachineBasicBlock :: iterator MI = MBB -> begin ( ) ; while ( MI != End ) { if ( MI -> isKill ( ) || MI -> getOpcode ( ) == <STR_LIT> || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) { MachineBasicBlock :: iterator DeleteMI = MI ; ++ MI ; MBB -> erase ( DeleteMI ) ; End = MBB -> end ( ) ; continue ; } ++ MI ; } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ;


<s> bool ignorePseudoInstruction ( MachineInstr * MI , MachineBasicBlock * MBB ) override { return false ;


<s> <ISA_LIT>PacketizerList ( MachineFunction & MF , MachineLoopInfo & MLI ) : VLIWPacketizerList ( MF , MLI ) , TII ( static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ) , TRI ( TII -> getRegisterInfo ( ) ) {

VLIW5 = ! MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasCaymanISA ( ) ;


<s> int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } unsigned Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( Dst == <STR_LIT> ) { continue ; } unsigned PVReg = <NUM_LIT> ; switch ( TRI . getHWRegChan ( Dst ) ) { case <NUM_LIT> : PVReg = <STR_LIT> ; break ; case <NUM_LIT> : PVReg = <STR_LIT> ; break ; case <NUM_LIT> : PVReg = <STR_LIT> ; break ; case <NUM_LIT> :


<s> default : case <STR_LIT> : return & <STR_LIT> ; }


<s> const RegClassWeight & <ISA_LIT>RegisterInfo :: getRegClassWeight ( const TargetRegisterClass * RC ) const {


<s> Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; for ( TargetRegisterClass :: iterator I = <STR_LIT> . begin ( ) , E = <STR_LIT> . end ( ) ; I != E ; ++ I ) { Reserved . set ( * I ) ; }

TII -> reserveIndirectRegisters ( Reserved , MF ) ;


<s> Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; for ( TargetRegisterClass :: iterator I = <STR_LIT> . begin ( ) , E = <STR_LIT> . end ( ) ; I != E ; ++ I ) { Reserved . set ( * I ) ; }


<s> assert ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) ; switch ( Reg ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; default : return true ;


<s> RCW . RegWeight = <NUM_LIT> ; RCW . WeightLimit = <NUM_LIT> ;


<s> unsigned <ISA_LIT>RegisterInfo :: getFrameRegister ( const MachineFunction & MF ) const {


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; for ( MCPhysReg R : <STR_LIT> ) reserveRegisterTuples ( Reserved , R ) ; TII -> reserveIndirectRegisters ( Reserved , MF , * this ) ; return Reserved ;


return SubRegFromChannelTable [ Channel ] ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getCFGStructurizerRegClass ( MVT VT ) const {


<s> Register <ISA_LIT>RegisterInfo :: getFrameRegister ( const MachineFunction & MF ) const { return <STR_LIT> ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; for ( TargetRegisterClass :: iterator I = <STR_LIT> . begin ( ) , E = <STR_LIT> . end ( ) ; I != E ; ++ I ) { reserveRegisterTuples ( Reserved , * I ) ;


<s> bool <ISA_LIT>RegisterInfo :: isPhysRegLiveAcrossClauses ( unsigned Reg ) const {

assert ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) ;


<s> <ISA_LIT>RegisterInfo :: <ISA_LIT>RegisterInfo ( ) : <ISA_LIT>GenRegisterInfo ( <NUM_LIT> ) {


<s> void <ISA_LIT>RegisterInfo :: reserveRegisterTuples ( BitVector & Reserved , unsigned Reg ) const {

MCRegAliasIterator R ( Reg , this , true ) ;


<s> switch ( Reg ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> <ISA_LIT>RegisterInfo ( ) : <ISA_LIT>GenRegisterInfo ( <NUM_LIT> ) {


<s> bool trackLivenessAfterRegAlloc ( const MachineFunction & MF ) const override {


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> <ISA_LIT>Subtarget & <ISA_LIT>Subtarget :: initializeSubtargetDependencies ( const Triple & TT , StringRef GPU , StringRef FS ) {

SmallString < <NUM_LIT> > FullFS ( <STR_LIT> ) ;


<s> <ISA_LIT>Subtarget :: <ISA_LIT>Subtarget ( const Triple & TT , StringRef GPU , StringRef FS , const TargetMachine & TM ) : <ISA_LIT>GenSubtargetInfo ( TT , GPU , GPU , FS ) , <ISA_LIT>Subtarget ( TT ) , InstrInfo ( * this ) , FrameLowering ( TargetFrameLowering :: StackGrowsUp , getStackAlignment ( ) , <NUM_LIT> ) , FMA ( false ) , CaymanISA ( false ) , CFALUBug ( false ) , HasVertexCache ( false ) , <ISA_LIT>ALUInst ( false ) , FP64 ( false ) , TexVTXClauseSize ( <NUM_LIT> ) , Gen ( <ISA_LIT> ) , TLInfo ( TM , initializeSubtargetDependencies ( TT , GPU , FS ) ) , InstrItins ( getInstrItineraryForCPU ( GPU ) ) {


<s> void <ISA_LIT>PassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>CFGStructurizerPass ( ) ) ; addPass ( create<ISA_LIT>ExpandSpecialInstrsPass ( ) ) ; addPass ( & FinalizeMachineBundlesID ) ; addPass ( create<ISA_LIT>Packetizer ( ) ) ;

addPass ( create<ISA_LIT>ControlFlowFinalizer ( ) ) ;


addPass ( create<ISA_LIT>Packetizer ( ) ) ;


<s> addPass ( & FinalizeMachineBundlesID ) ; addPass ( create<ISA_LIT>Packetizer ( ) ) ; addPass ( create<ISA_LIT>ControlFlowFinalizer ( ) ) ;


<s> if ( Enable<ISA_LIT>IfConvert ) addPass ( & IfConverterID ) ;

addPass ( create<ISA_LIT>ClauseMergePass ( ) ) ;


<s> return create<ISA_LIT>MachineScheduler ( C ) ;


<s> SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F ) ; I = std :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ;


<s> const TargetSubtargetInfo * <ISA_LIT>TargetMachine :: getSubtargetImpl ( const Function & F ) const { StringRef GPU = getGPUName ( F ) ; StringRef FS = getFeatureString ( F ) ; SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F ) ; I = std :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ;


<s> return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this , F ) ) ;


<s> return new <ISA_LIT>TextureIntrinsicsReplacer ( ) ;


<s> Mod = & M ; FloatType = Type :: getFloatTy ( Ctx ) ; Int32Type = Type :: getInt32Ty ( Ctx ) ; V4f32Type = VectorType :: get ( FloatType , <NUM_LIT> ) ; V4i32Type = VectorType :: get ( Int32Type , <NUM_LIT> ) ; Type * ArgsType [ ] = { V4f32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , } ; TexSign = FunctionType :: get ( V4f32Type , ArgsType , false ) ; Type * ArgsQType [ ] = {


<s> CT [ <NUM_LIT> ] = <NUM_LIT> ; } if ( TextureType == TEXTURE_CUBE_ARRAY || TextureType == TEXTURE_SHADOWCUBE_ARRAY ) CT [ <NUM_LIT> ] = <NUM_LIT> ; if ( TextureType == TEXTURE_1D_ARRAY || TextureType == TEXTURE_SHADOW1D_ARRAY ) { if ( hasLOD && useShadowVariant ) { CT [ <NUM_LIT> ] = <NUM_LIT> ; } else { CT [ <NUM_LIT> ] = <NUM_LIT> ; SrcSelect [ <NUM_LIT> ] = <NUM_LIT> ; } } else if ( TextureType == TEXTURE_2D_ARRAY || TextureType == TEXTURE_SHADOW2D_ARRAY ) { CT [ <NUM_LIT> ] = <NUM_LIT> ; }

if ( ( TextureType == TEXTURE_SHADOW1D || TextureType == TEXTURE_SHADOW2D || TextureType == TEXTURE_SHADOWRECT || TextureType == TEXTURE_SHADOW1D_ARRAY ) && ! ( hasLOD && useShadowVariant ) ) SrcSelect [ <NUM_LIT> ] = <NUM_LIT> ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> <ISA_LIT>TextureIntrinsicsReplacer ( ) : FunctionPass ( ID ) {


<s> void ReplaceCallInst ( CallInst & I , FunctionType * FT , const char * Name , unsigned SrcSelect [ <NUM_LIT> ] , Value * Offset [ <NUM_LIT> ] , Value * Resource , Value * Sampler , unsigned CT [ <NUM_LIT> ] , Value * Coord ) { IRBuilder < > Builder ( & I ) ; Constant * Mask [ ] = { ConstantInt :: get ( Int32Type , SrcSelect [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , SrcSelect [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , SrcSelect [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , SrcSelect [ <NUM_LIT> ] ) } ; Value * SwizzleMask = ConstantVector :: get ( Mask ) ; Value * SwizzledCoord = Builder . CreateShuffleVector ( Coord , Coord , SwizzleMask ) ; Value * Args [ ] = { SwizzledCoord , Offset [ <NUM_LIT> ] , Offset [ <NUM_LIT> ] , Offset [ <NUM_LIT> ] , Resource , Sampler , ConstantInt :: get ( Int32Type , CT [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , CT [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , CT [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , CT [ <NUM_LIT> ] ) } ; Function * F = Mod -> getFunction ( Name ) ; if ( ! F ) { F = Function :: Create ( FT , GlobalValue :: ExternalLinkage , Name , Mod ) ;

F -> addFnAttr ( Attribute :: ReadNone ) ;


<s> void ReplaceTexIntrinsic ( CallInst & I , bool hasLOD , FunctionType * FT , const char * VanillaInt , const char * ShadowInt ) { Value * Coord = I . getArgOperand ( <NUM_LIT> ) ; Value * ResourceId = I . getArgOperand ( <NUM_LIT> ) ; Value * SamplerId = I . getArgOperand ( <NUM_LIT> ) ; unsigned TextureType = cast < ConstantInt > ( I . getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ;

unsigned SrcSelect [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;


<s> unsigned SrcSelect [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; unsigned CT [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; Value * Offset [ <NUM_LIT> ] = {

ConstantInt :: get ( Int32Type , <NUM_LIT> ) , ConstantInt :: get ( Int32Type , <NUM_LIT> ) , ConstantInt :: get ( Int32Type , <NUM_LIT> ) } ;


<s> void ReplaceTXF ( CallInst & I ) { Value * Coord = I . getArgOperand ( <NUM_LIT> ) ; Value * ResourceId = I . getArgOperand ( <NUM_LIT> ) ; Value * SamplerId = I . getArgOperand ( <NUM_LIT> ) ;

unsigned TextureType = cast < ConstantInt > ( I . getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ;


<s> unsigned SrcSelect [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; unsigned CT [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; Value * Offset [ <NUM_LIT> ] = { I . getArgOperand ( <NUM_LIT> ) , I . getArgOperand ( <NUM_LIT> ) , I . getArgOperand ( <NUM_LIT> ) , } ; bool useShadowVariant ;

getAdjustmentFromTextureTarget ( TextureType , false , SrcSelect , CT , useShadowVariant ) ;


<s> bool runOnFunction ( Function & F ) override { visit ( F ) ; return false ;


<s> void visitCallInst ( CallInst & I ) { if ( ! I . getCalledFunction ( ) ) return ; StringRef Name = I . getCalledFunction ( ) -> getName ( ) ; if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , true , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , true , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTXF ( I ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexQSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ;


<s> ReplaceTXF ( I ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexQSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; }


<s> } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , true , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , true , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTXF ( I ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexQSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ;


<s> ReplaceTXF ( I ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexQSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ; return ; } if ( Name == <STR_LIT> ) { ReplaceTexIntrinsic ( I , false , TexSign , <STR_LIT> , <STR_LIT> ) ;


<s> int DstIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vdata ) ; MachineOperand * MO_Dmask = TII -> getNamedOperand ( MI , <STR_LIT> :: dmask ) ; assert ( MO_Dmask && <STR_LIT> ) ; unsigned dmask = MO_Dmask -> getImm ( ) ; unsigned ActiveLanes = TII -> isGather4 ( Opcode ) ? <NUM_LIT> : countPopulation ( dmask ) ; static_assert ( <STR_LIT> == <NUM_LIT> && <STR_LIT> == <NUM_LIT> , <STR_LIT> ) ; bool Packed = ! ST . hasUnpackedD16VMem ( ) ; unsigned InitIdx = D16Val && Packed ? ( ( ActiveLanes + <NUM_LIT> ) >> <NUM_LIT> ) + <NUM_LIT> : ActiveLanes + <NUM_LIT> ; uint32_t DstSize = RI -> getRegSizeInBits ( * TII -> getOpRegClass ( MI , DstIdx ) ) / <NUM_LIT> ; if ( DstSize < InitIdx ) continue ; Register PrevDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; unsigned NewDst = <NUM_LIT> ; unsigned SizeLeft = ST . usePRTStrictNull ( ) ? InitIdx : <NUM_LIT> ; unsigned CurrIdx = ST . usePRTStrictNull ( ) ? <NUM_LIT> : InitIdx ; if ( DstSize == <NUM_LIT> ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) . addImm ( <NUM_LIT> ) ; NewDst = PrevDst ; } else { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) ; for ( ; SizeLeft ; SizeLeft -- , CurrIdx ++ ) { NewDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; Register SubReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , NewDst ) . addReg ( PrevDst ) . addReg ( SubReg ) . addImm ( CurrIdx ) ; PrevDst = NewDst ; } } MachineInstrBuilder ( MF , MI ) . addReg ( NewDst , RegState :: Implicit ) ; MI . tieOperands ( DstIdx , MI . getNumOperands ( ) - <NUM_LIT> ) ; Changed = true ; } } } }


<s> static_assert ( <STR_LIT> == <NUM_LIT> && <STR_LIT> == <NUM_LIT> , <STR_LIT> ) ; bool Packed = ! ST . hasUnpackedD16VMem ( ) ; unsigned InitIdx = D16Val && Packed ? ( ( ActiveLanes + <NUM_LIT> ) >> <NUM_LIT> ) + <NUM_LIT> : ActiveLanes + <NUM_LIT> ; uint32_t DstSize = RI -> getRegSizeInBits ( * TII -> getOpRegClass ( MI , DstIdx ) ) / <NUM_LIT> ; if ( DstSize < InitIdx ) continue ; Register PrevDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; unsigned NewDst = <NUM_LIT> ; unsigned SizeLeft = ST . usePRTStrictNull ( ) ? InitIdx : <NUM_LIT> ; unsigned CurrIdx = ST . usePRTStrictNull ( ) ? <NUM_LIT> : InitIdx ; if ( DstSize == <NUM_LIT> ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) . addImm ( <NUM_LIT> ) ; NewDst = PrevDst ; } else { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) ; for ( ; SizeLeft ; SizeLeft -- , CurrIdx ++ ) { NewDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; Register SubReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , NewDst ) . addReg ( PrevDst ) . addReg ( SubReg ) . addImm ( CurrIdx ) ; PrevDst = NewDst ; } } MachineInstrBuilder ( MF , MI ) . addReg ( NewDst , RegState :: Implicit ) ; MI . tieOperands ( DstIdx , MI . getNumOperands ( ) - <NUM_LIT> ) ; Changed = true ; } } }


<s> assert ( MO_Dmask && <STR_LIT> ) ; unsigned dmask = MO_Dmask -> getImm ( ) ; unsigned ActiveLanes = TII -> isGather4 ( Opcode ) ? <NUM_LIT> : countPopulation ( dmask ) ; bool Packed = ! ST . hasUnpackedD16VMem ( ) ; unsigned InitIdx = D16Val && Packed ? ( ( ActiveLanes + <NUM_LIT> ) >> <NUM_LIT> ) + <NUM_LIT> : ActiveLanes + <NUM_LIT> ; uint32_t DstSize = RI -> getRegSizeInBits ( * TII -> getOpRegClass ( MI , DstIdx ) ) / <NUM_LIT> ; if ( DstSize < InitIdx ) continue ; Register PrevDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; unsigned NewDst = <NUM_LIT> ; unsigned SizeLeft = ST . usePRTStrictNull ( ) ? InitIdx : <NUM_LIT> ; unsigned CurrIdx = ST . usePRTStrictNull ( ) ? <NUM_LIT> : ( InitIdx - <NUM_LIT> ) ; if ( DstSize == <NUM_LIT> ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) . addImm ( <NUM_LIT> ) ; NewDst = PrevDst ; } else { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) ; for ( ; SizeLeft ; SizeLeft -- , CurrIdx ++ ) { NewDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; Register SubReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , NewDst ) . addReg ( PrevDst ) . addReg ( SubReg ) . addImm ( <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( CurrIdx ) ) ; PrevDst = NewDst ; } } MachineInstrBuilder ( MF , MI ) . addReg ( NewDst , RegState :: Implicit ) ; MI . tieOperands ( DstIdx , MI . getNumOperands ( ) - <NUM_LIT> ) ; Changed = true ; } } } }


<s> return new <ISA_LIT>AddIMGInit ( ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * RI = ST . getRegisterInfo ( ) ; bool Changed = false ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; auto Opcode = MI . getOpcode ( ) ; if ( TII -> isMIMG ( Opcode ) && ! MI . mayStore ( ) ) { MachineOperand * TFE = TII -> getNamedOperand ( MI , <STR_LIT> :: tfe ) ; MachineOperand * LWE = TII -> getNamedOperand ( MI , <STR_LIT> :: lwe ) ; MachineOperand * D16 = TII -> getNamedOperand ( MI , <STR_LIT> :: d16 ) ; assert ( ( TFE && LWE ) && <STR_LIT> ) ; unsigned TFEVal = TFE -> getImm ( ) ; unsigned LWEVal = LWE -> getImm ( ) ; unsigned D16Val = D16 ? D16 -> getImm ( ) : <NUM_LIT> ; if ( TFEVal || LWEVal ) { const DebugLoc & DL = MI . getDebugLoc ( ) ; int DstIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vdata ) ; MachineOperand * MO_Dmask = TII -> getNamedOperand ( MI , <STR_LIT> :: dmask ) ; assert ( MO_Dmask && <STR_LIT> ) ; unsigned dmask = MO_Dmask -> getImm ( ) ; unsigned ActiveLanes = TII -> isGather4 ( Opcode ) ? <NUM_LIT> : countPopulation ( dmask ) ; static_assert ( <STR_LIT> == <NUM_LIT> && <STR_LIT> == <NUM_LIT> , <STR_LIT> ) ; bool Packed = ! ST . hasUnpackedD16VMem ( ) ; unsigned InitIdx = D16Val && Packed ? ( ( ActiveLanes + <NUM_LIT> ) >> <NUM_LIT> ) + <NUM_LIT> : ActiveLanes + <NUM_LIT> ; uint32_t DstSize = RI -> getRegSizeInBits ( * TII -> getOpRegClass ( MI , DstIdx ) ) / <NUM_LIT> ; if ( DstSize < InitIdx ) continue ;

unsigned PrevDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ;


<s> initialize<ISA_LIT>AddIMGInitPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> <ISA_LIT>AddIMGInit ( ) : MachineFunctionPass ( ID ) { initialize<ISA_LIT>AddIMGInitPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> SmallVector < BasicBlock * , <NUM_LIT> > Preds ; for ( BasicBlock * Pred : predecessors ( BB ) ) { if ( ! is_contained ( Latches , Pred ) ) Preds . push_back ( Pred ) ; } BB = SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ; }


<s> FunctionPass * llvm :: create<ISA_LIT>AnnotateControlFlowPass ( ) {

return new <ISA_LIT>AnnotateControlFlow ( ) ;


<s> bool <ISA_LIT>AnnotateControlFlow :: doInitialization ( Module & M ) { LLVMContext & Context = M . getContext ( ) ; Void = Type :: getVoidTy ( Context ) ; Boolean = Type :: getInt1Ty ( Context ) ; Int64 = Type :: getInt64Ty ( Context ) ; ReturnStruct = StructType :: get ( Boolean , Int64 ) ; BoolTrue = ConstantInt :: getTrue ( Context ) ;

BoolFalse = ConstantInt :: getFalse ( Context ) ;


<s> void <ISA_LIT>AnnotateControlFlow :: eraseIfUnused ( PHINode * Phi ) {

if ( RecursivelyDeleteDeadPHINode ( Phi ) ) {


<s> AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < DivergenceAnalysis > ( ) ; AU . addPreserved < DominatorTreeWrapperPass > ( ) ;

FunctionPass :: getAnalysisUsage ( AU ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {

AU . addRequired < LoopInfoWrapperPass > ( ) ;


<s> llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; SmallVector < WeakTrackingVH , <NUM_LIT> > LoopPhiConditions ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term , LoopPhiConditions ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred == BB ? Arg : Int64Zero , Pred ) ; Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ; for ( WeakTrackingVH Val : llvm :: reverse ( LoopPhiConditions ) ) { if ( PHINode * Cond = cast_or_null < PHINode > ( Val ) ) eraseIfUnused ( Cond ) ; } push ( Term -> getSuccessor ( <NUM_LIT> ) , Arg ) ;


<s> Value * Incoming = Phi -> getIncomingValue ( i ) ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( isa < ConstantInt > ( Incoming ) ) { NewPhi -> addIncoming ( Broken , From ) ; continue ; } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term , LoopPhiConditions ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ;

while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ;


<s> BasicBlock * IDom = DT -> getNode ( Phi -> getParent ( ) ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { if ( Phi -> getIncomingBlock ( i ) == IDom ) { if ( Phi -> getIncomingValue ( i ) != BoolTrue ) return false ; } else { if ( Phi -> getIncomingValue ( i ) != BoolFalse ) return false ;


<s> if ( Phi -> getIncomingBlock ( i ) == IDom ) { if ( Phi -> getIncomingValue ( i ) != BoolTrue ) return false ; } else {

if ( Phi -> getIncomingValue ( i ) != BoolFalse ) return false ;


<s> bool <ISA_LIT>AnnotateControlFlow :: isTopOfStack ( BasicBlock * BB ) {

return ! Stack . empty ( ) && Stack . back ( ) . first == BB ;


<s> bool <ISA_LIT>AnnotateControlFlow :: isUniform ( BranchInst * T ) { return DA -> isUniform ( T -> getCondition ( ) ) || T -> getMetadata ( <STR_LIT> ) != nullptr ;


<s> Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;


<s> if ( isUniform ( Term ) ) return ; Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) , <STR_LIT> , Term ) ;

Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;


<s> return Stack . pop_back_val ( ) . second ;


<s> void <ISA_LIT>AnnotateControlFlow :: push ( BasicBlock * BB , Value * Saved ) {

Stack . push_back ( std :: make_pair ( BB , Saved ) ) ;


<s> BranchInst * Term = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == BB && isElse ( Phi ) ) { insertElse ( Term ) ; eraseIfUnused ( Phi ) ;


<s> } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == BB && isElse ( Phi ) ) { insertElse ( Term ) ; eraseIfUnused ( Phi ) ; continue ; }


<s> <ISA_LIT>AnnotateControlFlow ( ) : FunctionPass ( ID ) {


<s> if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; }

BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ;


<s> bool <ISA_LIT>AnnotateControlFlow :: doInitialization ( Module & M ) { LLVMContext & Context = M . getContext ( ) ; Void = Type :: getVoidTy ( Context ) ; Boolean = Type :: getInt1Ty ( Context ) ; Int64 = Type :: getInt64Ty ( Context ) ; ReturnStruct = StructType :: get ( Boolean , Int64 , ( Type * ) nullptr ) ; BoolTrue = ConstantInt :: getTrue ( Context ) ; BoolFalse = ConstantInt :: getFalse ( Context ) ; BoolUndef = UndefValue :: get ( Boolean ) ;

Int64Zero = ConstantInt :: get ( Int64 , <NUM_LIT> ) ;


<s> BoolUndef = UndefValue :: get ( Boolean ) ; Int64Zero = ConstantInt :: get ( Int64 , <NUM_LIT> ) ; If = M . getOrInsertFunction ( IfIntrinsic , ReturnStruct , Boolean , ( Type * ) nullptr ) ; Else = M . getOrInsertFunction ( ElseIntrinsic , ReturnStruct , Int64 , ( Type * ) nullptr ) ; Break = M . getOrInsertFunction ( BreakIntrinsic , Int64 , Int64 , ( Type * ) nullptr ) ; IfBreak = M . getOrInsertFunction ( IfBreakIntrinsic , Int64 , Boolean , Int64 , ( Type * ) nullptr ) ; ElseBreak = M . getOrInsertFunction ( ElseBreakIntrinsic , Int64 , Int64 , Int64 , ( Type * ) nullptr ) ; Loop = M . getOrInsertFunction ( LoopIntrinsic , Boolean , Int64 , ( Type * ) nullptr ) ;

EndCf = M . getOrInsertFunction ( EndCfIntrinsic , Void , Int64 , ( Type * ) nullptr ) ;


<s> AU . addRequired < DominatorTreeWrapperPass > ( ) ;

AU . addPreserved < DominatorTreeWrapperPass > ( ) ;


<s> BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } eraseIfUnused ( Phi ) ; return Ret ; } else if ( Instruction * Inst = dyn_cast < Instruction > ( Cond ) ) { BasicBlock * Parent = Inst -> getParent ( ) ; Instruction * Insert ; if ( L -> contains ( Inst ) ) { Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } else { llvm_unreachable ( <STR_LIT> ) ; }


<s> Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;

push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;


<s> push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;


<s> Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;


<s> DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; for ( df_iterator < BasicBlock * > I = df_begin ( & F . getEntryBlock ( ) ) , E = df_end ( & F . getEntryBlock ( ) ) ; I != E ; ++ I ) { BranchInst * Term = dyn_cast < BranchInst > ( ( * I ) -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( * I ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == * I && isElse ( Phi ) ) {


<s> bool <ISA_LIT>AnnotateControlFlow :: runOnFunction ( Function & F ) { DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; for ( df_iterator < BasicBlock * > I = df_begin ( & F . getEntryBlock ( ) ) , E = df_end ( & F . getEntryBlock ( ) ) ; I != E ; ++ I ) { BranchInst * Term = dyn_cast < BranchInst > ( ( * I ) -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( * I ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == * I && isElse ( Phi ) ) {


<s> assert ( Stack . back ( ) . first == BB ) ; if ( L && L -> getHeader ( ) == BB ) { SmallVector < BasicBlock * , <NUM_LIT> > Latches ; L -> getLoopLatches ( Latches ) ; SmallVector < BasicBlock * , <NUM_LIT> > Preds ; for ( BasicBlock * Pred : predecessors ( BB ) ) { if ( ! is_contained ( Latches , Pred ) ) Preds . push_back ( Pred ) ; } BB = SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , nullptr , false ) ;


<s> SmallVector < BasicBlock * , <NUM_LIT> > Preds ; for ( BasicBlock * Pred : predecessors ( BB ) ) { if ( ! is_contained ( Latches , Pred ) ) Preds . push_back ( Pred ) ; } BB = SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , nullptr , false ) ; } Value * Exec = popSaved ( ) ;

Instruction * FirstInsertionPt = & * BB -> getFirstInsertionPt ( ) ;


<s> if ( ! isa < UndefValue > ( Exec ) && ! isa < UnreachableInst > ( FirstInsertionPt ) ) { Instruction * ExecDef = cast < Instruction > ( Exec ) ; BasicBlock * DefBB = ExecDef -> getParent ( ) ; if ( ! DT -> dominates ( DefBB , BB ) ) { FirstInsertionPt = & * SplitEdge ( DefBB , BB , DT , LI ) -> getFirstInsertionPt ( ) ;


<s> BB = SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , nullptr , false ) ; } Value * Exec = popSaved ( ) ; Instruction * FirstInsertionPt = & * BB -> getFirstInsertionPt ( ) ; if ( ! isa < UndefValue > ( Exec ) && ! isa < UnreachableInst > ( FirstInsertionPt ) ) { Instruction * ExecDef = cast < Instruction > ( Exec ) ; BasicBlock * DefBB = ExecDef -> getParent ( ) ; if ( ! DT -> dominates ( DefBB , BB ) ) { FirstInsertionPt = & * SplitEdge ( DefBB , BB , DT , LI ) -> getFirstInsertionPt ( ) ;


<s> if ( Changed ) LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ;


<s> AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addPreserved < LoopInfoWrapperPass > ( ) ;


<s> AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addPreserved < LoopInfoWrapperPass > ( ) ; AU . addPreserved < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ;


<s> bool <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( isUniform ( Term ) ) return false ; BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return false ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( IntMask , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) {


<s> BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( IntMask , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) { Value * PHIValue = IntMaskZero ; if ( Pred == BB ) PHIValue = Arg ; else if ( L -> contains ( Pred ) && DT -> dominates ( Pred , BB ) ) PHIValue = Broken ; Broken -> addIncoming ( PHIValue , Pred ) ; } Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , Arg ) ;


<s> Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } if ( isa < Constant > ( Cond ) ) { Instruction * Insert = Cond == BoolTrue ? Term : L -> getHeader ( ) -> getTerminator ( ) ; Value * Args [ ] = { Cond , Broken } ;


<s> for ( const Instruction & I : * BB ) { if ( const CallInst * CI = dyn_cast < CallInst > ( & I ) ) if ( CI -> getIntrinsicID ( ) == <STR_LIT> ) return true ; } return false ;


<s> bool <ISA_LIT>AnnotateControlFlow :: hasKill ( const BasicBlock * BB ) { for ( const Instruction & I : * BB ) {

if ( const CallInst * CI = dyn_cast < CallInst > ( & I ) ) if ( CI -> getIntrinsicID ( ) == <STR_LIT> ) return true ;


<s> LLVMContext & Context = M . getContext ( ) ; Void = Type :: getVoidTy ( Context ) ; Boolean = Type :: getInt1Ty ( Context ) ; IntMask = ST . isWave32 ( ) ? Type :: getInt32Ty ( Context ) : Type :: getInt64Ty ( Context ) ; ReturnStruct = StructType :: get ( Boolean , IntMask ) ; BoolTrue = ConstantInt :: getTrue ( Context ) ; BoolFalse = ConstantInt :: getFalse ( Context ) ; BoolUndef = UndefValue :: get ( Boolean ) ; IntMaskZero = ConstantInt :: get ( IntMask , <NUM_LIT> ) ; If = <STR_LIT> ( & M , <STR_LIT> , { IntMask } ) ; Else = <STR_LIT> ( & M , <STR_LIT> , { IntMask , IntMask } ) ;

IfBreak = <STR_LIT> ( & M , <STR_LIT> , { IntMask } ) ;


<s> bool <ISA_LIT>AnnotateControlFlow :: insertElse ( BranchInst * Term ) { if ( isUniform ( Term ) ) { return false ; } Value * Ret = CallInst :: Create ( Else , popSaved ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;


<s> return false ; } Value * Ret = CallInst :: Create ( Else , popSaved ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; return true ;


<s> push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; return true ;


<s> if ( isUniform ( Term ) ) return false ;

Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) , <STR_LIT> , Term ) ;


<s> assert ( Stack . back ( ) . first == BB ) ; if ( L && L -> getHeader ( ) == BB ) { SmallVector < BasicBlock * , <NUM_LIT> > Latches ; L -> getLoopLatches ( Latches ) ; SmallVector < BasicBlock * , <NUM_LIT> > Preds ; for ( BasicBlock * Pred : predecessors ( BB ) ) { if ( ! is_contained ( Latches , Pred ) ) Preds . push_back ( Pred ) ; } BB = SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , nullptr , false ) ;


<s> AU . addPreserved < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ;

FunctionPass :: getAnalysisUsage ( AU ) ;


<s> AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addPreserved < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; FunctionPass :: getAnalysisUsage ( AU ) ;


<s> llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( IntMask , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) { Value * PHIValue = IntMaskZero ; if ( Pred == BB ) PHIValue = Arg ; else if ( L -> contains ( Pred ) && DT -> dominates ( Pred , BB ) ) PHIValue = Broken ; Broken -> addIncoming ( PHIValue , Pred ) ; } Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ;

push ( Term -> getSuccessor ( <NUM_LIT> ) , Arg ) ;


<s> Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } if ( isa < Constant > ( Cond ) ) { Instruction * Insert = Cond == BoolTrue ? Term : L -> getHeader ( ) -> getTerminator ( ) ; Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; }


<s> } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; }


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; TargetPassConfig & TPC = getAnalysis < TargetPassConfig > ( ) ; const TargetMachine & TM = TPC . getTM < TargetMachine > ( ) ; initialize ( * F . getParent ( ) , TM . getSubtarget < GCNSubtarget > ( F ) ) ; for ( df_iterator < BasicBlock * > I = df_begin ( & F . getEntryBlock ( ) ) , E = df_end ( & F . getEntryBlock ( ) ) ; I != E ; ++ I ) { BasicBlock * BB = * I ; BranchInst * Term = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; if ( DT -> dominates ( Term -> getSuccessor ( <NUM_LIT> ) , BB ) ) handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == BB && isElse ( Phi ) ) { insertElse ( Term ) ; eraseIfUnused ( Phi ) ; continue ; } closeControlFlow ( BB ) ; } openIf ( Term ) ; } if ( ! Stack . empty ( ) ) {


<s> L -> getLoopLatches ( Latches ) ; std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ; } Value * Exec = popSaved ( ) ;

if ( ! isa < UndefValue > ( Exec ) ) CallInst :: Create ( EndCf , Exec , <STR_LIT> , & * BB -> getFirstInsertionPt ( ) ) ;


<s> BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ;

Value * Cond = Term -> getCondition ( ) ;


<s> NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } eraseIfUnused ( Phi ) ; return Ret ; } else if ( Instruction * Inst = dyn_cast < Instruction > ( Cond ) ) { BasicBlock * Parent = Inst -> getParent ( ) ; Instruction * Insert ; if ( L -> contains ( Inst ) ) { Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } else if ( isa < ConstantInt > ( Cond ) ) { Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Term ) ; } else { llvm_unreachable ( <STR_LIT> ) ;


<s> Value * Ret = NewPhi ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( isa < ConstantInt > ( Incoming ) ) { NewPhi -> addIncoming ( Broken , From ) ; continue ; } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } eraseIfUnused ( Phi ) ; return Ret ; } else if ( Instruction * Inst = dyn_cast < Instruction > ( Cond ) ) { BasicBlock * Parent = Inst -> getParent ( ) ; Instruction * Insert ; if ( L -> contains ( Inst ) ) { Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; }


<s> Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) , <STR_LIT> , Term ) ;


<s> continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( * I ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == * I && isElse ( Phi ) ) {


<s> if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == BB && isElse ( Phi ) ) { insertElse ( Term ) ; eraseIfUnused ( Phi ) ; continue ;


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; DA = & getAnalysis < DivergenceAnalysis > ( ) ; for ( df_iterator < BasicBlock * > I = df_begin ( & F . getEntryBlock ( ) ) , E = df_end ( & F . getEntryBlock ( ) ) ; I != E ; ++ I ) { BasicBlock * BB = * I ; BranchInst * Term = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ;


<s> Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( pred_iterator PI = pred_begin ( Target ) , PE = pred_end ( Target ) ; PI != PE ; ++ PI ) { Broken -> addIncoming ( * PI == BB ? Arg : Int64Zero , * PI ) ; } Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , Arg ) ;


<s> BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( pred_iterator PI = pred_begin ( Target ) , PE = pred_end ( Target ) ;


<s> BasicBlock * Parent = Phi -> getParent ( ) ; PHINode * NewPhi = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Parent -> front ( ) ) ; Value * Ret = NewPhi ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( isa < ConstantInt > ( Incoming ) ) { NewPhi -> addIncoming ( Broken , From ) ; continue ; } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ;

NewPhi -> setIncomingValue ( i , PhiArg ) ;


<s> std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , nullptr , DT , LI , false ) ; }

CallInst :: Create ( EndCf , popSaved ( ) , <STR_LIT> , BB -> getFirstInsertionPt ( ) ) ;


<s> SmallVector < BasicBlock * , <NUM_LIT> > Latches ; L -> getLoopLatches ( Latches ) ; std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , nullptr , DT , LI , false ) ;


<s> BoolFalse = ConstantInt :: getFalse ( Context ) ; BoolUndef = UndefValue :: get ( Boolean ) ; Int64Zero = ConstantInt :: get ( Int64 , <NUM_LIT> ) ; If = <STR_LIT> ( & M , <STR_LIT> ) ; Else = <STR_LIT> ( & M , <STR_LIT> ) ; IfBreak = <STR_LIT> ( & M , <STR_LIT> ) ; Loop = <STR_LIT> ( & M , <STR_LIT> ) ; EndCf = <STR_LIT> ( & M , <STR_LIT> ) ; return false ;


<s> AU . addPreserved < DominatorTreeWrapperPass > ( ) ; FunctionPass :: getAnalysisUsage ( AU ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LoopInfoWrapperPass > ( ) ;

AU . addRequired < DominatorTreeWrapperPass > ( ) ;


<s> void <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( isUniform ( Term ) ) return ; BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) { Value * PHIValue = Int64Zero ;


<s> if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == BB && isElse ( Phi ) ) { insertElse ( Term ) ; eraseIfUnused ( Phi ) ; continue ; }


<s> BranchInst * Term = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) {

PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ;


<s> llvm :: Loop * L = LI -> getLoopFor ( BB ) ; assert ( Stack . back ( ) . first == BB ) ; if ( L && L -> getHeader ( ) == BB ) { SmallVector < BasicBlock * , <NUM_LIT> > Latches ; L -> getLoopLatches ( Latches ) ; std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( ! is_contained ( Latches , * PI ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ; } Value * Exec = popSaved ( ) ;


<s> Else = M . getOrInsertFunction ( ElseIntrinsic , ReturnStruct , Int64 , ( Type * ) nullptr ) ; Break = M . getOrInsertFunction ( BreakIntrinsic , Int64 , Int64 , ( Type * ) nullptr ) ; cast < Function > ( Break ) -> setDoesNotAccessMemory ( ) ; IfBreak = M . getOrInsertFunction ( IfBreakIntrinsic , Int64 , Boolean , Int64 , ( Type * ) nullptr ) ; cast < Function > ( IfBreak ) -> setDoesNotAccessMemory ( ) ; ; ElseBreak = M . getOrInsertFunction ( ElseBreakIntrinsic , Int64 , Int64 , Int64 , ( Type * ) nullptr ) ; cast < Function > ( ElseBreak ) -> setDoesNotAccessMemory ( ) ;

Loop = M . getOrInsertFunction ( LoopIntrinsic , Boolean , Int64 , ( Type * ) nullptr ) ;


<s> void <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( isUniform ( Term ) ) return ; BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ;

for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred == BB ? Arg : Int64Zero , Pred ) ;


<s> for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ;


<s> L -> getLoopLatches ( Latches ) ; std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ; } CallInst :: Create ( EndCf , popSaved ( ) , <STR_LIT> , BB -> getFirstInsertionPt ( ) ) ;


<s> ReturnStruct = StructType :: get ( Boolean , Int64 , ( Type * ) nullptr ) ; BoolTrue = ConstantInt :: getTrue ( Context ) ; BoolFalse = ConstantInt :: getFalse ( Context ) ; BoolUndef = UndefValue :: get ( Boolean ) ; Int64Zero = ConstantInt :: get ( Int64 , <NUM_LIT> ) ; If = <STR_LIT> ( & M , <STR_LIT> ) ; Else = <STR_LIT> ( & M , <STR_LIT> ) ; Break = <STR_LIT> ( & M , <STR_LIT> ) ; IfBreak = <STR_LIT> ( & M , <STR_LIT> ) ; ElseBreak = <STR_LIT> ( & M , <STR_LIT> ) ; Loop = <STR_LIT> ( & M , <STR_LIT> ) ; EndCf = <STR_LIT> ( & M , <STR_LIT> ) ;


<s> Value * Arg = handleLoopCondition ( Cond , Broken , L , Term , LoopPhiConditions ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred == BB ? Arg : Int64Zero , Pred ) ; Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ; for ( WeakVH Val : reverse ( LoopPhiConditions ) ) {

if ( PHINode * Cond = cast_or_null < PHINode > ( Val ) ) eraseIfUnused ( Cond ) ;


<s> } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term , LoopPhiConditions ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } LoopPhiConditions . push_back ( WeakVH ( Phi ) ) ; return Ret ; }

if ( Instruction * Inst = dyn_cast < Instruction > ( Cond ) ) {


<s> if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } LoopPhiConditions . push_back ( WeakVH ( Phi ) ) ; return Ret ; } if ( Instruction * Inst = dyn_cast < Instruction > ( Cond ) ) { BasicBlock * Parent = Inst -> getParent ( ) ; Instruction * Insert ; if ( L -> contains ( Inst ) ) { Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } if ( isa < Constant > ( Cond ) ) { Instruction * Insert = Cond == BoolTrue ? Term : L -> getHeader ( ) -> getTerminator ( ) ; Value * Args [ ] = { Cond , Broken } ;


<s> for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ; } Value * Exec = popSaved ( ) ; if ( ! isa < UndefValue > ( Exec ) ) CallInst :: Create ( EndCf , Exec , <STR_LIT> , & * BB -> getFirstInsertionPt ( ) ) ;


<s> if ( Stack . back ( ) . first != BB ) return ; if ( L && L -> getHeader ( ) == BB ) { SmallVector < BasicBlock * , <NUM_LIT> > Latches ; L -> getLoopLatches ( Latches ) ; std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) {

if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ;


<s> void <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( DA -> isUniform ( Term -> getCondition ( ) ) ) { return ; } BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ;


<s> Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( pred_iterator PI = pred_begin ( Target ) , PE = pred_end ( Target ) ; PI != PE ; ++ PI ) { Broken -> addIncoming ( * PI == BB ? Arg : Int64Zero , * PI ) ;


<s> continue ; } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ;


<s> Value * Ret = CallInst :: Create ( Else , popSaved ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;

push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;


<s> Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) , <STR_LIT> , Term ) ;

Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ;


<s> void <ISA_LIT>AnnotateControlFlow :: openIf ( BranchInst * Term ) { if ( DA -> isUniform ( Term -> getCondition ( ) ) ) { return ; }


<s> BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ;

Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ;


<s> Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term , LoopPhiConditions ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred == BB ? Arg : Int64Zero , Pred ) ;

Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ;


<s> PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; SmallVector < WeakTrackingVH , <NUM_LIT> > LoopPhiConditions ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term , LoopPhiConditions ) ;


<s> StringRef getPassName ( ) const override {


<s> auto DL = MI -> getDebugLoc ( ) ; if ( NopInserted . find ( DL . getLine ( ) ) == NopInserted . end ( ) ) { BuildMI ( MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;

NopInserted . insert ( DL . getLine ( ) ) ;


<s> if ( ! ST . debuggerInsertNops ( ) ) return false ; if ( ! MF . getMMI ( ) . hasDebugInfo ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DenseSet < unsigned > NopInserted ; for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( MI -> isDebugInstr ( ) || ! MI -> getDebugLoc ( ) ) continue ; auto DL = MI -> getDebugLoc ( ) ;

if ( NopInserted . find ( DL . getLine ( ) ) == NopInserted . end ( ) ) {


<s> if ( ! ST . debuggerInsertNops ( ) ) return false ; if ( ! MF . getMMI ( ) . hasDebugInfo ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DenseSet < unsigned > NopInserted ; for ( auto & MBB : MF ) {

for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) {


<s> DenseSet < unsigned > NopInserted ; for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( MI -> isDebugValue ( ) || ! MI -> getDebugLoc ( ) ) continue ; auto DL = MI -> getDebugLoc ( ) ; if ( NopInserted . find ( DL . getLine ( ) ) == NopInserted . end ( ) ) {

BuildMI ( MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( MI -> isDebugValue ( ) || ! MI -> getDebugLoc ( ) ) continue ; auto DL = MI -> getDebugLoc ( ) ; if ( NopInserted . find ( DL . getLine ( ) ) == NopInserted . end ( ) ) { BuildMI ( MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; NopInserted . insert ( DL . getLine ( ) ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>DebuggerInsertNopsPass ( ) {

return new <ISA_LIT>DebuggerInsertNops ( ) ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! ST . debuggerInsertNops ( ) ) return false ; if ( ! MF . getMMI ( ) . hasDebugInfo ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DenseSet < unsigned > NopInserted ; for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( MI -> isDebugValue ( ) || ! MI -> getDebugLoc ( ) ) continue ; auto DL = MI -> getDebugLoc ( ) ; if ( NopInserted . find ( DL . getLine ( ) ) == NopInserted . end ( ) ) { BuildMI ( MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; NopInserted . insert ( DL . getLine ( ) ) ; }


<s> auto DL = MI -> getDebugLoc ( ) ; if ( NopInserted . find ( DL . getLine ( ) ) == NopInserted . end ( ) ) { BuildMI ( MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; NopInserted . insert ( DL . getLine ( ) ) ;


<s> <ISA_LIT>DebuggerInsertNops ( ) : MachineFunctionPass ( ID ) {


<s> return new <ISA_LIT>FixControlFlowLiveIntervals ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LIS -> getInterval ( Reg ) . markNotSpillable ( ) ; break ; } default : break ; } }


<s> return new <ISA_LIT>FixSGPRCopies ( ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>FixSGPRCopiesPass ( ) {


<s> unsigned SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ; const TargetRegisterClass * DstRC = TargetRegisterInfo :: isVirtualRegister ( DstReg ) ? MRI . getRegClass ( DstReg ) : TRI . getPhysRegClass ( DstReg ) ;

return std :: make_pair ( SrcRC , DstRC ) ;


<s> static std :: pair < const TargetRegisterClass * , const TargetRegisterClass * > getCopyRegClasses ( const MachineInstr & Copy , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { unsigned DstReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) :


<s> const MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( i ) . getReg ( ) ) ) continue ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( MI . getOperand ( i ) . getReg ( ) ) ) ) return true ; } return false ;