<s> unsigned EltOffset1 = CI . Offset1 / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width0 == EltOffset1 || EltOffset1 + CI . Width1 == EltOffset0 ) && CI . GLC0 == CI . GLC1 && CI . DLC0 == CI . DLC1 && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC0 == CI . SLC1 ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { CI . Offset0 = EltOffset0 / <NUM_LIT> ; CI . Offset1 = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset0 = EltOffset0 ;


<s> continue ; } const unsigned Opc = MI . getOpcode ( ) ; CombineInfo CI ; CI . I = I ; CI . InstClass = getInstClass ( Opc ) ; switch ( CI . InstClass ) { default : break ; case DS_READ : CI . EltSize = ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeRead2Pair ( CI ) ; } else { ++ I ; } continue ; case DS_WRITE : CI . EltSize = ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeWrite2Pair ( CI ) ; } else { ++ I ; } continue ; case S_BUFFER_LOAD_IMM : CI . EltSize = <STR_LIT> ( * STM , <NUM_LIT> ) ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeSBufferLoadImmPair ( CI ) ; OptimizeAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } else { ++ I ; } continue ; case BUFFER_LOAD_OFFEN : case BUFFER_LOAD_OFFSET : case BUFFER_LOAD_OFFEN_exact :


<s> MachineInstr * Def = MRI -> getUniqueVRegDef ( Base . getReg ( ) ) ; if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || Def -> getNumOperands ( ) != <NUM_LIT> ) return ; MachineOperand BaseLo = Def -> getOperand ( <NUM_LIT> ) ; MachineOperand BaseHi = Def -> getOperand ( <NUM_LIT> ) ; if ( ! BaseLo . isReg ( ) || ! BaseHi . isReg ( ) ) return ; MachineInstr * BaseLoDef = MRI -> getUniqueVRegDef ( BaseLo . getReg ( ) ) ; MachineInstr * BaseHiDef = MRI -> getUniqueVRegDef ( BaseHi . getReg ( ) ) ;

if ( ! BaseLoDef || BaseLoDef -> getOpcode ( ) != <STR_LIT> || ! BaseHiDef || BaseHiDef -> getOpcode ( ) != <STR_LIT> ) return ;


<s> Src1 = TII -> getNamedOperand ( * BaseHiDef , <STR_LIT> :: src1 ) ; if ( Src0 -> isImm ( ) ) std :: swap ( Src0 , Src1 ) ; if ( ! Src1 -> isImm ( ) ) return ; uint64_t Offset1 = Src1 -> getImm ( ) ; BaseHi = * Src0 ; Addr . Base . LoReg = BaseLo . getReg ( ) ; Addr . Base . HiReg = BaseHi . getReg ( ) ; Addr . Base . LoSubReg = BaseLo . getSubReg ( ) ; Addr . Base . HiSubReg = BaseHi . getSubReg ( ) ;

Addr . Offset = ( * Offset0P & <NUM_LIT> ) | ( Offset1 << <NUM_LIT> ) ;


<s> if ( MAddrNext . Base . LoReg != MAddr . Base . LoReg || MAddrNext . Base . HiReg != MAddr . Base . HiReg || MAddrNext . Base . LoSubReg != MAddr . Base . LoSubReg || MAddrNext . Base . HiSubReg != MAddr . Base . HiSubReg ) continue ; InstsWCommonBase . push_back ( std :: make_pair ( & MINext , MAddrNext . Offset ) ) ; int64_t Dist = MAddr . Offset - MAddrNext . Offset ; TargetLoweringBase :: AddrMode AM ; AM . HasBaseReg = true ; AM . BaseOffs = Dist ; if ( TLI -> isLegalGlobalAddressingMode ( AM ) && ( uint32_t ) std :: abs ( Dist ) > MaxDist ) { MaxDist = std :: abs ( Dist ) ; AnchorAddr = MAddrNext ; AnchorInst = & MINext ; } } if ( AnchorInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AnchorInst -> dump ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << AnchorAddr . Offset << <STR_LIT> ) ; unsigned Base = computeBase ( MI , AnchorAddr ) ; updateBaseAndOffset ( MI , Base , MAddr . Offset - AnchorAddr . Offset ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; ) ; for ( auto P : InstsWCommonBase ) { TargetLoweringBase :: AddrMode AM ; AM . HasBaseReg = true ; AM . BaseOffs = P . second - AnchorAddr . Offset ; if ( TLI -> isLegalGlobalAddressingMode ( AM ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << P . second ; dbgs ( ) << <STR_LIT> ; P . first -> dump ( ) ) ; updateBaseAndOffset ( * P . first , Base , P . second - AnchorAddr . Offset ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; P . first -> dump ( ) ) ; } } AnchorList . insert ( AnchorInst ) ; return true ;


<s> if ( STM -> ldsRequiresM0Init ( ) ) return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ;


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: read2Opcode ( unsigned EltSize ) const { if ( STM -> ldsRequiresM0Init ( ) ) return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ;


<s> if ( STM -> ldsRequiresM0Init ( ) ) return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ;


<s> TII = STM -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : MF ) {


<s> <ISA_LIT>LoadStoreOptimizer ( ) : MachineFunctionPass ( ID ) {


<s> TII -> getNamedOperand ( MI , <STR_LIT> :: vaddr ) -> setReg ( NewBase ) ; TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> setImm ( NewOffset ) ;


<s> const unsigned Width = ( CI . Width0 + CI . Width1 ) ; switch ( CI . InstClass ) { default : return ( Width <= <NUM_LIT> ) && ( STM . hasDwordx3LoadStores ( ) || ( Width != <NUM_LIT> ) ) ; case S_BUFFER_LOAD_IMM : switch ( Width ) {


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: write2ST64Opcode ( unsigned EltSize ) const { if ( STM -> ldsRequiresM0Init ( ) ) return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ;


<s> void <ISA_LIT>LowerControlFlowPass :: Branch ( MachineInstr & MI ) { if ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) == MI . getParent ( ) -> getNextNode ( ) ) MI . eraseFromParent ( ) ;


<s> void <ISA_LIT>LowerControlFlowPass :: Branch ( MachineInstr & MI ) {

if ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) == MI . getParent ( ) -> getNextNode ( ) ) MI . eraseFromParent ( ) ;


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addReg ( Src ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> void <ISA_LIT>LowerControlFlowPass :: computeIndirectRegAndOffset ( unsigned VecReg , unsigned & Reg , int & Offset ) { unsigned SubReg = TRI -> getSubReg ( VecReg , <STR_LIT> ) ; if ( ! SubReg ) SubReg = VecReg ; const TargetRegisterClass * RC = TRI -> getPhysRegClass ( SubReg ) ;

int RegIdx = TRI -> getHWRegIndex ( SubReg ) + Offset ;


<s> const TargetRegisterClass * RC = TRI -> getPhysRegClass ( SubReg ) ; int RegIdx = TRI -> getHWRegIndex ( SubReg ) + Offset ; if ( RegIdx < <NUM_LIT> ) { Offset = RegIdx ; RegIdx = <NUM_LIT> ; } else {


<s> FunctionPass * llvm :: create<ISA_LIT>LowerControlFlowPass ( TargetMachine & tm ) { return new <ISA_LIT>LowerControlFlowPass ( tm ) ;


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Saved = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Saved ) . addReg ( Src ) ;


<s> unsigned Saved = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Saved ) . addReg ( Src ) ;


<s> unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ; MI . eraseFromParent ( ) ;


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ;

Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ;


<s> DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ;


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Vcc ) . addReg ( Src ) ;


<s> unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Vcc ) . addReg ( Src ) ;


<s> unsigned Val = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Reg ; computeIndirectRegAndOffset ( Dst , Reg , Off ) ; MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Define ) . addReg ( Val ) . addReg ( Dst , RegState :: Implicit ) ; LoadM0 ( MI , MovRel , Off ) ;


<s> MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Reg ) . addReg ( Vec , RegState :: Implicit ) ; LoadM0 ( MI , MovRel , Off ) ;


<s> const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MBB . getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; assert ( MFI -> getShaderType ( ) == ShaderType :: PIXEL || MFI -> getShaderType ( ) == ShaderType :: GEOMETRY ) ; if ( ( Op . isImm ( ) ) ) { if ( Op . getImm ( ) & <NUM_LIT> ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; }


<s> assert ( MFI -> getShaderType ( ) == ShaderType :: PIXEL || MFI -> getShaderType ( ) == ShaderType :: GEOMETRY ) ; if ( ( Op . isImm ( ) ) ) { if ( Op . getImm ( ) & <NUM_LIT> ) {

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) . addImm ( Offset ) ; } else { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) ; } MBB . insert ( I , MovRel ) ; } else { assert ( <STR_LIT> . contains ( Save ) ) ; assert ( <STR_LIT> . contains ( Idx ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Save ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> ) . addReg ( Idx ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; if ( Offset ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addImm ( Offset ) ; } MBB . insert ( I , MovRel ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( - <NUM_LIT> ) ;

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Save ) ;


<s> unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Src ) ;


<s> void <ISA_LIT>LowerControlFlowPass :: Loop ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Src ) ;

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> break ; case <STR_LIT> : Else ( MI ) ; break ; case <STR_LIT> : Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { SkipIfDead ( MI ) ; HaveKill = false ; } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) SkipIfDead ( MI ) ; else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectSrc ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectDst ( MI ) ; break ; } } } if ( NeedWQM && MFI -> getShaderType ( ) == ShaderType :: PIXEL ) { MachineBasicBlock & MBB = MF . front ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; } if ( NeedFlat && MFI -> IsKernel ) { const MachineFrameInfo * FrameInfo = MF . getFrameInfo ( ) ; uint64_t StackSizeBytes = FrameInfo -> getStackSize ( ) ; int IndirectBegin = static_cast < const <ISA_LIT>InstrInfo * > ( TII ) -> getIndirectIndexBegin ( MF ) ; uint64_t StackOffset = IndirectBegin < <NUM_LIT> ? <NUM_LIT> : ( <NUM_LIT> * IndirectBegin / <NUM_LIT> ) ; assert ( ( StackSizeBytes < <NUM_LIT> ) && StackOffset < <NUM_LIT> && <STR_LIT> ) ; MachineBasicBlock & MBB = MF . front ( ) ; DebugLoc NoDL ;

MachineBasicBlock :: iterator Start = MBB . getFirstNonPHI ( ) ;


<s> for ( MachineBasicBlock * MBB = From ; MBB != To && ! MBB -> succ_empty ( ) ; MBB = * MBB -> succ_begin ( ) ) { for ( MachineBasicBlock :: iterator I = MBB -> begin ( ) , E = MBB -> end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( I -> isBundle ( ) || ! I -> isBundled ( ) ) if ( ++ NumInstr >= SkipThreshold ) return true ; } } return false ;


<s> MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) {


<s> if ( ! shouldSkip ( * From . getParent ( ) -> succ_begin ( ) , To . getMBB ( ) ) ) return ; DebugLoc DL = From . getDebugLoc ( ) ; BuildMI ( * From . getParent ( ) , & From , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( To ) ;


<s> void <ISA_LIT>LowerControlFlowPass :: Skip ( MachineInstr & From , MachineOperand & To ) { if ( ! shouldSkip ( * From . getParent ( ) -> succ_begin ( ) , To . getMBB ( ) ) ) return ; DebugLoc DL = From . getDebugLoc ( ) ; BuildMI ( * From . getParent ( ) , & From , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( To ) ;


<s> BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) ;


<s> DebugLoc DL = MI . getDebugLoc ( ) ; if ( MBB . getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getShaderType ( ) != ShaderType :: PIXEL || ! shouldSkip ( & MBB , & MBB . getParent ( ) -> back ( ) ) ) return ; MachineBasicBlock :: iterator Insert = & MI ; ++ Insert ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ;


<s> AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . addPreservedID ( LiveVariablesID ) ; MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( Begin -> succ_begin ( ) , Begin -> succ_end ( ) ) ; while ( ! Worklist . empty ( ) ) {

MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ;


<s> MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( MBB == End || ! Visited . insert ( MBB ) . second ) continue ; for ( auto & Term : MBB -> terminators ( ) ) if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) return true ; Worklist . append ( MBB -> succ_begin ( ) , MBB -> succ_end ( ) ) ;


<s> break ; default : assert ( false && <STR_LIT> ) ; break ; } MachineBasicBlock :: iterator Next ; for ( I = Prev ? Prev -> getIterator ( ) : MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MaskMI = * I ; switch ( MaskMI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : combineMasks ( MaskMI ) ; break ; default : I = MBB . end ( ) ; break ; } }


<s> bool Redundant = true ; for ( auto & I : MBB . instrs ( ) ) { if ( ! I . isDebugInstr ( ) && ! I . isUnconditionalBranch ( ) ) Redundant = false ; } if ( Redundant ) { MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Preds ( MBB . predecessors ( ) ) ; for ( auto P : Preds ) { P -> replaceSuccessor ( & MBB , Succ ) ; MachineBasicBlock :: iterator I ( P -> getFirstInstrTerminator ( ) ) ; while ( I != P -> end ( ) ) { if ( I -> isBranch ( ) ) { if ( TII -> getBranchDestBlock ( * I ) == & MBB ) { I -> getOperand ( <NUM_LIT> ) . setMBB ( Succ ) ; break ; } } I ++ ; } if ( I == P -> end ( ) ) { MachineFunction * MF = P -> getParent ( ) ; MachineFunction :: iterator InsertPt = P -> getNextNode ( ) ? MachineFunction :: iterator ( P -> getNextNode ( ) ) : MF -> end ( ) ; MF -> splice ( InsertPt , Succ ) ; } } MBB . removeSuccessor ( Succ ) ; if ( LIS ) { for ( auto & I : MBB . instrs ( ) ) LIS -> RemoveMachineInstrFromMaps ( I ) ; } MBB . clear ( ) ; MBB . eraseFromParent ( ) ;


<s> SmallVector < MachineInstr * , <NUM_LIT> > Worklist ; MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; MachineBasicBlock * SplitMBB = MBB ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : SplitMBB = process ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( InsertKillCleanups ) Worklist . push_back ( & MI ) ; else SplitMBB = process ( MI ) ; break ; default :


<s> MachineBasicBlock * B = & MBB ; do { if ( ! Visited . insert ( B ) . second ) return MBB . end ( ) ; auto E = B -> end ( ) ; for ( ; It != E ; ++ It ) { if ( It -> getOpcode ( ) == <STR_LIT> ) continue ; if ( TII -> mayReadEXEC ( * MRI , * It ) ) break ; } if ( It != E ) return It ; if ( B -> succ_size ( ) != <NUM_LIT> ) return MBB . end ( ) ; MachineBasicBlock * Succ = * B -> succ_begin ( ) ; It = Succ -> begin ( ) ; B = Succ ;


<s> MachineBasicBlock :: iterator <ISA_LIT>LowerControlFlow :: skipIgnoreExecInstsTrivialSucc ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator It ) const { SmallSet < const MachineBasicBlock * , <NUM_LIT> > Visited ; MachineBasicBlock * B = & MBB ; do { if ( ! Visited . insert ( B ) . second ) return MBB . end ( ) ; auto E = B -> end ( ) ; for ( ; It != E ; ++ It ) { if ( It -> getOpcode ( ) == <STR_LIT> ) continue ; if ( TII -> mayReadEXEC ( * MRI , * It ) ) break ; }


<s> void <ISA_LIT>LowerControlFlow :: Branch ( MachineInstr & MI ) {


<s> if ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) == MI . getParent ( ) -> getNextNode ( ) ) MI . eraseFromParent ( ) ;


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> unsigned SubReg = TRI -> getSubReg ( VecReg , <STR_LIT> ) ; if ( ! SubReg ) SubReg = VecReg ; const TargetRegisterClass * RC = TRI -> getPhysRegClass ( SubReg ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>LowerControlFlowPass ( ) {

return new <ISA_LIT>LowerControlFlow ( ) ;


<s> return new <ISA_LIT>LowerControlFlow ( ) ;


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ;

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ;


<s> BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Saved ) . addReg ( Src ) ; MI . eraseFromParent ( ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ;

unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> void <ISA_LIT>LowerControlFlow :: If ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ;

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ;


<s> unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> computeIndirectRegAndOffset ( Dst , Reg , Off ) ;

MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Define ) . addReg ( Val ) . addReg ( Dst , RegState :: Implicit ) ;


<s> unsigned Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; int Off = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned Reg ; computeIndirectRegAndOffset ( Vec , Reg , Off ) ;

MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Reg ) . addReg ( Vec , RegState :: Implicit ) ;


<s> MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Reg ) . addReg ( Vec , RegState :: Implicit ) ;


<s> if ( Op . getImm ( ) & <NUM_LIT> ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; } } else { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addOperand ( Op ) ; } MI . eraseFromParent ( ) ;


<s> BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Src ) ;


<s> BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Src ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ;


<s> TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MF . getSubtarget ( ) . getRegisterInfo ( ) ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool HaveKill = false ; bool NeedWQM = false ; bool NeedFlat = false ; unsigned Depth = <NUM_LIT> ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock * EmptyMBBAtEnd = NULL ; MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( TII -> isWQM ( MI ) || TII -> isDS ( MI ) ) NeedWQM = true ; if ( TII -> isFLAT ( MI ) ) NeedFlat = true ; switch ( MI . getOpcode ( ) ) { default : break ; case <STR_LIT> : ++ Depth ; If ( MI ) ; break ; case <STR_LIT> : Else ( MI ) ; break ; case <STR_LIT> : Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { SkipIfDead ( MI ) ; HaveKill = false ; } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) SkipIfDead ( MI ) ; else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectSrc ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectDst ( MI ) ; break ; case <STR_LIT> : { if ( MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) break ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( * BI , I , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; } I -> eraseFromParent ( ) ; break ; } } } }

if ( NeedWQM && MFI -> getShaderType ( ) == ShaderType :: PIXEL ) {


<s> for ( MachineBasicBlock * MBB = From ; MBB != To && ! MBB -> succ_empty ( ) ; MBB = * MBB -> succ_begin ( ) ) {

for ( MachineBasicBlock :: iterator I = MBB -> begin ( ) , E = MBB -> end ( ) ;


<s> bool <ISA_LIT>LowerControlFlow :: shouldSkip ( MachineBasicBlock * From , MachineBasicBlock * To ) { unsigned NumInstr = <NUM_LIT> ; for ( MachineBasicBlock * MBB = From ; MBB != To && ! MBB -> succ_empty ( ) ; MBB = * MBB -> succ_begin ( ) ) {


<s> MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) {


<s> DebugLoc DL = From . getDebugLoc ( ) ; BuildMI ( * From . getParent ( ) , & From , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( To ) ;


<s> BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) ;


<s> if ( MBB . getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getShaderType ( ) != ShaderType :: PIXEL || ! shouldSkip ( & MBB , & MBB . getParent ( ) -> back ( ) ) ) return ; MachineBasicBlock :: iterator Insert = & MI ; ++ Insert ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;

BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ;


<s> Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) {


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; Register DstReg = getSaveExec ( & MI ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register CopyReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ;


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = Def && Def -> getParent ( ) == & MBB ? std :: next ( MachineBasicBlock :: iterator ( Def ) ) : MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;

if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ;


<s> unsigned CFMask = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( CFMask ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = Def && Def -> getParent ( ) == & MBB ? std :: next ( MachineBasicBlock :: iterator ( Def ) ) : MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ; MI . eraseFromParent ( ) ;

if ( LIS ) LIS -> handleMove ( * NewMI ) ;


<s> Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( Tmp ) ;

if ( ! SimpleIf ) LIS -> createAndComputeVirtRegInterval ( CopyReg ) ;


<s> MachineOperand & Cond = MI . getOperand ( <NUM_LIT> ) ; assert ( Cond . getSubReg ( ) == <STR_LIT> ) ; MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI , TII ) ; Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ;


<s> } } MachineInstr * And = nullptr , * Or = nullptr ; if ( ! SkipAnding ) { Register AndReg = MRI -> createVirtualRegister ( BoolRC ) ; And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , AndReg ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;

Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( AndReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> MachineInstr * And = nullptr , * Or = nullptr ; if ( ! SkipAnding ) { Register AndReg = MRI -> createVirtualRegister ( BoolRC ) ; And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , AndReg ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( AndReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> createAndComputeVirtRegInterval ( AndReg ) ; } else Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) { if ( And ) LIS -> InsertMachineInstrInMaps ( * And ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * Or ) ; }


<s> MachineInstr * Branch = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) {

LIS -> ReplaceMachineInstrInMaps ( MI , * AndN2 ) ;


<s> MachineInstr * AndN2 = BuildMI ( MBB , & MI , DL , TII -> get ( Andn2TermOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; MachineInstr * Branch = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) { LIS -> ReplaceMachineInstrInMaps ( MI , * AndN2 ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ;


<s> MachineBasicBlock :: iterator J = std :: next ( I ) ; if ( J != MBB -> end ( ) && J -> getOpcode ( ) == FalseTermOpc && J -> getOperand ( <NUM_LIT> ) . isReg ( ) && J -> getOperand ( <NUM_LIT> ) . getReg ( ) == SaveExecReg ) { SaveExecReg = J -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( MBB == EMBB || ! Visited . insert ( MBB ) . second ) continue ; for ( auto & Term : MBB -> terminators ( ) ) if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) return false ;

Worklist . append ( MBB -> succ_begin ( ) , MBB -> succ_end ( ) ) ;


<s> Exec = <STR_LIT> ; } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next , Last ; for ( I = MBB . begin ( ) , Last = MBB . end ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : emitIf ( MI ) ; break ; case <STR_LIT> : emitElse ( MI ) ; break ; case <STR_LIT> : emitIfBreak ( MI ) ; break ; case <STR_LIT> : emitLoop ( MI ) ; break ; case <STR_LIT> : emitEndCf ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : combineMasks ( MI ) ; Last = I ; continue ; default : Last = I ;


<s> MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ; LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ; LIS -> removeInterval ( DstReg ) ; LIS -> createAndComputeVirtRegInterval ( DstReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveReg ) ;

LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT> ) ;


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register SaveReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LV ) LV -> replaceKillInstruction ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MI , * OrSaveExec ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ;

if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ;


<s> NeedBlockSplit = true ; break ; } } unsigned Opcode = OrOpc ; MachineBasicBlock * SplitBB = & MBB ; if ( NeedBlockSplit ) { SplitBB = MBB . splitAt ( MI , true , LIS ) ; if ( MDT && SplitBB != & MBB ) { MachineDomTreeNode * MBBNode = ( * MDT ) [ & MBB ] ; SmallVector < MachineDomTreeNode * > Children ( MBBNode -> begin ( ) , MBBNode -> end ( ) ) ; MachineDomTreeNode * SplitBBNode = MDT -> addNewBlock ( SplitBB , & MBB ) ; for ( MachineDomTreeNode * Child : Children ) MDT -> changeImmediateDominator ( Child , SplitBBNode ) ;


<s> if ( SimpleIf ) { auto UseMI = MRI -> use_instr_nodbg_begin ( SaveExecReg ) ; SimpleIf = ! hasKill ( MI . getParent ( ) , UseMI -> getParent ( ) ) ; } Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; LoweredIf . insert ( CopyReg ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; if ( LV ) LV -> replaceKillInstruction ( Cond . getReg ( ) , MI , * And ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; }

MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ;


<s> SimpleIf = ! hasKill ( MI . getParent ( ) , UseMI -> getParent ( ) ) ; } Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; LoweredIf . insert ( CopyReg ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; if ( LV ) LV -> replaceKillInstruction ( Cond . getReg ( ) , MI , * And ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; if ( LV ) LV -> getVarInfo ( Tmp ) . Kills . push_back ( SetExec ) ; I = skipToUncondBrOrEnd ( MBB , I ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( Tmp ) ;


<s> Register AndReg = MRI -> createVirtualRegister ( BoolRC ) ; And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , AndReg ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LV ) LV -> replaceKillInstruction ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MI , * And ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( AndReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> createAndComputeVirtRegInterval ( AndReg ) ; } else { Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;

if ( LV ) LV -> replaceKillInstruction ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MI , * Or ) ;


<s> } else { Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LV ) LV -> replaceKillInstruction ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MI , * Or ) ; } if ( LV ) LV -> replaceKillInstruction ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MI , * Or ) ; if ( LIS ) { if ( And ) LIS -> InsertMachineInstrInMaps ( * And ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * Or ) ; } MI . eraseFromParent ( ) ;


<s> BuildMI ( * MBB , MBB -> begin ( ) , MI . getDebugLoc ( ) , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) ) ; if ( LIS ) LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; return ; } Register InputReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * FirstMI = & * MBB -> begin ( ) ; if ( InputReg . isVirtual ( ) ) { MachineInstr * DefInstr = MRI -> getVRegDef ( InputReg ) ; assert ( DefInstr && DefInstr -> isCopy ( ) ) ; if ( DefInstr -> getParent ( ) == MBB ) { if ( DefInstr != FirstMI ) { DefInstr -> removeFromParent ( ) ; MBB -> insert ( FirstMI , DefInstr ) ; if ( LIS ) LIS -> handleMove ( * DefInstr ) ; } else { FirstMI = & * std :: next ( FirstMI -> getIterator ( ) ) ; } } } const DebugLoc DL = MI . getDebugLoc ( ) ; const unsigned WavefrontSize = ST . getWavefrontSize ( ) ; const unsigned Mask = ( WavefrontSize << <NUM_LIT> ) - <NUM_LIT> ; Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto BfeMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( InputReg ) . addImm ( ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & Mask ) | <NUM_LIT> ) ; if ( LV ) LV -> recomputeForSingleDefVirtReg ( InputReg ) ; auto BfmMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( CountReg ) . addImm ( <NUM_LIT> ) ; auto CmpMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( CountReg , RegState :: Kill ) . addImm ( WavefrontSize ) ; if ( LV ) LV -> getVarInfo ( CountReg ) . Kills . push_back ( CmpMI ) ; auto CmovMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( - <NUM_LIT> ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * BfeMI ) ;

LIS -> InsertMachineInstrInMaps ( * BfmMI ) ;


<s> MBB -> insert ( FirstMI , DefInstr ) ; if ( LIS ) LIS -> handleMove ( * DefInstr ) ; } else { FirstMI = & * std :: next ( FirstMI -> getIterator ( ) ) ; } } } const DebugLoc DL = MI . getDebugLoc ( ) ; const unsigned WavefrontSize = ST . getWavefrontSize ( ) ; const unsigned Mask = ( WavefrontSize << <NUM_LIT> ) - <NUM_LIT> ; Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto BfeMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( InputReg ) . addImm ( ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & Mask ) | <NUM_LIT> ) ; if ( LV ) LV -> recomputeForSingleDefVirtReg ( InputReg ) ; auto BfmMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( CountReg ) . addImm ( <NUM_LIT> ) ; auto CmpMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( CountReg , RegState :: Kill ) . addImm ( WavefrontSize ) ; if ( LV ) LV -> getVarInfo ( CountReg ) . Kills . push_back ( CmpMI ) ; auto CmovMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( - <NUM_LIT> ) ; if ( ! LIS ) {


<s> if ( Next == MBB . end ( ) || ! LoweredEndCf . count ( & * Next ) ) continue ; Register SavedExec = TII -> getNamedOperand ( * Next , <STR_LIT> :: src1 ) -> getReg ( ) ; assert ( SavedExec . isVirtual ( ) && <STR_LIT> ) ;

const MachineInstr * Def = MRI -> getUniqueVRegDef ( SavedExec ) ;


<s> } if ( MDT ) { if ( MDT -> dominates ( & MBB , Succ ) ) MDT -> changeImmediateDominator ( MDT -> getNode ( Succ ) , MDT -> getNode ( & MBB ) -> getIDom ( ) ) ; MDT -> eraseNode ( & MBB ) ; } MBB . clear ( ) ; MBB . eraseFromParent ( ) ; if ( FallThrough && ! FallThrough -> isLayoutSuccessor ( Succ ) ) { if ( ! Succ -> canFallThrough ( ) ) { MachineFunction * MF = FallThrough -> getParent ( ) ;

MachineFunction :: iterator FallThroughPos ( FallThrough ) ;


<s> MachineBasicBlock * FallThrough = nullptr ; while ( ! MBB . predecessors ( ) . empty ( ) ) { MachineBasicBlock * P = * MBB . pred_begin ( ) ; if ( P -> getFallThrough ( ) == & MBB ) FallThrough = P ; P -> ReplaceUsesOfBlockWith ( & MBB , Succ ) ; } MBB . removeSuccessor ( Succ ) ; if ( LIS ) { for ( auto & I : MBB . instrs ( ) ) LIS -> RemoveMachineInstrFromMaps ( I ) ; } if ( MDT ) { if ( MDT -> dominates ( & MBB , Succ ) ) MDT -> changeImmediateDominator ( MDT -> getNode ( Succ ) , MDT -> getNode ( & MBB ) -> getIDom ( ) ) ;


<s> MDT = getAnalysisIfAvailable < MachineDominatorTree > ( ) ; MRI = & MF . getRegInfo ( ) ; BoolRC = TRI -> getBoolRC ( ) ; if ( ST . isWave32 ( ) ) { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; MachineBasicBlock * SplitMBB = MBB ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : SplitMBB = process ( MI ) ; break ; case <STR_LIT> :


<s> MachineBasicBlock & MBB = * MI -> getParent ( ) ; auto Next = skipIgnoreExecInstsTrivialSucc ( MBB , std :: next ( MI -> getIterator ( ) ) ) ; if ( Next == MBB . end ( ) || ! LoweredEndCf . count ( & * Next ) ) continue ; Register SavedExec = TII -> getNamedOperand ( * Next , <STR_LIT> :: src1 ) -> getReg ( ) ; assert ( SavedExec . isVirtual ( ) && <STR_LIT> ) ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( SavedExec ) ; if ( Def && LoweredIf . count ( SavedExec ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ) ; if ( LIS ) LIS -> RemoveMachineInstrFromMaps ( * MI ) ; Register Reg ; if ( LV ) Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) -> getReg ( ) ; MI -> eraseFromParent ( ) ; if ( LV ) LV -> recomputeForSingleDefVirtReg ( Reg ) ; removeMBBifRedundant ( MBB ) ; }


<s> void <ISA_LIT>LowerControlFlow :: combineMasks ( MachineInstr & MI ) { assert ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) ; SmallVector < MachineOperand , <NUM_LIT> > Ops ; unsigned OpToReplace = <NUM_LIT> ; findMaskOperands ( MI , <NUM_LIT> , Ops ) ; if ( Ops . size ( ) == <NUM_LIT> ) OpToReplace = <NUM_LIT> ; findMaskOperands ( MI , <NUM_LIT> , Ops ) ; if ( Ops . size ( ) != <NUM_LIT> ) return ; unsigned UniqueOpndIdx ; if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ;


<s> if ( Ops . size ( ) != <NUM_LIT> ) return ; unsigned UniqueOpndIdx ; if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else return ; unsigned Reg = MI . getOperand ( OpToReplace ) . getReg ( ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Or = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * Or ) ;


<s> MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( <STR_LIT> ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ;

LIS -> InsertMachineInstrInMaps ( * Xor ) ;


<s> MI . setDesc ( TII -> get ( <STR_LIT> ) ) ;


<s> void <ISA_LIT>LowerControlFlow :: emitElseBreak ( MachineInstr & MI ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ;


<s> void <ISA_LIT>LowerControlFlow :: emitEndCf ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> assert ( SaveExec . getSubReg ( ) == <STR_LIT> && Cond . getSubReg ( ) == <STR_LIT> ) ; unsigned SaveExecReg = SaveExec . getReg ( ) ; MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI ) ; unsigned CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> , RegState :: ImplicitDefine ) ; unsigned Tmp = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Tmp ) . addReg ( CopyReg ) . addReg ( Cond . getReg ( ) ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ; MI . eraseFromParent ( ) ; LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( Tmp ) ; if ( ! SimpleIf ) LIS -> createAndComputeVirtRegInterval ( CopyReg ) ;


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineInstr * AndN2 = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; MachineInstr * Branch = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) { LIS -> ReplaceMachineInstrInMaps ( MI , * AndN2 ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ;


<s> void <ISA_LIT>LowerControlFlow :: findMaskOperands ( MachineInstr & MI , unsigned OpNo , SmallVectorImpl < MachineOperand > & Src ) const { MachineOperand & Op = MI . getOperand ( OpNo ) ; if ( ! Op . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( Op . getReg ( ) ) ) { Src . push_back ( Op ) ; return ; } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ; for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) != <STR_LIT> ) ) return ;

for ( const auto & SrcOp : Def -> explicit_operands ( ) ) if ( SrcOp . isUse ( ) && ( ! SrcOp . isReg ( ) || TargetRegisterInfo :: isVirtualRegister ( SrcOp . getReg ( ) ) || SrcOp . getReg ( ) == <STR_LIT> ) ) Src . push_back ( SrcOp ) ;


<s> DenseSet < const MachineBasicBlock * > Visited ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( SMBB -> succ_begin ( ) , SMBB -> succ_end ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ;

if ( MBB == EMBB || ! Visited . insert ( MBB ) . second ) continue ;


<s> assert ( SaveExec . getSubReg ( ) == <STR_LIT> && Cond . getSubReg ( ) == <STR_LIT> ) ; unsigned SaveExecReg = SaveExec . getReg ( ) ; MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI , TII ) ; unsigned CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> , RegState :: ImplicitDefine ) ; unsigned Tmp = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Tmp ) . addReg ( CopyReg ) . addReg ( Cond . getReg ( ) ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ; MI . eraseFromParent ( ) ; LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( Tmp ) ; if ( ! SimpleIf ) LIS -> createAndComputeVirtRegInterval ( CopyReg ) ;


<s> if ( U == MRI -> use_instr_nodbg_end ( ) || std :: next ( U ) != MRI -> use_instr_nodbg_end ( ) || U -> getOpcode ( ) != <STR_LIT> ) return false ; auto SMBB = MI . getParent ( ) ; auto EMBB = U -> getParent ( ) ; DenseSet < const MachineBasicBlock * > Visited ;

SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( SMBB -> succ_begin ( ) , SMBB -> succ_end ( ) ) ;


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register SaveReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; ElsePt = skipToUncondBrOrEnd ( MBB , ElsePt ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ; LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ;


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = MBB . begin ( ) ; bool NeedBlockSplit = false ; Register DataReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineBasicBlock :: iterator I = InsPt , E = MI . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( DataReg , TRI ) ) { NeedBlockSplit = true ; break ; } } unsigned Opcode = OrOpc ; MachineBasicBlock * SplitBB = & MBB ; if ( NeedBlockSplit ) { SplitBB = MBB . splitAt ( MI , true , LIS ) ; if ( MDT && SplitBB != & MBB ) { MachineDomTreeNode * MBBNode = ( * MDT ) [ & MBB ] ; SmallVector < MachineDomTreeNode * > Children ( MBBNode -> begin ( ) , MBBNode -> end ( ) ) ; MachineDomTreeNode * SplitBBNode = MDT -> addNewBlock ( SplitBB , & MBB ) ; for ( MachineDomTreeNode * Child : Children ) MDT -> changeImmediateDominator ( Child , SplitBBNode ) ; } Opcode = OrTermrOpc ; InsPt = MI ; } MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( Opcode ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; LoweredEndCf . insert ( NewMI ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ; MI . eraseFromParent ( ) ; if ( LIS ) LIS -> handleMove ( * NewMI ) ; return SplitBB ;


<s> if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; I = skipToUncondBrOrEnd ( MBB , I ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ;


<s> MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI ) ; if ( SimpleIf ) { auto UseMI = MRI -> use_instr_nodbg_begin ( SaveExecReg ) ; SimpleIf = ! hasKill ( MI . getParent ( ) , UseMI -> getParent ( ) ) ; } Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; LoweredIf . insert ( CopyReg ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ;


<s> if ( ! SkipAnding ) { Register AndReg = MRI -> createVirtualRegister ( BoolRC ) ; And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , AndReg ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( AndReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> createAndComputeVirtRegInterval ( AndReg ) ;

} else Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineInstr * AndN2 = BuildMI ( MBB , & MI , DL , TII -> get ( Andn2TermOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; auto BranchPt = skipToUncondBrOrEnd ( MBB , MI . getIterator ( ) ) ; MachineInstr * Branch = BuildMI ( MBB , BranchPt , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) { LIS -> ReplaceMachineInstrInMaps ( MI , * AndN2 ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ;


<s> void <ISA_LIT>LowerControlFlow :: findMaskOperands ( MachineInstr & MI , unsigned OpNo , SmallVectorImpl < MachineOperand > & Src ) const { MachineOperand & Op = MI . getOperand ( OpNo ) ; if ( ! Op . isReg ( ) || ! Op . getReg ( ) . isVirtual ( ) ) { Src . push_back ( Op ) ; return ; } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ; for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) != Exec ) ) return ;

for ( const auto & SrcOp : Def -> explicit_operands ( ) ) if ( SrcOp . isReg ( ) && SrcOp . isUse ( ) && ( SrcOp . getReg ( ) . isVirtual ( ) || SrcOp . getReg ( ) == Exec ) ) Src . push_back ( SrcOp ) ;


<s> AU . addPreserved < LiveIntervals > ( ) ; AU . addPreservedID ( LiveVariablesID ) ; MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> bool <ISA_LIT>LowerControlFlow :: hasKill ( const MachineBasicBlock * Begin , const MachineBasicBlock * End ) { DenseSet < const MachineBasicBlock * > Visited ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( Begin -> successors ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( MBB == End || ! Visited . insert ( MBB ) . second ) continue ; if ( KillBlocks . contains ( MBB ) ) return true ; Worklist . append ( MBB -> succ_begin ( ) , MBB -> succ_end ( ) ) ; } return false ;


<s> if ( MBB == End || ! Visited . insert ( MBB ) . second ) continue ; if ( KillBlocks . contains ( MBB ) ) return true ; Worklist . append ( MBB -> succ_begin ( ) , MBB -> succ_end ( ) ) ;


<s> Register SaveExecReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

auto U = MRI -> use_instr_nodbg_begin ( SaveExecReg ) ;


<s> MachineInstr * FirstMI = & * MBB -> begin ( ) ; if ( InputReg . isVirtual ( ) ) { MachineInstr * DefInstr = MRI -> getVRegDef ( InputReg ) ; assert ( DefInstr && DefInstr -> isCopy ( ) ) ; if ( DefInstr -> getParent ( ) == MBB ) { if ( DefInstr != FirstMI ) { DefInstr -> removeFromParent ( ) ; MBB -> insert ( FirstMI , DefInstr ) ; if ( LIS ) LIS -> handleMove ( * DefInstr ) ; } else { FirstMI = & * std :: next ( FirstMI -> getIterator ( ) ) ; } } } const DebugLoc DL = MI . getDebugLoc ( ) ; const unsigned WavefrontSize = ST . getWavefrontSize ( ) ; const unsigned Mask = ( WavefrontSize << <NUM_LIT> ) - <NUM_LIT> ; Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto BfeMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( InputReg ) . addImm ( ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & Mask ) | <NUM_LIT> ) ; auto BfmMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( CountReg ) . addImm ( <NUM_LIT> ) ; auto CmpMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( CountReg , RegState :: Kill ) . addImm ( WavefrontSize ) ; auto CmovMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( - <NUM_LIT> ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ;


<s> FirstMI = & * std :: next ( FirstMI -> getIterator ( ) ) ; } } } const DebugLoc DL = MI . getDebugLoc ( ) ; const unsigned WavefrontSize = ST . getWavefrontSize ( ) ; const unsigned Mask = ( WavefrontSize << <NUM_LIT> ) - <NUM_LIT> ; Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto BfeMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( InputReg ) . addImm ( ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & Mask ) | <NUM_LIT> ) ; auto BfmMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( CountReg ) . addImm ( <NUM_LIT> ) ; auto CmpMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( CountReg , RegState :: Kill ) . addImm ( WavefrontSize ) ; auto CmovMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( - <NUM_LIT> ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * BfeMI ) ;


<s> for ( MachineInstr * MI : LoweredEndCf ) { MachineBasicBlock & MBB = * MI -> getParent ( ) ; auto Next = skipIgnoreExecInstsTrivialSucc ( MBB , std :: next ( MI -> getIterator ( ) ) ) ;

if ( Next == MBB . end ( ) || ! LoweredEndCf . count ( & * Next ) ) continue ;


<s> void <ISA_LIT>LowerControlFlow :: optimizeEndCf ( ) { if ( ! RemoveRedundantEndcf ) return ; for ( MachineInstr * MI : LoweredEndCf ) { MachineBasicBlock & MBB = * MI -> getParent ( ) ; auto Next = skipIgnoreExecInstsTrivialSucc ( MBB , std :: next ( MI -> getIterator ( ) ) ) ; if ( Next == MBB . end ( ) || ! LoweredEndCf . count ( & * Next ) ) continue ; Register SavedExec = TII -> getNamedOperand ( * Next , <STR_LIT> :: src1 ) -> getReg ( ) ; assert ( SavedExec . isVirtual ( ) && <STR_LIT> ) ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( SavedExec ) ; if ( Def && LoweredIf . count ( SavedExec ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ) ; if ( LIS ) LIS -> RemoveMachineInstrFromMaps ( * MI ) ; MI -> eraseFromParent ( ) ; removeMBBifRedundant ( MBB ) ;


<s> } MachineBasicBlock :: iterator Next ; for ( I = Prev ? Prev -> getIterator ( ) : MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MaskMI = * I ; switch ( MaskMI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : combineMasks ( MaskMI ) ; break ; default : I = MBB . end ( ) ; break ; }


<s> emitIfBreak ( MI ) ; break ; case <STR_LIT> : emitLoop ( MI ) ; break ; case <STR_LIT> : MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; break ; case <STR_LIT> : SplitBB = emitEndCf ( MI ) ; break ; default : assert ( false && <STR_LIT> ) ; break ; } MachineBasicBlock :: iterator Next ; for ( I = Prev ? Prev -> getIterator ( ) : MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MaskMI = * I ; switch ( MaskMI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : combineMasks ( MaskMI ) ; break ; default :


<s> if ( MDT -> dominates ( & MBB , Succ ) ) MDT -> changeImmediateDominator ( MDT -> getNode ( Succ ) , MDT -> getNode ( & MBB ) -> getIDom ( ) ) ; MDT -> eraseNode ( & MBB ) ; } MBB . clear ( ) ; MBB . eraseFromParent ( ) ; if ( FallThrough && ! FallThrough -> isLayoutSuccessor ( Succ ) ) { if ( ! GetFallThroughSucc ( Succ ) ) { MachineFunction * MF = FallThrough -> getParent ( ) ; MachineFunction :: iterator FallThroughPos ( FallThrough ) ; MF -> splice ( std :: next ( FallThroughPos ) , Succ ) ; } else BuildMI ( * FallThrough , FallThrough -> end ( ) , FallThrough -> findBranchDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( Succ ) ; } return true ;


<s> XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; MachineBasicBlock * SplitMBB = MBB ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ;


<s> MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; ImpDefSCC . setIsDead ( IsDead ) ;


<s> static void setImpSCCDefDead ( MachineInstr & MI , bool IsDead ) { MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ;


<s> <ISA_LIT>LowerControlFlow ( ) : MachineFunctionPass ( ID ) {


<s> SmallSet < const MachineBasicBlock * , <NUM_LIT> > Visited ; MachineBasicBlock * B = & MBB ; do { if ( ! Visited . insert ( B ) . second ) return MBB . end ( ) ; auto E = B -> end ( ) ; for ( ; It != E ; ++ It ) { if ( TII -> mayReadEXEC ( * MRI , * It ) ) break ; } if ( It != E ) return It ; if ( B -> succ_size ( ) != <NUM_LIT> ) return MBB . end ( ) ; MachineBasicBlock * Succ = * B -> succ_begin ( ) ; It = Succ -> begin ( ) ;


<s> for ( ; It != E ; ++ It ) { if ( TII -> mayReadEXEC ( * MRI , * It ) ) break ; } if ( It != E ) return It ; if ( B -> succ_size ( ) != <NUM_LIT> ) return MBB . end ( ) ;

MachineBasicBlock * Succ = * B -> succ_begin ( ) ;


<s> assert ( I -> isTerminator ( ) ) ; MachineBasicBlock :: iterator End = MBB . end ( ) ; while ( I != End && ! I -> isUnconditionalBranch ( ) ) ++ I ;


<s> MachineBasicBlock :: iterator skipToUncondBrOrEnd ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) const { assert ( I -> isTerminator ( ) ) ; MachineBasicBlock :: iterator End = MBB . end ( ) ; while ( I != End && ! I -> isUnconditionalBranch ( ) ) ++ I ; return I ;


<s> bool SkipAnding = false ; if ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) { if ( MachineInstr * Def = MRI -> getUniqueVRegDef ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { SkipAnding = Def -> getParent ( ) == MI . getParent ( ) && <ISA_LIT>InstrInfo :: isVALU ( * Def ) ; } } MachineInstr * And = nullptr , * Or = nullptr ; if ( ! SkipAnding ) { And = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;

} else Or = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ; for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) != <STR_LIT> ) ) return ; for ( const auto & SrcOp : Def -> explicit_operands ( ) ) if ( SrcOp . isReg ( ) && SrcOp . isUse ( ) && ( TargetRegisterInfo :: isVirtualRegister ( SrcOp . getReg ( ) ) || SrcOp . getReg ( ) == <STR_LIT> ) ) Src . push_back ( SrcOp ) ;


<s> return ; } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ;

for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) != <STR_LIT> ) ) return ;


<s> MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next , Last ; for ( I = MBB . begin ( ) , Last = MBB . end ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : emitIf ( MI ) ;


<s> MachineInstr * Or = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * Or ) ;


<s> void <ISA_LIT>LowerControlFlow :: emitElse ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) == <STR_LIT> ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; unsigned CopyReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; unsigned SaveReg = ExecModified ? MRI -> createVirtualRegister ( & <STR_LIT> ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( <STR_LIT> ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ;


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ;

if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ;


<s> assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; unsigned CopyReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> , RegState :: ImplicitDefine ) ; unsigned Tmp = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Tmp ) . addReg ( CopyReg ) . addReg ( Cond . getReg ( ) ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> MachineInstr * AndN2 = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; MachineInstr * Branch = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) { LIS -> ReplaceMachineInstrInMaps ( MI , * AndN2 ) ;

LIS -> InsertMachineInstrInMaps ( * Branch ) ;


<s> MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) , LIS ( nullptr ) , MRI ( nullptr ) {


MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) , LIS ( nullptr ) , MRI ( nullptr ) {


<s> Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; }

LIS -> RemoveMachineInstrFromMaps ( MI ) ;


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) == <STR_LIT> ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register CopyReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ;

MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ;


<s> void <ISA_LIT>LowerControlFlow :: emitEndCf ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ; MI . eraseFromParent ( ) ;

if ( LIS ) LIS -> handleMove ( * NewMI ) ;


<s> Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> if ( Ops . size ( ) != <NUM_LIT> ) return ; unsigned UniqueOpndIdx ; if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else return ; Register Reg = MI . getOperand ( OpToReplace ) . getReg ( ) ; MI . removeOperand ( OpToReplace ) ; MI . addOperand ( Ops [ UniqueOpndIdx ] ) ;

if ( MRI -> use_empty ( Reg ) ) MRI -> getUniqueVRegDef ( Reg ) -> eraseFromParent ( ) ;


<s> LV = getAnalysisIfAvailable < LiveVariables > ( ) ; MDT = getAnalysisIfAvailable < MachineDominatorTree > ( ) ; MRI = & MF . getRegInfo ( ) ; BoolRC = TRI -> getBoolRC ( ) ; if ( ST . isWave32 ( ) ) { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } }


<s> NeedBlockSplit = true ; break ; } } unsigned Opcode = OrOpc ; MachineBasicBlock * SplitBB = & MBB ; if ( NeedBlockSplit ) { SplitBB = MBB . splitAt ( MI , true , LIS ) ; if ( MDT && SplitBB != & MBB ) { MachineDomTreeNode * MBBNode = ( * MDT ) [ & MBB ] ; SmallVector < MachineDomTreeNode * > Children ( MBBNode -> begin ( ) , MBBNode -> end ( ) ) ; MachineDomTreeNode * SplitBBNode = MDT -> addNewBlock ( SplitBB , & MBB ) ; for ( MachineDomTreeNode * Child : Children ) MDT -> changeImmediateDominator ( Child , SplitBBNode ) ; } Opcode = OrTermrOpc ; InsPt = MI ; } MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( Opcode ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LV ) { LV -> replaceKillInstruction ( DataReg , MI , * NewMI ) ; if ( SplitBB != & MBB ) { DenseSet < Register > SplitDefs ; for ( MachineInstr & X : * SplitBB ) { for ( MachineOperand & Op : X . operands ( ) ) { if ( Op . isReg ( ) && Op . isDef ( ) && Op . getReg ( ) . isVirtual ( ) ) SplitDefs . insert ( Op . getReg ( ) ) ; } } for ( unsigned i = <NUM_LIT> , e = MRI -> getNumVirtRegs ( ) ; i != e ; ++ i ) { Register Reg = Register :: index2VirtReg ( i ) ; LiveVariables :: VarInfo & VI = LV -> getVarInfo ( Reg ) ; if ( VI . AliveBlocks . test ( MBB . getNumber ( ) ) ) VI . AliveBlocks . set ( SplitBB -> getNumber ( ) ) ; else { for ( MachineInstr * Kill : VI . Kills ) { if ( Kill -> getParent ( ) == SplitBB && ! SplitDefs . contains ( Reg ) ) VI . AliveBlocks . set ( MBB . getNumber ( ) ) ; } } }


<s> } MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( Opcode ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; LoweredEndCf . insert ( NewMI ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ; MI . eraseFromParent ( ) ; if ( LIS ) LIS -> handleMove ( * NewMI ) ;


<s> unsigned Opcode = OrOpc ; MachineBasicBlock * SplitBB = & MBB ; if ( NeedBlockSplit ) { SplitBB = MBB . splitAt ( MI , true , LIS ) ; Opcode = OrTermrOpc ; InsPt = MI ; } MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( Opcode ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> if ( ! I . isDebugInstr ( ) && ! I . isUnconditionalBranch ( ) ) return false ; } assert ( MBB . succ_size ( ) == <NUM_LIT> && <STR_LIT> ) ; MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; MachineBasicBlock * FallThrough = nullptr ; while ( ! MBB . predecessors ( ) . empty ( ) ) { MachineBasicBlock * P = * MBB . pred_begin ( ) ; if ( GetFallThroughSucc ( P ) == & MBB ) FallThrough = P ; P -> ReplaceUsesOfBlockWith ( & MBB , Succ ) ; } MBB . removeSuccessor ( Succ ) ; if ( LIS ) { for ( auto & I : MBB . instrs ( ) ) LIS -> RemoveMachineInstrFromMaps ( I ) ; } MBB . clear ( ) ; MBB . eraseFromParent ( ) ; if ( FallThrough && ! FallThrough -> isLayoutSuccessor ( Succ ) ) { if ( ! GetFallThroughSucc ( Succ ) ) { MachineFunction * MF = FallThrough -> getParent ( ) ;

MachineFunction :: iterator FallThroughPos ( FallThrough ) ;


<s> OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ;


<s> unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ; if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; } BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( Dst ) ; MI . eraseFromParent ( ) ;


<s> } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) { if ( skipIfDead ( MI , * NextBB ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : { assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( * BI , I , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ;


<s> MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) , SkipThreshold ( <NUM_LIT> ) {


<s> Register CopyReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) {

MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ;


<s> MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ;

LIS -> InsertMachineInstrInMaps ( * Xor ) ;


<s> MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI , TII ) ; Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ;


<s> MachineBasicBlock * MBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ;

if ( MBB == MI . getParent ( ) -> getNextNode ( ) ) MI . eraseFromParent ( ) ;


<s> MachineBasicBlock * MBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( MBB == MI . getParent ( ) -> getNextNode ( ) ) MI . eraseFromParent ( ) ;


<s> if ( Offset >= NumElts ) return std :: make_pair ( RC -> getRegister ( BaseRegIdx ) , Offset ) ; int RegIdx = BaseRegIdx + Offset ; if ( RegIdx < <NUM_LIT> ) { Offset = RegIdx ; RegIdx = <NUM_LIT> ; } else { Offset = <NUM_LIT> ; } unsigned Reg = RC -> getRegister ( RegIdx ) ; return std :: make_pair ( Reg , Offset ) ;


<s> void <ISA_LIT>LowerControlFlow :: Else ( MachineInstr & MI , bool ExecModified ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ; if ( ExecModified ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; } BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ;

BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( Dst ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ; if ( ExecModified ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; } BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( Dst ) ;


<s> BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addImm ( Offset ) ; } LoopBB . insert ( I , MovRel ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ;

BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & LoopBB ) ;


<s> BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; if ( Offset != <NUM_LIT> ) {

BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addImm ( Offset ) ;


<s> DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( Reg ) ;


<s> unsigned Reg ; const MachineOperand * Val = TII -> getNamedOperand ( MI , <STR_LIT> :: val ) ; std :: tie ( Reg , Offset ) = computeIndirectRegAndOffset ( Dst , Offset ) ; MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; if ( Idx -> getReg ( ) == <STR_LIT> ) {

BuildMI ( MBB , MI . getIterator ( ) , DL , TII -> get ( <STR_LIT> ) , Reg ) . addOperand ( * Val ) ;


<s> BuildMI ( MBB , MI . getIterator ( ) , DL , TII -> get ( <STR_LIT> ) , Reg ) . addOperand ( * Val ) ; MI . eraseFromParent ( ) ; return false ; }

MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Val -> getReg ( ) , getUndefRegState ( Val -> isUndef ( ) ) ) . addReg ( Dst , RegState :: Implicit ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand * SrcVec = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; unsigned Reg ;

std :: tie ( Reg , Offset ) = computeIndirectRegAndOffset ( SrcVec -> getReg ( ) , Offset ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand * SrcVec = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; unsigned Reg ; std :: tie ( Reg , Offset ) = computeIndirectRegAndOffset ( SrcVec -> getReg ( ) , Offset ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; if ( Idx -> getReg ( ) == <STR_LIT> ) { BuildMI ( MBB , MI . getIterator ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Reg , getUndefRegState ( SrcVec -> isUndef ( ) ) ) ; MI . eraseFromParent ( ) ; return false ; } MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Reg , getUndefRegState ( SrcVec -> isUndef ( ) ) ) . addReg ( SrcVec -> getReg ( ) , RegState :: Implicit ) ; return loadM0 ( MI , MovRel , Offset ) ;


<s> MachineBasicBlock * <ISA_LIT>LowerControlFlow :: insertSkipBlock ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) const { MachineFunction * MF = MBB . getParent ( ) ; MachineBasicBlock * SkipBB = MF -> CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF -> insert ( MBBI , SkipBB ) ; return SkipBB ;


<s> CallingConv :: ID CallConv = MBB . getParent ( ) -> getFunction ( ) -> getCallingConv ( ) ; assert ( CallConv == CallingConv :: <ISA_LIT>_PS || CallConv == CallingConv :: <ISA_LIT>_GS ) ; if ( ( Op . isImm ( ) ) ) {