<s> SmallVector < Type * , <NUM_LIT> > Types ; bool NeedToReplace = false ; Function * F = I . getCalledFunction ( ) ; if ( ! F ) return ; std :: string Name = F -> getName ( ) ; for ( unsigned i = <NUM_LIT> , e = I . getNumArgOperands ( ) ; i != e ; ++ i ) { Value * Arg = I . getArgOperand ( i ) ; if ( Arg -> getType ( ) == v16i8 ) { Args . push_back ( Builder . CreateBitCast ( Arg , v4i32 ) ) ; Types . push_back ( v4i32 ) ; NeedToReplace = true ; Name = Name + <STR_LIT> ; } else if ( Arg -> getType ( ) -> isVectorTy ( ) && Arg -> getType ( ) -> getVectorNumElements ( ) == <NUM_LIT> && Arg -> getType ( ) -> getVectorElementType ( ) == Type :: getInt32Ty ( I . getContext ( ) ) ) { Type * ElementTy = Arg -> getType ( ) -> getVectorElementType ( ) ; std :: string TypeName = <STR_LIT> ; InsertElementInst * Def = cast < InsertElementInst > ( Arg ) ; Args . push_back ( Def -> getOperand ( <NUM_LIT> ) ) ; Types . push_back ( ElementTy ) ; std :: string VecTypeName = <STR_LIT> + TypeName ; Name = Name . replace ( Name . find ( VecTypeName ) , VecTypeName . length ( ) , TypeName ) ; NeedToReplace = true ; } else { Args . push_back ( Arg ) ;

Types . push_back ( Arg -> getType ( ) ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> if ( A . isStringAttribute ( ) ) { StringRef Str = A . getValueAsString ( ) ; Str . getAsInteger ( <NUM_LIT> , ShaderType ) ; } if ( ShaderType == ShaderType :: COMPUTE ) return false ;


<s> Attribute A = F . getFnAttribute ( <STR_LIT> ) ; unsigned ShaderType = ShaderType :: COMPUTE ; if ( A . isStringAttribute ( ) ) { StringRef Str = A . getValueAsString ( ) ; Str . getAsInteger ( <NUM_LIT> , ShaderType ) ; }

if ( ShaderType == ShaderType :: COMPUTE ) return false ;


<s> const char GlobalFlags = analyzeFunction ( MF ) ; const bool NeedsLiveMask = ! ( KillInstrs . empty ( ) && LiveMaskQueries . empty ( ) ) ; LiveMaskReg = Exec ; if ( ! ( GlobalFlags & ( StateWQM | StateStrict ) ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) && KillInstrs . empty ( ) ) { lowerLiveMaskQueries ( ) ; return ! LiveMaskQueries . empty ( ) ; } MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( NeedsLiveMask || ( GlobalFlags & StateWQM ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } LLVM_DEBUG ( printInfo ( ) ) ; lowerLiveMaskQueries ( ) ;


<s> auto III = Instructions . find ( & MI ) ; if ( III == Instructions . end ( ) ) continue ; dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << PrintState ( III -> second . Needs ) << <STR_LIT> << PrintState ( III -> second . OutNeeds ) << '\n' ; }


<s> for ( const MachineInstr & MI : * BII . first ) { auto III = Instructions . find ( & MI ) ; if ( III == Instructions . end ( ) ) continue ; dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << PrintState ( III -> second . Needs ) << <STR_LIT> << PrintState ( III -> second . OutNeeds ) << '\n' ; }


<s> if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; SavedNonWWMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , Before , SavedWQMReg ) ; if ( SavedWQMReg ) { LIS -> createAndComputeVirtRegInterval ( SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = StateWQM ; } else {


<s> auto III = Instructions . find ( & MI ) ; if ( III != Instructions . end ( ) ) { if ( III -> second . Needs & StateWWM ) Needs = StateWWM ; else if ( III -> second . Needs & StateWQM ) Needs = StateWQM ; else Needs &= ~ III -> second . Disabled ; OutNeeds = III -> second . OutNeeds ; } } else { Needs = StateExact | StateWQM | StateWWM ; } if ( MI . isTerminator ( ) && OutNeeds == StateExact ) Needs = StateExact ; if ( MI . getOpcode ( ) == <STR_LIT> && BI . OutNeeds == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; ++ Next ; } else { if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; SavedNonWWMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , Before , SavedWQMReg ) ; if ( SavedWQMReg ) { LIS -> createAndComputeVirtRegInterval ( SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = StateWQM ;


<s> Instructions [ & MI ] . Needs = StateWQM ; II . Needs = StateWQM ; } if ( II . Needs & StateWQM ) { BI . Needs |= StateWQM ; if ( ! ( BI . InNeeds & StateWQM ) ) { BI . InNeeds |= StateWQM ; Worklist . push_back ( MBB ) ; } } if ( MachineInstr * PrevMI = MI . getPrevNode ( ) ) {

char InNeeds = ( II . Needs & ~ StateWWM ) | II . OutNeeds ;


<s> LiveMaskQueries . clear ( ) ; LowerToCopyInstrs . clear ( ) ; CallingConv = MF . getFunction ( ) -> getCallingConv ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerCopyInstrs ( ) ; return true ;


<s> MachineBasicBlock & MBB = * * BI ; BlockInfo & BBI = Blocks [ & MBB ] ; for ( auto II = MBB . begin ( ) , IE = MBB . end ( ) ; II != IE ; ++ II ) { MachineInstr & MI = * II ; InstrInfo & III = Instructions [ & MI ] ; unsigned Opcode = MI . getOpcode ( ) ; char Flags = <NUM_LIT> ; if ( TII -> isDS ( Opcode ) && CallingConv == CallingConv :: <ISA_LIT>_PS ) { Flags = StateWQM ; } else if ( TII -> isWQM ( Opcode ) ) { markInstructionUses ( MI , StateWQM , Worklist ) ; GlobalFlags |= StateWQM ; continue ; } else if ( Opcode == <STR_LIT> ) { Flags = StateWQM ; LowerToCopyInstrs . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { markInstructionUses ( MI , StateWWM , Worklist ) ; GlobalFlags |= StateWWM ; LowerToCopyInstrs . push_back ( & MI ) ; continue ; } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { III . Disabled = StateWWM ; MachineOperand & Inactive = MI . getOperand ( <NUM_LIT> ) ; if ( Inactive . isReg ( ) ) { if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { unsigned Reg = Inactive . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) {

for ( MachineInstr & DefMI : MRI -> def_instructions ( Reg ) ) markInstruction ( DefMI , StateWWM , Worklist ) ;


<s> char <ISA_LIT>WholeQuadMode :: scanInstructions ( MachineFunction & MF , std :: vector < WorkItem > & Worklist ) { char GlobalFlags = <NUM_LIT> ; bool WQMOutputs = MF . getFunction ( ) -> hasFnAttribute ( <STR_LIT> ) ; SmallVector < MachineInstr * , <NUM_LIT> > SetInactiveInstrs ; ReversePostOrderTraversal < MachineFunction * > RPOT ( & MF ) ; for ( auto BI = RPOT . begin ( ) , BE = RPOT . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * * BI ; BlockInfo & BBI = Blocks [ & MBB ] ; for ( auto II = MBB . begin ( ) , IE = MBB . end ( ) ; II != IE ; ++ II ) { MachineInstr & MI = * II ; InstrInfo & III = Instructions [ & MI ] ; unsigned Opcode = MI . getOpcode ( ) ; char Flags = <NUM_LIT> ; if ( TII -> isDS ( Opcode ) && CallingConv == CallingConv :: <ISA_LIT>_PS ) { Flags = StateWQM ; } else if ( TII -> isWQM ( Opcode ) ) { markInstructionUses ( MI , StateWQM , Worklist ) ; GlobalFlags |= StateWQM ; continue ;


<s> TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ;

if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) {


<s> MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerCopyInstrs ( ) ;


<s> void <ISA_LIT>WholeQuadMode :: fromWWM ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SavedOrig ) { MachineInstr * MI ; assert ( SavedOrig ) ;

MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) . addReg ( SavedOrig ) ;


<s> AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . setPreservesCFG ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LiveIntervals > ( ) ; AU . addPreserved < SlotIndexes > ( ) ;

AU . addPreserved < LiveIntervals > ( ) ;


<s> for ( MachineInstr * MI : LiveMaskQueries ) { const DebugLoc & DL = MI -> getDebugLoc ( ) ; Register Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Copy = BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; LIS -> ReplaceMachineInstrInMaps ( * MI , * Copy ) ; MI -> eraseFromParent ( ) ; }


<s> Register Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Copy = BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; LIS -> ReplaceMachineInstrInMaps ( * MI , * Copy ) ; MI -> eraseFromParent ( ) ; }


<s> if ( Reg == <STR_LIT> || Reg == <STR_LIT> ) continue ; for ( MCRegUnitIterator RegUnit ( Reg , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit ) { LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag , Worklist ) ; } continue ; } for ( MachineInstr & DefMI : MRI -> def_instructions ( Use . getReg ( ) ) ) markInstruction ( DefMI , Flag , Worklist ) ; }


<s> if ( III -> second . Needs & StateWWM ) Needs = StateWWM ; else if ( III -> second . Needs & StateWQM ) Needs = StateWQM ; else Needs &= ~ III -> second . Disabled ; OutNeeds = III -> second . OutNeeds ; } } else { Needs = StateExact | StateWQM | StateWWM ; } if ( MI . isTerminator ( ) && OutNeeds == StateExact ) Needs = StateExact ; if ( MI . getOpcode ( ) == <STR_LIT> && BI . OutNeeds == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; ++ Next ; } else { if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; SavedNonWWMReg = MRI -> createVirtualRegister ( BoolRC ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( BoolRC ) ; toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ;


<s> LowerToCopyInstrs . clear ( ) ; CallingConv = MF . getFunction ( ) . getCallingConv ( ) ; ST = & MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ;

if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ;


<s> LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> } else { unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ;

MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) . addReg ( LiveMaskReg ) ;


<s> MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , Exec ) . addReg ( SavedWQM ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; }

LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; }


<s> MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveOrig ) . addImm ( - <NUM_LIT> ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> void <ISA_LIT>WholeQuadMode :: lowerCopyInstrs ( ) { for ( MachineInstr * MI : LowerToMovInstrs ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; const Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const unsigned SubReg = MI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( TRI -> isVGPR ( * MRI , Reg ) ) { const TargetRegisterClass * regClass = Reg . isVirtual ( ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; if ( SubReg ) regClass = TRI -> getSubRegClass ( regClass , SubReg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; assert ( any_of ( MI -> implicit_operands ( ) , [ ] ( const MachineOperand & MO ) { return MO . isUse ( ) && MO . getReg ( ) == <STR_LIT> ; } ) ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; if ( MI -> getOperand ( <NUM_LIT> ) . isEarlyClobber ( ) ) { LIS -> removeInterval ( Reg ) ; MI -> getOperand ( <NUM_LIT> ) . setIsEarlyClobber ( false ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } int Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; while ( Index >= <NUM_LIT> ) { MI -> removeOperand ( Index ) ; Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> removeOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } else {


<s> Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } MachineInstr * VcmpMI ; const MachineOperand & Op0 = MI . getOperand ( <NUM_LIT> ) ; const MachineOperand & Op1 = MI . getOperand ( <NUM_LIT> ) ; Register VCC = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( TRI -> isVGPR ( * MRI , Op0 . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; VcmpMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( Op1 ) . add ( Op0 ) ; } else { VcmpMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . addReg ( VCC , RegState :: Define ) . addImm ( <NUM_LIT> ) . add ( Op1 ) . addImm ( <NUM_LIT> ) . add ( Op0 ) . addImm ( <NUM_LIT> ) ; }

MachineInstr * MaskUpdateMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , LiveMaskReg ) . addReg ( LiveMaskReg ) . addReg ( VCC ) ;


<s> XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; AndN2Opc = <STR_LIT> ; XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const char GlobalFlags = analyzeFunction ( MF ) ; const bool NeedsLiveMask = ! ( KillInstrs . empty ( ) && LiveMaskQueries . empty ( ) ) ; LiveMaskReg = Exec ; if ( ! ( GlobalFlags & ( StateWQM | StateStrict ) ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) && KillInstrs . empty ( ) ) { lowerLiveMaskQueries ( ) ; return ! LiveMaskQueries . empty ( ) ; } MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( NeedsLiveMask || ( GlobalFlags & StateWQM ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } LLVM_DEBUG ( printInfo ( ) ) ; lowerLiveMaskQueries ( ) ; lowerCopyInstrs ( ) ; if ( GlobalFlags == StateWQM ) { auto MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( WQMOpc ) , Exec ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> LowerToMovInstrs . clear ( ) ; KillInstrs . clear ( ) ; StateTransition . clear ( ) ; ST = & MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; if ( ST -> isWave32 ( ) ) { AndOpc = <STR_LIT> ; AndN2Opc = <STR_LIT> ; XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; AndN2Opc = <STR_LIT> ; XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const char GlobalFlags = analyzeFunction ( MF ) ; const bool NeedsLiveMask = ! ( KillInstrs . empty ( ) && LiveMaskQueries . empty ( ) ) ; LiveMaskReg = Exec ; if ( ! ( GlobalFlags & ( StateWQM | StateStrict ) ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) && KillInstrs . empty ( ) ) { lowerLiveMaskQueries ( ) ; return ! LiveMaskQueries . empty ( ) ;


<s> } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { III . Disabled = StateStrict ; MachineOperand & Inactive = MI . getOperand ( <NUM_LIT> ) ; if ( Inactive . isReg ( ) ) { if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { markOperand ( MI , Inactive , StateStrictWWM , Worklist ) ; } } SetInactiveInstrs . push_back ( & MI ) ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateStrict ; continue ; } else { if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { KillInstrs . push_back ( & MI ) ; BBI . NeedsLowering = true ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( ! Reg . isVirtual ( ) && TRI -> hasVectorRegisters ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } markInstruction ( MI , Flags , Worklist ) ;


<s> if ( TRI -> isVGPR ( * MRI , Reg ) ) { const TargetRegisterClass * regClass = TargetRegisterInfo :: isVirtualRegister ( Reg ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; MI -> addOperand ( MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ;


<s> if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag , Worklist ) ; } continue ; }

for ( MachineInstr & DefMI : MRI -> def_instructions ( Use . getReg ( ) ) ) markInstruction ( DefMI , Flag , Worklist ) ;


<s> TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; lowerCopyInstrs ( ) ; return true ; } } LLVM_DEBUG ( printInfo ( ) ) ; lowerCopyInstrs ( ) ; for ( auto BII : Blocks ) processBlock ( * BII . first , LiveMaskReg , BII . first == & * MF . begin ( ) ) ; LIS -> removeRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ;


<s> assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; const Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const unsigned SubReg = MI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( TRI -> isVGPR ( * MRI , Reg ) ) { const TargetRegisterClass * regClass = Reg . isVirtual ( ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; if ( SubReg ) regClass = TRI -> getSubRegClass ( regClass , SubReg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; MI -> addOperand ( MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; } else if ( ! MRI -> isSSA ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; if ( MI -> getOperand ( <NUM_LIT> ) . isEarlyClobber ( ) ) { LIS -> removeInterval ( Reg ) ; MI -> getOperand ( <NUM_LIT> ) . setIsEarlyClobber ( false ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } int Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; while ( Index >= <NUM_LIT> ) { MI -> RemoveOperand ( Index ) ; Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> RemoveOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } else { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ;


<s> do { const VNInfo * Value = ToProcess . pop_back_val ( ) ; Visited . insert ( Value ) ; if ( Value -> isPHIDef ( ) ) { const MachineBasicBlock * MBB = LIS -> getMBBFromIndex ( Value -> def ) ; assert ( MBB && <STR_LIT> ) ; for ( MachineBasicBlock :: const_pred_iterator PI = MBB -> pred_begin ( ) , PE = MBB -> pred_end ( ) ; PI != PE ; ++ PI ) { if ( const VNInfo * VN = LR . getVNInfoBefore ( LIS -> getMBBEndIdx ( * PI ) ) ) { if ( ! Visited . count ( VN ) ) ToProcess . push_back ( VN ) ; } } } else { MachineInstr * MI = LIS -> getInstructionFromIndex ( Value -> def ) ; assert ( MI && <STR_LIT> ) ; markInstruction ( * MI , Flag , Worklist ) ; for ( const MachineOperand & Op : MI -> operands ( ) ) {

if ( ! ( Op . isReg ( ) && Op . getReg ( ) == Reg ) ) continue ;


<s> void <ISA_LIT>WholeQuadMode :: markInstruction ( MachineInstr & MI , char Flag , std :: vector < WorkItem > & Worklist ) { InstrInfo & II = Instructions [ & MI ] ; assert ( ! ( Flag & StateExact ) && Flag != <NUM_LIT> ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintState ( Flag ) << <STR_LIT> << MI ) ;


<s> for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; Register Reg = Use . getReg ( ) ; if ( ! Reg . isVirtual ( ) ) { if ( Reg == <STR_LIT> || Reg == <STR_LIT> ) continue ; for ( MCRegUnitIterator RegUnit ( Reg . asMCReg ( ) , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit ) { LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( MRI -> isSSA ( ) ) { if ( Value -> isPHIDef ( ) ) continue ;

markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag , Worklist ) ;


<s> else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; LIS -> createAndComputeVirtRegInterval ( SavedNonWWMReg ) ; SavedNonWWMReg = <NUM_LIT> ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; assert ( ! SavedNonWWMReg ) ; SavedNonWWMReg = MRI -> createVirtualRegister ( BoolRC ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) { assert ( ! SavedWQMReg ) ; SavedWQMReg = MRI -> createVirtualRegister ( BoolRC ) ; } toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , Before , SavedWQMReg ) ; if ( SavedWQMReg ) { LIS -> createAndComputeVirtRegInterval ( SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = StateWQM ; } else { assert ( Needs & State ) ; } } } if ( Needs != ( StateExact | StateWQM | StateWWM ) ) { if ( Needs != ( StateExact | StateWQM ) ) FirstWQM = IE ; FirstWWM = IE ; } if ( II == IE ) break ;


<s> if ( ST -> isWave32 ( ) ) { AndOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { auto MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; lowerCopyInstrs ( ) ; return true ; }


<s> if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { Register Reg = Inactive . getReg ( ) ; if ( Reg . isVirtual ( ) ) { for ( MachineInstr & DefMI : MRI -> def_instructions ( Reg ) ) markInstruction ( DefMI , StateWWM , Worklist ) ; } } } SetInactiveInstrs . push_back ( & MI ) ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( & MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateWWM ; continue ; } else { if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( ! Reg . isVirtual ( ) && TRI -> hasVectorRegisters ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } markInstruction ( MI , Flags , Worklist ) ; GlobalFlags |= Flags ;


<s> void <ISA_LIT>WholeQuadMode :: fromStrictMode ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , Register SavedOrig , char NonStrictState , char CurrentStrictState ) { MachineInstr * MI ; assert ( SavedOrig ) ; assert ( CurrentStrictState == StateStrictWWM || CurrentStrictState == StateStrictWQM ) ;

if ( CurrentStrictState == StateStrictWWM ) {


<s> AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addPreserved < MachinePostDominatorTree > ( ) ;


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addPreserved < MachinePostDominatorTree > ( ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( MBB ) << <STR_LIT> ) ; SmallVector < MachineInstr * , <NUM_LIT> > SplitPoints ; char State = BI . InitialState ; auto II = MBB . getFirstNonPHI ( ) , IE = MBB . end ( ) ; while ( II != IE ) { auto Next = std :: next ( II ) ; MachineInstr & MI = * II ; if ( StateTransition . count ( & MI ) ) State = StateTransition [ & MI ] ; MachineInstr * SplitPoint = nullptr ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : SplitPoint = lowerKillI1 ( MBB , MI , State == StateWQM ) ; break ; case <STR_LIT> :

SplitPoint = lowerKillF32 ( MBB , MI ) ;


<s> return MO . isUse ( ) && MO . getReg ( ) == <STR_LIT> ; } ) ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; if ( MI -> getOperand ( <NUM_LIT> ) . isEarlyClobber ( ) ) { LIS -> removeInterval ( Reg ) ; MI -> getOperand ( <NUM_LIT> ) . setIsEarlyClobber ( false ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } int Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; while ( Index >= <NUM_LIT> ) { MI -> RemoveOperand ( Index ) ; Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) {


<s> } int Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; while ( Index >= <NUM_LIT> ) { MI -> RemoveOperand ( Index ) ; Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> RemoveOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ;


<s> if ( TRI -> isVGPR ( * MRI , Op0 . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; VcmpMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( Op1 ) . add ( Op0 ) ; } else { VcmpMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . addReg ( <STR_LIT> , RegState :: Define ) . addImm ( <NUM_LIT> ) . add ( Op1 ) . addImm ( <NUM_LIT> ) . add ( Op0 ) . addImm ( <NUM_LIT> ) ; } Register VCC = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; MachineInstr * MaskUpdateMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , LiveMaskReg ) . addReg ( LiveMaskReg ) . addReg ( VCC ) ; MachineInstr * EarlyTermMI = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; MachineInstr * ExecMaskMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , Exec ) . addReg ( Exec ) . addReg ( VCC ) ; assert ( MBB . succ_size ( ) == <NUM_LIT> ) ; MachineInstr * NewTerm = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( * MBB . succ_begin ( ) ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * VcmpMI ) ;


<s> MBB . remove ( & MI ) ; return NewTerm ; } } else { if ( ! KillVal ) { TmpReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; ComputeKilledMaskMI = BuildMI ( MBB , MI , DL , TII -> get ( XorOpc ) , TmpReg ) . add ( Op ) . addReg ( Exec ) ; MaskUpdateMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , LiveMaskReg ) . addReg ( LiveMaskReg ) . addReg ( TmpReg ) ; } else { MaskUpdateMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , LiveMaskReg ) . addReg ( LiveMaskReg ) . add ( Op ) ; } } MachineInstr * EarlyTermMI = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; MachineInstr * NewTerm ; MachineInstr * WQMMaskMI = nullptr ; Register LiveMaskWQM ; if ( IsDemote ) { LiveMaskWQM = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; WQMMaskMI = BuildMI ( MBB , MI , DL , TII -> get ( WQMOpc ) , LiveMaskWQM ) . addReg ( LiveMaskReg ) ; NewTerm = BuildMI ( MBB , MI , DL , TII -> get ( AndOpc ) , Exec ) . addReg ( Exec ) . addReg ( LiveMaskWQM ) ; } else { if ( Op . isImm ( ) ) { unsigned MovOpc = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; NewTerm = BuildMI ( MBB , & MI , DL , TII -> get ( MovOpc ) , Exec ) . addImm ( <NUM_LIT> ) ; } else if ( ! IsWQM ) { NewTerm = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , Exec ) . addReg ( Exec ) . addReg ( LiveMaskReg ) ; } else { unsigned Opcode = KillVal ? AndN2Opc : AndOpc ; NewTerm = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) , Exec ) . addReg ( Exec ) . add ( Op ) ; } } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MBB . remove ( & MI ) ; assert ( EarlyTermMI ) ; assert ( MaskUpdateMI ) ; assert ( NewTerm ) ; if ( ComputeKilledMaskMI ) LIS -> InsertMachineInstrInMaps ( * ComputeKilledMaskMI ) ;

LIS -> InsertMachineInstrInMaps ( * MaskUpdateMI ) ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * SplitPoint = nullptr ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> :

SplitPoint = lowerKillI1 ( * MBB , * MI , IsWQM ) ;


<s> MachineInstr * Copy = BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; LIS -> ReplaceMachineInstrInMaps ( * MI , * Copy ) ; MI -> eraseFromParent ( ) ;


<s> MachineInstr * Copy = BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; LIS -> ReplaceMachineInstrInMaps ( * MI , * Copy ) ; MI -> eraseFromParent ( ) ; }


<s> if ( const VNInfo * VN = LR . getVNInfoBefore ( LIS -> getMBBEndIdx ( * PI ) ) ) { if ( ! Visited . count ( VisitKey ( VN , DefinedLanes ) ) ) NextValue = VN ; } } if ( PI != PE ) PhiStack . emplace_back ( Value , Idx , DefinedLanes ) ; } else { MachineInstr * MI = LIS -> getInstructionFromIndex ( Value -> def ) ; assert ( MI && <STR_LIT> ) ; if ( Reg . isVirtual ( ) ) { bool HasDef = false ; for ( const MachineOperand & Op : MI -> operands ( ) ) { if ( ! ( Op . isReg ( ) && Op . isDef ( ) && Op . getReg ( ) == Reg ) ) continue ; LaneBitmask OpLanes = Op . isUndef ( ) ? LaneBitmask :: getAll ( ) : TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) ; LaneBitmask Overlap = ( UseLanes & OpLanes ) ; HasDef |= Overlap . any ( ) ; DefinedLanes |= OpLanes ; } if ( ( DefinedLanes & UseLanes ) != UseLanes ) { LiveQueryResult LRQ = LR . Query ( LIS -> getInstructionIndex ( * MI ) ) ; if ( const VNInfo * VN = LRQ . valueIn ( ) ) { if ( ! Visited . count ( VisitKey ( VN , DefinedLanes ) ) ) NextValue = VN ; } } if ( HasDef ) markInstruction ( * MI , Flag , Worklist ) ; } else { markInstruction ( * MI , Flag , Worklist ) ; } }

if ( ! NextValue && ! PhiStack . empty ( ) ) {


<s> assert ( ! ( Flag & StateExact ) && Flag != <NUM_LIT> ) ; Flag &= ~ II . Disabled ; if ( ( II . Needs & Flag ) == Flag ) return ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintState ( Flag ) << <STR_LIT> << MI ) ;


<s> if ( ( II . Needs & Flag ) == Flag ) return ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintState ( Flag ) << <STR_LIT> << MI ) ; II . Needs |= Flag ;


<s> for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; markOperand ( MI , Use , Flag , Worklist ) ; }


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintState ( Flag ) << <STR_LIT> << Op << <STR_LIT> << MI ) ; if ( Reg . isVirtual ( ) ) { LiveRange & LR = LIS -> getInterval ( Reg ) ; markDefs ( MI , LR , Reg , Op . getSubReg ( ) , Flag , Worklist ) ; } else { for ( MCRegUnitIterator RegUnit ( Reg . asMCReg ( ) , TRI ) ; RegUnit . isValid ( ) ;


<s> PhiEntry ( const VNInfo * Phi , unsigned PredIdx , LaneBitmask DefinedLanes ) : Phi ( Phi ) , PredIdx ( PredIdx ) , DefinedLanes ( DefinedLanes ) {


<s> LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } LLVM_DEBUG ( printInfo ( ) ) ; lowerLiveMaskQueries ( ) ; lowerCopyInstrs ( ) ; if ( GlobalFlags == StateWQM ) { auto MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( WQMOpc ) , Exec ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; lowerKillInstrs ( true ) ; } else { for ( auto BII : Blocks ) processBlock ( * BII . first , BII . first == & Entry ) ; for ( auto BII : Blocks ) lowerBlock ( * BII . first ) ; } if ( LiveMaskReg != Exec ) LIS -> createAndComputeVirtRegInterval ( LiveMaskReg ) ; LIS -> removeRegUnit ( * MCRegUnitIterator ( MCRegister :: from ( <STR_LIT> ) , TRI ) ) ; if ( ! KillInstrs . empty ( ) ) LIS -> removeRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ;


<s> KillInstrs . clear ( ) ; StateTransition . clear ( ) ; ST = & MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; if ( ST -> isWave32 ( ) ) { AndOpc = <STR_LIT> ; AndN2Opc = <STR_LIT> ; XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; AndN2Opc = <STR_LIT> ; XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const char GlobalFlags = analyzeFunction ( MF ) ; const bool NeedsLiveMask = ! ( KillInstrs . empty ( ) && LiveMaskQueries . empty ( ) ) ; LiveMaskReg = Exec ; if ( ! ( GlobalFlags & ( StateWQM | StateStrict ) ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) && KillInstrs . empty ( ) ) {


<s> MachineInstr * Save = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveReg ) . addReg ( <STR_LIT> ) ; MachineInstr * Restore = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SaveReg ) ; LIS -> InsertMachineInstrInMaps ( * Save ) ;

LIS -> InsertMachineInstrInMaps ( * Restore ) ;


<s> case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; default : break ; } if ( NewOpcode ) TermMI -> setDesc ( TII -> get ( NewOpcode ) ) ; if ( SplitBB != BB ) { using DomTreeT = DomTreeBase < MachineBasicBlock > ; SmallVector < DomTreeT :: UpdateType , <NUM_LIT> > DTUpdates ; for ( MachineBasicBlock * Succ : SplitBB -> successors ( ) ) { DTUpdates . push_back ( { DomTreeT :: Insert , SplitBB , Succ } ) ; DTUpdates . push_back ( { DomTreeT :: Delete , BB , Succ } ) ; } DTUpdates . push_back ( { DomTreeT :: Insert , BB , SplitBB } ) ; if ( MDT ) MDT -> getBase ( ) . applyUpdates ( DTUpdates ) ; if ( PDT ) PDT -> getBase ( ) . applyUpdates ( DTUpdates ) ; MachineInstr * MI = BuildMI ( * BB , BB -> end ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( SplitBB ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; }


<s> NewOpcode = <STR_LIT> ; break ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; default : break ; } if ( NewOpcode ) TermMI -> setDesc ( TII -> get ( NewOpcode ) ) ; if ( SplitBB != BB ) { using DomTreeT = DomTreeBase < MachineBasicBlock > ; SmallVector < DomTreeT :: UpdateType , <NUM_LIT> > DTUpdates ; for ( MachineBasicBlock * Succ : SplitBB -> successors ( ) ) { DTUpdates . push_back ( { DomTreeT :: Insert , SplitBB , Succ } ) ; DTUpdates . push_back ( { DomTreeT :: Delete , BB , Succ } ) ; } DTUpdates . push_back ( { DomTreeT :: Insert , BB , SplitBB } ) ; if ( MDT ) MDT -> getBase ( ) . applyUpdates ( DTUpdates ) ; if ( PDT ) PDT -> getBase ( ) . applyUpdates ( DTUpdates ) ; MachineInstr * MI = BuildMI ( * BB , BB -> end ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( SplitBB ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } return SplitBB ;


<s> } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( AndOpc ) , Exec ) . addReg ( Exec ) . addReg ( LiveMaskReg ) ; } LIS -> InsertMachineInstrInMaps ( * MI ) ; StateTransition [ MI ] = StateExact ;


<s> } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( AndOpc ) , Exec ) . addReg ( Exec ) . addReg ( LiveMaskReg ) ; } LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> void <ISA_LIT>WholeQuadMode :: toStrictMode ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , Register SaveOrig , char StrictStateNeeded ) { MachineInstr * MI ; assert ( SaveOrig ) ; assert ( StrictStateNeeded == StateStrictWWM || StrictStateNeeded == StateStrictWQM ) ; if ( StrictStateNeeded == StateStrictWWM ) { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveOrig ) . addImm ( - <NUM_LIT> ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveOrig ) . addImm ( - <NUM_LIT> ) ; } LIS -> InsertMachineInstrInMaps ( * MI ) ;

StateTransition [ MI ] = StateStrictWWM ;


<s> MachineInstr * MI ; if ( SavedWQM ) { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , Exec ) . addReg ( SavedWQM ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( WQMOpc ) , Exec ) . addReg ( Exec ) ; } LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> RemoveOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } else {


<s> void <ISA_LIT>WholeQuadMode :: lowerCopyInstrs ( ) { for ( MachineInstr * MI : LowerToMovInstrs ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; const Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TRI -> isVGPR ( * MRI , Reg ) ) { const TargetRegisterClass * regClass = Register :: isVirtualRegister ( Reg ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; MI -> addOperand ( MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; } else { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) {


<s> for ( ; ; ) { MachineBasicBlock :: iterator Next = II ; char Needs = StateExact | StateWQM ; char OutNeeds = <NUM_LIT> ; if ( FirstWQM == IE ) FirstWQM = II ; if ( FirstWWM == IE ) FirstWWM = II ; if ( II != IE ) { MachineInstr & MI = * II ; if ( MI . isTerminator ( ) || TII -> mayReadEXEC ( * MRI , MI ) ) { auto III = Instructions . find ( & MI ) ; if ( III != Instructions . end ( ) ) { if ( III -> second . Needs & StateWWM ) Needs = StateWWM ; else if ( III -> second . Needs & StateWQM ) Needs = StateWQM ; else Needs &= ~ III -> second . Disabled ; OutNeeds = III -> second . OutNeeds ; } } else { Needs = StateExact | StateWQM | StateWWM ; } if ( MI . isTerminator ( ) && OutNeeds == StateExact ) Needs = StateExact ; if ( MI . getOpcode ( ) == <STR_LIT> && BI . OutNeeds == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; ++ Next ; } else { if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else {


<s> LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; lowerCopyInstrs ( ) ; return true ; } } LLVM_DEBUG ( printInfo ( ) ) ; lowerCopyInstrs ( ) ; for ( auto BII : Blocks ) processBlock ( * BII . first , LiveMaskReg , BII . first == & * MF . begin ( ) ) ;

LIS -> removeRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ;


<s> char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) {

BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ;


<s> } else if ( Opcode == <STR_LIT> ) { markInstructionUses ( MI , StateWWM , Worklist ) ; GlobalFlags |= StateWWM ; LowerToMovInstrs . push_back ( & MI ) ; continue ; } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { III . Disabled = StateWWM ; MachineOperand & Inactive = MI . getOperand ( <NUM_LIT> ) ; if ( Inactive . isReg ( ) ) { if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { Register Reg = Inactive . getReg ( ) ; if ( Register :: isVirtualRegister ( Reg ) ) { for ( MachineInstr & DefMI : MRI -> def_instructions ( Reg ) ) markInstruction ( DefMI , StateWWM , Worklist ) ; } } } SetInactiveInstrs . push_back ( & MI ) ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( & MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateWWM ; continue ; } else { if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( ! Register :: isVirtualRegister ( Reg ) && TRI -> hasVectorRegisters ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ;


<s> const char * getPassName ( ) const override {


<s> unsigned Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ;

BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ;


<s> void <ISA_LIT>WholeQuadMode :: markInstruction ( MachineInstr & MI , char Flag , std :: vector < WorkItem > & Worklist ) { InstrInfo & II = Instructions [ & MI ] ; assert ( Flag == StateWQM || Flag == StateExact ) ; if ( II . Needs ) return ; II . Needs = Flag ;


<s> InstrInfo & II = Instructions [ & MI ] ; assert ( Flag == StateWQM || Flag == StateExact ) ; if ( II . Needs ) return ;


<s> } char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; auto InstrInfoIt = Instructions . find ( & MI ) ; if ( InstrInfoIt != Instructions . end ( ) ) { Needs = InstrInfoIt -> second . Needs ; OutNeeds = InstrInfoIt -> second . OutNeeds ; if ( OutNeeds == StateExact && MI . isTerminator ( ) ) { assert ( Needs == <NUM_LIT> ) ; Needs = StateExact ; } } if ( Needs && State != Needs ) { if ( Needs == StateExact ) { assert ( ! SavedWQMReg ) ; if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , & MI , SavedWQMReg , LiveMaskReg ) ; } else { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , & MI , SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = Needs ; } } if ( ( BI . OutNeeds & StateWQM ) && State != StateWQM ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ;

toWQM ( MBB , MBB . end ( ) , SavedWQMReg ) ;


<s> auto BII = Blocks . find ( & MBB ) ; if ( BII == Blocks . end ( ) ) return ; const BlockInfo & BI = BII -> second ; if ( ! ( BI . InNeeds & StateWQM ) ) return ; if ( ! isEntry && ! ( BI . Needs & StateExact ) && BI . OutNeeds != StateExact ) return ; unsigned SavedWQMReg = <NUM_LIT> ; bool WQMFromExec = isEntry ; char State = isEntry ? StateExact : StateWQM ; auto II = MBB . getFirstNonPHI ( ) , IE = MBB . end ( ) ; while ( II != IE ) { MachineInstr & MI = * II ; ++ II ; if ( TII -> isScalarUnit ( MI ) && ! MI . isTerminator ( ) ) continue ; if ( TargetInstrInfo :: isGenericOpcode ( MI . getOpcode ( ) ) ) { if ( MI . getNumExplicitOperands ( ) >= <NUM_LIT> ) { const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; if ( Op . isReg ( ) ) { if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { continue ; } } } } char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; auto InstrInfoIt = Instructions . find ( & MI ) ; if ( InstrInfoIt != Instructions . end ( ) ) { Needs = InstrInfoIt -> second . Needs ; OutNeeds = InstrInfoIt -> second . OutNeeds ; if ( OutNeeds == StateExact && MI . isTerminator ( ) ) { assert ( Needs == <NUM_LIT> ) ; Needs = StateExact ; } } if ( Needs && State != Needs ) { if ( Needs == StateExact ) { assert ( ! SavedWQMReg ) ; if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , & MI , SavedWQMReg , LiveMaskReg ) ; } else { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , & MI , SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = Needs ; }


<s> MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; } if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerLiveMaskQueries ( LiveMaskReg ) ; return true ; } } lowerLiveMaskQueries ( LiveMaskReg ) ; for ( auto BII : Blocks ) processBlock ( * BII . first , LiveMaskReg , BII . first == & * MF . begin ( ) ) ; return true ;


<s> } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } }


<s> BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( LiveMaskReg ) ; }


<s> BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( LiveMaskReg ) ;


<s> BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SavedWQM ) ; } else {

BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ;


<s> void <ISA_LIT>WholeQuadMode :: toWQM ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SavedWQM ) { if ( SavedWQM ) { BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SavedWQM ) ; } else { BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ;


<s> dbgs ( ) << <STR_LIT> << BII . first -> getNumber ( ) << <STR_LIT> << <STR_LIT> << PrintState ( BII . second . InNeeds ) << <STR_LIT> << PrintState ( BII . second . Needs ) << <STR_LIT> << PrintState ( BII . second . OutNeeds ) << <STR_LIT> ; for ( const MachineInstr & MI : * BII . first ) { auto III = Instructions . find ( & MI ) ; if ( III == Instructions . end ( ) ) continue ;

dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << PrintState ( III -> second . Needs ) << <STR_LIT> << PrintState ( III -> second . OutNeeds ) << '\n' ;


<s> LLVM_DUMP_METHOD void <ISA_LIT>WholeQuadMode :: printInfo ( ) { for ( const auto & BII : Blocks ) { dbgs ( ) << <STR_LIT> << BII . first -> getNumber ( ) << <STR_LIT> << <STR_LIT> << PrintState ( BII . second . InNeeds ) << <STR_LIT> << PrintState ( BII . second . Needs ) << <STR_LIT> << PrintState ( BII . second . OutNeeds ) << <STR_LIT> ; for ( const MachineInstr & MI : * BII . first ) {

auto III = Instructions . find ( & MI ) ;


<s> unsigned Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ;


<s> for ( MachineInstr * MI : LiveMaskQueries ) { DebugLoc DL = MI -> getDebugLoc ( ) ; unsigned Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; MI -> eraseFromParent ( ) ;


<s> char State = isEntry ? StateExact : StateWQM ; auto II = MBB . getFirstNonPHI ( ) , IE = MBB . end ( ) ; while ( II != IE ) { MachineInstr & MI = * II ; ++ II ; if ( MI . getDesc ( ) . TSFlags & ( <STR_LIT> | <STR_LIT> ) && ! MI . isBranch ( ) && ! MI . isTerminator ( ) ) continue ; if ( TargetInstrInfo :: isGenericOpcode ( MI . getOpcode ( ) ) ) { if ( MI . getNumExplicitOperands ( ) >= <NUM_LIT> ) { const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; if ( Op . isReg ( ) ) { if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { continue ; } } } } char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; auto InstrInfoIt = Instructions . find ( & MI ) ; if ( InstrInfoIt != Instructions . end ( ) ) { Needs = InstrInfoIt -> second . Needs ;

OutNeeds = InstrInfoIt -> second . OutNeeds ;


<s> } for ( const MachineBasicBlock * Pred : MBB . predecessors ( ) ) { BlockInfo & PredBI = Blocks [ Pred ] ; if ( ( PredBI . OutNeeds | BI . InNeeds ) == PredBI . OutNeeds ) continue ; PredBI . OutNeeds |= BI . InNeeds ; PredBI . InNeeds |= BI . InNeeds ; Worklist . push_back ( Pred ) ; } for ( const MachineBasicBlock * Succ : MBB . successors ( ) ) { BlockInfo & SuccBI = Blocks [ Succ ] ;

if ( ( SuccBI . InNeeds | BI . OutNeeds ) == SuccBI . InNeeds ) continue ;


<s> void <ISA_LIT>WholeQuadMode :: propagateBlock ( const MachineBasicBlock & MBB , std :: vector < WorkItem > & Worklist ) { BlockInfo BI = Blocks [ & MBB ] ; if ( ! MBB . empty ( ) ) { const MachineInstr * LastMI = & * MBB . rbegin ( ) ; InstrInfo & LastII = Instructions [ LastMI ] ; if ( ( LastII . OutNeeds | BI . OutNeeds ) != LastII . OutNeeds ) {


<s> if ( ( II . OutNeeds & StateWQM ) && ! ( II . Needs & StateWQM ) && ( MI . isBranch ( ) || MI . isTerminator ( ) || MI . getOpcode ( ) == <STR_LIT> ) ) { Instructions [ & MI ] . Needs = StateWQM ; II . Needs = StateWQM ; } BI . Needs |= II . Needs ; if ( ( BI . InNeeds | II . Needs ) != BI . InNeeds ) { BI . InNeeds |= II . Needs ; Worklist . push_back ( & MBB ) ; } if ( const MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = II . Needs | II . OutNeeds ; if ( ! PrevMI -> isPHI ( ) ) { InstrInfo & PrevII = Instructions [ PrevMI ] ; if ( ( PrevII . OutNeeds | InNeeds ) != PrevII . OutNeeds ) { PrevII . OutNeeds |= InNeeds ; Worklist . push_back ( PrevMI ) ; } } } assert ( II . Needs != ( StateWQM | StateExact ) ) ; if ( II . Needs != StateWQM ) return ; for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Use . getReg ( ) ) ) continue ; for ( const MachineOperand & Def : MRI -> def_operands ( Use . getReg ( ) ) ) { const MachineInstr * DefMI = Def . getParent ( ) ; InstrInfo & DefII = Instructions [ DefMI ] ; if ( DefII . Needs != <NUM_LIT> ) continue ; DefII . Needs = StateWQM ;


<s> char GlobalFlags = analyzeFunction ( MF ) ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; return ! LiveMaskQueries . empty ( ) ; } MachineBasicBlock & Entry = MF . front ( ) ; MachineInstr * EntryMI = Entry . getFirstNonPHI ( ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; } if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerLiveMaskQueries ( LiveMaskReg ) ; return true ;


<s> unsigned Opcode = MI . getOpcode ( ) ; char Flags ; if ( TII -> isWQM ( Opcode ) || TII -> isDS ( Opcode ) ) { Flags = StateWQM ; } else if ( TII -> get ( Opcode ) . mayStore ( ) && ( MI . getDesc ( ) . TSFlags & <STR_LIT> ) ) { Flags = StateExact ; } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } continue ;


<s> WorkItem ( const MachineInstr * MI ) : MI ( MI ) {


<s> Instructions . clear ( ) ; Blocks . clear ( ) ; ExecExports . clear ( ) ; LiveMaskQueries . clear ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; return ! LiveMaskQueries . empty ( ) ; } unsigned LiveMaskReg = <NUM_LIT> ; { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; } if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerLiveMaskQueries ( LiveMaskReg ) ;


<s> TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; return ! LiveMaskQueries . empty ( ) ; } unsigned LiveMaskReg = <NUM_LIT> ; { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; } if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerLiveMaskQueries ( LiveMaskReg ) ; return true ; } } lowerLiveMaskQueries ( LiveMaskReg ) ; for ( const auto & BII : Blocks ) processBlock ( const_cast < MachineBasicBlock & > ( * BII . first ) , LiveMaskReg , BII . first == & * MF . begin ( ) ) ;


<s> for ( auto BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( auto II = MBB . begin ( ) , IE = MBB . end ( ) ; II != IE ; ++ II ) { MachineInstr & MI = * II ; unsigned Opcode = MI . getOpcode ( ) ; char Flags = <NUM_LIT> ; if ( TII -> isWQM ( Opcode ) || TII -> isDS ( Opcode ) ) { Flags = StateWQM ; } else if ( TII -> get ( Opcode ) . mayStore ( ) && ( MI . getDesc ( ) . TSFlags & <STR_LIT> ) ) { Flags = StateExact ; } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } Instructions [ & MI ] . Needs = Flags ; Worklist . push_back ( & MI ) ; GlobalFlags |= Flags ; } if ( WQMOutputs && MBB . succ_empty ( ) ) { Blocks [ & MBB ] . OutNeeds = StateExact ; Worklist . push_back ( & MBB ) ; GlobalFlags |= StateExact ; } }


<s> } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } Instructions [ & MI ] . Needs = Flags ; Worklist . push_back ( & MI ) ; GlobalFlags |= Flags ; } if ( WQMOutputs && MBB . succ_empty ( ) ) { Blocks [ & MBB ] . OutNeeds = StateExact ;


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { if ( Reg == <STR_LIT> ) continue ; for ( MCRegUnitIterator RegUnit ( Reg , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit ) { LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , StateWQM , Worklist ) ; }


<s> unsigned Reg = Use . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { if ( Reg == <STR_LIT> ) continue ; for ( MCRegUnitIterator RegUnit ( Reg , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit ) { LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , StateWQM , Worklist ) ; } continue ;


<s> if ( BII == Blocks . end ( ) ) return ; const BlockInfo & BI = BII -> second ; if ( ! ( BI . InNeeds & StateWQM ) ) return ; if ( ! isEntry && ! ( BI . Needs & StateExact ) && BI . OutNeeds != StateExact ) return ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB . getNumber ( ) << <STR_LIT> ) ; unsigned SavedWQMReg = <NUM_LIT> ; bool WQMFromExec = isEntry ; char State = isEntry ? StateExact : StateWQM ; auto II = MBB . getFirstNonPHI ( ) , IE = MBB . end ( ) ; if ( isEntry ) ++ II ; MachineBasicBlock :: iterator First = IE ; for ( ; ; ) { MachineBasicBlock :: iterator Next = II ; char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; if ( First == IE ) First = II ; if ( II != IE ) { MachineInstr & MI = * II ; if ( requiresCorrectState ( MI ) ) {


<s> BI . Needs |= II . Needs ; if ( ( BI . InNeeds | II . Needs ) != BI . InNeeds ) { BI . InNeeds |= II . Needs ; Worklist . push_back ( MBB ) ; } if ( MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = II . Needs | II . OutNeeds ; if ( ! PrevMI -> isPHI ( ) ) { InstrInfo & PrevII = Instructions [ PrevMI ] ; if ( ( PrevII . OutNeeds | InNeeds ) != PrevII . OutNeeds ) { PrevII . OutNeeds |= InNeeds ; Worklist . push_back ( PrevMI ) ; } }


<s> TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; return ! LiveMaskQueries . empty ( ) ; } unsigned LiveMaskReg = <NUM_LIT> ; { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ;

MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ;


<s> MachineInstr & MI = * II ; unsigned Opcode = MI . getOpcode ( ) ; char Flags = <NUM_LIT> ; if ( TII -> isDS ( Opcode ) ) { Flags = StateWQM ; } else if ( TII -> isWQM ( Opcode ) ) { markUsesWQM ( MI , Worklist ) ; GlobalFlags |= StateWQM ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { Flags = StateExact ; } else { if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; }


<s> while ( II != IE ) { MachineInstr & MI = * II ; ++ II ; if ( TII -> isScalarUnit ( MI ) && ! MI . isTerminator ( ) ) continue ; if ( TargetInstrInfo :: isGenericOpcode ( MI . getOpcode ( ) ) ) { if ( MI . getNumExplicitOperands ( ) >= <NUM_LIT> ) { const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; if ( Op . isReg ( ) ) { if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { continue ; } } } } char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; auto InstrInfoIt = Instructions . find ( & MI ) ; if ( InstrInfoIt != Instructions . end ( ) ) { Needs = InstrInfoIt -> second . Needs ; OutNeeds = InstrInfoIt -> second . OutNeeds ; if ( OutNeeds == StateExact && MI . isTerminator ( ) ) { assert ( Needs == <NUM_LIT> ) ; Needs = StateExact ; } } if ( Needs && State != Needs ) { if ( Needs == StateExact ) { assert ( ! SavedWQMReg ) ; if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , & MI , SavedWQMReg , LiveMaskReg ) ; } else { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , & MI , SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = Needs ; } if ( MI . getOpcode ( ) == <STR_LIT> && State == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; } if ( ( BI . OutNeeds & StateWQM ) && State != StateWQM ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , MBB . end ( ) , SavedWQMReg ) ; } else if ( BI . OutNeeds == StateExact && State != StateExact ) {

toExact ( MBB , MBB . end ( ) , <NUM_LIT> , LiveMaskReg ) ;


<s> } } char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; auto InstrInfoIt = Instructions . find ( & MI ) ; if ( InstrInfoIt != Instructions . end ( ) ) { Needs = InstrInfoIt -> second . Needs ; OutNeeds = InstrInfoIt -> second . OutNeeds ; if ( OutNeeds == StateExact && MI . isTerminator ( ) ) { assert ( Needs == <NUM_LIT> ) ; Needs = StateExact ; } } if ( Needs && State != Needs ) { if ( Needs == StateExact ) { assert ( ! SavedWQMReg ) ; if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , & MI , SavedWQMReg , LiveMaskReg ) ; } else { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , & MI , SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = Needs ; } if ( MI . getOpcode ( ) == <STR_LIT> && State == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; } if ( ( BI . OutNeeds & StateWQM ) && State != StateWQM ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , MBB . end ( ) , SavedWQMReg ) ; } else if ( BI . OutNeeds == StateExact && State != StateExact ) { toExact ( MBB , MBB . end ( ) , <NUM_LIT> , LiveMaskReg ) ;


<s> const TargetRegisterClass * regClass = Reg . isVirtual ( ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; MI -> addOperand ( MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; } else { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> RemoveOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } else { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ;


<s> const TargetRegisterClass * regClass = Reg . isVirtual ( ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; MI -> addOperand ( MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; } else { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> RemoveOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } else { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; }


<s> if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; Register Reg = Use . getReg ( ) ; if ( ! Reg . isVirtual ( ) ) { if ( Reg == <STR_LIT> || Reg == <STR_LIT> ) continue ; for ( MCRegUnitIterator RegUnit ( Reg , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit ) { LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ;

markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag , Worklist ) ;


<s> for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; Register Reg = Use . getReg ( ) ; if ( ! Reg . isVirtual ( ) ) { if ( Reg == <STR_LIT> || Reg == <STR_LIT> ) continue ; for ( MCRegUnitIterator RegUnit ( Reg , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit ) { LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag , Worklist ) ; }


<s> lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; }


<s> if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { unsigned Reg = Inactive . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { for ( MachineInstr & DefMI : MRI -> def_instructions ( Reg ) ) markInstruction ( DefMI , StateWWM , Worklist ) ; } } } SetInactiveInstrs . push_back ( & MI ) ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( & MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateWWM ; continue ; } else { if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVectorRegisters ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } markInstruction ( MI , Flags , Worklist ) ; GlobalFlags |= Flags ; } }


<s> } BI . Needs |= II . Needs ; if ( ( BI . InNeeds | II . Needs ) != BI . InNeeds ) { BI . InNeeds |= II . Needs ; Worklist . push_back ( MBB ) ; } if ( MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = II . Needs | II . OutNeeds ; if ( ! PrevMI -> isPHI ( ) ) { InstrInfo & PrevII = Instructions [ PrevMI ] ; if ( ( PrevII . OutNeeds | InNeeds ) != PrevII . OutNeeds ) { PrevII . OutNeeds |= InNeeds ; Worklist . push_back ( PrevMI ) ; } } } assert ( II . Needs != ( StateWQM | StateExact ) ) ; if ( II . Needs != StateWQM ) return ; for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Use . getReg ( ) ) ) continue ; for ( MachineInstr & DefMI : MRI -> def_instructions ( Use . getReg ( ) ) ) { InstrInfo & DefII = Instructions [ & DefMI ] ; if ( DefII . Needs != <NUM_LIT> ) continue ; DefII . Needs = StateWQM ;


<s> void <ISA_LIT>WholeQuadMode :: propagateInstruction ( MachineInstr & MI , std :: vector < WorkItem > & Worklist ) { MachineBasicBlock * MBB = MI . getParent ( ) ; InstrInfo II = Instructions [ & MI ] ; BlockInfo & BI = Blocks [ MBB ] ; if ( ( II . OutNeeds & StateWQM ) && ! ( II . Needs & StateWQM ) && MI . isTerminator ( ) ) { Instructions [ & MI ] . Needs = StateWQM ; II . Needs = StateWQM ; } BI . Needs |= II . Needs ; if ( ( BI . InNeeds | II . Needs ) != BI . InNeeds ) { BI . InNeeds |= II . Needs ; Worklist . push_back ( MBB ) ; } if ( MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = II . Needs | II . OutNeeds ; if ( ! PrevMI -> isPHI ( ) ) { InstrInfo & PrevII = Instructions [ PrevMI ] ; if ( ( PrevII . OutNeeds | InNeeds ) != PrevII . OutNeeds ) { PrevII . OutNeeds |= InNeeds ; Worklist . push_back ( PrevMI ) ; } } } assert ( II . Needs != ( StateWQM | StateExact ) ) ; if ( II . Needs != StateWQM ) return ; for ( const MachineOperand & Use : MI . uses ( ) ) {

if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ;


<s> MRI = & MF . getRegInfo ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; return ! LiveMaskQueries . empty ( ) ; } unsigned LiveMaskReg = <NUM_LIT> ; { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; } if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerLiveMaskQueries ( LiveMaskReg ) ; return true ; }


<s> } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ;


<s> Flags = StateExact ; } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } Instructions [ & MI ] . Needs = Flags ; Worklist . push_back ( & MI ) ; GlobalFlags |= Flags ; } if ( WQMOutputs && MBB . succ_empty ( ) ) { Blocks [ & MBB ] . OutNeeds = StateExact ; Worklist . push_back ( & MBB ) ; GlobalFlags |= StateExact ;


<s> Idx = Next ; } } MachineBasicBlock :: iterator MBBI ; if ( MachineInstr * MI = LIS -> getInstructionFromIndex ( Idx ) ) MBBI = MI ; else { assert ( Idx == LIS -> getMBBEndIdx ( & MBB ) ) ; MBBI = MBB . end ( ) ; } while ( MBBI != Last ) { bool IsExecDef = false ; for ( const MachineOperand & MO : MBBI -> operands ( ) ) { if ( MO . isReg ( ) && MO . isDef ( ) ) { IsExecDef |= MO . getReg ( ) == <STR_LIT> || MO . getReg ( ) == <STR_LIT> ; } } if ( ! IsExecDef ) break ; MBBI ++ ;


<s> Exec = <STR_LIT> ; } char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) {

auto MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ;


<s> MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerCopyInstrs ( ) ; return true ; } } DEBUG ( printInfo ( ) ) ; lowerCopyInstrs ( ) ;

for ( auto BII : Blocks ) processBlock ( * BII . first , LiveMaskReg , BII . first == & * MF . begin ( ) ) ;


<s> if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) {


<s> char <ISA_LIT>WholeQuadMode :: analyzeFunction ( MachineFunction & MF ) { std :: vector < WorkItem > Worklist ; char GlobalFlags = scanInstructions ( MF , Worklist ) ; while ( ! Worklist . empty ( ) ) { WorkItem WI = Worklist . back ( ) ; Worklist . pop_back ( ) ; if ( WI . MI ) propagateInstruction ( * WI . MI , Worklist ) ; else propagateBlock ( * WI . MBB , Worklist ) ; } return GlobalFlags ;


<s> Worklist . pop_back ( ) ; if ( WI . MI ) propagateInstruction ( * WI . MI , Worklist ) ; else propagateBlock ( * WI . MBB , Worklist ) ;


<s> AU . addRequired < LiveIntervals > ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {

AU . addRequired < LiveIntervals > ( ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> void <ISA_LIT>WholeQuadMode :: lowerCopyInstrs ( ) { for ( MachineInstr * MI : LowerToCopyInstrs ) { for ( unsigned i = MI -> getNumExplicitOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; i -- ) MI -> RemoveOperand ( i ) ; MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; }


<s> for ( MachineInstr * MI : LowerToCopyInstrs ) {

for ( unsigned i = MI -> getNumExplicitOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; i -- ) MI -> RemoveOperand ( i ) ;


<s> unsigned Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Copy = BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; LIS -> ReplaceMachineInstrInMaps ( * MI , * Copy ) ; MI -> eraseFromParent ( ) ;


<s> unsigned Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Copy = BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; LIS -> ReplaceMachineInstrInMaps ( * MI , * Copy ) ;


<s> LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag , Worklist ) ; } continue ; }

for ( MachineInstr & DefMI : MRI -> def_instructions ( Use . getReg ( ) ) ) markInstruction ( DefMI , Flag , Worklist ) ;


<s> for ( ; ; ) { S = LR . getSegmentContaining ( Idx ) ; if ( ! S ) break ; if ( PreferLast ) { SlotIndex Next = S -> start . getBaseIndex ( ) ; if ( Next < FirstIdx ) break ; Idx = Next ; } else { SlotIndex Next = S -> end . getNextIndex ( ) . getBaseIndex ( ) ; if ( Next > LastIdx ) break ; Idx = Next ; } } MachineBasicBlock :: iterator MBBI ; if ( MachineInstr * MI = LIS -> getInstructionFromIndex ( Idx ) ) MBBI = MI ; else { assert ( Idx == LIS -> getMBBEndIdx ( & MBB ) ) ; MBBI = MBB . end ( ) ; } if ( S ) MBBI = saveSCC ( MBB , MBBI ) ;


<s> auto III = Instructions . find ( & MI ) ; if ( III == Instructions . end ( ) ) continue ;


<s> if ( III == Instructions . end ( ) ) continue ; dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << PrintState ( III -> second . Needs ) << <STR_LIT> << PrintState ( III -> second . OutNeeds ) << '\n' ;


<s> explicit PrintState ( int State ) : State ( State ) {


<s> } if ( MI . isTerminator ( ) && OutNeeds == StateExact ) Needs = StateExact ; if ( MI . getOpcode ( ) == <STR_LIT> && BI . OutNeeds == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; ++ Next ; } else { if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; SavedNonWWMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , Before , SavedWQMReg ) ; if ( SavedWQMReg ) { LIS -> createAndComputeVirtRegInterval ( SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = StateWQM ; } else {


<s> Worklist . push_back ( MBB ) ; } } if ( MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = ( II . Needs & ~ StateWWM ) | II . OutNeeds ; if ( ! PrevMI -> isPHI ( ) ) { InstrInfo & PrevII = Instructions [ PrevMI ] ; if ( ( PrevII . OutNeeds | InNeeds ) != PrevII . OutNeeds ) { PrevII . OutNeeds |= InNeeds ; Worklist . push_back ( PrevMI ) ; } }


<s> bool <ISA_LIT>WholeQuadMode :: requiresCorrectState ( const MachineInstr & MI ) const { if ( MI . isTerminator ( ) ) return true ; if ( TII -> isScalarUnit ( MI ) ) return false ; if ( MI . isTransient ( ) ) { if ( MI . getNumExplicitOperands ( ) >= <NUM_LIT> ) {

const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ;


<s> Instructions . clear ( ) ; Blocks . clear ( ) ; LiveMaskQueries . clear ( ) ; LowerToCopyInstrs . clear ( ) ; CallingConv = MF . getFunction ( ) . getCallingConv ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) {

LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ;


<s> continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( & MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateWWM ; continue ; } else { if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } markInstruction ( MI , Flags , Worklist ) ; GlobalFlags |= Flags ; } } if ( GlobalFlags & StateWQM ) { for ( MachineInstr * MI : SetInactiveInstrs ) markInstruction ( * MI , StateWQM , Worklist ) ;


<s> Flags = StateWQM ; LowerToCopyInstrs . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { markInstructionUses ( MI , StateWWM , Worklist ) ; GlobalFlags |= StateWWM ; LowerToCopyInstrs . push_back ( & MI ) ; continue ; } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { III . Disabled = StateWWM ; MachineOperand & Inactive = MI . getOperand ( <NUM_LIT> ) ; if ( Inactive . isReg ( ) ) { if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { unsigned Reg = Inactive . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { for ( MachineInstr & DefMI : MRI -> def_instructions ( Reg ) ) markInstruction ( DefMI , StateWWM , Worklist ) ; } } } SetInactiveInstrs . push_back ( & MI ) ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( & MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateWWM ; continue ;


<s> void <ISA_LIT>WholeQuadMode :: toExact ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SaveWQM , unsigned LiveMaskReg ) { MachineInstr * MI ; if ( SaveWQM ) { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveWQM ) . addReg ( LiveMaskReg ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( LiveMaskReg ) ; }

LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> void <ISA_LIT>WholeQuadMode :: toExact ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SaveWQM , unsigned LiveMaskReg ) { MachineInstr * MI ; if ( SaveWQM ) {

MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveWQM ) . addReg ( LiveMaskReg ) ;


<s> void <ISA_LIT>WholeQuadMode :: toWQM ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SavedWQM ) { MachineInstr * MI ; if ( SavedWQM ) { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SavedWQM ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; } LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> void <ISA_LIT>WholeQuadMode :: toWWM ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SaveOrig ) { MachineInstr * MI ; assert ( SaveOrig ) ; MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveOrig ) . addImm ( - <NUM_LIT> ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ;


<s> WorkItem ( MachineInstr * MI ) : MI ( MI ) {