<s> int VAddr0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; unsigned NewAddrDwords = Info -> VAddrDwords ; const TargetRegisterClass * RC ; if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI -> getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef


<s> if ( SOPKOpc == - <NUM_LIT> ) return ; if ( SOPKOpc == <STR_LIT> || SOPKOpc == <STR_LIT> ) { bool HasUImm ; if ( isKImmOrKUImmOperand ( Src1 , HasUImm ) ) { if ( ! HasUImm ) { SOPKOpc = ( SOPKOpc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; } MI . setDesc ( TII -> get ( SOPKOpc ) ) ; } return ; } const MCInstrDesc & NewDesc = TII -> get ( SOPKOpc ) ; if ( ( TII -> sopkIsZext ( SOPKOpc ) && isKUImmOperand ( Src1 ) ) || ( ! TII ->

sopkIsZext ( SOPKOpc ) && isKImmOperand ( Src1 ) ) ) {


<s> for ( const MachineOperand & MO : R ) { if ( ! MO . isReg ( ) ) continue ; if ( Register :: isPhysicalRegister ( Reg ) && Register :: isPhysicalRegister ( MO . getReg ( ) ) ) { if ( TRI . regsOverlap ( Reg , MO . getReg ( ) ) ) return true ; } else if ( MO . getReg ( ) == Reg && Register :: isVirtualRegister ( Reg ) ) { LaneBitmask Overlap = TRI . getSubRegIndexLaneMask ( SubReg ) &

TRI . getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ;


<s> if ( ! MO . isReg ( ) ) continue ; if ( Register :: isPhysicalRegister ( Reg ) && Register :: isPhysicalRegister ( MO . getReg ( ) ) ) { if ( TRI . regsOverlap ( Reg , MO . getReg ( ) ) ) return true ; } else if ( MO . getReg ( ) == Reg && Register :: isVirtualRegister ( Reg ) ) { LaneBitmask Overlap = TRI . getSubRegIndexLaneMask ( SubReg ) & TRI . getSubRegIndexLaneMask ( MO .


<s> if ( ! Info || Info -> MIMGEncoding != <STR_LIT> ) return ; MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; int VAddr0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; unsigned NewAddrDwords = Info -> VAddrDwords ; const TargetRegisterClass * RC ; if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords <= <NUM_LIT> ) { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI . getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i ) . isTied ( ) && MI . getOperand ( i ) . isImplicit


<s> Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && SrcImm == Src0 ) { if ( ! TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) NewImm = <NUM_LIT> ; } if ( NewImm != <NUM_LIT> ) { if ( Register :: isVirtualRegister ( Dest -> getReg ( ) ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return


<s> static TargetInstrInfo :: RegSubRegPair getSubRegForIndex ( unsigned Reg , unsigned Sub , unsigned I , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { if ( TRI . getRegSizeInBits ( Reg , MRI ) != <NUM_LIT> ) { if ( Register :: isPhysicalRegister ( Reg ) ) { Reg = TRI . getSubReg ( Reg , TRI . getSubRegFromChannel ( I ) ) ; } else { LaneBitmask LM = TRI . getSubRegIndexLaneMask ( Sub ) ; Sub = TRI . getSubRegFromChannel ( I + countTrailingZeros ( LM . getAsInteger ( ) ) ) ; } } return TargetInstrInfo :: RegSubRegPair (


<s> MachineInstr & MovY = * YTop . getParent ( ) ; if ( ( MovY . getOpcode ( ) != <STR_LIT> && MovY . getOpcode ( ) != <STR_LIT> ) || MovY . getOperand ( <NUM_LIT> ) . getSubReg ( ) != Tsub ) continue ; Register Y = MovY . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Ysub = MovY . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( ! TRI . isVGPR ( MRI , Y ) || MovT . getParent ( ) != MovY . getParent ( ) ) continue ; MachineInstr * MovX = nullptr ; auto I = std :: next ( MovT . getIterator ( ) ) , E = MovT . getParent ( ) -> instr_end ( ) ; for ( auto IY = MovY . getIterator ( ) ; I != E && I != IY ; ++ I ) { if ( instReadsReg ( & * I , X , Xsub , TRI ) || instModifiesReg ( & * I , Y , Ysub , TRI ) || instModifiesReg ( & * I , T , Tsub , TRI ) || ( MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) ) { MovX = nullptr ; break ; } if ( ! instReadsReg ( & * I , Y , Ysub , TRI ) ) { if ( ! MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) { MovX = nullptr ; break ; } continue ; } if ( MovX || ( I -> getOpcode ( ) != <STR_LIT> && I -> getOpcode ( ) != <STR_LIT> ) || I -> getOperand ( <NUM_LIT> ) . getReg ( ) != X || I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Xsub ) { MovX = nullptr ; break ; } MovX = &


<s> case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || ( Src1Mod && Src1Mod ->

getImm ( ) != <NUM_LIT> ) ) ) return false ;


<s> static bool isKImmOperand ( const <ISA_LIT>InstrInfo * TII , const MachineOperand & Src ) { return isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII

-> isInlineConstant ( Src , <NUM_LIT> ) ;


<s> static bool isKImmOperand ( const <ISA_LIT>InstrInfo * TII , const MachineOperand & Src ) { return isInt < <NUM_LIT> > ( Src . getImm ( ) ) && !

TII -> isInlineConstant ( Src , <NUM_LIT> ) ;


<s> MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; unsigned SReg = Src2 -> getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( SReg != <STR_LIT> ) continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << '\n' ; ) ; MachineInstrBuilder Inst32 = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( Op32 ) ) ; Inst32 . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; Inst32 . addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . addOperand ( * Src1 ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT>


<s> case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || TII

-> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ;


<s> } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && SrcImm == Src0 ) { if ( ! TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) NewImm = <NUM_LIT> ; } if ( NewImm != <NUM_LIT> ) { if ( TargetRegisterInfo :: isVirtualRegister ( Dest -> getReg ( ) ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ; } if ( SrcReg -> isReg ( ) && SrcReg -> getReg ( ) == Dest -> getReg ( ) ) { MI . setDesc ( TII -> get ( Opc ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { Src0 -> ChangeToImmediate ( NewImm )


<s> } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; unsigned SReg = Src2 -> getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( SReg != <STR_LIT> ) continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << '\n' ; ) ; MachineInstrBuilder Inst32 = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( Op32 ) ) ; int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: dst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && <STR_LIT> ) ; } Inst32 . addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . addOperand ( * Src1 ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . addOperand ( * Src2 ) ; } else { assert ( Src2 -> getReg ( ) == <STR_LIT> && <STR_LIT> ) ; Inst32 . addOperand ( copyRegOperandAsImplicit ( * Src2 ) ) ; } } ++ NumInstructionsShrunk ; MI . eraseFromParent ( ) ; foldImmediates ( * Inst32 , TII , MRI ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * Inst32 << '\n' ) ; } } return


<s> NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && SrcImm == Src0 ) { if ( ! TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) NewImm = <NUM_LIT> ; } if ( NewImm != <NUM_LIT> ) { if ( Register :: isVirtualRegister ( Dest -> getReg ( ) ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ; } if ( SrcReg -> isReg ( ) && SrcReg -> getReg ( ) == Dest -> getReg ( ) ) { MI . setDesc ( TII -> get ( Opc ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { Src0 -> ChangeToImmediate ( NewImm ) ; MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Dest -> getReg ( ) , false ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } else { SrcImm -> setImm


<s> MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . isCompare ( ) && TII -> isSOPC ( MI ) ) { shrinkScalarCompare ( TII , MI ) ; continue ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && Register :: isPhysicalRegister ( Dst . getReg ( ) ) ) { int32_t ReverseImm ; if ( isKImmOperand ( TII , Src ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; else if ( isReverseInlineImm ( TII , Src , ReverseImm ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; Src . setImm ( ReverseImm ) ; } } continue ; } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { if ( shrinkScalarLogicOp ( ST , MRI , TII , MI ) ) continue ; } if ( TII -> isMIMG ( MI . getOpcode ( ) ) && ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 && MF . getProperties ( ) . hasProperty ( MachineFunctionProperties :: Property :: NoVRegs ) ) { shrinkMIMG ( MI ) ; continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; if ( ! TII -> canShrink ( MI , MRI ) ) { if ( ! MI . isCommutable ( ) || ! TII -> commuteInstruction ( MI ) || ! TII -> canShrink ( MI , MRI ) ) continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , VCCReg ) ; continue ; } if ( DstReg != VCCReg ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; Register SReg = Src2 -> getReg ( ) ; if ( Register :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , VCCReg ) ; continue ; } if ( SReg != VCCReg ) continue ; } const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( SDst )


<s> MI . setDesc ( TII -> get ( Opc ) ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && isKImmOperand ( TII , Src ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; if ( ! canShrink ( MI , TII , TRI , MRI ) ) { if ( ! MI . isCommutable ( ) || ! TII -> commuteInstruction ( MI ) || ! canShrink ( MI , TII , TRI , MRI ) ) continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; unsigned SReg = Src2 -> getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( SReg != <STR_LIT> ) continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder Inst32 = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( Op32 ) ) ; int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: vdst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && <STR_LIT> ) ; } Inst32 . addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . addOperand ( * Src1 ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . addOperand ( * Src2 ) ; } else { copyFlagsToImplicitVCC ( * Inst32 , * Src2


<s> break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || ( Src1Mod && Src1Mod -> getImm ( ) != <NUM_LIT> ) ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; return ! TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp


<s> if ( ! MRI . isSSA ( ) ) return ; assert ( TII -> isVOP1 ( MI . getOpcode ( ) ) || TII -> isVOP2 ( MI . getOpcode ( ) ) || TII -> isVOPC ( MI . getOpcode ( ) ) ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; if ( Src0 . isImm ( ) && TII -> isLiteralConstant ( Src0 , TII -> getOpSize ( MI , Src0Idx ) ) ) return ; if ( Src0 . isReg ( ) && ! isVGPR ( & Src0 , TRI , MRI ) ) return ; if ( Src0 . isReg ( ) && MRI . hasOneUse ( Src0 . getReg ( ) ) ) { unsigned Reg = Src0 . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate (


<s> const char * getPassName ( ) const override { return <STR_LIT>


<s> bool <ISA_LIT>TypeRewriter :: runOnFunction ( Function & F ) { if ( <STR_LIT> ( F ) == ShaderType :: COMPUTE ) return false ; visit ( F ) ; visit ( F ) ; return false


<s> IRBuilder < > Builder ( & I ) ; if ( I . getDestTy ( ) != v4i32 ) { return ; } if ( BitCastInst * Op = dyn_cast < BitCastInst >


<s> void <ISA_LIT>TypeRewriter :: visitBitCast ( BitCastInst & I ) { IRBuilder < > Builder ( & I ) ; if ( I . getDestTy ( ) != v4i32 ) { return ; } if ( BitCastInst * Op = dyn_cast < BitCastInst > ( I . getOperand (


<s> Types . push_back ( v4i32 ) ; NeedToReplace = true ; Name = Name + <STR_LIT> ; } else if ( Arg -> getType ( ) -> isVectorTy ( ) && Arg -> getType ( ) -> getVectorNumElements ( ) == <NUM_LIT> && Arg -> getType ( ) -> getVectorElementType ( ) == Type :: getInt32Ty ( I . getContext ( ) ) ) { Type * ElementTy = Arg -> getType ( ) -> getVectorElementType ( ) ; std :: string TypeName = <STR_LIT> ; InsertElementInst * Def = cast < InsertElementInst > ( Arg ) ; Args . push_back ( Def -> getOperand ( <NUM_LIT> ) ) ; Types . push_back ( ElementTy )


<s> Type * PtrTy = Ptr -> getType ( ) ; Type * ElemTy = PtrTy -> getPointerElementType ( ) ; IRBuilder < > Builder ( & I ) ; if ( ElemTy == v16i8 ) { Value * BitCast = Builder . CreateBitCast ( Ptr , PointerType :: get ( v4i32 , PtrTy -> getPointerAddressSpace ( ) ) ) ; LoadInst * Load = Builder . CreateLoad ( BitCast ) ; SmallVector < std :: pair < unsigned , MDNode * > , <NUM_LIT> > MD ; I . getAllMetadataOtherThanDebugLoc ( MD ) ; for ( unsigned i = <NUM_LIT> , e = MD . size ( ) ; i != e ; ++ i ) { Load -> setMetadata ( MD [ i ] . first , MD [ i ] . second ) ; } Value * BitCastLoad = Builder . CreateBitCast ( Load , I . getType ( )


<s> if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ; visit ( F


<s> Types . push_back ( v4i32 ) ; NeedToReplace = true ; Name = Name + <STR_LIT> ; } else if ( Arg -> getType ( ) -> isVectorTy ( ) && Arg -> getType ( ) -> getVectorNumElements ( ) == <NUM_LIT> && Arg -> getType ( ) -> getVectorElementType ( ) == Type :: getInt32Ty ( I . getContext ( ) ) ) { Type * ElementTy = Arg -> getType ( ) -> getVectorElementType ( ) ; std :: string TypeName = <STR_LIT> ; InsertElementInst * Def = cast < InsertElementInst > ( Arg ) ; Args . push_back ( Def -> getOperand ( <NUM_LIT> ) ) ; Types . push_back ( ElementTy ) ; std :: string VecTypeName = <STR_LIT> + TypeName ; Name = Name . replace ( Name . find ( VecTypeName ) , VecTypeName . length ( ) , TypeName ) ; NeedToReplace = true ; }


<s> SmallVector < Type * , <NUM_LIT> > Types ; bool NeedToReplace = false ; Function * F = I . getCalledFunction ( ) ; if ( ! F ) return ; std :: string Name = F -> getName ( ) ; for ( unsigned i = <NUM_LIT> , e = I . getNumArgOperands ( ) ; i != e ; ++ i ) { Value * Arg = I . getArgOperand ( i ) ; if ( Arg -> getType ( ) == v16i8 ) { Args . push_back ( Builder . CreateBitCast ( Arg , v4i32 ) ) ; Types . push_back ( v4i32 ) ; NeedToReplace = true ; Name = Name + <STR_LIT> ; } else if ( Arg -> getType ( ) -> isVectorTy ( ) && Arg -> getType ( ) -> getVectorNumElements ( ) == <NUM_LIT> && Arg -> getType ( ) -> getVectorElementType ( ) == Type :: getInt32Ty ( I . getContext ( ) ) ) { Type * ElementTy = Arg -> getType ( ) -> getVectorElementType ( ) ; std :: string TypeName = <STR_LIT> ; InsertElementInst * Def = cast < InsertElementInst > ( Arg ) ; Args . push_back ( Def -> getOperand ( <NUM_LIT> ) ) ; Types . push_back ( ElementTy ) ; std :: string VecTypeName = <STR_LIT> + TypeName ; Name = Name . replace ( Name . find ( VecTypeName ) , VecTypeName . length ( ) , TypeName ) ; NeedToReplace = true ; } else { Args . push_back ( Arg ) ; Types . push_back ( Arg -> getType (


<s> if ( A . isStringAttribute ( ) ) { StringRef Str = A . getValueAsString ( ) ; Str . getAsInteger ( <NUM_LIT> , ShaderType ) ; } if ( ShaderType == ShaderType :: COMPUTE ) return false ; visit ( F


<s> Attribute A = F . getFnAttribute ( <STR_LIT> ) ; unsigned ShaderType = ShaderType :: COMPUTE ; if ( A . isStringAttribute ( ) ) { StringRef Str = A . getValueAsString ( ) ; Str . getAsInteger ( <NUM_LIT> , ShaderType ) ; } if ( ShaderType == ShaderType :: COMPUTE ) return


<s> const char GlobalFlags = analyzeFunction ( MF ) ; const bool NeedsLiveMask = ! ( KillInstrs . empty ( ) && LiveMaskQueries . empty ( ) ) ; LiveMaskReg = Exec ; if ( ! ( GlobalFlags & ( StateWQM | StateStrict ) ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) && KillInstrs . empty ( ) ) { lowerLiveMaskQueries ( ) ; return ! LiveMaskQueries . empty ( ) ; } MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( NeedsLiveMask || ( GlobalFlags & StateWQM ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } LLVM_DEBUG ( printInfo ( ) ) ; lowerLiveMaskQueries ( ) ; lowerCopyInstrs (


<s> if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; SavedNonWWMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , Before , SavedWQMReg ) ; if ( SavedWQMReg ) { LIS -> createAndComputeVirtRegInterval ( SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = StateWQM ; } else { assert ( Needs &


<s> auto III = Instructions . find ( & MI ) ; if ( III != Instructions . end ( ) ) { if ( III -> second . Needs & StateWWM ) Needs = StateWWM ; else if ( III -> second . Needs & StateWQM ) Needs = StateWQM ; else Needs &= ~ III -> second . Disabled ; OutNeeds = III -> second . OutNeeds ; } } else { Needs = StateExact | StateWQM | StateWWM ; } if ( MI . isTerminator ( ) && OutNeeds == StateExact ) Needs = StateExact ; if ( MI . getOpcode ( ) == <STR_LIT> && BI . OutNeeds == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; ++ Next ; } else { if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; SavedNonWWMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , Before , SavedWQMReg ) ; if ( SavedWQMReg ) { LIS -> createAndComputeVirtRegInterval ( SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = StateWQM ; } else


<s> Instructions [ & MI ] . Needs = StateWQM ; II . Needs = StateWQM ; } if ( II . Needs & StateWQM ) { BI . Needs |= StateWQM ; if ( ! ( BI . InNeeds & StateWQM ) ) { BI . InNeeds |= StateWQM ; Worklist . push_back ( MBB ) ; } } if ( MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = ( II . Needs & ~ StateWWM


<s> MachineBasicBlock & MBB = * * BI ; BlockInfo & BBI = Blocks [ & MBB ] ; for ( auto II = MBB . begin ( ) , IE = MBB . end ( ) ; II != IE ; ++ II ) { MachineInstr & MI = * II ; InstrInfo & III = Instructions [ & MI ] ; unsigned Opcode = MI . getOpcode ( ) ; char Flags = <NUM_LIT> ; if ( TII -> isDS ( Opcode ) && CallingConv == CallingConv :: <ISA_LIT>_PS ) { Flags = StateWQM ; } else if ( TII -> isWQM ( Opcode ) ) { markInstructionUses ( MI , StateWQM , Worklist ) ; GlobalFlags |= StateWQM ; continue ; } else if ( Opcode == <STR_LIT> ) { Flags = StateWQM ; LowerToCopyInstrs . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { markInstructionUses ( MI , StateWWM , Worklist ) ; GlobalFlags |= StateWWM ; LowerToCopyInstrs . push_back ( & MI ) ; continue ; } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { III . Disabled = StateWWM ; MachineOperand & Inactive = MI . getOperand ( <NUM_LIT> ) ; if ( Inactive . isReg ( ) ) { if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { unsigned Reg = Inactive . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { for ( MachineInstr & DefMI : MRI -> def_instructions ( Reg ) ) markInstruction ( DefMI


<s> char <ISA_LIT>WholeQuadMode :: scanInstructions ( MachineFunction & MF , std :: vector < WorkItem > & Worklist ) { char GlobalFlags = <NUM_LIT> ; bool WQMOutputs = MF . getFunction ( ) -> hasFnAttribute ( <STR_LIT> ) ; SmallVector < MachineInstr * , <NUM_LIT> > SetInactiveInstrs ; ReversePostOrderTraversal < MachineFunction * > RPOT ( & MF ) ; for ( auto BI = RPOT . begin ( ) , BE = RPOT . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * * BI ; BlockInfo & BBI = Blocks [ & MBB ] ; for ( auto II = MBB . begin ( ) , IE = MBB . end ( ) ; II != IE ; ++ II ) { MachineInstr & MI = * II ; InstrInfo & III = Instructions [ & MI ] ; unsigned Opcode = MI . getOpcode ( ) ; char Flags = <NUM_LIT> ; if ( TII -> isDS ( Opcode ) && CallingConv == CallingConv :: <ISA_LIT>_PS ) { Flags = StateWQM ; } else if ( TII -> isWQM ( Opcode ) ) { markInstructionUses ( MI , StateWQM , Worklist ) ; GlobalFlags |= StateWQM ; continue ; } else if ( Opcode == <STR_LIT>


<s> TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty (


<s> MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerCopyInstrs ( ) ; return


<s> void <ISA_LIT>WholeQuadMode :: fromWWM ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SavedOrig ) { MachineInstr * MI ; assert ( SavedOrig ) ; MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT>


<s> MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , ST -> isWave32 ( )

? <STR_LIT> : <STR_LIT> ) . addReg ( SavedOrig ) ;


<s> AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . setPreservesCFG ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LiveIntervals > ( ) ; AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved <


<s> if ( III -> second . Needs & StateWWM ) Needs = StateWWM ; else if ( III -> second . Needs & StateWQM ) Needs = StateWQM ; else Needs &= ~ III -> second . Disabled ; OutNeeds = III -> second . OutNeeds ; } } else { Needs = StateExact | StateWQM | StateWWM ; } if ( MI . isTerminator ( ) && OutNeeds == StateExact ) Needs = StateExact ; if ( MI . getOpcode ( ) == <STR_LIT> && BI . OutNeeds == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; ++ Next ; } else { if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; SavedNonWWMReg = MRI -> createVirtualRegister ( BoolRC ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( BoolRC ) ; toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB ,


<s> LowerToCopyInstrs . clear ( ) ; CallingConv = MF . getFunction ( ) . getCallingConv ( ) ; ST = & MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs .

empty ( ) ) return ! LiveMaskQueries . empty ( ) ;


<s> } else { unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) .

addReg ( Exec ) . addReg ( LiveMaskReg ) ;


<s> MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , Exec ) . addReg ( SavedWQM ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; } LIS -> InsertMachineInstrInMaps ( *


<s> } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; } LIS -> InsertMachineInstrInMaps (


<s> MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveOrig ) . addImm


<s> void <ISA_LIT>WholeQuadMode :: lowerCopyInstrs ( ) { for ( MachineInstr * MI : LowerToMovInstrs ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; const Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const unsigned SubReg = MI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( TRI -> isVGPR ( * MRI , Reg ) ) { const TargetRegisterClass * regClass = Reg . isVirtual ( ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; if ( SubReg ) regClass = TRI -> getSubRegClass ( regClass , SubReg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; assert ( any_of ( MI -> implicit_operands ( ) , [ ] ( const MachineOperand & MO ) { return MO . isUse ( ) && MO . getReg ( ) == <STR_LIT> ; } ) ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; if ( MI -> getOperand ( <NUM_LIT> ) . isEarlyClobber ( ) ) { LIS -> removeInterval ( Reg ) ; MI -> getOperand ( <NUM_LIT> ) . setIsEarlyClobber ( false ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } int Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; while ( Index >= <NUM_LIT> ) { MI -> removeOperand ( Index ) ; Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> removeOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } else { assert ( MI -> getNumExplicitOperands ( ) ==


<s> Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } MachineInstr * VcmpMI ; const MachineOperand & Op0 = MI . getOperand ( <NUM_LIT> ) ; const MachineOperand & Op1 = MI . getOperand ( <NUM_LIT> ) ; Register VCC = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( TRI -> isVGPR ( * MRI , Op0 . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; VcmpMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( Op1 ) . add ( Op0 ) ; } else { VcmpMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . addReg ( VCC , RegState :: Define ) . addImm ( <NUM_LIT> ) . add ( Op1 ) . addImm ( <NUM_LIT> ) . add ( Op0 ) . addImm ( <NUM_LIT> ) ; } MachineInstr * MaskUpdateMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , LiveMaskReg ) . addReg


<s> XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; AndN2Opc = <STR_LIT> ; XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const char GlobalFlags = analyzeFunction ( MF ) ; const bool NeedsLiveMask = ! ( KillInstrs . empty ( ) && LiveMaskQueries . empty ( ) ) ; LiveMaskReg = Exec ; if ( ! ( GlobalFlags & ( StateWQM | StateStrict ) ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) && KillInstrs . empty ( ) ) { lowerLiveMaskQueries ( ) ; return ! LiveMaskQueries . empty ( ) ; } MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( NeedsLiveMask || ( GlobalFlags & StateWQM ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } LLVM_DEBUG ( printInfo ( ) ) ; lowerLiveMaskQueries ( ) ; lowerCopyInstrs ( ) ; if ( GlobalFlags == StateWQM ) { auto MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( WQMOpc ) , Exec ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; lowerKillInstrs ( true )


<s> } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { III . Disabled = StateStrict ; MachineOperand & Inactive = MI . getOperand ( <NUM_LIT> ) ; if ( Inactive . isReg ( ) ) { if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { markOperand ( MI , Inactive , StateStrictWWM , Worklist ) ; } } SetInactiveInstrs . push_back ( & MI ) ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateStrict ; continue ; } else { if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { KillInstrs . push_back ( & MI ) ; BBI . NeedsLowering = true ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( ! Reg . isVirtual ( ) && TRI -> hasVectorRegisters ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } markInstruction ( MI , Flags , Worklist ) ; GlobalFlags |=


<s> if ( TRI -> isVGPR ( * MRI , Reg ) ) { const TargetRegisterClass * regClass = TargetRegisterInfo :: isVirtualRegister ( Reg ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; MI -> addOperand ( MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; } else


<s> if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag , Worklist ) ; } continue ; } for ( MachineInstr & DefMI : MRI -> def_instructions ( Use . getReg ( ) ) )

markInstruction ( DefMI , Flag , Worklist ) ;


<s> TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; lowerCopyInstrs ( ) ; return true ; } } LLVM_DEBUG ( printInfo ( ) ) ; lowerCopyInstrs ( ) ; for ( auto BII : Blocks ) processBlock ( * BII . first , LiveMaskReg , BII . first == & * MF . begin ( ) ) ; LIS -> removeRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ; return true


<s> do { const VNInfo * Value = ToProcess . pop_back_val ( ) ; Visited . insert ( Value ) ; if ( Value -> isPHIDef ( ) ) { const MachineBasicBlock * MBB = LIS -> getMBBFromIndex ( Value -> def ) ; assert ( MBB && <STR_LIT> ) ; for ( MachineBasicBlock :: const_pred_iterator PI = MBB -> pred_begin ( ) , PE = MBB -> pred_end ( ) ; PI != PE ; ++ PI ) { if ( const VNInfo * VN = LR . getVNInfoBefore ( LIS -> getMBBEndIdx ( * PI ) ) ) { if ( ! Visited . count ( VN ) ) ToProcess . push_back ( VN ) ; } } } else { MachineInstr * MI = LIS -> getInstructionFromIndex ( Value -> def ) ; assert ( MI && <STR_LIT> ) ; markInstruction ( * MI , Flag , Worklist ) ; for ( const MachineOperand & Op : MI -> operands ( ) ) { if ( ! ( Op . isReg ( ) && Op . getReg (


<s> void <ISA_LIT>WholeQuadMode :: markInstruction ( MachineInstr & MI , char Flag , std :: vector < WorkItem > & Worklist ) { InstrInfo & II = Instructions [ & MI ] ; assert ( ! ( Flag & StateExact ) && Flag != <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintState ( Flag ) <<


<s> for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; Register Reg = Use . getReg ( ) ; if ( ! Reg . isVirtual ( ) ) { if ( Reg == <STR_LIT> || Reg == <STR_LIT> ) continue ; for ( MCRegUnitIterator RegUnit ( Reg . asMCReg ( ) , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit ) { LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( MRI -> isSSA ( ) ) { if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag


<s> else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; LIS -> createAndComputeVirtRegInterval ( SavedNonWWMReg ) ; SavedNonWWMReg = <NUM_LIT> ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; assert ( ! SavedNonWWMReg ) ; SavedNonWWMReg = MRI -> createVirtualRegister ( BoolRC ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) { assert ( ! SavedWQMReg ) ; SavedWQMReg = MRI -> createVirtualRegister ( BoolRC ) ; } toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , Before , SavedWQMReg ) ; if ( SavedWQMReg ) { LIS -> createAndComputeVirtRegInterval ( SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = StateWQM ; } else { assert ( Needs & State ) ; } } } if ( Needs != ( StateExact | StateWQM | StateWWM ) ) { if ( Needs != ( StateExact | StateWQM ) ) FirstWQM = IE ; FirstWWM = IE ; } if ( II == IE ) break ; II =


<s> void <ISA_LIT>WholeQuadMode :: fromStrictMode ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , Register SavedOrig , char NonStrictState , char CurrentStrictState ) { MachineInstr * MI ; assert ( SavedOrig ) ; assert ( CurrentStrictState == StateStrictWWM || CurrentStrictState == StateStrictWQM ) ; if ( CurrentStrictState ==


<s> AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addPreserved < MachinePostDominatorTree > ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addPreserved < MachinePostDominatorTree > ( ) ; MachineFunctionPass :: getAnalysisUsage (


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( MBB ) << <STR_LIT> ) ; SmallVector < MachineInstr * , <NUM_LIT> > SplitPoints ; char State = BI . InitialState ; auto II = MBB . getFirstNonPHI ( ) , IE = MBB . end ( ) ; while ( II != IE ) { auto Next = std :: next ( II ) ; MachineInstr & MI = * II ; if ( StateTransition . count ( & MI ) ) State = StateTransition [ & MI ] ; MachineInstr * SplitPoint = nullptr ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : SplitPoint = lowerKillI1 ( MBB , MI , State == StateWQM ) ; break ; case <STR_LIT> : SplitPoint = lowerKillF32 ( MBB


<s> return MO . isUse ( ) && MO . getReg ( ) == <STR_LIT> ; } ) ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; if ( MI -> getOperand ( <NUM_LIT> ) . isEarlyClobber ( ) ) { LIS -> removeInterval ( Reg ) ; MI -> getOperand ( <NUM_LIT> ) . setIsEarlyClobber ( false ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } int Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; while ( Index >= <NUM_LIT> ) { MI -> RemoveOperand ( Index ) ; Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) ==


<s> } int Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; while ( Index >= <NUM_LIT> ) { MI -> RemoveOperand ( Index ) ; Index = MI -> findRegisterUseOperandIdx ( <STR_LIT> ) ; } MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> RemoveOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } else


<s> if ( TRI -> isVGPR ( * MRI , Op0 . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; VcmpMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( Op1 ) . add ( Op0 ) ; } else { VcmpMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . addReg ( <STR_LIT> , RegState :: Define ) . addImm ( <NUM_LIT> ) . add ( Op1 ) . addImm ( <NUM_LIT> ) . add ( Op0 ) . addImm ( <NUM_LIT> ) ; } Register VCC = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; MachineInstr * MaskUpdateMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , LiveMaskReg ) . addReg ( LiveMaskReg ) . addReg ( VCC ) ; MachineInstr * EarlyTermMI = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; MachineInstr * ExecMaskMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , Exec ) . addReg ( Exec ) . addReg ( VCC ) ; assert ( MBB . succ_size ( ) == <NUM_LIT> ) ; MachineInstr * NewTerm = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( * MBB . succ_begin ( ) ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * VcmpMI ) ; MBB . remove (


<s> MBB . remove ( & MI ) ; return NewTerm ; } } else { if ( ! KillVal ) { TmpReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; ComputeKilledMaskMI = BuildMI ( MBB , MI , DL , TII -> get ( XorOpc ) , TmpReg ) . add ( Op ) . addReg ( Exec ) ; MaskUpdateMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , LiveMaskReg ) . addReg ( LiveMaskReg ) . addReg ( TmpReg ) ; } else { MaskUpdateMI = BuildMI ( MBB , MI , DL , TII -> get ( AndN2Opc ) , LiveMaskReg ) . addReg ( LiveMaskReg ) . add ( Op ) ; } } MachineInstr * EarlyTermMI = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; MachineInstr * NewTerm ; MachineInstr * WQMMaskMI = nullptr ; Register LiveMaskWQM ; if ( IsDemote ) { LiveMaskWQM = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; WQMMaskMI = BuildMI ( MBB , MI , DL , TII -> get ( WQMOpc ) , LiveMaskWQM ) . addReg ( LiveMaskReg ) ; NewTerm = BuildMI ( MBB , MI , DL , TII -> get ( AndOpc ) , Exec ) . addReg ( Exec ) . addReg ( LiveMaskWQM ) ; } else { if ( Op . isImm ( ) ) { unsigned MovOpc = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; NewTerm = BuildMI ( MBB , & MI , DL , TII -> get ( MovOpc ) , Exec ) . addImm ( <NUM_LIT> ) ; } else if ( ! IsWQM ) { NewTerm = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , Exec ) . addReg ( Exec ) . addReg ( LiveMaskReg ) ; } else { unsigned Opcode = KillVal ? AndN2Opc : AndOpc ; NewTerm = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) , Exec ) . addReg ( Exec ) . add ( Op ) ; } } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MBB . remove ( & MI ) ; assert ( EarlyTermMI ) ; assert ( MaskUpdateMI ) ; assert ( NewTerm ) ; if ( ComputeKilledMaskMI ) LIS -> InsertMachineInstrInMaps ( * ComputeKilledMaskMI ) ; LIS -> InsertMachineInstrInMaps ( * MaskUpdateMI


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * SplitPoint = nullptr ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : SplitPoint = lowerKillI1 ( * MBB , * MI


<s> if ( const VNInfo * VN = LR . getVNInfoBefore ( LIS -> getMBBEndIdx ( * PI ) ) ) { if ( ! Visited . count ( VisitKey ( VN , DefinedLanes ) ) ) NextValue = VN ; } } if ( PI != PE ) PhiStack . emplace_back ( Value , Idx , DefinedLanes ) ; } else { MachineInstr * MI = LIS -> getInstructionFromIndex ( Value -> def ) ; assert ( MI && <STR_LIT> ) ; if ( Reg . isVirtual ( ) ) { bool HasDef = false ; for ( const MachineOperand & Op : MI -> operands ( ) ) { if ( ! ( Op . isReg ( ) && Op . isDef ( ) && Op . getReg ( ) == Reg ) ) continue ; LaneBitmask OpLanes = Op . isUndef ( ) ? LaneBitmask :: getAll ( ) : TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) ; LaneBitmask Overlap = ( UseLanes & OpLanes ) ; HasDef |= Overlap . any ( ) ; DefinedLanes |= OpLanes ; } if ( ( DefinedLanes & UseLanes ) != UseLanes ) { LiveQueryResult LRQ = LR . Query ( LIS -> getInstructionIndex ( * MI ) ) ; if ( const VNInfo * VN = LRQ . valueIn ( ) ) { if ( ! Visited . count ( VisitKey ( VN , DefinedLanes ) ) ) NextValue = VN ; } } if ( HasDef ) markInstruction ( * MI , Flag , Worklist ) ; } else { markInstruction ( * MI , Flag , Worklist ) ; } } if ( ! NextValue && ! PhiStack


<s> assert ( ! ( Flag & StateExact ) && Flag != <NUM_LIT> ) ; Flag &= ~ II . Disabled ; if ( ( II . Needs & Flag ) == Flag ) return ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintState ( Flag ) << <STR_LIT> << MI ) ; II . Needs |=


<s> if ( ( II . Needs & Flag ) == Flag ) return ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintState ( Flag ) << <STR_LIT> << MI ) ; II . Needs |= Flag ; Worklist . push_back ( & MI


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintState ( Flag ) << <STR_LIT> << Op << <STR_LIT> << MI ) ; if ( Reg . isVirtual ( ) ) { LiveRange & LR = LIS -> getInterval ( Reg ) ; markDefs ( MI , LR , Reg , Op . getSubReg ( ) , Flag , Worklist ) ; } else { for ( MCRegUnitIterator RegUnit ( Reg . asMCReg ( ) , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit )


<s> LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } LLVM_DEBUG ( printInfo ( ) ) ; lowerLiveMaskQueries ( ) ; lowerCopyInstrs ( ) ; if ( GlobalFlags == StateWQM ) { auto MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( WQMOpc ) , Exec ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; lowerKillInstrs ( true ) ; } else { for ( auto BII : Blocks ) processBlock ( * BII . first , BII . first == & Entry ) ; for ( auto BII : Blocks ) lowerBlock ( * BII . first ) ; } if ( LiveMaskReg != Exec ) LIS -> createAndComputeVirtRegInterval ( LiveMaskReg ) ; LIS -> removeRegUnit ( * MCRegUnitIterator ( MCRegister :: from ( <STR_LIT> ) , TRI ) ) ; if ( ! KillInstrs . empty ( ) ) LIS -> removeRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ; return true


<s> KillInstrs . clear ( ) ; StateTransition . clear ( ) ; ST = & MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; if ( ST -> isWave32 ( ) ) { AndOpc = <STR_LIT> ; AndN2Opc = <STR_LIT> ; XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; AndN2Opc = <STR_LIT> ; XorOpc = <STR_LIT> ; AndSaveExecOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; WQMOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const char GlobalFlags = analyzeFunction ( MF ) ; const bool NeedsLiveMask = ! ( KillInstrs . empty ( ) && LiveMaskQueries . empty ( ) ) ; LiveMaskReg = Exec ; if ( ! ( GlobalFlags & ( StateWQM | StateStrict ) ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) && KillInstrs . empty ( ) ) { lowerLiveMaskQueries (


<s> MachineInstr * Save = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveReg ) . addReg ( <STR_LIT> ) ; MachineInstr * Restore = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SaveReg ) ; LIS -> InsertMachineInstrInMaps ( * Save ) ; LIS -> InsertMachineInstrInMaps ( *


<s> case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; default : break ; } if ( NewOpcode ) TermMI -> setDesc ( TII -> get ( NewOpcode ) ) ; if ( SplitBB != BB ) { using DomTreeT = DomTreeBase < MachineBasicBlock > ; SmallVector < DomTreeT :: UpdateType , <NUM_LIT> > DTUpdates ; for ( MachineBasicBlock * Succ : SplitBB -> successors ( ) ) { DTUpdates . push_back ( { DomTreeT :: Insert , SplitBB , Succ } ) ; DTUpdates . push_back ( { DomTreeT :: Delete , BB , Succ } ) ; } DTUpdates . push_back ( { DomTreeT :: Insert , BB , SplitBB } ) ; if ( MDT ) MDT -> getBase ( ) . applyUpdates ( DTUpdates ) ; if ( PDT ) PDT -> getBase ( ) . applyUpdates ( DTUpdates ) ; MachineInstr * MI = BuildMI ( * BB , BB -> end ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( SplitBB ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } return


<s> } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( AndOpc ) , Exec ) . addReg ( Exec ) . addReg ( LiveMaskReg ) ; } LIS -> InsertMachineInstrInMaps ( * MI ) ; StateTransition [ MI


<s> void <ISA_LIT>WholeQuadMode :: toStrictMode ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , Register SaveOrig , char StrictStateNeeded ) { MachineInstr * MI ; assert ( SaveOrig ) ; assert ( StrictStateNeeded == StateStrictWWM || StrictStateNeeded == StateStrictWQM ) ; if ( StrictStateNeeded == StateStrictWWM ) { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveOrig ) . addImm ( - <NUM_LIT> ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveOrig ) . addImm ( - <NUM_LIT> ) ; } LIS -> InsertMachineInstrInMaps ( * MI ) ; StateTransition [ MI


<s> MachineInstr * MI ; if ( SavedWQM ) { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , Exec ) . addReg ( SavedWQM ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( WQMOpc ) , Exec ) . addReg ( Exec ) ; } LIS -> InsertMachineInstrInMaps ( * MI ) ; StateTransition [ MI ]


<s> for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; assert ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) ; MI -> RemoveOperand ( <NUM_LIT> ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } else { assert ( MI -> getNumExplicitOperands


<s> void <ISA_LIT>WholeQuadMode :: lowerCopyInstrs ( ) { for ( MachineInstr * MI : LowerToMovInstrs ) { assert ( MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) ; const Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TRI -> isVGPR ( * MRI , Reg ) ) { const TargetRegisterClass * regClass = Register :: isVirtualRegister ( Reg ) ? MRI -> getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; const unsigned MovOp = TII -> getMovOpcode ( regClass ) ; MI -> setDesc ( TII -> get ( MovOp ) ) ; MI -> addOperand ( MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; } else { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; } } for ( MachineInstr * MI : LowerToCopyInstrs ) { if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { assert ( MI -> getNumExplicitOperands (


<s> for ( ; ; ) { MachineBasicBlock :: iterator Next = II ; char Needs = StateExact | StateWQM ; char OutNeeds = <NUM_LIT> ; if ( FirstWQM == IE ) FirstWQM = II ; if ( FirstWWM == IE ) FirstWWM = II ; if ( II != IE ) { MachineInstr & MI = * II ; if ( MI . isTerminator ( ) || TII -> mayReadEXEC ( * MRI , MI ) ) { auto III = Instructions . find ( & MI ) ; if ( III != Instructions . end ( ) ) { if ( III -> second . Needs & StateWWM ) Needs = StateWWM ; else if ( III -> second . Needs & StateWQM ) Needs = StateWQM ; else Needs &= ~ III -> second . Disabled ; OutNeeds = III -> second . OutNeeds ; } } else { Needs = StateExact | StateWQM | StateWWM ; } if ( MI . isTerminator ( ) && OutNeeds == StateExact ) Needs = StateExact ; if ( MI . getOpcode ( ) == <STR_LIT> && BI . OutNeeds == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; ++ Next ; } else { if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM


<s> LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( Exec ) ; lowerCopyInstrs ( ) ; return true ; } } LLVM_DEBUG ( printInfo ( ) ) ; lowerCopyInstrs ( ) ; for ( auto BII : Blocks ) processBlock ( * BII . first , LiveMaskReg , BII . first == & * MF . begin ( ) ) ; LIS -> removeRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI )


<s> char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; unsigned Exec = ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT>

: <STR_LIT> ) , Exec ) . addReg ( Exec ) ;


<s> const char * getPassName ( ) const override { return <STR_LIT>


<s> const char * getPassName ( ) const override { return


<s> unsigned Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) ,


<s> void <ISA_LIT>WholeQuadMode :: markInstruction ( MachineInstr & MI , char Flag , std :: vector < WorkItem > & Worklist ) { InstrInfo & II = Instructions [ & MI ] ; assert ( Flag == StateWQM || Flag == StateExact ) ; if ( II . Needs ) return ; II . Needs = Flag ; Worklist . push_back ( &


<s> InstrInfo & II = Instructions [ & MI ] ; assert ( Flag == StateWQM || Flag == StateExact ) ; if ( II . Needs ) return ; II . Needs =


<s> } char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; auto InstrInfoIt = Instructions . find ( & MI ) ; if ( InstrInfoIt != Instructions . end ( ) ) { Needs = InstrInfoIt -> second . Needs ; OutNeeds = InstrInfoIt -> second . OutNeeds ; if ( OutNeeds == StateExact && MI . isTerminator ( ) ) { assert ( Needs == <NUM_LIT> ) ; Needs = StateExact ; } } if ( Needs && State != Needs ) { if ( Needs == StateExact ) { assert ( ! SavedWQMReg ) ; if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , & MI , SavedWQMReg , LiveMaskReg ) ; } else { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , & MI , SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = Needs ; } } if ( ( BI . OutNeeds & StateWQM ) && State != StateWQM ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , MBB . end ( )


<s> } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue


<s> BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SavedWQM ) ; } else { BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT>

) , <STR_LIT> ) . addReg ( <STR_LIT> ) ;


<s> dbgs ( ) << <STR_LIT> << BII . first -> getNumber ( ) << <STR_LIT> << <STR_LIT> << PrintState ( BII . second . InNeeds ) << <STR_LIT> << PrintState ( BII . second . Needs ) << <STR_LIT> << PrintState ( BII . second . OutNeeds ) << <STR_LIT> ; for ( const MachineInstr & MI : * BII . first ) { auto III = Instructions . find ( & MI ) ; if ( III == Instructions . end ( ) ) continue ; dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << PrintState ( III -> second . Needs ) << <STR_LIT> << PrintState ( III -> second .


<s> LLVM_DUMP_METHOD void <ISA_LIT>WholeQuadMode :: printInfo ( ) { for ( const auto & BII : Blocks ) { dbgs ( ) << <STR_LIT> << BII . first -> getNumber ( ) << <STR_LIT> << <STR_LIT> << PrintState ( BII . second . InNeeds ) << <STR_LIT> << PrintState ( BII . second . Needs ) << <STR_LIT> << PrintState ( BII . second . OutNeeds ) << <STR_LIT> ; for ( const MachineInstr & MI : * BII . first ) { auto III = Instructions . find ( &


<s> unsigned Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; MI -> eraseFromParent


<s> char State = isEntry ? StateExact : StateWQM ; auto II = MBB . getFirstNonPHI ( ) , IE = MBB . end ( ) ; while ( II != IE ) { MachineInstr & MI = * II ; ++ II ; if ( MI . getDesc ( ) . TSFlags & ( <STR_LIT> | <STR_LIT> ) && ! MI . isBranch ( ) && ! MI . isTerminator ( ) ) continue ; if ( TargetInstrInfo :: isGenericOpcode ( MI . getOpcode ( ) ) ) { if ( MI . getNumExplicitOperands ( ) >= <NUM_LIT> ) { const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; if ( Op . isReg ( ) ) { if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { continue ; } } } } char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; auto InstrInfoIt = Instructions . find ( & MI ) ; if ( InstrInfoIt != Instructions . end ( ) ) { Needs = InstrInfoIt -> second . Needs ; OutNeeds = InstrInfoIt -> second .


<s> } for ( const MachineBasicBlock * Pred : MBB . predecessors ( ) ) { BlockInfo & PredBI = Blocks [ Pred ] ; if ( ( PredBI . OutNeeds | BI . InNeeds ) == PredBI . OutNeeds ) continue ; PredBI . OutNeeds |= BI . InNeeds ; PredBI . InNeeds |= BI . InNeeds ; Worklist . push_back ( Pred ) ; } for ( const MachineBasicBlock * Succ : MBB . successors ( ) ) { BlockInfo & SuccBI = Blocks [ Succ ] ; if ( ( SuccBI . InNeeds | BI .

OutNeeds ) == SuccBI . InNeeds ) continue ;


<s> void <ISA_LIT>WholeQuadMode :: propagateBlock ( const MachineBasicBlock & MBB , std :: vector < WorkItem > & Worklist ) { BlockInfo BI = Blocks [ & MBB ] ; if ( ! MBB . empty ( ) ) { const MachineInstr * LastMI = & * MBB . rbegin ( ) ; InstrInfo & LastII = Instructions [ LastMI ] ; if ( ( LastII . OutNeeds | BI . OutNeeds ) != LastII . OutNeeds ) { LastII . OutNeeds |= BI


<s> if ( ( II . OutNeeds & StateWQM ) && ! ( II . Needs & StateWQM ) && ( MI . isBranch ( ) || MI . isTerminator ( ) || MI . getOpcode ( ) == <STR_LIT> ) ) { Instructions [ & MI ] . Needs = StateWQM ; II . Needs = StateWQM ; } BI . Needs |= II . Needs ; if ( ( BI . InNeeds | II . Needs ) != BI . InNeeds ) { BI . InNeeds |= II . Needs ; Worklist . push_back ( & MBB ) ; } if ( const MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = II . Needs | II . OutNeeds ; if ( ! PrevMI -> isPHI ( ) ) { InstrInfo & PrevII = Instructions [ PrevMI ] ; if ( ( PrevII . OutNeeds | InNeeds ) != PrevII . OutNeeds ) { PrevII . OutNeeds |= InNeeds ; Worklist . push_back ( PrevMI ) ; } } } assert ( II . Needs != ( StateWQM | StateExact ) ) ; if ( II . Needs != StateWQM ) return ; for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Use . getReg ( ) ) ) continue ; for ( const MachineOperand & Def : MRI -> def_operands ( Use . getReg ( ) ) ) { const MachineInstr * DefMI = Def . getParent ( ) ; InstrInfo & DefII = Instructions [ DefMI ] ; if ( DefII . Needs != <NUM_LIT> ) continue ; DefII . Needs = StateWQM ; Worklist . push_back ( DefMI


<s> Instructions . clear ( ) ; Blocks . clear ( ) ; ExecExports . clear ( ) ; LiveMaskQueries . clear ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; return ! LiveMaskQueries . empty ( ) ; } unsigned LiveMaskReg = <NUM_LIT> ; { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; } if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerLiveMaskQueries ( LiveMaskReg ) ; return


<s> TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; return ! LiveMaskQueries . empty ( ) ; } unsigned LiveMaskReg = <NUM_LIT> ; { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; } if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerLiveMaskQueries ( LiveMaskReg ) ; return true ; } } lowerLiveMaskQueries ( LiveMaskReg ) ; for ( const auto & BII : Blocks ) processBlock ( const_cast < MachineBasicBlock & > ( * BII . first ) , LiveMaskReg , BII . first == & * MF . begin ( ) ) ; return


<s> for ( auto BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( auto II = MBB . begin ( ) , IE = MBB . end ( ) ; II != IE ; ++ II ) { MachineInstr & MI = * II ; unsigned Opcode = MI . getOpcode ( ) ; char Flags = <NUM_LIT> ; if ( TII -> isWQM ( Opcode ) || TII -> isDS ( Opcode ) ) { Flags = StateWQM ; } else if ( TII -> get ( Opcode ) . mayStore ( ) && ( MI . getDesc ( ) . TSFlags & <STR_LIT> ) ) { Flags = StateExact ; } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } Instructions [ & MI ] . Needs = Flags ; Worklist . push_back ( & MI ) ; GlobalFlags |= Flags ; } if ( WQMOutputs && MBB . succ_empty ( ) ) { Blocks [ & MBB ] . OutNeeds = StateExact ; Worklist . push_back ( & MBB ) ; GlobalFlags |= StateExact ; } } return GlobalFlags


<s> } else { if ( Opcode == <STR_LIT> ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } Instructions [ & MI ] . Needs = Flags ; Worklist . push_back ( & MI ) ; GlobalFlags |= Flags ; } if ( WQMOutputs && MBB . succ_empty ( ) ) { Blocks [ & MBB ] . OutNeeds = StateExact ; Worklist . push_back (


<s> if ( BII == Blocks . end ( ) ) return ; const BlockInfo & BI = BII -> second ; if ( ! ( BI . InNeeds & StateWQM ) ) return ; if ( ! isEntry && ! ( BI . Needs & StateExact ) && BI . OutNeeds != StateExact ) return ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB . getNumber ( ) << <STR_LIT> ) ; unsigned SavedWQMReg = <NUM_LIT> ; bool WQMFromExec = isEntry ; char State = isEntry ? StateExact : StateWQM ; auto II = MBB . getFirstNonPHI ( ) , IE = MBB . end ( ) ; if ( isEntry ) ++ II ; MachineBasicBlock :: iterator First = IE ; for ( ; ; ) { MachineBasicBlock :: iterator Next = II ; char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; if ( First == IE ) First = II ; if ( II != IE ) { MachineInstr & MI = * II ; if ( requiresCorrectState ( MI ) ) { auto III = Instructions . find (


<s> TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; return ! LiveMaskQueries . empty ( ) ; } unsigned LiveMaskReg = <NUM_LIT> ; { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT>

) , LiveMaskReg ) . addReg ( <STR_LIT> ) ;


<s> while ( II != IE ) { MachineInstr & MI = * II ; ++ II ; if ( TII -> isScalarUnit ( MI ) && ! MI . isTerminator ( ) ) continue ; if ( TargetInstrInfo :: isGenericOpcode ( MI . getOpcode ( ) ) ) { if ( MI . getNumExplicitOperands ( ) >= <NUM_LIT> ) { const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; if ( Op . isReg ( ) ) { if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { continue ; } } } } char Needs = <NUM_LIT> ; char OutNeeds = <NUM_LIT> ; auto InstrInfoIt = Instructions . find ( & MI ) ; if ( InstrInfoIt != Instructions . end ( ) ) { Needs = InstrInfoIt -> second . Needs ; OutNeeds = InstrInfoIt -> second . OutNeeds ; if ( OutNeeds == StateExact && MI . isTerminator ( ) ) { assert ( Needs == <NUM_LIT> ) ; Needs = StateExact ; } } if ( Needs && State != Needs ) { if ( Needs == StateExact ) { assert ( ! SavedWQMReg ) ; if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , & MI , SavedWQMReg , LiveMaskReg ) ; } else { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , & MI , SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = Needs ; } if ( MI . getOpcode ( ) == <STR_LIT> && State == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; } if ( ( BI . OutNeeds & StateWQM ) && State != StateWQM ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , MBB . end ( ) , SavedWQMReg ) ; } else if ( BI . OutNeeds == StateExact && State != StateExact ) { toExact ( MBB , MBB . end


<s> if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; Register Reg = Use . getReg ( ) ; if ( ! Reg . isVirtual ( ) ) { if ( Reg == <STR_LIT> || Reg == <STR_LIT> ) continue ; for ( MCRegUnitIterator RegUnit ( Reg , TRI ) ; RegUnit . isValid ( ) ; ++ RegUnit ) { LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value


<s> lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg


<s> if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { unsigned Reg = Inactive . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { for ( MachineInstr & DefMI : MRI -> def_instructions ( Reg ) ) markInstruction ( DefMI , StateWWM , Worklist ) ; } } } SetInactiveInstrs . push_back ( & MI ) ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( & MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateWWM ; continue ; } else { if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVectorRegisters ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags = StateWQM ; break ; } } } if ( ! Flags ) continue ; } markInstruction ( MI , Flags , Worklist ) ; GlobalFlags |= Flags ; } } if ( GlobalFlags & StateWQM


<s> } BI . Needs |= II . Needs ; if ( ( BI . InNeeds | II . Needs ) != BI . InNeeds ) { BI . InNeeds |= II . Needs ; Worklist . push_back ( MBB ) ; } if ( MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = II . Needs | II . OutNeeds ; if ( ! PrevMI -> isPHI ( ) ) { InstrInfo & PrevII = Instructions [ PrevMI ] ; if ( ( PrevII . OutNeeds | InNeeds ) != PrevII . OutNeeds ) { PrevII . OutNeeds |= InNeeds ; Worklist . push_back ( PrevMI ) ; } } } assert ( II . Needs != ( StateWQM | StateExact ) ) ; if ( II . Needs != StateWQM ) return ; for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use . isUse ( ) ) continue ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Use . getReg ( ) ) ) continue ; for ( MachineInstr & DefMI : MRI -> def_instructions ( Use . getReg ( ) ) ) { InstrInfo & DefII = Instructions [ & DefMI ] ; if ( DefII . Needs != <NUM_LIT> ) continue ; DefII . Needs = StateWQM ; Worklist . push_back (


<s> void <ISA_LIT>WholeQuadMode :: propagateInstruction ( MachineInstr & MI , std :: vector < WorkItem > & Worklist ) { MachineBasicBlock * MBB = MI . getParent ( ) ; InstrInfo II = Instructions [ & MI ] ; BlockInfo & BI = Blocks [ MBB ] ; if ( ( II . OutNeeds & StateWQM ) && ! ( II . Needs & StateWQM ) && MI . isTerminator ( ) ) { Instructions [ & MI ] . Needs = StateWQM ; II . Needs = StateWQM ; } BI . Needs |= II . Needs ; if ( ( BI . InNeeds | II . Needs ) != BI . InNeeds ) { BI . InNeeds |= II . Needs ; Worklist . push_back ( MBB ) ; } if ( MachineInstr * PrevMI = MI . getPrevNode ( ) ) { char InNeeds = II . Needs | II . OutNeeds ; if ( ! PrevMI -> isPHI ( ) ) { InstrInfo & PrevII = Instructions [ PrevMI ] ; if ( ( PrevII . OutNeeds | InNeeds ) != PrevII . OutNeeds ) { PrevII . OutNeeds |= InNeeds ; Worklist . push_back ( PrevMI ) ; } } } assert ( II . Needs != ( StateWQM | StateExact ) ) ; if ( II . Needs != StateWQM ) return ; for ( const MachineOperand & Use : MI . uses ( ) ) { if ( ! Use . isReg ( ) || ! Use


<s> Idx = Next ; } } MachineBasicBlock :: iterator MBBI ; if ( MachineInstr * MI = LIS -> getInstructionFromIndex ( Idx ) ) MBBI = MI ; else { assert ( Idx == LIS -> getMBBEndIdx ( & MBB ) ) ; MBBI = MBB . end ( ) ; } while ( MBBI != Last ) { bool IsExecDef = false ; for ( const MachineOperand & MO : MBBI -> operands ( ) ) { if ( MO . isReg ( ) && MO . isDef ( ) ) { IsExecDef |= MO . getReg ( ) == <STR_LIT> || MO . getReg ( ) == <STR_LIT> ; } } if ( ! IsExecDef ) break ; MBBI ++ ; S = nullptr


<s> Exec = <STR_LIT> ; } char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( Exec ) ; if ( ! ( GlobalFlags & StateWWM ) && LowerToCopyInstrs . empty ( ) && LowerToMovInstrs . empty ( ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( TRI -> getBoolRC ( ) ) ; MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( Exec ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { auto MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( ST -> isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg


<s> MachineInstr * MI = BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , LiveMaskReg ) . addReg ( <STR_LIT> ) ; LIS -> InsertMachineInstrInMaps ( * MI ) ; } lowerLiveMaskQueries ( LiveMaskReg ) ; if ( GlobalFlags == StateWQM ) { BuildMI ( Entry , EntryMI , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; lowerCopyInstrs ( ) ; return true ; } } DEBUG ( printInfo ( ) ) ; lowerCopyInstrs ( ) ; for ( auto BII : Blocks ) processBlock ( * BII . first , LiveMaskReg , BII . first == & * MF


<s> if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ExecExports . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { LiveMaskQueries . push_back ( & MI ) ; } else if ( WQMOutputs ) { for ( const MachineOperand & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TRI -> isVirtualRegister ( Reg ) && TRI -> hasVGPRs ( TRI -> getPhysRegClass ( Reg ) ) ) { Flags =


<s> MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg


<s> AU . addRequired < LiveIntervals > ( ) ; AU . setPreservesCFG (


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LiveIntervals >


<s> for ( MachineInstr * MI : LowerToCopyInstrs ) { for ( unsigned i = MI -> getNumExplicitOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; i -- ) MI


<s> unsigned Dest = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Copy = BuildMI ( * MI -> getParent ( ) , MI , DL , TII -> get ( <STR_LIT> ) , Dest ) . addReg ( LiveMaskReg ) ; LIS -> ReplaceMachineInstrInMaps ( * MI , * Copy ) ; MI -> eraseFromParent


<s> LiveRange & LR = LIS -> getRegUnit ( * RegUnit ) ; const VNInfo * Value = LR . Query ( LIS -> getInstructionIndex ( MI ) ) . valueIn ( ) ; if ( ! Value ) continue ; if ( Value -> isPHIDef ( ) ) continue ; markInstruction ( * LIS -> getInstructionFromIndex ( Value -> def ) , Flag , Worklist ) ; } continue ; } for ( MachineInstr & DefMI : MRI -> def_instructions ( Use . getReg ( ) ) ) markInstruction (


<s> for ( ; ; ) { S = LR . getSegmentContaining ( Idx ) ; if ( ! S ) break ; if ( PreferLast ) { SlotIndex Next = S -> start . getBaseIndex ( ) ; if ( Next < FirstIdx ) break ; Idx = Next ; } else { SlotIndex Next = S -> end . getNextIndex ( ) . getBaseIndex ( ) ; if ( Next > LastIdx ) break ; Idx = Next ; } } MachineBasicBlock :: iterator MBBI ; if ( MachineInstr * MI = LIS -> getInstructionFromIndex ( Idx ) ) MBBI = MI ; else { assert ( Idx == LIS -> getMBBEndIdx ( & MBB ) ) ; MBBI = MBB . end ( ) ; } if ( S ) MBBI = saveSCC ( MBB , MBBI ) ; return


<s> auto III = Instructions . find ( & MI ) ; if ( III == Instructions . end ( ) ) continue ; dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << PrintState ( III -> second . Needs ) << <STR_LIT> << PrintState


<s> } if ( MI . isTerminator ( ) && OutNeeds == StateExact ) Needs = StateExact ; if ( MI . getOpcode ( ) == <STR_LIT> && BI . OutNeeds == StateExact ) MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; ++ Next ; } else { if ( BI . OutNeeds & StateWQM ) Needs = StateWQM ; else if ( BI . OutNeeds == StateExact ) Needs = StateExact ; else Needs = StateWQM | StateExact ; } if ( ! ( Needs & State ) ) { MachineBasicBlock :: iterator First ; if ( State == StateWWM || Needs == StateWWM ) { First = FirstWWM ; } else { First = FirstWQM ; } MachineBasicBlock :: iterator Before = prepareInsertion ( MBB , First , II , Needs == StateWQM , Needs == StateExact || WQMFromExec ) ; if ( State == StateWWM ) { assert ( SavedNonWWMReg ) ; fromWWM ( MBB , Before , SavedNonWWMReg ) ; State = NonWWMState ; } if ( Needs == StateWWM ) { NonWWMState = State ; SavedNonWWMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toWWM ( MBB , Before , SavedNonWWMReg ) ; State = StateWWM ; } else { if ( State == StateWQM && ( Needs & StateExact ) && ! ( Needs & StateWQM ) ) { if ( ! WQMFromExec && ( OutNeeds & StateWQM ) ) SavedWQMReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; toExact ( MBB , Before , SavedWQMReg , LiveMaskReg ) ; State = StateExact ; } else if ( State == StateExact && ( Needs & StateWQM ) && ! ( Needs & StateExact ) ) { assert ( WQMFromExec == ( SavedWQMReg == <NUM_LIT> ) ) ; toWQM ( MBB , Before , SavedWQMReg ) ; if ( SavedWQMReg ) { LIS -> createAndComputeVirtRegInterval ( SavedWQMReg ) ; SavedWQMReg = <NUM_LIT> ; } State = StateWQM ; } else { assert ( Needs & State


<s> bool <ISA_LIT>WholeQuadMode :: requiresCorrectState ( const MachineInstr & MI ) const { if ( MI . isTerminator ( ) ) return true ; if ( TII -> isScalarUnit ( MI ) ) return false ; if ( MI . isTransient ( ) ) { if ( MI . getNumExplicitOperands ( ) >= <NUM_LIT> ) { const MachineOperand & Op = MI


<s> Instructions . clear ( ) ; Blocks . clear ( ) ; LiveMaskQueries . clear ( ) ; LowerToCopyInstrs . clear ( ) ; CallingConv = MF . getFunction ( ) . getCallingConv ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; char GlobalFlags = analyzeFunction ( MF ) ; unsigned LiveMaskReg = <NUM_LIT> ; if ( ! ( GlobalFlags & StateWQM ) ) { lowerLiveMaskQueries ( <STR_LIT> ) ; if ( ! ( GlobalFlags & StateWWM ) ) return ! LiveMaskQueries . empty ( ) ; } else { MachineBasicBlock & Entry = MF . front ( ) ; MachineBasicBlock :: iterator EntryMI = Entry . getFirstNonPHI ( ) ; if ( GlobalFlags & StateExact || ! LiveMaskQueries . empty ( ) ) { LiveMaskReg = MRI -> createVirtualRegister ( & <STR_LIT>


<s> Flags = StateWQM ; LowerToCopyInstrs . push_back ( & MI ) ; } else if ( Opcode == <STR_LIT> ) { markInstructionUses ( MI , StateWWM , Worklist ) ; GlobalFlags |= StateWWM ; LowerToCopyInstrs . push_back ( & MI ) ; continue ; } else if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { III . Disabled = StateWWM ; MachineOperand & Inactive = MI . getOperand ( <NUM_LIT> ) ; if ( Inactive . isReg ( ) ) { if ( Inactive . isUndef ( ) ) { LowerToCopyInstrs . push_back ( & MI ) ; } else { unsigned Reg = Inactive . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { for ( MachineInstr & DefMI : MRI -> def_instructions ( Reg ) ) markInstruction ( DefMI , StateWWM , Worklist ) ; } } } SetInactiveInstrs . push_back ( & MI ) ; continue ; } else if ( TII -> isDisableWQM ( MI ) ) { BBI . Needs |= StateExact ; if ( ! ( BBI . InNeeds & StateExact ) ) { BBI . InNeeds |= StateExact ; Worklist . push_back ( & MBB ) ; } GlobalFlags |= StateExact ; III . Disabled = StateWQM | StateWWM ; continue ; }


<s> void <ISA_LIT>WholeQuadMode :: toExact ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SaveWQM , unsigned LiveMaskReg ) { MachineInstr * MI ; if ( SaveWQM ) { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SaveWQM ) . addReg ( LiveMaskReg ) ; } else { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( LiveMaskReg ) ; } LIS -> InsertMachineInstrInMaps (


<s> void <ISA_LIT>WholeQuadMode :: toExact ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator Before , unsigned SaveWQM , unsigned LiveMaskReg ) { MachineInstr * MI ; if ( SaveWQM ) { MI = BuildMI ( MBB , Before , DebugLoc ( ) , TII -> get ( <STR_LIT>

) , SaveWQM ) . addReg ( LiveMaskReg ) ;