<s> <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool UseKill = IsKill && i == NumSubRegs - <NUM_LIT> ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; auto MIB = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( UseKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; if ( i == <NUM_LIT> && NumSubRegs > <NUM_LIT> ) { MIB . addReg ( SuperReg , RegState :: ImplicitDefine ) ; } if ( NumSubRegs > <NUM_LIT> ) MIB . addReg ( SuperReg , getKillRegState ( UseKill ) | RegState :: Implicit ) ; } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; MachineInstrBuilder WriteLane = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PerVGPR ) . addReg ( TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ;

if ( i + <NUM_LIT> == NumSubRegs ) SuperKillState |= getKillRegState ( IsKill ) ;


<s> return SRC ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ;


<s> bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; Register SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned M0CopyReg = <STR_LIT> ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; Register TmpVGPR ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToVGPR ) { <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; auto MIB = BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) ;

if ( NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SuperReg , RegState :: ImplicitDefine ) ;


<s> DenseSet < unsigned > SGPRSpillVGPRDefinedSet ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > VGPRSpills = MFI -> getSGPRToVGPRSpills ( Index ) ; bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; Register SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; assert ( SpillToVGPR || ( SuperReg != MFI -> getStackPtrOffsetReg ( ) && SuperReg != MFI -> getFrameOffsetReg ( ) && SuperReg != MFI -> getScratchWaveOffsetReg ( ) ) ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned M0CopyReg = <STR_LIT> ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; Register TmpVGPR ;

unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ;


<s> unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( LIS ) { if ( i == <NUM_LIT> ) LIS -> ReplaceMachineInstrInMaps ( * MI , * WriteLane ) ; else LIS -> InsertMachineInstrInMaps ( * WriteLane ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == SB . NumSubRegs ) SuperKillState |= getKillRegState ( SB . IsKill ) ; WriteLane . addReg ( SB . SuperReg , RegState :: Implicit | SuperKillState ) ; } } SB . readWriteTmpVGPR ( Offset , false ) ; } SB . restore ( ) ; } MI -> eraseFromParent ( ) ;

SB . MFI . addToSpilledSGPRs ( SB . NumSubRegs ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ;

if ( ScratchWaveOffsetReg != <STR_LIT> ) {


<s> switch ( Size ) { case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ;


<s> bool Scavenged = false ; unsigned SOffset = ScratchOffset ; unsigned OriginalImmOffset = Offset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = <STR_LIT> ; if ( RS ) SOffset = RS -> FindUnusedReg ( & <STR_LIT> ) ; if ( SOffset == <STR_LIT> ) { RanOutOfSGPRs = true ; SOffset = ScratchOffset ; } else { Scavenged = true ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) { unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg ( Value , & <STR_LIT> , i ) : Value ; unsigned SOffsetRegState = <NUM_LIT> ; unsigned SrcDstRegState = getDefRegState ( ! IsStore ) ; if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; }

BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . addReg ( SubReg , getDefRegState ( ! IsStore ) ) . addReg ( ScratchRsrcReg ) . addReg ( SOffset , SOffsetRegState ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( Value , RegState :: Implicit | SrcDstRegState ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ;


<s> DebugLoc DL ; if ( Ins != MBB -> end ( ) ) DL = Ins -> getDebugLoc ( ) ; MachineFunction * MF = MBB -> getParent ( ) ; const <ISA_LIT>Subtarget & Subtarget = MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; const TargetInstrInfo * TII = Subtarget . getInstrInfo ( ) ; if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , Ins , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addFrameIndex ( FrameIdx ) ; return ; } MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; unsigned UnusedCarry = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Ins , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addReg ( UnusedCarry , RegState :: Define | RegState :: Dead ) . addImm ( Offset ) . addFrameIndex ( FrameIdx ) ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; const MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF -> getSubtarget ( ) . getInstrInfo ( ) ) ; LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool IsLoad = TII -> get ( LoadStoreOp ) . mayLoad ( ) ; bool RanOutOfSGPRs = false ; unsigned SOffset = ScratchOffset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ;


<s> unsigned SubReg = getPhysRegSubReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , & <STR_LIT> , i ) ; struct <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = MFI -> getSpilledReg ( MF , Index , i ) ; if ( Spill . VGPR == <STR_LIT> ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; } BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) . addReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , RegState :: ImplicitDefine ) ; } switch ( MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) . getGeneration ( ) ) { case <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS : TII -> insertNOPs ( MI , <NUM_LIT> ) ; break ; case <ISA_LIT>Subtarget :: SEA_ISLANDS : break ; default : TII -> insertNOPs ( MI , <NUM_LIT> ) ; } MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : buildScratchLoadStore ( MI , <STR_LIT> , TII -> getNamedOperand ( * MI , <STR_LIT> :: src ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_rsrc ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_offset ) -> getReg ( ) , FrameInfo -> getObjectOffset ( Index ) , RS ) ; MI -> eraseFromParent ( ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { buildScratchLoadStore ( MI , <STR_LIT> , TII -> getNamedOperand ( * MI , <STR_LIT> :: dst ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_rsrc ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_offset ) -> getReg ( ) , FrameInfo -> getObjectOffset ( Index ) , RS ) ; MI -> eraseFromParent ( ) ; break ; } default : { int64_t Offset = FrameInfo -> getObjectOffset ( Index ) ; FIOp . ChangeToImmediate ( Offset ) ; if ( ! TII -> isImmOperandLegal ( MI , FIOperandNum , FIOp ) ) { unsigned TmpReg = RS -> scavengeRegister ( & <STR_LIT> , MI , SPAdj ) ; BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . addImm ( Offset ) ; FIOp . ChangeToRegister ( TmpReg , false , false , true ) ;


<s> case <ISA_LIT>RegisterInfo :: SCRATCH_WAVE_OFFSET : if ( MFI -> getShaderType ( ) != ShaderType :: COMPUTE ) return MFI -> ScratchOffsetReg ; return <STR_LIT> . getRegister ( MFI -> NumUserSGPRs + <NUM_LIT> ) ; case <ISA_LIT>RegisterInfo :: SCRATCH_PTR : return <STR_LIT> ; case <ISA_LIT>RegisterInfo :: INPUT_PTR : return <STR_LIT> ; case <ISA_LIT>RegisterInfo :: TIDIG_X : return <STR_LIT> ; case <ISA_LIT>RegisterInfo :: TIDIG_Y : return <STR_LIT> ; case <ISA_LIT>RegisterInfo :: TIDIG_Z : return <STR_LIT> ; } llvm_unreachable ( <STR_LIT> ) ;


<s> if ( isSGPRClass ( * I ) ) { Limit = SGPRLimit / NumSubRegs ; } else { Limit = VGPRLimit / NumSubRegs ; } const int * Sets = getRegClassPressureSets ( * I ) ; assert ( Sets ) ; for ( unsigned i = <NUM_LIT> ; Sets [ i ] != - <NUM_LIT> ; ++ i ) { if ( Sets [ i ] == ( int ) Idx ) return Limit ; } }


<s> for ( regclass_iterator I = regclass_begin ( ) , E = regclass_end ( ) ; I != E ; ++ I ) { unsigned NumSubRegs = std :: max ( ( int ) ( * I ) -> getSize ( ) / <NUM_LIT> , <NUM_LIT> ) ; unsigned Limit ; if ( isSGPRClass ( * I ) ) { Limit = SGPRLimit / NumSubRegs ; } else { Limit = VGPRLimit / NumSubRegs ; } const int * Sets = getRegClassPressureSets ( * I ) ; assert ( Sets ) ; for ( unsigned i = <NUM_LIT> ; Sets [ i ] != - <NUM_LIT> ; ++ i ) {

if ( Sets [ i ] == ( int ) Idx ) return Limit ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasSGPRInitBug ( ) ) {

unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ;


<s> const MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( I -> isConditionalBranch ( ) ) return true ; if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ; } } return false ;


<s> if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ;


<s> if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ; } }


<s> if ( TII -> analyzeBranch ( SrcMBB , TrueMBB , FalseMBB , Cond ) ) return false ; if ( ! FalseMBB ) FalseMBB = SrcMBB . getNextNode ( ) ;


<s> unsigned NumInstr = <NUM_LIT> ; const MachineFunction * MF = From . getParent ( ) ; for ( MachineFunction :: const_iterator MBBI ( & From ) , ToI ( & To ) , End = MF -> end ( ) ; MBBI != End && MBBI != ToI ; ++ MBBI ) { const MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( I -> isConditionalBranch ( ) ) return true ; if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isKillTerminator ( I -> getOpcode ( ) ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ;


<s> if ( ( SrcMBB . getNumber ( ) >= TrueMBB -> getNumber ( ) ) || mustRetainExeczBranch ( * FalseMBB , * TrueMBB ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;


<s> MachineBasicBlock * TrueMBB = nullptr ; MachineBasicBlock * FalseMBB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; if ( ! getBlockDestinations ( SrcMBB , TrueMBB , FalseMBB , Cond ) ) return false ; if ( ( SrcMBB . getNumber ( ) >= TrueMBB -> getNumber ( ) ) || mustRetainExeczBranch ( * FalseMBB , * TrueMBB ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MI . eraseFromParent ( ) ;


<s> bool <ISA_LIT>RemoveShortExecBranches :: runOnMachineFunction ( MachineFunction & MF ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; MF . RenumberBlocks ( ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator MBBI = MBB . getFirstTerminator ( ) ; if ( MBBI == MBB . end ( ) ) continue ; MachineInstr & MI = * MBBI ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : Changed = removeExeczBranch ( MI , MBB ) ; break ;


<s> const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { switch ( MI . getOpcode ( ) ) { default : return false ; case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ;


<s> static MachineOperand copyRegOperandAsImplicit ( const MachineOperand & Orig ) { assert ( ! Orig . isImplicit ( ) ) ;

return MachineOperand :: CreateReg ( Orig . getReg ( ) , Orig . isDef ( ) , true , Orig . isKill ( ) , Orig . isDead ( ) , Orig . isUndef ( ) , Orig . isEarlyClobber ( ) , Orig . getSubReg ( ) , Orig . isDebug ( ) , Orig . isInternalRead ( ) ) ;


<s> assert ( ! Orig . isImplicit ( ) ) ;


<s> return new <ISA_LIT>ShrinkInstructions ( ) ;


<s> if ( Src0 . isImm ( ) && TII -> isLiteralConstant ( Src0 , TII -> getOpSize ( MI , Src0Idx ) ) ) return ; if ( Src0 . isReg ( ) && ! isVGPR ( & Src0 , TRI , MRI ) ) return ; if ( Src0 . isReg ( ) && MRI . hasOneUse ( Src0 . getReg ( ) ) ) { unsigned Reg = Src0 . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ;

if ( Def && Def -> isMoveImmediate ( ) ) {


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> static bool isVGPR ( const MachineOperand * MO , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { if ( ! MO -> isReg ( ) ) return false ; if ( TargetRegisterInfo :: isVirtualRegister ( MO -> getReg ( ) ) ) return TRI . hasVGPRs ( MRI . getRegClass ( MO -> getReg ( ) ) ) ;

return TRI . hasVGPRs ( TRI . getPhysRegClass ( MO -> getReg ( ) ) ) ;


<s> Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) ) { if ( isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( Src , <NUM_LIT> ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; if ( ! canShrink ( MI , TII , TRI , MRI ) ) { if ( ! MI . isCommutable ( ) || ! TII -> commuteInstruction ( & MI ) || ! canShrink ( MI , TII , TRI , MRI ) ) continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ;


<s> <ISA_LIT>ShrinkInstructions ( ) : MachineFunctionPass ( ID ) {


<s> int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; if ( Src0 . isReg ( ) ) { Register Reg = Src0 . getReg ( ) ; if ( Reg . isVirtual ( ) && MRI . hasOneUse ( Reg ) ) { MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { assert ( MRI . use_empty ( Reg ) ) ; Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return true ; } } } }

if ( TryToCommute && MI . isCommutable ( ) ) {


<s> if ( Reg . isVirtual ( ) && MRI . hasOneUse ( Reg ) ) { MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { assert ( MRI . use_empty ( Reg ) ) ; Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return true ; } } } } if ( TryToCommute && MI . isCommutable ( ) ) { if ( TII -> commuteInstruction ( MI ) ) { if ( foldImmediates ( MI , TII , MRI , false ) ) return true ; TII -> commuteInstruction ( MI ) ; } }


<s> if ( ( MovY -> getOpcode ( ) != <STR_LIT> && MovY -> getOpcode ( ) != <STR_LIT> ) || ! MovY -> getOperand ( <NUM_LIT> ) . isReg ( ) || MovY -> getOperand ( <NUM_LIT> ) . getReg ( ) != T || MovY -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Tsub || MovY -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; Register Y = MovY -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Ysub = MovY -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( ! TRI . isVGPR ( MRI , Y ) ) continue ; MachineInstr * MovX = nullptr ; for ( auto IY = MovY -> getIterator ( ) , I = std :: next ( MovT . getIterator ( ) ) ; I != IY ; ++ I ) { if ( instReadsReg ( & * I , X , Xsub , TRI ) || instModifiesReg ( & * I , Y , Ysub , TRI ) || instModifiesReg ( & * I , T , Tsub , TRI ) || ( MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) ) { MovX = nullptr ; break ; } if ( ! instReadsReg ( & * I , Y , Ysub , TRI ) ) { if ( ! MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) { MovX = nullptr ; break ; } continue ; } if ( MovX || ( I -> getOpcode ( ) != <STR_LIT> && I -> getOpcode ( ) != <STR_LIT> ) || I -> getOperand ( <NUM_LIT> ) . getReg ( ) != X || I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Xsub ) { MovX = nullptr ; break ; } if ( I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; if ( Size > <NUM_LIT> && ( I -> getNumImplicitOperands ( ) > ( I -> isCopy ( ) ? <NUM_LIT> : <NUM_LIT> ) ) ) continue ; MovX = & * I ; } if ( ! MovX ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MovT << * MovX << * MovY ) ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) { TargetInstrInfo :: RegSubRegPair X1 , Y1 ; X1 = getSubRegForIndex ( X , Xsub , I , TRI , MRI ) ; Y1 = getSubRegForIndex ( Y , Ysub , I , TRI , MRI ) ; MachineBasicBlock & MBB = * MovT . getParent ( ) ;

auto MIB = BuildMI ( MBB , MovX -> getIterator ( ) , MovT . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addDef ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . addDef ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . getInstr ( ) ;


<s> } } } if ( ST . hasSwap ( ) && ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) ) { if ( auto * NextMI = matchSwap ( MI , MRI , TII ) ) { Next = NextMI -> getIterator ( ) ; continue ; } } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand * Dest = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src0 = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src1 = & MI . getOperand ( <NUM_LIT> ) ; if ( ! Src0 -> isReg ( ) && Src1 -> isReg ( ) ) { if ( TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) std :: swap ( Src0 , Src1 ) ; } if ( Dest -> getReg ( ) . isVirtual ( ) && Src0 -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , Src0 -> getReg ( ) ) ; MRI . setRegAllocationHint ( Src0 -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; continue ; } if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Dest -> getReg ( ) ) { if ( Src1 -> isImm ( ) && isKImmOperand ( TII , * Src1 ) ) { unsigned Opc = ( MI . getOpcode ( ) == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; MI . setDesc ( TII -> get ( Opc ) ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . isCompare ( ) && TII -> isSOPC ( MI ) ) { shrinkScalarCompare ( TII , MI ) ; continue ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && Dst . getReg ( ) . isPhysical ( ) ) { int32_t ReverseImm ; if ( isKImmOperand ( TII , Src ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; else if ( isReverseInlineImm ( TII , Src , ReverseImm ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ;


<s> if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = ( TFEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = ( LWEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) {

if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i ) . isTied ( ) && MI . getOperand ( i ) . isImplicit ( ) ) {


<s> default : return false ; case <STR_LIT> : case <STR_LIT> : if ( TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) -> isImm ( ) ) return false ; return true ; case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ;

return ! TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) ;


<s> if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ;

if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ;


<s> static void foldImmediates ( MachineInstr & MI , const <ISA_LIT>InstrInfo * TII , MachineRegisterInfo & MRI , bool TryToCommute = true ) { if ( ! MRI . isSSA ( ) ) return ; assert ( TII -> isVOP1 ( MI ) || TII -> isVOP2 ( MI ) || TII -> isVOPC ( MI ) ) ; int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; if ( TII -> isLiteralConstant ( MI , Src0Idx ) ) return ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; if ( Src0 . isReg ( ) && MRI . hasOneUse ( Src0 . getReg ( ) ) ) { unsigned Reg = Src0 . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { if ( MRI . use_empty ( Reg ) ) Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return ; }


<s> if ( ! MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ; const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ; if ( ! Src1 . isImm ( ) ) return ; int SOPKOpc = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( SOPKOpc == - <NUM_LIT> ) return ; if ( SOPKOpc == <STR_LIT> || SOPKOpc == <STR_LIT> ) { bool HasUImm ; if ( isKImmOrKUImmOperand ( TII , Src1 , HasUImm ) ) { if ( ! HasUImm ) { SOPKOpc = ( SOPKOpc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; } MI . setDesc ( TII -> get ( SOPKOpc ) ) ; }


<s> const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { switch ( MI . getOpcode ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) , TRI , MRI ) ) return false ; return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :

if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ;


<s> static bool canShrink ( MachineInstr & MI , const <ISA_LIT>InstrInfo * TII , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { switch ( MI . getOpcode ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) , TRI , MRI ) ) return false ; return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } }

const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ;


<s> MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . setSubReg ( <NUM_LIT> ) ; Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . setSubReg ( <NUM_LIT> ) ; Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { assert ( MRI . use_empty ( Reg ) ) ; Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return true ; } } } } if ( TryToCommute && MI . isCommutable ( ) ) {

if ( TII -> commuteInstruction ( MI ) ) {


<s> if ( TRI . getRegSizeInBits ( Reg , MRI ) != <NUM_LIT> ) { if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) ) { Reg = TRI . getSubReg ( Reg , TRI . getSubRegFromChannel ( I ) ) ; } else { LaneBitmask LM = TRI . getSubRegIndexLaneMask ( Sub ) ;

Sub = TRI . getSubRegFromChannel ( I + countTrailingZeros ( LM . getAsInteger ( ) ) ) ;


<s> } else if ( MO . getReg ( ) == Reg && TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { LaneBitmask Overlap = TRI . getSubRegIndexLaneMask ( SubReg ) & TRI . getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ; if ( Overlap . any ( ) ) return true ;


<s> if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) && TargetRegisterInfo :: isPhysicalRegister ( MO . getReg ( ) ) ) { if ( TRI . regsOverlap ( Reg , MO . getReg ( ) ) ) return true ; } else if ( MO . getReg ( ) == Reg && TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { LaneBitmask Overlap = TRI . getSubRegIndexLaneMask ( SubReg ) & TRI . getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ; if ( Overlap . any ( ) ) return true ;


<s> if ( instReadsReg ( & * I , X , Xsub , TRI ) || instModifiesReg ( & * I , Y , Ysub , TRI ) || instModifiesReg ( & * I , T , Tsub , TRI ) || ( MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) ) { MovX = nullptr ; break ; } if ( ! instReadsReg ( & * I , Y , Ysub , TRI ) ) { if ( ! MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) { MovX = nullptr ; break ; } continue ; } if ( MovX || ( I -> getOpcode ( ) != <STR_LIT> && I -> getOpcode ( ) != <STR_LIT> ) || I -> getOperand ( <NUM_LIT> ) . getReg ( ) != X || I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Xsub ) { MovX = nullptr ; break ; } MovX = & * I ; } if ( ! MovX || I == E ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MovT << * MovX << MovY ) ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) { TargetInstrInfo :: RegSubRegPair X1 , Y1 ; X1 = getSubRegForIndex ( X , Xsub , I , TRI , MRI ) ; Y1 = getSubRegForIndex ( Y , Ysub , I , TRI , MRI ) ; BuildMI ( * MovT . getParent ( ) , MovX -> getIterator ( ) , MovT . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addDef ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . addDef ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . getInstr ( ) ; } MovX -> eraseFromParent ( ) ; MovY . eraseFromParent ( ) ; MachineInstr * Next = & * std :: next ( MovT . getIterator ( ) ) ; if ( MRI . use_nodbg_empty ( T ) ) MovT . eraseFromParent ( ) ; else Xop . setIsKill ( false ) ; return Next ; } return nullptr ;


<s> case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || ( Src1Mod && Src1Mod -> getImm ( ) != <NUM_LIT> ) ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ;

if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ;


<s> MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( MI . getOpcode ( ) == <STR_LIT> ) { MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && TargetRegisterInfo :: isPhysicalRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { int32_t ReverseImm ; if ( isReverseInlineImm ( TII , Src , ReverseImm ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; Src . setImm ( ReverseImm ) ; continue ; } } } if ( MI . getOpcode ( ) == <STR_LIT> && Next != MBB . end ( ) && ( * Next ) . getOpcode ( ) == <STR_LIT> ) { MachineInstr & NextMI = * Next ; uint8_t Nop0 = MI . getOperand ( <NUM_LIT> ) . getImm ( ) + <NUM_LIT> ; uint8_t Nop1 = NextMI . getOperand ( <NUM_LIT> ) . getImm ( ) + <NUM_LIT> ; if ( Nop0 + Nop1 <= <NUM_LIT> ) { NextMI . getOperand ( <NUM_LIT> ) . setImm ( Nop0 + Nop1 - <NUM_LIT> ) ; MI . eraseFromParent ( ) ; } continue ; } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand * Dest = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src0 = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src1 = & MI . getOperand ( <NUM_LIT> ) ; if ( ! Src0 -> isReg ( ) && Src1 -> isReg ( ) ) { if ( TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) std :: swap ( Src0 , Src1 ) ; } if ( TargetRegisterInfo :: isVirtualRegister ( Dest -> getReg ( ) ) && Src0 -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , Src0 -> getReg ( ) ) ; MRI . setRegAllocationHint ( Src0 -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; continue ; } if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Dest -> getReg ( ) ) { if ( Src1 -> isImm ( ) && isKImmOperand ( TII , * Src1 ) ) { unsigned Opc = ( MI . getOpcode ( ) == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; MI . setDesc ( TII -> get ( Opc ) ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } }

if ( MI . isCompare ( ) && TII -> isSOPC ( MI ) ) {


<s> for ( unsigned i = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) + MI . getDesc ( ) . getNumImplicitDefs ( ) , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & MO = MI . getOperand ( i ) ; if ( ( MO . isReg ( ) && MO . isImplicit ( ) ) || MO . isRegMask ( ) ) NewMI . addOperand ( MF , MO ) ; }


<s> static void dropInstructionKeepingImpDefs ( MachineInstr & MI , const <ISA_LIT>InstrInfo * TII ) {

for ( unsigned i = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) + MI . getDesc ( ) . getNumImplicitDefs ( ) , e = MI . getNumOperands ( ) ;


<s> BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Op . getReg ( ) ) ; } MI . eraseFromParent ( ) ;


<s> int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; if ( Src0 . isReg ( ) ) { Register Reg = Src0 . getReg ( ) ; if ( Reg . isVirtual ( ) && MRI . hasOneUse ( Reg ) ) { MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( TII -> isOperandLegal ( MI , Src0Idx , & MovSrc ) ) { if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ;


<s> StringRef getPassName ( ) const override {


<s> static TargetInstrInfo :: RegSubRegPair getSubRegForIndex ( Register Reg , unsigned Sub , unsigned I , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { if ( TRI . getRegSizeInBits ( Reg , MRI ) != <NUM_LIT> ) { if ( Reg . isPhysical ( ) ) { Reg = TRI . getSubReg ( Reg , TRI . getSubRegFromChannel ( I ) ) ; } else { Sub = TRI . getSubRegFromChannel ( I + TRI . getChannelFromSubReg ( Sub ) ) ; } } return TargetInstrInfo :: RegSubRegPair ( Reg , Sub ) ;


<s> if ( Reg . isPhysical ( ) && MO . getReg ( ) . isPhysical ( ) ) { if ( TRI . regsOverlap ( Reg , MO . getReg ( ) ) ) return true ; } else if ( MO . getReg ( ) == Reg && Reg . isVirtual ( ) ) { LaneBitmask Overlap = TRI . getSubRegIndexLaneMask ( SubReg ) & TRI . getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ; if ( Overlap . any ( ) ) return true ; } }


<s> static bool instModifiesReg ( const MachineInstr * MI , unsigned Reg , unsigned SubReg , const <ISA_LIT>RegisterInfo & TRI ) {

return instAccessReg ( MI -> defs ( ) , Reg , SubReg , TRI ) ;


<s> return instAccessReg ( MI -> defs ( ) , Reg , SubReg , TRI ) ;


<s> static bool instReadsReg ( const MachineInstr * MI , unsigned Reg , unsigned SubReg , const <ISA_LIT>RegisterInfo & TRI ) { return instAccessReg ( MI -> uses ( ) , Reg , SubReg , TRI ) ;


<s> return instAccessReg ( MI -> uses ( ) , Reg , SubReg , TRI ) ;


<s> return isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src . getParent ( ) , Src . getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> IsUnsigned = false ; return ! TII -> isInlineConstant ( Src ) ; } if ( isUInt < <NUM_LIT> > ( Src . getImm ( ) ) ) { IsUnsigned = true ; return ! TII -> isInlineConstant ( Src ) ; } return false ;


<s> return isUInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src . getParent ( ) , Src . getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> if ( ! isInt < <NUM_LIT> > ( Src . getImm ( ) ) || TII -> isInlineConstant ( Src ) ) return false ; ReverseImm = reverseBits < int32_t > ( static_cast < int32_t > ( Src . getImm ( ) ) ) ;

return ReverseImm >= - <NUM_LIT> && ReverseImm <= <NUM_LIT> ;


<s> if ( ! MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) { MovX = nullptr ; break ; } continue ; } if ( MovX || ( I -> getOpcode ( ) != <STR_LIT> && I -> getOpcode ( ) != <STR_LIT> ) || I -> getOperand ( <NUM_LIT> ) . getReg ( ) != X || I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Xsub ) { MovX = nullptr ; break ; } if ( I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; if ( Size > <NUM_LIT> && ( I -> getNumImplicitOperands ( ) > ( I -> isCopy ( ) ? <NUM_LIT> : <NUM_LIT> ) ) ) continue ; MovX = & * I ; } if ( ! MovX ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MovT << * MovX << * MovY ) ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) { TargetInstrInfo :: RegSubRegPair X1 , Y1 ; X1 = getSubRegForIndex ( X , Xsub , I , TRI , MRI ) ; Y1 = getSubRegForIndex ( Y , Ysub , I , TRI , MRI ) ; MachineBasicBlock & MBB = * MovT . getParent ( ) ; auto MIB = BuildMI ( MBB , MovX -> getIterator ( ) , MovT . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addDef ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . addDef ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . getInstr ( ) ; if ( MovX -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) { MIB -> RemoveOperand ( MIB -> getNumExplicitOperands ( ) ) ; MIB -> copyImplicitOps ( * MBB . getParent ( ) , * MovX ) ; } } MovX -> eraseFromParent ( ) ; dropInstructionKeepingImpDefs ( * MovY , TII ) ; MachineInstr * Next = & * std :: next ( MovT . getIterator ( ) ) ; if ( T . isVirtual ( ) && MRI . use_nodbg_empty ( T ) ) { dropInstructionKeepingImpDefs ( MovT , TII ) ; } else { Xop . setIsKill ( false ) ; for ( int I = MovT . getNumImplicitOperands ( ) - <NUM_LIT> ; I >= <NUM_LIT> ; -- I ) { unsigned OpNo = MovT . getNumExplicitOperands ( ) + I ;

const MachineOperand & Op = MovT . getOperand ( OpNo ) ;


<s> unsigned LWEVal = ( LWEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i ) . isTied ( ) && MI . getOperand ( i ) . isImplicit ( ) ) { assert ( ToUntie == - <NUM_LIT> && <STR_LIT> ) ; ToUntie = i ; MI . untieRegOperand ( ToUntie ) ; } } } unsigned NewOpcode = <STR_LIT> ( Info -> BaseOpcode , <STR_LIT> , Info -> VDataDwords , NewAddrDwords ) ; MI . setDesc ( TII -> get ( NewOpcode ) ) ; MI . getOperand ( VAddr0Idx ) . setReg ( RC -> getRegister ( VgprBase ) ) ; MI . getOperand ( VAddr0Idx ) . setIsUndef ( IsUndef ) ; MI . getOperand ( VAddr0Idx ) . setIsKill ( IsKill ) ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) MI . RemoveOperand ( VAddr0Idx + <NUM_LIT> ) ; if ( ToUntie >= <NUM_LIT> ) { MI . tieOperands ( <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vdata ) , ToUntie - ( Info -> VAddrDwords - <NUM_LIT> ) ) ;


<s> if ( ! Src1 . isImm ( ) ) return ; int SOPKOpc = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( SOPKOpc == - <NUM_LIT> ) return ; if ( SOPKOpc == <STR_LIT> || SOPKOpc == <STR_LIT> ) { bool HasUImm ; if ( isKImmOrKUImmOperand ( TII , Src1 , HasUImm ) ) { if ( ! HasUImm ) {

SOPKOpc = ( SOPKOpc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ;


<s> uint32_t NewImm = <NUM_LIT> ; if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( ~ Imm ) ) { NewImm = countTrailingOnes ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && SrcImm == Src0 ) { if ( ! TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) NewImm = <NUM_LIT> ; } if ( NewImm != <NUM_LIT> ) { if ( Dest -> getReg ( ) . isVirtual ( ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ; } if ( SrcReg -> isReg ( ) && SrcReg -> getReg ( ) == Dest -> getReg ( ) ) { const bool IsUndef = SrcReg -> isUndef ( ) ; const bool IsKill = SrcReg -> isKill ( ) ; MI . setDesc ( TII -> get ( Opc ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { Src0 -> ChangeToImmediate ( NewImm ) ; MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Dest -> getReg ( ) , false , false , IsKill , false , IsUndef ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } else { SrcImm -> setImm ( NewImm ) ; } }


<s> assert ( MovT . getOpcode ( ) == <STR_LIT> || MovT . getOpcode ( ) == <STR_LIT> ) ; Register T = MovT . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Tsub = MovT . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; MachineOperand & Xop = MovT . getOperand ( <NUM_LIT> ) ; if ( ! Xop . isReg ( ) ) return nullptr ; Register X = Xop . getReg ( ) ; unsigned Xsub = Xop . getSubReg ( ) ; unsigned Size = TII -> getOpSize ( MovT , <NUM_LIT> ) / <NUM_LIT> ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; if ( ! TRI . isVGPR ( MRI , X ) ) return nullptr ; if ( MovT . hasRegisterImplicitUseOperand ( <STR_LIT> ) ) return nullptr ; const unsigned SearchLimit = <NUM_LIT> ; unsigned Count = <NUM_LIT> ; bool KilledT = false ; for ( auto Iter = std :: next ( MovT . getIterator ( ) ) , E = MovT . getParent ( ) -> instr_end ( ) ; Iter != E && Count < SearchLimit && ! KilledT ; ++ Iter , ++ Count ) { MachineInstr * MovY = & * Iter ; KilledT = MovY -> killsRegister ( T , & TRI ) ; if ( ( MovY -> getOpcode ( ) != <STR_LIT> && MovY -> getOpcode ( ) != <STR_LIT> ) || ! MovY -> getOperand ( <NUM_LIT> ) . isReg ( ) || MovY -> getOperand ( <NUM_LIT> ) . getReg ( ) != T || MovY -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Tsub || MovY -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; Register Y = MovY -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Ysub = MovY -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( ! TRI . isVGPR ( MRI , Y ) ) continue ; MachineInstr * MovX = nullptr ;

for ( auto IY = MovY -> getIterator ( ) , I = std :: next ( MovT . getIterator ( ) ) ;


<s> if ( MI . getOpcode ( ) == <STR_LIT> ) { MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) ) { int32_t ReverseImm ; if ( isReverseInlineImm ( TII , Src , ReverseImm ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; Src . setImm ( ReverseImm ) ; continue ; } } } if ( ST . hasSwap ( ) && ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) ) { if ( auto * NextMI = matchSwap ( MI , MRI , TII ) ) { Next = NextMI -> getIterator ( ) ; continue ; } } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand * Dest = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src0 = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src1 = & MI . getOperand ( <NUM_LIT> ) ; if ( ! Src0 -> isReg ( ) && Src1 -> isReg ( ) ) { if ( TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) std :: swap ( Src0 , Src1 ) ; } if ( Dest -> getReg ( ) . isVirtual ( ) && Src0 -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , Src0 -> getReg ( ) ) ; MRI . setRegAllocationHint ( Src0 -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; continue ; } if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Dest -> getReg ( ) ) { if ( Src1 -> isImm ( ) && isKImmOperand ( TII , * Src1 ) ) { unsigned Opc = ( MI . getOpcode ( ) == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; MI . setDesc ( TII -> get ( Opc ) ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . isCompare ( ) && TII -> isSOPC ( MI ) ) { shrinkScalarCompare ( TII , MI ) ; continue ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && Dst . getReg ( ) . isPhysical ( ) ) { int32_t ReverseImm ;

if ( isKImmOperand ( TII , Src ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ;


<s> void <ISA_LIT>ShrinkInstructions :: shrinkMIMG ( MachineInstr & MI ) { const <STR_LIT> * Info = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( ! Info || Info -> MIMGEncoding != <STR_LIT> ) return ; MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; int VAddr0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; unsigned NewAddrDwords = Info -> VAddrDwords ; const TargetRegisterClass * RC ; if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI . getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = ( TFEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( TFEIdx ) . getImm ( ) ;

unsigned LWEVal = ( LWEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( LWEIdx ) . getImm ( ) ;


<s> RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI . getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = ( TFEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = ( LWEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i ) . isTied ( ) && MI . getOperand ( i ) . isImplicit ( ) ) { assert ( ToUntie == - <NUM_LIT> && <STR_LIT> ) ; ToUntie = i ; MI . untieRegOperand ( ToUntie ) ; } }


<s> Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( NewImm != <NUM_LIT> ) { if ( Dest -> getReg ( ) . isVirtual ( ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ; } if ( SrcReg -> isReg ( ) && SrcReg -> getReg ( ) == Dest -> getReg ( ) ) { const bool IsUndef = SrcReg -> isUndef ( ) ; const bool IsKill = SrcReg -> isKill ( ) ;


<s> case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 && ( ! Src1 -> isReg ( ) || ! TRI . isVGPR ( MRI , Src1 -> getReg ( ) ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ; return ! TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) ;


<s> MachineFunction & MF = * MI . getMF ( ) ; for ( unsigned i = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) + MI . getDesc ( ) . getNumImplicitDefs ( ) , e = MI . getNumOperands ( ) ; i != e ; ++ i ) {

const MachineOperand & MO = MI . getOperand ( i ) ;


<s> const MachineOperand & MO = MI . getOperand ( i ) ;

if ( ( MO . isReg ( ) && MO . isImplicit ( ) ) || MO . isRegMask ( ) ) NewMI . addOperand ( MF , MO ) ;


<s> for ( unsigned i = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) + MI . getDesc ( ) . getNumImplicitDefs ( ) , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & Op = MI . getOperand ( i ) ; if ( ! Op . isDef ( ) ) continue ; BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Op . getReg ( ) ) ; }


<s> void <ISA_LIT>ShrinkInstructions :: dropInstructionKeepingImpDefs ( MachineInstr & MI ) const { for ( unsigned i = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) + MI . getDesc ( ) . getNumImplicitDefs ( ) , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & Op = MI . getOperand ( i ) ; if ( ! Op . isDef ( ) ) continue ; BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Op . getReg ( ) ) ; } MI . eraseFromParent ( ) ;


<s> Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ; } } if ( ConstantFolded ) { if ( MRI -> use_nodbg_empty ( Reg ) ) Def -> eraseFromParent ( ) ;


<s> MachineInstr * Def = MRI -> getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( TII -> isOperandLegal ( MI , Src0Idx , & MovSrc ) ) { if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ;


<s> Reg = TRI -> getSubReg ( Reg , TRI -> getSubRegFromChannel ( I ) ) ; } else { Sub = TRI -> getSubRegFromChannel ( I + TRI -> getChannelFromSubReg ( Sub ) ) ;


<s> for ( const MachineOperand & MO : R ) { if ( ! MO . isReg ( ) ) continue ; if ( Reg . isPhysical ( ) && MO . getReg ( ) . isPhysical ( ) ) {

if ( TRI -> regsOverlap ( Reg , MO . getReg ( ) ) ) return true ;


<s> if ( TRI -> regsOverlap ( Reg , MO . getReg ( ) ) ) return true ; } else if ( MO . getReg ( ) == Reg && Reg . isVirtual ( ) ) { LaneBitmask Overlap = TRI -> getSubRegIndexLaneMask ( SubReg ) & TRI -> getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ; if ( Overlap . any ( ) ) return true ;


<s> bool <ISA_LIT>ShrinkInstructions :: isKImmOperand ( const MachineOperand & Src ) const {

return isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src . getParent ( ) , Src . getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> return isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src . getParent ( ) , Src . getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> bool <ISA_LIT>ShrinkInstructions :: isKImmOrKUImmOperand ( const MachineOperand & Src , bool & IsUnsigned ) const { if ( isInt < <NUM_LIT> > ( Src . getImm ( ) ) ) { IsUnsigned = false ; return ! TII -> isInlineConstant ( Src ) ; }

if ( isUInt < <NUM_LIT> > ( Src . getImm ( ) ) ) {


<s> bool <ISA_LIT>ShrinkInstructions :: isKUImmOperand ( const MachineOperand & Src ) const { return isUInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src . getParent ( ) , Src . getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> bool <ISA_LIT>ShrinkInstructions :: isKUImmOperand ( const MachineOperand & Src ) const {

return isUInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src . getParent ( ) , Src . getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> ReverseImm = reverseBits < int32_t > ( static_cast < int32_t > ( Src . getImm ( ) ) ) ; return ReverseImm >= - <NUM_LIT> && ReverseImm <= <NUM_LIT> ;


<s> bool <ISA_LIT>ShrinkInstructions :: isReverseInlineImm ( const MachineOperand & Src , int32_t & ReverseImm ) const { if ( ! isInt < <NUM_LIT> > ( Src . getImm ( ) ) || TII -> isInlineConstant ( Src ) ) return false ;

ReverseImm = reverseBits < int32_t > ( static_cast < int32_t > ( Src . getImm ( ) ) ) ;


<s> MovX = nullptr ; break ; } if ( ! instReadsReg ( & * I , Y , Ysub ) ) { if ( ! MovX && instModifiesReg ( & * I , X , Xsub ) ) { MovX = nullptr ; break ; } continue ; } if ( MovX || ( I -> getOpcode ( ) != <STR_LIT> && I -> getOpcode ( ) != <STR_LIT> ) || I -> getOperand ( <NUM_LIT> ) . getReg ( ) != X || I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Xsub ) { MovX = nullptr ; break ; } if ( I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; if ( Size > <NUM_LIT> && ( I -> getNumImplicitOperands ( ) > ( I -> isCopy ( ) ? <NUM_LIT> : <NUM_LIT> ) ) ) continue ; MovX = & * I ; } if ( ! MovX ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MovT << * MovX << * MovY ) ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) { TargetInstrInfo :: RegSubRegPair X1 , Y1 ; X1 = getSubRegForIndex ( X , Xsub , I ) ; Y1 = getSubRegForIndex ( Y , Ysub , I ) ; MachineBasicBlock & MBB = * MovT . getParent ( ) ; auto MIB = BuildMI ( MBB , MovX -> getIterator ( ) , MovT . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addDef ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . addDef ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . getInstr ( ) ; if ( MovX -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) { MIB -> removeOperand ( MIB -> getNumExplicitOperands ( ) ) ; MIB -> copyImplicitOps ( * MBB . getParent ( ) , * MovX ) ; } } MovX -> eraseFromParent ( ) ; dropInstructionKeepingImpDefs ( * MovY ) ; MachineInstr * Next = & * std :: next ( MovT . getIterator ( ) ) ;

if ( T . isVirtual ( ) && MRI -> use_nodbg_empty ( T ) ) {


<s> break ; } if ( I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; if ( Size > <NUM_LIT> && ( I -> getNumImplicitOperands ( ) > ( I -> isCopy ( ) ? <NUM_LIT> : <NUM_LIT> ) ) ) continue ; MovX = & * I ; } if ( ! MovX ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MovT << * MovX << * MovY ) ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) { TargetInstrInfo :: RegSubRegPair X1 , Y1 ; X1 = getSubRegForIndex ( X , Xsub , I ) ; Y1 = getSubRegForIndex ( Y , Ysub , I ) ; MachineBasicBlock & MBB = * MovT . getParent ( ) ; auto MIB = BuildMI ( MBB , MovX -> getIterator ( ) , MovT . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addDef ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . addDef ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . getInstr ( ) ; if ( MovX -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) { MIB -> removeOperand ( MIB -> getNumExplicitOperands ( ) ) ; MIB -> copyImplicitOps ( * MBB . getParent ( ) , * MovX ) ; } } MovX -> eraseFromParent ( ) ; dropInstructionKeepingImpDefs ( * MovY ) ; MachineInstr * Next = & * std :: next ( MovT . getIterator ( ) ) ; if ( T . isVirtual ( ) && MRI -> use_nodbg_empty ( T ) ) { dropInstructionKeepingImpDefs ( MovT ) ; } else { Xop . setIsKill ( false ) ; for ( int I = MovT . getNumImplicitOperands ( ) - <NUM_LIT> ; I >= <NUM_LIT> ; -- I ) { unsigned OpNo = MovT . getNumExplicitOperands ( ) + I ; const MachineOperand & Op = MovT . getOperand ( OpNo ) ; if ( Op . isKill ( ) && TRI -> regsOverlap ( X , Op . getReg ( ) ) ) MovT . removeOperand ( OpNo ) ; } } return Next ; } return nullptr ;


<s> NewOpcode = <STR_LIT> ; break ; } } if ( Src2 . isReg ( ) && TRI -> isVGPR ( * MRI , Src2 . getReg ( ) ) ) { if ( Src1 . isImm ( ) && ! TII -> isInlineConstant ( Src1 ) ) Swap = false ; else if ( Src0 . isImm ( ) && ! TII -> isInlineConstant ( Src0 ) ) Swap = true ; else return ; switch ( Opcode ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; } } if ( NewOpcode == <STR_LIT> ) return ; if ( Swap ) { BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( NewOpcode ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . add ( Src1 ) . add ( Src0 ) . add ( Src2 ) . setMIFlags ( MI . getFlags ( ) ) ; MI . eraseFromParent ( ) ;


<s> RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI -> getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = ( TFEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = ( LWEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i ) . isTied ( ) && MI . getOperand ( i ) . isImplicit ( ) ) { assert ( ToUntie == - <NUM_LIT> && <STR_LIT> ) ; ToUntie = i ; MI . untieRegOperand ( ToUntie ) ; } } } unsigned NewOpcode = <STR_LIT> ( Info -> BaseOpcode , <STR_LIT> , Info -> VDataDwords , NewAddrDwords ) ; MI . setDesc ( TII -> get ( NewOpcode ) ) ; MI . getOperand ( VAddr0Idx ) . setReg ( RC -> getRegister ( VgprBase ) ) ; MI . getOperand ( VAddr0Idx ) . setIsUndef ( IsUndef ) ; MI . getOperand ( VAddr0Idx ) . setIsKill ( IsKill ) ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) MI . removeOperand ( VAddr0Idx + <NUM_LIT> ) ;


<s> int VAddr0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; unsigned NewAddrDwords = Info -> VAddrDwords ; const TargetRegisterClass * RC ; if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI -> getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ;

if ( ! Op . isUndef ( ) ) IsUndef = false ;


<s> if ( SOPKOpc == - <NUM_LIT> ) return ; if ( SOPKOpc == <STR_LIT> || SOPKOpc == <STR_LIT> ) { bool HasUImm ; if ( isKImmOrKUImmOperand ( Src1 , HasUImm ) ) { if ( ! HasUImm ) { SOPKOpc = ( SOPKOpc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; } MI . setDesc ( TII -> get ( SOPKOpc ) ) ; } return ; } const MCInstrDesc & NewDesc = TII -> get ( SOPKOpc ) ;

if ( ( TII -> sopkIsZext ( SOPKOpc ) && isKUImmOperand ( Src1 ) ) || ( ! TII -> sopkIsZext ( SOPKOpc ) && isKImmOperand ( Src1 ) ) ) {


<s> NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST -> hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST -> hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( NewImm != <NUM_LIT> ) { if ( Dest -> getReg ( ) . isVirtual ( ) && SrcReg -> isReg ( ) ) { MRI -> setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI -> setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ;


<s> ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { assert ( MRI . use_empty ( Reg ) ) ; Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return true ; } } } } if ( TryToCommute && MI . isCommutable ( ) ) { if ( TII -> commuteInstruction ( MI ) ) { if ( foldImmediates ( MI , TII , MRI , false ) ) return true ; TII -> commuteInstruction ( MI ) ; }


<s> for ( const MachineOperand & MO : R ) { if ( ! MO . isReg ( ) ) continue ; if ( Register :: isPhysicalRegister ( Reg ) && Register :: isPhysicalRegister ( MO . getReg ( ) ) ) { if ( TRI . regsOverlap ( Reg , MO . getReg ( ) ) ) return true ; } else if ( MO . getReg ( ) == Reg && Register :: isVirtualRegister ( Reg ) ) {

LaneBitmask Overlap = TRI . getSubRegIndexLaneMask ( SubReg ) & TRI . getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ;


<s> if ( ! MO . isReg ( ) ) continue ; if ( Register :: isPhysicalRegister ( Reg ) && Register :: isPhysicalRegister ( MO . getReg ( ) ) ) { if ( TRI . regsOverlap ( Reg , MO . getReg ( ) ) ) return true ; } else if ( MO . getReg ( ) == Reg && Register :: isVirtualRegister ( Reg ) ) {


<s> if ( ! Info || Info -> MIMGEncoding != <STR_LIT> ) return ; MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; int VAddr0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; unsigned NewAddrDwords = Info -> VAddrDwords ; const TargetRegisterClass * RC ; if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords <= <NUM_LIT> ) { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI . getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) {

if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i ) . isTied ( ) && MI . getOperand ( i ) . isImplicit ( ) ) {


<s> if ( ! SrcImm -> isImm ( ) || <STR_LIT> ( SrcImm -> getImm ( ) , ST . hasInv2PiInlineImm ( ) ) ) return false ; uint32_t Imm = static_cast < uint32_t > ( SrcImm -> getImm ( ) ) ; uint32_t NewImm = <NUM_LIT> ; if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( ~ Imm ) ) { NewImm = countTrailingOnes ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ;


<s> } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i ) . isTied ( ) && MI . getOperand ( i ) . isImplicit ( ) ) { assert ( ToUntie == - <NUM_LIT> && <STR_LIT> ) ; ToUntie = i ; MI . untieRegOperand ( ToUntie ) ; } } } unsigned NewOpcode = <STR_LIT> ( Info -> BaseOpcode , <STR_LIT> , Info -> VDataDwords , NewAddrDwords ) ; MI . setDesc ( TII -> get ( NewOpcode ) ) ; MI . getOperand ( VAddr0Idx ) . setReg ( RC -> getRegister ( VgprBase ) ) ; MI . getOperand ( VAddr0Idx ) . setIsUndef ( IsUndef ) ; MI . getOperand ( VAddr0Idx ) . setIsKill ( IsKill ) ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) MI . RemoveOperand ( VAddr0Idx + <NUM_LIT> ) ; if ( ToUntie >= <NUM_LIT> ) { MI . tieOperands ( <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vdata ) , ToUntie - ( Info -> VAddrDwords - <NUM_LIT> ) ) ; }


<s> Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && SrcImm == Src0 ) { if ( ! TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) NewImm = <NUM_LIT> ; } if ( NewImm != <NUM_LIT> ) { if ( Register :: isVirtualRegister ( Dest -> getReg ( ) ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ;


<s> if ( isPowerOf2_32 ( ~ Imm ) ) { NewImm = countTrailingOnes ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && SrcImm == Src0 ) { if ( ! TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) NewImm = <NUM_LIT> ; } if ( NewImm != <NUM_LIT> ) { if ( Register :: isVirtualRegister ( Dest -> getReg ( ) ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ; } if ( SrcReg -> isReg ( ) && SrcReg -> getReg ( ) == Dest -> getReg ( ) ) { MI . setDesc ( TII -> get ( Opc ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { Src0 -> ChangeToImmediate ( NewImm ) ; MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Dest -> getReg ( ) , false ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } else { SrcImm -> setImm ( NewImm ) ; }


<s> static TargetInstrInfo :: RegSubRegPair getSubRegForIndex ( unsigned Reg , unsigned Sub , unsigned I , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { if ( TRI . getRegSizeInBits ( Reg , MRI ) != <NUM_LIT> ) { if ( Register :: isPhysicalRegister ( Reg ) ) { Reg = TRI . getSubReg ( Reg , TRI . getSubRegFromChannel ( I ) ) ; } else { LaneBitmask LM = TRI . getSubRegIndexLaneMask ( Sub ) ; Sub = TRI . getSubRegFromChannel ( I + countTrailingZeros ( LM . getAsInteger ( ) ) ) ; } }

return TargetInstrInfo :: RegSubRegPair ( Reg , Sub ) ;


<s> MachineInstr & MovY = * YTop . getParent ( ) ; if ( ( MovY . getOpcode ( ) != <STR_LIT> && MovY . getOpcode ( ) != <STR_LIT> ) || MovY . getOperand ( <NUM_LIT> ) . getSubReg ( ) != Tsub ) continue ; Register Y = MovY . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Ysub = MovY . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( ! TRI . isVGPR ( MRI , Y ) || MovT . getParent ( ) != MovY . getParent ( ) ) continue ; MachineInstr * MovX = nullptr ; auto I = std :: next ( MovT . getIterator ( ) ) , E = MovT . getParent ( ) -> instr_end ( ) ; for ( auto IY = MovY . getIterator ( ) ; I != E && I != IY ; ++ I ) { if ( instReadsReg ( & * I , X , Xsub , TRI ) || instModifiesReg ( & * I , Y , Ysub , TRI ) || instModifiesReg ( & * I , T , Tsub , TRI ) || ( MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) ) { MovX = nullptr ; break ; } if ( ! instReadsReg ( & * I , Y , Ysub , TRI ) ) { if ( ! MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) { MovX = nullptr ; break ; } continue ; } if ( MovX || ( I -> getOpcode ( ) != <STR_LIT> && I -> getOpcode ( ) != <STR_LIT> ) || I -> getOperand ( <NUM_LIT> ) . getReg ( ) != X || I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Xsub ) { MovX = nullptr ; break ; }


<s> case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ;

if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || ( Src1Mod && Src1Mod -> getImm ( ) != <NUM_LIT> ) ) ) return false ;


<s> static bool isKImmOperand ( const <ISA_LIT>InstrInfo * TII , const MachineOperand & Src ) {

return isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( Src , <NUM_LIT> ) ;


<s> unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; unsigned SReg = Src2 -> getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( SReg != <STR_LIT> ) continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder Inst32 = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( Op32 ) ) ; int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: vdst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && <STR_LIT> ) ; } Inst32 . addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . addOperand ( * Src1 ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . addOperand ( * Src2 ) ; } else { assert ( Src2 -> getReg ( ) == <STR_LIT> && <STR_LIT> ) ; Inst32 . addOperand ( copyRegOperandAsImplicit ( * Src2 ) ) ; } } ++ NumInstructionsShrunk ; MI . eraseFromParent ( ) ; foldImmediates ( * Inst32 , TII , MRI ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * Inst32 << '\n' ) ; }


<s> MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; unsigned SReg = Src2 -> getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( SReg != <STR_LIT> ) continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << '\n' ; ) ; MachineInstrBuilder Inst32 = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( Op32 ) ) ; Inst32 . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; Inst32 . addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . addOperand ( * Src1 ) ;


<s> if ( Src0 . isImm ( ) && TII -> isLiteralConstant ( Src0 , TII -> getOpSize ( MI , Src0Idx ) ) ) return ; if ( Src0 . isReg ( ) && ! isVGPR ( & Src0 , TRI , MRI ) ) return ; if ( Src0 . isReg ( ) && MRI . hasOneUse ( Src0 . getReg ( ) ) ) { unsigned Reg = Src0 . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) { Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { if ( MRI . use_empty ( Reg ) ) Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return ; } } } if ( TryToCommute && MI . isCommutable ( ) && TII -> commuteInstruction ( MI ) ) foldImmediates ( MI , TII , MRI , false ) ;


<s> case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ;

if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ;


<s> if ( TII -> isMIMG ( MI . getOpcode ( ) ) && ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 && MF . getProperties ( ) . hasProperty ( MachineFunctionProperties :: Property :: NoVRegs ) ) { shrinkMIMG ( MI ) ; continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; if ( ! TII -> canShrink ( MI , MRI ) ) { if ( ! MI . isCommutable ( ) || ! TII -> commuteInstruction ( MI ) || ! TII -> canShrink ( MI , MRI ) ) continue ; } int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( DstReg . isVirtual ( ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , VCCReg ) ; continue ; } if ( DstReg != VCCReg ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; Register SReg = Src2 -> getReg ( ) ; if ( SReg . isVirtual ( ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , VCCReg ) ; continue ; } if ( SReg != VCCReg ) continue ; } const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst ) { bool Next = false ; if ( SDst -> getReg ( ) != VCCReg ) { if ( SDst -> getReg ( ) . isVirtual ( ) ) MRI . setRegAllocationHint ( SDst -> getReg ( ) , <NUM_LIT> , VCCReg ) ; Next = true ; } const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 && Src2 -> getReg ( ) != VCCReg ) { if ( Src2 -> getReg ( ) . isVirtual ( ) ) MRI . setRegAllocationHint ( Src2 -> getReg ( ) , <NUM_LIT> , VCCReg ) ; Next = true ; } if ( Next ) continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstr * Inst32 = TII -> buildShrunkInst ( MI , Op32 ) ; ++ NumInstructionsShrunk ; copyExtraImplicitOps ( * Inst32 , MF , MI ) ; if ( SDst && SDst -> isDead ( ) ) Inst32 -> findRegisterDefOperand ( VCCReg ) -> setIsDead ( ) ; MI . eraseFromParent ( ) ; foldImmediates ( * Inst32 , TII , MRI ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Inst32 << '\n' ) ; }


<s> } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && SrcImm == Src0 ) { if ( ! TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) NewImm = <NUM_LIT> ; } if ( NewImm != <NUM_LIT> ) { if ( TargetRegisterInfo :: isVirtualRegister ( Dest -> getReg ( ) ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ; } if ( SrcReg -> isReg ( ) && SrcReg -> getReg ( ) == Dest -> getReg ( ) ) { MI . setDesc ( TII -> get ( Opc ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) {


<s> } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; unsigned SReg = Src2 -> getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( SReg != <STR_LIT> ) continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << '\n' ; ) ; MachineInstrBuilder Inst32 = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( Op32 ) ) ; int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: dst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && <STR_LIT> ) ; } Inst32 . addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . addOperand ( * Src1 ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . addOperand ( * Src2 ) ; } else { assert ( Src2 -> getReg ( ) == <STR_LIT> && <STR_LIT> ) ; Inst32 . addOperand ( copyRegOperandAsImplicit ( * Src2 ) ) ; } } ++ NumInstructionsShrunk ; MI . eraseFromParent ( ) ; foldImmediates ( * Inst32 , TII , MRI ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * Inst32 << '\n' ) ; } }


<s> if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . setSubReg ( <NUM_LIT> ) ; Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . setSubReg ( <NUM_LIT> ) ; Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { assert ( MRI . use_empty ( Reg ) ) ; Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return true ; } } } } if ( TryToCommute && MI . isCommutable ( ) ) { if ( TII -> commuteInstruction ( MI ) ) { if ( foldImmediates ( MI , TII , MRI , false ) ) return true ; TII -> commuteInstruction ( MI ) ;


<s> NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && SrcImm == Src0 ) { if ( ! TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) NewImm = <NUM_LIT> ; } if ( NewImm != <NUM_LIT> ) { if ( Register :: isVirtualRegister ( Dest -> getReg ( ) ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ; } if ( SrcReg -> isReg ( ) && SrcReg -> getReg ( ) == Dest -> getReg ( ) ) { MI . setDesc ( TII -> get ( Opc ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { Src0 -> ChangeToImmediate ( NewImm ) ; MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Dest -> getReg ( ) , false ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } else {


<s> MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . isCompare ( ) && TII -> isSOPC ( MI ) ) { shrinkScalarCompare ( TII , MI ) ; continue ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && Register :: isPhysicalRegister ( Dst . getReg ( ) ) ) { int32_t ReverseImm ; if ( isKImmOperand ( TII , Src ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; else if ( isReverseInlineImm ( TII , Src , ReverseImm ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; Src . setImm ( ReverseImm ) ; } } continue ; } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { if ( shrinkScalarLogicOp ( ST , MRI , TII , MI ) ) continue ; } if ( TII -> isMIMG ( MI . getOpcode ( ) ) && ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 && MF . getProperties ( ) . hasProperty ( MachineFunctionProperties :: Property :: NoVRegs ) ) { shrinkMIMG ( MI ) ; continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; if ( ! TII -> canShrink ( MI , MRI ) ) { if ( ! MI . isCommutable ( ) || ! TII -> commuteInstruction ( MI ) || ! TII -> canShrink ( MI , MRI ) ) continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Register :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , VCCReg ) ; continue ; } if ( DstReg != VCCReg ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; Register SReg = Src2 -> getReg ( ) ; if ( Register :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , VCCReg ) ; continue ; } if ( SReg != VCCReg ) continue ; } const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ;


<s> MI . setDesc ( TII -> get ( Opc ) ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && isKImmOperand ( TII , Src ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; if ( ! canShrink ( MI , TII , TRI , MRI ) ) { if ( ! MI . isCommutable ( ) || ! TII -> commuteInstruction ( MI ) || ! canShrink ( MI , TII , TRI , MRI ) ) continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg ( ) ) continue ; unsigned SReg = Src2 -> getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SReg ) ) { MRI . setRegAllocationHint ( SReg , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( SReg != <STR_LIT> ) continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder Inst32 = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( Op32 ) ) ; int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: vdst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && <STR_LIT> ) ; } Inst32 . addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . addOperand ( * Src1 ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . addOperand ( * Src2 ) ; } else {

copyFlagsToImplicitVCC ( * Inst32 , * Src2 ) ;


<s> break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || ( Src1Mod && Src1Mod -> getImm ( ) != <NUM_LIT> ) ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ;

return ! TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) ;


<s> if ( MI . getOpcode ( ) == <STR_LIT> ) { MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && TargetRegisterInfo :: isPhysicalRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { int64_t Imm = Src . getImm ( ) ; if ( isInt < <NUM_LIT> > ( Imm ) && ! TII -> isInlineConstant ( Src , <NUM_LIT> ) ) { int32_t ReverseImm = reverseBits < int32_t > ( static_cast < int32_t > ( Imm ) ) ; if ( ReverseImm >= - <NUM_LIT> && ReverseImm <= <NUM_LIT> ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; Src . setImm ( ReverseImm ) ; continue ; } } } } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; if ( ! canShrink ( MI , TII , TRI , MRI ) ) { if ( ! MI . isCommutable ( ) || ! TII -> commuteInstruction ( & MI ) || ! canShrink ( MI , TII , TRI , MRI ) ) continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ;


<s> MRI . setRegAllocationHint ( SReg , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( SReg != <STR_LIT> ) continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder Inst32 = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( Op32 ) ) ; int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: vdst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && <STR_LIT> ) ; } Inst32 . addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . addOperand ( * Src1 ) ; const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . addOperand ( * Src2 ) ; } else { assert ( Src2 -> getReg ( ) == <STR_LIT> && <STR_LIT> ) ; Inst32 . addOperand ( copyRegOperandAsImplicit ( * Src2 ) ) ; } } ++ NumInstructionsShrunk ; MI . eraseFromParent ( ) ; foldImmediates ( * Inst32 , TII , MRI ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * Inst32 << '\n' ) ; } } return false ;


<s> if ( ! MRI . isSSA ( ) ) return ; assert ( TII -> isVOP1 ( MI . getOpcode ( ) ) || TII -> isVOP2 ( MI . getOpcode ( ) ) || TII -> isVOPC ( MI . getOpcode ( ) ) ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; if ( Src0 . isImm ( ) && TII -> isLiteralConstant ( Src0 , TII -> getOpSize ( MI , Src0Idx ) ) ) return ; if ( Src0 . isReg ( ) && ! isVGPR ( & Src0 , TRI , MRI ) ) return ; if ( Src0 . isReg ( ) && MRI . hasOneUse ( Src0 . getReg ( ) ) ) { unsigned Reg = Src0 . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ;

if ( Def && Def -> isMoveImmediate ( ) ) {


<s> return new <ISA_LIT>TypeRewriter ( ) ;


<s> v4i32 = VectorType :: get ( Type :: getInt32Ty ( M . getContext ( ) ) , <NUM_LIT> ) ; return false ;


<s> const char * getPassName ( ) const override {


<s> bool <ISA_LIT>TypeRewriter :: runOnFunction ( Function & F ) { if ( <STR_LIT> ( F ) == ShaderType :: COMPUTE ) return false ; visit ( F ) ; visit ( F ) ;


<s> <ISA_LIT>TypeRewriter ( ) : FunctionPass ( ID ) {


<s> IRBuilder < > Builder ( & I ) ; if ( I . getDestTy ( ) != v4i32 ) { return ; }

if ( BitCastInst * Op = dyn_cast < BitCastInst > ( I . getOperand ( <NUM_LIT> ) ) ) {


<s> void <ISA_LIT>TypeRewriter :: visitBitCast ( BitCastInst & I ) { IRBuilder < > Builder ( & I ) ; if ( I . getDestTy ( ) != v4i32 ) { return ; }


<s> Types . push_back ( v4i32 ) ; NeedToReplace = true ; Name = Name + <STR_LIT> ; } else if ( Arg -> getType ( ) -> isVectorTy ( ) && Arg -> getType ( ) -> getVectorNumElements ( ) == <NUM_LIT> && Arg -> getType ( ) -> getVectorElementType ( ) == Type :: getInt32Ty ( I . getContext ( ) ) ) { Type * ElementTy = Arg -> getType ( ) -> getVectorElementType ( ) ; std :: string TypeName = <STR_LIT> ; InsertElementInst * Def = cast < InsertElementInst > ( Arg ) ; Args . push_back ( Def -> getOperand ( <NUM_LIT> ) ) ;


<s> Type * PtrTy = Ptr -> getType ( ) ; Type * ElemTy = PtrTy -> getPointerElementType ( ) ; IRBuilder < > Builder ( & I ) ; if ( ElemTy == v16i8 ) { Value * BitCast = Builder . CreateBitCast ( Ptr , PointerType :: get ( v4i32 , PtrTy -> getPointerAddressSpace ( ) ) ) ; LoadInst * Load = Builder . CreateLoad ( BitCast ) ; SmallVector < std :: pair < unsigned , MDNode * > , <NUM_LIT> > MD ; I . getAllMetadataOtherThanDebugLoc ( MD ) ; for ( unsigned i = <NUM_LIT> , e = MD . size ( ) ; i != e ; ++ i ) { Load -> setMetadata ( MD [ i ] . first , MD [ i ] . second ) ; }

Value * BitCastLoad = Builder . CreateBitCast ( Load , I . getType ( ) ) ;


<s> visit ( F ) ; visit ( F ) ; return false ;


<s> if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ;


<s> Types . push_back ( v4i32 ) ; NeedToReplace = true ; Name = Name + <STR_LIT> ; } else if ( Arg -> getType ( ) -> isVectorTy ( ) && Arg -> getType ( ) -> getVectorNumElements ( ) == <NUM_LIT> && Arg -> getType ( ) -> getVectorElementType ( ) == Type :: getInt32Ty ( I . getContext ( ) ) ) { Type * ElementTy = Arg -> getType ( ) -> getVectorElementType ( ) ; std :: string TypeName = <STR_LIT> ; InsertElementInst * Def = cast < InsertElementInst > ( Arg ) ; Args . push_back ( Def -> getOperand ( <NUM_LIT> ) ) ; Types . push_back ( ElementTy ) ; std :: string VecTypeName = <STR_LIT> + TypeName ; Name = Name . replace ( Name . find ( VecTypeName ) , VecTypeName . length ( ) , TypeName ) ; NeedToReplace = true ;