<s> case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_LS : return


<s> getOriginalFunctionArgs ( DAG , MF . getFunction ( ) , Ins , LocalIns ) ; AnalyzeFormalArguments ( CCInfo , LocalIns ) ; for ( unsigned i = <NUM_LIT> , e = Ins . size ( ) ; i < e ; ++ i ) { CCValAssign & VA = ArgLocs [ i ] ; const <STR_LIT> & In = Ins [ i ] ; EVT VT = In . VT ; EVT MemVT = VA . getLocVT ( ) ; if ( ! VT . isVector ( ) && MemVT . isVector ( ) ) { MemVT = MemVT . getVectorElementType ( ) ; } if ( <STR_LIT> ( CallConv ) ) { unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ; } PointerType * PtrTy = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; <STR_LIT> Ext = <STR_LIT> ; if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) { Ext = <STR_LIT> ; } unsigned ValBase = ArgLocs [ In . getOrigArgIndex ( ) ] .


<s> getOriginalFunctionArgs ( DAG , MF . getFunction ( ) , Ins , LocalIns ) ; AnalyzeFormalArguments ( CCInfo , LocalIns ) ; for ( unsigned i = <NUM_LIT> , e = Ins . size ( ) ; i < e ; ++ i ) { CCValAssign & VA = ArgLocs [ i ] ; const <STR_LIT> & In = Ins [ i ] ; EVT VT = In . VT ; EVT MemVT = VA . getLocVT ( ) ; if ( ! VT . isVector ( ) && MemVT . isVector ( ) ) { MemVT = MemVT . getVectorElementType ( ) ; } if ( <STR_LIT> ( CallConv ) ) { unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ; } PointerType * PtrTy = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; <STR_LIT> Ext = <STR_LIT> ; if ( MemVT . getScalarSizeInBits ( ) !=


<s> SDValue <ISA_LIT>TargetLowering :: LowerImplicitParameter ( SelectionDAG & DAG , EVT VT , const SDLoc & DL , unsigned DwordOffset ) const { unsigned ByteOffset = DwordOffset * <NUM_LIT> ; PointerType * PtrType = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; assert ( isInt < <NUM_LIT> > ( ByteOffset ) ) ; return DAG . getLoad ( VT , DL , DAG . getEntryNode ( ) , DAG . getConstant ( ByteOffset , DL , <STR_LIT>

) , MachinePointerInfo ( ConstantPointerNull :: get ( PtrType ) ) , false , false , false , <NUM_LIT> ) ;


<s> return DAG . getLoad ( VT , DL , DAG . getEntryNode ( ) , DAG . getConstant ( ByteOffset , DL , <STR_LIT> ) , MachinePointerInfo ( ConstantPointerNull :: get ( PtrType ) ) , false , false ,


<s> if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS && ExtType != <STR_LIT> && MemVT . bitsLT ( <STR_LIT> ) ) { return lowerPrivateExtLoad ( Op , DAG ) ; } SDLoc DL ( Op ) ; EVT VT = Op . getValueType ( ) ; SDValue Chain = LoadNode -> getChain ( ) ; SDValue Ptr = LoadNode -> getBasePtr ( ) ; if ( LoadNode -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS && VT . isVector ( ) ) { SDValue MergedValues [ <NUM_LIT> ] = { scalarizeVectorLoad ( LoadNode , DAG ) , Chain } ; return DAG . getMergeValues ( MergedValues , DL ) ; } int ConstantBlock = ConstantAddressBlock ( LoadNode -> getAddressSpace ( ) ) ; if ( ConstantBlock > - <NUM_LIT> && ( ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) || ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) ) ) { SDValue Result ; if ( isa < ConstantExpr > ( LoadNode -> getMemOperand ( ) -> getValue ( ) ) || isa < Constant > ( LoadNode -> getMemOperand ( ) -> getValue ( ) ) || isa < ConstantSDNode > ( Ptr ) ) { SDValue Slots [ <NUM_LIT> ] ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { SDValue NewPtr = DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , Ptr , DAG . getConstant ( <NUM_LIT> * i + ConstantBlock * <NUM_LIT> , DL , <STR_LIT> ) ) ; Slots [ i ] = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , NewPtr ) ; } EVT NewVT = <STR_LIT> ; unsigned NumElements = <NUM_LIT> ; if ( VT . isVector ( ) ) { NewVT = VT ; NumElements = VT . getVectorNumElements ( ) ; } Result = DAG . getBuildVector ( NewVT , DL , makeArrayRef ( Slots , NumElements ) ) ; } else { Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) , DAG . getConstant ( LoadNode -> getAddressSpace ( ) - <ISA_LIT>AS :: CONSTANT_BUFFER_0 , DL , <STR_LIT> ) ) ; } if ( ! VT .


<s> SDValue <ISA_LIT>TargetLowering :: LowerImplicitParameter ( SelectionDAG & DAG , EVT VT , const SDLoc & DL , unsigned DwordOffset ) const { unsigned ByteOffset = DwordOffset * <NUM_LIT> ; PointerType * PtrType = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , AMDGPUA<ISA_LIT> . PARAM_I_ADDRESS ) ; assert ( isInt < <NUM_LIT>


<s> if ( ConstantBlock > - <NUM_LIT> && ( ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) || ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) ) ) { SDValue Result ; if ( isa < Constant > ( LoadNode -> getMemOperand ( ) -> getValue ( ) ) || isa < ConstantSDNode > ( Ptr ) ) { return constBufferLoad ( LoadNode , LoadNode -> getAddressSpace ( ) , DAG ) ; } else { Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) , DAG . getConstant ( LoadNode -> getAddressSpace ( ) - AMDGPUA<ISA_LIT> . CONSTANT_BUFFER_0 , DL , <STR_LIT> ) ) ; } if ( ! VT . isVector ( ) ) { Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Result , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; } SDValue MergedValues [ <NUM_LIT> ] = { Result , Chain } ; return DAG . getMergeValues ( MergedValues , DL ) ; } if ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) { EVT MemVT = LoadNode -> getMemoryVT ( ) ; assert ( ! MemVT . isVector ( ) && ( MemVT == <STR_LIT> || MemVT == <STR_LIT>


<s> if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) { Ext = <STR_LIT> ; } unsigned ValBase = ArgLocs [ In . getOrigArgIndex ( ) ] . getLocMemOffset ( ) ; unsigned PartOffset = VA . getLocMemOffset ( ) ; unsigned Offset = Subtarget -> getExplicitKernelArgOffset ( MF ) + VA . getLocMemOffset ( ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) , PartOffset - ValBase ) ; SDValue Arg = DAG . getLoad ( <STR_LIT> , Ext , VT , DL , Chain , DAG . getConstant ( Offset , DL , <STR_LIT> ) , DAG . getUNDEF ( <STR_LIT> ) , PtrInfo , MemVT , <NUM_LIT> , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; InVals . push_back ( Arg ) ; MFI -> setABIArgOffset ( Offset + MemVT . getStoreSize ( ) ) ; } return


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; ReverseVector ( orderedBlks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; printOrderedBlocks ( ) ; ) ; int NumIter = <NUM_LIT> ; bool Finish = false ; MachineBasicBlock * MBB ; bool MakeProgress = false ; int NumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; do { ++ NumIter ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NumIter << <STR_LIT> << NumRemainedBlk << <STR_LIT> ; ) ; ( void ) NumIter ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator It = OrderedBlks . begin ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator E = OrderedBlks . end ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator SccBeginIter = It ; MachineBasicBlock * SccBeginMBB = nullptr ; int SccNumBlk = <NUM_LIT> ; int SccNumIter ; while ( It != E ) { MBB = * It ; if ( ! SccBeginMBB ) { SccBeginIter = It ; SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ( void ) SccNumIter ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else { ContNextScc = false ; } if ( ContNextScc ) SccBeginMBB = nullptr ; } MachineBasicBlock * EntryMBB = * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ; if ( EntryMBB -> succ_empty ( ) ) { Finish = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ; for ( auto & It : BlockInfoMap


<s> insertInstrEnd ( DummyExitBlk , <STR_LIT> ) ; for ( MachineBasicBlock * MBB : RetMBB ) { if ( MachineInstr * MI = getReturnInstr ( MBB ) ) MI -> eraseFromParent ( ) ; MBB -> addSuccessor ( DummyExitBlk ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber


<s> MachineFunction * Func = MBB -> getParent ( ) ; MachineBasicBlock * NewMBB = Func -> CreateMachineBasicBlock


<s> int Cloned = <NUM_LIT> ; assert ( PreMBB -> isSuccessor ( SrcMBB ) ) ; while ( SrcMBB && SrcMBB != DstMBB ) { assert ( SrcMBB -> succ_size ( ) == <NUM_LIT> ) ; if ( SrcMBB -> pred_size ( ) > <NUM_LIT> ) { SrcMBB = cloneBlockForPredecessor ( SrcMBB , PreMBB ) ; ++ Cloned ; } PreMBB = SrcMBB ; SrcMBB = * SrcMBB -> succ_begin ( ) ; } return


<s> for ( MachineBasicBlock * Succ : SrcMBB -> successors ( ) ) DstMBB ->


<s> int <ISA_LIT>MachineCFGStructurizer :: countActiveBlock ( MBBVector :: const_iterator It , MBBVector :: const_iterator E ) const { int Count = <NUM_LIT> ; while ( It != E ) { if ( ! isRetiredBlock ( * It ) ) ++ Count ; ++ It ; } return Count


<s> return new <ISA_LIT>MachineCFGStructurizer


<s> int <ISA_LIT>MachineCFGStructurizer :: getContinueNzeroOpcode ( int OldOpcode ) { switch ( OldOpcode


<s> for ( MachineInstr & MI : * MBB ) if ( MI . getDebugLoc ( ) ) DL = MI . getDebugLoc ( ) ; return DL


<s> for ( MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) , E = MBB -> rend ( ) ; It != E ; ++ It ) { MachineInstr * MI = & * It ; if ( MI )


<s> MachineBasicBlock * <ISA_LIT>MachineCFGStructurizer :: getLoopLandInfo ( MachineLoop * LoopRep ) const { LoopLandInfoMap :: const_iterator It = LLInfoMap


<s> MachineInstr * <ISA_LIT>MachineCFGStructurizer :: getNormalBlockBranchInstr ( MachineBasicBlock * MBB ) { MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) ; MachineInstr * MI = & * It ; if ( MI && ( isCondBranch ( MI )

|| isUncondBranch ( MI ) ) ) return MI ;


<s> MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) ; if ( It != MBB -> rend ( ) ) { MachineInstr * instr = & ( * It ) ; if ( instr -> getOpcode ( ) == <STR_LIT>


<s> if ( It == BlockInfoMap . end ( ) ) return INVALIDSCCNUM ; return ( * It ) . second


<s> insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } Register InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } if ( LandBlkHasOtherPred ) { insertInstrBefore ( I , <STR_LIT> ) ; for ( MachineBasicBlock * MBB : LandBlk -> predecessors ( ) ) if ( MBB != TrueMBB && MBB != FalseMBB ) report_fatal_error ( <STR_LIT> ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; * LandMBBPtr = LandBlk ; return NumNewBlk


<s> MachineInstr * OldMI = & ( * I ) ; MachineBasicBlock * MBB = OldMI -> getParent ( ) ; MachineInstr * NewMBB = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ; MBB -> insert ( I , NewMBB ) ; SHOWNEWINSTR ( NewMBB ) ; return NewMBB


<s> MachineBasicBlock * MBB = OldMI -> getParent ( ) ; MachineInstr * NewMBB = MBB -> getParent ( ) -> CreateMachineInstr ( TII ->


<s> if ( ! LoopLand ) return true ; if ( ! isRetiredBlock ( LoopLand ) ) return true ; LoopRep = LoopRep -> getParentLoop ( ) ; } return false


<s> bool <ISA_LIT>MachineCFGStructurizer :: isCondBranch ( MachineInstr * MI ) { switch ( MI -> getOpcode ( ) ) { case <STR_LIT>


<s> bool <ISA_LIT>MachineCFGStructurizer :: isRetiredBlock ( MachineBasicBlock * MBB ) const { MBBInfoMap :: const_iterator It = BlockInfoMap . find ( MBB ) ; if ( It == BlockInfoMap . end ( ) ) return false ; return ( * It ) . second


<s> bool <ISA_LIT>MachineCFGStructurizer :: isReturnBlock ( MachineBasicBlock * MBB ) { MachineInstr * MI = getReturnInstr ( MBB ) ; bool IsReturn = MBB -> succ_empty


<s> if ( TheEntry ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Src1MBB -> getNumber ( ) << <STR_LIT> << Src2MBB -> getNumber (


<s> for ( MachineLoop * ExaminedLoop : NestedLoops ) { if ( ExaminedLoop -> getNumBlocks ( ) == <NUM_LIT> || Visited [ ExaminedLoop ] ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ExaminedLoop -> dump ( ) ; ) ; int NumBreak = mergeLoop ( ExaminedLoop ) ; if ( NumBreak == - <NUM_LIT> ) break ; Num += NumBreak


<s> for ( auto & It : * MLI ) for ( MachineLoop * ML : depth_first ( It ) ) NestedLoops . push_front ( ML ) ; if ( NestedLoops . empty ( ) ) return <NUM_LIT> ; int Num = <NUM_LIT> ; for ( MachineLoop * ExaminedLoop : NestedLoops ) { if ( ExaminedLoop -> getNumBlocks ( ) == <NUM_LIT> || Visited [ ExaminedLoop ] ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ExaminedLoop -> dump ( ) ; ) ; int NumBreak = mergeLoop ( ExaminedLoop ) ; if ( NumBreak == - <NUM_LIT> ) break ; Num += NumBreak ; } return Num


<s> void <ISA_LIT>MachineCFGStructurizer :: mergeIfthenelseBlock ( MachineInstr * BranchMI , MachineBasicBlock * MBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB , MachineBasicBlock * LandMBB ) { assert ( TrueMBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) ; dbgs ( ) << <STR_LIT> ; if ( TrueMBB ) { dbgs ( ) << <STR_LIT> << TrueMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( ! LandMBB ) { dbgs ( ) << <STR_LIT> ; } else { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; ) ; int OldOpcode = BranchMI -> getOpcode ( ) ; DebugLoc BranchDL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock :: iterator I = BranchMI ; insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ; if ( TrueMBB ) { MBB -> splice ( I , TrueMBB , TrueMBB -> begin ( ) , TrueMBB -> end ( ) ) ; MBB -> removeSuccessor ( TrueMBB , true ) ; if ( LandMBB && TrueMBB -> succ_size ( ) != <NUM_LIT> )

TrueMBB -> removeSuccessor ( LandMBB , true ) ;


<s> void <ISA_LIT>MachineCFGStructurizer :: mergeLoopbreakBlock ( MachineBasicBlock * ExitingMBB , MachineBasicBlock * LandMBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ExitingMBB -> getNumber ( ) << <STR_LIT> << LandMBB -> getNumber ( ) << <STR_LIT> ; ) ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( ExitingMBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; DebugLoc DL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock * TrueBranch = getTrueBranch ( BranchMI ) ; MachineBasicBlock :: iterator I = BranchMI ; if ( TrueBranch != LandMBB ) reversePredicateSetter ( I , * I -> getParent ( ) ) ; insertCondBranchBefore ( ExitingMBB , I , <STR_LIT> , <STR_LIT> , DL ) ; insertInstrBefore ( I , <STR_LIT> ) ; insertInstrBefore ( I , <STR_LIT> )


<s> DstMBB -> splice ( DstMBB -> end ( ) , SrcMBB , SrcMBB -> begin ( ) , SrcMBB -> end ( ) ) ; DstMBB -> removeSuccessor ( SrcMBB , true ) ; cloneSuccessorList ( DstMBB , SrcMBB ) ; removeSuccessor ( SrcMBB ) ; MLI -> removeBlock ( SrcMBB )


<s> } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; SpliceEnd = BranchMI ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << <STR_LIT> ; ) ; DstMBB -> splice ( I , SrcMBB , SrcMBB -> begin ( ) , SpliceEnd ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size


<s> unsigned BlkSize = MBB -> size ( ) ; return ( ( BlkSize > BlockSizeThreshold ) && ( BlkSize * ( MBB -> pred_size ( )

- <NUM_LIT> ) > CloneInstrThreshold ) ) ;


<s> if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ; FuncRep -> push_back ( DummyExitBlk


<s> MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; MachineBasicBlock * LoopLatch = LoopRep -> getLoopLatch ( ) ; if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ; FuncRep -> push_back ( DummyExitBlk ) ; SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI << <STR_LIT> ; ) ; LLVMContext & Ctx = LoopHeader -> getParent ( ) -> getFunction ( ) . getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; return nullptr


<s> for ( MachineBasicBlock * MBB : SccNext ) { OrderedBlks . push_back ( MBB ) ; recordSccnum ( MBB , SccNum ) ; } } for ( auto * MBB : nodes ( MF ) ) { SccNum = getSCCNum ( MBB ) ; if ( SccNum == INVALIDSCCNUM ) dbgs ( ) <<

<STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ;


<s> int NumMatch = <NUM_LIT> ; int CurMatch ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; while ( ( CurMatch = patternMatchGroup ( MBB ) ) > <NUM_LIT> ) NumMatch += CurMatch ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> <<


<s> MBBVector ExitingMBBs ; LoopRep -> getExitingBlocks ( ExitingMBBs ) ; if ( ExitingMBBs . size ( ) == <NUM_LIT> ) { MachineBasicBlock * DummyExitBlk = normalizeInfiniteLoopExit ( LoopRep ) ; if ( DummyExitBlk ) RetBlks . push_back ( DummyExitBlk ) ; } } for ( MachineBasicBlock * MBB : OrderedBlks ) { removeUnconditionalBranch ( MBB ) ; removeRedundantConditionalBranch ( MBB ) ; if ( isReturnBlock ( MBB ) ) { RetBlks . push_back (


<s> void <ISA_LIT>MachineCFGStructurizer :: recordSccnum ( MachineBasicBlock * MBB , int SccNum ) { BlockInformation * & srcBlkInfo = BlockInfoMap [


<s> assert ( BranchMI && isCondBranch ( BranchMI ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; BranchMI -> eraseFromParent ( ) ; SHOWNEWBLK ( MBB1 , <STR_LIT> ) ; MBB -> removeSuccessor ( MBB1


<s> void <ISA_LIT>MachineCFGStructurizer :: removeRedundantConditionalBranch ( MachineBasicBlock * MBB ) { if ( MBB -> succ_size ( ) != <NUM_LIT> ) return ; MachineBasicBlock * MBB1 = * MBB -> succ_begin ( ) ; MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ; if ( MBB1 != MBB2 ) return ; MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI


<s> while ( ( BranchMI = getLoopendBlockBranchInstr ( MBB ) ) && isUncondBranch ( BranchMI ) ) { LLVM_DEBUG ( dbgs ( ) <<


<s> void <ISA_LIT>MachineCFGStructurizer :: replaceInstrUseOfBlockWith ( MachineBasicBlock * SrcMBB , MachineBasicBlock

* OldMBB , MachineBasicBlock * NewBlk ) {


<s> void <ISA_LIT>MachineCFGStructurizer :: reversePredicateSetter ( MachineBasicBlock :: iterator I , MachineBasicBlock & MBB ) { assert ( I . isValid ( ) && <STR_LIT> ) ; for ( ; ; -- I ) { if ( I == MBB . end ( ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( I -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT>


<s> MachineBasicBlock * childBlk = * MBB -> succ_begin ( ) ; if ( childBlk -> pred_size ( ) != <NUM_LIT> || isActiveLoophead ( childBlk ) ) return <NUM_LIT> ; mergeSerialBlock ( MBB


<s> insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; } else { int BranchOpcode = TrueBranch == ContMBB ? getContinueNzeroOpcode ( OldOpcode ) : getContinueZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; } MI -> eraseFromParent ( ) ; } else { insertInstrEnd ( ContingMBB , <STR_LIT> , getLastDebugLocInBB ( ContingMBB )


<s> if ( ! UseContinueLogical ) { int BranchOpcode = TrueBranch == ContMBB ? getBranchNzeroOpcode ( OldOpcode ) : getBranchZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; } else { int BranchOpcode = TrueBranch == ContMBB ? getContinueNzeroOpcode ( OldOpcode ) : getContinueZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; } MI -> eraseFromParent (


<s> MI -> getOperand ( <NUM_LIT> ) . setMBB ( MBB


<s> dbgs ( ) << <STR_LIT> ; } } if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) << <STR_LIT> << FalseMBB -> size ( ) << <STR_LIT> << FalseMBB -> pred_size ( ) ; if ( Detail ) { dbgs ( ) << <STR_LIT> ; FalseMBB -> print ( dbgs ( ) ) ; dbgs ( ) << <STR_LIT> ; } } if ( LandMBB ) { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) << <STR_LIT> << LandMBB

-> size ( ) << <STR_LIT> << LandMBB -> pred_size ( ) ;


<s> <ISA_LIT>MachineCFGStructurizer :: PathToKind <ISA_LIT>MachineCFGStructurizer :: singlePathTo ( MachineBasicBlock * SrcMBB , MachineBasicBlock * DstMBB , bool AllowSideEntry ) const { assert ( DstMBB ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; while ( SrcMBB && SrcMBB -> succ_size ( ) == <NUM_LIT> ) { SrcMBB = * SrcMBB -> succ_begin ( ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; if ( ! AllowSideEntry && SrcMBB -> pred_size ( ) > <NUM_LIT> ) return Not_SinglePath ; } if ( SrcMBB && SrcMBB -> succ_size ( ) == <NUM_LIT> ) return SinglePath_NotInPath ; return Not_SinglePath


<s> while ( It != E ) { if ( Pre -> getOpcode ( ) == <STR_LIT> && It -> getOpcode ( ) == <STR_LIT> ) ContInstr . push_back ( & * Pre ) ; Pre = It ; ++ It ; } for ( unsigned i = <NUM_LIT> ; i < ContInstr . size ( ) ; ++

i ) ContInstr [ i ] -> eraseFromParent ( ) ;


<s> static const AluKind IndexToID [ ] = { AluT_X , AluT_Y , AluT_Z , AluT_W } ; SUnit * SlotedSU = PopInst ( AvailableAlus [ IndexToID [ Slot ] ] , AnyAlu ) ; if ( SlotedSU ) return SlotedSU ; SUnit * UnslotedSU = PopInst ( AvailableAlus


<s> int <ISA_LIT>SchedStrategy :: getInstKind ( SUnit * SU ) { int Opcode = SU -> getInstr ( ) -> getOpcode ( ) ; if ( TII -> usesTextureCache ( Opcode ) || TII -> usesVertexCache ( Opcode ) ) return IDFetch ; if ( TII -> isALUInstr ( Opcode ) ) { return IDAlu


<s> assert ( GPRCount && <STR_LIT> ) ; return <NUM_LIT> / GPRCount


<s> static unsigned getWFCountLimitedByGPR ( unsigned GPRCount ) { assert ( GPRCount && <STR_LIT> ) ; return <NUM_LIT> / GPRCount


<s> dbgs ( ) << <STR_LIT> ; SU -> dump (


<s> static bool isPhysicalRegCopy ( MachineInstr * MI ) { if ( MI -> getOpcode ( ) != <STR_LIT> ) return false ; return ! TargetRegisterInfo :: isVirtualRegister ( MI -> getOperand ( <NUM_LIT> ) . getReg


<s> if ( MI -> getOpcode ( ) != <STR_LIT> ) return


<s> AluKind AK = getAluKind ( QSrc [ i ] ) ; AvailableAlus [ AK ] . push_back (


<s> QDst . insert ( QDst . end ( ) , QSrc . begin ( ) , QSrc


<s> if ( NeededWF > getWFCountLimitedByGPR ( NearRegisterRequirement ) ) AllowSwitchFromAlu = true ; } } if ( ! SU && ( ( AllowSwitchToAlu && CurInstKind != IDAlu ) || ( ! AllowSwitchFromAlu && CurInstKind == IDAlu ) ) ) { SU = pickAlu ( ) ; if ( ! SU && ! PhysicalRegCopy . empty ( ) ) { SU = PhysicalRegCopy . front ( ) ; PhysicalRegCopy . erase ( PhysicalRegCopy . begin ( ) ) ; } if ( SU ) { if ( CurEmitted >= InstKindLimit [ IDAlu ] ) CurEmitted = <NUM_LIT> ; NextInstKind = IDAlu ; } } if ( ! SU ) { SU = pickOther ( IDFetch ) ; if ( SU ) NextInstKind = IDFetch ; } if ( ! SU ) { SU = pickOther ( IDOther ) ; if ( SU ) NextInstKind = IDOther ; } DEBUG ( if ( SU ) { dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG


<s> if ( ! AQ . empty ( ) ) { SU = AQ . back ( ) ; AQ . resize ( AQ . size ( ) - <NUM_LIT>


<s> SUnit * <ISA_LIT>SchedStrategy :: pickOther ( int QID ) { SUnit * SU = nullptr ; std :: vector < SUnit * > & AQ = Available [ QID ] ; if ( AQ . empty ( ) ) { MoveUnits ( Pending [ QID ] , AQ ) ; } if ( ! AQ . empty ( ) ) { SU = AQ . back ( ) ; AQ . resize ( AQ . size ( ) - <NUM_LIT> ) ; } return SU


<s> InstructionsGroupCandidate . push_back ( SU -> getInstr ( ) ) ; if ( TII -> fitsConstReadLimitations ( InstructionsGroupCandidate ) && ( ! AnyALU || ! TII -> isVectorOnly ( SU -> getInstr ( ) ) ) ) { InstructionsGroupCandidate . pop_back ( ) ; Q . erase ( ( It + <NUM_LIT> ) . base ( ) ) ; return SU ; } else { InstructionsGroupCandidate . pop_back (


<s> void <ISA_LIT>SchedStrategy :: releaseBottomNode ( SUnit * SU ) { DEBUG ( dbgs ( ) << <STR_LIT> ;


<s> void <ISA_LIT>SchedStrategy :: releaseTopNode ( SUnit * SU ) { DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG )


<s> void <ISA_LIT>SchedStrategy :: schedNode ( SUnit * SU , bool IsTopNode ) { if ( NextInstKind != CurInstKind ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( NextInstKind != IDAlu ) OccupedSlotsMask |= <NUM_LIT> ; CurEmitted = <NUM_LIT> ; CurInstKind = NextInstKind ; } if ( CurInstKind == IDAlu ) { AluInstCount ++ ; switch ( getAluKind ( SU ) ) { case AluT_XYZW : CurEmitted += <NUM_LIT> ; break ; case AluDiscarded : break ; default : { ++ CurEmitted ; for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO .

getReg ( ) == <STR_LIT> ) ++ CurEmitted ;


<s> Register DestReg = MI -> getOperand ( DstIndex ) . getReg ( ) ; for ( MachineInstr :: mop_iterator It = MI -> operands_begin ( ) , E = MI -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && ! MO . isDef ( ) && MO . getReg ( ) == DestReg ) return ; } switch ( Slot ) { case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> : MRI -> constrainRegClass ( DestReg


<s> if ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) { return AluDiscarded ; } break ; default : break ; } if ( TII -> isVector ( * MI ) || TII -> isCubeOp ( MI -> getOpcode ( ) ) || TII -> isReductionOp ( MI -> getOpcode ( ) ) || MI -> getOpcode ( ) == <STR_LIT> ) { return AluT_XYZW ; } if ( TII -> isLD<ISA_LIT>nstr ( MI -> getOpcode ( ) ) ) { return AluT_X ; } unsigned DestSubReg = MI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; switch ( DestSubReg ) { case <STR_LIT> : return AluT_X ; case <STR_LIT> : return AluT_Y


<s> void <ISA_LIT>SchedStrategy :: initialize ( ScheduleDAGMI * dag ) { assert ( dag -> hasVRegLiveness ( ) && <STR_LIT> ) ; DAG = static_cast < ScheduleDAGMILive * > ( dag ) ; const <ISA_LIT>Subtarget & ST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = static_cast < const <ISA_LIT>InstrInfo * > ( DAG -> TII ) ; TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( DAG -> TRI ) ; VLIW5 = ! ST . hasCaymanISA ( ) ; MRI = & DAG -> MRI ; CurInstKind = IDOther ; CurEmitted = <NUM_LIT> ; OccupiedSlotsMask = <NUM_LIT> ; InstKindLimit [ IDAlu ] = TII -> getMaxAlusPerClause ( ) ; InstKindLimit [ IDOther ] = <NUM_LIT> ; InstKindLimit [ IDFetch ] = ST . getTexVTXClauseSize


<s> static bool isPhysicalRegCopy ( MachineInstr * MI ) { if ( MI -> getOpcode ( ) !=


<s> void <ISA_LIT>SchedStrategy :: MoveUnits ( std :: vector < SUnit * > & QSrc , std ::


<s> while ( AvailablesAluCount ( ) || ! Pending [ IDAlu ] . empty ( ) ) { if ( ! OccupiedSlotsMask ) { if ( ! AvailableAlus [ AluPredX ] . empty ( ) ) { OccupiedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluPredX ] , false ) ; } if ( ! AvailableAlus [ AluDiscarded ] . empty ( ) ) { OccupiedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluDiscarded ] , false ) ; } if ( ! AvailableAlus [ AluT_XYZW ] . empty ( ) ) { OccupiedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluT_XYZW ] , false ) ; } } bool TransSlotOccupied = OccupiedSlotsMask & <NUM_LIT> ; if ( ! TransSlotOccupied && VLIW5 ) { if ( ! AvailableAlus [ AluTrans ] . empty ( ) ) { OccupiedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluTrans ] , false ) ; } SUnit * SU = AttemptFillSlot ( <NUM_LIT> , true ) ; if ( SU ) { OccupiedSlotsMask |= <NUM_LIT> ; return SU ; } } for ( int Chan = <NUM_LIT> ; Chan > - <NUM_LIT> ; -- Chan ) { bool isOccupied = OccupiedSlotsMask & ( <NUM_LIT> << Chan ) ; if ( ! isOccupied ) { SUnit * SU = AttemptFillSlot ( Chan , false


<s> float ALUFetchRationEstimate = ( AluInstCount + AvailablesAluCount ( ) + Pending [ IDAlu ] . size ( ) ) / ( FetchInstCount + Available [ IDFetch ] . size ( ) ) ; if ( ALUFetchRationEstimate == <NUM_LIT> ) { AllowSwitchFromAlu = true ; } else { unsigned NeededWF = <NUM_LIT> / ALUFetchRationEstimate ; LLVM_DEBUG ( dbgs ( ) << NeededWF << <STR_LIT> ) ; unsigned NearRegisterRequirement = <NUM_LIT> * Available [ IDFetch ] . size ( ) ; if ( NeededWF > getWFCountLimitedByGPR ( NearRegisterRequirement ) ) AllowSwitchFromAlu = true ; } } if ( ! SU && ( ( AllowSwitchToAlu && CurInstKind != IDAlu ) || ( ! AllowSwitchFromAlu && CurInstKind == IDAlu ) ) ) { SU = pickAlu ( ) ; if ( ! SU && ! PhysicalRegCopy . empty ( ) ) { SU = PhysicalRegCopy . front ( ) ; PhysicalRegCopy . erase ( PhysicalRegCopy . begin ( ) ) ; } if ( SU ) { if ( CurEmitted >= InstKindLimit [ IDAlu ] ) CurEmitted = <NUM_LIT> ; NextInstKind = IDAlu ; } } if ( ! SU


<s> SUnit * SU = nullptr ; std :: vector < SUnit * > & AQ = Available [ QID ] ; if ( AQ . empty ( ) ) { MoveUnits ( Pending [ QID ] , AQ ) ; } if ( ! AQ . empty ( ) ) { SU = AQ . back ( ) ; AQ . pop_back


<s> It != E ; ++ It ) { SUnit * SU = * It ; InstructionsGroupCandidate . push_back ( SU -> getInstr ( ) ) ; if ( TII -> fitsConstReadLimitations ( InstructionsGroupCandidate ) && ( ! AnyALU || ! TII -> isVectorOnly ( * SU -> getInstr ( ) ) ) ) { InstructionsGroupCandidate . pop_back ( ) ; Q . erase ( ( It + <NUM_LIT> ) . base ( ) ) ; return SU ; }


<s> assert ( OccupiedSlotsMask && <STR_LIT> ) ; OccupiedSlotsMask = <NUM_LIT> ; InstructionsGroupCandidate . clear


<s> bool <ISA_LIT>SchedStrategy :: regBelongsToClass ( Register Reg , const TargetRegisterClass * RC ) const { if ( ! Reg . isVirtual ( ) ) { return RC -> contains ( Reg ) ; } else


<s> int IK = getInstKind ( SU ) ; if ( IK == IDOther ) Available [ IDOther ] . push_back ( SU ) ; else Pending [ IK ] . push_back ( SU


<s> if ( IK == IDOther ) Available [ IDOther ] . push_back ( SU ) ; else Pending [ IK ] .


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SU


<s> void <ISA_LIT>SchedStrategy :: releaseTopNode ( SUnit * SU


<s> switch ( getAluKind ( SU ) ) { case AluT_XYZW : CurEmitted += <NUM_LIT> ; break ; case AluDiscarded : break ; default : { ++ CurEmitted ; for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ; } } } } else { ++ CurEmitted ; } LLVM_DEBUG ( dbgs ( ) << CurEmitted << <STR_LIT>


<s> dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG


<s> void <ISA_LIT>SchedStrategy :: initialize ( ScheduleDAGMI * dag ) { assert ( dag -> hasVRegLiveness ( ) && <STR_LIT> ) ; DAG = static_cast < ScheduleDAGMILive * > ( dag ) ; const <ISA_LIT>Subtarget & ST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = static_cast < const <ISA_LIT>InstrInfo * > ( DAG -> TII ) ; TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( DAG -> TRI ) ; VLIW5 = ! ST . hasCaymanISA ( ) ; MRI = & DAG -> MRI ; CurInstKind = IDOther ; CurEmitted = <NUM_LIT> ; OccupedSlotsMask = <NUM_LIT> ; InstKindLimit [ IDAlu ] = TII -> getMaxAlusPerClause


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ; ) ; if ( isPhysicalRegCopy ( SU -> getInstr ( ) ) ) { PhysicalRegCopy . push_back ( SU ) ; return ; } int IK = getInstKind ( SU ) ; if ( IK == IDOther ) Available [ IDOther ] .


<s> void <ISA_LIT>SchedStrategy :: releaseTopNode ( SUnit * SU ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump (


<s> if ( CurInstKind == IDAlu ) { AluInstCount ++ ; switch ( getAluKind ( SU ) ) { case AluT_XYZW : CurEmitted += <NUM_LIT> ; break ; case AluDiscarded : break ; default : { ++ CurEmitted ; for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ; } } } } else { ++ CurEmitted ; } LLVM_DEBUG ( dbgs ( ) << CurEmitted << <STR_LIT> ) ; if ( CurInstKind != IDFetch ) { MoveUnits ( Pending [ IDFetch ] , Available [ IDFetch


<s> if ( DstIndex == - <NUM_LIT> ) { return ; } unsigned DestReg = MI -> getOperand ( DstIndex ) . getReg ( ) ; for ( MachineInstr :: mop_iterator It = MI -> operands_begin ( ) , E = MI -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && ! MO . isDef ( ) && MO . getReg ( ) == DestReg ) return ; } switch ( Slot ) { case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT>


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return AluT_XYZW ; case <STR_LIT> : if ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) { return AluDiscarded ; } break ; default : break ; } if ( TII -> isVector ( * MI ) || TII -> isCubeOp ( MI -> getOpcode ( ) ) || TII -> isReductionOp ( MI -> getOpcode ( ) ) || MI -> getOpcode ( ) == <STR_LIT> ) { return AluT_XYZW ; } if ( TII -> isLD<ISA_LIT>nstr ( MI -> getOpcode ( ) ) ) { return AluT_X ; } unsigned DestSubReg = MI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; switch ( DestSubReg ) { case <STR_LIT> : return AluT_X ; case <STR_LIT> : return AluT_Y ; case <STR_LIT> : return AluT_Z ; case <STR_LIT>


<s> return ! TargetRegisterInfo :: isVirtualRegister ( MI -> getOperand ( <NUM_LIT> )


<s> if ( MI -> getOpcode ( ) != <STR_LIT> ) return false ; return ! TargetRegisterInfo :: isVirtualRegister ( MI -> getOperand ( <NUM_LIT> ) . getReg (


<s> if ( SU ) { if ( CurEmitted >= InstKindLimit [ IDAlu ] ) CurEmitted = <NUM_LIT> ; NextInstKind = IDAlu ; } } if ( ! SU ) { SU = pickOther ( IDFetch ) ; if ( SU ) NextInstKind = IDFetch ; } if ( ! SU ) { SU = pickOther ( IDOther ) ; if ( SU ) NextInstKind = IDOther ; } LLVM_DEBUG ( if ( SU ) { dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SU ) ; } else { dbgs ( ) << <STR_LIT> ; for ( unsigned i = <NUM_LIT> ; i < DAG -> SUnits . size ( ) ; i ++ ) { const SUnit & S = DAG


<s> for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ; } } } } else { ++ CurEmitted ; } LLVM_DEBUG ( dbgs ( ) << CurEmitted << <STR_LIT> ) ; if ( CurInstKind != IDFetch ) { MoveUnits ( Pending [ IDFetch ] , Available [ IDFetch ] ) ; } else FetchInstCount ++


<s> if ( MI -> getOpcode ( ) != <STR_LIT> ) return


<s> switch ( DestSubReg ) { case <STR_LIT> : return AluT_X ; case <STR_LIT> : return AluT_Y ; case <STR_LIT> : return AluT_Z ; case <STR_LIT> : return AluT_W ; default : break ; } unsigned DestReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) || regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_X ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_Y ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_Z ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_W ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_XYZW ; if ( TII -> readsLDSSrcReg ( * MI ) ) return AluT_XYZW ; return AluAny


<s> return new <ISA_LIT>MCCodeEmitter ( MCII , * Ctx .


<s> void <ISA_LIT>MCCodeEmitter :: Emit ( uint64_t Value , raw_ostream


<s> Emit ( InstWord01 , OS ) ; Emit ( InstWord2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else if ( IS_TEX ( Desc ) ) { int64_t Sampler = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; int64_t SrcSelect [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ; Emit ( Word01 , OS ) ; Emit ( Word2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else { uint64_t Inst = getBinaryCodeForInstr ( MI , Fixups , STI ) ; if ( ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) && ( ( Desc . TSFlags & <STR_LIT> ) || Desc . TSFlags & <STR_LIT> ) ) { uint64_t ISAOpCode = Inst & (


<s> unsigned <ISA_LIT>MCCodeEmitter :: getHWRegChan ( unsigned reg ) const { return MRI . getEncodingValue ( reg


<s> const unsigned offset = ( & MO == & MI . getOperand ( <NUM_LIT> ) ) ? <NUM_LIT> : <NUM_LIT> ; Fixups . push_back ( MCFixup :: create ( offset , Expr , FK_SecRel_4 , MI . getLoc ( ) ) ) ; return <NUM_LIT> ; } assert ( MO . isImm ( ) ) ; return MO . getImm


<s> MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ; Emit ( Word01 ,


<s> int64_t SrcSelect [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ; Emit ( Word01 ,


<s> unsigned <ISA_LIT>MCCodeEmitter :: getHWReg ( unsigned RegNo ) const { return MRI . getEncodingValue ( RegNo ) &


<s> if ( HAS_NATIVE_OPERANDS ( MCII . get ( MI . getOpcode ( ) ) . TSFlags ) ) return MRI . getEncodingValue ( MO . getReg ( ) ) ; return getHWReg ( MO . getReg ( ) ) ; } assert ( MO . isImm ( ) ) ; return MO .


<s> uint64_t <ISA_LIT>MCCodeEmitter :: getMachineOpValue ( const MCInst & MI , const MCOperand & MO , SmallVectorImpl < MCFixup > & Fixup , const MCSubtargetInfo & STI ) const { if ( MO . isReg ( ) ) { if ( HAS_NATIVE_OPERANDS ( MCII . get ( MI . getOpcode ( ) ) . TSFlags ) ) return MRI . getEncodingValue ( MO . getReg ( ) ) ; return getHWReg ( MO . getReg ( ) ) ; } assert ( MO . isImm ( ) ) ; return MO . getImm ( )


<s> const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { return ; } else if ( IS_VTX ( Desc ) ) { uint64_t InstWord01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t InstWord2 = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ! ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) { InstWord2 |= <NUM_LIT> << <NUM_LIT> ; } Emit ( InstWord01 , OS ) ; Emit ( InstWord2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else if ( IS_TEX ( Desc ) ) { int64_t Sampler = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; int64_t SrcSelect [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ; Emit ( Word01 , OS ) ; Emit ( Word2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else { uint64_t Inst = getBinaryCodeForInstr ( MI , Fixups , STI ) ; if ( ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) && ( ( Desc . TSFlags & <STR_LIT> ) || Desc . TSFlags & <STR_LIT> ) ) { uint64_t ISAOpCode = Inst & ( <NUM_LIT>


<s> return ; } else if ( IS_VTX ( Desc ) ) { uint64_t InstWord01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t InstWord2 = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ! ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) { InstWord2 |= <NUM_LIT> << <NUM_LIT> ; } Emit ( InstWord01 , OS ) ; Emit ( InstWord2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else if ( IS_TEX ( Desc ) ) { int64_t Sampler = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; int64_t SrcSelect [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ; Emit ( Word01 , OS ) ; Emit ( Word2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else { uint64_t Inst = getBinaryCodeForInstr ( MI , Fixups , STI ) ; if ( ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) && ( ( Desc . TSFlags & <STR_LIT> ) || Desc . TSFlags & <STR_LIT> ) ) { uint64_t ISAOpCode = Inst & ( <NUM_LIT> << <NUM_LIT> ) ; Inst &= ~ ( <NUM_LIT> << <NUM_LIT> ) ; Inst |= ISAOpCode << <NUM_LIT>


<s> ArgTypes . push_back ( ImageSizeType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageSizeArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; ArgTypes . push_back ( ImageFormatType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageFormatArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; Modified = true ; } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getAddressSpace ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ; auto NewFArgIt = NewF


<s> ArgTypes . push_back ( FT -> getParamType ( i ) ) ; MDVector ArgMD = GetArgMD ( KernelMDNode , i + <NUM_LIT> ) ; PushArgMD ( NewArgMDs , ArgMD ) ; if ( ! IsImageType ( ArgTypeFromMD ( KernelMDNode , i ) ) ) continue ; ArgTypes . push_back ( ImageSizeType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageSizeArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; ArgTypes . push_back ( ImageFormatType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageFormatArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; Modified = true ; } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getAddressSpace ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ; auto NewFArgIt = NewF -> arg_begin


<s> } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ; auto NewFArgIt = NewF -> arg_begin ( ) ; for ( auto & Arg : F -> args ( ) ) { auto ArgName = Arg . getName ( ) ; NewFArgIt -> setName ( ArgName ) ; VMap [ & Arg ] = & ( * NewFArgIt ++ ) ; if ( IsImageType ( ArgTypeFromMD ( KernelMDNode , Arg . getArgNo ( ) ) ) ) { ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; } } SmallVector < ReturnInst * , <NUM_LIT> > Returns ; CloneFunctionInto ( NewF , F , VMap , CloneFunctionChangeType :: LocalChangesOnly , Returns ) ; SmallVector < Metadata * , <NUM_LIT> > KernelMDArgs ; KernelMDArgs . push_back ( ConstantAsMetadata :: get ( NewF ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) KernelMDArgs . push_back ( MDNode :: get ( * Context , NewArgMDs . ArgVector [ i ] ) ) ; MDNode * NewMDNode = MDNode :: get


<s> StringRef getPassName ( ) const override { return


<s> bool <ISA_LIT>VectorRegMerger :: areAllUsesSwizzeable ( unsigned Reg ) const { for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { if ( ! canSwizzle ( * It ) ) return


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ; AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < MachineLoopInfo


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ; AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved <


<s> const char * getPassName ( ) const override { return <STR_LIT>


<s> unsigned Reg = R<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: iterator Pos = R<ISA_LIT> -> Instr ; MachineBasicBlock & MBB = * Pos -> getParent ( ) ; DebugLoc DL = Pos -> getDebugLoc ( ) ; unsigned SrcVec = BaseR<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; DenseMap < unsigned , unsigned > UpdatedRegToChan = BaseR<ISA_LIT> -> RegToChan ; std :: vector < unsigned > UpdatedUndef = BaseR<ISA_LIT> -> UndefReg ; for ( DenseMap < unsigned , unsigned > :: iterator It = R<ISA_LIT> -> RegToChan . begin ( ) , E = R<ISA_LIT> -> RegToChan . end ( ) ; It != E ; ++ It ) { unsigned DstReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned SubReg = ( * It ) . first ; unsigned Swizzle = ( * It ) . second ; unsigned Chan = getReassignedChan ( RemapChan , Swizzle ) ; MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < unsigned > :: iterator ChanPos = std :: find ( UpdatedUndef . begin ( ) , UpdatedUndef . end ( ) , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( std :: find ( UpdatedUndef . begin ( ) , UpdatedUndef . end ( ) , Chan ) == UpdatedUndef . end ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } Pos = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; Pos -> dump ( ) ; ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ( * It ) . dump ( ) ; dbgs ( ) << <STR_LIT> ) ; SwizzleInput ( * It , RemapChan ) ; DEBUG ( ( * It ) . dump ( ) ) ; } R<ISA_LIT> -> Instr -> eraseFromParent ( ) ; R<ISA_LIT> -> Instr = Pos ; R<ISA_LIT> -> RegToChan = UpdatedRegToChan ; R<ISA_LIT> -> UndefReg = UpdatedUndef ; return


<s> MIs . erase ( std :: find ( MIs . begin ( ) , MIs . end ( ) , MI ) , MIs . end ( ) ) ; } for ( InstructionSetMap :: iterator It = PreviousRegSeqByUndefCount . begin ( ) , E = PreviousRegSeqByUndefCount . end ( ) ; It != E


<s> MI -> dump ( ) ; ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned > > RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } trackR<ISA_LIT> ( R<ISA_LIT> ) ; } } return


<s> continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , MI ) ; unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned > > RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( )


<s> for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { unsigned Swizzle = MI . getOperand ( i + Offset ) . getImm ( ) + <NUM_LIT> ; for ( unsigned j = <NUM_LIT> , e = RemapChan . size ( ) ; j


<s> for ( DenseMap < unsigned , unsigned > :: const_iterator It = R<ISA_LIT> . RegToChan . begin ( ) , E = R<ISA_LIT> . RegToChan . end ( ) ; It != E ; ++ It ) { PreviousRegSeqByReg [ ( * It ) . first ] . push_back ( R<ISA_LIT> . Instr ) ; } PreviousRegSeqByUndefCount [ R<ISA_LIT> . UndefReg . size ( ) ] . push_back ( R<ISA_LIT> . Instr ) ; PreviousRegSeq [ R<ISA_LIT> .


<s> if ( PreviousRegSeqByUndefCount [ NeededUndefs ] . empty ( ) ) return false ; std :: vector < MachineInstr * > & MIs = PreviousRegSeqByUndefCount [ NeededUndefs ] ; CompatibleR<ISA_LIT> = PreviousRegSeq [ MIs . back ( ) ] ; tryMergeVector ( & CompatibleR<ISA_LIT> , & R<ISA_LIT> , RemapChan ) ; return


<s> std :: vector < MachineInstr * > & MIs = PreviousRegSeqByUndefCount [ NeededUndefs ] ; CompatibleR<ISA_LIT> = PreviousRegSeq [ MIs . back ( ) ] ; tryMergeVector ( & CompatibleR<ISA_LIT> , & R<ISA_LIT> , RemapChan ) ; return


<s> bool <ISA_LIT>VectorRegMerger :: tryMergeVector ( const RegSeqInfo * Untouched , RegSeqInfo * ToMerge , std :: vector < std :: pair < unsigned , unsigned > > & Remap ) const { unsigned CurrentUndexIdx = <NUM_LIT> ; for ( DenseMap < unsigned , unsigned > :: iterator It = ToMerge -> RegToChan . begin ( ) , E = ToMerge -> RegToChan . end ( ) ; It != E ; ++ It ) { DenseMap < unsigned , unsigned > :: const_iterator PosInUntouched = Untouched -> RegToChan . find ( ( * It ) . first ) ; if ( PosInUntouched != Untouched -> RegToChan


<s> for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT>


<s> RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> ,


<s> MachineInstr * <ISA_LIT>VectorRegMerger :: RebuildVector ( RegSeqInfo * R<ISA_LIT> , const RegSeqInfo * BaseR<ISA_LIT> , const std :: vector < std :: pair < unsigned , unsigned >> & RemapChan ) const { Register Reg = R<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: iterator Pos = R<ISA_LIT> -> Instr ; MachineBasicBlock & MBB = * Pos -> getParent ( ) ; DebugLoc DL = Pos -> getDebugLoc ( ) ; Register SrcVec = BaseR<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; DenseMap < Register , unsigned > UpdatedRegToChan = BaseR<ISA_LIT> -> RegToChan ; std :: vector < Register > UpdatedUndef = BaseR<ISA_LIT> -> UndefReg ; for ( const auto & It : R<ISA_LIT> -> RegToChan ) { Register DstReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned SubReg = It . first ; unsigned Swizzle = It . second ; unsigned Chan = getReassignedChan ( RemapChan , Swizzle ) ; MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < Register > :: iterator ChanPos = llvm :: find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } MachineInstr * NewMI = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; NewMI -> dump ( ) ; ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ( * It ) . dump ( ) ; dbgs ( ) << <STR_LIT> ) ; SwizzleInput ( * It


<s> LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT>


<s> unsigned Offset ; if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) Offset = <NUM_LIT> ; else Offset = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { unsigned Swizzle = MI . getOperand ( i + Offset ) . getImm ( ) + <NUM_LIT> ; for ( const auto & J : RemapChan ) { if ( J . first == Swizzle ) { MI . getOperand ( i + Offset ) . setImm ( J


<s> if ( PosInUntouched != Untouched -> RegToChan . end ( ) ) { Remap . push_back ( std :: pair < unsigned , unsigned > ( It . second , ( * PosInUntouched ) . second ) ) ; continue ; } if ( CurrentUndexIdx >= Untouched -> UndefReg . size ( ) ) return false ; Remap . push_back ( std :: pair < unsigned , unsigned > ( It . second , Untouched -> UndefReg [ CurrentUndexIdx ++ ] ) ) ; } return true


<s> DebugLoc DL = Pos -> getDebugLoc ( ) ; unsigned SrcVec = BaseR<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; DenseMap < unsigned , unsigned > UpdatedRegToChan = BaseR<ISA_LIT> -> RegToChan ; std :: vector < unsigned > UpdatedUndef = BaseR<ISA_LIT> -> UndefReg ; for ( DenseMap < unsigned , unsigned > :: iterator It = R<ISA_LIT> -> RegToChan . begin ( ) , E = R<ISA_LIT> -> RegToChan . end ( ) ; It != E ; ++ It ) { unsigned DstReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned SubReg = ( * It ) . first ; unsigned Swizzle = ( * It ) . second ; unsigned Chan = getReassignedChan ( RemapChan , Swizzle ) ; MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < unsigned > :: iterator ChanPos = llvm :: find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec =


<s> MachineInstr & MI = * MII ; if ( MI . getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg )


<s> for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { MachineBasicBlock * MB = & * MBB ; PreviousRegSeq . clear ( ) ; PreviousRegSeqByReg . clear ( ) ; PreviousRegSeqByUndefCount . clear ( ) ; for ( MachineBasicBlock :: iterator MII = MB -> begin ( ) , MIIE = MB -> end ( ) ; MII != MIIE ; ++ MII ) { MachineInstr & MI = * MII ; if ( MI . getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT>


<s> if ( skipFunction ( * Fn . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & ST = Fn . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { MachineBasicBlock * MB = & * MBB ; PreviousRegSeq . clear ( ) ; PreviousRegSeqByReg . clear ( ) ; PreviousRegSeqByUndefCount . clear ( ) ; for ( MachineBasicBlock :: iterator MII = MB -> begin ( ) , MIIE = MB -> end ( ) ; MII != MIIE ; ++ MII ) { MachineInstr & MI = * MII ; if ( MI . getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } trackR<ISA_LIT> ( R<ISA_LIT>


<s> std :: vector < unsigned > :: iterator ChanPos = find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } MachineInstr * NewMI = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; NewMI -> dump ( ) ; ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ( * It ) . dump ( ) ; dbgs ( ) << <STR_LIT> ) ; SwizzleInput ( * It , RemapChan ) ; DEBUG ( ( * It ) . dump ( ) ) ; } R<ISA_LIT> -> Instr -> eraseFromParent ( ) ; R<ISA_LIT> -> Instr = NewMI ; R<ISA_LIT> -> RegToChan = UpdatedRegToChan ; R<ISA_LIT> -> UndefReg = UpdatedUndef ; return NewMI


<s> PreviousRegSeq . clear ( ) ; PreviousRegSeqByReg . clear ( ) ; PreviousRegSeqByUndefCount . clear ( ) ; for ( MachineBasicBlock :: iterator MII = MB -> begin ( ) , MIIE = MB -> end ( ) ; MII != MIIE ; ++ MII ) { MachineInstr * MI = MII ; if ( MI -> getOpcode ( ) != <STR_LIT> ) { if ( TII -> get ( MI -> getOpcode ( ) ) . TSFlags & <STR_LIT> ) { unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , MI ) ; unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( )


<s> MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < unsigned > :: iterator ChanPos = llvm :: find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } MachineInstr * NewMI = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; NewMI -> dump ( ) ; ) ; LLVM_DEBUG ( dbgs (


<s> if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) ) return false ; const MachineInstr * MI = MRI . getUniqueVRegDef ( Reg ) ; return MI && MI ->


<s> bool <ISA_LIT>VectorRegMerger :: canSwizzle ( const MachineInstr & MI ) const { if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ; default : return


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ; default : return


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>VectorRegMerger ( ) { return new <ISA_LIT>VectorRegMerger


<s> StringRef getPassName ( ) const override


<s> static unsigned getReassignedChan ( const std :: vector < std :: pair < unsigned , unsigned >> & RemapChan , unsigned


<s> const MachineInstr * MI = MRI . getUniqueVRegDef ( Reg ) ; return MI && MI ->


<s> DenseMap < Register , unsigned > UpdatedRegToChan = BaseR<ISA_LIT> -> RegToChan ; std :: vector < Register > UpdatedUndef = BaseR<ISA_LIT> -> UndefReg ; for ( DenseMap < Register , unsigned > :: iterator It = R<ISA_LIT> -> RegToChan . begin ( ) , E = R<ISA_LIT> -> RegToChan . end ( ) ; It != E ; ++ It ) { Register DstReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned SubReg = ( * It ) . first ; unsigned Swizzle = ( * It ) . second ; unsigned Chan = getReassignedChan ( RemapChan , Swizzle ) ; MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < Register > :: iterator ChanPos = llvm :: find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef


<s> Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT>


<s> for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { unsigned Swizzle = MI . getOperand ( i + Offset ) . getImm ( ) + <NUM_LIT> ; for ( unsigned j = <NUM_LIT> , e = RemapChan . size ( ) ; j < e ; j ++ ) { if ( RemapChan [ j ] . first == Swizzle ) { MI . getOperand ( i + Offset ) . setImm (

RemapChan [ j ] . second - <NUM_LIT> ) ;


<s> unsigned CurrentUndexIdx = <NUM_LIT> ; for ( DenseMap < Register , unsigned > :: iterator It = ToMerge -> RegToChan . begin ( ) , E = ToMerge -> RegToChan . end ( ) ; It != E ; ++ It ) { DenseMap < Register , unsigned > :: const_iterator PosInUntouched = Untouched -> RegToChan . find ( ( * It ) . first ) ; if ( PosInUntouched != Untouched -> RegToChan . end ( ) ) { Remap . push_back ( std :: pair < unsigned , unsigned > ( ( * It ) . second , ( * PosInUntouched ) . second ) ) ; continue ; } if ( CurrentUndexIdx >= Untouched -> UndefReg . size ( ) ) return false ; Remap . push_back ( std :: pair < unsigned , unsigned > ( ( * It ) . second , Untouched -> UndefReg [ CurrentUndexIdx ++ ] ) ) ; } return


<s> MachineBasicBlock :: iterator FirstInBundle = CurrentPacketMIs . empty ( ) ? & MI : CurrentPacketMIs . front ( ) ; const DenseMap < unsigned , unsigned > & PV = getPreviousVector ( FirstInBundle ) ; std :: vector < <ISA_LIT>InstrInfo :: BankSwizzle > BS ; bool isTransSlot ; if ( isBundlableWithCurrentPMI ( MI , PV , BS , isTransSlot ) ) { for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) ; i < e ; i ++ ) { MachineInstr * MI = CurrentPacketMIs [ i ] ; unsigned Op = TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: bank_swizzle ) ; MI -> getOperand ( Op ) . setImm ( BS [ i ] ) ; } unsigned Op = TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: bank_swizzle ) ; MI . getOperand ( Op ) . setImm ( BS . back ( ) ) ; if ( ! CurrentPacketMIs . empty ( ) ) setIsLastBit

( CurrentPacketMIs . back ( ) , <NUM_LIT> ) ;


<s> for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) - <NUM_LIT> ; i < e ; i ++ ) { CurrentPacketMIs [ i ] -> dump ( ) ; dbgs ( ) << <STR_LIT> ; } dbgs ( ) <<


<s> unsigned getSlot ( const MachineInstr & MI ) const { return TRI . getHWRegChan ( MI . getOperand (


<s> return TRI . getHWRegChan ( MI . getOperand ( <NUM_LIT> ) . getReg (


<s> bool ignorePseudoInstruction ( const MachineInstr & MI , const MachineBasicBlock * MBB ) override { return false


<s> bool isLegalToPruneDependencies ( SUnit * SUI , SUnit * SUJ ) override { return


<s> if ( ! TII -> isALUInstr ( MI . getOpcode ( ) ) ) return true ; if ( MI . getOpcode ( ) ==


<s> <ISA_LIT>PacketizerList ( MachineFunction & MF , const <ISA_LIT>Subtarget & ST , MachineLoopInfo & MLI ) : VLIWPacketizerList ( MF , MLI , nullptr ) , TII ( ST . getInstrInfo ( ) ) , TRI ( TII -> getRegisterInfo ( ) ) { VLIW5 = ! ST . hasCaymanISA


<s> <ISA_LIT>PacketizerList ( MachineFunction & MF , const <ISA_LIT>Subtarget & ST , MachineLoopInfo & MLI ) : VLIWPacketizerList ( MF , MLI , nullptr ) , TII ( ST . getInstrInfo ( ) ) ,


<s> } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ; for ( ; I != MBB -> begin ( ) ; -- I , -- RemainingCount ) { if ( TII -> isSchedulingBoundary ( * std :: prev ( I ) , & * MBB , Fn ) ) break ; } I = MBB -> begin ( ) ; if ( I == RegionEnd ) { RegionEnd = std :: prev ( RegionEnd ) ; -- RemainingCount ; continue ; } if ( I == std :: prev ( RegionEnd ) ) { RegionEnd = std :: prev ( RegionEnd ) ; continue ; } Packetizer . PacketizeMIs ( & * MBB , & * I , RegionEnd ) ; RegionEnd = I ; } } return


<s> void setIsLastBit ( MachineInstr * MI , unsigned Bit ) const { unsigned LastOp = TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: last ) ; MI -> getOperand ( LastOp ) .


<s> const DenseMap < unsigned , unsigned > :: const_iterator It = PVs . find ( Src ) ; if ( It != PVs . end ( ) ) MI . getOperand

( OperandIdx ) . setReg ( It -> second ) ;


<s> <STR_LIT> :: src0 , <STR_LIT> :: src1 , <STR_LIT> :: src2 } ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { int OperandIdx = TII -> getOperandIdx ( MI . getOpcode ( ) , Ops [ i ] ) ; if ( OperandIdx < <NUM_LIT> ) continue ; Register Src = MI . getOperand ( OperandIdx )


<s> MI . getOperand ( Op ) . setImm ( BS . back ( ) ) ; if ( ! CurrentPacketMIs . empty ( ) ) setIsLastBit ( CurrentPacketMIs . back ( ) , <NUM_LIT> ) ; substitutePV ( MI , PV ) ; MachineBasicBlock :: iterator It = VLIWPacketizerList :: addToPacket ( MI ) ; if ( isTransSlot ) { endPacket ( std :: next ( It ) -> getParent ( ) , std :: next ( It ) ) ; } return It ; } endPacket ( MI . getParent ( ) , MI ) ; if ( TII -> isTransOnly ( MI ) )


<s> if ( ! TII -> isALUInstr ( I -> getOpcode ( ) ) && ! I -> isBundle ( ) ) return Result ; MachineBasicBlock :: instr_iterator BI = I . getInstrIterator ( ) ; if ( I -> isBundle ( ) ) BI ++ ; int LastDstChan = - <NUM_LIT> ; do { bool isTrans = false ; int BISlot = getSlot ( * BI ) ; if ( LastDstChan >= BISlot ) isTrans = true ; LastDstChan = BISlot ; if ( TII -> isPredicated ( * BI ) ) continue ; int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } unsigned Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( * BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) { Result [ Dst ]


<s> if ( PredI != PredJ ) return false ; if ( SUJ -> isSucc ( SUI ) ) { for ( unsigned i = <NUM_LIT> , e = SUJ -> Succs . size ( ) ; i < e ; ++ i ) { const SDep & Dep = SUJ -> Succs [ i ] ; if ( Dep . getSUnit ( ) != SUI ) continue ; if ( Dep . getKind ( ) == SDep :: Anti ) continue ; if ( Dep . getKind ( ) == SDep :: Output ) if ( MII -> getOperand ( <NUM_LIT> ) . getReg ( ) != MIJ -> getOperand ( <NUM_LIT> ) . getReg ( ) ) continue ; return false ; } } bool ARDef = TII -> definesAddressRegister ( * MII

) || TII -> definesAddressRegister ( * MIJ ) ;


<s> if ( TII -> isPredicated ( & * BI ) ) continue ; int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } unsigned Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( & * BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( Dst == <STR_LIT> ) { continue ; } unsigned PVReg = <NUM_LIT> ; switch ( TRI . getHWRegChan ( Dst ) ) { case <NUM_LIT> : PVReg = <STR_LIT> ; break ; case <NUM_LIT> : PVReg =


<s> return TRI . getHWRegChan ( MI -> getOperand


<s> bool ignorePseudoInstruction ( const MachineInstr * MI ,

const MachineBasicBlock * MBB ) override {


<s> bool ignorePseudoInstruction ( const MachineInstr * MI , const MachineBasicBlock *


<s> DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; ) ; } else return false ; } } CurrentPacketMIs . push_back ( MI ) ; if ( ! TII -> fitsConstReadLimitations ( CurrentPacketMIs ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; dbgs ( ) << <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) - <NUM_LIT> ; i < e ; i ++ ) { CurrentPacketMIs [ i ] -> dump ( ) ; dbgs ( ) << <STR_LIT> ; } dbgs ( ) << <STR_LIT> ; ) ; CurrentPacketMIs . pop_back ( ) ; return false ; } if ( ! TII -> fitsReadPortLimitations ( CurrentPacketMIs , PV , BS , isTransSlot ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; dbgs ( ) << <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) - <NUM_LIT> ; i < e ; i ++ ) { CurrentPacketMIs [ i ]


<s> if ( TII -> isVector ( * MI ) ) return true ; if ( ! TII -> isALUInstr ( MI ->


<s> <STR_LIT> :: src0 , <STR_LIT> :: src1 , <STR_LIT> :: src2 } ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { int OperandIdx = TII -> getOperandIdx ( MI -> getOpcode ( ) , Ops [ i ] ) ; if ( OperandIdx < <NUM_LIT> ) continue ; unsigned Src = MI -> getOperand ( OperandIdx ) .


<s> MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = CurrentPacketMIs . size ( ) - <NUM_LIT> ; i < e ; i ++ ) { CurrentPacketMIs [ i ] -> dump ( ) ; dbgs ( )


<s> const char * getPassName ( ) const override { return


<s> unsigned LastOp = TII -> getOperandIdx ( MI -> getOpcode (


<s> while ( MI != End ) { if ( MI -> isKill ( ) || MI -> getOpcode ( ) == <STR_LIT> || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) { MachineBasicBlock :: iterator DeleteMI = MI ; ++ MI ; MBB -> erase ( DeleteMI ) ; End = MBB -> end ( ) ; continue ; } ++ MI ; } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ; for ( ; I != MBB -> begin ( ) ; -- I ,


<s> bool <ISA_LIT>Packetizer :: runOnMachineFunction ( MachineFunction & Fn ) { const TargetInstrInfo * TII = Fn . getSubtarget ( ) . getInstrInfo ( ) ; MachineLoopInfo & MLI = getAnalysis < MachineLoopInfo > ( ) ; <ISA_LIT>PacketizerList Packetizer ( Fn , MLI ) ; assert ( Packetizer . getResourceTracker ( ) && <STR_LIT> ) ; if ( Packetizer . getResourceTracker ( ) -> getInstrItins ( ) -> isEmpty ( ) ) return false ; for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { MachineBasicBlock :: iterator End = MBB -> end ( ) ; MachineBasicBlock :: iterator MI = MBB -> begin ( ) ; while ( MI != End ) { if ( MI -> isKill ( ) || MI -> getOpcode ( ) == <STR_LIT> || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) { MachineBasicBlock :: iterator DeleteMI = MI ; ++ MI ; MBB -> erase ( DeleteMI ) ; End = MBB -> end ( ) ; continue ; } ++ MI ; } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ;


<s> <ISA_LIT>PacketizerList ( MachineFunction & MF , MachineLoopInfo & MLI ) : VLIWPacketizerList ( MF , MLI ) , TII ( static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ) , TRI ( TII -> getRegisterInfo ( ) ) { VLIW5 = ! MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasCaymanISA


<s> int OperandIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: write ) ; if ( OperandIdx > - <NUM_LIT> && BI -> getOperand ( OperandIdx ) . getImm ( ) == <NUM_LIT> ) continue ; int DstIdx = TII -> getOperandIdx ( BI -> getOpcode ( ) , <STR_LIT> :: dst ) ; if ( DstIdx == - <NUM_LIT> ) { continue ; } unsigned Dst = BI -> getOperand ( DstIdx ) . getReg ( ) ; if ( isTrans || TII -> isTransOnly ( BI ) ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( BI -> getOpcode ( ) == <STR_LIT> || BI -> getOpcode ( ) == <STR_LIT> ) { Result [ Dst ] = <STR_LIT> ; continue ; } if ( Dst == <STR_LIT> ) { continue ; } unsigned PVReg = <NUM_LIT> ; switch ( TRI . getHWRegChan ( Dst ) ) { case <NUM_LIT> : PVReg = <STR_LIT> ; break ; case <NUM_LIT> : PVReg = <STR_LIT> ; break ; case <NUM_LIT> : PVReg = <STR_LIT> ; break ; case <NUM_LIT> : PVReg = <STR_LIT>


<s> <ISA_LIT>PacketizerList ( MachineFunction & MF , MachineLoopInfo & MLI ) : VLIWPacketizerList ( MF , MLI , true ) , TII ( static_cast < const <ISA_LIT>InstrInfo * >

( MF . getSubtarget ( ) . getInstrInfo ( ) ) ) , TRI ( TII -> getRegisterInfo ( ) ) {


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getCFGStructurizerRegClass (


<s> unsigned <ISA_LIT>RegisterInfo :: getHWRegChan ( unsigned reg )


<s> unsigned <ISA_LIT>RegisterInfo :: getHWRegChan ( unsigned


<s> unsigned <ISA_LIT>RegisterInfo :: getHWRegIndex ( unsigned Reg )


<s> return GET_REG_INDEX ( getEncodingValue (


<s> const RegClassWeight & <ISA_LIT>RegisterInfo :: getRegClassWeight ( const TargetRegisterClass * RC ) const { return RCW


<s> Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; for ( TargetRegisterClass :: iterator I = <STR_LIT> . begin ( ) , E = <STR_LIT> . end ( ) ; I != E ; ++ I ) { Reserved . set ( * I ) ; } TII -> reserveIndirectRegisters ( Reserved


<s> Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; for ( TargetRegisterClass :: iterator I = <STR_LIT> . begin ( ) , E = <STR_LIT> . end ( ) ; I != E ; ++ I ) { Reserved . set ( * I ) ; } TII -> reserveIndirectRegisters ( Reserved ,


<s> unsigned <ISA_LIT>RegisterInfo :: getFrameRegister ( const MachineFunction & MF ) const { return


<s> void <ISA_LIT>RegisterInfo :: eliminateFrameIndex ( MachineBasicBlock :: iterator MI , int SPAdj , unsigned FIOperandNum


<s> const MCPhysReg * <ISA_LIT>RegisterInfo :: getCalleeSavedRegs


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getCFGStructurizerRegClass ( MVT VT ) const { switch ( VT . SimpleTy )


<s> bool <ISA_LIT>RegisterInfo :: isPhysRegLiveAcrossClauses ( unsigned Reg ) const { assert ( ! TargetRegisterInfo :: isVirtualRegister ( Reg


<s> <ISA_LIT>RegisterInfo :: <ISA_LIT>RegisterInfo ( ) : <ISA_LIT>GenRegisterInfo ( <NUM_LIT> ) { RCW . RegWeight = <NUM_LIT>


<s> void <ISA_LIT>RegisterInfo :: reserveRegisterTuples ( BitVector & Reserved , unsigned Reg ) const { MCRegAliasIterator R ( Reg , this ,


<s> switch ( Reg ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return


<s> bool trackLivenessAfterRegAlloc ( const MachineFunction & MF ) const override { return false


<s> bool trackLivenessAfterRegAlloc ( const MachineFunction &


<s> <ISA_LIT>Subtarget & <ISA_LIT>Subtarget :: initializeSubtargetDependencies ( const Triple & TT , StringRef GPU , StringRef FS ) { SmallString < <NUM_LIT> > FullFS (


<s> <ISA_LIT>Subtarget & <ISA_LIT>Subtarget :: initializeSubtargetDependencies ( const Triple & TT , StringRef GPU , StringRef FS ) { SmallString < <NUM_LIT> > FullFS


<s> void <ISA_LIT>PassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>CFGStructurizerPass ( ) ) ; addPass ( create<ISA_LIT>ExpandSpecialInstrsPass ( ) ) ; addPass ( & FinalizeMachineBundlesID ) ; addPass ( create<ISA_LIT>Packetizer ( ) ) ; addPass ( create<ISA_LIT>ControlFlowFinalizer ( ) )


<s> if ( Enable<ISA_LIT>IfConvert ) addPass ( &


<s> if ( Enable<ISA_LIT>IfConvert ) addPass ( & IfConverterID ) ; addPass ( create<ISA_LIT>ClauseMergePass ( )


<s> return create<ISA_LIT>MachineScheduler ( C


<s> return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this


<s> Mod = & M ; FloatType = Type :: getFloatTy ( Ctx ) ; Int32Type = Type :: getInt32Ty ( Ctx ) ; V4f32Type = VectorType :: get ( FloatType , <NUM_LIT> ) ; V4i32Type = VectorType :: get ( Int32Type , <NUM_LIT> ) ; Type * ArgsType [ ] = { V4f32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , } ; TexSign = FunctionType :: get ( V4f32Type , ArgsType , false ) ; Type * ArgsQType [ ] = { V4i32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type , Int32Type


<s> CT [ <NUM_LIT> ] = <NUM_LIT> ; } if ( TextureType == TEXTURE_CUBE_ARRAY || TextureType == TEXTURE_SHADOWCUBE_ARRAY ) CT [ <NUM_LIT> ] = <NUM_LIT> ; if ( TextureType == TEXTURE_1D_ARRAY || TextureType == TEXTURE_SHADOW1D_ARRAY ) { if ( hasLOD && useShadowVariant ) { CT [ <NUM_LIT> ] = <NUM_LIT> ; } else { CT [ <NUM_LIT> ] = <NUM_LIT> ; SrcSelect [ <NUM_LIT> ] = <NUM_LIT> ; } } else if ( TextureType == TEXTURE_2D_ARRAY || TextureType == TEXTURE_SHADOW2D_ARRAY ) { CT [ <NUM_LIT> ] = <NUM_LIT> ; } if ( ( TextureType == TEXTURE_SHADOW1D || TextureType == TEXTURE_SHADOW2D || TextureType == TEXTURE_SHADOWRECT || TextureType == TEXTURE_SHADOW1D_ARRAY ) &&

! ( hasLOD && useShadowVariant ) ) SrcSelect [ <NUM_LIT> ] = <NUM_LIT> ;


<s> void ReplaceCallInst ( CallInst & I , FunctionType * FT , const char * Name , unsigned SrcSelect [ <NUM_LIT> ] , Value * Offset [ <NUM_LIT> ] , Value * Resource , Value * Sampler , unsigned CT [ <NUM_LIT> ] , Value * Coord ) { IRBuilder < > Builder ( & I ) ; Constant * Mask [ ] = { ConstantInt :: get ( Int32Type , SrcSelect [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , SrcSelect [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , SrcSelect [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , SrcSelect [ <NUM_LIT> ] ) } ; Value * SwizzleMask = ConstantVector :: get ( Mask ) ; Value * SwizzledCoord = Builder . CreateShuffleVector ( Coord , Coord , SwizzleMask ) ; Value * Args [ ] = { SwizzledCoord , Offset [ <NUM_LIT> ] , Offset [ <NUM_LIT> ] , Offset [ <NUM_LIT> ] , Resource , Sampler , ConstantInt :: get ( Int32Type , CT [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , CT [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , CT [ <NUM_LIT> ] ) , ConstantInt :: get ( Int32Type , CT [ <NUM_LIT> ] ) } ; Function * F = Mod -> getFunction ( Name ) ; if ( ! F ) { F = Function :: Create ( FT , GlobalValue :: ExternalLinkage , Name , Mod ) ; F -> addFnAttr ( Attribute :: ReadNone )


<s> void ReplaceTexIntrinsic ( CallInst & I , bool hasLOD , FunctionType * FT , const char * VanillaInt , const char * ShadowInt ) { Value * Coord = I . getArgOperand ( <NUM_LIT> ) ; Value * ResourceId = I . getArgOperand ( <NUM_LIT> ) ; Value * SamplerId = I . getArgOperand ( <NUM_LIT> ) ; unsigned TextureType = cast < ConstantInt > ( I . getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; unsigned SrcSelect [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> ,


<s> unsigned SrcSelect [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; unsigned CT [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; Value * Offset [ <NUM_LIT> ] = { ConstantInt :: get ( Int32Type , <NUM_LIT> ) , ConstantInt :: get ( Int32Type , <NUM_LIT> )

, ConstantInt :: get ( Int32Type , <NUM_LIT> ) } ;


<s> void ReplaceTXF ( CallInst & I ) { Value * Coord = I . getArgOperand ( <NUM_LIT> ) ; Value * ResourceId = I . getArgOperand ( <NUM_LIT> ) ; Value * SamplerId = I . getArgOperand ( <NUM_LIT> ) ; unsigned TextureType = cast < ConstantInt > ( I . getArgOperand


<s> unsigned SrcSelect [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; unsigned CT [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; Value * Offset [ <NUM_LIT> ] = { I . getArgOperand ( <NUM_LIT> ) , I . getArgOperand ( <NUM_LIT> ) , I . getArgOperand ( <NUM_LIT> ) , } ; bool useShadowVariant ; getAdjustmentFromTextureTarget ( TextureType , false , SrcSelect , CT ,


<s> int DstIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vdata ) ; MachineOperand * MO_Dmask = TII -> getNamedOperand ( MI , <STR_LIT> :: dmask ) ; assert ( MO_Dmask && <STR_LIT> ) ; unsigned dmask = MO_Dmask -> getImm ( ) ; unsigned ActiveLanes = TII -> isGather4 ( Opcode ) ? <NUM_LIT> : countPopulation ( dmask ) ; static_assert ( <STR_LIT> == <NUM_LIT> && <STR_LIT> == <NUM_LIT> , <STR_LIT> ) ; bool Packed = ! ST . hasUnpackedD16VMem ( ) ; unsigned InitIdx = D16Val && Packed ? ( ( ActiveLanes + <NUM_LIT> ) >> <NUM_LIT> ) + <NUM_LIT> : ActiveLanes + <NUM_LIT> ; uint32_t DstSize = RI -> getRegSizeInBits ( * TII -> getOpRegClass ( MI , DstIdx ) ) / <NUM_LIT> ; if ( DstSize < InitIdx ) continue ; Register PrevDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; unsigned NewDst = <NUM_LIT> ; unsigned SizeLeft = ST . usePRTStrictNull ( ) ? InitIdx : <NUM_LIT> ; unsigned CurrIdx = ST . usePRTStrictNull ( ) ? <NUM_LIT> : InitIdx ; if ( DstSize == <NUM_LIT> ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) . addImm ( <NUM_LIT> ) ; NewDst = PrevDst ; } else { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) ; for ( ; SizeLeft ; SizeLeft -- , CurrIdx ++ ) { NewDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; Register SubReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , NewDst ) . addReg ( PrevDst ) . addReg ( SubReg ) . addImm ( CurrIdx ) ; PrevDst = NewDst ; } } MachineInstrBuilder ( MF , MI ) . addReg ( NewDst , RegState :: Implicit ) ; MI . tieOperands ( DstIdx , MI . getNumOperands ( ) - <NUM_LIT> ) ; Changed = true ; } } } } return Changed


<s> assert ( MO_Dmask && <STR_LIT> ) ; unsigned dmask = MO_Dmask -> getImm ( ) ; unsigned ActiveLanes = TII -> isGather4 ( Opcode ) ? <NUM_LIT> : countPopulation ( dmask ) ; bool Packed = ! ST . hasUnpackedD16VMem ( ) ; unsigned InitIdx = D16Val && Packed ? ( ( ActiveLanes + <NUM_LIT> ) >> <NUM_LIT> ) + <NUM_LIT> : ActiveLanes + <NUM_LIT> ; uint32_t DstSize = RI -> getRegSizeInBits ( * TII -> getOpRegClass ( MI , DstIdx ) ) / <NUM_LIT> ; if ( DstSize < InitIdx ) continue ; Register PrevDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; unsigned NewDst = <NUM_LIT> ; unsigned SizeLeft = ST . usePRTStrictNull ( ) ? InitIdx : <NUM_LIT> ; unsigned CurrIdx = ST . usePRTStrictNull ( ) ? <NUM_LIT> : ( InitIdx - <NUM_LIT> ) ; if ( DstSize == <NUM_LIT> ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) . addImm ( <NUM_LIT> ) ; NewDst = PrevDst ; } else { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , PrevDst ) ; for ( ; SizeLeft ; SizeLeft -- , CurrIdx ++ ) { NewDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI , DstIdx ) ) ; Register SubReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , NewDst ) . addReg ( PrevDst ) . addReg ( SubReg ) . addImm ( <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( CurrIdx ) ) ; PrevDst = NewDst ; } } MachineInstrBuilder ( MF , MI ) . addReg ( NewDst , RegState :: Implicit ) ; MI . tieOperands ( DstIdx , MI . getNumOperands ( ) - <NUM_LIT> ) ; Changed = true ; } } } } return


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * RI = ST . getRegisterInfo ( ) ; bool Changed = false ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; auto Opcode = MI . getOpcode ( ) ; if ( TII -> isMIMG ( Opcode ) && ! MI . mayStore ( ) ) { MachineOperand * TFE = TII -> getNamedOperand ( MI , <STR_LIT> :: tfe ) ; MachineOperand * LWE = TII -> getNamedOperand ( MI , <STR_LIT> :: lwe ) ; MachineOperand * D16 = TII -> getNamedOperand ( MI , <STR_LIT> :: d16 ) ; assert ( ( TFE && LWE ) && <STR_LIT> ) ; unsigned TFEVal = TFE -> getImm ( ) ; unsigned LWEVal = LWE -> getImm ( ) ; unsigned D16Val = D16 ? D16 -> getImm ( ) : <NUM_LIT> ; if ( TFEVal || LWEVal ) { const DebugLoc & DL = MI . getDebugLoc ( ) ; int DstIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vdata ) ; MachineOperand * MO_Dmask = TII -> getNamedOperand ( MI , <STR_LIT> :: dmask ) ; assert ( MO_Dmask && <STR_LIT> ) ; unsigned dmask = MO_Dmask -> getImm ( ) ; unsigned ActiveLanes = TII -> isGather4 ( Opcode ) ? <NUM_LIT> : countPopulation ( dmask ) ; static_assert ( <STR_LIT> == <NUM_LIT> && <STR_LIT> == <NUM_LIT> , <STR_LIT> ) ; bool Packed = ! ST . hasUnpackedD16VMem ( ) ; unsigned InitIdx = D16Val && Packed ? ( ( ActiveLanes + <NUM_LIT> ) >> <NUM_LIT> ) + <NUM_LIT> : ActiveLanes + <NUM_LIT> ; uint32_t DstSize = RI -> getRegSizeInBits ( * TII -> getOpRegClass ( MI , DstIdx ) ) / <NUM_LIT> ; if ( DstSize < InitIdx ) continue ; unsigned PrevDst = MRI . createVirtualRegister ( TII -> getOpRegClass ( MI ,


<s> SmallVector < BasicBlock * , <NUM_LIT> > Preds ; for ( BasicBlock * Pred : predecessors ( BB ) ) { if ( ! is_contained ( Latches , Pred ) ) Preds . push_back ( Pred ) ; } BB = SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ; } Value * Exec = popSaved (


<s> FunctionPass * llvm :: create<ISA_LIT>AnnotateControlFlowPass ( ) { return new <ISA_LIT>AnnotateControlFlow (


<s> return new <ISA_LIT>AnnotateControlFlow


<s> bool <ISA_LIT>AnnotateControlFlow :: doInitialization ( Module & M ) { LLVMContext & Context = M . getContext ( ) ; Void = Type :: getVoidTy ( Context ) ; Boolean = Type :: getInt1Ty ( Context ) ; Int64 = Type :: getInt64Ty ( Context ) ; ReturnStruct = StructType :: get ( Boolean , Int64 ) ; BoolTrue = ConstantInt :: getTrue ( Context ) ; BoolFalse = ConstantInt :: getFalse (


<s> void <ISA_LIT>AnnotateControlFlow :: eraseIfUnused ( PHINode * Phi ) { if ( RecursivelyDeleteDeadPHINode (


<s> AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < DivergenceAnalysis > ( ) ; AU . addPreserved < DominatorTreeWrapperPass > ( ) ; FunctionPass :: getAnalysisUsage


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired <


<s> Value * Incoming = Phi -> getIncomingValue ( i ) ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( isa < ConstantInt > ( Incoming ) ) { NewPhi -> addIncoming ( Broken , From ) ; continue ; } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term , LoopPhiConditions ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst >


<s> if ( Phi -> getIncomingBlock ( i ) == IDom ) { if ( Phi -> getIncomingValue ( i ) != BoolTrue ) return false ; } else { if ( Phi -> getIncomingValue ( i ) != BoolFalse


<s> bool <ISA_LIT>AnnotateControlFlow :: isTopOfStack ( BasicBlock * BB ) { return ! Stack . empty ( ) && Stack .


<s> bool <ISA_LIT>AnnotateControlFlow :: isTopOfStack ( BasicBlock *


<s> if ( isUniform ( Term ) ) return ; Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret


<s> void <ISA_LIT>AnnotateControlFlow :: push ( BasicBlock * BB , Value * Saved ) { Stack . push_back ( std :: make_pair ( BB


<s> Stack . push_back ( std :: make_pair


<s> } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == BB && isElse ( Phi ) ) { insertElse ( Term ) ; eraseIfUnused ( Phi ) ; continue ; } closeControlFlow ( BB


<s> if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds


<s> bool <ISA_LIT>AnnotateControlFlow :: doInitialization ( Module & M ) { LLVMContext & Context = M . getContext ( ) ; Void = Type :: getVoidTy ( Context ) ; Boolean = Type :: getInt1Ty ( Context ) ; Int64 = Type :: getInt64Ty ( Context ) ; ReturnStruct = StructType :: get ( Boolean , Int64 , ( Type * ) nullptr ) ; BoolTrue = ConstantInt :: getTrue ( Context ) ; BoolFalse = ConstantInt :: getFalse ( Context ) ; BoolUndef = UndefValue :: get ( Boolean ) ; Int64Zero = ConstantInt :: get (


<s> BoolUndef = UndefValue :: get ( Boolean ) ; Int64Zero = ConstantInt :: get ( Int64 , <NUM_LIT> ) ; If = M . getOrInsertFunction ( IfIntrinsic , ReturnStruct , Boolean , ( Type * ) nullptr ) ; Else = M . getOrInsertFunction ( ElseIntrinsic , ReturnStruct , Int64 , ( Type * ) nullptr ) ; Break = M . getOrInsertFunction ( BreakIntrinsic , Int64 , Int64 , ( Type * ) nullptr ) ; IfBreak = M . getOrInsertFunction ( IfBreakIntrinsic , Int64 , Boolean , Int64 , ( Type * ) nullptr ) ; ElseBreak = M . getOrInsertFunction ( ElseBreakIntrinsic , Int64 , Int64 , Int64 , ( Type * ) nullptr ) ; Loop = M . getOrInsertFunction ( LoopIntrinsic , Boolean , Int64 , ( Type * ) nullptr ) ; EndCf = M . getOrInsertFunction ( EndCfIntrinsic , Void , Int64 ,


<s> if ( ! Phi -> hasNUsesOrMore ( <NUM_LIT> ) ) Phi


<s> void <ISA_LIT>AnnotateControlFlow :: eraseIfUnused (


<s> AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addPreserved < DominatorTreeWrapperPass


<s> BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } eraseIfUnused ( Phi ) ; return Ret ; } else if ( Instruction * Inst = dyn_cast < Instruction > ( Cond ) ) { BasicBlock * Parent = Inst -> getParent ( ) ; Instruction * Insert ; if ( L -> contains ( Inst ) ) { Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } else { llvm_unreachable ( <STR_LIT> ) ; } return


<s> Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create


<s> push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> ,