<s> } SDValue DWordAddr = DAG . getNode ( <STR_LIT> , DL , VT , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; SDValue ByteIndex = DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , Ptr , DAG . getConstant ( <NUM_LIT> , DL , VT ) ) ; SDValue TruncValue = DAG . getNode ( <STR_LIT> , DL , VT , Value , MaskConstant ) ; SDValue Shift = DAG . getNode ( <STR_LIT> , DL , VT , ByteIndex , DAG . getConstant ( <NUM_LIT> , DL , VT ) ) ; SDValue ShiftedValue = DAG . getNode ( <STR_LIT> , DL , VT , TruncValue , Shift ) ; SDValue Mask = DAG . getNode ( <STR_LIT> , DL , VT , MaskConstant , Shift ) ; SDValue Src [ <NUM_LIT> ] = { ShiftedValue , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , Mask } ; SDValue Input = DAG . getBuildVector ( <STR_LIT> , DL , Src ) ; SDValue Args [ <NUM_LIT> ] = { Chain , Input , DWordAddr } ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Args , MemVT , StoreNode -> getMemOperand ( ) ) ; } else if ( Ptr -> getOpcode ( ) != <STR_LIT> && ValueVT . bitsGE ( <STR_LIT> ) ) { Ptr = DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ) ; if ( StoreNode -> isTruncatingStore ( ) || StoreNode -> isIndexed ( ) ) { llvm_unreachable ( <STR_LIT> ) ; } else { Chain = DAG . getStore ( Chain , DL , Value , Ptr , StoreNode -> getMemOperand ( ) ) ;


<s> EVT VT = Value . getValueType ( ) ; assert ( VT . bitsLE ( <STR_LIT> ) ) ; EVT MemVT = StoreNode -> getMemoryVT ( ) ; SDValue MaskConstant ; if ( MemVT == <STR_LIT> ) { MaskConstant = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } else { assert ( MemVT == <STR_LIT> ) ; MaskConstant = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } SDValue DWordAddr = DAG . getNode ( <STR_LIT> , DL , VT , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; SDValue ByteIndex = DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , Ptr , DAG . getConstant ( <NUM_LIT> , DL , VT ) ) ; SDValue TruncValue = DAG . getNode ( <STR_LIT> , DL , VT , Value , MaskConstant ) ; SDValue Shift = DAG . getNode ( <STR_LIT> , DL , VT , ByteIndex , DAG . getConstant ( <NUM_LIT> , DL , VT ) ) ; SDValue ShiftedValue = DAG . getNode ( <STR_LIT> , DL , VT , TruncValue , Shift ) ; SDValue Mask = DAG . getNode ( <STR_LIT> , DL , VT , MaskConstant , Shift ) ; SDValue Src [ <NUM_LIT> ] = { ShiftedValue , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , Mask } ; SDValue Input = DAG . getBuildVector ( <STR_LIT> , DL , Src ) ; SDValue Args [ <NUM_LIT> ] = { Chain , Input , DWordAddr } ;

return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Args , MemVT , StoreNode -> getMemOperand ( ) ) ;


<s> setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; if ( Subtarget -> hasCARRY ( ) ) setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; if ( Subtarget -> hasBORROW ( ) ) setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; if ( ! Subtarget -> hasBFE ( ) ) setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; if ( ! Subtarget -> hasBFE ( ) ) setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; if ( ! Subtarget -> hasBFE ( ) ) setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Legal ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Expand ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; setOperationAction ( <STR_LIT> , <STR_LIT> , Custom ) ; const MVT ScalarIntVTs [ ] = { <STR_LIT> , <STR_LIT> } ; for ( MVT VT : ScalarIntVTs ) { setOperationAction ( <STR_LIT> , VT , Expand ) ; setOperationAction ( <STR_LIT> , VT , Expand ) ; setOperationAction ( <STR_LIT> , VT , Expand ) ; setOperationAction ( <STR_LIT> , VT , Expand ) ; }

setSchedulingPreference ( Sched :: Source ) ;


<s> <ISA_LIT>TargetLowering :: ReplaceNodeResults ( N , Results , DAG ) ; return ; case <STR_LIT> : if ( N -> getValueType ( <NUM_LIT> ) == <STR_LIT> ) { Results . push_back ( lowerFP_TO_UINT ( N -> getOperand ( <NUM_LIT> ) , DAG ) ) ; return ; } case <STR_LIT> : { if ( N -> getValueType ( <NUM_LIT> ) == <STR_LIT> ) { Results . push_back ( lowerFP_TO_<ISA_LIT>NT ( N -> getOperand ( <NUM_LIT> ) , DAG ) ) ; return ; } SDValue Result ; if ( expandFP_TO_<ISA_LIT>NT ( N , Result , DAG ) ) Results . push_back ( Result ) ; return ; } case <STR_LIT> : { SDValue Op = SDValue ( N , <NUM_LIT> ) ; SDValue RES = LowerSDIVREM ( Op , DAG ) ; Results . push_back ( RES ) ; Results . push_back ( RES . getValue ( <NUM_LIT> ) ) ; break ; } case <STR_LIT> : { SDValue Op = SDValue ( N , <NUM_LIT> ) ; LowerUDIVREM64 ( Op , DAG , Results ) ; break ; }


<s> SDValue <ISA_LIT>TargetLowering :: lowerFrameIndex ( SDValue Op , SelectionDAG & DAG ) const { MachineFunction & MF = DAG . getMachineFunction ( ) ;

const <ISA_LIT>FrameLowering * TFL = Subtarget -> getFrameLowering ( ) ;


<s> unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ; } PointerType * PtrTy = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; <STR_LIT> Ext = <STR_LIT> ; if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) { Ext = <STR_LIT> ; } unsigned ValBase = ArgLocs [ In . getOrigArgIndex ( ) ] . getLocMemOffset ( ) ; unsigned PartOffset = VA . getLocMemOffset ( ) ; unsigned Offset = <NUM_LIT> + VA . getLocMemOffset ( ) ;

MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) , PartOffset - ValBase ) ;


<s> CCValAssign & VA = ArgLocs [ i ] ; const <STR_LIT> & In = Ins [ i ] ; EVT VT = In . VT ; EVT MemVT = VA . getLocVT ( ) ; if ( ! VT . isVector ( ) && MemVT . isVector ( ) ) { MemVT = MemVT . getVectorElementType ( ) ; } if ( <STR_LIT> ( CallConv ) ) { unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ; } PointerType * PtrTy = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; <STR_LIT> Ext = <STR_LIT> ; if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) { Ext = <STR_LIT> ; } unsigned ValBase = ArgLocs [ In . getOrigArgIndex ( ) ] . getLocMemOffset ( ) ; unsigned PartOffset = VA . getLocMemOffset ( ) ; unsigned Offset = <NUM_LIT> + VA . getLocMemOffset ( ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) , PartOffset - ValBase ) ; SDValue Arg = DAG . getLoad ( <STR_LIT> , Ext , VT , DL , Chain , DAG . getConstant ( Offset , DL , <STR_LIT> ) , DAG . getUNDEF ( <STR_LIT> ) , PtrInfo , MemVT , <NUM_LIT> , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MOInvariant ) ; InVals . push_back ( Arg ) ; MFI -> ABIArgOffset = Offset + MemVT . getStoreSize ( ) ;


<s> SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , ArgLocs , * DAG . getContext ( ) ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; SmallVector < <STR_LIT> , <NUM_LIT> > LocalIns ; if ( <STR_LIT> ( CallConv ) ) { CCInfo . AnalyzeFormalArguments ( Ins , CCAssignFnForCall ( CallConv , isVarArg ) ) ; } else { analyzeFormalArgumentsCompute ( CCInfo , Ins ) ; } for ( unsigned i = <NUM_LIT> , e = Ins . size ( ) ; i < e ; ++ i ) { CCValAssign & VA = ArgLocs [ i ] ; const <STR_LIT> & In = Ins [ i ] ; EVT VT = In . VT ; EVT MemVT = VA . getLocVT ( ) ;

if ( ! VT . isVector ( ) && MemVT . isVector ( ) ) {


<s> if ( HasDst ) { OtherSrcIdx -- ; OtherSelIdx -- ; } if ( RegisterSDNode * Reg = dyn_cast < RegisterSDNode > ( ParentNode -> getOperand ( OtherSrcIdx ) ) ) { if ( Reg -> getReg ( ) == <STR_LIT> ) { ConstantSDNode * Cst = cast < ConstantSDNode > ( ParentNode -> getOperand ( OtherSelIdx ) ) ; Consts . push_back ( Cst -> getZExtValue ( ) ) ; } } } ConstantSDNode * Cst = cast < ConstantSDNode > ( CstOffset ) ; Consts . push_back ( Cst -> getZExtValue ( ) ) ; if ( ! TII -> fitsConstReadLimitations ( Consts ) ) { return false ; } Sel = CstOffset ; Src = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; return true ; } case <STR_LIT> : if ( cast < ConstantSDNode > ( Imm ) -> getZExtValue ( ) ) return false ; Imm = Src . getOperand ( <NUM_LIT> ) ; Src = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; return true ; case <STR_LIT> : case <STR_LIT> : { unsigned ImmReg = <STR_LIT> ; uint64_t ImmValue = <NUM_LIT> ; if ( Src . getMachineOpcode ( ) == <STR_LIT> ) { ConstantFPSDNode * FPC = dyn_cast < ConstantFPSDNode > ( Src . getOperand ( <NUM_LIT> ) ) ; float FloatValue = FPC -> getValueAPF ( ) . convertToFloat ( ) ; if ( FloatValue == <NUM_LIT> ) { ImmReg = <STR_LIT> ; } else if ( FloatValue == <NUM_LIT> ) { ImmReg = <STR_LIT> ; } else if ( FloatValue == <NUM_LIT> ) { ImmReg = <STR_LIT> ; } else { ImmValue = FPC -> getValueAPF ( ) . bitcastToAPInt ( ) . getZExtValue ( ) ; } } else { ConstantSDNode * C = dyn_cast < ConstantSDNode > ( Src . getOperand ( <NUM_LIT> ) ) ; uint64_t Value = C -> getZExtValue ( ) ; if ( Value == <NUM_LIT> ) { ImmReg = <STR_LIT> ; } else if ( Value == <NUM_LIT> ) { ImmReg = <STR_LIT> ; } else { ImmValue = Value ; } } if ( ImmReg == <STR_LIT> ) { if ( ! Imm . getNode ( ) ) return false ; ConstantSDNode * C = dyn_cast < ConstantSDNode > ( Imm ) ; assert ( C ) ; if ( C -> getZExtValue ( ) ) return false ; Imm = DAG . getTargetConstant ( ImmValue , SDLoc ( ParentNode ) , <STR_LIT> ) ;


<s> if ( <STR_LIT> ( CallConv ) ) { unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ; } PointerType * PtrTy = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; <STR_LIT> Ext = <STR_LIT> ; if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) { Ext = <STR_LIT> ; } unsigned ValBase = ArgLocs [ In . getOrigArgIndex ( ) ] . getLocMemOffset ( ) ; unsigned PartOffset = VA . getLocMemOffset ( ) ;

unsigned Offset = <NUM_LIT> + VA . getLocMemOffset ( ) ;


<s> unsigned Offset = TFL -> getFrameIndexReference ( MF , FrameIndex , IgnoredFrameReg ) ;

return DAG . getConstant ( Offset * <NUM_LIT> * TFL -> getStackWidth ( MF ) , SDLoc ( Op ) , Op . getValueType ( ) ) ;


<s> scalarizeVectorLoad ( LoadNode , DAG ) , Chain } ; return DAG . getMergeValues ( MergedValues , DL ) ; } int ConstantBlock = ConstantAddressBlock ( LoadNode -> getAddressSpace ( ) ) ; if ( ConstantBlock > - <NUM_LIT> && ( ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) || ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) ) ) { SDValue Result ; if ( isa < ConstantExpr > ( LoadNode -> getMemOperand ( ) -> getValue ( ) ) || isa < Constant > ( LoadNode -> getMemOperand ( ) -> getValue ( ) ) || isa < ConstantSDNode > ( Ptr ) ) { SDValue Slots [ <NUM_LIT> ] ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { SDValue NewPtr = DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , Ptr , DAG . getConstant ( <NUM_LIT> * i + ConstantBlock * <NUM_LIT> , DL , <STR_LIT> ) ) ; Slots [ i ] = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , NewPtr ) ; } EVT NewVT = <STR_LIT> ; unsigned NumElements = <NUM_LIT> ; if ( VT . isVector ( ) ) { NewVT = VT ; NumElements = VT . getVectorNumElements ( ) ; } Result = DAG . getBuildVector ( NewVT , DL , makeArrayRef ( Slots , NumElements ) ) ; } else { Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) , DAG . getConstant ( LoadNode -> getAddressSpace ( ) - <ISA_LIT>AS :: CONSTANT_BUFFER_0 , DL , <STR_LIT> ) ) ; } if ( ! VT . isVector ( ) ) { Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Result , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; } SDValue MergedValues [ <NUM_LIT> ] = { Result , Chain } ; return DAG . getMergeValues ( MergedValues , DL ) ; }


<s> if ( StoreNode -> isTruncatingStore ( ) ) { EVT VT = Value . getValueType ( ) ; assert ( VT . bitsLE ( <STR_LIT> ) ) ; EVT MemVT = StoreNode -> getMemoryVT ( ) ; SDValue MaskConstant ; if ( MemVT == <STR_LIT> ) { MaskConstant = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } else { assert ( MemVT == <STR_LIT> ) ; MaskConstant = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } SDValue DWordAddr = DAG . getNode ( <STR_LIT> , DL , VT , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; SDValue ByteIndex = DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , Ptr , DAG . getConstant ( <NUM_LIT> , DL , VT ) ) ; SDValue TruncValue = DAG . getNode ( <STR_LIT> , DL , VT , Value , MaskConstant ) ; SDValue Shift = DAG . getNode ( <STR_LIT> , DL , VT , ByteIndex , DAG . getConstant ( <NUM_LIT> , DL , VT ) ) ; SDValue ShiftedValue = DAG . getNode ( <STR_LIT> , DL , VT , TruncValue , Shift ) ; SDValue Mask = DAG . getNode ( <STR_LIT> , DL , VT , MaskConstant , Shift ) ; SDValue Src [ <NUM_LIT> ] = { ShiftedValue , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , Mask } ; SDValue Input = DAG . getBuildVector ( <STR_LIT> , DL , Src ) ; SDValue Args [ <NUM_LIT> ] = { Chain , Input , DWordAddr } ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Args , MemVT , StoreNode -> getMemOperand ( ) ) ; } else if ( Ptr -> getOpcode ( ) != <STR_LIT> && ValueVT . bitsGE ( <STR_LIT> ) ) { Ptr = DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ) ;

if ( StoreNode -> isTruncatingStore ( ) || StoreNode -> isIndexed ( ) ) {


<s> return expandUnalignedStore ( StoreNode , DAG ) ; } SDValue DWordAddr = DAG . getNode ( <STR_LIT> , DL , PtrVT , Ptr , DAG . getConstant ( <NUM_LIT> , DL , PtrVT ) ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( TruncatingStore ) { assert ( VT . bitsLE ( <STR_LIT> ) ) ; SDValue MaskConstant ; if ( MemVT == <STR_LIT> ) { MaskConstant = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } else { assert ( MemVT == <STR_LIT> ) ; assert ( StoreNode -> getAlignment ( ) >= <NUM_LIT> ) ; MaskConstant = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } SDValue ByteIndex = DAG . getNode ( <STR_LIT> , DL , PtrVT , Ptr , DAG . getConstant ( <NUM_LIT> , DL , PtrVT ) ) ; SDValue BitShift = DAG . getNode ( <STR_LIT> , DL , VT , ByteIndex , DAG . getConstant ( <NUM_LIT> , DL , VT ) ) ; SDValue Mask = DAG . getNode ( <STR_LIT> , DL , VT , MaskConstant , BitShift ) ; SDValue TruncValue = DAG . getNode ( <STR_LIT> , DL , VT , Value , MaskConstant ) ; SDValue ShiftedValue = DAG . getNode ( <STR_LIT> , DL , VT , TruncValue , BitShift ) ; SDValue Src [ <NUM_LIT> ] = { ShiftedValue , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , Mask } ; SDValue Input = DAG . getBuildVector ( <STR_LIT> , DL , Src ) ; SDValue Args [ <NUM_LIT> ] = { Chain , Input , DWordAddr } ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Args , MemVT , StoreNode -> getMemOperand ( ) ) ; } else if ( Ptr -> getOpcode ( ) != <STR_LIT> && VT . bitsGE ( <STR_LIT> ) ) { Ptr = DAG . getNode ( <STR_LIT> , DL , PtrVT , DWordAddr ) ; if ( StoreNode -> isIndexed ( ) ) { llvm_unreachable ( <STR_LIT> ) ; } else { Chain = DAG . getStore ( Chain , DL , Value , Ptr , StoreNode -> getMemOperand ( ) ) ; }


<s> MaskConstant = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } else { assert ( MemVT == <STR_LIT> ) ; assert ( StoreNode -> getAlignment ( ) >= <NUM_LIT> ) ; MaskConstant = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } SDValue ByteIndex = DAG . getNode ( <STR_LIT> , DL , PtrVT , Ptr , DAG . getConstant ( <NUM_LIT> , DL , PtrVT ) ) ; SDValue BitShift = DAG . getNode ( <STR_LIT> , DL , VT , ByteIndex , DAG . getConstant ( <NUM_LIT> , DL , VT ) ) ; SDValue Mask = DAG . getNode ( <STR_LIT> , DL , VT , MaskConstant , BitShift ) ; SDValue TruncValue = DAG . getNode ( <STR_LIT> , DL , VT , Value , MaskConstant ) ; SDValue ShiftedValue = DAG . getNode ( <STR_LIT> , DL , VT , TruncValue , BitShift ) ; SDValue Src [ <NUM_LIT> ] = { ShiftedValue , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , Mask } ; SDValue Input = DAG . getBuildVector ( <STR_LIT> , DL , Src ) ; SDValue Args [ <NUM_LIT> ] = { Chain , Input , DWordAddr } ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Args , MemVT , StoreNode -> getMemOperand ( ) ) ; } else if ( Ptr -> getOpcode ( ) != <STR_LIT> && VT . bitsGE ( <STR_LIT> ) ) { Ptr = DAG . getNode ( <STR_LIT> , DL , PtrVT , DWordAddr ) ; if ( StoreNode -> isIndexed ( ) ) { llvm_unreachable ( <STR_LIT> ) ; } else { Chain = DAG . getStore ( Chain , DL , Value , Ptr , StoreNode -> getMemOperand ( ) ) ; } return Chain ; } }

if ( AS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return SDValue ( ) ;


<s> if ( isHWTrueValue ( True ) && isHWFalseValue ( False ) && ( CompareVT == VT || VT == <STR_LIT> ) ) { return DAG . getNode ( <STR_LIT> , DL , VT , LHS , RHS , True , False , CC ) ; } if ( isZero ( LHS ) ) { <STR_LIT> CCOpcode = cast < CondCodeSDNode > ( CC ) -> get ( ) ; <STR_LIT> CCSwapped = <STR_LIT> ( CCOpcode ) ; if ( isCondCodeLegal ( CCSwapped , CompareVT . getSimpleVT ( ) ) ) { std :: swap ( LHS , RHS ) ; CC = DAG . getCondCode ( CCSwapped ) ; } else { <STR_LIT> CCInv = <STR_LIT> ( CCOpcode , CompareVT . isInteger ( ) ) ; CCSwapped = <STR_LIT> ( CCInv ) ; if ( isCondCodeLegal ( CCSwapped , CompareVT . getSimpleVT ( ) ) ) { std :: swap ( True , False ) ; std :: swap ( LHS , RHS ) ; CC = DAG . getCondCode ( CCSwapped ) ; } } } if ( isZero ( RHS ) ) { SDValue Cond = LHS ; SDValue Zero = RHS ; <STR_LIT> CCOpcode = cast < CondCodeSDNode > ( CC ) -> get ( ) ; if ( CompareVT != VT ) { True = DAG . getNode ( <STR_LIT> , DL , CompareVT , True ) ; False = DAG . getNode ( <STR_LIT> , DL , CompareVT , False ) ; } switch ( CCOpcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : CCOpcode = <STR_LIT> ( CCOpcode , CompareVT == <STR_LIT> ) ; Temp = True ; True = False ; False = Temp ; break ; default :


<s> return CC_<ISA_LIT>_Kernel ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_LS : return CC_<ISA_LIT> ; default : report_fatal_error ( <STR_LIT> ) ;


<s> case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_LS :


<s> SDValue <ISA_LIT>TargetLowering :: LowerFormalArguments ( SDValue Chain , CallingConv :: ID CallConv , bool isVarArg , const SmallVectorImpl < <STR_LIT> > & Ins , const SDLoc & DL , SelectionDAG & DAG , SmallVectorImpl < SDValue > & InVals ) const { SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , ArgLocs , * DAG . getContext ( ) ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; SmallVector < <STR_LIT> , <NUM_LIT> > LocalIns ; if ( <STR_LIT> ( CallConv ) ) { CCInfo . AnalyzeFormalArguments ( Ins , CCAssignFnForCall ( CallConv , isVarArg ) ) ; } else { analyzeFormalArgumentsCompute ( CCInfo , Ins ) ; } for ( unsigned i = <NUM_LIT> , e = Ins . size ( ) ; i < e ; ++ i ) { CCValAssign & VA = ArgLocs [ i ] ; const <STR_LIT> & In = Ins [ i ] ; EVT VT = In . VT ; EVT MemVT = VA . getLocVT ( ) ; if ( ! VT . isVector ( ) && MemVT . isVector ( ) ) { MemVT = MemVT . getVectorElementType ( ) ; } if ( <STR_LIT> ( CallConv ) ) { unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ;


<s> getOriginalFunctionArgs ( DAG , MF . getFunction ( ) , Ins , LocalIns ) ; AnalyzeFormalArguments ( CCInfo , LocalIns ) ; for ( unsigned i = <NUM_LIT> , e = Ins . size ( ) ; i < e ; ++ i ) { CCValAssign & VA = ArgLocs [ i ] ; const <STR_LIT> & In = Ins [ i ] ; EVT VT = In . VT ; EVT MemVT = VA . getLocVT ( ) ; if ( ! VT . isVector ( ) && MemVT . isVector ( ) ) { MemVT = MemVT . getVectorElementType ( ) ; } if ( <STR_LIT> ( CallConv ) ) { unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ; } PointerType * PtrTy = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; <STR_LIT> Ext = <STR_LIT> ; if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) { Ext = <STR_LIT> ; }

unsigned ValBase = ArgLocs [ In . getOrigArgIndex ( ) ] . getLocMemOffset ( ) ;


<s> getOriginalFunctionArgs ( DAG , MF . getFunction ( ) , Ins , LocalIns ) ; AnalyzeFormalArguments ( CCInfo , LocalIns ) ; for ( unsigned i = <NUM_LIT> , e = Ins . size ( ) ; i < e ; ++ i ) { CCValAssign & VA = ArgLocs [ i ] ; const <STR_LIT> & In = Ins [ i ] ; EVT VT = In . VT ; EVT MemVT = VA . getLocVT ( ) ; if ( ! VT . isVector ( ) && MemVT . isVector ( ) ) { MemVT = MemVT . getVectorElementType ( ) ; } if ( <STR_LIT> ( CallConv ) ) { unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ; } PointerType * PtrTy = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; <STR_LIT> Ext = <STR_LIT> ;

if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) {


<s> SDValue <ISA_LIT>TargetLowering :: LowerImplicitParameter ( SelectionDAG & DAG , EVT VT , const SDLoc & DL , unsigned DwordOffset ) const { unsigned ByteOffset = DwordOffset * <NUM_LIT> ; PointerType * PtrType = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_BUFFER_0 ) ; assert ( isInt < <NUM_LIT> > ( ByteOffset ) ) ;

return DAG . getLoad ( VT , DL , DAG . getEntryNode ( ) , DAG . getConstant ( ByteOffset , DL , <STR_LIT> ) , MachinePointerInfo ( ConstantPointerNull :: get ( PtrType ) ) , false , false , false , <NUM_LIT> ) ;


<s> if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS && ExtType != <STR_LIT> && MemVT . bitsLT ( <STR_LIT> ) ) { return lowerPrivateExtLoad ( Op , DAG ) ; } SDLoc DL ( Op ) ; EVT VT = Op . getValueType ( ) ; SDValue Chain = LoadNode -> getChain ( ) ; SDValue Ptr = LoadNode -> getBasePtr ( ) ; if ( LoadNode -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS && VT . isVector ( ) ) { SDValue MergedValues [ <NUM_LIT> ] = { scalarizeVectorLoad ( LoadNode , DAG ) , Chain } ; return DAG . getMergeValues ( MergedValues , DL ) ; } int ConstantBlock = ConstantAddressBlock ( LoadNode -> getAddressSpace ( ) ) ; if ( ConstantBlock > - <NUM_LIT> && ( ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) || ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) ) ) { SDValue Result ; if ( isa < ConstantExpr > ( LoadNode -> getMemOperand ( ) -> getValue ( ) ) || isa < Constant > ( LoadNode -> getMemOperand ( ) -> getValue ( ) ) || isa < ConstantSDNode > ( Ptr ) ) { SDValue Slots [ <NUM_LIT> ] ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { SDValue NewPtr = DAG . getNode ( <STR_LIT> , DL , Ptr . getValueType ( ) , Ptr , DAG . getConstant ( <NUM_LIT> * i + ConstantBlock * <NUM_LIT> , DL , <STR_LIT> ) ) ; Slots [ i ] = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , NewPtr ) ; } EVT NewVT = <STR_LIT> ; unsigned NumElements = <NUM_LIT> ; if ( VT . isVector ( ) ) { NewVT = VT ; NumElements = VT . getVectorNumElements ( ) ; } Result = DAG . getBuildVector ( NewVT , DL , makeArrayRef ( Slots , NumElements ) ) ; } else { Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) , DAG . getConstant ( LoadNode -> getAddressSpace ( ) - <ISA_LIT>AS :: CONSTANT_BUFFER_0 , DL , <STR_LIT> ) ) ; }


<s> if ( <STR_LIT> ( CallConv ) ) { CCInfo . AnalyzeFormalArguments ( Ins , CCAssignFnForCall ( CallConv , isVarArg ) ) ; } else { analyzeFormalArgumentsCompute ( CCInfo , Ins ) ; } for ( unsigned i = <NUM_LIT> , e = Ins . size ( ) ; i < e ; ++ i ) { CCValAssign & VA = ArgLocs [ i ] ; const <STR_LIT> & In = Ins [ i ] ; EVT VT = In . VT ; EVT MemVT = VA . getLocVT ( ) ; if ( ! VT . isVector ( ) && MemVT . isVector ( ) ) { MemVT = MemVT . getVectorElementType ( ) ; } if ( <STR_LIT> ( CallConv ) ) { unsigned Reg = MF . addLiveIn ( VA . getLocReg ( ) , & <STR_LIT> ) ; SDValue Register = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Register ) ; continue ; } PointerType * PtrTy = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , AMDGPUA<ISA_LIT> . PARAM_I_ADDRESS ) ; <STR_LIT> Ext = <STR_LIT> ; if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) { Ext = <STR_LIT> ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerImplicitParameter ( SelectionDAG & DAG , EVT VT , const SDLoc & DL , unsigned DwordOffset ) const { unsigned ByteOffset = DwordOffset * <NUM_LIT> ; PointerType * PtrType = PointerType :: get ( VT . getTypeForEVT ( * DAG . getContext ( ) ) , AMDGPUA<ISA_LIT> . PARAM_I_ADDRESS ) ;

assert ( isInt < <NUM_LIT> > ( ByteOffset ) ) ;


<s> if ( ConstantBlock > - <NUM_LIT> && ( ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) || ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) ) ) { SDValue Result ; if ( isa < Constant > ( LoadNode -> getMemOperand ( ) -> getValue ( ) ) || isa < ConstantSDNode > ( Ptr ) ) { return constBufferLoad ( LoadNode , LoadNode -> getAddressSpace ( ) , DAG ) ; } else { Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) , DAG . getConstant ( LoadNode -> getAddressSpace ( ) - AMDGPUA<ISA_LIT> . CONSTANT_BUFFER_0 , DL , <STR_LIT> ) ) ; } if ( ! VT . isVector ( ) ) { Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Result , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; } SDValue MergedValues [ <NUM_LIT> ] = { Result , Chain } ; return DAG . getMergeValues ( MergedValues , DL ) ; } if ( LoadNode -> getExtensionType ( ) == <STR_LIT> ) { EVT MemVT = LoadNode -> getMemoryVT ( ) ;

assert ( ! MemVT . isVector ( ) && ( MemVT == <STR_LIT> || MemVT == <STR_LIT> ) ) ;


<s> if ( MemVT . getScalarSizeInBits ( ) != VT . getScalarSizeInBits ( ) ) { Ext = <STR_LIT> ; } unsigned ValBase = ArgLocs [ In . getOrigArgIndex ( ) ] . getLocMemOffset ( ) ; unsigned PartOffset = VA . getLocMemOffset ( ) ; unsigned Offset = Subtarget -> getExplicitKernelArgOffset ( MF ) + VA . getLocMemOffset ( ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) , PartOffset - ValBase ) ; SDValue Arg = DAG . getLoad ( <STR_LIT> , Ext , VT , DL , Chain , DAG . getConstant ( Offset , DL , <STR_LIT> ) , DAG . getUNDEF ( <STR_LIT> ) , PtrInfo , MemVT , <NUM_LIT> , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; InVals . push_back ( Arg ) ; MFI -> setABIArgOffset ( Offset + MemVT . getStoreSize ( ) ) ; }


<s> } else { Chain = DAG . getStore ( Chain , DL , Value , Ptr , StoreNode -> getMemOperand ( ) ) ; } return Chain ; } } if ( AS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return SDValue ( ) ; EVT MemVT = StoreNode -> getMemoryVT ( ) ; if ( MemVT . bitsLT ( <STR_LIT> ) ) return lowerPrivateTruncStore ( StoreNode , DAG ) ; const MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>FrameLowering * TFL = static_cast < const <ISA_LIT>FrameLowering * > ( Subtarget -> getFrameLowering ( ) ) ; unsigned StackWidth = TFL -> getStackWidth ( MF ) ; Ptr = stackPtrToRegIndex ( Ptr , StackWidth , DAG ) ; if ( ValueVT . isVector ( ) ) { unsigned NumElemVT = ValueVT . getVectorNumElements ( ) ; EVT ElemVT = ValueVT . getVectorElementType ( ) ; SmallVector < SDValue , <NUM_LIT> > Stores ( NumElemVT ) ; assert ( NumElemVT >= StackWidth && <STR_LIT> <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i < NumElemVT ; ++ i ) { unsigned Channel , PtrIncr ; getStackAddress ( StackWidth , i , Channel , PtrIncr ) ; Ptr = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( PtrIncr , DL , <STR_LIT> ) ) ; SDValue Elem = DAG . getNode ( <STR_LIT> , DL , ElemVT , Value , DAG . getConstant ( i , DL , <STR_LIT> ) ) ; Stores [ i ] = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Chain , Elem , Ptr , DAG . getTargetConstant ( Channel , DL , <STR_LIT> ) ) ; } Chain = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Stores ) ; } else { if ( ValueVT == <STR_LIT> ) { Value = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Value ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; ReverseVector ( orderedBlks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; printOrderedBlocks ( ) ; ) ; int NumIter = <NUM_LIT> ; bool Finish = false ; MachineBasicBlock * MBB ; bool MakeProgress = false ; int NumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; do { ++ NumIter ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NumIter << <STR_LIT> << NumRemainedBlk << <STR_LIT> ; ) ; ( void ) NumIter ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator It = OrderedBlks . begin ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator E = OrderedBlks . end ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator SccBeginIter = It ; MachineBasicBlock * SccBeginMBB = nullptr ; int SccNumBlk = <NUM_LIT> ; int SccNumIter ; while ( It != E ) { MBB = * It ; if ( ! SccBeginMBB ) { SccBeginIter = It ; SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ( void ) SccNumIter ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else { ContNextScc = false ; } if ( ContNextScc ) SccBeginMBB = nullptr ; } MachineBasicBlock * EntryMBB = * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ; if ( EntryMBB -> succ_empty ( ) ) { Finish = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ;


<s> MBB -> addSuccessor ( DummyExitBlk ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; } SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ;


<s> insertInstrEnd ( DummyExitBlk , <STR_LIT> ) ; for ( MachineBasicBlock * MBB : RetMBB ) { if ( MachineInstr * MI = getReturnInstr ( MBB ) ) MI -> eraseFromParent ( ) ; MBB -> addSuccessor ( DummyExitBlk ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ;


<s> Func -> push_back ( NewMBB ) ; for ( const MachineInstr & It : * MBB ) NewMBB -> push_back ( Func -> CloneMachineInstr ( & It ) ) ; return NewMBB ;


<s> MachineFunction * Func = MBB -> getParent ( ) ;

MachineBasicBlock * NewMBB = Func -> CreateMachineBasicBlock ( ) ;


<s> MachineBasicBlock * <ISA_LIT>MachineCFGStructurizer :: cloneBlockForPredecessor ( MachineBasicBlock * MBB , MachineBasicBlock * PredMBB ) { assert ( PredMBB -> isSuccessor ( MBB ) && <STR_LIT> ) ; MachineBasicBlock * CloneMBB = clone ( MBB ) ; replaceInstrUseOfBlockWith ( PredMBB , MBB , CloneMBB ) ; PredMBB -> replaceSuccessor ( MBB , CloneMBB ) ; cloneSuccessorList ( CloneMBB , MBB ) ; numClonedInstr += MBB -> size ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> size ( ) << <STR_LIT> ; ) ; SHOWNEWBLK ( CloneMBB , <STR_LIT> ) ; return CloneMBB ;


<s> int Cloned = <NUM_LIT> ; assert ( PreMBB -> isSuccessor ( SrcMBB ) ) ; while ( SrcMBB && SrcMBB != DstMBB ) { assert ( SrcMBB -> succ_size ( ) == <NUM_LIT> ) ; if ( SrcMBB -> pred_size ( ) > <NUM_LIT> ) { SrcMBB = cloneBlockForPredecessor ( SrcMBB , PreMBB ) ; ++ Cloned ; } PreMBB = SrcMBB ; SrcMBB = * SrcMBB -> succ_begin ( ) ; }


<s> for ( MachineBasicBlock * Succ : SrcMBB -> successors ( ) ) DstMBB -> addSuccessor ( Succ ) ;


<s> int <ISA_LIT>MachineCFGStructurizer :: countActiveBlock ( MBBVector :: const_iterator It , MBBVector :: const_iterator E ) const { int Count = <NUM_LIT> ; while ( It != E ) { if ( ! isRetiredBlock ( * It ) ) ++ Count ; ++ It ; }


<s> int <ISA_LIT>MachineCFGStructurizer :: getBranchNzeroOpcode ( int OldOpcode ) { switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } return - <NUM_LIT> ;


<s> int <ISA_LIT>MachineCFGStructurizer :: getContinueNzeroOpcode ( int OldOpcode ) { switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> int <ISA_LIT>MachineCFGStructurizer :: getContinueNzeroOpcode ( int OldOpcode ) {


<s> case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } return - <NUM_LIT> ;


<s> for ( MachineInstr & MI : * MBB ) if ( MI . getDebugLoc ( ) ) DL = MI . getDebugLoc ( ) ;


<s> for ( MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) , E = MBB -> rend ( ) ; It != E ; ++ It ) { MachineInstr * MI = & * It ;


<s> MachineBasicBlock * <ISA_LIT>MachineCFGStructurizer :: getLoopLandInfo ( MachineLoop * LoopRep ) const { LoopLandInfoMap :: const_iterator It = LLInfoMap . find ( LoopRep ) ; if ( It == LLInfoMap . end ( ) ) return nullptr ; return ( * It ) . second ;


<s> MachineBasicBlock * <ISA_LIT>MachineCFGStructurizer :: getLoopLandInfo ( MachineLoop * LoopRep ) const {

LoopLandInfoMap :: const_iterator It = LLInfoMap . find ( LoopRep ) ;


<s> MachineInstr * <ISA_LIT>MachineCFGStructurizer :: getNormalBlockBranchInstr ( MachineBasicBlock * MBB ) { MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) ; MachineInstr * MI = & * It ;

if ( MI && ( isCondBranch ( MI ) || isUncondBranch ( MI ) ) ) return MI ;


<s> MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) ; MachineInstr * MI = & * It ; if ( MI && ( isCondBranch ( MI ) || isUncondBranch ( MI ) ) ) return MI ; return nullptr ;


<s> MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) ; if ( It != MBB -> rend ( ) ) { MachineInstr * instr = & ( * It ) ;

if ( instr -> getOpcode ( ) == <STR_LIT> ) return instr ;


<s> if ( It == BlockInfoMap . end ( ) ) return INVALIDSCCNUM ;


<s> MachineBasicBlock * <ISA_LIT>MachineCFGStructurizer :: getTrueBranch ( MachineInstr * MI ) { return MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ;


<s> int <ISA_LIT>MachineCFGStructurizer :: handleJumpintoIf ( MachineBasicBlock * HeadMBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB ) { int Num = handleJumpintoIfImp ( HeadMBB , TrueMBB , FalseMBB ) ; if ( Num == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ; ) ; Num = handleJumpintoIfImp ( HeadMBB , FalseMBB , TrueMBB ) ; } return Num ;


<s> if ( ! LoopRep ) return false ; MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; return MBB -> isSuccessor ( LoopHeader ) ;


<s> LandBlk = FalseMBB ; FalseMBB = nullptr ; } else if ( FalseMBB -> succ_size ( ) == <NUM_LIT> && isSameloopDetachedContbreak ( TrueMBB , FalseMBB ) ) { LandBlk = * FalseMBB -> succ_begin ( ) ; } else if ( TrueMBB -> succ_size ( ) == <NUM_LIT> && isSameloopDetachedContbreak ( FalseMBB , TrueMBB ) ) { LandBlk = * TrueMBB -> succ_begin ( ) ; } else { return NumMatch + handleJumpintoIf ( MBB , TrueMBB , FalseMBB ) ; } if ( LandBlk && ( ( TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) || ( FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) ) ) { Cloned += improveSimpleJumpintoIf ( MBB , TrueMBB , FalseMBB , & LandBlk ) ; } if ( TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) { TrueMBB = cloneBlockForPredecessor ( TrueMBB , MBB ) ; ++ Cloned ; } if ( FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) { FalseMBB = cloneBlockForPredecessor ( FalseMBB , MBB ) ; ++ Cloned ;


<s> insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } Register InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } if ( LandBlkHasOtherPred ) { insertInstrBefore ( I , <STR_LIT> ) ; for ( MachineBasicBlock * MBB : LandBlk -> predecessors ( ) ) if ( MBB != TrueMBB && MBB != FalseMBB ) report_fatal_error ( <STR_LIT> ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; * LandMBBPtr = LandBlk ;


<s> blk -> insert ( I , NewInstr ) ; MachineInstrBuilder ( * MF , NewInstr ) . addReg ( RegNum , false ) ; SHOWNEWINSTR ( NewInstr ) ;


<s> MachineInstr * OldMI = & ( * I ) ; MachineBasicBlock * MBB = OldMI -> getParent ( ) ; MachineInstr * NewMBB = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ; MBB -> insert ( I , NewMBB ) ; SHOWNEWINSTR ( NewMBB ) ;


<s> MachineBasicBlock * MBB = OldMI -> getParent ( ) ;

MachineInstr * NewMBB = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ;


<s> MachineInstr * MI = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ; MBB -> push_back ( MI ) ; SHOWNEWINSTR ( MI ) ;


<s> if ( ! LoopLand ) return true ; if ( ! isRetiredBlock ( LoopLand ) ) return true ; LoopRep = LoopRep -> getParentLoop ( ) ; }


<s> if ( ! isRetiredBlock ( LoopLand ) ) return true ; LoopRep = LoopRep -> getParentLoop ( ) ; } return false ;


<s> bool <ISA_LIT>MachineCFGStructurizer :: isCondBranch ( MachineInstr * MI ) { switch ( MI -> getOpcode ( ) ) {


<s> bool <ISA_LIT>MachineCFGStructurizer :: isRetiredBlock ( MachineBasicBlock * MBB ) const { MBBInfoMap :: const_iterator It = BlockInfoMap . find ( MBB ) ; if ( It == BlockInfoMap . end ( ) ) return false ;


<s> bool <ISA_LIT>MachineCFGStructurizer :: isReturnBlock ( MachineBasicBlock * MBB ) { MachineInstr * MI = getReturnInstr ( MBB ) ;


<s> else if ( IsReturn ) LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; return IsReturn ;


LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Src1MBB -> getNumber ( ) << <STR_LIT> << Src2MBB -> getNumber ( ) << <STR_LIT> ; ) ;


<s> for ( MachineLoop * ExaminedLoop : NestedLoops ) { if ( ExaminedLoop -> getNumBlocks ( ) == <NUM_LIT> || Visited [ ExaminedLoop ] ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ExaminedLoop -> dump ( ) ; ) ; int NumBreak = mergeLoop ( ExaminedLoop ) ; if ( NumBreak == - <NUM_LIT> ) break ;


<s> for ( auto & It : * MLI ) for ( MachineLoop * ML : depth_first ( It ) ) NestedLoops . push_front ( ML ) ; if ( NestedLoops . empty ( ) ) return <NUM_LIT> ; int Num = <NUM_LIT> ; for ( MachineLoop * ExaminedLoop : NestedLoops ) { if ( ExaminedLoop -> getNumBlocks ( ) == <NUM_LIT> || Visited [ ExaminedLoop ] ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ExaminedLoop -> dump ( ) ; ) ; int NumBreak = mergeLoop ( ExaminedLoop ) ; if ( NumBreak == - <NUM_LIT> ) break ; Num += NumBreak ; }


<s> void <ISA_LIT>MachineCFGStructurizer :: mergeIfthenelseBlock ( MachineInstr * BranchMI , MachineBasicBlock * MBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB , MachineBasicBlock * LandMBB ) { assert ( TrueMBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) ; dbgs ( ) << <STR_LIT> ; if ( TrueMBB ) { dbgs ( ) << <STR_LIT> << TrueMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( ! LandMBB ) { dbgs ( ) << <STR_LIT> ; } else { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; ) ; int OldOpcode = BranchMI -> getOpcode ( ) ; DebugLoc BranchDL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock :: iterator I = BranchMI ; insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ; if ( TrueMBB ) { MBB -> splice ( I , TrueMBB , TrueMBB -> begin ( ) , TrueMBB -> end ( ) ) ; MBB -> removeSuccessor ( TrueMBB , true ) ;

if ( LandMBB && TrueMBB -> succ_size ( ) != <NUM_LIT> ) TrueMBB -> removeSuccessor ( LandMBB , true ) ;


<s> void <ISA_LIT>MachineCFGStructurizer :: mergeLoopbreakBlock ( MachineBasicBlock * ExitingMBB , MachineBasicBlock * LandMBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ExitingMBB -> getNumber ( ) << <STR_LIT> << LandMBB -> getNumber ( ) << <STR_LIT> ; ) ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( ExitingMBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; DebugLoc DL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock * TrueBranch = getTrueBranch ( BranchMI ) ; MachineBasicBlock :: iterator I = BranchMI ; if ( TrueBranch != LandMBB ) reversePredicateSetter ( I , * I -> getParent ( ) ) ; insertCondBranchBefore ( ExitingMBB , I , <STR_LIT> , <STR_LIT> , DL ) ; insertInstrBefore ( I , <STR_LIT> ) ;


<s> insertInstrBefore ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ; insertInstrEnd ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ; DstBlk -> replaceSuccessor ( DstBlk , LandMBB ) ;


<s> DstMBB -> splice ( DstMBB -> end ( ) , SrcMBB , SrcMBB -> begin ( ) , SrcMBB -> end ( ) ) ; DstMBB -> removeSuccessor ( SrcMBB , true ) ; cloneSuccessorList ( DstMBB , SrcMBB ) ; removeSuccessor ( SrcMBB ) ;


<s> } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; SpliceEnd = BranchMI ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << <STR_LIT> ; ) ; DstMBB -> splice ( I , SrcMBB , SrcMBB -> begin ( ) , SpliceEnd ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << '\n' ; ) ;


<s> unsigned BlkSize = MBB -> size ( ) ;

return ( ( BlkSize > BlockSizeThreshold ) && ( BlkSize * ( MBB -> pred_size ( ) - <NUM_LIT> ) > CloneInstrThreshold ) ) ;


<s> return ( ( BlkSize > BlockSizeThreshold ) && ( BlkSize * ( MBB -> pred_size ( ) - <NUM_LIT> ) > CloneInstrThreshold ) ) ;


<s> if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ;


<s> MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; MachineBasicBlock * LoopLatch = LoopRep -> getLoopLatch ( ) ; if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ; FuncRep -> push_back ( DummyExitBlk ) ; SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI << <STR_LIT> ; ) ; LLVMContext & Ctx = LoopHeader -> getParent ( ) -> getFunction ( ) . getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ;


<s> for ( scc_iterator < MachineFunction * > It = scc_begin ( MF ) ; ! It . isAtEnd ( ) ; ++ It , ++ SccNum ) { const std :: vector < MachineBasicBlock * > & SccNext = * It ; for ( MachineBasicBlock * MBB : SccNext ) { OrderedBlks . push_back ( MBB ) ; recordSccnum ( MBB , SccNum ) ; }


<s> for ( MachineBasicBlock * MBB : SccNext ) { OrderedBlks . push_back ( MBB ) ; recordSccnum ( MBB , SccNum ) ; } } for ( auto * MBB : nodes ( MF ) ) { SccNum = getSCCNum ( MBB ) ;

if ( SccNum == INVALIDSCCNUM ) dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ;


<s> int NumMatch = <NUM_LIT> ; int CurMatch ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; while ( ( CurMatch = patternMatchGroup ( MBB ) ) > <NUM_LIT> ) NumMatch += CurMatch ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << NumMatch << <STR_LIT> ; ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; while ( ( CurMatch = patternMatchGroup ( MBB ) ) > <NUM_LIT> ) NumMatch += CurMatch ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << NumMatch << <STR_LIT> ; ) ; return NumMatch ;


<s> NumMatch += serialPatternMatch ( MBB ) ; NumMatch += ifPatternMatch ( MBB ) ; return NumMatch ;


<s> NumMatch += loopendPatternMatch ( ) ; NumMatch += serialPatternMatch ( MBB ) ; NumMatch += ifPatternMatch ( MBB ) ; return NumMatch ;


<s> MBBVector ExitingMBBs ; LoopRep -> getExitingBlocks ( ExitingMBBs ) ; if ( ExitingMBBs . size ( ) == <NUM_LIT> ) { MachineBasicBlock * DummyExitBlk = normalizeInfiniteLoopExit ( LoopRep ) ; if ( DummyExitBlk ) RetBlks . push_back ( DummyExitBlk ) ; } } for ( MachineBasicBlock * MBB : OrderedBlks ) { removeUnconditionalBranch ( MBB ) ; removeRedundantConditionalBranch ( MBB ) ; if ( isReturnBlock ( MBB ) ) {


<s> void <ISA_LIT>MachineCFGStructurizer :: recordSccnum ( MachineBasicBlock * MBB , int SccNum ) {

BlockInformation * & srcBlkInfo = BlockInfoMap [ MBB ] ;


<s> assert ( BranchMI && isCondBranch ( BranchMI ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; BranchMI -> eraseFromParent ( ) ; SHOWNEWBLK ( MBB1 , <STR_LIT> ) ;

MBB -> removeSuccessor ( MBB1 , true ) ;


<s> void <ISA_LIT>MachineCFGStructurizer :: removeRedundantConditionalBranch ( MachineBasicBlock * MBB ) { if ( MBB -> succ_size ( ) != <NUM_LIT> ) return ; MachineBasicBlock * MBB1 = * MBB -> succ_begin ( ) ; MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ; if ( MBB1 != MBB2 ) return ; MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ;


<s> void <ISA_LIT>MachineCFGStructurizer :: removeSuccessor ( MachineBasicBlock * MBB ) { while ( MBB -> succ_size ( ) ) MBB -> removeSuccessor ( * MBB -> succ_begin ( ) ) ;


<s> while ( MBB -> succ_size ( ) ) MBB -> removeSuccessor ( * MBB -> succ_begin ( ) ) ;


<s> while ( ( BranchMI = getLoopendBlockBranchInstr ( MBB ) ) && isUncondBranch ( BranchMI ) ) {


<s> if ( BranchMI && isCondBranch ( BranchMI ) && getTrueBranch ( BranchMI ) == OldMBB ) setTrueBranch ( BranchMI , NewBlk ) ;


<s> if ( ! SrcBlkInfo ) SrcBlkInfo = new BlockInformation ( ) ; SrcBlkInfo -> IsRetired = true ; assert ( MBB -> succ_empty ( ) && MBB -> pred_empty ( ) && <STR_LIT> ) ;


<s> void <ISA_LIT>MachineCFGStructurizer :: reversePredicateSetter ( MachineBasicBlock :: iterator I , MachineBasicBlock & MBB ) { assert ( I . isValid ( ) && <STR_LIT> ) ; for ( ; ; -- I ) { if ( I == MBB . end ( ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( I -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ; case <STR_LIT> : I -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ) ; return ;


<s> while ( It != E ) { MBB = * It ; if ( ! SccBeginMBB ) { SccBeginIter = It ; SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ;


<s> MachineBasicBlock * childBlk = * MBB -> succ_begin ( ) ; if ( childBlk -> pred_size ( ) != <NUM_LIT> || isActiveLoophead ( childBlk ) ) return <NUM_LIT> ;


<s> insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; } else { int BranchOpcode = TrueBranch == ContMBB ? getContinueNzeroOpcode ( OldOpcode ) : getContinueZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; } MI -> eraseFromParent ( ) ; } else {

insertInstrEnd ( ContingMBB , <STR_LIT> , getLastDebugLocInBB ( ContingMBB ) ) ;


<s> if ( ! UseContinueLogical ) { int BranchOpcode = TrueBranch == ContMBB ? getBranchNzeroOpcode ( OldOpcode ) : getBranchZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; } else { int BranchOpcode = TrueBranch == ContMBB ? getContinueNzeroOpcode ( OldOpcode ) : getContinueZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; }


<s> MI -> getOperand ( <NUM_LIT> ) . setMBB ( MBB ) ;


<s> dbgs ( ) << <STR_LIT> ; } } if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) << <STR_LIT> << FalseMBB -> size ( ) << <STR_LIT> << FalseMBB -> pred_size ( ) ; if ( Detail ) { dbgs ( ) << <STR_LIT> ; FalseMBB -> print ( dbgs ( ) ) ; dbgs ( ) << <STR_LIT> ; } } if ( LandMBB ) {

dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) << <STR_LIT> << LandMBB -> size ( ) << <STR_LIT> << LandMBB -> pred_size ( ) ;


<s> assert ( DstMBB ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; while ( SrcMBB && SrcMBB -> succ_size ( ) == <NUM_LIT> ) { SrcMBB = * SrcMBB -> succ_begin ( ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; if ( ! AllowSideEntry && SrcMBB -> pred_size ( ) > <NUM_LIT> ) return Not_SinglePath ; } if ( SrcMBB && SrcMBB -> succ_size ( ) == <NUM_LIT> ) return SinglePath_NotInPath ; return Not_SinglePath ;


<s> <ISA_LIT>MachineCFGStructurizer :: PathToKind <ISA_LIT>MachineCFGStructurizer :: singlePathTo ( MachineBasicBlock * SrcMBB , MachineBasicBlock * DstMBB , bool AllowSideEntry ) const { assert ( DstMBB ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; while ( SrcMBB && SrcMBB -> succ_size ( ) == <NUM_LIT> ) { SrcMBB = * SrcMBB -> succ_begin ( ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; if ( ! AllowSideEntry && SrcMBB -> pred_size ( ) > <NUM_LIT> ) return Not_SinglePath ; } if ( SrcMBB && SrcMBB -> succ_size ( ) == <NUM_LIT> ) return SinglePath_NotInPath ;


<s> while ( It != E ) { if ( Pre -> getOpcode ( ) == <STR_LIT> && It -> getOpcode ( ) == <STR_LIT> ) ContInstr . push_back ( & * Pre ) ; Pre = It ; ++ It ; }

for ( unsigned i = <NUM_LIT> ; i < ContInstr . size ( ) ; ++ i ) ContInstr [ i ] -> eraseFromParent ( ) ;


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) {


<s> static const AluKind IndexToID [ ] = { AluT_X , AluT_Y , AluT_Z , AluT_W } ; SUnit * SlotedSU = PopInst ( AvailableAlus [ IndexToID [ Slot ] ] , AnyAlu ) ; if ( SlotedSU ) return SlotedSU ;

SUnit * UnslotedSU = PopInst ( AvailableAlus [ AluAny ] , AnyAlu ) ;


<s> unsigned <ISA_LIT>SchedStrategy :: AvailablesAluCount ( ) const { return AvailableAlus [ AluAny ] . size ( ) + AvailableAlus [ AluT_XYZW ] . size ( ) + AvailableAlus [ AluT_X ] . size ( ) + AvailableAlus [ AluT_Y ] . size ( ) + AvailableAlus [ AluT_Z ] . size ( ) + AvailableAlus [ AluT_W ] . size ( ) + AvailableAlus [ AluTrans ] . size ( ) + AvailableAlus [ AluDiscarded ] . size ( ) + AvailableAlus [ AluPredX ] . size ( ) ;


<s> int <ISA_LIT>SchedStrategy :: getInstKind ( SUnit * SU ) { int Opcode = SU -> getInstr ( ) -> getOpcode ( ) ; if ( TII -> usesTextureCache ( Opcode ) || TII -> usesVertexCache ( Opcode ) ) return IDFetch ; if ( TII -> isALUInstr ( Opcode ) ) {


<s> static unsigned getWFCountLimitedByGPR ( unsigned GPRCount ) { assert ( GPRCount && <STR_LIT> ) ;


<s> DEBUG ( if ( SU ) { dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ;


<s> static bool isPhysicalRegCopy ( MachineInstr * MI ) { if ( MI -> getOpcode ( ) != <STR_LIT> ) return false ;

return ! TargetRegisterInfo :: isVirtualRegister ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> AluKind AK = getAluKind ( QSrc [ i ] ) ;

AvailableAlus [ AK ] . push_back ( QSrc [ i ] ) ;


<s> while ( AvailablesAluCount ( ) || ! Pending [ IDAlu ] . empty ( ) ) { if ( ! OccupedSlotsMask ) { if ( ! AvailableAlus [ AluPredX ] . empty ( ) ) { OccupedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluPredX ] , false ) ; } if ( ! AvailableAlus [ AluDiscarded ] . empty ( ) ) { OccupedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluDiscarded ] , false ) ; } if ( ! AvailableAlus [ AluT_XYZW ] . empty ( ) ) { OccupedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluT_XYZW ] , false ) ; }


<s> if ( NeededWF > getWFCountLimitedByGPR ( NearRegisterRequirement ) ) AllowSwitchFromAlu = true ; } } if ( ! SU && ( ( AllowSwitchToAlu && CurInstKind != IDAlu ) || ( ! AllowSwitchFromAlu && CurInstKind == IDAlu ) ) ) { SU = pickAlu ( ) ; if ( ! SU && ! PhysicalRegCopy . empty ( ) ) { SU = PhysicalRegCopy . front ( ) ; PhysicalRegCopy . erase ( PhysicalRegCopy . begin ( ) ) ; } if ( SU ) { if ( CurEmitted >= InstKindLimit [ IDAlu ] ) CurEmitted = <NUM_LIT> ; NextInstKind = IDAlu ; } } if ( ! SU ) { SU = pickOther ( IDFetch ) ; if ( SU ) NextInstKind = IDFetch ; } if ( ! SU ) { SU = pickOther ( IDOther ) ; if ( SU ) NextInstKind = IDOther ; } DEBUG ( if ( SU ) { dbgs ( ) << <STR_LIT> ;


<s> if ( ! AQ . empty ( ) ) { SU = AQ . back ( ) ;

AQ . resize ( AQ . size ( ) - <NUM_LIT> ) ;


<s> SUnit * <ISA_LIT>SchedStrategy :: pickOther ( int QID ) { SUnit * SU = nullptr ; std :: vector < SUnit * > & AQ = Available [ QID ] ; if ( AQ . empty ( ) ) { MoveUnits ( Pending [ QID ] , AQ ) ; } if ( ! AQ . empty ( ) ) { SU = AQ . back ( ) ; AQ . resize ( AQ . size ( ) - <NUM_LIT> ) ; }


<s> InstructionsGroupCandidate . push_back ( SU -> getInstr ( ) ) ; if ( TII -> fitsConstReadLimitations ( InstructionsGroupCandidate ) && ( ! AnyALU || ! TII -> isVectorOnly ( SU -> getInstr ( ) ) ) ) { InstructionsGroupCandidate . pop_back ( ) ; Q . erase ( ( It + <NUM_LIT> ) . base ( ) ) ; return SU ; } else {


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; assert ( OccupedSlotsMask && <STR_LIT> ) ; OccupedSlotsMask = <NUM_LIT> ; InstructionsGroupCandidate . clear ( ) ; LoadAlu ( ) ;


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { return RC -> contains ( Reg ) ; } else { return MRI -> getRegClass ( Reg ) == RC ; }


<s> void <ISA_LIT>SchedStrategy :: releaseBottomNode ( SUnit * SU ) {

DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ; ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ; ) ; if ( isPhysicalRegCopy ( SU -> getInstr ( ) ) ) { PhysicalRegCopy . push_back ( SU ) ;


<s> void <ISA_LIT>SchedStrategy :: releaseTopNode ( SUnit * SU ) {


<s> void <ISA_LIT>SchedStrategy :: releaseTopNode ( SUnit * SU ) { DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ; ) ;


<s> CurEmitted += <NUM_LIT> ; break ; case AluDiscarded : break ; default : { ++ CurEmitted ; for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ; }


<s> void <ISA_LIT>SchedStrategy :: schedNode ( SUnit * SU , bool IsTopNode ) { if ( NextInstKind != CurInstKind ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( NextInstKind != IDAlu ) OccupedSlotsMask |= <NUM_LIT> ; CurEmitted = <NUM_LIT> ; CurInstKind = NextInstKind ; } if ( CurInstKind == IDAlu ) { AluInstCount ++ ; switch ( getAluKind ( SU ) ) { case AluT_XYZW : CurEmitted += <NUM_LIT> ; break ; case AluDiscarded : break ; default : { ++ CurEmitted ; for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ;

if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ;


<s> virtual ~ <ISA_LIT>SchedStrategy ( ) {


<s> Register DestReg = MI -> getOperand ( DstIndex ) . getReg ( ) ; for ( MachineInstr :: mop_iterator It = MI -> operands_begin ( ) , E = MI -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && ! MO . isDef ( ) && MO . getReg ( ) == DestReg ) return ; } switch ( Slot ) { case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> :


<s> if ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) { return AluDiscarded ; } break ; default : break ; } if ( TII -> isVector ( * MI ) || TII -> isCubeOp ( MI -> getOpcode ( ) ) || TII -> isReductionOp ( MI -> getOpcode ( ) ) || MI -> getOpcode ( ) == <STR_LIT> ) { return AluT_XYZW ; } if ( TII -> isLD<ISA_LIT>nstr ( MI -> getOpcode ( ) ) ) { return AluT_X ; } unsigned DestSubReg = MI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; switch ( DestSubReg ) { case <STR_LIT> : return AluT_X ; case <STR_LIT> :


<s> return IDAlu ; } switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return IDAlu ; default : return IDOther ; }


<s> LLVM_DEBUG ( if ( SU ) { dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SU ) ;


<s> void <ISA_LIT>SchedStrategy :: initialize ( ScheduleDAGMI * dag ) { assert ( dag -> hasVRegLiveness ( ) && <STR_LIT> ) ; DAG = static_cast < ScheduleDAGMILive * > ( dag ) ; const <ISA_LIT>Subtarget & ST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = static_cast < const <ISA_LIT>InstrInfo * > ( DAG -> TII ) ; TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( DAG -> TRI ) ; VLIW5 = ! ST . hasCaymanISA ( ) ; MRI = & DAG -> MRI ; CurInstKind = IDOther ; CurEmitted = <NUM_LIT> ; OccupiedSlotsMask = <NUM_LIT> ; InstKindLimit [ IDAlu ] = TII -> getMaxAlusPerClause ( ) ; InstKindLimit [ IDOther ] = <NUM_LIT> ;

InstKindLimit [ IDFetch ] = ST . getTexVTXClauseSize ( ) ;


<s> static bool isPhysicalRegCopy ( MachineInstr * MI ) {

if ( MI -> getOpcode ( ) != <STR_LIT> ) return false ;


<s> static bool isPhysicalRegCopy ( MachineInstr * MI ) { if ( MI -> getOpcode ( ) != <STR_LIT> ) return false ; return ! MI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ;


<s> while ( AvailablesAluCount ( ) || ! Pending [ IDAlu ] . empty ( ) ) { if ( ! OccupiedSlotsMask ) { if ( ! AvailableAlus [ AluPredX ] . empty ( ) ) { OccupiedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluPredX ] , false ) ; } if ( ! AvailableAlus [ AluDiscarded ] . empty ( ) ) { OccupiedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluDiscarded ] , false ) ; } if ( ! AvailableAlus [ AluT_XYZW ] . empty ( ) ) { OccupiedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluT_XYZW ] , false ) ; } } bool TransSlotOccupied = OccupiedSlotsMask & <NUM_LIT> ; if ( ! TransSlotOccupied && VLIW5 ) { if ( ! AvailableAlus [ AluTrans ] . empty ( ) ) { OccupiedSlotsMask |= <NUM_LIT> ; return PopInst ( AvailableAlus [ AluTrans ] , false ) ; } SUnit * SU = AttemptFillSlot ( <NUM_LIT> , true ) ; if ( SU ) { OccupiedSlotsMask |= <NUM_LIT> ; return SU ; } } for ( int Chan = <NUM_LIT> ; Chan > - <NUM_LIT> ; -- Chan ) { bool isOccupied = OccupiedSlotsMask & ( <NUM_LIT> << Chan ) ; if ( ! isOccupied ) {

SUnit * SU = AttemptFillSlot ( Chan , false ) ;


<s> float ALUFetchRationEstimate = ( AluInstCount + AvailablesAluCount ( ) + Pending [ IDAlu ] . size ( ) ) / ( FetchInstCount + Available [ IDFetch ] . size ( ) ) ; if ( ALUFetchRationEstimate == <NUM_LIT> ) { AllowSwitchFromAlu = true ; } else { unsigned NeededWF = <NUM_LIT> / ALUFetchRationEstimate ; LLVM_DEBUG ( dbgs ( ) << NeededWF << <STR_LIT> ) ; unsigned NearRegisterRequirement = <NUM_LIT> * Available [ IDFetch ] . size ( ) ; if ( NeededWF > getWFCountLimitedByGPR ( NearRegisterRequirement ) ) AllowSwitchFromAlu = true ; } } if ( ! SU && ( ( AllowSwitchToAlu && CurInstKind != IDAlu ) || ( ! AllowSwitchFromAlu && CurInstKind == IDAlu ) ) ) { SU = pickAlu ( ) ; if ( ! SU && ! PhysicalRegCopy . empty ( ) ) { SU = PhysicalRegCopy . front ( ) ; PhysicalRegCopy . erase ( PhysicalRegCopy . begin ( ) ) ; } if ( SU ) { if ( CurEmitted >= InstKindLimit [ IDAlu ] ) CurEmitted = <NUM_LIT> ; NextInstKind = IDAlu ; } }


<s> AllowSwitchFromAlu = true ; } else { unsigned NeededWF = <NUM_LIT> / ALUFetchRationEstimate ; LLVM_DEBUG ( dbgs ( ) << NeededWF << <STR_LIT> ) ; unsigned NearRegisterRequirement = <NUM_LIT> * Available [ IDFetch ] . size ( ) ; if ( NeededWF > getWFCountLimitedByGPR ( NearRegisterRequirement ) ) AllowSwitchFromAlu = true ; } } if ( ! SU && ( ( AllowSwitchToAlu && CurInstKind != IDAlu ) || ( ! AllowSwitchFromAlu && CurInstKind == IDAlu ) ) ) { SU = pickAlu ( ) ; if ( ! SU && ! PhysicalRegCopy . empty ( ) ) { SU = PhysicalRegCopy . front ( ) ; PhysicalRegCopy . erase ( PhysicalRegCopy . begin ( ) ) ;


<s> SUnit * <ISA_LIT>SchedStrategy :: pickOther ( int QID ) { SUnit * SU = nullptr ; std :: vector < SUnit * > & AQ = Available [ QID ] ; if ( AQ . empty ( ) ) { MoveUnits ( Pending [ QID ] , AQ ) ; } if ( ! AQ . empty ( ) ) { SU = AQ . back ( ) ; AQ . pop_back ( ) ;


<s> SUnit * SU = nullptr ; std :: vector < SUnit * > & AQ = Available [ QID ] ; if ( AQ . empty ( ) ) { MoveUnits ( Pending [ QID ] , AQ ) ; } if ( ! AQ . empty ( ) ) { SU = AQ . back ( ) ;


<s> It != E ; ++ It ) { SUnit * SU = * It ; InstructionsGroupCandidate . push_back ( SU -> getInstr ( ) ) ; if ( TII -> fitsConstReadLimitations ( InstructionsGroupCandidate ) && ( ! AnyALU || ! TII -> isVectorOnly ( * SU -> getInstr ( ) ) ) ) { InstructionsGroupCandidate . pop_back ( ) ; Q . erase ( ( It + <NUM_LIT> ) . base ( ) ) ; return SU ;


<s> assert ( OccupiedSlotsMask && <STR_LIT> ) ; OccupiedSlotsMask = <NUM_LIT> ;

InstructionsGroupCandidate . clear ( ) ;


<s> bool <ISA_LIT>SchedStrategy :: regBelongsToClass ( Register Reg , const TargetRegisterClass * RC ) const { if ( ! Reg . isVirtual ( ) ) { return RC -> contains ( Reg ) ;


<s> int IK = getInstKind ( SU ) ; if ( IK == IDOther ) Available [ IDOther ] . push_back ( SU ) ;

else Pending [ IK ] . push_back ( SU ) ;


<s> if ( IK == IDOther ) Available [ IDOther ] . push_back ( SU ) ;


<s> case AluDiscarded : break ; default : { ++ CurEmitted ; for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ; } } } } else { ++ CurEmitted ; } LLVM_DEBUG ( dbgs ( ) << CurEmitted << <STR_LIT> ) ; if ( CurInstKind != IDFetch ) { MoveUnits ( Pending [ IDFetch ] , Available [ IDFetch ] ) ; } else FetchInstCount ++ ;


<s> switch ( getAluKind ( SU ) ) { case AluT_XYZW : CurEmitted += <NUM_LIT> ; break ; case AluDiscarded : break ; default : { ++ CurEmitted ; for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ; } } } } else { ++ CurEmitted ; }

LLVM_DEBUG ( dbgs ( ) << CurEmitted << <STR_LIT> ) ;


<s> if ( TII -> isVector ( * MI ) || TII -> isCubeOp ( MI -> getOpcode ( ) ) || TII -> isReductionOp ( MI -> getOpcode ( ) ) || MI -> getOpcode ( ) == <STR_LIT> ) { return AluT_XYZW ; } if ( TII -> isLD<ISA_LIT>nstr ( MI -> getOpcode ( ) ) ) { return AluT_X ; } unsigned DestSubReg = MI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; switch ( DestSubReg ) { case <STR_LIT> : return AluT_X ; case <STR_LIT> : return AluT_Y ; case <STR_LIT> : return AluT_Z ; case <STR_LIT> : return AluT_W ; default : break ;


<s> LLVM_DEBUG ( if ( SU ) { dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ;


<s> void <ISA_LIT>SchedStrategy :: initialize ( ScheduleDAGMI * dag ) { assert ( dag -> hasVRegLiveness ( ) && <STR_LIT> ) ; DAG = static_cast < ScheduleDAGMILive * > ( dag ) ; const <ISA_LIT>Subtarget & ST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = static_cast < const <ISA_LIT>InstrInfo * > ( DAG -> TII ) ; TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( DAG -> TRI ) ; VLIW5 = ! ST . hasCaymanISA ( ) ; MRI = & DAG -> MRI ; CurInstKind = IDOther ; CurEmitted = <NUM_LIT> ; OccupedSlotsMask = <NUM_LIT> ;

InstKindLimit [ IDAlu ] = TII -> getMaxAlusPerClause ( ) ;


<s> CurInstKind = IDOther ; CurEmitted = <NUM_LIT> ; OccupedSlotsMask = <NUM_LIT> ; InstKindLimit [ IDAlu ] = TII -> getMaxAlusPerClause ( ) ; InstKindLimit [ IDOther ] = <NUM_LIT> ; InstKindLimit [ IDFetch ] = ST . getTexVTXClauseSize ( ) ; AluInstCount = <NUM_LIT> ; FetchInstCount = <NUM_LIT> ;


<s> void <ISA_LIT>SchedStrategy :: PrepareNextSlot ( ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; assert ( OccupedSlotsMask && <STR_LIT> ) ; OccupedSlotsMask = <NUM_LIT> ; InstructionsGroupCandidate . clear ( ) ; LoadAlu ( ) ;


<s> OccupedSlotsMask = <NUM_LIT> ; InstructionsGroupCandidate . clear ( ) ; LoadAlu ( ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ; ) ; if ( isPhysicalRegCopy ( SU -> getInstr ( ) ) ) { PhysicalRegCopy . push_back ( SU ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ; ) ; if ( isPhysicalRegCopy ( SU -> getInstr ( ) ) ) { PhysicalRegCopy . push_back ( SU ) ; return ; } int IK = getInstKind ( SU ) ;

if ( IK == IDOther ) Available [ IDOther ] . push_back ( SU ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ; ) ;


<s> void <ISA_LIT>SchedStrategy :: releaseTopNode ( SUnit * SU ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ; ) ;


<s> if ( CurInstKind == IDAlu ) { AluInstCount ++ ; switch ( getAluKind ( SU ) ) { case AluT_XYZW : CurEmitted += <NUM_LIT> ; break ; case AluDiscarded : break ; default : { ++ CurEmitted ; for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ; } } } } else { ++ CurEmitted ; } LLVM_DEBUG ( dbgs ( ) << CurEmitted << <STR_LIT> ) ; if ( CurInstKind != IDFetch ) {

MoveUnits ( Pending [ IDFetch ] , Available [ IDFetch ] ) ;


<s> if ( DstIndex == - <NUM_LIT> ) { return ; } unsigned DestReg = MI -> getOperand ( DstIndex ) . getReg ( ) ; for ( MachineInstr :: mop_iterator It = MI -> operands_begin ( ) , E = MI -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && ! MO . isDef ( ) && MO . getReg ( ) == DestReg ) return ; } switch ( Slot ) { case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ; }


<s> if ( DstIndex == - <NUM_LIT> ) { return ; } unsigned DestReg = MI -> getOperand ( DstIndex ) . getReg ( ) ; for ( MachineInstr :: mop_iterator It = MI -> operands_begin ( ) , E = MI -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && ! MO . isDef ( ) && MO . getReg ( ) == DestReg ) return ; } switch ( Slot ) { case <NUM_LIT> : MRI -> constrainRegClass ( DestReg , & <STR_LIT> ) ; break ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return AluT_XYZW ; case <STR_LIT> : if ( MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ) { return AluDiscarded ; } break ; default : break ; } if ( TII -> isVector ( * MI ) || TII -> isCubeOp ( MI -> getOpcode ( ) ) || TII -> isReductionOp ( MI -> getOpcode ( ) ) || MI -> getOpcode ( ) == <STR_LIT> ) { return AluT_XYZW ; } if ( TII -> isLD<ISA_LIT>nstr ( MI -> getOpcode ( ) ) ) { return AluT_X ; } unsigned DestSubReg = MI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; switch ( DestSubReg ) { case <STR_LIT> : return AluT_X ; case <STR_LIT> : return AluT_Y ; case <STR_LIT> : return AluT_Z ;


<s> if ( MI -> getOpcode ( ) != <STR_LIT> ) return false ;

return ! TargetRegisterInfo :: isVirtualRegister ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> if ( SU ) { if ( CurEmitted >= InstKindLimit [ IDAlu ] ) CurEmitted = <NUM_LIT> ; NextInstKind = IDAlu ; } } if ( ! SU ) { SU = pickOther ( IDFetch ) ; if ( SU ) NextInstKind = IDFetch ; } if ( ! SU ) { SU = pickOther ( IDOther ) ; if ( SU ) NextInstKind = IDOther ; } LLVM_DEBUG ( if ( SU ) { dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SU ) ; } else { dbgs ( ) << <STR_LIT> ; for ( unsigned i = <NUM_LIT> ; i < DAG -> SUnits . size ( ) ; i ++ ) {


<s> for ( MachineInstr :: mop_iterator It = SU -> getInstr ( ) -> operands_begin ( ) , E = SU -> getInstr ( ) -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ CurEmitted ; } } } } else { ++ CurEmitted ; } LLVM_DEBUG ( dbgs ( ) << CurEmitted << <STR_LIT> ) ; if ( CurInstKind != IDFetch ) { MoveUnits ( Pending [ IDFetch ] , Available [ IDFetch ] ) ;


<s> } else { return MRI -> getRegClass ( Reg ) == RC ; }


<s> switch ( DestSubReg ) { case <STR_LIT> : return AluT_X ; case <STR_LIT> : return AluT_Y ; case <STR_LIT> : return AluT_Z ; case <STR_LIT> : return AluT_W ; default : break ; } unsigned DestReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) || regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_X ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_Y ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_Z ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_W ; if ( regBelongsToClass ( DestReg , & <STR_LIT> ) ) return AluT_XYZW ; if ( TII -> readsLDSSrcReg ( * MI ) ) return AluT_XYZW ;


<s> std :: vector < SUnit * > & QSrc = Pending [ IDAlu ] ; for ( SUnit * SU : QSrc ) { AluKind AK = getAluKind ( SU ) ; AvailableAlus [ AK ] . push_back ( SU ) ;


<s> return new <ISA_LIT>MCCodeEmitter ( MCII , * Ctx . getRegisterInfo ( ) ) ;


<s> <STR_LIT> :: Writer < <STR_LIT> > ( OS ) . write ( Value ) ;


<s> void <ISA_LIT>MCCodeEmitter :: EmitByte ( unsigned int Byte , raw_ostream & OS ) const { OS . write ( ( uint8_t ) Byte & <NUM_LIT> ) ;


<s> OS . write ( ( uint8_t ) Byte & <NUM_LIT> ) ;


<s> Emit ( InstWord01 , OS ) ; Emit ( InstWord2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else if ( IS_TEX ( Desc ) ) { int64_t Sampler = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; int64_t SrcSelect [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ; Emit ( Word01 , OS ) ; Emit ( Word2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else { uint64_t Inst = getBinaryCodeForInstr ( MI , Fixups , STI ) ; if ( ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) && ( ( Desc . TSFlags & <STR_LIT> ) || Desc . TSFlags & <STR_LIT> ) ) {

uint64_t ISAOpCode = Inst & ( <NUM_LIT> << <NUM_LIT> ) ;


<s> unsigned <ISA_LIT>MCCodeEmitter :: getHWRegChan ( unsigned reg ) const {

return MRI . getEncodingValue ( reg ) >> HW_CHAN_SHIFT ;


<s> unsigned <ISA_LIT>MCCodeEmitter :: getHWRegChan ( unsigned reg ) const { return MRI . getEncodingValue ( reg ) >> HW_CHAN_SHIFT ;


<s> const unsigned offset = ( & MO == & MI . getOperand ( <NUM_LIT> ) ) ? <NUM_LIT> : <NUM_LIT> ; Fixups . push_back ( MCFixup :: create ( offset , Expr , FK_SecRel_4 , MI . getLoc ( ) ) ) ; return <NUM_LIT> ; } assert ( MO . isImm ( ) ) ;


<s> <ISA_LIT>MCCodeEmitter ( const MCInstrInfo & mcii , const MCRegisterInfo & mri ) : MCII ( mcii ) , MRI ( mri ) {


<s> return new <ISA_LIT>MCCodeEmitter ( MCII , MRI ) ;


<s> void <ISA_LIT>MCCodeEmitter :: Emit ( uint64_t Value , raw_ostream & OS ) const { <STR_LIT> :: write ( OS , Value , <STR_LIT> ) ;


<s> MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ;


<s> int64_t SrcSelect [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ;


<s> unsigned <ISA_LIT>MCCodeEmitter :: getHWReg ( unsigned RegNo ) const {

return MRI . getEncodingValue ( RegNo ) & HW_REG_MASK ;


<s> unsigned <ISA_LIT>MCCodeEmitter :: getHWReg ( unsigned RegNo ) const { return MRI . getEncodingValue ( RegNo ) & HW_REG_MASK ;


<s> Fixups . push_back ( MCFixup :: create ( offset , MO . getExpr ( ) , FK_SecRel_4 , MI . getLoc ( ) ) ) ; return <NUM_LIT> ; } assert ( MO . isImm ( ) ) ; return MO . getImm ( ) ;


<s> <ISA_LIT>MCCodeEmitter ( const MCInstrInfo & mcii , const MCRegisterInfo & mri ) : MRI ( mri ) , MCII ( mcii ) {


<s> if ( HAS_NATIVE_OPERANDS ( MCII . get ( MI . getOpcode ( ) ) . TSFlags ) ) return MRI . getEncodingValue ( MO . getReg ( ) ) ; return getHWReg ( MO . getReg ( ) ) ; } assert ( MO . isImm ( ) ) ;


<s> uint64_t <ISA_LIT>MCCodeEmitter :: getMachineOpValue ( const MCInst & MI , const MCOperand & MO , SmallVectorImpl < MCFixup > & Fixup , const MCSubtargetInfo & STI ) const { if ( MO . isReg ( ) ) { if ( HAS_NATIVE_OPERANDS ( MCII . get ( MI . getOpcode ( ) ) . TSFlags ) ) return MRI . getEncodingValue ( MO . getReg ( ) ) ; return getHWReg ( MO . getReg ( ) ) ; } assert ( MO . isImm ( ) ) ;


<s> const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { return ; } else if ( IS_VTX ( Desc ) ) { uint64_t InstWord01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t InstWord2 = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ! ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) { InstWord2 |= <NUM_LIT> << <NUM_LIT> ; } Emit ( InstWord01 , OS ) ; Emit ( InstWord2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else if ( IS_TEX ( Desc ) ) { int64_t Sampler = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; int64_t SrcSelect [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ; Emit ( Word01 , OS ) ; Emit ( Word2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else { uint64_t Inst = getBinaryCodeForInstr ( MI , Fixups , STI ) ; if ( ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) && ( ( Desc . TSFlags & <STR_LIT> ) || Desc . TSFlags & <STR_LIT> ) ) {


<s> return ; } else if ( IS_VTX ( Desc ) ) { uint64_t InstWord01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t InstWord2 = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ! ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) { InstWord2 |= <NUM_LIT> << <NUM_LIT> ; } Emit ( InstWord01 , OS ) ; Emit ( InstWord2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else if ( IS_TEX ( Desc ) ) { int64_t Sampler = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; int64_t SrcSelect [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) , MI . getOperand ( <NUM_LIT> ) . getImm ( ) } ; int64_t Offsets [ <NUM_LIT> ] = { MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> , MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> } ; uint64_t Word01 = getBinaryCodeForInstr ( MI , Fixups , STI ) ; uint32_t Word2 = Sampler << <NUM_LIT> | SrcSelect [ ELEMENT_X ] << <NUM_LIT> | SrcSelect [ ELEMENT_Y ] << <NUM_LIT> | SrcSelect [ ELEMENT_Z ] << <NUM_LIT> | SrcSelect [ ELEMENT_W ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> | Offsets [ <NUM_LIT> ] << <NUM_LIT> ; Emit ( Word01 , OS ) ; Emit ( Word2 , OS ) ; Emit ( ( uint32_t ) <NUM_LIT> , OS ) ; } else { uint64_t Inst = getBinaryCodeForInstr ( MI , Fixups , STI ) ; if ( ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) && ( ( Desc . TSFlags & <STR_LIT> ) || Desc . TSFlags & <STR_LIT> ) ) { uint64_t ISAOpCode = Inst & ( <NUM_LIT> << <NUM_LIT> ) ; Inst &= ~ ( <NUM_LIT> << <NUM_LIT> ) ;


<s> <ISA_LIT>MCCodeEmitter ( const MCInstrInfo & mcii , const MCRegisterInfo & mri ) : <ISA_LIT>MCCodeEmitter ( mcii ) , MRI ( mri ) {


<s> <ISA_LIT>MCInstLower :: <ISA_LIT>MCInstLower ( MCContext & Ctx , const <ISA_LIT>Subtarget & ST , const AsmPrinter & AP ) : <ISA_LIT>MCInstLower ( Ctx , ST , AP ) {


<s> MCInstrInfo * X = new MCInstrInfo ( ) ; Init<ISA_LIT>MCInstrInfo ( X ) ; return X ;


<s> MCInstrInfo * llvm :: create<ISA_LIT>MCInstrInfo ( ) { MCInstrInfo * X = new MCInstrInfo ( ) ; Init<ISA_LIT>MCInstrInfo ( X ) ; return X ;


<s> ArgTypes . push_back ( ImageSizeType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageSizeArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; ArgTypes . push_back ( ImageFormatType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageFormatArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; Modified = true ; } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getAddressSpace ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ;

auto NewFArgIt = NewF -> arg_begin ( ) ;


<s> ArgTypes . push_back ( FT -> getParamType ( i ) ) ; MDVector ArgMD = GetArgMD ( KernelMDNode , i + <NUM_LIT> ) ; PushArgMD ( NewArgMDs , ArgMD ) ; if ( ! IsImageType ( ArgTypeFromMD ( KernelMDNode , i ) ) ) continue ; ArgTypes . push_back ( ImageSizeType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageSizeArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; ArgTypes . push_back ( ImageFormatType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageFormatArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; Modified = true ; } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getAddressSpace ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ;


<s> } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ; auto NewFArgIt = NewF -> arg_begin ( ) ; for ( auto & Arg : F -> args ( ) ) { auto ArgName = Arg . getName ( ) ; NewFArgIt -> setName ( ArgName ) ; VMap [ & Arg ] = & ( * NewFArgIt ++ ) ; if ( IsImageType ( ArgTypeFromMD ( KernelMDNode , Arg . getArgNo ( ) ) ) ) { ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; } } SmallVector < ReturnInst * , <NUM_LIT> > Returns ; CloneFunctionInto ( NewF , F , VMap , CloneFunctionChangeType :: LocalChangesOnly , Returns ) ; SmallVector < Metadata * , <NUM_LIT> > KernelMDArgs ; KernelMDArgs . push_back ( ConstantAsMetadata :: get ( NewF ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) KernelMDArgs . push_back ( MDNode :: get ( * Context , NewArgMDs . ArgVector [ i ] ) ) ;

MDNode * NewMDNode = MDNode :: get ( * Context , KernelMDArgs ) ;


<s> StringRef getPassName ( ) const override {


<s> <ISA_LIT>OpenCLImageTypeLoweringPass ( ) : ModulePass ( ID ) {


<s> bool <ISA_LIT>VectorRegMerger :: areAllUsesSwizzeable ( unsigned Reg ) const { for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) {

if ( ! canSwizzle ( * It ) ) return false ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ; default : return false ; }


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>VectorRegMerger ( TargetMachine & tm ) { return new <ISA_LIT>VectorRegMerger ( tm ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ; AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ;

AU . addPreserved < MachineLoopInfo > ( ) ;


<s> const char * getPassName ( ) const override {


<s> if ( RemapChan [ j ] . first == Chan ) return RemapChan [ j ] . second ; } llvm_unreachable ( <STR_LIT> ) ;


<s> <ISA_LIT>VectorRegMerger ( TargetMachine & tm ) : MachineFunctionPass ( ID ) , TII ( nullptr ) {


<s> unsigned Reg = R<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: iterator Pos = R<ISA_LIT> -> Instr ; MachineBasicBlock & MBB = * Pos -> getParent ( ) ; DebugLoc DL = Pos -> getDebugLoc ( ) ; unsigned SrcVec = BaseR<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; DenseMap < unsigned , unsigned > UpdatedRegToChan = BaseR<ISA_LIT> -> RegToChan ; std :: vector < unsigned > UpdatedUndef = BaseR<ISA_LIT> -> UndefReg ; for ( DenseMap < unsigned , unsigned > :: iterator It = R<ISA_LIT> -> RegToChan . begin ( ) , E = R<ISA_LIT> -> RegToChan . end ( ) ; It != E ; ++ It ) { unsigned DstReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned SubReg = ( * It ) . first ; unsigned Swizzle = ( * It ) . second ; unsigned Chan = getReassignedChan ( RemapChan , Swizzle ) ; MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < unsigned > :: iterator ChanPos = std :: find ( UpdatedUndef . begin ( ) , UpdatedUndef . end ( ) , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( std :: find ( UpdatedUndef . begin ( ) , UpdatedUndef . end ( ) , Chan ) == UpdatedUndef . end ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } Pos = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; Pos -> dump ( ) ; ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ( * It ) . dump ( ) ; dbgs ( ) << <STR_LIT> ) ; SwizzleInput ( * It , RemapChan ) ; DEBUG ( ( * It ) . dump ( ) ) ; } R<ISA_LIT> -> Instr -> eraseFromParent ( ) ; R<ISA_LIT> -> Instr = Pos ; R<ISA_LIT> -> RegToChan = UpdatedRegToChan ; R<ISA_LIT> -> UndefReg = UpdatedUndef ;


<s> MIs . erase ( std :: find ( MIs . begin ( ) , MIs . end ( ) , MI ) , MIs . end ( ) ) ; }

for ( InstructionSetMap :: iterator It = PreviousRegSeqByUndefCount . begin ( ) , E = PreviousRegSeqByUndefCount . end ( ) ; It != E ; ++ It ) {


<s> MI -> dump ( ) ; ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned > > RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } trackR<ISA_LIT> ( R<ISA_LIT> ) ; } }


<s> continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , MI ) ; unsigned Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned > > RemapChan ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ;


<s> for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { unsigned Swizzle = MI . getOperand ( i + Offset ) . getImm ( ) + <NUM_LIT> ;

for ( unsigned j = <NUM_LIT> , e = RemapChan . size ( ) ; j < e ; j ++ ) {


<s> if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) Offset = <NUM_LIT> ; else Offset = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { unsigned Swizzle = MI . getOperand ( i + Offset ) . getImm ( ) + <NUM_LIT> ; for ( unsigned j = <NUM_LIT> , e = RemapChan . size ( ) ; j < e ; j ++ ) { if ( RemapChan [ j ] . first == Swizzle ) { MI . getOperand ( i + Offset ) . setImm ( RemapChan [ j ] . second - <NUM_LIT> ) ; break ; } } }


<s> for ( DenseMap < unsigned , unsigned > :: const_iterator It = R<ISA_LIT> . RegToChan . begin ( ) , E = R<ISA_LIT> . RegToChan . end ( ) ; It != E ; ++ It ) { PreviousRegSeqByReg [ ( * It ) . first ] . push_back ( R<ISA_LIT> . Instr ) ; } PreviousRegSeqByUndefCount [ R<ISA_LIT> . UndefReg . size ( ) ] . push_back ( R<ISA_LIT> . Instr ) ;

PreviousRegSeq [ R<ISA_LIT> . Instr ] = R<ISA_LIT> ;


<s> for ( MachineInstr * MI : PreviousRegSeqByReg [ MOp -> getReg ( ) ] ) { CompatibleR<ISA_LIT> = PreviousRegSeq [ MI ] ; if ( R<ISA_LIT> == CompatibleR<ISA_LIT> ) continue ; if ( tryMergeVector ( & CompatibleR<ISA_LIT> , & R<ISA_LIT> , RemapChan ) ) return true ; } } return false ;


<s> if ( PreviousRegSeqByUndefCount [ NeededUndefs ] . empty ( ) ) return false ; std :: vector < MachineInstr * > & MIs = PreviousRegSeqByUndefCount [ NeededUndefs ] ; CompatibleR<ISA_LIT> = PreviousRegSeq [ MIs . back ( ) ] ; tryMergeVector ( & CompatibleR<ISA_LIT> , & R<ISA_LIT> , RemapChan ) ;


<s> std :: vector < MachineInstr * > & MIs = PreviousRegSeqByUndefCount [ NeededUndefs ] ; CompatibleR<ISA_LIT> = PreviousRegSeq [ MIs . back ( ) ] ; tryMergeVector ( & CompatibleR<ISA_LIT> , & R<ISA_LIT> , RemapChan ) ;


<s> bool <ISA_LIT>VectorRegMerger :: tryMergeVector ( const RegSeqInfo * Untouched , RegSeqInfo * ToMerge , std :: vector < std :: pair < unsigned , unsigned > > & Remap ) const { unsigned CurrentUndexIdx = <NUM_LIT> ; for ( DenseMap < unsigned , unsigned > :: iterator It = ToMerge -> RegToChan . begin ( ) , E = ToMerge -> RegToChan . end ( ) ; It != E ; ++ It ) { DenseMap < unsigned , unsigned > :: const_iterator PosInUntouched = Untouched -> RegToChan . find ( ( * It ) . first ) ;

if ( PosInUntouched != Untouched -> RegToChan . end ( ) ) {


<s> assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } MachineInstr * NewMI = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; NewMI -> dump ( ) ; ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ( * It ) . dump ( ) ; dbgs ( ) << <STR_LIT> ) ; SwizzleInput ( * It , RemapChan ) ; LLVM_DEBUG ( ( * It ) . dump ( ) ) ;


<s> for ( MachineRegisterInfo :: def_instr_iterator It = MRI -> def_instr_begin ( Reg ) , E = MRI -> def_instr_end ( ) ; It != E ; ++ It ) { RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ;

if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) {


<s> RemoveMI ( & ( * It ) ) ; } } continue ; } RegSeqInfo R<ISA_LIT> ( * MRI , & MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! areAllUsesSwizzeable ( Reg ) ) continue ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ;


<s> for ( DenseMap < unsigned , unsigned > :: iterator It = ToMerge -> RegToChan . begin ( ) , E = ToMerge -> RegToChan . end ( ) ; It != E ; ++ It ) { DenseMap < unsigned , unsigned > :: const_iterator PosInUntouched = Untouched -> RegToChan . find ( ( * It ) . first ) ; if ( PosInUntouched != Untouched -> RegToChan . end ( ) ) { Remap . push_back ( std :: pair < unsigned , unsigned > ( ( * It ) . second , ( * PosInUntouched ) . second ) ) ; continue ;


<s> unsigned CurrentUndexIdx = <NUM_LIT> ; for ( DenseMap < unsigned , unsigned > :: iterator It = ToMerge -> RegToChan . begin ( ) , E = ToMerge -> RegToChan . end ( ) ; It != E ; ++ It ) { DenseMap < unsigned , unsigned > :: const_iterator PosInUntouched = Untouched -> RegToChan . find ( ( * It ) . first ) ; if ( PosInUntouched != Untouched -> RegToChan . end ( ) ) { Remap . push_back ( std :: pair < unsigned , unsigned > ( ( * It ) . second , ( * PosInUntouched ) . second ) ) ; continue ; } if ( CurrentUndexIdx >= Untouched -> UndefReg . size ( ) ) return false ; Remap . push_back ( std :: pair < unsigned , unsigned > ( ( * It ) . second , Untouched -> UndefReg [ CurrentUndexIdx ++ ] ) ) ;


<s> static unsigned getReassignedChan ( const std :: vector < std :: pair < unsigned , unsigned >> & RemapChan , unsigned Chan ) { for ( const auto & J : RemapChan ) { if ( J . first == Chan ) return J . second ; } llvm_unreachable ( <STR_LIT> ) ;


<s> MachineInstr * <ISA_LIT>VectorRegMerger :: RebuildVector ( RegSeqInfo * R<ISA_LIT> , const RegSeqInfo * BaseR<ISA_LIT> , const std :: vector < std :: pair < unsigned , unsigned >> & RemapChan ) const { Register Reg = R<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: iterator Pos = R<ISA_LIT> -> Instr ; MachineBasicBlock & MBB = * Pos -> getParent ( ) ; DebugLoc DL = Pos -> getDebugLoc ( ) ; Register SrcVec = BaseR<ISA_LIT> -> Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; DenseMap < Register , unsigned > UpdatedRegToChan = BaseR<ISA_LIT> -> RegToChan ; std :: vector < Register > UpdatedUndef = BaseR<ISA_LIT> -> UndefReg ; for ( const auto & It : R<ISA_LIT> -> RegToChan ) { Register DstReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned SubReg = It . first ; unsigned Swizzle = It . second ; unsigned Chan = getReassignedChan ( RemapChan , Swizzle ) ; MachineInstr * Tmp = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( SrcVec ) . addReg ( SubReg ) . addImm ( Chan ) ; UpdatedRegToChan [ SubReg ] = Chan ; std :: vector < Register > :: iterator ChanPos = llvm :: find ( UpdatedUndef , Chan ) ; if ( ChanPos != UpdatedUndef . end ( ) ) UpdatedUndef . erase ( ChanPos ) ; assert ( ! is_contained ( UpdatedUndef , Chan ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Tmp -> dump ( ) ; ) ; ( void ) Tmp ; SrcVec = DstReg ; } MachineInstr * NewMI = BuildMI ( MBB , Pos , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( SrcVec ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; NewMI -> dump ( ) ; ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( MachineRegisterInfo :: use_instr_iterator It = MRI -> use_instr_begin ( Reg ) , E = MRI -> use_instr_end ( ) ; It != E ; ++ It ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ( * It ) . dump ( ) ; dbgs ( ) << <STR_LIT> ) ;


<s> std :: vector < MachineInstr * > & MIs = It . second ; MIs . erase ( llvm :: find ( MIs , MI ) , MIs . end ( ) ) ; }


<s> LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; } ) ; RegSeqInfo CandidateR<ISA_LIT> ; std :: vector < std :: pair < unsigned , unsigned >> RemapChan ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; if ( tryMergeUsingCommonSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) { RemoveMI ( CandidateR<ISA_LIT> . Instr ) ; MII = RebuildVector ( & R<ISA_LIT> , & CandidateR<ISA_LIT> , RemapChan ) ; trackR<ISA_LIT> ( R<ISA_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; RemapChan . clear ( ) ; if ( tryMergeUsingFreeSlot ( R<ISA_LIT> , CandidateR<ISA_LIT> , RemapChan ) ) {


<s> unsigned Offset ; if ( TII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) Offset = <NUM_LIT> ; else Offset = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) { unsigned Swizzle = MI . getOperand ( i + Offset ) . getImm ( ) + <NUM_LIT> ; for ( const auto & J : RemapChan ) { if ( J . first == Swizzle ) {

MI . getOperand ( i + Offset ) . setImm ( J . second - <NUM_LIT> ) ;


<s> if ( PosInUntouched != Untouched -> RegToChan . end ( ) ) { Remap . push_back ( std :: pair < unsigned , unsigned > ( It . second , ( * PosInUntouched ) . second ) ) ; continue ; } if ( CurrentUndexIdx >= Untouched -> UndefReg . size ( ) ) return false ; Remap . push_back ( std :: pair < unsigned , unsigned > ( It . second , Untouched -> UndefReg [ CurrentUndexIdx ++ ] ) ) ; }