<s> MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) { MIRBuilder . buildCopy ( ( Register ) ArgReg . first , ArgReg . second ) ; MIB . addReg ( ArgReg . first , RegState :: Implicit ) ; } unsigned NumBytes = CCInfo . getNextStackOffset ( ) ; if ( MIB -> getOperand ( <NUM_LIT> ) . isReg ( ) ) { MIB -> getOperand ( <NUM_LIT> ) . setReg ( constrainOperandRegClass ( MF , * TRI , MRI , * ST . getInstrInfo ( ) , * ST . getRegBankInfo ( ) , * MIB , MIB -> getDesc ( ) , MIB -> getOperand ( <NUM_LIT> ) , <NUM_LIT> ) ) ; } auto OrigInsertPt = MIRBuilder . getInsertPt ( ) ; MIRBuilder . insertInstr ( MIB ) ; MachineInstrBuilder CallSeqEnd = MIRBuilder . buildInstr ( <STR_LIT> ) ; SmallVector < ArgInfo , <NUM_LIT> > InArgs ; if ( ! Info . CanLowerReturn ) { insertSRetLoads ( MIRBuilder , Info . OrigRet . Ty , Info . OrigRet . Regs , Info . DemoteRegister , Info . DemoteStackIndex ) ; } else if ( ! Info . OrigRet . Ty -> isVoidTy ( ) ) { SmallVector < ArgInfo , <NUM_LIT> > PreSplitRetInfos ; splitToValueTypes ( MIRBuilder , Info . OrigRet , PreSplitRetInfos , DL , Info . CallConv ) ; processSplitArgs ( MIRBuilder , Info . OrigRet , PreSplitRetInfos , InArgs , DL , Info . CallConv , false , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { assert ( DstReg == Info . OrigRet . Regs [ VTSplitIdx ] ) ; packSplitRegsToOrigType ( MIRBuilder , Info . OrigRet . Regs [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; } MIRBuilder . setInsertPt ( MIRBuilder . getMBB ( ) , CallSeqEnd ) ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) { CCAssignFn * RetAssignFn = TLI . CCAssignFnForReturn ( Info . CallConv , Info . IsVarArg ) ; CallReturnHandler Handler ( MIRBuilder , MRI , MIB , RetAssignFn ) ;

if ( ! handleAssignments ( MIRBuilder , InArgs , Handler ) ) return false ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , AssignFnFixed , AssignFnVarArg , false ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , MIRBuilder , OutArgs , Handler ) ) return false ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! ST . enableFlatScratch ( ) ) { auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) , MFI -> getScratchRSrcReg ( ) ) ; MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) { MIRBuilder . buildCopy ( ( Register ) ArgReg . first , ArgReg . second ) ; MIB . addReg ( ArgReg . first , RegState :: Implicit ) ;


<s> uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { const bool IsByRef = Arg . hasByRefAttr ( ) ; Type * ArgTy = IsByRef ? Arg . getParamByRefType ( ) : Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; MaybeAlign ABIAlign = IsByRef ? Arg . getParamAlign ( ) : None ; if ( ! ABIAlign ) ABIAlign = DL . getABITypeAlign ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; if ( Arg . use_empty ( ) ) { ++ i ; continue ; } Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; if ( IsByRef ) { unsigned ByRefAS = cast < PointerType > ( Arg . getType ( ) ) -> getAddressSpace ( ) ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; if ( ByRefAS == <ISA_LIT>AS :: CONSTANT_ADDRESS ) { lowerParameterPtr ( VRegs [ i ] [ <NUM_LIT> ] , B , ArgTy , ArgOffset ) ; } else { const LLT ConstPtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register PtrReg = MRI . createGenericVirtualRegister ( ConstPtrTy ) ; lowerParameterPtr ( PtrReg , B , ArgTy , ArgOffset ) ; B . buildAddrSpaceCast ( VRegs [ i ] [ <NUM_LIT> ] , PtrReg ) ; } } else { ArrayRef < Register > OrigArgRegs = VRegs [ i ] ;

Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ;


<s> Register PtrReg = B . getMRI ( ) -> createGenericVirtualRegister ( PtrTy ) ; lowerParameterPtr ( PtrReg , B , ParamTy , Offset ) ;

MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Alignment ) ;


<s> unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register PtrReg = B . getMRI ( ) -> createGenericVirtualRegister ( PtrTy ) ; lowerParameterPtr ( PtrReg , B , ParamTy , Offset ) ;


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; auto OffsetReg = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) ;

B . buildPtrAdd ( DstReg , KernArgSegmentVReg , OffsetReg ) ;


<s> Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ;

auto OffsetReg = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) ;


<s> SmallVector < ArgInfo , <NUM_LIT> > PreSplitRetInfos ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; for ( unsigned i = <NUM_LIT> ; i < SplitEVTs . size ( ) ; ++ i ) { EVT VT = SplitEVTs [ i ] ; Register Reg = VRegs [ i ] ; ArgInfo RetInfo ( Reg , VT . getTypeForEVT ( Ctx ) ) ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; if ( VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( RetInfo . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( RetInfo . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { RetInfo . Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLT ExtTy = getLLTForType ( * RetInfo . Ty , DL ) ;

Reg = B . buildInstr ( ExtendOp , { ExtTy } , { Reg } ) . getReg ( <NUM_LIT> ) ;


<s> SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; for ( unsigned i = <NUM_LIT> ; i < SplitEVTs . size ( ) ; ++ i ) { EVT VT = SplitEVTs [ i ] ; Register Reg = VRegs [ i ] ; ArgInfo RetInfo ( Reg , VT . getTypeForEVT ( Ctx ) ) ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; if ( VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( RetInfo . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( RetInfo . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { RetInfo . Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLT ExtTy = getLLTForType ( * RetInfo . Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { ExtTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } if ( Reg != RetInfo . Regs [ <NUM_LIT> ] ) { RetInfo . Regs [ <NUM_LIT> ] = Reg ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; } splitToValueTypes ( B , RetInfo , PreSplitRetInfos , DL , CC ) ; processSplitArgs ( B , RetInfo , PreSplitRetInfos , SplitRetInfos , DL , CC , true , [ & ] ( ArrayRef < Register > Regs , Register SrcReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( B , Regs , SrcReg , PreSplitRetInfos [ VTSplitIdx ] , LLTy , PartLLT ) ; } ) ; PreSplitRetInfos . clear ( ) ; } CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ;

<ISA_LIT>OutgoingValueHandler RetHandler ( B , * MRI , Ret , AssignFn ) ;


<s> LLVMContext & Ctx = OrigArg . Ty -> getContext ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; for ( int SplitIdx = <NUM_LIT> , e = SplitArg . size ( ) ; SplitIdx != e ; ++ SplitIdx ) { const ArgInfo & CurSplitArg = SplitArg [ SplitIdx ] ; Register Reg = OrigArg . Regs [ SplitIdx ] ; EVT VT = EVT :: getEVT ( CurSplitArg . Ty ) ; LLT LLTy = getLLTForType ( * CurSplitArg . Ty , DL ) ; unsigned NumParts = TLI . getNumRegistersForCallingConv ( Ctx , CallConv , VT ) ; MVT RegVT = TLI . getRegisterTypeForCallingConv ( Ctx , CallConv , VT ) ; if ( NumParts == <NUM_LIT> ) { SplitArgs . emplace_back ( Reg , CurSplitArg . Ty , OrigArg . Flags , OrigArg . IsFixed ) ; continue ; } SmallVector < Register , <NUM_LIT> > SplitRegs ; Type * PartTy = EVT ( RegVT ) . getTypeForEVT ( Ctx ) ; LLT PartLLT = getLLTForType ( * PartTy , DL ) ;

MachineRegisterInfo & MRI = * B . getMRI ( ) ;


SplitArgs . back ( ) . Flags [ <NUM_LIT> ] . setInConsecutiveRegsLast ( ) ;


<s> LLT DstTy = MRI . getType ( DstRegs [ <NUM_LIT> ] ) ; LLT LCMTy = getLCMType ( SrcTy , PartTy ) ; const unsigned LCMSize = LCMTy . getSizeInBits ( ) ; const unsigned DstSize = DstTy . getSizeInBits ( ) ; const unsigned SrcSize = SrcTy . getSizeInBits ( ) ; Register UnmergeSrc = SrcReg ; if ( LCMSize != SrcSize ) { Register Undef = B . buildUndef ( SrcTy ) . getReg ( <NUM_LIT> ) ; SmallVector < Register , <NUM_LIT> > MergeParts ( <NUM_LIT> , SrcReg ) ; for ( unsigned Size = SrcSize ; Size != LCMSize ; Size += SrcSize ) MergeParts . push_back ( Undef ) ;

UnmergeSrc = B . buildMerge ( LCMTy , MergeParts ) . getReg ( <NUM_LIT> ) ;


<s> const LLT RegTy = MRI . getType ( ValVReg ) ; MemSize = std :: min ( MemSize , ( uint64_t ) RegTy . getSizeInBytes ( ) ) ; assignValueToAddress ( ValVReg , Addr , MemSize , MPO , VA ) ;


<s> MIB . addUse ( PhysReg , RegState :: Implicit ) ; Register ExtReg = extendRegisterMin32 ( * this , ValVReg , VA ) ;

MIRBuilder . buildCopy ( PhysReg , ExtReg ) ;


<s> void <ISA_LIT>CallLowering :: handleImplicitCallArguments ( MachineIRBuilder & MIRBuilder , MachineInstrBuilder & CallInst , const GCNSubtarget & ST , const <ISA_LIT>MachineFunctionInfo & FuncInfo , ArrayRef < std :: pair < MCRegister , Register >> ImplicitArgRegs ) const { if ( ! ST . enableFlatScratch ( ) ) { auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) , FuncInfo . getScratchRSrcReg ( ) ) ;

MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ;


<s> auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) , FuncInfo . getScratchRSrcReg ( ) ) ; MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; CallInst . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) { MIRBuilder . buildCopy ( ( Register ) ArgReg . first , ArgReg . second ) ; CallInst . addReg ( ArgReg . first , RegState :: Implicit ) ;


<s> MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , false ) ; if ( ! handleAssignments ( Handler , OutArgs , CCInfo , ArgLocs , MIRBuilder ) ) return false ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; handleImplicitCallArguments ( MIRBuilder , MIB , ST , * MFI , ImplicitArgRegs ) ; unsigned NumBytes = CCInfo . getNextStackOffset ( ) ; if ( MIB -> getOperand ( <NUM_LIT> ) . isReg ( ) ) { MIB -> getOperand ( <NUM_LIT> ) . setReg ( constrainOperandRegClass ( MF , * TRI , MRI , * ST . getInstrInfo ( ) , * ST . getRegBankInfo ( ) , * MIB , MIB -> getDesc ( ) , MIB -> getOperand ( <NUM_LIT> ) , <NUM_LIT> ) ) ; } MIRBuilder . insertInstr ( MIB ) ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) {

CCAssignFn * RetAssignFn = TLI . CCAssignFnForReturn ( Info . CallConv , Info . IsVarArg ) ;


<s> CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; allocateHSAUserSGPRs ( CCInfo , B , MF , * TRI , * Info ) ; unsigned i = <NUM_LIT> ; const Align KernArgBaseAlign ( <NUM_LIT> ) ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { const bool IsByRef = Arg . hasByRefAttr ( ) ; Type * ArgTy = IsByRef ? Arg . getParamByRefType ( ) : Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; MaybeAlign ABIAlign = IsByRef ? Arg . getParamAlign ( ) : None ; if ( ! ABIAlign ) ABIAlign = DL . getABITypeAlign ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; if ( Arg . use_empty ( ) ) { ++ i ; continue ; } Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; if ( IsByRef ) { unsigned ByRefAS = cast < PointerType > ( Arg . getType ( ) ) -> getAddressSpace ( ) ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; if ( ByRefAS == <ISA_LIT>AS :: CONSTANT_ADDRESS ) { lowerParameterPtr ( VRegs [ i ] [ <NUM_LIT> ] , B , ArgTy , ArgOffset ) ; } else { const LLT ConstPtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register PtrReg = MRI . createGenericVirtualRegister ( ConstPtrTy ) ; lowerParameterPtr ( PtrReg , B , ArgTy , ArgOffset ) ; B . buildAddrSpaceCast ( VRegs [ i ] [ <NUM_LIT> ] , PtrReg ) ; } } else {

ArrayRef < Register > OrigArgRegs = VRegs [ i ] ;


<s> assert ( VRegs . size ( ) == SplitEVTs . size ( ) && <STR_LIT> ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; for ( unsigned i = <NUM_LIT> ; i < SplitEVTs . size ( ) ; ++ i ) { EVT VT = SplitEVTs [ i ] ; Register Reg = VRegs [ i ] ; ArgInfo RetInfo ( Reg , VT . getTypeForEVT ( Ctx ) ) ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; if ( VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( RetInfo . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( RetInfo . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { RetInfo . Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLT ExtTy = getLLTForType ( * RetInfo . Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { ExtTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } if ( Reg != RetInfo . Regs [ <NUM_LIT> ] ) { RetInfo . Regs [ <NUM_LIT> ] = Reg ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; } splitToValueTypes ( RetInfo , SplitRetInfos , DL , CC ) ; } CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; OutgoingValueAssigner Assigner ( AssignFn ) ;

<ISA_LIT>OutgoingValueHandler RetHandler ( B , * MRI , Ret ) ;


<s> continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; setArgFlags ( OrigArg , Idx + AttributeList :: FirstArgIndex , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { packSplitRegsToOrigType ( B , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; FormalArgHandler Handler ( B , MRI , AssignFn ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , B , SplitArgs , Handler ) ) return false ; if ( ! IsEntryFunc ) { TLI . allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsShader ) ; } else { CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; CCInfo . AllocateReg ( Info -> getScratchWaveOffsetReg ( ) ) ; CCInfo . AllocateReg ( Info -> getFrameOffsetReg ( ) ) ; TLI . allocateSpecialInputSGPRs ( CCInfo , MF , * TRI , * Info ) ; }


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; allocateHSAUserSGPRs ( CCInfo , B , MF , * TRI , * Info ) ; unsigned i = <NUM_LIT> ; const unsigned KernArgBaseAlign = <NUM_LIT> ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; unsigned ABIAlign = DL . getABITypeAlignment ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ;

unsigned Align = MinAlign ( KernArgBaseAlign , ArgOffset ) ;


<s> void <ISA_LIT>CallLowering :: lowerParameter ( MachineIRBuilder & B , Type * ParamTy , uint64_t Offset , unsigned Align , Register DstReg ) const { MachineFunction & MF = B . getMF ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Align ) ;

B . buildLoad ( DstReg , PtrReg , * MMO ) ;


<s> const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Align ) ;


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ;

PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ;


<s> PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; Register DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ;

Register OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ;


<s> B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } auto const & ST = B . getMF ( ) . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! lowerReturnVal ( B , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LiveInReturn = MF . addLiveIn ( TRI -> getReturnAddressReg ( MF ) , & <STR_LIT> ) ; B . buildCopy ( ReturnAddrVReg , LiveInReturn ) ;


<s> assert ( ! Val == VRegs . empty ( ) && <STR_LIT> ) ; CallingConv :: ID CC = B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ; const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } auto const & ST = B . getMF ( ) . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! lowerReturnVal ( B , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LiveInReturn = MF . addLiveIn ( TRI -> getReturnAddressReg ( MF ) , & <STR_LIT> ) ;

B . buildCopy ( ReturnAddrVReg , LiveInReturn ) ;


<s> CallingConv :: ID CC = F . getCallingConv ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; ArgInfo OrigRetInfo ( VRegs , Val -> getType ( ) ) ; setArgFlags ( OrigRetInfo , AttributeList :: ReturnIndex , DL , F ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ;

splitToValueTypes ( OrigRetInfo , SplitRetInfos , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) {


<s> return ; } if ( LLTy . isVector ( ) && PartLLT . isVector ( ) ) { assert ( LLTy . getElementType ( ) == PartLLT . getElementType ( ) ) ; int DstElts = LLTy . getNumElements ( ) ; int PartElts = PartLLT . getNumElements ( ) ; if ( DstElts % PartElts == <NUM_LIT> ) B . buildConcatVectors ( OrigRegs [ <NUM_LIT> ] , Regs ) ; else { assert ( PartElts == <NUM_LIT> && DstElts % <NUM_LIT> != <NUM_LIT> ) ; int RoundedElts = PartElts * ( ( DstElts + PartElts - <NUM_LIT> ) / PartElts ) ; LLT RoundedDestTy = LLT :: vector ( RoundedElts , PartLLT . getElementType ( ) ) ; auto RoundedConcat = B . buildConcatVectors ( RoundedDestTy , Regs ) ; B . buildExtract ( OrigRegs [ <NUM_LIT> ] , RoundedConcat , <NUM_LIT> ) ; } return ; } assert ( LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) ; LLT DstEltTy = LLTy . getElementType ( ) ; if ( DstEltTy == PartLLT ) {

B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , Regs ) ;


<s> } if ( SrcSize % PartSize == <NUM_LIT> ) { B . buildUnmerge ( DstRegs , SrcReg ) ; return ; } const int NumRoundedParts = ( SrcSize + PartSize - <NUM_LIT> ) / PartSize ; LLT BigTy = getMultipleType ( PartTy , NumRoundedParts ) ; auto ImpDef = B . buildUndef ( BigTy ) ; Register BigReg = MRI . createGenericVirtualRegister ( BigTy ) ; B . buildInsert ( BigReg , ImpDef . getReg ( <NUM_LIT> ) , SrcReg , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ;


<s> const unsigned PartSize = PartTy . getSizeInBits ( ) ; if ( SrcTy . isVector ( ) && ! PartTy . isVector ( ) && PartSize > SrcTy . getElementType ( ) . getSizeInBits ( ) ) { auto UnmergeToEltTy = B . buildUnmerge ( SrcTy . getElementType ( ) , SrcReg ) ; for ( int i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i ) B . buildAnyExt ( DstRegs [ i ] , UnmergeToEltTy . getReg ( i ) ) ; return ; } if ( SrcSize % PartSize == <NUM_LIT> ) { B . buildUnmerge ( DstRegs , SrcReg ) ; return ; } const int NumRoundedParts = ( SrcSize + PartSize - <NUM_LIT> ) / PartSize ; LLT BigTy = getMultipleType ( PartTy , NumRoundedParts ) ; auto ImpDef = B . buildUndef ( BigTy ) ; Register BigReg = MRI . createGenericVirtualRegister ( BigTy ) ; B . buildInsert ( BigReg , ImpDef . getReg ( <NUM_LIT> ) , SrcReg , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; int64_t Offset = <NUM_LIT> ;

for ( unsigned i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i , Offset += PartSize ) B . buildExtract ( DstRegs [ i ] , BigReg , Offset ) ;


<s> void assignValueToReg ( Register ValVReg , Register PhysReg , CCValAssign & VA ) override {


<s> void assignValueToReg ( Register ValVReg , Register PhysReg , CCValAssign & VA ) override { MIB . addUse ( PhysReg ) ;

MIRBuilder . buildCopy ( PhysReg , ValVReg ) ;


<s> PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy , Offset ) ;

MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MONonTemporal | MachineMemOperand :: MOInvariant , TypeSize , Align ) ;


<s> PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ;

Register PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy , Offset ) ;


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MFI -> setIfReturnsVoid ( ! Val ) ; if ( ! Val ) { MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } Register VReg = VRegs [ <NUM_LIT> ] ; const Function & F = MF . getFunction ( ) ; auto & DL = F . getParent ( ) -> getDataLayout ( ) ; if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; SmallVector < EVT , <NUM_LIT> > SplitVTs ; SmallVector < uint64_t , <NUM_LIT> > Offsets ;

ArgInfo OrigArg { VReg , Val -> getType ( ) } ;


<s> OutgoingArgHandler ( MachineIRBuilder & MIRBuilder , MachineRegisterInfo & MRI , MachineInstrBuilder MIB , CCAssignFn * AssignFn ) : ValueHandler ( MIRBuilder , MRI , AssignFn ) , MIB ( MIB ) {


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = * MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( OffsetReg , Offset ) ;

MIRBuilder . buildGEP ( DstReg , KernArgSegmentVReg , OffsetReg ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = * MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ;

unsigned KernArgSegmentPtr = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ;


<s> void assignValueToAddress ( const CallLowering :: ArgInfo & Arg , Register Addr , uint64_t Size , MachinePointerInfo & MPO , CCValAssign & VA ) override { Register ValVReg = VA . getLocInfo ( ) != CCValAssign :: LocInfo :: FPExt ? extendRegister ( Arg . Regs [ <NUM_LIT> ] , VA ) : Arg . Regs [ <NUM_LIT> ] ; const LLT RegTy = MRI . getType ( ValVReg ) ; if ( RegTy . getSizeInBytes ( ) > Size ) Size = RegTy . getSizeInBytes ( ) ; assignValueToAddress ( ValVReg , Addr , Size , MPO , VA ) ;


<s> Register ValVReg = VA . getLocInfo ( ) != CCValAssign :: LocInfo :: FPExt ? extendRegister ( Arg . Regs [ <NUM_LIT> ] , VA ) : Arg . Regs [ <NUM_LIT> ] ; const LLT RegTy = MRI . getType ( ValVReg ) ; if ( RegTy . getSizeInBytes ( ) > Size ) Size = RegTy . getSizeInBytes ( ) ;

assignValueToAddress ( ValVReg , Addr , Size , MPO , VA ) ;


<s> for ( int I = <NUM_LIT> , E = VRegs [ Idx ] . size ( ) ; I != E ; ++ I ) B . buildUndef ( VRegs [ Idx ] [ I ] ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( B , OrigArg , SplitArgs , DL , CC , false , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { assert ( DstReg == VRegs [ Idx ] [ VTSplitIdx ] ) ; packSplitRegsToOrigType ( B , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; if ( ! IsEntryFunc ) { if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) TLI . allocateSpecialInputVGPRsFixed ( CCInfo , MF , * TRI , * Info ) ; } FormalArgHandler Handler ( B , MRI , AssignFn ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , B , SplitArgs , Handler ) ) return false ; if ( ! IsEntryFunc && ! <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) { TLI . allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsEntryFunc ) {

TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsGraphics ) ;


<s> ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( B , OrigArg , SplitArgs , DL , CC , false , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { assert ( DstReg == VRegs [ Idx ] [ VTSplitIdx ] ) ; packSplitRegsToOrigType ( B , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; if ( ! IsEntryFunc ) { if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) TLI . allocateSpecialInputVGPRsFixed ( CCInfo , MF , * TRI , * Info ) ; } FormalArgHandler Handler ( B , MRI , AssignFn ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , B , SplitArgs , Handler ) ) return false ; if ( ! IsEntryFunc && ! <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) { TLI . allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsGraphics ) ; } else { if ( ! Subtarget . enableFlatScratch ( ) ) CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; TLI . allocateSpecialInputSGPRs ( CCInfo , MF , * TRI , * Info ) ; } B . setMBB ( MBB ) ; return true ;


<s> setArgFlags ( OrigRetInfo , AttributeList :: ReturnIndex , DL , F ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; splitToValueTypes ( B , OrigRetInfo , SplitRetInfos , DL , CC , true , [ & ] ( ArrayRef < Register > Regs , Register SrcReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( B , Regs , SrcReg , SplitRetInfos [ VTSplitIdx ] , LLTy , PartLLT ) ; } ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ;

<ISA_LIT>OutgoingValueHandler RetHandler ( B , * MRI , Ret , AssignFn ) ;


<s> assert ( OrigArg . Regs . size ( ) == SplitVTs . size ( ) ) ; int SplitIdx = <NUM_LIT> ; for ( EVT VT : SplitVTs ) { Register Reg = OrigArg . Regs [ SplitIdx ] ; Type * Ty = VT . getTypeForEVT ( Ctx ) ; LLT LLTy = getLLTForType ( * Ty , DL ) ; if ( IsOutgoing && VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( OrigArg . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( OrigArg . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT . getSizeInBits ( ) != VT . getSizeInBits ( ) ) { VT = ExtVT ; Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLTy = getLLTForType ( * Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { LLTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } unsigned NumParts = TLI . getNumRegistersForCallingConv ( Ctx , CallConv , VT ) ; MVT RegVT = TLI . getRegisterTypeForCallingConv ( Ctx , CallConv , VT ) ; if ( NumParts == <NUM_LIT> ) { SplitArgs . emplace_back ( Reg , Ty , OrigArg . Flags , OrigArg . IsFixed ) ; ++ SplitIdx ; continue ; } SmallVector < Register , <NUM_LIT> > SplitRegs ; Type * PartTy = EVT ( RegVT ) . getTypeForEVT ( Ctx ) ; LLT PartLLT = getLLTForType ( * PartTy , DL ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) { Register PartReg = MRI . createGenericVirtualRegister ( PartLLT ) ; SplitRegs . push_back ( PartReg ) ;

SplitArgs . emplace_back ( ArrayRef < Register > ( PartReg ) , PartTy , OrigArg . Flags ) ;


<s> if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( Register R : VRegs [ Idx ] ) B . buildUndef ( R ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg , Idx ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , CC ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ;

if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ;


<s> const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsGraphics && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( Register R : VRegs [ Idx ] ) B . buildUndef ( R ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg , Idx ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , CC ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; if ( ! IsEntryFunc && ! IsGraphics ) { TLI . allocateSpecialInputVGPRsFixed ( CCInfo , MF , * TRI , * Info ) ; } IncomingValueAssigner Assigner ( AssignFn ) ; if ( ! determineAssignments ( Assigner , SplitArgs , CCInfo ) ) return false ;


<s> for ( auto & Arg : F . args ( ) ) { const bool IsByRef = Arg . hasByRefAttr ( ) ; Type * ArgTy = IsByRef ? Arg . getParamByRefType ( ) : Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; MaybeAlign ABIAlign = IsByRef ? Arg . getParamAlign ( ) : None ; if ( ! ABIAlign ) ABIAlign = DL . getABITypeAlign ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; if ( Arg . use_empty ( ) ) { ++ i ; continue ; } Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; if ( IsByRef ) { unsigned ByRefAS = cast < PointerType > ( Arg . getType ( ) ) -> getAddressSpace ( ) ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; if ( ByRefAS == <ISA_LIT>AS :: CONSTANT_ADDRESS ) { lowerParameterPtr ( VRegs [ i ] [ <NUM_LIT> ] , B , ArgOffset ) ; } else { const LLT ConstPtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register PtrReg = MRI . createGenericVirtualRegister ( ConstPtrTy ) ; lowerParameterPtr ( PtrReg , B , ArgOffset ) ;

B . buildAddrSpaceCast ( VRegs [ i ] [ <NUM_LIT> ] , PtrReg ) ;


<s> MachineFunction & MF = B . getMF ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( B , ParamTy , Offset ) ;

MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Align ) ;


<s> unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Align ) ;

B . buildLoad ( DstReg , PtrReg , * MMO ) ;


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; Register DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; Register OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildConstant ( OffsetReg , Offset ) ; B . buildPtrAdd ( DstReg , KernArgSegmentVReg , OffsetReg ) ;


<s> auto const & ST = MIRBuilder . getMF ( ) . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = MIRBuilder . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! lowerReturnVal ( MIRBuilder , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LiveInReturn = MF . addLiveIn ( TRI -> getReturnAddressReg ( MF ) , & <STR_LIT> ) ; MIRBuilder . buildCopy ( ReturnAddrVReg , LiveInReturn ) ; } MIRBuilder . insertInstr ( Ret ) ;


<s> auto MMO = MF . getMachineMemOperand ( MPO , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant , Size , Align ) ; MIRBuilder . buildLoad ( ValVReg , Addr , * MMO ) ;


<s> unsigned Align = inferAlignmentFromPtrInfo ( MF , MPO ) ; auto MMO = MF . getMachineMemOperand ( MPO , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant , Size , Align ) ;

MIRBuilder . buildLoad ( ValVReg , Addr , * MMO ) ;


<s> assert ( OrigArg . Regs . size ( ) == SplitVTs . size ( ) ) ; int SplitIdx = <NUM_LIT> ; for ( EVT VT : SplitVTs ) { Register Reg = OrigArg . Regs [ SplitIdx ] ; Type * Ty = VT . getTypeForEVT ( Ctx ) ; LLT LLTy = getLLTForType ( * Ty , DL ) ; if ( OrigArgIdx == AttributeList :: ReturnIndex && VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( OrigArg . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( OrigArg . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { VT = ExtVT ; Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLTy = getLLTForType ( * Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { LLTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } unsigned NumParts = TLI . getNumRegistersForCallingConv ( Ctx , CallConv , VT ) ; MVT RegVT = TLI . getRegisterTypeForCallingConv ( Ctx , CallConv , VT ) ; if ( NumParts == <NUM_LIT> ) { SplitArgs . emplace_back ( Reg , Ty , OrigArg . Flags , OrigArg . IsFixed ) ; ++ SplitIdx ; continue ; } SmallVector < Register , <NUM_LIT> > SplitRegs ; Type * PartTy = EVT ( RegVT ) . getTypeForEVT ( Ctx ) ; LLT PartLLT = getLLTForType ( * PartTy , DL ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ;

for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) {


<s> CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) MIRBuilder . setInstr ( * MBB . begin ( ) ) ; FormalArgHandler Handler ( MIRBuilder , MRI , AssignFn ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , MIRBuilder , SplitArgs , Handler ) ) return false ; if ( ! IsEntryFunc ) { TLI . allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsShader ) ; } else { CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; CCInfo . AllocateReg ( Info -> getScratchWaveOffsetReg ( ) ) ; CCInfo . AllocateReg ( Info -> getFrameOffsetReg ( ) ) ; TLI . allocateSpecialInputSGPRs ( CCInfo , MF , * TRI , * Info ) ; } MIRBuilder . setMBB ( MBB ) ; return true ;


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; CallingConv :: ID CallConv = F . getCallingConv ( ) ; if ( ! <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && CallConv != CallingConv :: <ISA_LIT>_Gfx ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( <STR_LIT> ( CallConv ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } SmallVector < ArgInfo , <NUM_LIT> > OutArgs ; for ( auto & OrigArg : Info . OrigArgs ) splitToValueTypes ( OrigArg , OutArgs , DL , Info . CallConv ) ; SmallVector < ArgInfo , <NUM_LIT> > InArgs ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) splitToValueTypes ( Info . OrigRet , InArgs , DL , Info . CallConv ) ; bool CanTailCallOpt = isEligibleForTailCallOptimization ( MIRBuilder , Info , InArgs , OutArgs ) ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( CanTailCallOpt ) return lowerTailCall ( MIRBuilder , Info , OutArgs ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; }

SmallVector < Register , <NUM_LIT> > PhysRegs ;


<s> Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && ! OrigArg . Flags . isInReg ( ) && ! OrigArg . Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { if ( CurOrigArg -> use_empty ( ) && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ) { Skipped . set ( i ) ; ++ P<ISA_LIT>nputNum ; continue ; } Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! CurOrigArg -> use_empty ( ) ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; ++ P<ISA_LIT>nputNum ; } CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; if ( ValEVT . isVector ( ) ) { EVT ElemVT = ValEVT . getVectorElementType ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ElemVT . getSimpleVT ( ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( ! Res ) return false ; } else { MVT ValVT = ValEVT . getSimpleVT ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS || F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ) { for ( unsigned i = <NUM_LIT> , OrigArgIdx = <NUM_LIT> ; OrigArgIdx != NumArgs && i != ArgLocs . size ( ) ; ++ Arg , ++ OrigArgIdx ) { if ( Skipped . test ( OrigArgIdx ) ) continue ; CCValAssign & VA = ArgLocs [ i ++ ] ; MRI . addLiveIn ( VA . getLocReg ( ) , VRegs [ OrigArgIdx ] ) ; MIRBuilder . getMBB ( ) . addLiveIn ( VA . getLocReg ( ) ) ;

MIRBuilder . buildCopy ( VRegs [ OrigArgIdx ] , VA . getLocReg ( ) ) ;


<s> B . buildMerge ( OrigRegs [ <NUM_LIT> ] , Regs ) ; return ; } if ( LLTy . isVector ( ) && PartLLT . isVector ( ) ) { assert ( LLTy . getElementType ( ) == PartLLT . getElementType ( ) ) ; int DstElts = LLTy . getNumElements ( ) ; int PartElts = PartLLT . getNumElements ( ) ; if ( DstElts % PartElts == <NUM_LIT> ) B . buildConcatVectors ( OrigRegs [ <NUM_LIT> ] , Regs ) ; else { assert ( PartElts == <NUM_LIT> && DstElts % <NUM_LIT> != <NUM_LIT> ) ; int RoundedElts = PartElts * ( ( DstElts + PartElts - <NUM_LIT> ) / PartElts ) ; LLT RoundedDestTy = LLT :: vector ( RoundedElts , PartLLT . getElementType ( ) ) ; auto RoundedConcat = B . buildConcatVectors ( RoundedDestTy , Regs ) ; B . buildExtract ( OrigRegs [ <NUM_LIT> ] , RoundedConcat , <NUM_LIT> ) ; } return ; } MachineRegisterInfo & MRI = * B . getMRI ( ) ; assert ( LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) ; LLT DstEltTy = LLTy . getElementType ( ) ; LLT RealDstEltTy = MRI . getType ( OrigRegs [ <NUM_LIT> ] ) . getElementType ( ) ; assert ( DstEltTy . getSizeInBits ( ) == RealDstEltTy . getSizeInBits ( ) ) ; if ( DstEltTy == PartLLT ) { if ( RealDstEltTy . isPointer ( ) ) { for ( Register Reg : Regs ) MRI . setType ( Reg , RealDstEltTy ) ; } B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , Regs ) ; } else if ( DstEltTy . getSizeInBits ( ) > PartLLT . getSizeInBits ( ) ) { SmallVector < Register , <NUM_LIT> > EltMerges ; int PartsPerElt = DstEltTy . getSizeInBits ( ) / PartLLT . getSizeInBits ( ) ; assert ( DstEltTy . getSizeInBits ( ) % PartLLT . getSizeInBits ( ) == <NUM_LIT> ) ; for ( int I = <NUM_LIT> , NumElts = LLTy . getNumElements ( ) ; I != NumElts ; ++ I ) { auto Merge = B . buildMerge ( RealDstEltTy , Regs . take_front ( PartsPerElt ) ) ; MRI . setType ( Merge . getReg ( <NUM_LIT> ) , RealDstEltTy ) ; EltMerges . push_back ( Merge . getReg ( <NUM_LIT> ) ) ; Regs = Regs . drop_front ( PartsPerElt ) ; } B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , EltMerges ) ;


<s> } if ( SrcSize % PartSize == <NUM_LIT> ) { B . buildUnmerge ( DstRegs , SrcReg ) ; return ; } const int NumRoundedParts = ( SrcSize + PartSize - <NUM_LIT> ) / PartSize ; LLT BigTy = getMultipleType ( PartTy , NumRoundedParts ) ; auto ImpDef = B . buildUndef ( BigTy ) ; auto Big = B . buildInsert ( BigTy , ImpDef . getReg ( <NUM_LIT> ) , SrcReg , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; int64_t Offset = <NUM_LIT> ;

for ( unsigned i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i , Offset += PartSize ) B . buildExtract ( DstRegs [ i ] , Big , Offset ) ;


<s> LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ;

unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ;


<s> } bool CanTailCallOpt = false ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , AssignFnFixed , AssignFnVarArg , false ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , MIRBuilder , OutArgs , Handler ) ) return false ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! ST . enableFlatScratch ( ) ) { auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) , MFI -> getScratchRSrcReg ( ) ) ; MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) { MIRBuilder . buildCopy ( ( Register ) ArgReg . first , ArgReg . second ) ; MIB . addReg ( ArgReg . first , RegState :: Implicit ) ; } unsigned NumBytes = CCInfo . getNextStackOffset ( ) ;

if ( MIB -> getOperand ( <NUM_LIT> ) . isReg ( ) ) {


<s> return true ; } auto const & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! lowerReturnVal ( B , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LiveInReturn = MF . addLiveIn ( TRI -> getReturnAddressReg ( MF ) , & <STR_LIT> ) ; B . buildCopy ( ReturnAddrVReg , LiveInReturn ) ;


<s> void assignValueToAddress ( unsigned ValVReg , unsigned Addr , uint64_t Size , MachinePointerInfo & MPO , CCValAssign & VA ) override { llvm_unreachable ( <STR_LIT> ) ;


<s> MIRBuilder . buildCopy ( PhysReg , ValVReg ) ;


<s> void assignValueToReg ( unsigned ValVReg , unsigned PhysReg , CCValAssign & VA ) override {


<s> static unsigned findFirstFreeSGPR ( CCState & CCInfo ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned Reg = <NUM_LIT> ; Reg < NumSGPRs ; ++ Reg ) { if ( ! CCInfo . isAllocated ( <STR_LIT> + Reg ) ) { return <STR_LIT> + Reg ; } } llvm_unreachable ( <STR_LIT> ) ;


<s> unsigned getStackAddress ( uint64_t Size , int64_t Offset , MachinePointerInfo & MPO ) override {


<s> bool <ISA_LIT>CallLowering :: lowerFormalArguments ( MachineIRBuilder & MIRBuilder , const Function :: ArgumentListType & Args , ArrayRef < unsigned > VRegs ) const {


<s> MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; if ( Info . hasWorkItemIDX ( ) ) { Register Reg = <STR_LIT> ; MRI . setType ( MF . addLiveIn ( Reg , & <STR_LIT> ) , S32 ) ; CCInfo . AllocateReg ( Reg ) ; Info . setWorkItemIDX ( ArgDescriptor :: createRegister ( Reg ) ) ; } if ( Info . hasWorkItemIDY ( ) ) { Register Reg = <STR_LIT> ; MRI . setType ( MF . addLiveIn ( Reg , & <STR_LIT> ) , S32 ) ; CCInfo . AllocateReg ( Reg ) ; Info . setWorkItemIDY ( ArgDescriptor :: createRegister ( Reg ) ) ; } if ( Info . hasWorkItemIDZ ( ) ) { Register Reg = <STR_LIT> ; MRI . setType ( MF . addLiveIn ( Reg , & <STR_LIT> ) , S32 ) ; CCInfo . AllocateReg ( Reg ) ; Info . setWorkItemIDZ ( ArgDescriptor :: createRegister ( Reg ) ) ; }


<s> MachineFunction & MF = MIRBuilder . getMF ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>RegisterInfo * TRI = MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; bool IsShader = <STR_LIT> ( F . getCallingConv ( ) ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; if ( Info -> hasImplicitBufferPtr ( ) ) { unsigned ImplicitBufferPtrReg = Info -> addImplicitBufferPtr ( * TRI ) ; MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; BitVector Skipped ( NumArgs ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; <STR_LIT> Flags ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ArgInfo OrigArg { VRegs [ i ] [ <NUM_LIT> ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && ! OrigArg . Flags . isInReg ( ) && ! OrigArg . Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { if ( CurOrigArg -> use_empty ( ) && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ) { Skipped . set ( i ) ; ++ P<ISA_LIT>nputNum ; continue ; } Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! CurOrigArg -> use_empty ( ) ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; ++ P<ISA_LIT>nputNum ; } CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; if ( ValEVT . isVector ( ) ) { EVT ElemVT = ValEVT . getVectorElementType ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ElemVT . getSimpleVT ( ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( ! Res ) return false ;


<s> uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; unsigned Align = MinAlign ( KernArgBaseAlign , ArgOffset ) ; ArgOffset = alignTo ( ArgOffset , DL . getABITypeAlignment ( ArgTy ) ) ; lowerParameter ( MIRBuilder , ArgTy , ArgOffset , Align , ArgReg ) ;

if ( OrigArgRegs . size ( ) > <NUM_LIT> ) unpackRegs ( OrigArgRegs , ArgReg , ArgTy , MIRBuilder ) ;


<s> bool <ISA_LIT>CallLowering :: lowerFormalArguments ( MachineIRBuilder & MIRBuilder , const Function :: ArgumentListType & Args , const SmallVectorImpl < unsigned > & VRegs ) const {


<s> bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( int I = <NUM_LIT> , E = VRegs [ Idx ] . size ( ) ; I != E ; ++ I ) MIRBuilder . buildUndef ( VRegs [ Idx ] [ I ] ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; setArgFlags ( OrigArg , Idx + AttributeList :: FirstArgIndex , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { packSplitRegsToOrigType ( MIRBuilder , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) MIRBuilder . setInstr ( * MBB . begin ( ) ) ; FormalArgHandler Handler ( MIRBuilder , MRI , AssignFn ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , MIRBuilder , SplitArgs , Handler ) ) return false ; if ( ! IsEntryFunc ) { TLI . allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsShader ) ; } else { CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; CCInfo . AllocateReg ( Info -> getScratchWaveOffsetReg ( ) ) ;

CCInfo . AllocateReg ( Info -> getFrameOffsetReg ( ) ) ;


<s> MachineFunction & MF = B . getMF ( ) ; MachineBasicBlock & MBB = B . getMBB ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget . getRegisterInfo ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CC , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; if ( ! IsEntryFunc ) { Register ReturnAddrReg = TRI -> getReturnAddressReg ( MF ) ; Register LiveInReturn = MF . addLiveIn ( ReturnAddrReg , & <STR_LIT> ) ; MBB . addLiveIn ( ReturnAddrReg ) ; B . buildCopy ( LiveInReturn , ReturnAddrReg ) ; } if ( Info -> hasImplicitBufferPtr ( ) ) { Register ImplicitBufferPtrReg = Info -> addImplicitBufferPtr ( * TRI ) ; MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; unsigned Idx = <NUM_LIT> ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsShader && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ;


<s> MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; unsigned Idx = <NUM_LIT> ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsShader && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( int I = <NUM_LIT> , E = VRegs [ Idx ] . size ( ) ; I != E ; ++ I ) B . buildUndef ( VRegs [ Idx ] [ I ] ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ;

splitToValueTypes ( B , OrigArg , OrigArgIdx , SplitArgs , DL , CC , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) {


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; allocateHSAUserSGPRs ( CCInfo , B , MF , * TRI , * Info ) ; unsigned i = <NUM_LIT> ; const Align KernArgBaseAlign ( <NUM_LIT> ) ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; Align ABIAlign = DL . getABITypeAlign ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; if ( Arg . use_empty ( ) ) { ++ i ;


<s> if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) splitToValueTypes ( Info . OrigRet , InArgs , DL , Info . CallConv ) ; bool CanTailCallOpt = isEligibleForTailCallOptimization ( MIRBuilder , Info , InArgs , OutArgs ) ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( CanTailCallOpt ) return lowerTailCall ( MIRBuilder , Info , OutArgs ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , false ) ; if ( ! handleAssignments ( Handler , OutArgs , CCInfo , ArgLocs , MIRBuilder ) ) return false ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; handleImplicitCallArguments ( MIRBuilder , MIB , ST , * MFI , ImplicitArgRegs ) ; unsigned NumBytes = CCInfo . getNextStackOffset ( ) ;


<s> if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; if ( ! IsEntryFunc && ! IsGraphics ) { TLI . allocateSpecialInputVGPRsFixed ( CCInfo , MF , * TRI , * Info ) ; } IncomingValueAssigner Assigner ( AssignFn ) ; if ( ! determineAssignments ( Assigner , SplitArgs , CCInfo ) ) return false ; FormalArgHandler Handler ( B , MRI ) ; if ( ! handleAssignments ( Handler , SplitArgs , CCInfo , ArgLocs , B ) ) return false ; uint64_t StackOffset = Assigner . StackOffset ; if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsGraphics ) ; } else { if ( ! Subtarget . enableFlatScratch ( ) ) CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; TLI . allocateSpecialInputSGPRs ( CCInfo , MF , * TRI , * Info ) ;


<s> } else if ( InputID == <STR_LIT> ) { LI -> getImplicitArgPtr ( InputReg , MRI , MIRBuilder ) ; } else { MIRBuilder . buildUndef ( InputReg ) ; } if ( OutgoingArg -> isRegister ( ) ) { ArgRegs . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; if ( ! CCInfo . AllocateReg ( OutgoingArg -> getRegister ( ) ) ) report_fatal_error ( <STR_LIT> ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; LLT ArgTy ; std :: tie ( OutgoingArg , ArgRC , ArgTy ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , ArgTy ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , ArgTy ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return false ; auto WorkitemIDX = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; auto WorkitemIDY = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; auto WorkitemIDZ = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; const ArgDescriptor * IncomingArgX = std :: get < <NUM_LIT> > ( WorkitemIDX ) ; const ArgDescriptor * IncomingArgY = std :: get < <NUM_LIT> > ( WorkitemIDY ) ; const ArgDescriptor * IncomingArgZ = std :: get < <NUM_LIT> > ( WorkitemIDZ ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const bool NeedWorkItemIDX = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDY = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDZ = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; Register InputReg ; if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX && NeedWorkItemIDX ) { InputReg = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( InputReg , MIRBuilder , IncomingArgX , std :: get < <NUM_LIT> > ( WorkitemIDX ) , std :: get < <NUM_LIT> > ( WorkitemIDX ) ) ;


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ;

unsigned KernArgSegmentPtr = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ;


<s> MachineFunction & MF = MIRBuilder . getMF ( ) ; const <ISA_LIT>RegisterInfo * TRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = * MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ;

MIRBuilder . buildConstant ( OffsetReg , Offset ) ;


<s> CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; BitVector Skipped ( NumArgs ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; <STR_LIT> Flags ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ArgInfo OrigArg { VRegs [ i ] [ <NUM_LIT> ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && ! OrigArg . Flags . isInReg ( ) && ! OrigArg . Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { if ( CurOrigArg -> use_empty ( ) && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ) { Skipped . set ( i ) ; ++ P<ISA_LIT>nputNum ; continue ; } Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! CurOrigArg -> use_empty ( ) ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; ++ P<ISA_LIT>nputNum ; } CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; if ( ValEVT . isVector ( ) ) { EVT ElemVT = ValEVT . getVectorElementType ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ElemVT . getSimpleVT ( ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( ! Res ) return false ; } else { MVT ValVT = ValEVT . getSimpleVT ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } } Function :: const_arg_iterator Arg = F . arg_begin ( ) ;

if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS || F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ) {


<s> static bool addCallTargetOperands ( MachineInstrBuilder & CallInst , MachineIRBuilder & MIRBuilder , <ISA_LIT>CallLowering :: CallLoweringInfo & Info ) { if ( Info . Callee . isReg ( ) ) { CallInst . addReg ( Info . Callee . getReg ( ) ) ; CallInst . addImm ( <NUM_LIT> ) ; } else if ( Info . Callee . isGlobal ( ) && Info . Callee . getOffset ( ) == <NUM_LIT> ) { const GlobalValue * GV = Info . Callee . getGlobal ( ) ; auto Ptr = MIRBuilder . buildGlobalValue ( LLT :: pointer ( GV -> getAddressSpace ( ) , <NUM_LIT> ) , GV ) ; CallInst . addReg ( Ptr . getReg ( <NUM_LIT> ) ) ; CallInst . add ( Info . Callee ) ; } else return false ;


<s> } if ( Info . hasKernargSegmentPtr ( ) ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Register InputPtrReg = Info . addKernargSegmentPtr ( TRI ) ; const LLT P4 = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register VReg = MRI . createGenericVirtualRegister ( P4 ) ; MRI . addLiveIn ( InputPtrReg , VReg ) ; B . getMBB ( ) . addLiveIn ( InputPtrReg ) ; B . buildCopy ( VReg , InputPtrReg ) ; CCInfo . AllocateReg ( InputPtrReg ) ; } if ( Info . hasDispatchID ( ) ) {

Register DispatchIDReg = Info . addDispatchID ( TRI ) ;


<s> <ISA_LIT>CallLowering :: <ISA_LIT>CallLowering ( const <ISA_LIT>TargetLowering & TLI ) : CallLowering ( & TLI ) {


<s> <ISA_LIT>IncomingArgHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI ) : IncomingValueHandler ( B , MRI ) {


<s> <ISA_LIT>OutgoingArgHandler ( MachineIRBuilder & MIRBuilder , MachineRegisterInfo & MRI , MachineInstrBuilder MIB , bool IsTailCall = false , int FPDiff = <NUM_LIT> ) : <ISA_LIT>OutgoingValueHandler ( MIRBuilder , MRI , MIB ) , FPDiff ( FPDiff ) , IsTailCall ( IsTailCall ) {


<s> <ISA_LIT>OutgoingValueHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI , MachineInstrBuilder MIB ) : OutgoingValueHandler ( B , MRI ) , MIB ( MIB ) {


<s> CallingConv :: ID CallerCC = CallerF . getCallingConv ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( CalleeCC , TLI ) ; SmallVector < CCValAssign , <NUM_LIT> > OutLocs ; CCState OutInfo ( CalleeCC , false , MF , OutLocs , CallerF . getContext ( ) ) ; OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , OutInfo ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( OutInfo . getNextStackOffset ( ) > FuncInfo -> getBytesInStackArgArea ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; }

const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ;


<s> Register ExtReg = extendRegisterMin32 ( * this , ValVReg , VA ) ; MIRBuilder . buildCopy ( PhysReg , ExtReg ) ;


<s> void assignValueToReg ( Register ValVReg , Register PhysReg , CCValAssign VA ) override { MIB . addUse ( PhysReg , RegState :: Implicit ) ; Register ExtReg = extendRegisterMin32 ( * this , ValVReg , VA ) ;

MIRBuilder . buildCopy ( PhysReg , ExtReg ) ;


<s> CallReturnHandler ( MachineIRBuilder & MIRBuilder , MachineRegisterInfo & MRI , MachineInstrBuilder MIB ) : <ISA_LIT>IncomingArgHandler ( MIRBuilder , MRI ) , MIB ( MIB ) {


<s> bool <ISA_LIT>CallLowering :: canLowerReturn ( MachineFunction & MF , CallingConv :: ID CallConv , SmallVectorImpl < BaseArgInfo > & Outs , bool IsVarArg ) const { if ( <STR_LIT> ( CallConv ) ) return true ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ;

const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ;


<s> bool <ISA_LIT>CallLowering :: canLowerReturn ( MachineFunction & MF , CallingConv :: ID CallConv , SmallVectorImpl < BaseArgInfo > & Outs , bool IsVarArg ) const { if ( <STR_LIT> ( CallConv ) ) return true ;

SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ;


<s> const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * CalleeAssignFnFixed ; CCAssignFn * CalleeAssignFnVarArg ; std :: tie ( CalleeAssignFnFixed , CalleeAssignFnVarArg ) = getAssignFnsForCC ( CalleeCC , TLI ) ; CCAssignFn * CallerAssignFnFixed ; CCAssignFn * CallerAssignFnVarArg ; std :: tie ( CallerAssignFnFixed , CallerAssignFnVarArg ) = getAssignFnsForCC ( CallerCC , TLI ) ;

IncomingValueAssigner CalleeAssigner ( CalleeAssignFnFixed , CalleeAssignFnVarArg ) ;


<s> if ( VA . getLocVT ( ) . getSizeInBits ( ) < <NUM_LIT> ) { return Handler . MIRBuilder . buildAnyExt ( LLT :: scalar ( <NUM_LIT> ) , ValVReg ) . getReg ( <NUM_LIT> ) ;


<s> case TargetOpcode :: G_ZEXT : return <STR_LIT> ; case TargetOpcode :: G_ANYEXT : return <STR_LIT> ; default :


<s> FormalArgHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI ) : <ISA_LIT>IncomingArgHandler ( B , MRI ) {


<s> return { TLI . CCAssignFnForCall ( CC , false ) , TLI . CCAssignFnForCall ( CC , true ) } ;


<s> static std :: pair < CCAssignFn * , CCAssignFn * > getAssignFnsForCC ( CallingConv :: ID CC , const <ISA_LIT>TargetLowering & TLI ) {

return { TLI . CCAssignFnForCall ( CC , false ) , TLI . CCAssignFnForCall ( CC , true ) } ;


<s> static unsigned getCallOpcode ( const MachineFunction & CallerF , bool IsIndirect , bool IsTailCall ) { return IsTailCall ? <STR_LIT> : <STR_LIT> ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! SPReg ) SPReg = MIRBuilder . buildCopy ( PtrTy , MFI -> getStackPtrOffsetReg ( ) ) . getReg ( <NUM_LIT> ) ; auto OffsetReg = MIRBuilder . buildConstant ( S32 , Offset ) ; auto AddrReg = MIRBuilder . buildPtrAdd ( PtrTy , SPReg , OffsetReg ) ; MPO = MachinePointerInfo :: getStack ( MF , Offset ) ;


<s> auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) , FuncInfo . getScratchRSrcReg ( ) ) ; MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; CallInst . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) { MIRBuilder . buildCopy ( ( Register ) ArgReg . first , ArgReg . second ) ; CallInst . addReg ( ArgReg . first , RegState :: Implicit ) ;


<s> if ( ! ST . enableFlatScratch ( ) ) { auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) , FuncInfo . getScratchRSrcReg ( ) ) ; MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; CallInst . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) {

MIRBuilder . buildCopy ( ( Register ) ArgReg . first , ArgReg . second ) ;


<s> } if ( any_of ( CallerF . args ( ) , [ ] ( const Argument & A ) { return A . hasByValAttr ( ) || A . hasSwiftErrorAttr ( ) ; } ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( MF . getTarget ( ) . Options . GuaranteedTailCallOpt ) return canGuaranteeTCO ( CalleeCC ) && CalleeCC == CallerF . getCallingConv ( ) ; if ( ! doCallerAndCalleePassArgsTheSameWay ( Info , MF , InArgs ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! areCalleeOutgoingArgsTailCallable ( Info , MF , OutArgs ) ) return false ;


<s> return false ; } SmallVector < ArgInfo , <NUM_LIT> > OutArgs ; for ( auto & OrigArg : Info . OrigArgs ) splitToValueTypes ( OrigArg , OutArgs , DL , Info . CallConv ) ; SmallVector < ArgInfo , <NUM_LIT> > InArgs ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) splitToValueTypes ( Info . OrigRet , InArgs , DL , Info . CallConv ) ; bool CanTailCallOpt = isEligibleForTailCallOptimization ( MIRBuilder , Info , InArgs , OutArgs ) ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( CanTailCallOpt ) return lowerTailCall ( MIRBuilder , Info , OutArgs ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ;

CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ;


<s> CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ;

<ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , false ) ;


<s> <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget . getRegisterInfo ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; Info -> allocateModuleLDSGlobal ( F . getParent ( ) ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CC , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; if ( ! IsEntryFunc ) { Register ReturnAddrReg = TRI -> getReturnAddressReg ( MF ) ; Register LiveInReturn = MF . addLiveIn ( ReturnAddrReg , & <STR_LIT> ) ; MBB . addLiveIn ( ReturnAddrReg ) ; B . buildCopy ( LiveInReturn , ReturnAddrReg ) ; } if ( Info -> hasImplicitBufferPtr ( ) ) { Register ImplicitBufferPtrReg = Info -> addImplicitBufferPtr ( * TRI ) ; MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; unsigned Idx = <NUM_LIT> ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; if ( ! FLI . CanLowerReturn ) insertSRetIncomingArgument ( F , SplitArgs , FLI . DemoteRegister , MRI , DL ) ; for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsGraphics && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) {

const bool ArgUsed = ! Arg . use_empty ( ) ;


<s> LLT ArgTy = getLLTForType ( * SplitArg . Ty , DL ) ; if ( SplitArg . Flags [ <NUM_LIT> ] . isPointer ( ) ) { LLT PtrTy = LLT :: pointer ( SplitArg . Flags [ <NUM_LIT> ] . getPointerAddrSpace ( ) , ArgTy . getScalarSizeInBits ( ) ) ; ArgTy = ArgTy . isVector ( ) ? LLT :: vector ( ArgTy . getElementCount ( ) , PtrTy ) : PtrTy ; } MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , ArgTy , commonAlignment ( Alignment , FieldOffsets [ Idx ] ) ) ; assert ( SplitArg . Regs . size ( ) == <NUM_LIT> ) ;

B . buildLoad ( SplitArg . Regs [ <NUM_LIT> ] , PtrReg , * MMO ) ;


<s> void <ISA_LIT>CallLowering :: lowerParameter ( MachineIRBuilder & B , ArgInfo & OrigArg , uint64_t Offset , Align Alignment ) const { MachineFunction & MF = B . getMF ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; SmallVector < uint64_t > FieldOffsets ; splitToValueTypes ( OrigArg , SplitArgs , DL , F . getCallingConv ( ) , & FieldOffsets ) ; unsigned Idx = <NUM_LIT> ; for ( ArgInfo & SplitArg : SplitArgs ) { Register PtrReg = B . getMRI ( ) -> createGenericVirtualRegister ( PtrTy ) ; lowerParameterPtr ( PtrReg , B , Offset + FieldOffsets [ Idx ] ) ; LLT ArgTy = getLLTForType ( * SplitArg . Ty , DL ) ; if ( SplitArg . Flags [ <NUM_LIT> ] . isPointer ( ) ) { LLT PtrTy = LLT :: pointer ( SplitArg . Flags [ <NUM_LIT> ] . getPointerAddrSpace ( ) , ArgTy . getScalarSizeInBits ( ) ) ; ArgTy = ArgTy . isVector ( ) ? LLT :: vector ( ArgTy . getElementCount ( ) , PtrTy ) : PtrTy ; }

MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , ArgTy , commonAlignment ( Alignment , FieldOffsets [ Idx ] ) ) ;


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ;

auto OffsetReg = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) ;


<s> void <ISA_LIT>CallLowering :: lowerParameterPtr ( Register DstReg , MachineIRBuilder & B , uint64_t Offset ) const { MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

MachineRegisterInfo & MRI = MF . getRegInfo ( ) ;


<s> const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } auto const & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! FLI . CanLowerReturn ) insertSRetStores ( B , Val -> getType ( ) , VRegs , FLI . DemoteRegister ) ; else if ( ! lowerReturnVal ( B , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LiveInReturn = MF . addLiveIn ( TRI -> getReturnAddressReg ( MF ) , & <STR_LIT> ) ; B . buildCopy ( ReturnAddrVReg , LiveInReturn ) ;


<s> if ( ! Val ) return true ; auto & MF = B . getMF ( ) ; const auto & F = MF . getFunction ( ) ; const DataLayout & DL = MF . getDataLayout ( ) ; MachineRegisterInfo * MRI = B . getMRI ( ) ; LLVMContext & Ctx = F . getContext ( ) ; CallingConv :: ID CC = F . getCallingConv ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; SmallVector < EVT , <NUM_LIT> > SplitEVTs ; ComputeValueVTs ( TLI , DL , Val -> getType ( ) , SplitEVTs ) ; assert ( VRegs . size ( ) == SplitEVTs . size ( ) && <STR_LIT> ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; for ( unsigned i = <NUM_LIT> ; i < SplitEVTs . size ( ) ; ++ i ) { EVT VT = SplitEVTs [ i ] ; Register Reg = VRegs [ i ] ; ArgInfo RetInfo ( Reg , VT . getTypeForEVT ( Ctx ) , <NUM_LIT> ) ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; if ( VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( RetInfo . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( RetInfo . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { RetInfo . Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLT ExtTy = getLLTForType ( * RetInfo . Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { ExtTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } if ( Reg != RetInfo . Regs [ <NUM_LIT> ] ) { RetInfo . Regs [ <NUM_LIT> ] = Reg ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; } splitToValueTypes ( RetInfo , SplitRetInfos , DL , CC ) ; } CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; OutgoingValueAssigner Assigner ( AssignFn ) ; <ISA_LIT>OutgoingValueHandler RetHandler ( B , * MRI , Ret ) ; return determineAndHandleAssignments ( RetHandler , Assigner , SplitRetInfos , B , CC , F . isVarArg ( ) ) ;


<s> MIB . addDef ( PhysReg , RegState :: Implicit ) ;


<s> case CallingConv :: C : case CallingConv :: <ISA_LIT>_Gfx : return true ;


<s> <ISA_LIT>CodeGenPrepare ( ) : FunctionPass ( ID ) {


<s> bool <ISA_LIT>CodeGenPrepare :: canWidenScalarExtLoad ( LoadInst & I ) const { Type * Ty = I . getType ( ) ; const DataLayout & DL = Mod -> getDataLayout ( ) ; int TySize = DL . getTypeSizeInBits ( Ty ) ; unsigned Align = I . getAlignment ( ) ? I . getAlignment ( ) : DL . getABITypeAlignment ( Ty ) ; return I . isSimple ( ) && TySize < <NUM_LIT> && Align >= <NUM_LIT> && DA -> isUniform ( & I ) ;


<s> int TySize = DL . getTypeSizeInBits ( Ty ) ; unsigned Align = I . getAlignment ( ) ? I . getAlignment ( ) : DL . getABITypeAlignment ( Ty ) ; return I . isSimple ( ) && TySize < <NUM_LIT> && Align >= <NUM_LIT> && DA -> isUniform ( & I ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>CodeGenPreparePass ( ) {

return new <ISA_LIT>CodeGenPrepare ( ) ;


<s> bool <ISA_LIT>CodeGenPrepare :: doInitialization ( Module & M ) { Mod = & M ; return false ;


<s> bool <ISA_LIT>CodeGenPrepare :: doInitialization ( Module & M ) {


<s> if ( RHSSignBits < <NUM_LIT> ) return nullptr ; unsigned SignBits = std :: min ( LHSSignBits , RHSSignBits ) ; unsigned DivBits = <NUM_LIT> - SignBits ; if ( IsSigned ) ++ DivBits ; Type * Ty = Num -> getType ( ) ; Type * I32Ty = Builder . getInt32Ty ( ) ; Type * F32Ty = Builder . getFloatTy ( ) ; ConstantInt * One = Builder . getInt32 ( <NUM_LIT> ) ; Value * JQ = One ; if ( IsSigned ) { JQ = Builder . CreateXor ( Num , Den ) ; JQ = Builder . CreateAShr ( JQ , Builder . getInt32 ( <NUM_LIT> ) ) ; JQ = Builder . CreateOr ( JQ , One ) ; } Value * IA = Num ; Value * IB = Den ; Value * FA = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IA , F32Ty ) : Builder . CreateUIToFP ( IA , F32Ty ) ; Value * FB = IsSigned ? Builder . Create<ISA_LIT>ToFP ( IB , F32Ty ) : Builder . CreateUIToFP ( IB , F32Ty ) ; Value * RCP = Builder . CreateFDiv ( ConstantFP :: get ( F32Ty , <NUM_LIT> ) , FB ) ; Value * FQM = Builder . CreateFMul ( FA , RCP ) ; CallInst * FQ = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FQM ) ; FQ -> copyFastMathFlags ( Builder . getFastMathFlags ( ) ) ; Value * FQNeg = Builder . CreateFNeg ( FQ ) ; Value * FR = Builder . CreateIntrinsic ( <STR_LIT> , { FQNeg -> getType ( ) } , { FQNeg , FB , FA } , FQ ) ; Value * IQ = IsSigned ? Builder . CreateFPTo<ISA_LIT> ( FQ , I32Ty ) : Builder . CreateFPToUI ( FQ , I32Ty ) ; FR = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FR , FQ ) ; FB = Builder . CreateUnaryIntrinsic ( <STR_LIT> , FB , FQ ) ;

Value * CV = Builder . CreateFCmpOGE ( FR , FB ) ;