<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> initializeBaseYamlFields ( YamlMFI , MF , PFS , Error , SourceRange ) ) return true ; if ( MFI -> Occupancy == <NUM_LIT> ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; MFI -> Occupancy = ST . computeOccupancy ( MF . getFunction ( ) , MFI -> getLDSSize ( ) ) ; } auto parseRegister = [ & ] ( const yaml :: StringValue & RegName , Register & RegVal ) { Register TempReg ; if ( parseNamedRegisterReference ( PFS , TempReg , RegName . Value , Error ) ) { SourceRange = RegName . SourceRange ; return true ; } RegVal = TempReg ; return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs , unsigned SystemSGPRs ) { if ( ! A ) return false ; if ( A -> IsRegister ) { Register Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; } if ( ! RC . contains ( Reg ) ) return

diagnoseRegisterClass ( A -> RegisterName ) ;


<s> AAM . registerFunctionAnalysis < <ISA_LIT>AA > (


<s> AAM . registerFunctionAnalysis < <ISA_LIT>AA >


<s> return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaToVectorPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>LowerKernelAttributesPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteKernelArgumentsPass ( ) ) ; return true ; } return false ; } ) ; PB . registerAnalysisRegistrationCallback ( [ ] ( FunctionAnalysisManager & FAM ) { FAM . registerPass ( [ & ] { return <ISA_LIT>AA ( ) ; } ) ; } ) ; PB . registerParseAACallback ( [ ] ( StringRef AAName , AAManager & AAM ) { if ( AAName == <STR_LIT> ) { AAM . registerFunctionAnalysis < <ISA_LIT>AA > ( ) ; return true ; } return false ; } ) ; PB . registerPipelineStartEPCallback ( [ this ] ( ModulePassManager & PM , OptimizationLevel Level ) { FunctionPassManager FPM ; FPM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; FPM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ; if ( EnableLibCallSimplify && Level != OptimizationLevel :: O0 ) FPM . addPass ( <ISA_LIT>SimplifyLibCallsPass ( *


<s> static FunctionPass * useDefaultRegisterAllocator (


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass * RegAllocPass ) { addPass ( & <ISA_LIT>FixSGPRLiveRangesID ) ; TargetPassConfig :: addFastRegAlloc ( RegAllocPass


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass *


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & LiveVariablesID , & <ISA_LIT>FixSGPRLiveRangesID ,


<ISA_LIT>FixSGPRLiveRangesID , false ) ;


<s> addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ,


<s> const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) { insertPass ( & MachineSchedulerID ,

& <ISA_LIT>FixControlFlowLiveIntervalsID ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , CPU , FS , Options , RM , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , Subtarget ( TT , CPU , FS , * this ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , CPU , FS , Options , RM , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , Subtarget ( TT , CPU , FS , * this ) , IntrinsicInfo ( ) { setRequiresStructuredCFG (


<s> if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return create<ISA_LIT>MachineScheduler


<s> const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return create<ISA_LIT>MachineScheduler ( C ) ; return


<s> static ScheduleDAGInstrs * create<ISA_LIT>MachineScheduler ( MachineSchedContext * C


<s> static std :: unique_ptr < TargetLoweringObjectFile > createTLOF ( const Triple & TT ) { if ( TT . getOS ( ) == Triple :: AMDHSA )

return make_unique < <ISA_LIT>HSATargetObjectFile > ( ) ;


<s> return getTM < <ISA_LIT>TargetMachine >


<s> void LLVMInitialize<ISA_LIT>Target ( ) { RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ; RegisterTargetMachine < GCNTargetMachine > Y ( TheGCNTarget ) ; PassRegistry * PR = PassRegistry :: getPassRegistry ( ) ; initialize<ISA_LIT>LowerI1CopiesPass ( *


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ; TargetPassConfig :: addCodeGenPrepare (


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & <ISA_LIT>LowerControlFlowID , & <ISA_LIT>FixWWMLivenessID ,


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & <ISA_LIT>LowerControlFlowID , & <ISA_LIT>FixWWMLivenessID , false


<s> void GCNPassConfig :: addMachineSSAOptimization ( ) { TargetPassConfig :: addMachineSSAOptimization ( ) ; addPass ( & <ISA_LIT>FoldOperandsID ) ; addPass ( & DeadMachineInstructionElimID ) ; addPass ( & <ISA_LIT>LoadStoreOptimizerID ) ; if ( EnableSDWAPeephole ) { addPass ( & <ISA_LIT>PeepholeSDWAID


<s> insertPass ( & <ISA_LIT>LowerControlFlowID , & <ISA_LIT>FixWWMLivenessID , false ) ; TargetPassConfig :: addOptimizedRegAlloc (


<s> insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & <ISA_LIT>LowerControlFlowID , & <ISA_LIT>FixWWMLivenessID , false ) ; TargetPassConfig :: addOptimizedRegAlloc ( RegAllocPass


<s> void GCNPassConfig :: addPostRegAlloc ( ) { addPass ( & <ISA_LIT>FixVGPRCopiesID )


<s> addPass ( & <ISA_LIT>FixVGPRCopiesID ) ; addPass ( & <ISA_LIT>OptimizeExecMaskingID ) ; TargetPassConfig :: addPostRegAlloc


<s> addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ; addPass ( & BranchRelaxationPassID


<s> } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) { addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ; } return false


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , getEffectiveCodeModel ( CM ) , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) { initAsmInfo (


<s> if ( TT . getArch ( ) == Triple :: r600


<s> if ( auto * WrapperPass = P . getAnalysisIfAvailable < <ISA_LIT>AAWrapperPass > (

) ) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ;


<s> ScheduleDAGMILive * DAG = new GCNScheduleDAGMILive ( C , make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( createStoreClusterDAGMutation ( DAG ->


<s> ScheduleDAGInstrs * createMachineScheduler ( MachineSchedContext * C ) const override { ScheduleDAGMILive * DAG = createGenericSchedLive ( C ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG


<s> ScheduleDAGMILive * DAG = createGenericSchedLive ( C ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG ->


<s> return new ScheduleDAGMILive ( C , llvm ::

make_unique < <ISA_LIT>SchedStrategy > ( ) ) ;


<s> static ScheduleDAGInstrs * create<ISA_LIT>MachineScheduler ( MachineSchedContext * C ) { return new ScheduleDAGMILive ( C , llvm :: make_unique


<s> static std :: unique_ptr < TargetLoweringObjectFile > createTLOF


<s> return FSAttr . hasAttribute ( Attribute :: None )


<s> StringRef <ISA_LIT>TargetMachine :: getGPUName ( const Function & F ) const { Attribute GPUAttr = F . getFnAttribute ( <STR_LIT>


<s> if ( ! GPU . empty ( ) ) return GPU ; if ( TT . getArch ( ) == Triple :: amdgcn ) return <STR_LIT> ; return <STR_LIT>


<s> TargetPassConfig :: addCodeGenPrepare ( ) ; if ( EnableLoadStoreVectorizer ) addPass ( createLoadStoreVectorizerPass ( )


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass * RegAllocPass


<s> void GCNPassConfig :: addIRPasses ( ) { addPass ( create<ISA_LIT>CodeGenPreparePass ( & getGCNTargetMachine ( ) )


<s> TargetPassConfig :: addOptimizedRegAlloc (


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) { TargetPassConfig :: addOptimizedRegAlloc (


<s> addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>LowerControlFlowPass ( )


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ; } if ( getOptLevel ( ) > CodeGenOpt :: None ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>LoadStoreOptimizerID ) ; insertPass ( & MachineSchedulerID , & RegisterCoalescerID ) ; } addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) )


<s> addPass ( create<ISA_LIT>EmitClauseMarkers ( ) , false ) ; if ( Enable<ISA_LIT>IfConvert ) addPass ( & IfConverterID , false ) ; addPass ( create<ISA_LIT>ClauseMergePass ( * TM ) ,


<s> addPass ( createSpeculativeExecutionPass ( ) ) ; addPass ( createStraightLineStrengthReducePass ( ) ) ; addEarlyCSEOrGVNPass ( ) ; addPass ( createNaryReassociatePass


<s> addPass ( createSpeculativeExecutionPass ( ) ) ; addPass ( createStraightLineStrengthReducePass ( ) ) ; addEarlyCSEOrGVNPass ( )


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true ) ; initAsmInfo (


<s> static StringRef computeDataLayout ( const Triple & TT ) { if ( TT . getArch ( )


<s> const <ISA_LIT>Subtarget & ST = C -> MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . enable<ISA_LIT>Scheduler ( ) )

return create<ISA_LIT>MachineScheduler ( C ) ;


<s> StringRef FS = getFeatureString ( F ) ; SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F ) ; I = llvm :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ; GISelAccessor * GISel = new GISelAccessor ( ) ; <ISA_LIT>GISelActualAccessor * GISel = new <ISA_LIT>GISelActualAccessor ( ) ; GISel -> CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * I -> getTargetLowering ( ) ) ) ; I -> setGISelAccessor ( *


<s> return TargetIRAnalysis ( [ this ] ( const Function & F ) { return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this ,


<s> TargetIRAnalysis <ISA_LIT>TargetMachine :: getTargetIRAnalysis ( ) { return TargetIRAnalysis ( [ this ] (


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ; TargetPassConfig :: addCodeGenPrepare


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ; TargetPassConfig :: addCodeGenPrepare ( )


<s> void GCNPassConfig :: addFastRegAlloc ( ) { insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & RegisterCoalescerID , & <ISA_LIT>PreAllocateWWMRegsID , false ) ; TargetPassConfig :: addFastRegAlloc (


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; insertPass ( & <ISA_LIT>OptimizeExecMaskingPreRAID , & <ISA_LIT>FormMemoryClausesID ) ; } else { insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; } insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & RegisterCoalescerID , & <ISA_LIT>PreAllocateWWMRegsID ,


<s> void GCNPassConfig :: addOptimizedRegAlloc ( ) { if ( OptExecMaskPreRA ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; insertPass ( & <ISA_LIT>OptimizeExecMaskingPreRAID , &


<s> addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( & BranchRelaxationPassID


<s> addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID


<s> } addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { addPass ( createStructurizeCFGPass ( true ) ) ; } addPass ( createSinkingPass ( )


<s> const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> .


<s> } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ; addPass ( & <ISA_LIT>PerfHintAnalysisID ) ; TargetPassConfig :: addCodeGenPrepare


<s> <ISA_LIT>PassConfig :: addPreISel ( ) ; if ( EnableAtomicOptimizations ) { addPass ( create<ISA_LIT>AtomicOptimizerPass ( ) ) ; } addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { addPass ( createStructurizeCFGPass ( true ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues (


<s> if ( ! LateCFGStructurize ) { addPass ( createStructurizeCFGPass ( true ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) { addPass ( create<ISA_LIT>AnnotateControlFlowPass


<s> static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler ( MachineSchedContext * C ) { ScheduleDAGMILive * DAG = new GCNScheduleDAGMILive ( C , std :: make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG


<s> return false ; } ; if ( YamlMFI . ArgInfo && ( parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentBuffer , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentBuffer , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchPtr , <STR_LIT> , MFI -> ArgInfo . DispatchPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> QueuePtr , <STR_LIT> , MFI -> ArgInfo . QueuePtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> KernargSegmentPtr , <STR_LIT> , MFI -> ArgInfo . KernargSegmentPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchID , <STR_LIT> , MFI -> ArgInfo . DispatchID , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> FlatScratchInit , <STR_LIT> , MFI -> ArgInfo . FlatScratchInit , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentSize , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentSize , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDX , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDY , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDZ , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDZ , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupInfo , <STR_LIT> , MFI -> ArgInfo . WorkGroupInfo , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentWaveByteOffset , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentWaveByteOffset , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitArgPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitArgPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitBufferPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitBufferPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDX , <STR_LIT> , MFI -> ArgInfo . WorkItemIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDY , <STR_LIT> , MFI -> ArgInfo . WorkItemIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDZ , <STR_LIT> , MFI -> ArgInfo . WorkItemIDZ , <NUM_LIT> , <NUM_LIT> ) ) ) return true ; MFI -> Mode . IEEE = YamlMFI . Mode . IEEE ; MFI -> Mode . DX10Clamp = YamlMFI . Mode . DX10Clamp ; MFI -> Mode . FP32Denormals = YamlMFI


<s> addPass ( & FinalizeISelID ) ; addPass ( create<ISA_LIT>FixupVectorISelPass ( ) ) ; addPass ( create<ISA_LIT>AddIMGInitPass ( ) ) ; return false


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & RegisterCoalescerID , & <ISA_LIT>PreAllocateWWMRegsID , false ) ; if ( EnableDCEInRA ) insertPass ( & DetectDeadLanesID , & DeadMachineInstructionElimID ) ; TargetPassConfig :: addOptimizedRegAlloc


<s> addPass ( & PostRAHazardRecognizerID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>RemoveShortExecBranchesID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID )


<s> addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID


<s> } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) { addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ; } addPass ( createLCSSAPass (


<s> if ( match ( const_cast < Value * > ( V ) , m_c_And ( m_Not ( m_Intrinsic < <STR_LIT> > ( m_Value ( Ptr ) ) ) , m_Not ( m_Intrinsic < <STR_LIT> > ( m_Deferred ( Ptr ) ) ) )

) ) return std :: make_pair ( Ptr , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ;


<s> } insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & RegisterCoalescerID , & <ISA_LIT>PreAllocateWWMRegsID , false ) ; if ( EnableDCEInRA ) insertPass ( & DetectDeadLanesID , &


<s> void GCNPassConfig :: addPreLegalizeMachineIR ( ) { bool IsOptNone = getOptLevel ( ) == CodeGenOpt :: None ; addPass ( create<ISA_LIT>PreLegalizeCombiner ( IsOptNone


<s> void GCNPassConfig :: addPreLegalizeMachineIR ( ) { bool IsOptNone = getOptLevel ( )


<s> addPass ( & <ISA_LIT>PerfHintAnalysisID ) ; TargetPassConfig :: addCodeGenPrepare ( ) ; if ( EnableLoadStoreVectorizer ) addPass ( createLoadStoreVectorizerPass ( ) ) ; addPass ( createLowerSwitchPass ( )


<s> void GCNPassConfig :: addFastRegAlloc ( ) { insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & TwoAddressInstructionPassID , & <ISA_LIT>WholeQuadModeID ) ; insertPass ( & TwoAddressInstructionPassID , &


<s> addPass ( new InstructionSelect ( ) ) ; return false


<s> addPass ( & <ISA_LIT>FixSGPRCopiesID ) ; addPass ( create<ISA_LIT>LowerI1CopiesPass ( ) )


<s> addPass ( & <ISA_LIT>FoldOperandsID ) ; if ( EnableDPPCombine ) addPass ( & GCNDPPCombineID ) ; addPass ( & DeadMachineInstructionElimID ) ; addPass ( & <ISA_LIT>LoadStoreOptimizerID ) ; if ( EnableSDWAPeephole ) { addPass ( & <ISA_LIT>PeepholeSDWAID ) ; addPass ( & EarlyMachineLICMID ) ; addPass ( & MachineCSEID ) ; addPass ( & <ISA_LIT>FoldOperandsID ) ; addPass ( & DeadMachineInstructionElimID )


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>WholeQuadModeID ) ; insertPass ( & MachineSchedulerID , & <ISA_LIT>PreAllocateWWMRegsID ) ; if ( OptExecMaskPreRA ) insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false


<s> addPass ( & <ISA_LIT>FixVGPRCopiesID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>OptimizeExecMaskingID ) ; TargetPassConfig :: addPostRegAlloc ( ) ; addPass ( &


<s> addPass ( & <ISA_LIT>FixVGPRCopiesID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( &


<s> addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>RemoveShortExecBranchesID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( & <ISA_LIT>PreEmitPeepholeID ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( & BranchRelaxationPassID


<s> bool GCNPassConfig :: addPreISel ( ) { <ISA_LIT>PassConfig :: addPreISel ( ) ; addPass ( create<ISA_LIT>LateCodeGenPreparePass ( ) ) ; if ( EnableAtomicOptimizations ) { addPass ( create<ISA_LIT>AtomicOptimizerPass ( ) ) ; } addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { if (


<s> addPass ( & GCNRegBankReassignID ) ; } return


<s> if ( Enable<ISA_LIT>IfConvert ) addPass ( & IfConverterID , false ) ; addPass ( create<ISA_LIT>ClauseMergePass ( )


<s> <ISA_LIT>PassConfig ( LLVMTargetMachine & TM , PassManagerBase & PM ) : TargetPassConfig


<s> <ISA_LIT>PassConfig ( LLVMTargetMachine & TM , PassManagerBase & PM ) : TargetPassConfig (


<s> yaml :: MachineFunctionInfo * GCNTargetMachine :: convertFuncInfoToYAML ( const MachineFunction & MF ) const { const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return new yaml :: <ISA_LIT>MachineFunctionInfo ( * MFI , * MF . getSubtarget ( )


<s> ScheduleDAGInstrs * createMachineScheduler ( MachineSchedContext * C ) const override { return create<ISA_LIT>MachineScheduler ( C


<s> TargetPassConfig * <ISA_LIT>TargetMachine :: createPassConfig (


<s> return new ScheduleDAGMILive ( C , std :: make_unique < <ISA_LIT>SchedStrategy > ( )


<s> GCNPassConfig ( LLVMTargetMachine & TM , PassManagerBase & PM ) : <ISA_LIT>PassConfig ( TM , PM ) { setRequiresCodeGenSCCOrder ( true


<s> SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F )


<s> StringRef FS = getFeatureString ( F ) ; SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( !


<s> TargetTransformInfo <ISA_LIT>TargetMachine :: getTargetTransformInfo ( const Function & F ) { return TargetTransformInfo ( <ISA_LIT>TTIImpl (


<s> if ( A -> IsRegister ) { Register Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; } if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A -> RegisterName ) ; Arg = ArgDescriptor :: createRegister ( Reg ) ; } else Arg = ArgDescriptor :: createStack ( A -> StackOffset ) ; if ( A -> Mask ) Arg = ArgDescriptor :: createArg ( Arg , A -> Mask . getValue ( ) ) ; MFI -> NumUserSGPRs += UserSGPRs ; MFI -> NumSystemSGPRs += SystemSGPRs ; return false ; } ; if ( YamlMFI . ArgInfo && ( parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentBuffer , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentBuffer , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchPtr , <STR_LIT> , MFI -> ArgInfo . DispatchPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> QueuePtr , <STR_LIT> , MFI -> ArgInfo . QueuePtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> KernargSegmentPtr , <STR_LIT> , MFI -> ArgInfo . KernargSegmentPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchID , <STR_LIT> , MFI -> ArgInfo . DispatchID , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> FlatScratchInit , <STR_LIT> , MFI -> ArgInfo . FlatScratchInit , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentSize , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentSize , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDX , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDY , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDZ , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDZ , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupInfo , <STR_LIT> , MFI -> ArgInfo . WorkGroupInfo , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentWaveByteOffset , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentWaveByteOffset , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitArgPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitArgPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitBufferPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitBufferPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDX , <STR_LIT> , MFI -> ArgInfo . WorkItemIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDY , <STR_LIT> , MFI -> ArgInfo . WorkItemIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDZ , <STR_LIT> , MFI -> ArgInfo . WorkItemIDZ , <NUM_LIT> , <NUM_LIT> ) ) ) return true ; MFI -> Mode . IEEE = YamlMFI . Mode


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OL , bool JIT ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) { setRequiresStructuredCFG ( true )


addPass ( <ISA_LIT>AlwaysInlinePass ( ) ) ;


<s> Builder . DivergentTarget = true ; bool EnableOpt = getOptLevel ( ) > CodeGenOpt :: None ; bool Internalize = InternalizeSymbols ; bool EarlyInline = EarlyInlineAll && EnableOpt && ! Enable<ISA_LIT>FunctionCalls ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && EnableOpt ; bool LibCallSimplify = EnableLibCallSimplify && EnableOpt ; if ( Enable<ISA_LIT>FunctionCalls ) { delete Builder . Inliner ; Builder . Inliner = create<ISA_LIT>FunctionInliningPass ( ) ; } if ( Internalize ) { Builder . addExtension ( PassManagerBuilder :: EP_EnabledOnOptLevel0 , [ ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { PM . add ( createInternalizePass ( mustPreserveGV ) ) ; } ) ; } Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass (


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , getEffectiveCodeModel ( CM ) , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) { AS = <STR_LIT> ( TT ) ; initAsmInfo ( )


<s> addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( &


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ; addPass ( &


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass * RegAllocPass


<s> addPass ( create<ISA_LIT>ISelDag ( get<ISA_LIT>TargetMachine ( ) , getOptLevel ( ) ) ) ; return


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; TargetPassConfig :: addOptimizedRegAlloc (


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID ,


<s> addPass ( & PostRAHazardRecognizerID ) ; addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( &


<s> void GCNPassConfig :: addPreRegAlloc ( ) { addPass ( create<ISA_LIT>ShrinkInstructionsPass (


<s> PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true ; } } return ! GV . use_empty ( ) ; } ) ) ; PM . add ( createGlobalDCEPass ( )


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine (

T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) {


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options ,

getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) {


<s> DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( createStoreClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; return DAG


<s> if ( ST . enable<ISA_LIT>Scheduler ( ) ) return create<ISA_LIT>MachineScheduler ( C ) ; return createGCNMaxOccupancyMachineScheduler (


<s> if ( ! I ) { resetTargetOptions ( F ) ; I = llvm :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ; GISelAccessor * GISel = new GISelAccessor ( ) ; <ISA_LIT>GISelActualAccessor * GISel = new <ISA_LIT>GISelActualAccessor ( ) ; GISel -> CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * I ->


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID


<s> addPass ( & PostRAHazardRecognizerID ) ; addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; if ( Enable<ISA_LIT>InsertWaitcntsPass ) addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; else addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ; addPass ( &


<s> } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( mustPreserveGV ) ) ; PM . add ( createGlobalDCEPass ( ) ) ; } if ( EarlyInline ) PM . add ( create<ISA_LIT>AlwaysInlinePass ( false ) ) ; } ) ; const auto & Opt = Options ; Builder . addExtension ( PassManagerBuilder :: EP_EarlyAsPossible , [ <ISA_LIT>AA , LibCallSimplify , & Opt ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( llvm :: create<ISA_LIT>UseNativeCallsPass (


<s> if ( TT . getArch ( ) == Triple :: r600 ) { return <STR_LIT> <STR_LIT> ; } return <STR_LIT> <STR_LIT>


<s> auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F ) ; I = llvm :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ; } I -> setScalarizeGlobalBehavior ( ScalarizeGlobal


<s> return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this


<s> bool isPassEnabled ( const cl :: opt < bool > & Opt , CodeGenOpt :: Level Level = CodeGenOpt :: Default ) const { if ( Opt . getNumOccurrences ( ) ) return


<s> if ( Opt . getNumOccurrences ( ) ) return Opt ; if ( TM -> getOptLevel ( ) < Level


<s> addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; addPass ( createSROAPass ( ) ) ; } TargetPassConfig :: addCodeGenPrepare


<s> addPass ( create<ISA_LIT>InsertWaitsPass ( ) , false ) ; addPass ( create<ISA_LIT>LowerControlFlowPass ( ) , false ) ; if (


<s> bool <ISA_LIT>PassConfig :: addPreISel ( ) { <ISA_LIT>PassConfig :: addPreISel ( ) ; const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( ST . IsIRStructurizerEnabled ( ) ) addPass ( createStructurizeCFGPass ( )


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , RM , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , Subtarget ( TT , getTargetCPU ( ) , FS , * this ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true


<s> ScheduleDAGInstrs * createMachineScheduler ( MachineSchedContext * C ) const override { const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return create<ISA_LIT>MachineScheduler ( C ) ; else if ( ST . enable<ISA_LIT>Scheduler ( ) ) return create<ISA_LIT>MachineScheduler ( C ) ; return


<s> LLVM_READNONE static StringRef getGPUOrDefault ( const Triple & TT , StringRef GPU ) { if ( ! GPU . empty ( ) ) return


<s> if ( AddrSpace == AS . LOCAL_ADDRESS || AddrSpace

== AS . REGION_ADDRESS ) return - <NUM_LIT> ;


<s> <ISA_LIT>PassConfig :: addPreISel ( ) ; if ( EnableAtomicOptimizations ) { addPass ( create<ISA_LIT>AtomicOptimizerPass ( ) ) ; } addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { if ( EnableStructurizerWorkarounds ) { addPass ( createUnifyLoopExitsPass ( ) ) ; } addPass ( createStructurizeCFGPass ( false ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize )


<s> const <ISA_LIT>Subtarget * getSubtargetImpl ( const Function &


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( & PostRAHazardRecognizerID ) ; if ( Enable<ISA_LIT>InsertWaitcntsPass ) addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; else addPass ( create<ISA_LIT>InsertWaitsPass (


<s> addPass ( & PostRAHazardRecognizerID ) ; if ( Enable<ISA_LIT>InsertWaitcntsPass ) addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; else addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( )


<s> addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { addPass ( createStructurizeCFGPass ( true ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>TypeRewriter ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) { addPass ( create<ISA_LIT>AnnotateControlFlowPass


<s> Builder . DivergentTarget = true ; bool Internalize = InternalizeSymbols && ( getOptLevel ( ) > CodeGenOpt :: None ) && ( getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) ; bool EarlyInline = EarlyInlineAll && ( getOptLevel ( ) > CodeGenOpt :: None ) ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && getOptLevel ( ) > CodeGenOpt :: None ; Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) { AS = <STR_LIT> ( TT ) ; initAsmInfo ( )


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) { AS = <STR_LIT> ( TT )


<s> if ( TT . getEnvironmentName ( ) == <STR_LIT> || TT . getEnvironmentName ( ) == <STR_LIT> ) return <STR_LIT> <STR_LIT> <STR_LIT> ; return <STR_LIT> <STR_LIT>


<s> } return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg ) { if ( ! A ) return false ; if ( A -> IsRegister ) { unsigned Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; } if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A -> RegisterName ) ; Arg = ArgDescriptor :: createRegister ( Reg )


&& EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ;


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ; TargetPassConfig :: addCodeGenPrepare (


<s> addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>PreEmitPeepholeID ) ; addPass ( &


<s> if ( TM -> getOptLevel ( ) > CodeGenOpt :: None ) addPass

( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;


<s> addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( )


<s> addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>MemoryLegalizerPass (


<s> } if ( TT . getEnvironmentName ( ) == <STR_LIT> || TT . getEnvironmentName ( ) == <STR_LIT> ) return <STR_LIT>


<s> } return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs , unsigned SystemSGPRs ) { if ( ! A ) return false ; if ( A -> IsRegister ) { unsigned Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; } if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A


<s> bool mustPreserveGV ( const GlobalValue & GV ) { if ( const Function * F = dyn_cast < Function > ( & GV ) ) return F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ; return ! GV .


<s> addPass ( create<ISA_LIT>InsertWaitsPass ( )


<s> addPass ( create<ISA_LIT>InsertWaitsPass ( ) , false ) ; addPass ( create<ISA_LIT>LowerControlFlowPass ( )


<s> void GCNPassConfig :: addPreRegAlloc ( )


<s> bool Internalize = InternalizeSymbols && ( getOptLevel ( ) > CodeGenOpt :: None ) && ( getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) ; bool EarlyInline = EarlyInlineAll && ( getOptLevel ( ) > CodeGenOpt :: None ) ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && getOptLevel ( ) > CodeGenOpt :: None ; Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv


<s> Builder . DivergentTarget = true ; bool Internalize = InternalizeSymbols && ( getOptLevel ( ) > CodeGenOpt :: None ) && ( getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) ; bool EarlyInline = EarlyInlineAll && ( getOptLevel ( ) > CodeGenOpt :: None ) ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && getOptLevel ( ) > CodeGenOpt :: None ; Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true ; } } return ! GV . use_empty ( ) ; } ) ) ; PM . add ( createGlobalDCEPass ( ) ) ; } if ( EarlyInline ) PM . add ( create<ISA_LIT>AlwaysInlinePass ( false ) ) ; } ) ; Builder . addExtension ( PassManagerBuilder :: EP_EarlyAsPossible , [ <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( )


<s> addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass (


<s> void <ISA_LIT>TargetMachine :: addEarlyAsPossiblePasses ( PassManagerBase &


<s> PM . add ( create<ISA_LIT>UnifyMetadataPass


<s> bool GCNPassConfig :: addGlobalInstructionSelect ( ) { return


<s> addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; addStraightLineScalarOptimizationPasses ( ) ; } TargetPassConfig :: addIRPasses ( ) ; if ( getOptLevel ( ) != CodeGenOpt


<s> bool GCNPassConfig :: addLegalizeMachineIR


<s> bool GCNPassConfig :: addLegalizeMachineIR (


<s> static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler ( MachineSchedContext * C ) { ScheduleDAGMILive * DAG = new ScheduleDAGMILive ( C , llvm :: make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG


<s> SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F ) ; I = llvm :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ; GISelAccessor * GISel = new GISelAccessor ( ) ; <ISA_LIT>GISelActualAccessor * GISel = new <ISA_LIT>GISelActualAccessor ( ) ; GISel -> CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * I -> getTargetLowering ( ) ) ) ; I -> setGISelAccessor ( * GISel ) ; } I -> setScalarizeGlobalBehavior ( ScalarizeGlobal


<s> Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ & ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { PM . add ( create<ISA_LIT>UnifyMetadataPass (


<s> TargetLoweringObjectFile * getObjFileLowering (


<s> TargetTransformInfo GCNTargetMachine :: getTargetTransformInfo ( const Function


<s> return TargetTransformInfo ( GCNTTIImpl ( this , F


<s> if ( TT . getArch ( ) == Triple :: r600 ) { return <STR_LIT> <STR_LIT> ; } return <STR_LIT> <STR_LIT> <STR_LIT>


<s> addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( isPassEnabled ( EnableSetWavePriority , CodeGenOpt :: Less ) ) addPass ( create<ISA_LIT>SetWavePriorityPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>PreEmitPeepholeID ) ; addPass ( &


<s> addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( isPassEnabled ( EnableSetWavePriority , CodeGenOpt :: Less

) ) addPass ( create<ISA_LIT>SetWavePriorityPass ( ) ) ;


<s> DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation (


<s> static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler ( MachineSchedContext * C ) { ScheduleDAGMILive * DAG = new GCNScheduleDAGMILive ( C , std :: make_unique

< GCNMaxOccupancySchedStrategy > ( C ) ) ;


<s> DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( ST . createFillMFMAShadowMutation ( DAG -> TII ) ) ; DAG -> addMutation ( createMFMAIGroupLPDAGMutation (


<s> case PseudoSourceValue :: GOT : case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue ::


<s> if ( parseNamedRegisterReference ( PFS , RegVal , RegName . Value , Error ) ) { SourceRange = RegName . SourceRange ; return true ; } return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs , unsigned SystemSGPRs ) { if ( ! A ) return false ; if ( A -> IsRegister ) { unsigned Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error


<s> MFI -> Occupancy = ST . computeOccupancy ( MF . getFunction ( ) , MFI -> getLDSSize ( ) ) ; } auto parseRegister = [ & ] ( const yaml :: StringValue & RegName , Register & RegVal ) { Register TempReg ; if ( parseNamedRegisterReference ( PFS , TempReg , RegName . Value , Error ) ) { SourceRange = RegName . SourceRange ; return true ; } RegVal = TempReg ; return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs


<s> if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A -> RegisterName ) ; Arg = ArgDescriptor :: createRegister ( Reg ) ; } else Arg = ArgDescriptor :: createStack ( A -> StackOffset ) ; if ( A -> Mask ) Arg = ArgDescriptor :: createArg ( Arg , A -> Mask . getValue ( ) ) ; MFI -> NumUserSGPRs += UserSGPRs ; MFI -> NumSystemSGPRs += SystemSGPRs ; return false ; } ; if ( YamlMFI . ArgInfo && ( parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentBuffer , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentBuffer , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchPtr , <STR_LIT> , MFI -> ArgInfo . DispatchPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> QueuePtr , <STR_LIT> , MFI -> ArgInfo . QueuePtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> KernargSegmentPtr , <STR_LIT> , MFI -> ArgInfo . KernargSegmentPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchID , <STR_LIT> , MFI -> ArgInfo . DispatchID , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> FlatScratchInit , <STR_LIT> , MFI -> ArgInfo . FlatScratchInit , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentSize , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentSize , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDX , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDY , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDZ , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDZ , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupInfo , <STR_LIT> , MFI -> ArgInfo . WorkGroupInfo , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentWaveByteOffset , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentWaveByteOffset , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitArgPtr

, <STR_LIT> , MFI -> ArgInfo . ImplicitArgPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitBufferPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitBufferPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDX , <STR_LIT> , MFI -> ArgInfo . WorkItemIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDY , <STR_LIT> , MFI -> ArgInfo . WorkItemIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDZ , <STR_LIT> , MFI -> ArgInfo . WorkItemIDZ , <NUM_LIT> , <NUM_LIT> ) ) ) return true ;


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass (


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>RemoveShortExecBranchesID ) ; addPass ( & <ISA_LIT>PreEmitPeepholeID


<s> if ( isPassEnabled ( EnablePreRAOptimizations ) ) insertPass ( & RenameIndependentSubregsID , & GCNPreRAOptimizationsID ) ; if ( TM -> getOptLevel ( ) > CodeGenOpt :: Less ) insertPass ( & MachineSchedulerID ,


<s> void GCNPassConfig :: addOptimizedRegAlloc ( ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>WholeQuadModeID ) ; insertPass ( & MachineSchedulerID , & <ISA_LIT>PreAllocateWWMRegsID ) ; if ( OptExecMaskPreRA ) insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; if ( isPassEnabled ( EnablePreRAOptimizations ) ) insertPass ( & RenameIndependentSubregsID , & GCNPreRAOptimizationsID ) ; if ( TM -> getOptLevel ( ) > CodeGenOpt :: Less ) insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; if ( OptVGPRLiveRange ) insertPass ( & LiveVariablesID , & <ISA_LIT>OptimizeVGPRLiveRangeID , false ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; if ( EnableDCEInRA ) insertPass ( & DetectDeadLanesID , & DeadMachineInstructionElimID


<s> bool EnableOpt = getOptLevel ( ) > CodeGenOpt :: None ; bool Internalize = InternalizeSymbols ; bool EarlyInline = EarlyInlineAll && EnableOpt && ! EnableFunctionCalls ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && EnableOpt ; bool LibCallSimplify = EnableLibCallSimplify && EnableOpt ; if ( EnableFunctionCalls ) { delete Builder . Inliner ; Builder . Inliner = createFunctionInliningPass ( ) ; } Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA , this ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass (


<s> static bool mustPreserveGV ( const GlobalValue & GV ) { if ( const Function * F = dyn_cast < Function > ( & GV ) ) return F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ; GV . removeDeadConstantUsers ( ) ; return ! GV .


<s> if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>AlwaysInlinePass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>ReplaceLDSUseWithPointerPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>LowerModuleLDSPass ( ) ) ; return true ; } return false ; } ) ; PB . registerPipelineParsingCallback ( [ this ] ( StringRef PassName , FunctionPassManager & PM , ArrayRef < PassBuilder :: PipelineElement > ) { if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>SimplifyLibCallsPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaToVectorPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>LowerKernelAttributesPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; return true ; } return false ; } ) ; PB . registerAnalysisRegistrationCallback ( [ ] ( FunctionAnalysisManager & FAM ) { FAM . registerPass ( [ & ] { return <ISA_LIT>AA ( ) ; } ) ; } ) ; PB . registerParseAACallback ( [ ] ( StringRef AAName , AAManager & AAM ) { if ( AAName == <STR_LIT> ) { AAM . registerFunctionAnalysis < <ISA_LIT>AA > ( ) ; return true ; } return false ; } ) ; PB . registerPipelineStartEPCallback ( [ this ] ( ModulePassManager & PM , OptimizationLevel Level ) { FunctionPassManager FPM ; FPM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; FPM . addPass (


) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ;


<s> } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaToVectorPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>LowerKernelAttributesPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; return true ; } return false ; } ) ; PB . registerAnalysisRegistrationCallback ( [ ] ( FunctionAnalysisManager & FAM ) { FAM . registerPass ( [ & ] { return <ISA_LIT>AA ( ) ; } ) ; } ) ; PB . registerParseAACallback ( [ ] ( StringRef AAName , AAManager & AAM ) { if ( AAName == <STR_LIT> ) { AAM . registerFunctionAnalysis < <ISA_LIT>AA > ( ) ; return true ; } return false ; } ) ; PB . registerPipelineStartEPCallback ( [ this ] ( ModulePassManager & PM , PassBuilder :: OptimizationLevel Level ) { FunctionPassManager FPM ; FPM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; FPM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ; if ( EnableLibCallSimplify && Level != PassBuilder :: OptimizationLevel :: O0 ) FPM . addPass ( <ISA_LIT>SimplifyLibCallsPass ( * this ) ) ; PM . addPass ( createModuleToFunctionPassAdaptor ( std :: move ( FPM ) ) ) ; } )


<s> disablePass ( & PatchableFunctionID ) ; addPass ( create<ISA_LIT>AlwaysInlinePass ( ) ) ; addPass ( createAlwaysInlinerPass ( ) ) ; addPass ( createBarrierNoopPass (


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( & PostRAHazardRecognizerID ) ; addPass ( create<ISA_LIT>InsertWaitsPass ( )


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , Subtarget ( TT , getTargetCPU ( ) , FS , * this ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true ) ; initAsmInfo ( )


<s> if ( ! RM . hasValue ( ) ) return Reloc :: PIC_ ; return *


<s> static Reloc :: Model getEffectiveRelocModel ( Optional < Reloc :: Model > RM ) { if ( ! RM . hasValue ( ) ) return Reloc :: PIC_ ; return * RM


<s> void LLVMInitialize<ISA_LIT>Target ( ) { RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ; RegisterTargetMachine < GCNTargetMachine > Y ( TheGCNTarget ) ; PassRegistry * PR = PassRegistry :: getPassRegistry ( ) ; initialize<ISA_LIT>LowerI1CopiesPass ( * PR ) ; initialize<ISA_LIT>FixSGPRCopiesPass ( * PR


<s> bool GCNPassConfig :: addPreISel ( ) { <ISA_LIT>PassConfig :: addPreISel ( ) ; addPass ( & <ISA_LIT>AnnotateKernelFeaturesID ) ; addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>TypeRewriter ( ) ) ; addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ; return


<s> addPass ( create<ISA_LIT>AlwaysInlinePass ( ) ) ; addPass ( createAlwaysInlinerPass ( )


<s> addPass ( createBarrierNoopPass ( ) ) ; TargetPassConfig :: addIRPasses (


<s> return TargetIRAnalysis ( [ this ] ( Function & F ) { return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this , F . getParent ( ) -> getDataLayout ( ) ) ) ; }


<s> RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ; RegisterTargetMachine < GCNTargetMachine > Y


<s> void LLVMInitialize<ISA_LIT>Target ( ) { RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ; RegisterTargetMachine < GCNTargetMachine > Y ( TheGCNTarget


<s> <ISA_LIT>TargetMachine :: ~ <ISA_LIT>TargetMachine (


( ) > CodeGenOpt :: Less ) addEarlyCSEOrGVNPass ( ) ;


<s> if ( TM -> getOptLevel ( ) > CodeGenOpt :: Less ) insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; if ( EnableDCEInRA ) insertPass ( & DetectDeadLanesID , & DeadMachineInstructionElimID ) ; TargetPassConfig :: addOptimizedRegAlloc ( )


<s> addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>PreEmitPeepholeID ) ; addPass ( & PostRAHazardRecognizerID


<s> addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>PreEmitPeepholeID ) ; addPass ( & PostRAHazardRecognizerID


<s> <ISA_LIT>PassConfig :: addPreISel ( ) ; addPass ( create<ISA_LIT>LateCodeGenPreparePass ( ) ) ; if ( EnableAtomicOptimizations ) { addPass ( create<ISA_LIT>AtomicOptimizerPass ( ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { if ( EnableStructurizerWorkarounds ) { addPass ( createFixIrreduciblePass ( ) ) ; addPass ( createUnifyLoopExitsPass ( ) ) ; } addPass ( createStructurizeCFGPass ( false ) ) ; } addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( !


<s> const <ISA_LIT>TargetMachine & TM = get<ISA_LIT>TargetMachine ( ) ; if ( TM . getOptLevel ( ) > CodeGenOpt :: None ) { addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; } addStraightLineScalarOptimizationPasses ( ) ; TargetPassConfig :: addIRPasses


<s> const <ISA_LIT>TargetMachine & TM = get<ISA_LIT>TargetMachine ( ) ; if ( TM . getOptLevel ( ) > CodeGenOpt :: None ) { addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; } addStraightLineScalarOptimizationPasses ( ) ; TargetPassConfig :: addIRPasses ( ) ; if ( getOptLevel ( ) != CodeGenOpt ::


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ; insertPass ( & RenameIndependentSubregsID , & <ISA_LIT>LowerControlFlowID ) ; TargetPassConfig :: addOptimizedRegAlloc ( RegAllocPass )


<s> if ( getOptLevel ( ) > CodeGenOpt :: None ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>LoadStoreOptimizerID ) ; insertPass ( & MachineSchedulerID , & RegisterCoalescerID


<s> return <STR_LIT> <STR_LIT> ; } if ( TT . getEnvironmentName ( ) == <STR_LIT> || TT . getEnvironmentName ( )


<s> <ISA_LIT>PassConfig :: addInstSelector ( ) ; addPass ( & <ISA_LIT>FixSGPRCopiesID ) ; addPass ( create<ISA_LIT>LowerI1CopiesPass ( )


<s> ScheduleDAGMI * DAG = createGenericSchedPostRA ( C ) ; const GCNSubtarget & ST = C -> MF -> getSubtarget < GCNSubtarget > ( ) ; DAG -> addMutation ( ST . createFillMFMAShadowMutation ( DAG ->


<s> void GCNPassConfig :: addFastRegAlloc ( ) { insertPass ( & PHIEliminationID , &


<s> insertPass ( & <ISA_LIT>LowerControlFlowID , & <ISA_LIT>FixWWMLivenessID , false ) ; TargetPassConfig :: addFastRegAlloc


<s> } insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & <ISA_LIT>LowerControlFlowID , &


<s> bool GCNPassConfig :: addGlobalInstructionSelect ( ) { addPass ( new InstructionSelect ( ) ) ; addPass ( create<ISA_LIT>AddIMGInitPass ( ) ) ; return false


<s> return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs , unsigned SystemSGPRs ) { if ( ! A ) return false ; if ( A -> IsRegister ) { Register Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; } if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass (


<s> if ( TT . getEnvironmentName ( ) == <STR_LIT> || TT . getEnvironmentName


<s> addPass ( & PostRAHazardRecognizerID ) ; addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>LowerControlFlowPass ( ) ,


<s> addPass ( & PostRAHazardRecognizerID ) ; addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>LowerControlFlowPass ( )


<s> void <ISA_LIT>PassConfig :: addIRPasses ( ) { const <ISA_LIT>TargetMachine & TM = get<ISA_LIT>TargetMachine ( ) ; disablePass ( & StackMapLivenessID ) ; disablePass ( & FuncletLayoutID ) ; disablePass ( & PatchableFunctionID ) ; addPass ( createAtomicExpandPass ( ) ) ; addPass ( create<ISA_LIT>LowerIntrinsicsPass ( ) ) ; addPass ( create<ISA_LIT>AlwaysInlinePass ( ) ) ; addPass ( createAlwaysInlinerLegacyPass ( ) ) ; addPass ( createBarrierNoopPass ( ) ) ; if ( TM . getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) { addPass ( create<ISA_LIT>CodeGenPreparePass (


<s> <ISA_LIT>PassConfig :: addInstSelector ( ) ; addPass ( create<ISA_LIT>LowerI1CopiesPass ( )


<s> addPass ( create<ISA_LIT>LowerI1CopiesPass ( ) ) ; addPass ( create<ISA_LIT>FixSGPRCopiesPass ( * TM


<s> void GCNPassConfig :: addPostRegAlloc ( ) { addPass ( create<ISA_LIT>PrepareScratchRegs ( )


<s> if ( TT . getOS ( ) == Triple :: AMDHSA

) return make_unique < <ISA_LIT>HSATargetObjectFile > ( ) ;


<s> if ( TT . getOS ( ) == Triple :: AMDHSA ) return make_unique < <ISA_LIT>HSATargetObjectFile >


<s> void LLVMInitialize<ISA_LIT>Target ( ) { RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ; RegisterTargetMachine < GCNTargetMachine > Y ( TheGCNTarget ) ; PassRegistry * PR = PassRegistry ::


<s> if ( Kind . isReadOnly ( ) && <STR_LIT> ( GV ) && TM . getTargetTriple ( ) . getOS ( ) != Triple :: AMDHSA ) return TextSection ; return TargetLoweringObjectFileELF :: SelectSectionForGlobal ( GV ,


<s> MCSection * <ISA_LIT>TargetObjectFile :: SelectSectionForGlobal ( const GlobalValue * GV , SectionKind Kind , Mangler & Mang , const TargetMachine & TM ) const { if ( Kind . isReadOnly ( ) && <STR_LIT> ( GV ) && TM . getTargetTriple ( ) . getOS ( ) != Triple :: AMDHSA ) return TextSection ; return TargetLoweringObjectFileELF :: SelectSectionForGlobal ( GV ,


<s> if ( Kind . isReadOnly ( ) && <STR_LIT> ( GO , AS ) && <STR_LIT> ( TM . getTargetTriple ( ) ) ) return TextSection ; return TargetLoweringObjectFileELF :: SelectSectionForGlobal ( GO , Kind


<s> MCSection * <ISA_LIT>TargetObjectFile :: getExplicitSectionGlobal ( const GlobalObject * GO , SectionKind


<s> MCSection * <ISA_LIT>TargetObjectFile :: SelectSectionForGlobal ( const GlobalValue * GV , SectionKind Kind , Mangler & Mang , const TargetMachine & TM ) const { if ( Kind . isReadOnly ( ) && <STR_LIT> ( GV ) ) return TextSection ; return TargetLoweringObjectFileELF :: SelectSectionForGlobal ( GV ,


<s> void <ISA_LIT>HSATargetObjectFile :: Initialize ( MCContext & Ctx , const TargetMachine & TM ) { TargetLoweringObjectFileELF :: Initialize ( Ctx , TM ) ; InitializeELF ( TM . Options . UseInitArray ) ; TextSection = <STR_LIT> ( Ctx ) ; DataGlobalAgentSection = <STR_LIT> (


<s> return <STR_LIT> ( GV ) || ( <STR_LIT> ( GV ) && GV ->

hasSection ( ) && isAgentAllocationSection ( GV -> getSection ( ) ) ) ;


<s> bool <ISA_LIT>HSATargetObjectFile :: isAgentAllocationSection ( const char * SectionName ) const { return cast < MCSectionELF > ( DataGlobalAgentSection ) -> getSectionName ( ) . equals (


<s> bool <ISA_LIT>HSATargetObjectFile :: isProgramAllocation ( const GlobalValue * GV ) const { return <STR_LIT> ( GV ) && !


<s> bool <ISA_LIT>HSATargetObjectFile :: isProgramAllocation ( const GlobalValue * GV ) const { return <STR_LIT> ( GV ) &&


<s> MCSection * <ISA_LIT>HSATargetObjectFile :: SelectSectionForGlobal ( const GlobalValue * GV , SectionKind Kind , Mangler & Mang , const TargetMachine & TM ) const { if ( Kind . isText ( ) && ! GV -> hasComdat ( ) ) return getTextSection ( ) ; if ( <STR_LIT> ( GV ) ) { if ( isAgentAllocation ( GV ) ) return DataGlobalAgentSection ; if ( isProgramAllocation ( GV ) ) return DataGlobalProgramSection ; } return <ISA_LIT>TargetObjectFile :: SelectSectionForGlobal ( GV , Kind , Mang , TM


<s> StringRef SectionName = GO -> getSection ( ) ; if ( SectionName . startswith ( <STR_LIT> ) ) SK = SectionKind :: getMetadata ( ) ; return TargetLoweringObjectFileELF :: getExplicitSectionGlobal ( GO , SK , TM


<s> StringRef SectionName = GO -> getSection ( ) ; if ( SectionName . startswith ( <STR_LIT> ) ) SK = SectionKind :: getMetadata ( ) ; return TargetLoweringObjectFileELF :: getExplicitSectionGlobal ( GO


<s> MCSection * <ISA_LIT>TargetObjectFile :: SelectSectionForGlobal ( const GlobalObject * GO , SectionKind Kind , const TargetMachine & TM ) const { if ( Kind . isReadOnly ( ) && <STR_LIT> ( GO ) && <STR_LIT> ( TM . getTargetTriple ( ) ) ) return TextSection ; return TargetLoweringObjectFileELF :: SelectSectionForGlobal ( GO ,


<s> Symbol -> setType ( ELF :: STT_OBJECT ) ; Symbol -> setBinding ( ELF


<s> MCSymbolELF * Symbol = cast < MCSymbolELF > ( getStreamer ( ) . getContext ( ) . getOrCreateSymbol ( GlobalName ) ) ; Symbol -> setType ( ELF :: STT_OBJECT ) ; Symbol -> setBinding ( ELF :: STB_LOCAL


<s> S . EmitValue ( DescSZ , <NUM_LIT> ) ; S . EmitIntValue ( Type , <NUM_LIT> ) ; S . EmitBytes ( StringRef ( PT_NOTE :: NoteName , NameSZ ) ) ; S . EmitValueToAlignment ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) ; EmitDesc ( S ) ; S . EmitValueToAlignment ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) ; S . PopSection


<s> void <ISA_LIT>TargetELFStreamer :: Emit<ISA_LIT>SymbolType ( StringRef SymbolName , unsigned Type ) { MCSymbolELF * Symbol = cast < MCSymbolELF > ( getStreamer ( ) . getContext ( ) . getOrCreateSymbol ( SymbolName ) ) ; Symbol -> setType ( ELF


<s> void <ISA_LIT>TargetELFStreamer :: EmitAMDKernelCodeT ( const amd_kernel_code_t & Header ) { MCStreamer & OS = getStreamer ( ) ; OS . PushSection ( ) ; OS . EmitBytes ( StringRef ( ( const char * ) & Header , sizeof ( Header ) ) ) ; OS . PopSection (


<s> void <ISA_LIT>TargetELFStreamer :: EmitRuntimeMetadata ( Module & M


<s> <ISA_LIT>PALMetadata * getPALMetadata ( ) { return & PALMetadata


<s> case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_REDWOOD : AK = GK_REDWOOD ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_SUMO : AK = GK_SUMO ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_BARTS : AK = GK_BARTS ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAICOS : AK = GK_CAICOS ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAYMAN : AK = GK_CAYMAN ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_TURKS : AK = GK_TURKS ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX600 : AK = GK_GFX600 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX601 : AK = GK_GFX601 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX700 : AK = GK_GFX700 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX701 : AK = GK_GFX701 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX702 : AK = GK_GFX702 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX703 : AK = GK_GFX703 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX704 : AK = GK_GFX704 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX801 : AK = GK_GFX801 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX802 : AK = GK_GFX802 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX803 : AK = GK_GFX803 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX810 : AK = GK_GFX810 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX900 : AK = GK_GFX900 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX902 : AK = GK_GFX902 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX904 : AK = GK_GFX904 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX906 : AK = GK_GFX906 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX908 : AK = GK_GFX908 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX909 : AK = GK_GFX909 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1010 : AK = GK_GFX1010 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1011 : AK = GK_GFX1011 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1012 : AK = GK_GFX1012 ; break ; case ELF :: EF_<ISA_LIT>_MACH_NONE : AK = GK_NONE


<s> case GK_GFX700 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX700 ; case GK_GFX701 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX701 ; case GK_GFX702 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX702 ; case GK_GFX703 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX703 ; case GK_GFX704 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX704 ; case GK_GFX801 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX801 ; case GK_GFX802 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX802 ; case GK_GFX803 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX803 ; case GK_GFX810 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX810 ; case GK_GFX900 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX900 ; case GK_GFX902 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX902 ; case GK_GFX904 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX904 ; case GK_GFX906 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX906 ; case GK_GFX908 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX908 ; case GK_GFX909 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX909 ; case GK_GFX1010 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1010 ; case GK_GFX1011 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1011 ; case GK_GFX1012 : return ELF ::


<s> case GK_RV730 : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_RV730 ; case GK_RV770 : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_RV770 ; case GK_CEDAR : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CEDAR ; case GK_CYPRESS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CYPRESS ; case GK_JUNIPER : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_JUNIPER ; case GK_REDWOOD : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_REDWOOD ; case GK_SUMO : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_SUMO ; case GK_BARTS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_BARTS ; case GK_CAICOS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAICOS ; case GK_CAYMAN : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAYMAN ; case GK_TURKS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_TURKS ; case GK_GFX600 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX600 ; case GK_GFX601 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX601 ; case GK_GFX700 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX700 ; case GK_GFX701 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX701 ; case GK_GFX702 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX702 ; case GK_GFX703 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX703 ; case GK_GFX704 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX704 ; case GK_GFX801 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX801 ; case GK_GFX802 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX802 ; case GK_GFX803 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX803 ; case GK_GFX810 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX810 ; case GK_GFX900 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX900 ; case GK_GFX902 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX902 ; case GK_GFX904 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX904 ; case GK_GFX906 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX906 ; case GK_GFX908 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX908 ; case GK_GFX909 : return ELF


<s> Optional < <STR_LIT> :: <ISA_LIT>TargetID > & getTargetID ( ) { return


<s> initializeTargetID ( STI ) ; assert ( getTargetID ( ) != None && <STR_LIT> ) ; getTargetID ( ) -> setTargetIDFromFeaturesString ( FeatureString )


<s> void initializeTargetID ( const MCSubtargetInfo & STI , StringRef FeatureString ) { initializeTargetID ( STI )


<s> case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_BARTS : AK = GK_BARTS ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAICOS : AK = GK_CAICOS ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAYMAN : AK = GK_CAYMAN ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_TURKS : AK = GK_TURKS ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX600 : AK = GK_GFX600 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX601 : AK = GK_GFX601 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX602 : AK = GK_GFX602 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX700 : AK = GK_GFX700 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX701 : AK = GK_GFX701 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX702 : AK = GK_GFX702 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX703 : AK = GK_GFX703 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX704 : AK = GK_GFX704 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX705 : AK = GK_GFX705 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX801 : AK = GK_GFX801 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX802 : AK = GK_GFX802 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX803 : AK = GK_GFX803 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX805 : AK = GK_GFX805 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX810 : AK = GK_GFX810 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX900 : AK =


<s> assert ( STI . getTargetTriple ( ) . getOS ( ) == Triple :: AMDHSA ) ; if ( Optional < uint8_t > HsaAbiVer = getHsaAbiVersion ( & STI ) ) { switch ( * HsaAbiVer ) { case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V2 : case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V3 : return getEFlagsV3 (


<s> switch ( * HsaAbiVer ) { case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V2 : case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V3 : return getEFlagsV3 ( ) ; case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V4 : return getEFlagsV4 ( ) ; } } llvm_unreachable (


<s> unsigned <ISA_LIT>TargetStreamer :: getElfMach ( StringRef GPU ) { <STR_LIT> AK = parseArchAMDGCN ( GPU ) ; if ( AK == <STR_LIT> :: GK_NONE ) AK = parseArch<ISA_LIT> ( GPU ) ; switch ( AK ) { case GK_<ISA_LIT> : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_<ISA_LIT> ; case GK_R630 : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_R630 ; case GK_RS880 : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_RS880 ; case GK_RV670 : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_RV670 ; case GK_RV710 : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_RV710 ; case GK_RV730 : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_RV730 ; case GK_RV770 : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_RV770 ; case GK_CEDAR : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CEDAR ; case GK_CYPRESS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CYPRESS ; case GK_JUNIPER : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_JUNIPER ; case GK_REDWOOD : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_REDWOOD ; case GK_SUMO : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_SUMO ; case GK_BARTS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_BARTS ; case GK_CAICOS : return

ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAICOS ;


<s> <ISA_LIT>TargetELFStreamer :: <ISA_LIT>TargetELFStreamer ( MCStreamer & S , const MCSubtargetInfo & STI ) : <ISA_LIT>TargetStreamer ( S ) , Streamer ( S ) { MCAssembler & MCA = getStreamer ( ) . getAssembler ( ) ; unsigned EFlags = MCA . getELFHeaderEFlags ( ) ; EFlags &= ~ ELF :: EF_<ISA_LIT>_MACH ; EFlags |= getElfMach ( STI . getCPU ( ) ) ; EFlags &= ~ ELF :: EF_<ISA_LIT>_XNACK ; if ( <STR_LIT> ( STI ) ) EFlags |= ELF :: EF_<ISA_LIT>_XNACK ; EFlags &= ~


<s> if ( KernelCodeSymbol -> getVisibility ( ) == ELF :: STV_DEFAULT ) KernelCodeSymbol -> setVisibility ( ELF :: STV_PROTECTED ) ; Streamer . EmitLabel ( KernelDescriptorSymbol ) ; Streamer . EmitBytes ( StringRef ( ( const char * ) & ( KernelDescriptor ) , offsetof ( amdhsa :: kernel_descriptor_t , kernel_code_entry_byte_offset ) ) ) ; Streamer . EmitValue ( MCBinaryExpr :: createSub ( MCSymbolRefExpr :: create ( KernelCodeSymbol , MCSymbolRefExpr :: VK_<ISA_LIT>_REL64 , Context ) , MCSymbolRefExpr :: create ( KernelDescriptorSymbol , MCSymbolRefExpr :: VK_None , Context ) , Context ) , sizeof (

KernelDescriptor . kernel_code_entry_byte_offset ) ) ;


<s> OS . EmitValueToAlignment ( <NUM_LIT> , Encoded_s_code_end , <NUM_LIT> ) ; for ( unsigned I = <NUM_LIT> ; I < <NUM_LIT> ; ++ I ) OS .

EmitIntValue ( Encoded_s_code_end , <NUM_LIT> ) ;


<s> auto * DescBegin = Context . createTempSymbol ( ) ; auto * DescEnd = Context . createTempSymbol ( ) ; auto * DescSZ = MCBinaryExpr :: createSub ( MCSymbolRefExpr :: create ( DescEnd , Context ) , MCSymbolRefExpr :: create ( DescBegin , Context ) , Context ) ; EmitNote ( ElfNote :: NoteNameV2 , DescSZ , ELF :: NT_AMD_<ISA_LIT>_HSA_METADATA , [ & ] ( MCELFStreamer & OS ) { OS . EmitLabel ( DescBegin ) ; OS . EmitBytes


<s> EmitNote ( ElfNote :: NoteNameV2 , DescSZ , ELF :: NT_AMD_<ISA_LIT>_ISA , [ & ] ( MCELFStreamer & OS ) { OS . EmitLabel ( DescBegin ) ; OS . EmitBytes ( IsaVersionString ) ; OS . EmitLabel


<s> getPALMetadata ( ) -> toBlob ( Type , Blob ) ; if ( Blob . empty ( ) ) return ; EmitNote ( Vendor , MCConstantExpr :: create ( Blob . size ( ) , getContext ( ) ) , Type , [ & ] ( MCELFStreamer

& OS ) { OS . EmitBytes ( Blob ) ; } ) ;


<s> EmitRuntimeMetadata ( getRuntimeMDYAMLString ( Features , M )


<s> case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX904 : AK = GK_GFX904 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX906 : AK = GK_GFX906 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX908 : AK = GK_GFX908 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX909 : AK = GK_GFX909 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX90A : AK = GK_GFX90A ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX90C : AK = GK_GFX90C ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX940 : AK = GK_GFX940 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1010 : AK = GK_GFX1010 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1011 : AK = GK_GFX1011 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1012 : AK = GK_GFX1012 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1013 : AK = GK_GFX1013 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1030 : AK = GK_GFX1030 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1031 : AK = GK_GFX1031 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1032 : AK = GK_GFX1032 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1033 : AK = GK_GFX1033 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1034 : AK = GK_GFX1034 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1035 : AK = GK_GFX1035 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1036 : AK = GK_GFX1036 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1100 : AK = GK_GFX1100 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1101 : AK = GK_GFX1101 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1102 : AK = GK_GFX1102 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1103 : AK = GK_GFX1103 ; break ; case ELF :: EF_<ISA_LIT>_MACH_NONE : AK = GK_NONE ; break ; } StringRef GPUName = getArchNameAMDGCN ( AK


<s> case GK_CYPRESS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CYPRESS ; case GK_JUNIPER : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_JUNIPER ; case GK_REDWOOD : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_REDWOOD ; case GK_SUMO : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_SUMO ; case GK_BARTS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_BARTS ; case GK_CAICOS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAICOS ; case GK_CAYMAN : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAYMAN ; case GK_TURKS : return ELF :: EF_AMDGPU_MACH_<ISA_LIT>_TURKS ; case GK_GFX600 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX600 ; case GK_GFX601 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX601 ; case GK_GFX602 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX602 ; case GK_GFX700 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX700 ; case GK_GFX701 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX701 ; case GK_GFX702 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX702 ; case GK_GFX703 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX703 ; case GK_GFX704 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX704 ; case GK_GFX705 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX705 ; case GK_GFX801 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX801 ; case GK_GFX802 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX802 ; case GK_GFX803 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX803 ; case GK_GFX805 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX805 ; case GK_GFX810 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX810 ; case GK_GFX900 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX900 ; case GK_GFX902 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX902 ; case GK_GFX904 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX904 ; case GK_GFX906 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX906 ; case GK_GFX908 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX908 ; case GK_GFX909 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX909 ; case GK_GFX90A : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX90A ; case GK_GFX90C : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX90C ; case GK_GFX940 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX940 ; case GK_GFX1010 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1010 ; case GK_GFX1011 : return ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX1011


<s> case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CYPRESS : AK = GK_CYPRESS ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_JUNIPER : AK = GK_JUNIPER ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_REDWOOD : AK = GK_REDWOOD ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_SUMO : AK = GK_SUMO ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_BARTS : AK = GK_BARTS ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAICOS : AK = GK_CAICOS ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_CAYMAN : AK = GK_CAYMAN ; break ; case ELF :: EF_AMDGPU_MACH_<ISA_LIT>_TURKS : AK = GK_TURKS ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX600 : AK = GK_GFX600 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX601 : AK = GK_GFX601 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX700 : AK = GK_GFX700 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX701 : AK = GK_GFX701 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX702 : AK = GK_GFX702 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX703 : AK = GK_GFX703 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX704 : AK = GK_GFX704 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX801 : AK = GK_GFX801 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX802 : AK = GK_GFX802 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX803 : AK = GK_GFX803 ; break ; case ELF :: EF_<ISA_LIT>_MACH_AMDGCN_GFX810 : AK


<s> S . SwitchSection ( Context . getELFSection ( ElfNote :: SectionName , ELF :: SHT_NOTE , ELF :: SHF_ALLOC ) ) ; S . emitInt32 ( NameSZ ) ; S . emitValue ( DescSZ , <NUM_LIT> ) ; S . emitInt32 ( NoteType ) ; S . emitBytes ( Name ) ; S . emitValueToAlignment ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) ; EmitDesc ( S


<s> EmitNote ( Vendor , MCConstantExpr :: create ( Blob . size ( ) , getContext ( ) ) , Type , [ & ] ( MCELFStreamer & OS ) { OS .


<s> if ( SymbolELF -> declareCommon ( Size , Alignment . value ( ) , true ) ) { report_fatal_error ( <STR_LIT> + Symbol -> getName ( ) + <STR_LIT> ) ; } SymbolELF -> setIndex ( ELF


<s> if ( SymbolELF -> declareCommon ( Size , Alignment . value ( ) , true ) ) { report_fatal_error ( <STR_LIT> + Symbol -> getName ( ) + <STR_LIT> ) ; } SymbolELF -> setIndex ( ELF :: SHN_<ISA_LIT>_LDS ) ; SymbolELF -> setSize ( MCConstantExpr :: create ( Size , getContext (


<s> Streamer . emitLabel ( KernelDescriptorSymbol ) ; Streamer . emitInt32 ( KernelDescriptor . group_segment_fixed_size ) ; Streamer . emitInt32 ( KernelDescriptor . private_segment_fixed_size ) ; Streamer . emitInt32 ( KernelDescriptor . kernarg_size ) ; for ( uint8_t Res : KernelDescriptor . reserved0 ) Streamer . emitInt8 ( Res ) ; Streamer . emitValue ( MCBinaryExpr :: createSub ( MCSymbolRefExpr :: create ( KernelCodeSymbol , MCSymbolRefExpr :: VK_<ISA_LIT>_REL64 , Context ) , MCSymbolRefExpr :: create ( KernelDescriptorSymbol , MCSymbolRefExpr :: VK_None , Context ) , Context ) , sizeof ( KernelDescriptor . kernel_code_entry_byte_offset ) ) ; for ( uint8_t Res : KernelDescriptor . reserved1 ) Streamer . emitInt8 ( Res ) ; Streamer . emitInt32 ( KernelDescriptor . compute_pgm_rsrc3 ) ; Streamer . emitInt32 ( KernelDescriptor . compute_pgm_rsrc1 ) ; Streamer . emitInt32 ( KernelDescriptor . compute_pgm_rsrc2 ) ; Streamer . emitInt16 ( KernelDescriptor . kernel_code_properties


<s> uint16_t VendorNameSize = VendorName . size ( ) + <NUM_LIT> ; uint16_t ArchNameSize = ArchName . size ( ) + <NUM_LIT> ; unsigned DescSZ = sizeof ( VendorNameSize ) + sizeof ( ArchNameSize ) + sizeof ( Major ) + sizeof ( Minor ) + sizeof ( Stepping ) + VendorNameSize + ArchNameSize ; convertIsaVersionV2 ( Major , Minor , Stepping , TargetID -> isSramEccOnOrAny ( ) , TargetID -> isXnackOnOrAny ( ) ) ; EmitNote ( ElfNote :: NoteNameV2 , MCConstantExpr :: create ( DescSZ , getContext ( ) ) ,

ELF :: NT_AMD_HSA_ISA_VER<ISA_LIT>ON , [ & ] ( MCELFStreamer & OS ) {


<s> void <ISA_LIT>TargetELFStreamer :: EmitDirectiveHSACodeObjectVersion ( uint32_t Major , uint32_t Minor ) { EmitNote ( ElfNote :: NoteNameV2 , MCConstantExpr :: create ( <NUM_LIT> , getContext ( ) ) , ELF :: NT_AMD_HSA_CODE_OBJECT_VER<ISA_LIT>ON , [


<s> bool <ISA_LIT>TargetELFStreamer :: EmitHSAMetadata ( const <STR_LIT> :: Metadata & HSAMetadata ) { std :: string HSAMetadataString ; if ( HSAMD :: toString ( HSAMetadata , HSAMetadataString ) ) return false ; auto & Context = getContext ( ) ; auto * DescBegin = Context . createTempSymbol ( ) ; auto * DescEnd = Context . createTempSymbol ( ) ; auto * DescSZ = MCBinaryExpr :: createSub ( MCSymbolRefExpr :: create ( DescEnd , Context ) , MCSymbolRefExpr :: create ( DescBegin , Context ) , Context ) ; EmitNote ( ElfNote :: NoteNameV2 , DescSZ , ELF :: NT_AMD_HSA_METADATA , [ & ] ( MCELFStreamer & OS ) { OS . emitLabel ( DescBegin ) ; OS . emitBytes ( HSAMetadataString ) ; OS . emitLabel