<s> GCNPassConfig ( LLVMTargetMachine & TM , PassManagerBase & PM ) : <ISA_LIT>PassConfig ( TM , PM ) { setRequiresCodeGenSCCOrder ( true ) ;

substitutePass ( & PostRASchedulerID , & PostMachineSchedulerID ) ;


<s> GCNTargetMachine :: GCNTargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OL , bool JIT ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) {


<s> assert ( V -> getType ( ) -> isPointerTy ( ) && V -> getType ( ) -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) ; const auto * Ptr = LD -> getPointerOperand ( ) ;

if ( Ptr -> getType ( ) -> getPointerAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return <ISA_LIT>AS :: UNKNOWN_ADDRESS_SPACE ;


<s> if ( ! LD ) return <ISA_LIT>AS :: UNKNOWN_ADDRESS_SPACE ; assert ( V -> getType ( ) -> isPointerTy ( ) && V -> getType ( ) -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) ; const auto * Ptr = LD -> getPointerOperand ( ) ; if ( Ptr -> getType ( ) -> getPointerAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return <ISA_LIT>AS :: UNKNOWN_ADDRESS_SPACE ; return <ISA_LIT>AS :: GLOBAL_ADDRESS ;


<s> static Reloc :: Model getEffectiveRelocModel ( Optional < Reloc :: Model > RM ) { return Reloc :: PIC_ ;


<s> StringRef <ISA_LIT>TargetMachine :: getFeatureString ( const Function & F ) const {

Attribute FSAttr = F . getFnAttribute ( <STR_LIT> ) ;


<s> StringRef <ISA_LIT>TargetMachine :: getFeatureString ( const Function & F ) const { Attribute FSAttr = F . getFnAttribute ( <STR_LIT> ) ; return FSAttr . isValid ( ) ? FSAttr . getValueAsString ( ) : getTargetFeatureString ( ) ;


<s> return getTM < GCNTargetMachine > ( ) ;


<s> Attribute GPUAttr = F . getFnAttribute ( <STR_LIT> ) ; return GPUAttr . isValid ( ) ? GPUAttr . getValueAsString ( ) : getTargetCPU ( ) ;


<s> if ( ! GPU . empty ( ) ) return GPU ;

if ( TT . getArch ( ) == Triple :: amdgcn ) return TT . getOS ( ) == Triple :: AMDHSA ? <STR_LIT> : <STR_LIT> ;


<s> const <ISA_LIT>IntrinsicInfo * getIntrinsicInfo ( ) const override {


<s> return ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ? - <NUM_LIT> : <NUM_LIT> ;


<s> uint64_t getNullPointerValue ( unsigned AddrSpace ) const {

return ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ? - <NUM_LIT> : <NUM_LIT> ;


<s> TargetLoweringObjectFile * getObjFileLowering ( ) const override { return TLOF . get ( ) ;


<s> const TargetSubtargetInfo * GCNTargetMachine :: getSubtargetImpl ( const Function & F ) const { StringRef GPU = getGPUName ( F ) ; StringRef FS = getFeatureString ( F ) ; SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ;

auto & I = SubtargetMap [ SubtargetKey ] ;


<s> return TargetTransformInfo ( GCNTTIImpl ( this , F ) ) ;


<s> RegisterRegAlloc :: FunctionPassCtor Ctor = SGPRRegisterRegAlloc :: getDefault ( ) ; if ( ! Ctor ) {


<s> if ( ! Ctor ) { Ctor = VGPRRegAlloc ; VGPRRegisterRegAlloc :: setDefault ( VGPRRegAlloc ) ; }


<s> bool isMachineVerifierClean ( ) const override {


<s> return <STR_LIT> ( SrcAS ) && <STR_LIT> ( DestAS ) ;


<s> bool <ISA_LIT>TargetMachine :: isNoopAddrSpaceCast ( unsigned SrcAS , unsigned DestAS ) const { return <STR_LIT> ( SrcAS ) && <STR_LIT> ( DestAS ) ;


<s> static bool mustPreserveGV ( const GlobalValue & GV ) { if ( const Function * F = dyn_cast < Function > ( & GV ) ) return F -> isDeclaration ( ) || F -> getName ( ) . startswith ( <STR_LIT> ) || F -> getName ( ) . startswith ( <STR_LIT> ) || <STR_LIT> ( F -> getCallingConv ( ) ) ;


<s> static bool onlyAllocateSGPRs ( const TargetRegisterInfo & TRI , const TargetRegisterClass & RC ) {

return static_cast < const <ISA_LIT>RegisterInfo & > ( TRI ) . isSGPRClass ( & RC ) ;


<s> return ! static_cast < const <ISA_LIT>RegisterInfo & > ( TRI ) . isSGPRClass ( & RC ) ;


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> initializeBaseYamlFields ( YamlMFI , MF , PFS , Error , SourceRange ) ) return true ; if ( MFI -> Occupancy == <NUM_LIT> ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; MFI -> Occupancy = ST . computeOccupancy ( MF . getFunction ( ) , MFI -> getLDSSize ( ) ) ; } auto parseRegister = [ & ] ( const yaml :: StringValue & RegName , Register & RegVal ) { Register TempReg ; if ( parseNamedRegisterReference ( PFS , TempReg , RegName . Value , Error ) ) { SourceRange = RegName . SourceRange ; return true ; } RegVal = TempReg ; return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs , unsigned SystemSGPRs ) { if ( ! A ) return false ; if ( A -> IsRegister ) { Register Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; }

if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A -> RegisterName ) ;


<s> return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaToVectorPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>LowerKernelAttributesPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteKernelArgumentsPass ( ) ) ; return true ; } return false ; } ) ; PB . registerAnalysisRegistrationCallback ( [ ] ( FunctionAnalysisManager & FAM ) { FAM . registerPass ( [ & ] { return <ISA_LIT>AA ( ) ; } ) ; } ) ; PB . registerParseAACallback ( [ ] ( StringRef AAName , AAManager & AAM ) { if ( AAName == <STR_LIT> ) { AAM . registerFunctionAnalysis < <ISA_LIT>AA > ( ) ; return true ; } return false ; } ) ; PB . registerPipelineStartEPCallback ( [ this ] ( ModulePassManager & PM , OptimizationLevel Level ) { FunctionPassManager FPM ; FPM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; FPM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ;

if ( EnableLibCallSimplify && Level != OptimizationLevel :: O0 ) FPM . addPass ( <ISA_LIT>SimplifyLibCallsPass ( * this ) ) ;


<s> SGPRRegisterRegAlloc ( const char * N , const char * D , FunctionPassCtor C ) : RegisterRegAllocBase ( N , D , C ) {


<s> bool useIPRA ( ) const override { return true ;


<s> VGPRRegisterRegAlloc ( const char * N , const char * D , FunctionPassCtor C ) : RegisterRegAllocBase ( N , D , C ) {


<s> if ( ST . isPromoteAllocaEnabled ( ) ) { addPass ( create<ISA_LIT>PromoteAlloca ( ST ) ) ; addPass ( createSROAPass ( ) ) ;


<s> addPass ( create<ISA_LIT>PromoteAlloca ( ST ) ) ; addPass ( createSROAPass ( ) ) ; } TargetPassConfig :: addCodeGenPrepare ( ) ;


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass * RegAllocPass ) { addPass ( & <ISA_LIT>FixSGPRLiveRangesID ) ;

TargetPassConfig :: addFastRegAlloc ( RegAllocPass ) ;


<s> bool <ISA_LIT>PassConfig :: addGCPasses ( ) { return false ;


<s> bool GCNPassConfig :: addInstSelector ( ) { <ISA_LIT>PassConfig :: addInstSelector ( ) ; addPass ( create<ISA_LIT>LowerI1CopiesPass ( ) ) ; addPass ( & <ISA_LIT>FixSGPRCopiesID ) ; addPass ( create<ISA_LIT>FoldOperandsPass ( ) ) ; return false ;


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) {

insertPass ( & LiveVariablesID , & <ISA_LIT>FixSGPRLiveRangesID , false ) ;


<s> void GCNPassConfig :: addPostRegAlloc ( ) { addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) , false ) ;


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>InsertWaits ( * TM ) , false ) ; addPass ( create<ISA_LIT>LowerControlFlowPass ( * TM ) , false ) ;


<s> addPass ( create<ISA_LIT>LowerControlFlowPass ( * TM ) , false ) ;


<s> addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; return false ;


<s> const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) {

insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ;


<s> void GCNPassConfig :: addPreSched2 ( ) {


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , CPU , FS , Options , RM , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , Subtarget ( TT , CPU , FS , * this ) , IntrinsicInfo ( ) {


<s> if ( TT . getArch ( ) == Triple :: amdgcn ) { Ret += <STR_LIT> ; } Ret += <STR_LIT> <STR_LIT> ; return Ret ;


<s> const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return create<ISA_LIT>MachineScheduler ( C ) ;


<s> return new GCNPassConfig ( this , PM ) ;


<s> return new ScheduleDAGMILive ( C , make_unique < <ISA_LIT>SchedStrategy > ( ) ) ;


<s> if ( TT . getOS ( ) == Triple :: AMDHSA ) return make_unique < <ISA_LIT>HSATargetObjectFile > ( ) ; return make_unique < <ISA_LIT>TargetObjectFile > ( ) ;


<s> static std :: unique_ptr < TargetLoweringObjectFile > createTLOF ( const Triple & TT ) {

if ( TT . getOS ( ) == Triple :: AMDHSA ) return make_unique < <ISA_LIT>HSATargetObjectFile > ( ) ;


<s> GCNPassConfig ( TargetMachine * TM , PassManagerBase & PM ) : <ISA_LIT>PassConfig ( TM , PM ) {


<s> GCNTargetMachine :: GCNTargetMachine ( const Target & T , const Triple & TT , StringRef FS , StringRef CPU , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) : <ISA_LIT>TargetMachine ( T , TT , FS , CPU , Options , RM , CM , OL ) {


<s> return getTM < <ISA_LIT>TargetMachine > ( ) ;


<s> return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this , F . getParent ( ) -> getDataLayout ( ) ) ) ; } ) ;


<s> void LLVMInitialize<ISA_LIT>Target ( ) { RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ; RegisterTargetMachine < GCNTargetMachine > Y ( TheGCNTarget ) ; PassRegistry * PR = PassRegistry :: getPassRegistry ( ) ;

initialize<ISA_LIT>LowerI1CopiesPass ( * PR ) ;


<s> <ISA_LIT>PassConfig ( TargetMachine * TM , PassManagerBase & PM ) : <ISA_LIT>PassConfig ( TM , PM ) {


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef FS , StringRef CPU , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) : <ISA_LIT>TargetMachine ( T , TT , FS , CPU , Options , RM , CM , OL ) {


<s> <ISA_LIT>TargetMachine :: ~ <ISA_LIT>TargetMachine ( ) {


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ;

TargetPassConfig :: addCodeGenPrepare ( ) ;


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ;

insertPass ( & <ISA_LIT>LowerControlFlowID , & <ISA_LIT>FixWWMLivenessID , false ) ;


<s> if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; addStraightLineScalarOptimizationPasses ( ) ; if ( Enable<ISA_LIT>AliasAnalysis ) { addPass ( create<ISA_LIT>AAWrapperPass ( ) ) ; addPass ( createExternalAAWrapperPass ( [ ] ( Pass & P , Function & , AAResults & AAR ) { if ( auto * WrapperPass = P . getAnalysisIfAvailable < <ISA_LIT>AAWrapperPass > ( ) ) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ; } ) ) ; } } TargetPassConfig :: addIRPasses ( ) ; if ( getOptLevel ( ) != CodeGenOpt :: None ) addEarlyCSEOrGVNPass ( ) ;


<s> void GCNPassConfig :: addMachineSSAOptimization ( ) { TargetPassConfig :: addMachineSSAOptimization ( ) ; addPass ( & <ISA_LIT>FoldOperandsID ) ; addPass ( & DeadMachineInstructionElimID ) ; addPass ( & <ISA_LIT>LoadStoreOptimizerID ) ; if ( EnableSDWAPeephole ) {


<s> insertPass ( & <ISA_LIT>LowerControlFlowID , & <ISA_LIT>FixWWMLivenessID , false ) ;

TargetPassConfig :: addOptimizedRegAlloc ( RegAllocPass ) ;


<s> insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & <ISA_LIT>LowerControlFlowID , & <ISA_LIT>FixWWMLivenessID , false ) ;


<s> void GCNPassConfig :: addPostRegAlloc ( ) {

addPass ( & <ISA_LIT>FixVGPRCopiesID ) ;


<s> addPass ( & <ISA_LIT>FixVGPRCopiesID ) ; addPass ( & <ISA_LIT>OptimizeExecMaskingID ) ;

TargetPassConfig :: addPostRegAlloc ( ) ;


<s> addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ;


<s> } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) { addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ; }


<s> } addPass ( create<ISA_LIT>WholeQuadModePass ( ) ) ;


addPass ( create<ISA_LIT>WholeQuadModePass ( ) ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , getEffectiveCodeModel ( CM ) , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) {


<s> if ( TT . getArch ( ) == Triple :: r600 ) { return <STR_LIT> <STR_LIT> ;


<s> return createExternalAAWrapperPass ( [ ] ( Pass & P , Function & , AAResults & AAR ) { if ( auto * WrapperPass = P . getAnalysisIfAvailable < <ISA_LIT>AAWrapperPass > ( ) ) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ; } ) ;


<s> ScheduleDAGMILive * DAG = new GCNScheduleDAGMILive ( C , make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ;

DAG -> addMutation ( createStoreClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ;


<s> DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( createStoreClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( create<ISA_LIT>MacroFusionDAGMutation ( ) ) ; return DAG ;


<s> auto DAG = new GCNIterativeScheduler ( C , GCNIterativeScheduler :: SCHEDULE_ILP ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( createStoreClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( create<ISA_LIT>MacroFusionDAGMutation ( ) ) ; return DAG ;


<s> ScheduleDAGInstrs * createMachineScheduler ( MachineSchedContext * C ) const override { ScheduleDAGMILive * DAG = createGenericSchedLive ( C ) ;

DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ;


<s> ScheduleDAGMILive * DAG = createGenericSchedLive ( C ) ;


<s> static ScheduleDAGInstrs * create<ISA_LIT>MachineScheduler ( MachineSchedContext * C ) {

return new ScheduleDAGMILive ( C , llvm :: make_unique < <ISA_LIT>SchedStrategy > ( ) ) ;


<s> return llvm :: make_unique < <ISA_LIT>TargetObjectFile > ( ) ;


<s> static CodeModel :: Model getEffectiveCodeModel ( Optional < CodeModel :: Model > CM ) { if ( CM ) return * CM ; return CodeModel :: Small ;


<s> StringRef <ISA_LIT>TargetMachine :: getGPUName ( const Function & F ) const {

Attribute GPUAttr = F . getFnAttribute ( <STR_LIT> ) ;


<s> Attribute GPUAttr = F . getFnAttribute ( <STR_LIT> ) ; return GPUAttr . hasAttribute ( Attribute :: None ) ? getTargetCPU ( ) : GPUAttr . getValueAsString ( ) ;


<s> LLVM_READNONE static StringRef getGPUOrDefault ( const Triple & TT , StringRef GPU ) { if ( ! GPU . empty ( ) ) return GPU ; if ( TT . getArch ( ) == Triple :: amdgcn ) return <STR_LIT> ; return <STR_LIT> ;


<s> if ( ! GPU . empty ( ) ) return GPU ; if ( TT . getArch ( ) == Triple :: amdgcn ) return <STR_LIT> ;


<s> I = llvm :: make_unique < GCNSubtarget > ( TargetTriple , GPU , FS , * this ) ; } I -> setScalarizeGlobalBehavior ( ScalarizeGlobal ) ; return I . get ( ) ;


initialize<ISA_LIT>UseNativeCallsPass ( * PR ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OL , bool JIT ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) { setRequiresStructuredCFG ( true ) ;


<s> if ( EnableLoadStoreVectorizer ) addPass ( createLoadStoreVectorizerPass ( ) ) ;


<s> TargetPassConfig :: addCodeGenPrepare ( ) ;

if ( EnableLoadStoreVectorizer ) addPass ( createLoadStoreVectorizerPass ( ) ) ;


<s> void GCNPassConfig :: addIRPasses ( ) {

addPass ( create<ISA_LIT>CodeGenPreparePass ( & getGCNTargetMachine ( ) ) ) ;


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) {


<s> addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;

addPass ( create<ISA_LIT>LowerControlFlowPass ( ) ) ;


<s> addPass ( create<ISA_LIT>LowerControlFlowPass ( ) ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ;


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ; } if ( getOptLevel ( ) > CodeGenOpt :: None ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>LoadStoreOptimizerID ) ; insertPass ( & MachineSchedulerID , & RegisterCoalescerID ) ; }

addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;


<s> addPass ( create<ISA_LIT>EmitClauseMarkers ( ) , false ) ; if ( Enable<ISA_LIT>IfConvert ) addPass ( & IfConverterID , false ) ;

addPass ( create<ISA_LIT>ClauseMergePass ( * TM ) , false ) ;


<s> addPass ( createSpeculativeExecutionPass ( ) ) ; addPass ( createStraightLineStrengthReducePass ( ) ) ; addEarlyCSEOrGVNPass ( ) ;

addPass ( createNaryReassociatePass ( ) ) ;


<s> addPass ( createSpeculativeExecutionPass ( ) ) ; addPass ( createStraightLineStrengthReducePass ( ) ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , IntrinsicInfo ( ) {


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true ) ;


<s> if ( TT . getArch ( ) == Triple :: r600 ) { return <STR_LIT> <STR_LIT> ; } return <STR_LIT> <STR_LIT> <STR_LIT> ;


<s> static StringRef computeDataLayout ( const Triple & TT ) {

if ( TT . getArch ( ) == Triple :: r600 ) {


<s> const <ISA_LIT>Subtarget & ST = C -> MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) ;

if ( ST . enable<ISA_LIT>Scheduler ( ) ) return create<ISA_LIT>MachineScheduler ( C ) ;


<s> return make_unique < <ISA_LIT>TargetObjectFile > ( ) ;


<s> static std :: unique_ptr < TargetLoweringObjectFile > createTLOF ( const Triple & TT ) { return make_unique < <ISA_LIT>TargetObjectFile > ( ) ;


<s> GCNTargetMachine :: GCNTargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) {


<s> if ( TT . getArch ( ) == Triple :: amdgcn ) return ( TT . getOS ( ) == Triple :: AMDHSA ) ? <STR_LIT> : <STR_LIT> ; return <STR_LIT> ;


<s> StringRef FS = getFeatureString ( F ) ; SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F ) ; I = llvm :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ; GISelAccessor * GISel = new GISelAccessor ( ) ; <ISA_LIT>GISelActualAccessor * GISel = new <ISA_LIT>GISelActualAccessor ( ) ; GISel -> CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * I -> getTargetLowering ( ) ) ) ;


<s> return TargetIRAnalysis ( [ this ] ( const Function & F ) {

return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this , F ) ) ;


<s> TargetIRAnalysis <ISA_LIT>TargetMachine :: getTargetIRAnalysis ( ) {

return TargetIRAnalysis ( [ this ] ( const Function & F ) {


initialize<ISA_LIT>WholeQuadModePass ( * PR ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) {


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ;

TargetPassConfig :: addCodeGenPrepare ( ) ;


<s> void GCNPassConfig :: addFastRegAlloc ( ) { insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & RegisterCoalescerID , & <ISA_LIT>PreAllocateWWMRegsID , false ) ;

TargetPassConfig :: addFastRegAlloc ( ) ;


if ( TM . getOptLevel ( ) > CodeGenOpt :: None ) {


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; insertPass ( & <ISA_LIT>OptimizeExecMaskingPreRAID , & <ISA_LIT>FormMemoryClausesID ) ; } else { insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; } insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ;

insertPass ( & RegisterCoalescerID , & <ISA_LIT>PreAllocateWWMRegsID , false ) ;


<s> void GCNPassConfig :: addOptimizedRegAlloc ( ) { if ( OptExecMaskPreRA ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ;

insertPass ( & <ISA_LIT>OptimizeExecMaskingPreRAID , & <ISA_LIT>FormMemoryClausesID ) ;


<s> addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ;


<s> addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ;

addPass ( & <ISA_LIT>InsertSkipsPassID ) ;


<s> } addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { addPass ( createStructurizeCFGPass ( true ) ) ; }


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , getEffectiveCodeModel ( CM , CodeModel :: Small ) , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) { initAsmInfo ( ) ;


<s> static StringRef computeDataLayout ( const Triple & TT ) { if ( TT . getArch ( ) == Triple :: r600 ) { return <STR_LIT> <STR_LIT> ;


initialize<ISA_LIT>LowerControlFlowPass ( * PR ) ;


<s> const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; }

if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) {


<s> } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) {

return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ;


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ; addPass ( & <ISA_LIT>PerfHintAnalysisID ) ;


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ; addPass ( & <ISA_LIT>PerfHintAnalysisID ) ; TargetPassConfig :: addCodeGenPrepare ( ) ; if ( EnableLoadStoreVectorizer ) addPass ( createLoadStoreVectorizerPass ( ) ) ;


<s> <ISA_LIT>PassConfig :: addPreISel ( ) ; if ( EnableAtomicOptimizations ) { addPass ( create<ISA_LIT>AtomicOptimizerPass ( ) ) ; } addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { addPass ( createStructurizeCFGPass ( true ) ) ; } addPass ( createSinkingPass ( ) ) ;

addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ;


<s> if ( ! LateCFGStructurize ) { addPass ( createStructurizeCFGPass ( true ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) {

addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ;


<s> static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler ( MachineSchedContext * C ) { ScheduleDAGMILive * DAG = new GCNScheduleDAGMILive ( C , std :: make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ;

DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ;


initialize<ISA_LIT>UnifyMetadataPass ( * PR ) ;


<s> return false ; } ; if ( YamlMFI . ArgInfo && ( parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentBuffer , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentBuffer , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchPtr , <STR_LIT> , MFI -> ArgInfo . DispatchPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> QueuePtr , <STR_LIT> , MFI -> ArgInfo . QueuePtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> KernargSegmentPtr , <STR_LIT> , MFI -> ArgInfo . KernargSegmentPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchID , <STR_LIT> , MFI -> ArgInfo . DispatchID , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> FlatScratchInit , <STR_LIT> , MFI -> ArgInfo . FlatScratchInit , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentSize , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentSize , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDX , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDY , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDZ , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDZ , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupInfo , <STR_LIT> , MFI -> ArgInfo . WorkGroupInfo , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentWaveByteOffset , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentWaveByteOffset , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitArgPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitArgPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitBufferPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitBufferPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDX , <STR_LIT> , MFI -> ArgInfo . WorkItemIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDY , <STR_LIT> , MFI -> ArgInfo . WorkItemIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDZ , <STR_LIT> , MFI -> ArgInfo . WorkItemIDZ , <NUM_LIT> , <NUM_LIT> ) ) ) return true ; MFI -> Mode . IEEE = YamlMFI . Mode . IEEE ; MFI -> Mode . DX10Clamp = YamlMFI . Mode . DX10Clamp ;

MFI -> Mode . FP32Denormals = YamlMFI . Mode . FP32Denormals ;


<s> addPass ( & FinalizeISelID ) ; addPass ( create<ISA_LIT>FixupVectorISelPass ( ) ) ; addPass ( create<ISA_LIT>AddIMGInitPass ( ) ) ;


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & RegisterCoalescerID , & <ISA_LIT>PreAllocateWWMRegsID , false ) ; if ( EnableDCEInRA ) insertPass ( & DetectDeadLanesID , & DeadMachineInstructionElimID ) ;

TargetPassConfig :: addOptimizedRegAlloc ( ) ;


<s> addPass ( & PostRAHazardRecognizerID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>RemoveShortExecBranchesID ) ;


<s> addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ;

addPass ( & PostRAHazardRecognizerID ) ;


<s> } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) { addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ; }


<s> static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler ( MachineSchedContext * C ) { ScheduleDAGMILive * DAG = new GCNScheduleDAGMILive ( C , std :: make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( createStoreClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( create<ISA_LIT>MacroFusionDAGMutation ( ) ) ;


<s> break ; } return std :: make_pair ( nullptr , - <NUM_LIT> ) ; } Value * Ptr ; if ( match ( const_cast < Value * > ( V ) , m_c_And ( m_Not ( m_Intrinsic < <STR_LIT> > ( m_Value ( Ptr ) ) ) , m_Not ( m_Intrinsic < <STR_LIT> > ( m_Deferred ( Ptr ) ) ) ) ) ) return std :: make_pair ( Ptr , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; return std :: make_pair ( nullptr , - <NUM_LIT> ) ;


<s> } insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & RegisterCoalescerID , & <ISA_LIT>PreAllocateWWMRegsID , false ) ;

if ( EnableDCEInRA ) insertPass ( & DetectDeadLanesID , & DeadMachineInstructionElimID ) ;


<s> addPass ( & <ISA_LIT>RemoveShortExecBranchesID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( & BranchRelaxationPassID ) ;


<s> void GCNPassConfig :: addPreLegalizeMachineIR ( ) { bool IsOptNone = getOptLevel ( ) == CodeGenOpt :: None ;

addPass ( create<ISA_LIT>PreLegalizeCombiner ( IsOptNone ) ) ;


<s> void GCNPassConfig :: addPreLegalizeMachineIR ( ) {

bool IsOptNone = getOptLevel ( ) == CodeGenOpt :: None ;


initialize<ISA_LIT>AAWrapperPassPass ( * PR ) ;


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ; addPass ( & <ISA_LIT>PerfHintAnalysisID ) ;

TargetPassConfig :: addCodeGenPrepare ( ) ;


<s> addPass ( & <ISA_LIT>PerfHintAnalysisID ) ; TargetPassConfig :: addCodeGenPrepare ( ) ; if ( EnableLoadStoreVectorizer ) addPass ( createLoadStoreVectorizerPass ( ) ) ;


<s> void GCNPassConfig :: addFastRegAlloc ( ) { insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; insertPass ( & TwoAddressInstructionPassID , & <ISA_LIT>WholeQuadModeID ) ;

insertPass ( & TwoAddressInstructionPassID , & <ISA_LIT>PreAllocateWWMRegsID ) ;


<s> addPass ( new InstructionSelect ( ) ) ;


<s> addPass ( & <ISA_LIT>FixSGPRCopiesID ) ;

addPass ( create<ISA_LIT>LowerI1CopiesPass ( ) ) ;


<s> addPass ( create<ISA_LIT>LowerI1CopiesPass ( ) ) ; addPass ( create<ISA_LIT>AddIMGInitPass ( ) ) ; return false ;


if ( TM . getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) {


<s> addPass ( & <ISA_LIT>FoldOperandsID ) ; if ( EnableDPPCombine ) addPass ( & GCNDPPCombineID ) ; addPass ( & DeadMachineInstructionElimID ) ; addPass ( & <ISA_LIT>LoadStoreOptimizerID ) ; if ( EnableSDWAPeephole ) { addPass ( & <ISA_LIT>PeepholeSDWAID ) ; addPass ( & EarlyMachineLICMID ) ; addPass ( & MachineCSEID ) ; addPass ( & <ISA_LIT>FoldOperandsID ) ;


<s> addPass ( & DeadMachineInstructionElimID ) ; addPass ( & <ISA_LIT>LoadStoreOptimizerID ) ; if ( EnableSDWAPeephole ) { addPass ( & <ISA_LIT>PeepholeSDWAID ) ; addPass ( & EarlyMachineLICMID ) ; addPass ( & MachineCSEID ) ; addPass ( & <ISA_LIT>FoldOperandsID ) ; addPass ( & DeadMachineInstructionElimID ) ; } addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>WholeQuadModeID ) ; insertPass ( & MachineSchedulerID , & <ISA_LIT>PreAllocateWWMRegsID ) ; if ( OptExecMaskPreRA ) insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ;

insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ;


<s> addPass ( & <ISA_LIT>FixVGPRCopiesID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>OptimizeExecMaskingID ) ; TargetPassConfig :: addPostRegAlloc ( ) ;

addPass ( & <ISA_LIT>LowerSGPRSpillsID ) ;


<s> addPass ( & <ISA_LIT>FixVGPRCopiesID ) ;

if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>OptimizeExecMaskingID ) ;


<s> addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>RemoveShortExecBranchesID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( & <ISA_LIT>PreEmitPeepholeID ) ; addPass ( & PostRAHazardRecognizerID ) ;


<s> bool GCNPassConfig :: addPreISel ( ) { <ISA_LIT>PassConfig :: addPreISel ( ) ; addPass ( create<ISA_LIT>LateCodeGenPreparePass ( ) ) ; if ( EnableAtomicOptimizations ) { addPass ( create<ISA_LIT>AtomicOptimizerPass ( ) ) ; } addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) {


<s> addPass ( create<ISA_LIT>VectorRegMerger ( ) ) ;


<s> void <ISA_LIT>PassConfig :: addPreRegAlloc ( ) { addPass ( create<ISA_LIT>VectorRegMerger ( ) ) ;


<s> addPass ( & GCNRegBankReassignID ) ; }


<s> if ( Enable<ISA_LIT>IfConvert ) addPass ( & IfConverterID , false ) ;

addPass ( create<ISA_LIT>ClauseMergePass ( ) , false ) ;


PM . add ( llvm :: create<ISA_LIT>UseNativeCallsPass ( ) ) ;


<s> yaml :: MachineFunctionInfo * GCNTargetMachine :: convertFuncInfoToYAML ( const MachineFunction & MF ) const { const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

return new yaml :: <ISA_LIT>MachineFunctionInfo ( * MFI , * MF . getSubtarget ( ) . getRegisterInfo ( ) ) ;


<s> return new yaml :: <ISA_LIT>MachineFunctionInfo ( * MFI , * MF . getSubtarget ( ) . getRegisterInfo ( ) ) ;


<s> ScheduleDAGInstrs * createMachineScheduler ( MachineSchedContext * C ) const override {

return create<ISA_LIT>MachineScheduler ( C ) ;


<s> return create<ISA_LIT>MachineScheduler ( C ) ;


<s> TargetPassConfig * <ISA_LIT>TargetMachine :: createPassConfig ( PassManagerBase & PM ) { return new <ISA_LIT>PassConfig ( * this , PM ) ;


<s> return new ScheduleDAGMILive ( C , std :: make_unique < <ISA_LIT>SchedStrategy > ( ) ) ;


<s> GCNPassConfig ( LLVMTargetMachine & TM , PassManagerBase & PM ) : <ISA_LIT>PassConfig ( TM , PM ) {


<s> std :: unique_ptr < CSEConfigBase > <ISA_LIT>PassConfig :: getCSEConfig ( ) const { return getStandardCSEConfigForOpt ( TM -> getOptLevel ( ) ) ;


<s> return getStandardCSEConfigForOpt ( TM -> getOptLevel ( ) ) ;


<s> SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) {


<s> StringRef FS = getFeatureString ( F ) ; SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ;


<s> TargetTransformInfo <ISA_LIT>TargetMachine :: getTargetTransformInfo ( const Function & F ) {

return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this , F ) ) ;


<s> TargetTransformInfo <ISA_LIT>TargetMachine :: getTargetTransformInfo ( const Function & F ) { return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this , F ) ) ;


<s> if ( const Function * F = dyn_cast < Function > ( & GV ) ) return F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ; return ! GV . use_empty ( ) ;


<s> static bool mustPreserveGV ( const GlobalValue & GV ) { if ( const Function * F = dyn_cast < Function > ( & GV ) ) return F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ; return ! GV . use_empty ( ) ;


<s> if ( A -> IsRegister ) { Register Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; } if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A -> RegisterName ) ; Arg = ArgDescriptor :: createRegister ( Reg ) ; } else Arg = ArgDescriptor :: createStack ( A -> StackOffset ) ; if ( A -> Mask ) Arg = ArgDescriptor :: createArg ( Arg , A -> Mask . getValue ( ) ) ; MFI -> NumUserSGPRs += UserSGPRs ; MFI -> NumSystemSGPRs += SystemSGPRs ; return false ; } ; if ( YamlMFI . ArgInfo && ( parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentBuffer , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentBuffer , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchPtr , <STR_LIT> , MFI -> ArgInfo . DispatchPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> QueuePtr , <STR_LIT> , MFI -> ArgInfo . QueuePtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> KernargSegmentPtr , <STR_LIT> , MFI -> ArgInfo . KernargSegmentPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchID , <STR_LIT> , MFI -> ArgInfo . DispatchID , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> FlatScratchInit , <STR_LIT> , MFI -> ArgInfo . FlatScratchInit , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentSize , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentSize , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDX , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDY , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDZ , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDZ , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupInfo , <STR_LIT> , MFI -> ArgInfo . WorkGroupInfo , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentWaveByteOffset , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentWaveByteOffset , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitArgPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitArgPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitBufferPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitBufferPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDX , <STR_LIT> , MFI -> ArgInfo . WorkItemIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDY , <STR_LIT> , MFI -> ArgInfo . WorkItemIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDZ , <STR_LIT> , MFI -> ArgInfo . WorkItemIDZ , <NUM_LIT> , <NUM_LIT> ) ) ) return true ;

MFI -> Mode . IEEE = YamlMFI . Mode . IEEE ;


<s> <ISA_LIT>PassConfig ( LLVMTargetMachine & TM , PassManagerBase & PM ) : <ISA_LIT>PassConfig ( TM , PM ) {


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OL , bool JIT ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) {


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OL , bool JIT ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) { setRequiresStructuredCFG ( true ) ; if ( EnableFunctionCalls && Enable<ISA_LIT>FunctionCallsOpt . getNumOccurrences ( ) == <NUM_LIT> ) EnableFunctionCalls = false ;


FunctionPassManager FPM ( DebugPassManager ) ;


if ( EarlyInlineAll && ! EnableFunctionCalls ) PM . addPass ( <ISA_LIT>AlwaysInlinePass ( ) ) ;


<s> Builder . DivergentTarget = true ; bool EnableOpt = getOptLevel ( ) > CodeGenOpt :: None ; bool Internalize = InternalizeSymbols ; bool EarlyInline = EarlyInlineAll && EnableOpt && ! Enable<ISA_LIT>FunctionCalls ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && EnableOpt ; bool LibCallSimplify = EnableLibCallSimplify && EnableOpt ; if ( Enable<ISA_LIT>FunctionCalls ) { delete Builder . Inliner ; Builder . Inliner = create<ISA_LIT>FunctionInliningPass ( ) ; } if ( Internalize ) { Builder . addExtension ( PassManagerBuilder :: EP_EnabledOnOptLevel0 , [ ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { PM . add ( createInternalizePass ( mustPreserveGV ) ) ; } ) ; } Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ;

PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , Optional < CodeModel :: Model > CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , getEffectiveCodeModel ( CM ) , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) { AS = <STR_LIT> ( TT ) ;


<s> addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ;

addPass ( & <ISA_LIT>InsertSkipsPassID ) ;


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ;


<s> addPass ( create<ISA_LIT>ISelDag ( get<ISA_LIT>TargetMachine ( ) , getOptLevel ( ) ) ) ; return false ;


<s> addPass ( create<ISA_LIT>ISelDag ( get<ISA_LIT>TargetMachine ( ) , getOptLevel ( ) ) ) ;


<s> TargetPassConfig :: addMachineSSAOptimization ( ) ; addPass ( & <ISA_LIT>FoldOperandsID ) ; addPass ( & DeadMachineInstructionElimID ) ; addPass ( & <ISA_LIT>LoadStoreOptimizerID ) ;


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ;

TargetPassConfig :: addOptimizedRegAlloc ( RegAllocPass ) ;


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ;


<s> addPass ( & PostRAHazardRecognizerID ) ; addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;


<s> addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ; addPass ( & BranchRelaxationPassID ) ;


<s> addPass ( create<ISA_LIT>WholeQuadModePass ( ) ) ;


<s> void GCNPassConfig :: addPreRegAlloc ( ) {

addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;


<s> PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true ; } } return ! GV . use_empty ( ) ; } ) ) ;


<s> ScheduleDAGMILive * DAG = new GCNScheduleDAGMILive ( C , make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ; DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( createStoreClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; return DAG ;


<s> DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( createStoreClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ;


<s> if ( ST . enable<ISA_LIT>Scheduler ( ) ) return create<ISA_LIT>MachineScheduler ( C ) ;

return createGCNMaxOccupancyMachineScheduler ( C ) ;


<s> if ( ! I ) { resetTargetOptions ( F ) ; I = llvm :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ; GISelAccessor * GISel = new GISelAccessor ( ) ; <ISA_LIT>GISelActualAccessor * GISel = new <ISA_LIT>GISelActualAccessor ( ) ;

GISel -> CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * I -> getTargetLowering ( ) ) ) ;


initialize<ISA_LIT>LowerControlFlowPass ( * PR ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) { setRequiresStructuredCFG ( true ) ;


<s> insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ;

insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ;


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ;


<s> addPass ( & PostRAHazardRecognizerID ) ; addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; if ( Enable<ISA_LIT>InsertWaitcntsPass ) addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; else addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ;


<s> } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( mustPreserveGV ) ) ; PM . add ( createGlobalDCEPass ( ) ) ; } if ( EarlyInline ) PM . add ( create<ISA_LIT>AlwaysInlinePass ( false ) ) ; } ) ; const auto & Opt = Options ; Builder . addExtension ( PassManagerBuilder :: EP_EarlyAsPossible , [ <ISA_LIT>AA , LibCallSimplify , & Opt ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; }


<s> if ( TT . getArch ( ) == Triple :: r600 ) { return <STR_LIT> <STR_LIT> ; }


<s> auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F ) ; I = llvm :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ; }

I -> setScalarizeGlobalBehavior ( ScalarizeGlobal ) ;


<s> return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this , F ) ) ;


initialize<ISA_LIT>MemoryLegalizerPass ( * PR ) ;


<s> bool isPassEnabled ( const cl :: opt < bool > & Opt , CodeGenOpt :: Level Level = CodeGenOpt :: Default ) const {

if ( Opt . getNumOccurrences ( ) ) return Opt ;


<s> if ( Opt . getNumOccurrences ( ) ) return Opt ;

if ( TM -> getOptLevel ( ) < Level ) return false ;


<s> const <ISA_LIT>TargetMachine & TM = get<ISA_LIT>TargetMachine ( ) ; const <ISA_LIT>Subtarget & ST = * TM . getSubtargetImpl ( ) ; if ( TM . getOptLevel ( ) > CodeGenOpt :: None && ST . isPromoteAllocaEnabled ( ) ) { addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; addPass ( createSROAPass ( ) ) ;


<s> addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; addPass ( createSROAPass ( ) ) ; }

TargetPassConfig :: addCodeGenPrepare ( ) ;


<s> addPass ( create<ISA_LIT>InsertWaitsPass ( ) , false ) ; addPass ( create<ISA_LIT>LowerControlFlowPass ( ) , false ) ;


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>InsertWaitsPass ( ) , false ) ; addPass ( create<ISA_LIT>LowerControlFlowPass ( ) , false ) ; if ( InsertNops ) { addPass ( create<ISA_LIT>InsertNopsPass ( ) , false ) ; }


<s> bool <ISA_LIT>PassConfig :: addPreISel ( ) { <ISA_LIT>PassConfig :: addPreISel ( ) ; const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ;

if ( ST . IsIRStructurizerEnabled ( ) ) addPass ( createStructurizeCFGPass ( ) ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , RM , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , Subtarget ( TT , getTargetCPU ( ) , FS , * this ) , IntrinsicInfo ( ) {


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , RM , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , Subtarget ( TT , getTargetCPU ( ) , FS , * this ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true ) ; initAsmInfo ( ) ;


<s> ScheduleDAGInstrs * createMachineScheduler ( MachineSchedContext * C ) const override { const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return create<ISA_LIT>MachineScheduler ( C ) ; else if ( ST . enable<ISA_LIT>Scheduler ( ) ) return create<ISA_LIT>MachineScheduler ( C ) ;


<s> GCNTargetMachine :: GCNTargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) {


<s> if ( TT . getArch ( ) == Triple :: amdgcn ) return ( TT . getOS ( ) == Triple :: AMDHSA ) ? <STR_LIT> : <STR_LIT> ; return <STR_LIT> ;


<s> LLVM_READNONE static StringRef getGPUOrDefault ( const Triple & TT , StringRef GPU ) {


initialize<ISA_LIT>PromoteAllocaPass ( * PR ) ;


initialize<ISA_LIT>InsertWaitsPass ( * PR ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) : <ISA_LIT>TargetMachine ( T , TT , CPU , FS , Options , RM , CM , OL ) {


<s> static int64_t getNullPointerValue ( unsigned AddrSpace ) { return ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ? - <NUM_LIT> : <NUM_LIT> ;


<s> return ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) ? - <NUM_LIT> : <NUM_LIT> ;


initialize<ISA_LIT>SimplifyLibCallsPass ( * PR ) ;


<s> addPass ( & <ISA_LIT>RemoveShortExecBranchesID ) ; addPass ( & <ISA_LIT>PreEmitPeepholeID ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( & BranchRelaxationPassID ) ;


<s> <ISA_LIT>PassConfig :: addPreISel ( ) ; if ( EnableAtomicOptimizations ) { addPass ( create<ISA_LIT>AtomicOptimizerPass ( ) ) ; } addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { if ( EnableStructurizerWorkarounds ) { addPass ( createUnifyLoopExitsPass ( ) ) ; } addPass ( createStructurizeCFGPass ( false ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ;


<s> addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) { addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ; } addPass ( createLCSSAPass ( ) ) ; return false ;


PM . add ( createInferAddressSpacesPass ( ) ) ;


<s> const <ISA_LIT>Subtarget * getSubtargetImpl ( const Function & ) const override { return & Subtarget ;


initialize<ISA_LIT>UseNativeCallsPass ( * PR ) ;


if ( EnableSROA ) addPass ( createSROAPass ( ) ) ;


<s> if ( TM . getOptLevel ( ) > CodeGenOpt :: None ) { addPass ( createInferAddressSpacesPass ( ) ) ; addPass ( create<ISA_LIT>PromoteAlloca ( ) ) ; if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; addStraightLineScalarOptimizationPasses ( ) ; if ( Enable<ISA_LIT>AliasAnalysis ) { addPass ( create<ISA_LIT>AAWrapperPass ( ) ) ; addPass ( createExternalAAWrapperPass ( [ ] ( Pass & P , Function & , AAResults & AAR ) { if ( auto * WrapperPass = P . getAnalysisIfAvailable < <ISA_LIT>AAWrapperPass > ( ) ) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ; } ) ) ; }


<s> addPass ( & <ISA_LIT>FoldOperandsID ) ; addPass ( & DeadMachineInstructionElimID ) ; addPass ( & <ISA_LIT>LoadStoreOptimizerID ) ; if ( EnableSDWAPeephole ) { addPass ( & <ISA_LIT>PeepholeSDWAID ) ; addPass ( & MachineLICMID ) ; addPass ( & MachineCSEID ) ; addPass ( & <ISA_LIT>FoldOperandsID ) ; addPass ( & DeadMachineInstructionElimID ) ; } addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( & PostRAHazardRecognizerID ) ; if ( Enable<ISA_LIT>InsertWaitcntsPass ) addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ;

else addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ;


<s> addPass ( & PostRAHazardRecognizerID ) ; if ( Enable<ISA_LIT>InsertWaitcntsPass ) addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; else addPass ( create<ISA_LIT>InsertWaitsPass ( ) ) ;

addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;


<s> addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { addPass ( createStructurizeCFGPass ( true ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>TypeRewriter ( ) ) ; addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ; if ( ! LateCFGStructurize ) {

addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ;


<s> Builder . DivergentTarget = true ; bool Internalize = InternalizeSymbols && ( getOptLevel ( ) > CodeGenOpt :: None ) && ( getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) ; bool EarlyInline = EarlyInlineAll && ( getOptLevel ( ) > CodeGenOpt :: None ) ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && getOptLevel ( ) > CodeGenOpt :: None ; Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL :


<s> PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true ; } } return ! GV . use_empty ( ) ; } ) ) ; PM . add ( createGlobalDCEPass ( ) ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) { AS = <STR_LIT> ( TT ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) {


<s> if ( TT . getEnvironmentName ( ) == <STR_LIT> || TT . getEnvironmentName ( ) == <STR_LIT> ) return <STR_LIT> <STR_LIT> <STR_LIT> ;


<s> if ( TT . getEnvironmentName ( ) == <STR_LIT> || TT . getEnvironmentName ( ) == <STR_LIT> ) return <STR_LIT> <STR_LIT> <STR_LIT> ; return <STR_LIT> <STR_LIT> <STR_LIT> ;


<s> GCNPassConfig ( LLVMTargetMachine & TM , PassManagerBase & PM ) : <ISA_LIT>PassConfig ( TM , PM ) {


initializeGCNRegBankReassignPass ( * PR ) ;


<s> } return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg ) { if ( ! A ) return false ; if ( A -> IsRegister ) { unsigned Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; } if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A -> RegisterName ) ;

Arg = ArgDescriptor :: createRegister ( Reg ) ;


<s> if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg ) { if ( ! A ) return false ; if ( A -> IsRegister ) { unsigned Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; } if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A -> RegisterName ) ; Arg = ArgDescriptor :: createRegister ( Reg ) ; } else Arg = ArgDescriptor :: createStack ( A -> StackOffset ) ; if ( A -> Mask ) Arg = ArgDescriptor :: createArg ( Arg , A -> Mask . getValue ( ) ) ; return false ; } ; if ( YamlMFI . ArgInfo && ( parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentBuffer , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentBuffer ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchPtr , <STR_LIT> , MFI -> ArgInfo . DispatchPtr ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> QueuePtr , <STR_LIT> , MFI -> ArgInfo . QueuePtr ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> KernargSegmentPtr , <STR_LIT> , MFI -> ArgInfo . KernargSegmentPtr ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchID , <STR_LIT> , MFI -> ArgInfo . DispatchID ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> FlatScratchInit , <STR_LIT> , MFI -> ArgInfo . FlatScratchInit ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentSize , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentSize ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDX , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDX ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDY , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDY ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDZ , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDZ ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupInfo , <STR_LIT> , MFI -> ArgInfo . WorkGroupInfo ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentWaveByteOffset , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentWaveByteOffset ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitArgPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitArgPtr ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitBufferPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitBufferPtr ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDX , <STR_LIT> , MFI -> ArgInfo . WorkItemIDX ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDY , <STR_LIT> , MFI -> ArgInfo . WorkItemIDY ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDZ , <STR_LIT> , MFI -> ArgInfo . WorkItemIDZ ) ) ) return true ; return false ;


if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ;


<s> if ( TM -> getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn && EnableLowerKernelArguments ) addPass ( create<ISA_LIT>LowerKernelArgumentsPass ( ) ) ;


addPass ( create<ISA_LIT>PromoteAlloca ( ) ) ;


<s> addPass ( create<ISA_LIT>AAWrapperPass ( ) ) ; addPass ( createExternalAAWrapperPass ( [ ] ( Pass & P , Function & , AAResults & AAR ) { if ( auto * WrapperPass = P . getAnalysisIfAvailable < <ISA_LIT>AAWrapperPass > ( ) ) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ; } ) ) ; } if ( TM . getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) { addPass ( create<ISA_LIT>CodeGenPreparePass ( ) ) ; } } TargetPassConfig :: addIRPasses ( ) ; if ( isPassEnabled ( EnableScalarIRPasses ) ) addEarlyCSEOrGVNPass ( ) ;


<s> addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>PreEmitPeepholeID ) ;

addPass ( & PostRAHazardRecognizerID ) ;


<s> if ( TM -> getOptLevel ( ) > CodeGenOpt :: None ) addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & <ISA_LIT>PostRABundlerID ) ;


<s> addPass ( & <ISA_LIT>InsertSkipsPassID ) ; addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ;

addPass ( create<ISA_LIT>DebuggerInsertNopsPass ( ) ) ;


<s> addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ;

addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ;


if ( TT . getEnvironmentName ( ) == <STR_LIT> || TT . getEnvironmentName ( ) == <STR_LIT> ) return <STR_LIT> <STR_LIT> <STR_LIT> ;


<s> } return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs , unsigned SystemSGPRs ) { if ( ! A ) return false ; if ( A -> IsRegister ) { unsigned Reg ; if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) { SourceRange = A -> RegisterName . SourceRange ; return true ; }

if ( ! RC . contains ( Reg ) ) return diagnoseRegisterClass ( A -> RegisterName ) ;


initialize<ISA_LIT>InsertWaitcntsPass ( * PR ) ;


initialize<ISA_LIT>AAWrapperPassPass ( * PR ) ;


<s> bool mustPreserveGV ( const GlobalValue & GV ) { if ( const Function * F = dyn_cast < Function > ( & GV ) ) return F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ;


<s> void <ISA_LIT>PassConfig :: addCodeGenPrepare ( ) { const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( ST . isPromoteAllocaEnabled ( ) ) { addPass ( create<ISA_LIT>PromoteAlloca ( TM ) ) ; addPass ( createSROAPass ( ) ) ; } TargetPassConfig :: addCodeGenPrepare ( ) ;


<s> addPass ( create<ISA_LIT>InsertWaitsPass ( ) , false ) ;

addPass ( create<ISA_LIT>LowerControlFlowPass ( ) , false ) ;


<s> if ( TM . getOptLevel ( ) > CodeGenOpt :: None ) { addPass ( createInferAddressSpacesPass ( ) ) ; addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; addStraightLineScalarOptimizationPasses ( ) ; if ( Enable<ISA_LIT>AliasAnalysis ) { addPass ( create<ISA_LIT>AAWrapperPass ( ) ) ; addPass ( createExternalAAWrapperPass ( [ ] ( Pass & P , Function & , AAResults & AAR ) { if ( auto * WrapperPass = P . getAnalysisIfAvailable < <ISA_LIT>AAWrapperPass > ( ) ) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ; } ) ) ; } } TargetPassConfig :: addIRPasses ( ) ; if ( getOptLevel ( ) != CodeGenOpt :: None ) addEarlyCSEOrGVNPass ( ) ;


<s> addPass ( create<ISA_LIT>WholeQuadModePass ( ) ) ;


<s> bool Internalize = InternalizeSymbols && ( getOptLevel ( ) > CodeGenOpt :: None ) && ( getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) ; bool EarlyInline = EarlyInlineAll && ( getOptLevel ( ) > CodeGenOpt :: None ) ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && getOptLevel ( ) > CodeGenOpt :: None ; Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS :


<s> Builder . DivergentTarget = true ; bool Internalize = InternalizeSymbols && ( getOptLevel ( ) > CodeGenOpt :: None ) && ( getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) ; bool EarlyInline = EarlyInlineAll && ( getOptLevel ( ) > CodeGenOpt :: None ) ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && getOptLevel ( ) > CodeGenOpt :: None ; Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true ; } } return ! GV . use_empty ( ) ; } ) ) ; PM . add ( createGlobalDCEPass ( ) ) ; } if ( EarlyInline ) PM . add ( create<ISA_LIT>AlwaysInlinePass ( false ) ) ; } ) ; Builder . addExtension ( PassManagerBuilder :: EP_EarlyAsPossible , [ <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ;


<s> addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( & <ISA_LIT>InsertSkipsPassID ) ;


initialize<ISA_LIT>MemoryLegalizerPass ( * PR ) ;


<s> bool GCNPassConfig :: addGlobalInstructionSelect ( ) {


<s> if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; addStraightLineScalarOptimizationPasses ( ) ; } TargetPassConfig :: addIRPasses ( ) ; if ( getOptLevel ( ) != CodeGenOpt :: None ) addEarlyCSEOrGVNPass ( ) ;


<s> addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; addStraightLineScalarOptimizationPasses ( ) ; } TargetPassConfig :: addIRPasses ( ) ;

if ( getOptLevel ( ) != CodeGenOpt :: None ) addEarlyCSEOrGVNPass ( ) ;


<s> TargetPassConfig :: addPostRegAlloc ( ) ;


<s> static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler ( MachineSchedContext * C ) { ScheduleDAGMILive * DAG = new ScheduleDAGMILive ( C , llvm :: make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ;

DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ;


<s> SmallString < <NUM_LIT> > SubtargetKey ( GPU ) ; SubtargetKey . append ( FS ) ; auto & I = SubtargetMap [ SubtargetKey ] ; if ( ! I ) { resetTargetOptions ( F ) ; I = llvm :: make_unique < <ISA_LIT>Subtarget > ( TargetTriple , GPU , FS , * this ) ; GISelAccessor * GISel = new GISelAccessor ( ) ; <ISA_LIT>GISelActualAccessor * GISel = new <ISA_LIT>GISelActualAccessor ( ) ; GISel -> CallLoweringInfo . reset ( new <ISA_LIT>CallLowering ( * I -> getTargetLowering ( ) ) ) ; I -> setGISelAccessor ( * GISel ) ; }

I -> setScalarizeGlobalBehavior ( ScalarizeGlobal ) ;


initialize<ISA_LIT>InsertWaitsPass ( * PR ) ;


<s> Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ & ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; } ) ;


<s> Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ & ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) {

PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ;


<s> addPass ( create<ISA_LIT>FixupVectorISelPass ( ) ) ; addPass ( create<ISA_LIT>AddIMGInitPass ( ) ) ; addPass ( & UnreachableMachineBlockElimID ) ; return false ;


<s> if ( TT . getArch ( ) == Triple :: r600 ) { return <STR_LIT> <STR_LIT> ; }


<s> static StringRef computeDataLayout ( const Triple & TT ) { if ( TT . getArch ( ) == Triple :: r600 ) { return <STR_LIT> <STR_LIT> ; } return <STR_LIT> <STR_LIT> <STR_LIT> ;


<s> addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( isPassEnabled ( EnableSetWavePriority , CodeGenOpt :: Less ) ) addPass ( create<ISA_LIT>SetWavePriorityPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>PreEmitPeepholeID ) ;


<s> addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ;

if ( isPassEnabled ( EnableSetWavePriority , CodeGenOpt :: Less ) ) addPass ( create<ISA_LIT>SetWavePriorityPass ( ) ) ;


<s> DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ;

DAG -> addMutation ( createMFMAIGroupLPDAGMutation ( ) ) ;


<s> static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler ( MachineSchedContext * C ) {

ScheduleDAGMILive * DAG = new GCNScheduleDAGMILive ( C , std :: make_unique < GCNMaxOccupancySchedStrategy > ( C ) ) ;


<s> DAG -> addMutation ( createLoadClusterDAGMutation ( DAG -> TII , DAG -> TRI ) ) ; DAG -> addMutation ( ST . createFillMFMAShadowMutation ( DAG -> TII ) ) ;


<s> case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return <ISA_LIT>AS :: CONSTANT_ADDRESS ; } return <ISA_LIT>AS :: FLAT_ADDRESS ;


<s> case PseudoSourceValue :: GOT : case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry :

case PseudoSourceValue :: TargetCustom :


<s> if ( A -> Mask ) Arg = ArgDescriptor :: createArg ( Arg , A -> Mask . getValue ( ) ) ; MFI -> NumUserSGPRs += UserSGPRs ; MFI -> NumSystemSGPRs += SystemSGPRs ; return false ; } ; if ( YamlMFI . ArgInfo && ( parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentBuffer , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentBuffer , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchPtr , <STR_LIT> , MFI -> ArgInfo . DispatchPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> QueuePtr , <STR_LIT> , MFI -> ArgInfo . QueuePtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> KernargSegmentPtr , <STR_LIT> , MFI -> ArgInfo . KernargSegmentPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> DispatchID , <STR_LIT> , MFI -> ArgInfo . DispatchID , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> FlatScratchInit , <STR_LIT> , MFI -> ArgInfo . FlatScratchInit , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentSize , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentSize , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDX , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDY , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupIDZ , <STR_LIT> , MFI -> ArgInfo . WorkGroupIDZ , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkGroupInfo , <STR_LIT> , MFI -> ArgInfo . WorkGroupInfo , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> PrivateSegmentWaveByteOffset , <STR_LIT> , MFI -> ArgInfo . PrivateSegmentWaveByteOffset , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitArgPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitArgPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> ImplicitBufferPtr , <STR_LIT> , MFI -> ArgInfo . ImplicitBufferPtr , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDX , <STR_LIT> , MFI -> ArgInfo . WorkItemIDX , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDY , <STR_LIT> , MFI -> ArgInfo . WorkItemIDY , <NUM_LIT> , <NUM_LIT> ) || parseAndCheckArgument ( YamlMFI . ArgInfo -> WorkItemIDZ , <STR_LIT> , MFI -> ArgInfo . WorkItemIDZ , <NUM_LIT> , <NUM_LIT> ) ) ) return true ; MFI -> Mode . IEEE = YamlMFI . Mode . IEEE ; MFI -> Mode . DX10Clamp = YamlMFI . Mode . DX10Clamp ; MFI -> Mode . FP32InputDenormals = YamlMFI . Mode . FP32InputDenormals ; MFI -> Mode . FP32OutputDenormals = YamlMFI . Mode . FP32OutputDenormals ; MFI -> Mode . FP64FP16InputDenormals = YamlMFI . Mode . FP64FP16InputDenormals ; MFI -> Mode . FP64FP16OutputDenormals = YamlMFI . Mode . FP64FP16OutputDenormals ; return false ;


<s> if ( getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) return static_cast < const GCNTargetMachine * > ( this ) -> getSubtargetImpl ( F ) ; return static_cast < const <ISA_LIT>TargetMachine * > ( this ) -> getSubtargetImpl ( F ) ;


<s> if ( parseNamedRegisterReference ( PFS , RegVal , RegName . Value , Error ) ) { SourceRange = RegName . SourceRange ; return true ; } return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . ScratchWaveOffsetReg , MFI -> ScratchWaveOffsetReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> ScratchWaveOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchWaveOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchWaveOffsetReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; } auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs , unsigned SystemSGPRs ) { if ( ! A ) return false ; if ( A -> IsRegister ) { unsigned Reg ;

if ( parseNamedRegisterReference ( PFS , Reg , A -> RegisterName . Value , Error ) ) {


<s> MFI -> Occupancy = ST . computeOccupancy ( MF . getFunction ( ) , MFI -> getLDSSize ( ) ) ; } auto parseRegister = [ & ] ( const yaml :: StringValue & RegName , Register & RegVal ) { Register TempReg ; if ( parseNamedRegisterReference ( PFS , TempReg , RegName . Value , Error ) ) { SourceRange = RegName . SourceRange ; return true ; } RegVal = TempReg ; return false ; } ; auto diagnoseRegisterClass = [ & ] ( const yaml :: StringValue & RegName ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , RegName . Value . size ( ) , SourceMgr :: DK_Error , <STR_LIT> , RegName . Value , None , None ) ; SourceRange = RegName . SourceRange ; return true ; } ; if ( parseRegister ( YamlMFI . ScratchRSrcReg , MFI -> ScratchRSrcReg ) || parseRegister ( YamlMFI . FrameOffsetReg , MFI -> FrameOffsetReg ) || parseRegister ( YamlMFI . StackPtrOffsetReg , MFI -> StackPtrOffsetReg ) ) return true ; if ( MFI -> ScratchRSrcReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> ScratchRSrcReg ) ) { return diagnoseRegisterClass ( YamlMFI . ScratchRSrcReg ) ; } if ( MFI -> FrameOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> FrameOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . FrameOffsetReg ) ; } if ( MFI -> StackPtrOffsetReg != <STR_LIT> && ! <STR_LIT> . contains ( MFI -> StackPtrOffsetReg ) ) { return diagnoseRegisterClass ( YamlMFI . StackPtrOffsetReg ) ; }

auto parseAndCheckArgument = [ & ] ( const Optional < yaml :: <ISA_LIT>Argument > & A , const TargetRegisterClass & RC , ArgDescriptor & Arg , unsigned UserSGPRs , unsigned SystemSGPRs ) {


addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ;


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;

addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ;


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( create<ISA_LIT>MemoryLegalizerPass ( ) ) ; addPass ( create<ISA_LIT>InsertWaitcntsPass ( ) ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ; addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; addPass ( & PostRAHazardRecognizerID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>RemoveShortExecBranchesID ) ;

addPass ( & <ISA_LIT>PreEmitPeepholeID ) ;


<s> if ( isPassEnabled ( EnablePreRAOptimizations ) ) insertPass ( & RenameIndependentSubregsID , & GCNPreRAOptimizationsID ) ;

if ( TM -> getOptLevel ( ) > CodeGenOpt :: Less ) insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ;


<s> void GCNPassConfig :: addOptimizedRegAlloc ( ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>WholeQuadModeID ) ; insertPass ( & MachineSchedulerID , & <ISA_LIT>PreAllocateWWMRegsID ) ; if ( OptExecMaskPreRA ) insertPass ( & MachineSchedulerID , & <ISA_LIT>OptimizeExecMaskingPreRAID ) ; if ( isPassEnabled ( EnablePreRAOptimizations ) ) insertPass ( & RenameIndependentSubregsID , & GCNPreRAOptimizationsID ) ; if ( TM -> getOptLevel ( ) > CodeGenOpt :: Less ) insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; if ( OptVGPRLiveRange ) insertPass ( & LiveVariablesID , & <ISA_LIT>OptimizeVGPRLiveRangeID , false ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ;

if ( EnableDCEInRA ) insertPass ( & DetectDeadLanesID , & DeadMachineInstructionElimID ) ;


<s> bool EnableOpt = getOptLevel ( ) > CodeGenOpt :: None ; bool Internalize = InternalizeSymbols ; bool EarlyInline = EarlyInlineAll && EnableOpt && ! EnableFunctionCalls ; bool <ISA_LIT>AA = Enable<ISA_LIT>AliasAnalysis && EnableOpt ; bool LibCallSimplify = EnableLibCallSimplify && EnableOpt ; if ( EnableFunctionCalls ) { delete Builder . Inliner ; Builder . Inliner = createFunctionInliningPass ( ) ; } Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA , this ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ;

PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ;


<s> disablePass ( & GCLoweringID ) ; disablePass ( & ShadowStackGCLoweringID ) ;


initialize<ISA_LIT>AAWrapperPassPass ( * PR ) ;


initialize<ISA_LIT>WholeQuadModePass ( * PR ) ;


<s> static bool mustPreserveGV ( const GlobalValue & GV ) { if ( const Function * F = dyn_cast < Function > ( & GV ) ) return F -> isDeclaration ( ) || <STR_LIT> ( F -> getCallingConv ( ) ) ; GV . removeDeadConstantUsers ( ) ;


<s> if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>AlwaysInlinePass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>ReplaceLDSUseWithPointerPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>LowerModuleLDSPass ( ) ) ; return true ; } return false ; } ) ; PB . registerPipelineParsingCallback ( [ this ] ( StringRef PassName , FunctionPassManager & PM , ArrayRef < PassBuilder :: PipelineElement > ) { if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>SimplifyLibCallsPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaToVectorPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>LowerKernelAttributesPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; return true ; } return false ; } ) ; PB . registerAnalysisRegistrationCallback ( [ ] ( FunctionAnalysisManager & FAM ) { FAM . registerPass ( [ & ] { return <ISA_LIT>AA ( ) ; } ) ; } ) ; PB . registerParseAACallback ( [ ] ( StringRef AAName , AAManager & AAM ) { if ( AAName == <STR_LIT> ) { AAM . registerFunctionAnalysis < <ISA_LIT>AA > ( ) ; return true ; } return false ; } ) ; PB . registerPipelineStartEPCallback ( [ this ] ( ModulePassManager & PM , OptimizationLevel Level ) { FunctionPassManager FPM ; FPM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ;

FPM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ;


if ( auto * WrapperPass = P . getAnalysisIfAvailable < <ISA_LIT>AAWrapperPass > ( ) ) AAR . addAAResult ( WrapperPass -> getResult ( ) ) ;


PM . addPass ( <ISA_LIT>PrintfRuntimeBindingPass ( ) ) ;


<s> } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PromoteAllocaToVectorPass ( * this ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>LowerKernelAttributesPass ( ) ) ; return true ; } if ( PassName == <STR_LIT> ) { PM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; return true ; } return false ; } ) ; PB . registerAnalysisRegistrationCallback ( [ ] ( FunctionAnalysisManager & FAM ) { FAM . registerPass ( [ & ] { return <ISA_LIT>AA ( ) ; } ) ; } ) ; PB . registerParseAACallback ( [ ] ( StringRef AAName , AAManager & AAM ) { if ( AAName == <STR_LIT> ) { AAM . registerFunctionAnalysis < <ISA_LIT>AA > ( ) ; return true ; } return false ; } ) ; PB . registerPipelineStartEPCallback ( [ this ] ( ModulePassManager & PM , PassBuilder :: OptimizationLevel Level ) { FunctionPassManager FPM ; FPM . addPass ( <ISA_LIT>PropagateAttributesEarlyPass ( * this ) ) ; FPM . addPass ( <ISA_LIT>UseNativeCallsPass ( ) ) ; if ( EnableLibCallSimplify && Level != PassBuilder :: OptimizationLevel :: O0 ) FPM . addPass ( <ISA_LIT>SimplifyLibCallsPass ( * this ) ) ; PM . addPass ( createModuleToFunctionPassAdaptor ( std :: move ( FPM ) ) ) ;


<s> Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( [ = ] ( const GlobalValue & GV ) -> bool { if ( const Function * F = dyn_cast < Function > ( & GV ) ) { if ( F -> isDeclaration ( ) ) return true ; switch ( F -> getCallingConv ( ) ) { default : return false ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true ;


<s> disablePass ( & PatchableFunctionID ) ; addPass ( create<ISA_LIT>AlwaysInlinePass ( ) ) ; addPass ( createAlwaysInlinerPass ( ) ) ;


<s> void GCNPassConfig :: addPreEmitPass ( ) { addPass ( & PostRAHazardRecognizerID ) ;

addPass ( create<ISA_LIT>InsertWaitsPass ( ) , false ) ;


<s> void GCNPassConfig :: addPreRegAlloc ( ) { const <ISA_LIT>Subtarget & ST = * get<ISA_LIT>TargetMachine ( ) . getSubtargetImpl ( ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ;


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Optional < Reloc :: Model > RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , getGPUOrDefault ( TT , CPU ) , FS , Options , getEffectiveRelocModel ( RM ) , CM , OptLevel ) , TLOF ( createTLOF ( getTargetTriple ( ) ) ) , Subtarget ( TT , getTargetCPU ( ) , FS , * this ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true ) ;


<s> if ( ! RM . hasValue ( ) ) return Reloc :: PIC_ ;


<s> static Reloc :: Model getEffectiveRelocModel ( Optional < Reloc :: Model > RM ) { if ( ! RM . hasValue ( ) ) return Reloc :: PIC_ ;


<s> void LLVMInitialize<ISA_LIT>Target ( ) { RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ; RegisterTargetMachine < GCNTargetMachine > Y ( TheGCNTarget ) ; PassRegistry * PR = PassRegistry :: getPassRegistry ( ) ; initialize<ISA_LIT>LowerI1CopiesPass ( * PR ) ;

initialize<ISA_LIT>FixSGPRCopiesPass ( * PR ) ;


initialize<ISA_LIT>LowerControlFlowPass ( * PR ) ;


<s> <ISA_LIT>PassConfig :: addPreISel ( ) ; addPass ( & <ISA_LIT>AnnotateKernelFeaturesID ) ; addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>TypeRewriter ( ) ) ; addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ; return false ;


<s> bool GCNPassConfig :: addPreISel ( ) { <ISA_LIT>PassConfig :: addPreISel ( ) ; addPass ( & <ISA_LIT>AnnotateKernelFeaturesID ) ; addPass ( createSinkingPass ( ) ) ; addPass ( create<ISA_LIT>TypeRewriter ( ) ) ; addPass ( create<ISA_LIT>AnnotateControlFlowPass ( ) ) ;


<s> addPass ( create<ISA_LIT>AlwaysInlinePass ( ) ) ;

addPass ( createAlwaysInlinerPass ( ) ) ;


<s> addPass ( createBarrierNoopPass ( ) ) ;


insertPass ( & MachineSchedulerID , & RegisterCoalescerID ) ;


<s> <ISA_LIT>PassConfig ( TargetMachine * TM , PassManagerBase & PM ) : TargetPassConfig ( TM , PM ) {


<s> <ISA_LIT>TargetMachine :: <ISA_LIT>TargetMachine ( const Target & T , const Triple & TT , StringRef CPU , StringRef FS , TargetOptions Options , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OptLevel ) : LLVMTargetMachine ( T , computeDataLayout ( TT ) , TT , CPU , FS , Options , RM , CM , OptLevel ) , TLOF ( new TargetLoweringObjectFileELF ( ) ) , Subtarget ( TT , CPU , FS , * this ) , IntrinsicInfo ( ) { setRequiresStructuredCFG ( true ) ; initAsmInfo ( ) ;


<s> return TargetIRAnalysis ( [ this ] ( Function & F ) { return TargetTransformInfo ( <ISA_LIT>TTIImpl ( this , F . getParent ( ) -> getDataLayout ( ) ) ) ;


<s> RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ;

RegisterTargetMachine < GCNTargetMachine > Y ( TheGCNTarget ) ;


<s> void LLVMInitialize<ISA_LIT>Target ( ) { RegisterTargetMachine < <ISA_LIT>TargetMachine > X ( The<ISA_LIT>Target ) ;


<s> TargetPassConfig :: addCodeGenPrepare ( ) ; if ( EnableLoadStoreVectorizer . getNumOccurrences ( ) ? EnableLoadStoreVectorizer : TM -> getOptLevel ( ) > CodeGenOpt :: Less ) addPass ( createLoadStoreVectorizerPass ( ) ) ; addPass ( createLowerSwitchPass ( ) ) ;


if ( EnableScalarIRPasses . getNumOccurrences ( ) ? EnableScalarIRPasses : TM . getOptLevel ( ) > CodeGenOpt :: Less ) addEarlyCSEOrGVNPass ( ) ;


<s> addPass ( & EarlyMachineLICMID ) ; addPass ( & MachineCSEID ) ; addPass ( & <ISA_LIT>FoldOperandsID ) ; } addPass ( & DeadMachineInstructionElimID ) ; addPass ( create<ISA_LIT>ShrinkInstructionsPass ( ) ) ;


<s> if ( TM -> getOptLevel ( ) > CodeGenOpt :: Less ) insertPass ( & MachineSchedulerID , & <ISA_LIT>FormMemoryClausesID ) ; insertPass ( & PHIEliminationID , & <ISA_LIT>LowerControlFlowID , false ) ; if ( EnableDCEInRA ) insertPass ( & DetectDeadLanesID , & DeadMachineInstructionElimID ) ;

TargetPassConfig :: addOptimizedRegAlloc ( ) ;


<s> addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>PreEmitPeepholeID ) ;

addPass ( & PostRAHazardRecognizerID ) ;


<s> addPass ( create<ISA_LIT>ModeRegisterPass ( ) ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>InsertHardClausesID ) ; addPass ( & <ISA_LIT>LateBranchLoweringPassID ) ; if ( getOptLevel ( ) > CodeGenOpt :: None ) addPass ( & <ISA_LIT>PreEmitPeepholeID ) ;


<s> <ISA_LIT>PassConfig :: addPreISel ( ) ; addPass ( create<ISA_LIT>LateCodeGenPreparePass ( ) ) ; if ( EnableAtomicOptimizations ) { addPass ( create<ISA_LIT>AtomicOptimizerPass ( ) ) ; } addPass ( createSinkingPass ( ) ) ; addPass ( & <ISA_LIT>UnifyDivergentExitNodesID ) ; if ( ! LateCFGStructurize ) { if ( EnableStructurizerWorkarounds ) { addPass ( createFixIrreduciblePass ( ) ) ; addPass ( createUnifyLoopExitsPass ( ) ) ; } addPass ( createStructurizeCFGPass ( false ) ) ; } addPass ( create<ISA_LIT>AnnotateUniformValues ( ) ) ;


PM . addPass ( createCGSCCToFunctionPassAdaptor ( std :: move ( FPM ) ) ) ;


initializeGCNRegBankReassignPass ( * PR ) ;


<s> } Builder . addExtension ( PassManagerBuilder :: EP_ModuleOptimizerEarly , [ Internalize , EarlyInline , <ISA_LIT>AA ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ; } PM . add ( create<ISA_LIT>UnifyMetadataPass ( ) ) ; if ( Internalize ) { PM . add ( createInternalizePass ( mustPreserveGV ) ) ; PM . add ( createGlobalDCEPass ( ) ) ; } if ( EarlyInline ) PM . add ( create<ISA_LIT>AlwaysInlinePass ( false ) ) ; } ) ; const auto & Opt = Options ; Builder . addExtension ( PassManagerBuilder :: EP_EarlyAsPossible , [ <ISA_LIT>AA , LibCallSimplify , & Opt ] ( const PassManagerBuilder & , legacy :: PassManagerBase & PM ) { if ( <ISA_LIT>AA ) { PM . add ( create<ISA_LIT>AAWrapperPass ( ) ) ; PM . add ( create<ISA_LIT>ExternalAAWrapperPass ( ) ) ;


<s> void GCNPassConfig :: addFastRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & TwoAddressInstructionPassID , & <ISA_LIT>LowerControlFlowID , false ) ; TargetPassConfig :: addFastRegAlloc ( RegAllocPass ) ;


<s> const <ISA_LIT>TargetMachine & TM = get<ISA_LIT>TargetMachine ( ) ; if ( TM . getOptLevel ( ) > CodeGenOpt :: None ) { addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; } addStraightLineScalarOptimizationPasses ( ) ;


<s> const <ISA_LIT>TargetMachine & TM = get<ISA_LIT>TargetMachine ( ) ; if ( TM . getOptLevel ( ) > CodeGenOpt :: None ) { addPass ( create<ISA_LIT>PromoteAlloca ( & TM ) ) ; if ( EnableSROA ) addPass ( createSROAPass ( ) ) ; } addStraightLineScalarOptimizationPasses ( ) ; TargetPassConfig :: addIRPasses ( ) ;

if ( getOptLevel ( ) != CodeGenOpt :: None ) addEarlyCSEOrGVNPass ( ) ;


<s> void GCNPassConfig :: addOptimizedRegAlloc ( FunctionPass * RegAllocPass ) { insertPass ( & MachineSchedulerID , & <ISA_LIT>FixControlFlowLiveIntervalsID ) ; insertPass ( & RenameIndependentSubregsID , & <ISA_LIT>LowerControlFlowID ) ;

TargetPassConfig :: addOptimizedRegAlloc ( RegAllocPass ) ;