<s> if ( MI -> isBundle ( ) ) { const MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineBasicBlock :: const_instr_iterator I = ++ MI -> getIterator ( ) ; while ( I != MBB -> instr_end ( ) && I -> isInsideBundle ( ) ) { emitInstruction ( & * I ) ; ++ I ; } } else { if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> isMetaInstruction ( ) ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } MCInst TmpInst ; MCInstLowering . lower ( MI , TmpInst ) ; EmitToStreamer ( * OutStreamer , TmpInst ) ; if ( ! MI -> isPseudo ( ) && STI . isCPUStringValid ( STI . getCPU ( ) ) && ( ! STI . hasOffset3fBug ( ) || ! MI -> isBranch ( ) ) ) { SmallVector < MCFixup , <NUM_LIT> > Fixups ; SmallVector < char , <NUM_LIT> > CodeBytes ; raw_svector_ostream CodeStream ( CodeBytes ) ; std :: unique_ptr < MCCodeEmitter > InstEmitter ( create<ISA_LIT>MCCodeEmitter ( * STI . getInstrInfo ( ) , * OutContext . getRegisterInfo ( ) , OutContext ) ) ; InstEmitter -> encodeInstruction ( TmpInst , CodeStream , Fixups , STI ) ;

assert ( CodeBytes . size ( ) == STI . getInstrInfo ( ) -> getInstSizeInBytes ( * MI ) ) ;


<s> StringRef Err ; if ( ! STI . getInstrInfo ( ) -> verifyInstruction ( * MI , Err ) ) { LLVMContext & C = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) . getContext ( ) ; C . emitError ( <STR_LIT> + Err ) ; MI -> print ( errs ( ) ) ; } if ( MI -> isBundle ( ) ) { const MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineBasicBlock :: const_instr_iterator I = ++ MI -> getIterator ( ) ; while ( I != MBB -> instr_end ( ) && I -> isInsideBundle ( ) ) { emitInstruction ( & * I ) ; ++ I ; } } else { if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> isMetaInstruction ( ) ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } MCInst TmpInst ; MCInstLowering . lower ( MI , TmpInst ) ; EmitToStreamer ( * OutStreamer , TmpInst ) ; if ( ! MI -> isPseudo ( ) && STI . isCPUStringValid ( STI . getCPU ( ) ) && ( ! STI . hasOffset3fBug ( ) || ! MI -> isBranch ( ) ) ) { SmallVector < MCFixup , <NUM_LIT> > Fixups ; SmallVector < char , <NUM_LIT> > CodeBytes ; raw_svector_ostream CodeStream ( CodeBytes ) ; std :: unique_ptr < MCCodeEmitter > InstEmitter ( create<ISA_LIT>MCCodeEmitter ( * STI . getInstrInfo ( ) , * OutContext . getRegisterInfo ( ) , OutContext ) ) ; InstEmitter -> encodeInstruction ( TmpInst , CodeStream , Fixups , STI ) ; assert ( CodeBytes . size ( ) == STI . getInstrInfo ( ) -> getInstSizeInBytes ( * MI ) ) ;


<s> if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) Opcode = <STR_LIT> ; else if ( Opcode == <STR_LIT> ) { OutMI . setOpcode ( TII -> pseudoToMCOpcode ( <STR_LIT> ) ) ; MCOperand Dest , Src ; lowerOperand ( MI -> getOperand ( <NUM_LIT> ) , Dest ) ; lowerOperand ( MI -> getOperand ( <NUM_LIT> ) , Src ) ; OutMI . addOperand ( Dest ) ; OutMI . addOperand ( Src ) ; return ; } else if ( Opcode == <STR_LIT> ) { Opcode = <STR_LIT> ; } int MCOpcode = TII -> pseudoToMCOpcode ( Opcode ) ; if ( MCOpcode == - <NUM_LIT> ) { LLVMContext & C = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) . getContext ( ) ; C . emitError ( <STR_LIT> <STR_LIT> + Twine ( MI -> getOpcode ( ) ) ) ; } OutMI . setOpcode ( MCOpcode ) ; for ( const MachineOperand & MO : MI -> explicit_operands ( ) ) { MCOperand MCOp ; lowerOperand ( MO , MCOp ) ; OutMI . addOperand ( MCOp ) ; }

int FIIdx = <STR_LIT> ( MCOpcode , <STR_LIT> :: fi ) ;


<s> return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } MCInst TmpInst ; MCInstLowering . lower ( MI , TmpInst ) ; EmitToStreamer ( * OutStreamer , TmpInst ) ; if ( STI . dumpCode ( ) ) { DisasmLines . resize ( DisasmLines . size ( ) + <NUM_LIT> ) ; std :: string & DisasmLine = DisasmLines . back ( ) ; raw_string_ostream DisasmStream ( DisasmLine ) ; <ISA_LIT>InstPrinter InstPrinter ( * TM . getMCAsmInfo ( ) , * STI . getInstrInfo ( ) , * STI . getRegisterInfo ( ) ) ; InstPrinter . printInst ( & TmpInst , DisasmStream , StringRef ( ) , STI ) ; SmallVector < MCFixup , <NUM_LIT> > Fixups ; SmallVector < char , <NUM_LIT> > CodeBytes ; raw_svector_ostream CodeStream ( CodeBytes ) ; auto & ObjStreamer = static_cast < MCObjectStreamer & > ( * OutStreamer ) ; MCCodeEmitter & InstEmitter = ObjStreamer . getAssembler ( ) . getEmitter ( ) ; InstEmitter . encodeInstruction ( TmpInst , CodeStream , Fixups , MF -> getSubtarget < MCSubtargetInfo > ( ) ) ; HexLines . resize ( HexLines . size ( ) + <NUM_LIT> ) ; std :: string & HexLine = HexLines . back ( ) ; raw_string_ostream HexStream ( HexLine ) ; for ( size_t i = <NUM_LIT> ; i < CodeBytes . size ( ) ; i += <NUM_LIT> ) { unsigned int CodeDWord = * ( unsigned int * ) & CodeBytes [ i ] ;

HexStream << format ( <STR_LIT> , ( i > <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) , CodeDWord ) ;


<s> break ; case MachineOperand :: MO_MachineBasicBlock : MCOp = MCOperand :: createExpr ( MCSymbolRefExpr :: create ( MO . getMBB ( ) -> getSymbol ( ) , Ctx ) ) ; break ; case MachineOperand :: MO_GlobalAddress : { const GlobalValue * GV = MO . getGlobal ( ) ; MCSymbol * Sym = Ctx . getOrCreateSymbol ( StringRef ( GV -> getName ( ) ) ) ; const MCExpr * SymExpr = MCSymbolRefExpr :: create ( Sym , Ctx ) ; const MCExpr * Expr = MCBinaryExpr :: createAdd ( SymExpr , MCConstantExpr :: create ( MO . getOffset ( ) , Ctx ) , Ctx ) ; MCOp = MCOperand :: createExpr ( Expr ) ; break ; } case MachineOperand :: MO_ExternalSymbol : { MCSymbol * Sym = Ctx . getOrCreateSymbol ( StringRef ( MO . getSymbolName ( ) ) ) ; Sym -> setExternal ( true ) ; const MCSymbolRefExpr * Expr = MCSymbolRefExpr :: create ( Sym , Ctx ) ; MCOp = MCOperand :: createExpr ( Expr ) ; break ; } }


<s> DisasmLines . resize ( DisasmLines . size ( ) + <NUM_LIT> ) ; std :: string & DisasmLine = DisasmLines . back ( ) ; raw_string_ostream DisasmStream ( DisasmLine ) ; <ISA_LIT>InstPrinter InstPrinter ( * TM . getMCAsmInfo ( ) , * STI . getInstrInfo ( ) , * STI . getRegisterInfo ( ) ) ; InstPrinter . printInst ( & TmpInst , DisasmStream , StringRef ( ) , STI ) ; SmallVector < MCFixup , <NUM_LIT> > Fixups ; SmallVector < char , <NUM_LIT> > CodeBytes ; raw_svector_ostream CodeStream ( CodeBytes ) ; auto & ObjStreamer = static_cast < MCObjectStreamer & > ( * OutStreamer ) ; MCCodeEmitter & InstEmitter = ObjStreamer . getAssembler ( ) . getEmitter ( ) ; InstEmitter . encodeInstruction ( TmpInst , CodeStream , Fixups , MF -> getSubtarget < MCSubtargetInfo > ( ) ) ; HexLines . resize ( HexLines . size ( ) + <NUM_LIT> ) ;

std :: string & HexLine = HexLines . back ( ) ;


<s> while ( I != MBB -> instr_end ( ) && I -> isInsideBundle ( ) ) { EmitInstruction ( & * I ) ; ++ I ; } } else { if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) { SmallVector < char , <NUM_LIT> > BBStr ; raw_svector_ostream Str ( BBStr ) ; const MachineBasicBlock * MBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; const MCSymbolRefExpr * Expr = MCSymbolRefExpr :: create ( MBB -> getSymbol ( ) , OutContext ) ; Expr -> print ( Str , MAI ) ; OutStreamer -> emitRawComment ( <STR_LIT> + BBStr ) ; } return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } MCInst TmpInst ; MCInstLowering . lower ( MI , TmpInst ) ; EmitToStreamer ( * OutStreamer , TmpInst ) ;


<s> default : return MCSymbolRefExpr :: VK_None ; case <ISA_LIT>InstrInfo :: MO_GOTPCREL : return MCSymbolRefExpr :: VK_GOTPCREL ;


<s> switch ( MOFlags ) { default : return MCSymbolRefExpr :: VK_None ;

case <ISA_LIT>InstrInfo :: MO_GOTPCREL : return MCSymbolRefExpr :: VK_GOTPCREL ;


<s> void <ISA_LIT>MCInstLower :: lower ( const MachineInstr * MI , MCInst & OutMI ) const { int MCOpcode = ST . getInstrInfo ( ) -> pseudoToMCOpcode ( MI -> getOpcode ( ) ) ; if ( MCOpcode == - <NUM_LIT> ) { LLVMContext & C = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) -> getContext ( ) ; C . emitError ( <STR_LIT> <STR_LIT> + Twine ( MI -> getOpcode ( ) ) ) ; } OutMI . setOpcode ( MCOpcode ) ; for ( const MachineOperand & MO : MI -> explicit_operands ( ) ) { MCOperand MCOp ; switch ( MO . getType ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case MachineOperand :: MO_Immediate : MCOp = MCOperand :: createImm ( MO . getImm ( ) ) ; break ; case MachineOperand :: MO_Register : MCOp = MCOperand :: createReg ( <STR_LIT> ( MO . getReg ( ) , ST ) ) ; break ;

case MachineOperand :: MO_MachineBasicBlock :


<s> MachineBasicBlock :: const_instr_iterator I = ++ MI -> getIterator ( ) ; while ( I != MBB -> instr_end ( ) && I -> isInsideBundle ( ) ) { EmitInstruction ( & * I ) ; ++ I ; } } else { if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) { SmallVector < char , <NUM_LIT> > BBStr ; raw_svector_ostream Str ( BBStr ) ; const MachineBasicBlock * MBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; const MCSymbolRefExpr * Expr = MCSymbolRefExpr :: create ( MBB -> getSymbol ( ) , OutContext ) ; Expr -> print ( Str , MAI ) ; OutStreamer -> emitRawComment ( Twine ( <STR_LIT> ) + BBStr ) ; } return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) {


<s> LLVMContext & C = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) . getContext ( ) ; C . emitError ( <STR_LIT> + Err ) ; MI -> print ( errs ( ) ) ; } if ( MI -> isBundle ( ) ) { const MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineBasicBlock :: const_instr_iterator I = ++ MI -> getIterator ( ) ; while ( I != MBB -> instr_end ( ) && I -> isInsideBundle ( ) ) { EmitInstruction ( & * I ) ; ++ I ; } } else { if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) { SmallVector < char , <NUM_LIT> > BBStr ; raw_svector_ostream Str ( BBStr ) ; const MachineBasicBlock * MBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; const MCSymbolRefExpr * Expr = MCSymbolRefExpr :: create ( MBB -> getSymbol ( ) , OutContext ) ; Expr -> print ( Str , MAI ) ; OutStreamer -> emitRawComment ( Twine ( <STR_LIT> ) + BBStr ) ; } return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) OutStreamer -> emitRawComment ( <STR_LIT> ) ; return ; } MCInst TmpInst ; MCInstLowering . lower ( MI , TmpInst ) ;

EmitToStreamer ( * OutStreamer , TmpInst ) ;


<s> void <ISA_LIT>AsmPrinter :: EmitInstruction ( const MachineInstr * MI ) { if ( emitPseudoExpansionLowering ( * OutStreamer , MI ) ) return ; const GCNSubtarget & STI = MF -> getSubtarget < GCNSubtarget > ( ) ; <ISA_LIT>MCInstLower MCInstLowering ( OutContext , STI , * this ) ; StringRef Err ; if ( ! STI . getInstrInfo ( ) -> verifyInstruction ( * MI , Err ) ) { LLVMContext & C = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) . getContext ( ) ; C . emitError ( <STR_LIT> + Err ) ; MI -> print ( errs ( ) ) ; } if ( MI -> isBundle ( ) ) { const MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineBasicBlock :: const_instr_iterator I = ++ MI -> getIterator ( ) ; while ( I != MBB -> instr_end ( ) && I -> isInsideBundle ( ) ) { EmitInstruction ( & * I ) ; ++ I ; } } else { if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( isVerbose ( ) ) { SmallVector < char , <NUM_LIT> > BBStr ; raw_svector_ostream Str ( BBStr ) ; const MachineBasicBlock * MBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ;

const MCSymbolRefExpr * Expr = MCSymbolRefExpr :: create ( MBB -> getSymbol ( ) , OutContext ) ;


<s> auto SrcAddr = Op -> getType ( ) -> getPointerAddressSpace ( ) ; if ( Op -> isNullValue ( ) && AT . getNullPointerValue ( SrcAddr ) == <NUM_LIT> ) { auto DstAddr = CE -> getType ( ) -> getPointerAddressSpace ( ) ; return MCConstantExpr :: create ( AT . getNullPointerValue ( DstAddr ) , OutContext ) ;


<s> while ( I != MBB -> end ( ) && I -> isInsideBundle ( ) ) { EmitInstruction ( I ) ; ++ I ; } } else { MCInst TmpInst ; MCInstLowering . lower ( MI , TmpInst ) ; EmitToStreamer ( * OutStreamer , TmpInst ) ; if ( STI . dumpCode ( ) ) { DisasmLines . resize ( DisasmLines . size ( ) + <NUM_LIT> ) ; std :: string & DisasmLine = DisasmLines . back ( ) ; raw_string_ostream DisasmStream ( DisasmLine ) ; <ISA_LIT>InstPrinter InstPrinter ( * TM . getMCAsmInfo ( ) , * MF -> getSubtarget ( ) . getInstrInfo ( ) , * MF -> getSubtarget ( ) . getRegisterInfo ( ) ) ; InstPrinter . printInst ( & TmpInst , DisasmStream , StringRef ( ) , MF -> getSubtarget ( ) ) ; SmallVector < MCFixup , <NUM_LIT> > Fixups ; SmallVector < char , <NUM_LIT> > CodeBytes ;

raw_svector_ostream CodeStream ( CodeBytes ) ;


<s> MCOp = MCOperand :: createImm ( MO . getImm ( ) ) ; break ; case MachineOperand :: MO_Register : MCOp = MCOperand :: createReg ( MO . getReg ( ) ) ; break ; case MachineOperand :: MO_MachineBasicBlock : MCOp = MCOperand :: createExpr ( MCSymbolRefExpr :: create ( MO . getMBB ( ) -> getSymbol ( ) , Ctx ) ) ; break ; case MachineOperand :: MO_GlobalAddress : { const GlobalValue * GV = MO . getGlobal ( ) ; MCSymbol * Sym = Ctx . getOrCreateSymbol ( StringRef ( GV -> getName ( ) ) ) ; MCOp = MCOperand :: createExpr ( MCSymbolRefExpr :: create ( Sym , Ctx ) ) ;


<s> break ; case MachineOperand :: MO_Register : MCOp = MCOperand :: createReg ( MO . getReg ( ) ) ; break ; case MachineOperand :: MO_MachineBasicBlock : MCOp = MCOperand :: createExpr ( MCSymbolRefExpr :: create ( MO . getMBB ( ) -> getSymbol ( ) , Ctx ) ) ; break ; case MachineOperand :: MO_GlobalAddress : { const GlobalValue * GV = MO . getGlobal ( ) ; MCSymbol * Sym = Ctx . getOrCreateSymbol ( StringRef ( GV -> getName ( ) ) ) ; MCOp = MCOperand :: createExpr ( MCSymbolRefExpr :: create ( Sym , Ctx ) ) ; break ; } case MachineOperand :: MO_TargetIndex : { assert ( MO . getIndex ( ) == <STR_LIT> ) ; MCSymbol * Sym = Ctx . getOrCreateSymbol ( StringRef ( END_OF_TEXT_LABEL_NAME ) ) ; const MCSymbolRefExpr * Expr = MCSymbolRefExpr :: create ( Sym , Ctx ) ; MCOp = MCOperand :: createExpr ( Expr ) ; break ; } case MachineOperand :: MO_ExternalSymbol : { MCSymbol * Sym = Ctx . getOrCreateSymbol ( StringRef ( MO . getSymbolName ( ) ) ) ; const MCSymbolRefExpr * Expr = MCSymbolRefExpr :: create ( Sym , Ctx ) ;


<s> if ( TT . getArch ( ) == Triple :: r600 ) Init<ISA_LIT>MCRegisterInfo ( X , <NUM_LIT> ) ; else Init<ISA_LIT>MCRegisterInfo ( X , <NUM_LIT> ) ;


<s> MCRegisterInfo * X = new MCRegisterInfo ( ) ; if ( TT . getArch ( ) == Triple :: r600 ) Init<ISA_LIT>MCRegisterInfo ( X , <NUM_LIT> ) ;

else Init<ISA_LIT>MCRegisterInfo ( X , <NUM_LIT> ) ;


<s> if ( TT . getArch ( ) == Triple :: r600 ) return create<ISA_LIT>MCSubtargetInfoImpl ( TT , CPU , FS ) ; return create<ISA_LIT>MCSubtargetInfoImpl ( TT , CPU , FS ) ;


<s> static MCSubtargetInfo * create<ISA_LIT>MCSubtargetInfo ( const Triple & TT , StringRef CPU , StringRef FS ) { if ( TT . getArch ( ) == Triple :: r600 ) return create<ISA_LIT>MCSubtargetInfoImpl ( TT , CPU , FS ) ; return create<ISA_LIT>MCSubtargetInfoImpl ( TT , CPU , FS ) ;


<s> RegisterMCAsmInfo < <ISA_LIT>MCAsmInfo > X ( * T ) ; TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( * T , create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCAsmBackend ( * T , create<ISA_LIT>AsmBackend ) ; TargetRegistry :: RegisterELFStreamer ( * T , createMCStreamer ) ; } TargetRegistry :: RegisterMCCodeEmitter ( getThe<ISA_LIT>Target ( ) , create<ISA_LIT>MCCodeEmitter ) ;

TargetRegistry :: RegisterObjectTargetStreamer ( getThe<ISA_LIT>Target ( ) , create<ISA_LIT>ObjectTargetStreamer ) ;


<s> } TargetRegistry :: RegisterMCCodeEmitter ( getThe<ISA_LIT>Target ( ) , create<ISA_LIT>MCCodeEmitter ) ; TargetRegistry :: RegisterObjectTargetStreamer ( getThe<ISA_LIT>Target ( ) , create<ISA_LIT>ObjectTargetStreamer ) ; TargetRegistry :: RegisterMCCodeEmitter ( getTheGCNTarget ( ) , create<ISA_LIT>MCCodeEmitter ) ; TargetRegistry :: RegisterAsmTargetStreamer ( getTheGCNTarget ( ) , create<ISA_LIT>AsmTargetStreamer ) ; TargetRegistry :: RegisterObjectTargetStreamer ( getTheGCNTarget ( ) , create<ISA_LIT>ObjectTargetStreamer ) ;


<s> TargetRegistry :: RegisterMCAsmBackend ( * T , create<ISA_LIT>AsmBackend ) ; TargetRegistry :: RegisterELFStreamer ( * T , createMCStreamer ) ; } TargetRegistry :: RegisterMCCodeEmitter ( getThe<ISA_LIT>Target ( ) , create<ISA_LIT>MCCodeEmitter ) ; TargetRegistry :: RegisterObjectTargetStreamer ( getThe<ISA_LIT>Target ( ) , create<ISA_LIT>ObjectTargetStreamer ) ; TargetRegistry :: RegisterMCCodeEmitter ( getTheGCNTarget ( ) , create<ISA_LIT>MCCodeEmitter ) ; TargetRegistry :: RegisterAsmTargetStreamer ( getTheGCNTarget ( ) , create<ISA_LIT>AsmTargetStreamer ) ;

TargetRegistry :: RegisterObjectTargetStreamer ( getTheGCNTarget ( ) , create<ISA_LIT>ObjectTargetStreamer ) ;


<s> static MCTargetStreamer * create<ISA_LIT>AsmTargetStreamer ( MCStreamer & S , formatted_raw_ostream & OS , MCInstPrinter * InstPrint , bool isVerboseAsm ) {

return new <ISA_LIT>TargetAsmStreamer ( S , OS ) ;


<s> if ( TT . getArch ( ) == Triple :: r600 ) Init<ISA_LIT>MCRegisterInfo ( X , <NUM_LIT> ) ; else Init<ISA_LIT>MCRegisterInfo ( X , <STR_LIT> ) ;


<s> static MCInstPrinter * create<ISA_LIT>MCInstPrinter ( const Triple & T , unsigned SyntaxVariant , const MCAsmInfo & MAI , const MCInstrInfo & MII , const MCRegisterInfo & MRI ) { return new <ISA_LIT>InstPrinter ( MAI , MII , MRI ) ;


<s> MCRegisterInfo * X = new MCRegisterInfo ( ) ; Init<ISA_LIT>MCRegisterInfo ( X , <NUM_LIT> ) ;


<s> static MCSubtargetInfo * create<ISA_LIT>MCSubtargetInfo ( const Triple & TT , StringRef CPU , StringRef FS ) {

return create<ISA_LIT>MCSubtargetInfoImpl ( TT , CPU , FS ) ;


<s> return new <ISA_LIT>TargetELFStreamer ( S ) ;


<s> static MCStreamer * createMCStreamer ( const Triple & T , MCContext & Context , MCAsmBackend & MAB , raw_pwrite_stream & OS , MCCodeEmitter * Emitter , bool RelaxAll ) {

if ( T . getOS ( ) == Triple :: AMDHSA ) return create<ISA_LIT>ELFStreamer ( Context , MAB , OS , Emitter , RelaxAll ) ;


<s> RegisterMCAsmInfo < <ISA_LIT>MCAsmInfo > X ( * T ) ; TargetRegistry :: RegisterMCInstrInfo ( * T , create<ISA_LIT>MCInstrInfo ) ; TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( * T , create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCAsmBackend ( * T , create<ISA_LIT>AsmBackend ) ; TargetRegistry :: RegisterELFStreamer ( * T , createMCStreamer ) ; }

TargetRegistry :: RegisterMCCodeEmitter ( getThe<ISA_LIT>Target ( ) , create<ISA_LIT>MCCodeEmitter ) ;


<s> explicit <ISA_LIT>MCInstrAnalysis ( const MCInstrInfo * Info ) : MCInstrAnalysis ( Info ) {


<s> return new <ISA_LIT>TargetAsmStreamer ( S , OS ) ;


<s> static MCTargetStreamer * create<ISA_LIT>AsmTargetStreamer ( MCStreamer & S , formatted_raw_ostream & OS , MCInstPrinter * InstPrint , bool isVerboseAsm ) { return new <ISA_LIT>TargetAsmStreamer ( S , OS ) ;


<s> if ( T . getArch ( ) == Triple :: r600 ) return new <ISA_LIT>InstPrinter ( MAI , MII , MRI ) ;

else return new <ISA_LIT>InstPrinter ( MAI , MII , MRI ) ;


<s> static MCInstPrinter * create<ISA_LIT>MCInstPrinter ( const Triple & T , unsigned SyntaxVariant , const MCAsmInfo & MAI , const MCInstrInfo & MII , const MCRegisterInfo & MRI ) { if ( T . getArch ( ) == Triple :: r600 ) return new <ISA_LIT>InstPrinter ( MAI , MII , MRI ) ;


<s> static MCInstrAnalysis * create<ISA_LIT>MCInstrAnalysis ( const MCInstrInfo * Info ) { return new <ISA_LIT>MCInstrAnalysis ( Info ) ;


<s> return new <ISA_LIT>MCInstrAnalysis ( Info ) ;


<s> Init<ISA_LIT>MCInstrInfo ( X ) ; return X ;


<s> MCInstrInfo * X = new MCInstrInfo ( ) ; Init<ISA_LIT>MCInstrInfo ( X ) ;


<s> MCRegisterInfo * X = new MCRegisterInfo ( ) ; if ( TT . getArch ( ) == Triple :: r600 ) Init<ISA_LIT>MCRegisterInfo ( X , <NUM_LIT> ) ; else Init<ISA_LIT>MCRegisterInfo ( X , <STR_LIT> ) ;


<s> if ( TT . getArch ( ) == Triple :: r600 ) Init<ISA_LIT>MCRegisterInfo ( X , <NUM_LIT> ) ; else Init<ISA_LIT>MCRegisterInfo ( X , <STR_LIT> ) ; return X ;


<s> static MCSubtargetInfo * create<ISA_LIT>MCSubtargetInfo ( const Triple & TT , StringRef CPU , StringRef FS ) { if ( TT . getArch ( ) == Triple :: r600 ) return create<ISA_LIT>MCSubtargetInfoImpl ( TT , CPU , CPU , FS ) ;


<s> static MCTargetStreamer * create<ISA_LIT>ObjectTargetStreamer ( MCStreamer & S , const MCSubtargetInfo & STI ) {

return new <ISA_LIT>TargetELFStreamer ( S , STI ) ;


<s> MCRegisterInfo * X = new MCRegisterInfo ( ) ; Init<ISA_LIT>MCRegisterInfo ( X , <STR_LIT> , DwarfFlavour ) ; return X ;


<s> static MCStreamer * createMCStreamer ( const Triple & T , MCContext & Context , std :: unique_ptr < MCAsmBackend > && MAB , std :: unique_ptr < MCObjectWriter > && OW , std :: unique_ptr < MCCodeEmitter > && Emitter , bool RelaxAll ) { return create<ISA_LIT>ELFStreamer ( T , Context , std :: move ( MAB ) , std :: move ( OW ) , std :: move ( Emitter ) , RelaxAll ) ;


<s> bool evaluateBranch ( const MCInst & Inst , uint64_t Addr , uint64_t Size , uint64_t & Target ) const override { if ( Inst . getNumOperands ( ) == <NUM_LIT> || ! Inst . getOperand ( <NUM_LIT> ) . isImm ( ) || Info -> get ( Inst . getOpcode ( ) ) . OpInfo [ <NUM_LIT> ] . OperandType != MCOI :: OPERAND_PCREL ) return false ;

int64_t Imm = Inst . getOperand ( <NUM_LIT> ) . getImm ( ) ;


<s> LLVM_EXTERNAL_VI<ISA_LIT>BILITY void LLVMInitialize<ISA_LIT>TargetMC ( ) { TargetRegistry :: RegisterMCInstrInfo ( getTheGCNTarget ( ) , create<ISA_LIT>MCInstrInfo ) ; TargetRegistry :: RegisterMCInstrInfo ( getThe<ISA_LIT>Target ( ) , create<ISA_LIT>MCInstrInfo ) ; for ( Target * T : { & getThe<ISA_LIT>Target ( ) , & getTheGCNTarget ( ) } ) { RegisterMCAsmInfo < <ISA_LIT>MCAsmInfo > X ( * T ) ; TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( * T , create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCInstrAnalysis ( * T , create<ISA_LIT>MCInstrAnalysis ) ; TargetRegistry :: RegisterMCAsmBackend ( * T , create<ISA_LIT>AsmBackend ) ; TargetRegistry :: RegisterELFStreamer ( * T , createMCStreamer ) ; }


<s> MCCodeGenInfo * X = new MCCodeGenInfo ( ) ; X -> initMCCodeGenInfo ( RM , CM , OL ) ; return X ;


<s> for ( Target * T : { & The<ISA_LIT>Target , & TheGCNTarget } ) { RegisterMCAsmInfo < <ISA_LIT>MCAsmInfo > X ( * T ) ; TargetRegistry :: RegisterMCCodeGenInfo ( * T , create<ISA_LIT>MCCodeGenInfo ) ; TargetRegistry :: RegisterMCInstrInfo ( * T , create<ISA_LIT>MCInstrInfo ) ;

TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ;


<s> TargetRegistry :: RegisterMCInstrInfo ( * T , create<ISA_LIT>MCInstrInfo ) ; TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( * T , create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ;

TargetRegistry :: RegisterMCAsmBackend ( * T , create<ISA_LIT>AsmBackend ) ;


<s> TargetRegistry :: RegisterMCSubtargetInfo ( * T , create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCAsmBackend ( * T , create<ISA_LIT>AsmBackend ) ; TargetRegistry :: RegisterELFStreamer ( * T , createMCStreamer ) ; } TargetRegistry :: RegisterMCCodeEmitter ( The<ISA_LIT>Target , create<ISA_LIT>MCCodeEmitter ) ; TargetRegistry :: RegisterMCCodeEmitter ( TheGCNTarget , create<ISA_LIT>MCCodeEmitter ) ; TargetRegistry :: RegisterAsmTargetStreamer ( TheGCNTarget , create<ISA_LIT>AsmTargetStreamer ) ; TargetRegistry :: RegisterObjectTargetStreamer ( TheGCNTarget , create<ISA_LIT>ObjectTargetStreamer ) ;


<s> for ( Target * T : { & The<ISA_LIT>Target , & TheGCNTarget } ) { RegisterMCAsmInfo < <ISA_LIT>MCAsmInfo > X ( * T ) ; TargetRegistry :: RegisterMCInstrInfo ( * T , create<ISA_LIT>MCInstrInfo ) ; TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( * T , create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCAsmBackend ( * T , create<ISA_LIT>AsmBackend ) ; TargetRegistry :: RegisterELFStreamer ( * T , createMCStreamer ) ;


<s> static MCInstPrinter * create<ISA_LIT>MCInstPrinter ( const Triple & T , unsigned SyntaxVariant , const MCAsmInfo & MAI , const MCInstrInfo & MII , const MCRegisterInfo & MRI ) { return T . getArch ( ) == Triple :: r600 ? new <ISA_LIT>InstPrinter ( MAI , MII , MRI ) :

new <ISA_LIT>InstPrinter ( MAI , MII , MRI ) ;


<s> return T . getArch ( ) == Triple :: r600 ? new <ISA_LIT>InstPrinter ( MAI , MII , MRI ) : new <ISA_LIT>InstPrinter ( MAI , MII , MRI ) ;


<s> return create<ISA_LIT>ELFStreamer ( T , Context , std :: move ( MAB ) , OS , std :: move ( Emitter ) , RelaxAll ) ;


<s> TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( * T , create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCAsmBackend ( * T , create<ISA_LIT>AsmBackend ) ; TargetRegistry :: RegisterELFStreamer ( * T , createMCStreamer ) ; }


<s> TargetRegistry :: RegisterMCCodeGenInfo ( * T , create<ISA_LIT>MCCodeGenInfo ) ; TargetRegistry :: RegisterMCInstrInfo ( * T , create<ISA_LIT>MCInstrInfo ) ; TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( * T , create<ISA_LIT>MCSubtargetInfo ) ;

TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ;


<s> return false ; default : break ; } } const auto checkNoAlias = [ AA , Ptr ] ( auto I ) -> bool { return I && AA -> isNoAlias ( I -> getPointerOperand ( ) , Ptr ) ; } ;

if ( checkNoAlias ( dyn_cast < AtomicCmpXchgInst > ( DefInst ) ) || checkNoAlias ( dyn_cast < AtomicRMWInst > ( DefInst ) ) ) return false ;


<s> case <STR_LIT> : case <STR_LIT> : return false ; default : break ; } } const auto checkNoAlias = [ AA , Ptr ] ( auto I ) -> bool { return I && AA -> isNoAlias ( I -> getPointerOperand ( ) , Ptr ) ;


<s> MemoryAccess * MA = WorkList . pop_back_val ( ) ; if ( ! Visited . insert ( MA ) . second ) continue ; if ( MSSA -> isLiveOnEntryDef ( MA ) ) continue ; if ( MemoryDef * Def = dyn_cast < MemoryDef > ( MA ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Def -> getMemoryInst ( ) << '\n' ) ;

if ( isReallyAClobber ( Load -> getPointerOperand ( ) , Def , AA ) ) {


<s> default : break ; } } const auto checkNoAlias = [ AA , Ptr ] ( auto I ) -> bool { return I && AA -> isNoAlias ( I -> getPointerOperand ( ) , Ptr ) ; } ; if ( checkNoAlias ( dyn_cast < AtomicCmpXchgInst > ( DefInst ) ) || checkNoAlias ( dyn_cast < AtomicRMWInst > ( DefInst ) ) ) return false ;


<s> auto StageB = PipelineOrder [ j ] ; for ( auto SUnitA : StageA -> Collection ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SUnitA ) ; ) ; for ( auto SUnitB : StageB -> Collection ) { if ( DAG -> canAddEdge ( SUnitB , SUnitA ) ) { DAG -> addEdge ( SUnitB , SDep ( SUnitA , SDep :: Artificial ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SUnitB ) ; ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SUnitB ) ; ) ; } } }


<s> static void addPipelineEdges ( const llvm :: ArrayRef < InstructionClass * > PipelineOrder , ScheduleDAGInstrs * DAG ) { for ( int i = <NUM_LIT> ; i < ( int ) PipelineOrder . size ( ) - <NUM_LIT> ; i ++ ) { auto StageA = PipelineOrder [ i ] ; for ( int j = i + <NUM_LIT> ; j < ( int ) PipelineOrder . size ( ) ; j ++ ) { auto StageB = PipelineOrder [ j ] ; for ( auto SUnitA : StageA -> Collection ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SUnitA ) ; ) ; for ( auto SUnitB : StageB -> Collection ) { if ( DAG -> canAddEdge ( SUnitB , SUnitA ) ) { DAG -> addEdge ( SUnitB , SDep ( SUnitA , SDep :: Artificial ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SUnitB ) ; ) ; } else {


<s> const GCNSubtarget & ST = DAGInstrs -> MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; if ( ! ST . hasMAIInsts ( ) ) return ; DAG = static_cast < ScheduleDAGMI * > ( DAGInstrs ) ; const TargetSchedModel * TSchedModel = DAGInstrs -> getSchedModel ( ) ; if ( ! TSchedModel || DAG -> SUnits . empty ( ) ) return ; const IsInstructionType isMFMAFn = [ this ] ( const MachineInstr & MI ) { if ( TII -> isMAI ( MI ) && MI . getOpcode ( ) != <STR_LIT> && MI . getOpcode ( ) != <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; return true ; } return false ; } ; InstructionClass MFMASUnits ( isMFMAFn , MFMAGroupMaxSize ) ; const IsInstructionType isVMEMReadFn = [ this ] ( const MachineInstr & MI ) { if ( ( ( TII -> isFLAT ( MI ) && ! TII -> isDS ( MI ) ) || TII -> isVMEM ( MI ) ) && MI . mayLoad ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; return true ; } return false ; } ; InstructionClass VMEMReadSUnits ( isVMEMReadFn , VMEMGroupMaxSize ) ; const IsInstructionType isDSWriteFn = [ this ] ( const MachineInstr & MI ) { if ( TII -> isDS ( MI ) && MI . mayStore ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; return true ; } return false ; } ; InstructionClass DSWriteSUnits ( isDSWriteFn , LDWGroupMaxSize ) ;

const IsInstructionType isDSReadFn = [ this ] ( const MachineInstr & MI ) {


<s> InstructionClass MFMASUnits ( isMFMAFn , MFMAGroupMaxSize ) ; const IsInstructionType isVMEMReadFn = [ this ] ( const MachineInstr & MI ) { if ( ( ( TII -> isFLAT ( MI ) && ! TII -> isDS ( MI ) ) || TII -> isVMEM ( MI ) ) && MI . mayLoad ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; return true ; } return false ; } ; InstructionClass VMEMReadSUnits ( isVMEMReadFn , VMEMGroupMaxSize ) ; const IsInstructionType isDSWriteFn = [ this ] ( const MachineInstr & MI ) { if ( TII -> isDS ( MI ) && MI . mayStore ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; return true ; } return false ; } ; InstructionClass DSWriteSUnits ( isDSWriteFn , LDWGroupMaxSize ) ; const IsInstructionType isDSReadFn = [ this ] ( const MachineInstr & MI ) { if ( TII -> isDS ( MI ) && MI . mayLoad ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; return true ; } return false ; } ; InstructionClass DSReadSUnits ( isDSReadFn , LDRGroupMaxSize ) ; SmallVector < InstructionClass * , <NUM_LIT> > PipelineOrder = { & VMEMReadSUnits , & DSReadSUnits , & MFMASUnits , & DSWriteSUnits } ;

collectSUnits ( PipelineOrder , TII , DAG ) ;


return ! ( MaxSize <= <NUM_LIT> ) && ( int ) Collection . size ( ) >= MaxSize ;


<s> if ( Src == <STR_LIT> ) { PSV = MFI -> getImagePSV ( TM ) ; return false ; } if ( Src == <STR_LIT> ) { PSV = MFI -> getGWSPSV ( TM ) ; return false ;


<s> } if ( Src == <STR_LIT> ) { PSV = MFI -> getImagePSV ( TII ) ; return false ; } if ( Src == <STR_LIT> ) { PSV = MFI -> getGWSPSV ( TII ) ; return false ; } llvm_unreachable ( <STR_LIT> ) ;


<s> explicit <ISA_LIT>OpenCLEnqueuedBlockLowering ( ) : ModulePass ( ID ) {


<s> if ( auto * CI = dyn_cast < CallInst > ( & * U ) ) { auto * Caller = CI -> getParent ( ) -> getParent ( ) ; if ( Callers . insert ( Caller ) . second ) collectCallers ( Caller , Callers ) ;


<s> for ( auto U : F -> users ( ) ) { if ( auto * CI = dyn_cast < CallInst > ( & * U ) ) { auto * Caller = CI -> getParent ( ) -> getParent ( ) ; if ( Callers . insert ( Caller ) . second ) collectCallers ( Caller , Callers ) ;


<s> static void collectFunctionUsers ( User * U , DenseSet < Function * > & Funcs ) { if ( auto * I = dyn_cast < Instruction > ( U ) ) { auto * F = I -> getParent ( ) -> getParent ( ) ; if ( Funcs . insert ( F ) . second ) collectCallers ( F , Funcs ) ; return ; }

if ( ! isa < Constant > ( U ) ) return ;


<s> static void collectFunctionUsers ( User * U , DenseSet < Function * > & Funcs ) { if ( auto * I = dyn_cast < Instruction > ( U ) ) { auto * F = I -> getParent ( ) -> getParent ( ) ; if ( Funcs . insert ( F ) . second ) collectCallers ( F , Funcs ) ; return ;


return new <ISA_LIT>OpenCLEnqueuedBlockLowering ( ) ;


<s> return new <ISA_LIT>OpenCLEnqueuedBlockLowering ( ) ;


<s> if ( auto * CI = dyn_cast < CallInst > ( & * U ) ) {

auto * Caller = CI -> getParent ( ) -> getParent ( ) ;


<s> auto * Caller = CI -> getParent ( ) -> getParent ( ) ; if ( Callers . count ( Caller ) ) continue ; Callers . insert ( Caller ) ; collectCallers ( Caller , Callers ) ; }


<s> } auto RuntimeHandle = ( F . getName ( ) + <STR_LIT> ) . str ( ) ; auto * GV = new GlobalVariable ( M , Type :: getInt8Ty ( C ) -> getPointerTo ( <ISA_LIT>AS :: GLOBAL_ADDRESS ) , true , GlobalValue :: ExternalLinkage , nullptr , RuntimeHandle , nullptr , GlobalValue :: NotThreadLocal , <ISA_LIT>AS :: GLOBAL_ADDRESS , true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * GV << '\n' ) ; for ( auto U : F . users ( ) ) { if ( ! isa < ConstantExpr > ( & * U ) ) continue ; auto * BitCast = cast < ConstantExpr > ( & * U ) ; auto * NewPtr = ConstantExpr :: getPointerCast ( GV , BitCast -> getType ( ) ) ; BitCast -> replaceAllUsesWith ( NewPtr ) ; F . addFnAttr ( <STR_LIT> , RuntimeHandle ) ; F . setLinkage ( GlobalValue :: ExternalLinkage ) ; for ( auto U : NewPtr -> users ( ) ) { if ( auto * I = dyn_cast < Instruction > ( & * U ) ) { auto * F = I -> getParent ( ) -> getParent ( ) ; Callers . insert ( F ) ; collectCallers ( F , Callers ) ; } }


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) { if ( ! F . hasOneUse ( ) || ! F . user_begin ( ) -> hasOneUse ( ) || ! isa < ConstantExpr > ( * F . user_begin ( ) ) || ! isa < ConstantExpr > ( * F . user_begin ( ) -> user_begin ( ) ) ) { continue ; } auto * BitCast = cast < ConstantExpr > ( * F . user_begin ( ) ) ; auto * AddrCast = cast < ConstantExpr > ( * BitCast -> user_begin ( ) ) ; auto RuntimeHandle = ( F . getName ( ) + <STR_LIT> ) . str ( ) ; auto * GV = new GlobalVariable ( M , Type :: getInt8Ty ( C ) -> getPointerTo ( <ISA_LIT>AS :: GLOBAL_ADDRESS ) , true , GlobalValue :: ExternalLinkage , nullptr , RuntimeHandle , nullptr , GlobalValue :: NotThreadLocal , <ISA_LIT>AS :: GLOBAL_ADDRESS , true ) ;

DEBUG ( dbgs ( ) << <STR_LIT> << * GV << '\n' ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << * GV << '\n' ) ; auto * NewPtr = ConstantExpr :: getPointerCast ( GV , AddrCast -> getType ( ) ) ; AddrCast -> replaceAllUsesWith ( NewPtr ) ; F . addFnAttr ( <STR_LIT> , RuntimeHandle ) ; F . setLinkage ( GlobalValue :: ExternalLinkage ) ; for ( auto U : NewPtr -> users ( ) ) { if ( auto * I = dyn_cast < Instruction > ( & * U ) ) { auto * F = I -> getParent ( ) -> getParent ( ) ; Callers . insert ( F ) ; collectCallers ( F , Callers ) ; } } Changed = true ; } } for ( auto F : Callers ) {

if ( F -> getCallingConv ( ) != CallingConv :: <ISA_LIT>_KERNEL ) continue ;


<s> bool <ISA_LIT>OpenCLEnqueuedBlockLowering :: runOnModule ( Module & M ) { DenseSet < Function * > Callers ; auto & C = M . getContext ( ) ; bool Changed = false ; for ( auto & F : M . functions ( ) ) { if ( F . hasFnAttribute ( <STR_LIT> ) ) { if ( ! F . hasName ( ) ) { SmallString < <NUM_LIT> > Name ; Mangler :: getNameWithPrefix ( Name , <STR_LIT> , M . getDataLayout ( ) ) ; F . setName ( Name ) ; } DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << '\n' ) ; auto RuntimeHandle = ( F . getName ( ) + <STR_LIT> ) . str ( ) ;

auto T = Type :: getInt8Ty ( C ) -> getPointerTo ( <ISA_LIT>AS :: GLOBAL_ADDRESS ) ;


<s> FunctionType * FT = F -> getFunctionType ( ) ; SmallVector < Type * , <NUM_LIT> > ArgTypes ; KernelArgMD NewArgMDs ; PushArgMD ( NewArgMDs , GetArgMD ( KernelMDNode , <NUM_LIT> ) ) ; for ( unsigned i = <NUM_LIT> ; i < FT -> getNumParams ( ) ; ++ i ) { ArgTypes . push_back ( FT -> getParamType ( i ) ) ; MDVector ArgMD = GetArgMD ( KernelMDNode , i + <NUM_LIT> ) ; PushArgMD ( NewArgMDs , ArgMD ) ; if ( ! IsImageType ( ArgTypeFromMD ( KernelMDNode , i ) ) ) continue ; ArgTypes . push_back ( ImageSizeType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageSizeArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; ArgTypes . push_back ( ImageFormatType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageFormatArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; Modified = true ; } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ; auto NewFArgIt = NewF -> arg_begin ( ) ; for ( auto & Arg : F -> args ( ) ) { auto ArgName = Arg . getName ( ) ; NewFArgIt -> setName ( ArgName ) ; VMap [ & Arg ] = & ( * NewFArgIt ++ ) ; if ( IsImageType ( ArgTypeFromMD ( KernelMDNode , Arg . getArgNo ( ) ) ) ) { ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; } }

SmallVector < ReturnInst * , <NUM_LIT> > Returns ;


<s> auto F = mdconst :: dyn_extract < Function > ( Node -> getOperand ( <NUM_LIT> ) ) ; if ( ! F ) return nullptr ; size_t ExpectNumArgNodeOps = F -> arg_size ( ) + <NUM_LIT> ; for ( size_t i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) { MDNode * ArgNode = dyn_cast_or_null < MDNode > ( Node -> getOperand ( i + <NUM_LIT> ) ) ; if ( ArgNode -> getNumOperands ( ) != ExpectNumArgNodeOps ) return nullptr ; if ( ! ArgNode -> getOperand ( <NUM_LIT> ) ) return nullptr ;

assert ( cast < MDString > ( ArgNode -> getOperand ( <NUM_LIT> ) ) -> getString ( ) == KernelArgMDNodeNames [ i ] && <STR_LIT> ) ;


<s> return cast < MDString > ( ArgAQNode -> getOperand ( ArgIdx + <NUM_LIT> ) ) -> getString ( ) ;


<s> MDNode * ArgAQNode = cast < MDNode > ( KernelMDNode -> getOperand ( <NUM_LIT> ) ) ;

return cast < MDString > ( ArgAQNode -> getOperand ( ArgIdx + <NUM_LIT> ) ) -> getString ( ) ;


<s> FunctionType * FT = F -> getFunctionType ( ) ; SmallVector < Type * , <NUM_LIT> > ArgTypes ; KernelArgMD NewArgMDs ; PushArgMD ( NewArgMDs , GetArgMD ( KernelMDNode , <NUM_LIT> ) ) ; for ( unsigned i = <NUM_LIT> ; i < FT -> getNumParams ( ) ; ++ i ) { ArgTypes . push_back ( FT -> getParamType ( i ) ) ; MDVector ArgMD = GetArgMD ( KernelMDNode , i + <NUM_LIT> ) ; PushArgMD ( NewArgMDs , ArgMD ) ; if ( ! IsImageType ( ArgTypeFromMD ( KernelMDNode , i ) ) ) continue ; ArgTypes . push_back ( ImageSizeType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageSizeArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; ArgTypes . push_back ( ImageFormatType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageFormatArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; Modified = true ; } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ; auto NewFArgIt = NewF -> arg_begin ( ) ; for ( auto & Arg : F -> args ( ) ) { auto ArgName = Arg . getName ( ) ; NewFArgIt -> setName ( ArgName ) ; VMap [ & Arg ] = & ( * NewFArgIt ++ ) ; if ( IsImageType ( ArgTypeFromMD ( KernelMDNode , Arg . getArgNo ( ) ) ) ) { ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; } } SmallVector < ReturnInst * , <NUM_LIT> > Returns ;

CloneFunctionInto ( NewF , F , VMap , false , Returns ) ;


<s> <ISA_LIT>OpenCLImageTypeLoweringPass ( ) : ModulePass ( ID ) {


<s> MDNode * ArgTypeNode = cast < MDNode > ( KernelMDNode -> getOperand ( <NUM_LIT> ) ) ;

return cast < MDString > ( ArgTypeNode -> getOperand ( ArgIdx + <NUM_LIT> ) ) -> getString ( ) ;


<s> static StringRef ArgTypeFromMD ( MDNode * KernelMDNode , unsigned ArgIdx ) { MDNode * ArgTypeNode = cast < MDNode > ( KernelMDNode -> getOperand ( <NUM_LIT> ) ) ;


<s> return new <ISA_LIT>OpenCLImageTypeLoweringPass ( ) ;


<s> static MDVector GetArgMD ( MDNode * KernelMDNode , unsigned OpIdx ) { MDVector Res ; for ( unsigned i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) {

MDNode * Node = cast < MDNode > ( KernelMDNode -> getOperand ( i + <NUM_LIT> ) ) ;


<s> MDNode * Node = cast < MDNode > ( KernelMDNode -> getOperand ( i + <NUM_LIT> ) ) ; Res . push_back ( Node -> getOperand ( OpIdx ) ) ;


<s> for ( size_t i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) { MDNode * ArgNode = dyn_cast_or_null < MDNode > ( Node -> getOperand ( i + <NUM_LIT> ) ) ; if ( ArgNode -> getNumOperands ( ) != ExpectNumArgNodeOps ) return nullptr ; if ( ! ArgNode -> getOperand ( <NUM_LIT> ) ) return nullptr ; MDString * StringNode = dyn_cast < MDString > ( ArgNode -> getOperand ( <NUM_LIT> ) ) ;

if ( ! StringNode || StringNode -> getString ( ) != KernelArgMDNodeNames [ i ] ) return nullptr ;


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> return TypeString == <STR_LIT> || TypeString == <STR_LIT> ;


<s> static inline bool IsImageType ( StringRef TypeString ) { return TypeString == <STR_LIT> || TypeString == <STR_LIT> ;


<s> for ( unsigned i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) { MD . ArgVector [ i ] . push_back ( V [ i ] ) ; }


<s> uint32_t ResourceID ; if ( AccessQual == <STR_LIT> ) { ResourceID = NumReadOnlyImageArgs ++ ; } else if ( AccessQual == <STR_LIT> ) { ResourceID = NumWriteOnlyImageArgs ++ ; } else { llvm_unreachable ( <STR_LIT> ) ; } Argument & SizeArg = * ( ++ ArgI ) ; Argument & FormatArg = * ( ++ ArgI ) ; Modified |= replaceImageUses ( Arg , ResourceID , SizeArg , FormatArg ) ; } else if ( IsSamplerType ( Type ) ) { uint32_t ResourceID = NumSamplerArgs ++ ; Modified |= replaceSamplerUses ( Arg , ResourceID ) ; } } for ( unsigned i = <NUM_LIT> ; i < InstsToErase . size ( ) ; ++ i ) {

InstsToErase [ i ] -> eraseFromParent ( ) ;


<s> auto Inst = dyn_cast < CallInst > ( Use . getUser ( ) ) ; if ( ! Inst ) { continue ; } Function * F = Inst -> getCalledFunction ( ) ; if ( ! F ) continue ; Value * Replacement = nullptr ; StringRef Name = F -> getName ( ) ; if ( Name . startswith ( GetImageResourceIDFunc ) ) { Replacement = ConstantInt :: get ( Int32Type , ResourceID ) ; } else if ( Name . startswith ( GetImageSizeFunc ) ) { Replacement = & ImageSizeArg ; } else if ( Name . startswith ( GetImageFormatFunc ) ) { Replacement = & ImageFormatArg ; } else { continue ; }

Inst -> replaceAllUsesWith ( Replacement ) ;


<s> bool Modified = false ; for ( const auto & Use : SamplerArg . uses ( ) ) { auto Inst = dyn_cast < CallInst > ( Use . getUser ( ) ) ; if ( ! Inst ) { continue ; } Function * F = Inst -> getCalledFunction ( ) ; if ( ! F ) continue ; Value * Replacement = nullptr ; StringRef Name = F -> getName ( ) ; if ( Name == GetSamplerResourceIDFunc ) { Replacement = ConstantInt :: get ( Int32Type , ResourceID ) ; } else { continue ; } Inst -> replaceAllUsesWith ( Replacement ) ; InstsToErase . push_back ( Inst ) ; Modified = true ;


<s> if ( Name == GetSamplerResourceIDFunc ) { Replacement = ConstantInt :: get ( Int32Type , ResourceID ) ; } else { continue ; } Inst -> replaceAllUsesWith ( Replacement ) ; InstsToErase . push_back ( Inst ) ; Modified = true ; } return Modified ;


<s> Context = & M . getContext ( ) ; Int32Type = Type :: getInt32Ty ( M . getContext ( ) ) ; ImageSizeType = ArrayType :: get ( Int32Type , <NUM_LIT> ) ; ImageFormatType = ArrayType :: get ( Int32Type , <NUM_LIT> ) ;


<s> Function * NewF ; MDNode * NewMDNode ; std :: tie ( NewF , NewMDNode ) = addImplicitArgs ( F , KernelMDNode ) ; if ( NewF ) { F -> eraseFromParent ( ) ; M . getFunctionList ( ) . push_back ( NewF ) ; M . getOrInsertFunction ( NewF -> getName ( ) , NewF -> getFunctionType ( ) , NewF -> getAttributes ( ) ) ; KernelsMDNode -> setOperand ( i , NewMDNode ) ; F = NewF ; KernelMDNode = NewMDNode ; Modified = true ; } Modified |= replaceImageAndSamplerUses ( F , KernelMDNode ) ;


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getShaderFunction ( StringRef Name ) { auto Functions = getShaderFunctions ( ) ; return Functions [ Name ] . getMap ( true ) ;


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getShaderFunction ( StringRef Name ) {

auto Functions = getShaderFunctions ( ) ;


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getShaderFunctions ( ) { if ( ShaderFunctions . isEmpty ( ) ) ShaderFunctions = refShaderFunctions ( ) ;


<s> case CallingConv :: <ISA_LIT>_PS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_VS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_GS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_ES : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_HS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_LS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_Gfx : llvm_unreachable ( <STR_LIT> ) ; default : return <STR_LIT> ; }


<s> case CallingConv :: <ISA_LIT>_PS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_VS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_GS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_ES : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_HS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_LS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_Gfx : llvm_unreachable ( <STR_LIT> ) ; default :


<s> Node [ <STR_LIT> ] = MsgPackDoc . getNode ( Val ) ;


<s> auto Node = getShaderFunction ( MF . getFunction ( ) . getName ( ) ) ; Node [ <STR_LIT> ] = MsgPackDoc . getNode ( Val ) ;


<s> void <ISA_LIT>PALMetadata :: readFromIR ( Module & M ) { auto NamedMD = M . getNamedMetadata ( <STR_LIT> ) ; if ( NamedMD && NamedMD -> getNumOperands ( ) ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; auto MDN = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( MDN && MDN -> getNumOperands ( ) ) { if ( auto MDS = dyn_cast < MDString > ( MDN -> getOperand ( <NUM_LIT> ) ) ) setFromMsgPackBlob ( MDS -> getString ( ) ) ; } return ; } BlobType = ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ; NamedMD = M . getNamedMetadata ( <STR_LIT> ) ; if ( ! NamedMD || ! NamedMD -> getNumOperands ( ) ) return ;

auto Tuple = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ;


<s> if ( NamedMD && NamedMD -> getNumOperands ( ) ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; auto MDN = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( MDN && MDN -> getNumOperands ( ) ) { if ( auto MDS = dyn_cast < MDString > ( MDN -> getOperand ( <NUM_LIT> ) ) ) setFromMsgPackBlob ( MDS -> getString ( ) ) ; } return ; } BlobType = ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ; NamedMD = M . getNamedMetadata ( <STR_LIT> ) ; if ( ! NamedMD || ! NamedMD -> getNumOperands ( ) ) return ; auto Tuple = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( ! Tuple ) return ; for ( unsigned I = <NUM_LIT> , E = Tuple -> getNumOperands ( ) & - <NUM_LIT> ; I != E ; I += <NUM_LIT> ) {

auto Key = mdconst :: dyn_extract < ConstantInt > ( Tuple -> getOperand ( I ) ) ;


<s> bool <ISA_LIT>PALMetadata :: isLegacy ( ) const { return BlobType == ELF :: NT_AMD_PAL_METADATA ;


<s> if ( ! NamedMD || ! NamedMD -> getNumOperands ( ) ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; return ; } auto Tuple = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( ! Tuple ) return ; for ( unsigned I = <NUM_LIT> , E = Tuple -> getNumOperands ( ) & - <NUM_LIT> ; I != E ; I += <NUM_LIT> ) { auto Key = mdconst :: dyn_extract < ConstantInt > ( Tuple -> getOperand ( I ) ) ; auto Val = mdconst :: dyn_extract < ConstantInt > ( Tuple -> getOperand ( I + <NUM_LIT> ) ) ; if ( ! Key || ! Val ) continue ; setRegister ( Key -> getZExtValue ( ) , Val -> getZExtValue ( ) ) ; }


<s> auto Node = getShaderFunction ( MF . getFunction ( ) . getName ( ) ) ;

Node [ <STR_LIT> ] = MsgPackDoc . getNode ( Val ) ;


<s> void <ISA_LIT>PALMetadata :: setLegacy ( ) { BlobType = ELF :: NT_AMD_PAL_METADATA ;


<s> BlobType = ELF :: NT_AMD_PAL_METADATA ;


<s> void <ISA_LIT>PALMetadata :: setNumUsedAgprs ( CallingConv :: ID CC , unsigned Val ) { getHwStage ( CC ) [ <STR_LIT> ] = Val ;


<s> getHwStage ( CC ) [ <STR_LIT> ] = Val ;


<s> if ( Type == ELF :: NT_AMD_PAL_METADATA ) toLegacyBlob ( Blob ) ;

else if ( Type ) toMsgPackBlob ( Blob ) ;


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getHwStage ( unsigned CC ) {

if ( HwStages . isEmpty ( ) ) HwStages = MsgPackDoc . getRoot ( ) . getMap ( true ) [ <STR_LIT> ] . getArray ( true ) [ <NUM_LIT> ] . getMap ( true ) [ <STR_LIT> ] . getMap ( true ) ;


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getHwStage ( unsigned CC ) { if ( HwStages . isEmpty ( ) ) HwStages = MsgPackDoc . getRoot ( ) . getMap ( true ) [ <STR_LIT> ] . getArray ( true ) [ <NUM_LIT> ] . getMap ( true ) [ <STR_LIT> ] . getMap ( true ) ;

return HwStages . getMap ( ) [ getStageName ( CC ) ] . getMap ( true ) ;


<s> msgpack :: Document * getMsgPackDoc ( ) { return & MsgPackDoc ;


if ( N . getKind ( ) != msgpack :: Type :: UInt ) return <NUM_LIT> ;


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getRegisters ( ) { if ( Registers . isEmpty ( ) ) Registers = refRegisters ( ) ;


<s> case CallingConv :: <ISA_LIT>_ES : return PALMD :: R_2CCA_SPI_SHADER_PGM_RSRC1_ES ; case CallingConv :: <ISA_LIT>_GS : return PALMD :: R_2C8A_SPI_SHADER_PGM_RSRC1_GS ; case CallingConv :: <ISA_LIT>_VS :

return PALMD :: R_2C4A_SPI_SHADER_PGM_RSRC1_VS ;


<s> return PALMD :: Key :: VS_SCRATCH_<ISA_LIT>ZE ; case CallingConv :: <ISA_LIT>_GS : return PALMD :: Key :: GS_SCRATCH_<ISA_LIT>ZE ; case CallingConv :: <ISA_LIT>_ES : return PALMD :: Key :: ES_SCRATCH_<ISA_LIT>ZE ; case CallingConv :: <ISA_LIT>_HS : return PALMD :: Key :: HS_SCRATCH_<ISA_LIT>ZE ; case CallingConv :: <ISA_LIT>_LS : return PALMD :: Key :: LS_SCRATCH_<ISA_LIT>ZE ; default : return PALMD :: Key :: CS_SCRATCH_<ISA_LIT>ZE ;


<s> switch ( CC ) { case CallingConv :: <ISA_LIT>_PS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_VS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_GS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_ES : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_HS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_LS : return <STR_LIT> ; default :


<s> unsigned <ISA_LIT>PALMetadata :: getType ( ) const {


<s> const char * <ISA_LIT>PALMetadata :: getVendor ( ) const { return isLegacy ( ) ? ElfNote :: NoteNameV2 : ElfNote :: NoteNameV3 ;


<s> return isLegacy ( ) ? ElfNote :: NoteNameV2 : ElfNote :: NoteNameV3 ;


<s> return BlobType == ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ;


<s> void <ISA_LIT>PALMetadata :: readFromIR ( Module & M ) { auto NamedMD = M . getNamedMetadata ( <STR_LIT> ) ; if ( NamedMD && NamedMD -> getNumOperands ( ) ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; auto MDN = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( MDN && MDN -> getNumOperands ( ) ) { if ( auto MDS = dyn_cast < MDString > ( MDN -> getOperand ( <NUM_LIT> ) ) ) setFromMsgPackBlob ( MDS -> getString ( ) ) ;


<s> if ( NamedMD && NamedMD -> getNumOperands ( ) ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; auto MDN = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( MDN && MDN -> getNumOperands ( ) ) { if ( auto MDS = dyn_cast < MDString > ( MDN -> getOperand ( <NUM_LIT> ) ) ) setFromMsgPackBlob ( MDS -> getString ( ) ) ; } return ; } BlobType = ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ; NamedMD = M . getNamedMetadata ( <STR_LIT> ) ; if ( ! NamedMD || ! NamedMD -> getNumOperands ( ) ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; return ;


<s> void <ISA_LIT>PALMetadata :: reset ( ) { MsgPackDoc . clear ( ) ; Registers = MsgPackDoc . getEmptyNode ( ) ; HwStages = MsgPackDoc . getEmptyNode ( ) ;


<s> MsgPackDoc . clear ( ) ; Registers = MsgPackDoc . getEmptyNode ( ) ; HwStages = MsgPackDoc . getEmptyNode ( ) ;


<s> if ( isLegacy ( ) ) return ; getHwStage ( CC ) [ <STR_LIT> ] = MsgPackDoc . getNode ( Name , true ) ;


<s> if ( Type == ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ) return setFromLegacyBlob ( Blob ) ; return setFromMsgPackBlob ( Blob ) ;


<s> msgpack :: Reader Reader ( Blob ) ; return MsgPackDoc . readFromBlob ( Blob , false ) ;


<s> bool <ISA_LIT>PALMetadata :: setFromString ( StringRef S ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; if ( ! MsgPackDoc . fromYAML ( S ) ) return false ; auto & RegsObj = refRegisters ( ) ; auto OrigRegs = RegsObj ; RegsObj = MsgPackDoc . getMapNode ( ) ; Registers = RegsObj . getMap ( ) ; bool Ok = true ; for ( auto I : OrigRegs . getMap ( ) ) { auto Key = I . first ; if ( Key . getKind ( ) == msgpack :: Type :: String ) { StringRef S = Key . getString ( ) ; uint64_t Val ;

if ( S . consumeInteger ( <NUM_LIT> , Val ) ) {


<s> void <ISA_LIT>PALMetadata :: setLegacy ( ) {

BlobType = ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ;


<s> unsigned NumUsedSgprsKey = getScratchSizeKey ( CC ) + PALMD :: Key :: VS_NUM_USED_SGPRS - PALMD :: Key :: VS_SCRATCH_<ISA_LIT>ZE ; setRegister ( NumUsedSgprsKey , Val ) ; return ; }

getHwStage ( CC ) [ <STR_LIT> ] = MsgPackDoc . getNode ( Val ) ;


<s> unsigned NumUsedVgprsKey = getScratchSizeKey ( CC ) + PALMD :: Key :: VS_NUM_USED_VGPRS - PALMD :: Key :: VS_SCRATCH_<ISA_LIT>ZE ; setRegister ( NumUsedVgprsKey , Val ) ; return ; }


<s> unsigned NumUsedVgprsKey = getScratchSizeKey ( CC ) + PALMD :: Key :: VS_NUM_USED_VGPRS - PALMD :: Key :: VS_SCRATCH_<ISA_LIT>ZE ; setRegister ( NumUsedVgprsKey , Val ) ; return ;


<s> auto & N = getRegisters ( ) [ MsgPackDoc . getNode ( Reg ) ] ; if ( N . getKind ( ) == msgpack :: Type :: UInt ) Val |= N . getUInt ( ) ;

N = N . getDocument ( ) -> getNode ( Val ) ;


<s> void <ISA_LIT>PALMetadata :: setRsrc1 ( CallingConv :: ID CC , unsigned Val ) { setRegister ( getRsrc1Reg ( CC ) , Val ) ;


<s> void <ISA_LIT>PALMetadata :: setRsrc2 ( CallingConv :: ID CC , unsigned Val ) { setRegister ( getRsrc1Reg ( CC ) + <NUM_LIT> , Val ) ;


<s> setRegister ( PALMD :: R_A1B4_SPI_PS_INPUT_ADDR , Val ) ;


<s> void <ISA_LIT>PALMetadata :: setSpiPsInputEna ( unsigned Val ) {

setRegister ( PALMD :: R_A1B3_SPI_PS_INPUT_ENA , Val ) ;


<s> setRegister ( PALMD :: R_A1B3_SPI_PS_INPUT_ENA , Val ) ;


<s> setRegister ( PALMD :: R_A2D5_VGT_SHADER_STAGES_EN , S_028B54_HS_W32_EN ( <NUM_LIT> ) ) ; break ; case CallingConv :: <ISA_LIT>_GS : setRegister ( PALMD :: R_A2D5_VGT_SHADER_STAGES_EN , S_028B54_GS_W32_EN ( <NUM_LIT> ) ) ; break ; case CallingConv :: <ISA_LIT>_VS : setRegister ( PALMD :: R_A2D5_VGT_SHADER_STAGES_EN , S_028B54_VS_W32_EN ( <NUM_LIT> ) ) ; break ; case CallingConv :: <ISA_LIT>_PS : setRegister ( PALMD :: R_A1B6_SPI_PS_IN_CONTROL , S_0286D8_PS_W32_EN ( <NUM_LIT> ) ) ;


<s> if ( Type == ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ) toLegacyBlob ( Blob ) ;


<s> auto Registers = getRegisters ( ) ; if ( Registers . getMap ( ) . empty ( ) ) return ; raw_string_ostream OS ( Blob ) ; <STR_LIT> :: Writer EW ( OS , <STR_LIT> :: little ) ; for ( auto I : Registers . getMap ( ) ) { EW . write ( uint32_t ( I . first . getUInt ( ) ) ) ; EW . write ( uint32_t ( I . second . getUInt ( ) ) ) ;


<s> raw_string_ostream OS ( Blob ) ; <STR_LIT> :: Writer EW ( OS , <STR_LIT> :: little ) ; for ( auto I : Registers . getMap ( ) ) {

EW . write ( uint32_t ( I . first . getUInt ( ) ) ) ;


<s>
raw_string_ostream
Stream
(
String
)
;
if
(
isLegacy
(
)
)
{
if
(
MsgPackDoc
.
getRoot
(
)
.
getKind
(
)
==
msgpack
::
Type
::
)
return
;
Stream
<<
'\t'
<<
<STR_LIT>
::
AssemblerDirective
<<
' '
;
auto
Regs
=
getRegisters
(
)
;
for
(
auto
I
=
Regs
.
begin
(
)
,
E
=
Regs
.
end
(
)
;
I
!=
E
;
++
I
)
{
if
(
I
!=
Regs
.
begin
(
)
)
Stream
<<
','
;
unsigned
Reg
=
I
->
first
.
getUInt
(
)
;
unsigned
Val
=
I
->
second
.
getUInt
(
)
;
Stream
<<
<STR_LIT>
<<
Twine
::
utohexstr
(
Reg
)
<<
<STR_LIT>
<<
Twine
::
utohexstr
(
Val
)
;
}
Stream
<<
'\n'
;
return
;
}
MsgPackDoc
.
setHexMode
(
)
;


<s> for ( auto I = Regs . begin ( ) , E = Regs . end ( ) ; I != E ; ++ I ) { if ( I != Regs . begin ( ) ) Stream << ',' ; unsigned Reg = I -> first . getUInt ( ) ; unsigned Val = I -> second . getUInt ( ) ; Stream << <STR_LIT> << Twine :: utohexstr ( Reg ) << <STR_LIT> << Twine :: utohexstr ( Val ) ; } Stream << '\n' ; return ; } MsgPackDoc . setHexMode ( ) ; auto & RegsObj = refRegisters ( ) ; auto OrigRegs = RegsObj . getMap ( ) ; RegsObj = MsgPackDoc . getMapNode ( ) ; for ( auto I : OrigRegs ) { auto Key = I . first ; if ( const char * RegName = getRegisterName ( Key . getUInt ( ) ) ) { std :: string KeyName = Key . toString ( ) ; KeyName += <STR_LIT> ; KeyName += RegName ; KeyName += ')' ; Key = MsgPackDoc . getNode ( KeyName , true ) ;


<s> <ISA_LIT>PerfHint ( <ISA_LIT>PerfHintAnalysis :: FuncInfoMap & FIM_ , const TargetLowering * TLI_ ) : FIM ( FIM_ ) , DL ( nullptr ) , TLI ( TLI_ ) {


<s> static std :: pair < const Value * , const Type * > getMemoryInstrPtrAndType ( const Instruction * Inst ) { if ( auto LI = dyn_cast < LoadInst > ( Inst ) ) return { LI -> getPointerOperand ( ) , LI -> getType ( ) } ; if ( auto <ISA_LIT> = dyn_cast < StoreInst > ( Inst ) ) return { <ISA_LIT> -> getPointerOperand ( ) , <ISA_LIT> -> getValueOperand ( ) -> getType ( ) } ; if ( auto AI = dyn_cast < AtomicCmpXchgInst > ( Inst ) ) return { AI -> getPointerOperand ( ) , AI -> getCompareOperand ( ) -> getType ( ) } ; if ( auto AI = dyn_cast < AtomicRMWInst > ( Inst ) ) return { AI -> getPointerOperand ( ) , AI -> getValOperand ( ) -> getType ( ) } ; if ( auto MI = dyn_cast < AnyMemIntrinsic > ( Inst ) ) return { MI -> getRawDest ( ) , Type :: getInt8Ty ( MI -> getContext ( ) ) } ;


<s> if ( auto AI = dyn_cast < AtomicCmpXchgInst > ( Inst ) ) return { AI -> getPointerOperand ( ) , AI -> getCompareOperand ( ) -> getType ( ) } ; if ( auto AI = dyn_cast < AtomicRMWInst > ( Inst ) ) return { AI -> getPointerOperand ( ) , AI -> getValOperand ( ) -> getType ( ) } ; if ( auto MI = dyn_cast < AnyMemIntrinsic > ( Inst ) ) return { MI -> getRawDest ( ) , Type :: getInt8Ty ( MI -> getContext ( ) ) } ; return { nullptr , nullptr } ;


<s> if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) { unsigned As = PT -> getAddressSpace ( ) ; return As == <ISA_LIT>AS :: CONSTANT_ADDRESS || As == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ; } return false ;


<s> if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) {

unsigned As = PT -> getAddressSpace ( ) ;


<s> return As == <ISA_LIT>AS :: GLOBAL_ADDRESS || As == <ISA_LIT>AS :: FLAT_ADDRESS ; } return false ;


<s> bool <ISA_LIT>PerfHint :: isIndirectAccess ( const Instruction * Inst ) const { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Inst << '\n' ) ; SmallSet < const Value * , <NUM_LIT> > WorkSet ; SmallSet < const Value * , <NUM_LIT> > Visited ; if ( const Value * MO = getMemoryInstrPtrAndType ( Inst ) . first ) { if ( isGlobalAddr ( MO ) ) WorkSet . insert ( MO ) ; } while ( ! WorkSet . empty ( ) ) { const Value * V = * WorkSet . begin ( ) ; WorkSet . erase ( * WorkSet . begin ( ) ) ; if ( ! Visited . insert ( V ) . second ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * V << '\n' ) ; if ( auto LD = dyn_cast < LoadInst > ( V ) ) { auto M = LD -> getPointerOperand ( ) ; if ( isGlobalAddr ( M ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return true ; } continue ; } if ( auto GEP = dyn_cast < GetElementPtrInst > ( V ) ) { auto P = GEP -> getPointerOperand ( ) ; WorkSet . insert ( P ) ; for ( unsigned I = <NUM_LIT> , E = GEP -> getNumIndices ( ) + <NUM_LIT> ; I != E ; ++ I ) WorkSet . insert ( GEP -> getOperand ( I ) ) ; continue ; } if ( auto U = dyn_cast < UnaryInstruction > ( V ) ) { WorkSet . insert ( U -> getOperand ( <NUM_LIT> ) ) ; continue ; } if ( auto BO = dyn_cast < BinaryOperator > ( V ) ) { WorkSet . insert ( BO -> getOperand ( <NUM_LIT> ) ) ; WorkSet . insert ( BO -> getOperand ( <NUM_LIT> ) ) ; continue ; } if ( auto S = dyn_cast < SelectInst > ( V ) ) { WorkSet . insert ( S -> getFalseValue ( ) ) ; WorkSet . insert ( S -> getTrueValue ( ) ) ; continue ; } if ( auto E = dyn_cast < ExtractElementInst > ( V ) ) { WorkSet . insert ( E -> getVectorOperand ( ) ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; }


<s> } if ( auto GEP = dyn_cast < GetElementPtrInst > ( V ) ) { auto P = GEP -> getPointerOperand ( ) ; WorkSet . insert ( P ) ; for ( unsigned I = <NUM_LIT> , E = GEP -> getNumIndices ( ) + <NUM_LIT> ; I != E ; ++ I ) WorkSet . insert ( GEP -> getOperand ( I ) ) ; continue ; } if ( auto U = dyn_cast < UnaryInstruction > ( V ) ) { WorkSet . insert ( U -> getOperand ( <NUM_LIT> ) ) ; continue ; } if ( auto BO = dyn_cast < BinaryOperator > ( V ) ) { WorkSet . insert ( BO -> getOperand ( <NUM_LIT> ) ) ; WorkSet . insert ( BO -> getOperand ( <NUM_LIT> ) ) ; continue ; } if ( auto S = dyn_cast < SelectInst > ( V ) ) { WorkSet . insert ( S -> getFalseValue ( ) ) ;

WorkSet . insert ( S -> getTrueValue ( ) ) ;


<s> MemAccessInfo MAI = makeMemAccessInfo ( const_cast < Instruction * > ( Inst ) ) ; bool IsLargeStride = MAI . isLargeStride ( LastAccess ) ; if ( MAI . Base ) LastAccess = std :: move ( MAI ) ; return IsLargeStride ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Inst << '\n' ) ; MemAccessInfo MAI = makeMemAccessInfo ( const_cast < Instruction * > ( Inst ) ) ;

bool IsLargeStride = MAI . isLargeStride ( LastAccess ) ;


<s> bool <ISA_LIT>PerfHint :: isLocalAddr ( const Value * V ) const { if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) return PT -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ;


<s> bool <ISA_LIT>PerfHint :: isMemBound ( const <ISA_LIT>PerfHintAnalysis :: FuncInfo & FI ) {

return FI . MemInstCost * <NUM_LIT> / FI . InstCost > MemBoundThresh ;


<s> return FI . MemInstCost * <NUM_LIT> / FI . InstCost > MemBoundThresh ;


<s> auto FI = FIM . find ( F ) ; if ( FI == FIM . end ( ) ) return false ; return <ISA_LIT>PerfHint :: isMemBound ( FI -> second ) ;


<s> bool <ISA_LIT>PerfHintAnalysis :: isMemoryBound ( const Function * F ) const { auto FI = FIM . find ( F ) ; if ( FI == FIM . end ( ) ) return false ; return <ISA_LIT>PerfHint :: isMemBound ( FI -> second ) ;


<s> <ISA_LIT>PerfHint :: MemAccessInfo <ISA_LIT>PerfHint :: makeMemAccessInfo ( Instruction * Inst ) const { MemAccessInfo MAI ;

const Value * MO = getMemoryInstrPtrAndType ( Inst ) . first ;


<s> <ISA_LIT>PerfHint :: MemAccessInfo <ISA_LIT>PerfHint :: makeMemAccessInfo ( Instruction * Inst ) const { MemAccessInfo MAI ; const Value * MO = getMemoryInstrPtrAndType ( Inst ) . first ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MO << '\n' ) ; if ( isLocalAddr ( MO ) ) return MAI ; MAI . V = MO ;

MAI . Base = GetPointerBaseWithConstantOffset ( MO , MAI . Offset , * DL ) ;


<s> MemAccessInfo ( ) : V ( nullptr ) , Base ( nullptr ) , Offset ( <NUM_LIT> ) {


<s> bool <ISA_LIT>PerfHint :: needLimitWave ( const <ISA_LIT>PerfHintAnalysis :: FuncInfo & FI ) {

return ( ( FI . MemInstCost + FI . IAMInstCost * IAWeight + FI . LSMInstCost * LSWeight ) * <NUM_LIT> / FI . InstCost ) > LimitWaveThresh ;


if ( FI == FIM . end ( ) ) return false ;


<s> return Printable ( [ this ] ( raw_ostream & OS ) {


<s> DL = & M . getDataLayout ( ) ; if ( F . hasFnAttribute ( <STR_LIT> ) && F . hasFnAttribute ( <STR_LIT> ) ) return false ; const <ISA_LIT>PerfHintAnalysis :: FuncInfo * Info = visit ( F ) ; LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << Info -> MemInstCost << '\n' << <STR_LIT> << Info -> IAMInstCost << '\n' << <STR_LIT> << Info -> LSMInstCost << '\n' << <STR_LIT> << Info -> InstCost << '\n' ) ; bool Changed = false ; if ( isMemBound ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumMemBound ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; Changed = true ; }

if ( <STR_LIT> ( F . getCallingConv ( ) ) && needLimitWave ( * Info ) ) {


<s> if ( ! F || F -> isDeclaration ( ) ) continue ; const TargetSubtargetInfo * ST = TM . getSubtargetImpl ( * F ) ;

<ISA_LIT>PerfHint Analyzer ( FIM , ST -> getTargetLowering ( ) ) ;


<s> if ( isIndirectAccess ( & I ) ) FI . IAMInstCost += Size ; if ( isLargeStride ( & I ) ) FI . LSMInstCost += Size ; FI . MemInstCost += Size ; FI . InstCost += Size ; continue ; } if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { Function * Callee = CB -> getCalledFunction ( ) ; if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCost ; continue ; } if ( & F == Callee ) continue ; auto Loc = FIM . find ( Callee ) ; if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCost += Loc -> second . MemInstCost ; FI . InstCost += Loc -> second . InstCost ; FI . IAMInstCost += Loc -> second . IAMInstCost ; FI . LSMInstCost += Loc -> second . LSMInstCost ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ;

AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ;


<s> if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCost ; continue ; } if ( & F == Callee ) continue ; auto Loc = FIM . find ( Callee ) ; if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCost += Loc -> second . MemInstCost ; FI . InstCost += Loc -> second . InstCost ; FI . IAMInstCost += Loc -> second . IAMInstCost ; FI . LSMInstCost += Loc -> second . LSMInstCost ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI . InstCost ;


<s> bool <ISA_LIT>PerfHint :: runOnFunction ( Function & F ) { const Module & M = * F . getParent ( ) ; DL = & M . getDataLayout ( ) ; if ( F . hasFnAttribute ( <STR_LIT> ) && F . hasFnAttribute ( <STR_LIT> ) ) return false ; const <ISA_LIT>PerfHintAnalysis :: FuncInfo * Info = visit ( F ) ; LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << Info -> MemInstCost << '\n' << <STR_LIT> << Info -> IAMInstCost << '\n' << <STR_LIT> << Info -> LSMInstCost << '\n' << <STR_LIT> << Info -> InstCost << '\n' ) ; if ( isMemBound ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumMemBound ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; } if ( <STR_LIT> ( F . getCallingConv ( ) ) && needLimitWave ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ;


<s> NumMemBound ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; } if ( <STR_LIT> ( F . getCallingConv ( ) ) && needLimitWave ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumLimitWave ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; } return true ;


<s> for ( auto & I : B ) { if ( const Value * Ptr = getMemoryInstrPtr ( & I ) ) { unsigned Size = divideCeil ( Ptr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , <NUM_LIT> ) ; if ( isIndirectAccess ( & I ) ) FI . IAMInstCost += Size ; if ( isLargeStride ( & I ) ) FI . LSMInstCost += Size ; FI . MemInstCost += Size ; FI . InstCost += Size ; continue ; } if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { Function * Callee = CB -> getCalledFunction ( ) ; if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCost ; continue ; } if ( & F == Callee ) continue ; auto Loc = FIM . find ( Callee ) ; if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCost += Loc -> second . MemInstCost ; FI . InstCost += Loc -> second . InstCost ; FI . IAMInstCost += Loc -> second . IAMInstCost ; FI . LSMInstCost += Loc -> second . LSMInstCost ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI . InstCost ; } else { ++ FI . InstCost ; } }


<s> NumMemBound ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; } if ( <STR_LIT> ( F . getCallingConv ( ) ) && needLimitWave ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumLimitWave ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; }


<s> if ( isMemBound ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumMemBound ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; } if ( <STR_LIT> ( F . getCallingConv ( ) ) && needLimitWave ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumLimitWave ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ;


<s> Function * Callee = CS . getCalledFunction ( ) ; if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCount ; continue ; } if ( & F == Callee ) continue ; auto Loc = FIM . find ( Callee ) ; if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCount += Loc -> second . MemInstCount ; FI . InstCount += Loc -> second . InstCount ; FI . IAMInstCount += Loc -> second . IAMInstCount ; FI . LSMInstCount += Loc -> second . LSMInstCount ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI . InstCount ; } else {


<s> assert ( Loc != FIM . end ( ) && <STR_LIT> ) ;

LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << Loc -> second . MemInstCount << '\n' << <STR_LIT> << Loc -> second . IAMInstCount << '\n' << <STR_LIT> << Loc -> second . LSMInstCount << '\n' << <STR_LIT> << Loc -> second . InstCount << '\n' ) ;


<s> LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << Loc -> second . MemInstCount << '\n' << <STR_LIT> << Loc -> second . IAMInstCount << '\n' << <STR_LIT> << Loc -> second . LSMInstCount << '\n' << <STR_LIT> << Loc -> second . InstCount << '\n' ) ; auto & FI = Loc -> second ; if ( isMemBound ( FI ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << '\n' ) ; for ( auto & B : F ) { LastAccess = MemAccessInfo ( ) ; for ( auto & I : B ) { if ( getMemoryInstrPtr ( & I ) ) { if ( isIndirectAccess ( & I ) ) ++ FI . IAMInstCount ; if ( isLargeStride ( & I ) ) ++ FI . LSMInstCount ; ++ FI . MemInstCount ; ++ FI . InstCount ; continue ; } if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { Function * Callee = CB -> getCalledFunction ( ) ;

if ( ! Callee || Callee -> isDeclaration ( ) ) {


<s> if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCount += Loc -> second . MemInstCount ; FI . InstCount += Loc -> second . InstCount ; FI . IAMInstCount += Loc -> second . IAMInstCount ; FI . LSMInstCount += Loc -> second . LSMInstCount ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ;


<s> } while ( ! WorkSet . empty ( ) ) { const Value * V = * WorkSet . begin ( ) ; WorkSet . erase ( * WorkSet . begin ( ) ) ; if ( ! Visited . insert ( V ) . second ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * V << '\n' ) ; if ( auto LD = dyn_cast < LoadInst > ( V ) ) { auto M = LD -> getPointerOperand ( ) ; if ( isGlobalAddr ( M ) || isLocalAddr ( M ) || isConstantAddr ( M ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return true ; } continue ; } if ( auto GEP = dyn_cast < GetElementPtrInst > ( V ) ) { auto P = GEP -> getPointerOperand ( ) ; WorkSet . insert ( P ) ; for ( unsigned I = <NUM_LIT> , E = GEP -> getNumIndices ( ) + <NUM_LIT> ; I != E ; ++ I ) WorkSet . insert ( GEP -> getOperand ( I ) ) ; continue ; } if ( auto U = dyn_cast < UnaryInstruction > ( V ) ) { WorkSet . insert ( U -> getOperand ( <NUM_LIT> ) ) ;


<s> <ISA_LIT>PerfHintAnalysis ( ) : CallGraphSCCPass ( ID ) {


<s> FuncInfo ( ) : MemInstCost ( <NUM_LIT> ) , InstCost ( <NUM_LIT> ) , IAMInstCost ( <NUM_LIT> ) , LSMInstCost ( <NUM_LIT> ) {


<s> <ISA_LIT>PerfHintAnalysis ( ) : FunctionPass ( ID ) {


<s> FuncInfo ( ) : MemInstCount ( <NUM_LIT> ) , InstCount ( <NUM_LIT> ) , IAMInstCount ( <NUM_LIT> ) , LSMInstCount ( <NUM_LIT> ) {


<s> if ( auto AI = dyn_cast < AtomicRMWInst > ( Inst ) ) { return AI -> getPointerOperand ( ) ; } if ( auto MI = dyn_cast < AnyMemIntrinsic > ( Inst ) ) { return MI -> getRawDest ( ) ;


<s> return As == AS . CONSTANT_ADDRESS || As == AS . CONSTANT_ADDRESS_32BIT ; } return false ;


<s> if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) { unsigned As = PT -> getAddressSpace ( ) ; return As == AS . CONSTANT_ADDRESS || As == AS . CONSTANT_ADDRESS_32BIT ; } return false ;


<s> if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) { unsigned As = PT -> getAddressSpace ( ) ; return As == AS . GLOBAL_ADDRESS || As == AS . FLAT_ADDRESS ; } return false ;


<s> bool <ISA_LIT>PerfHint :: isGlobalAddr ( const Value * V ) const { if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) {


<s> return FI . MemInstCount * <NUM_LIT> / FI . InstCount > MemBoundThresh ;


<s> const Value * MO = getMemoryInstrPtr ( Inst ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MO << '\n' ) ;


<s> return ( ( FI . MemInstCount + FI . IAMInstCount * IAWeight + FI . LSMInstCount * LSWeight ) * <NUM_LIT> / FI . InstCount ) > LimitWaveThresh ;


<s> bool <ISA_LIT>PerfHintAnalysis :: runOnFunction ( Function & F ) { auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; if ( ! TPC ) return false ; const TargetMachine & TM = TPC -> getTM < TargetMachine > ( ) ;

const TargetSubtargetInfo * ST = TM . getSubtargetImpl ( F ) ;


<s> FI . IAMInstCount += Loc -> second . IAMInstCount ; FI . LSMInstCount += Loc -> second . LSMInstCount ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ;


<s> ++ FI . MemInstCount ; ++ FI . InstCount ; continue ; } CallSite CS ( const_cast < Instruction * > ( & I ) ) ; if ( CS ) { Function * Callee = CS . getCalledFunction ( ) ; if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCount ; continue ; } if ( & F == Callee ) continue ; visit ( * Callee ) ; auto Loc = FIM . find ( Callee ) ; assert ( Loc != FIM . end ( ) && <STR_LIT> ) ; FI . MemInstCount += Loc -> second . MemInstCount ; FI . InstCount += Loc -> second . InstCount ; FI . IAMInstCount += Loc -> second . IAMInstCount ; FI . LSMInstCount += Loc -> second . LSMInstCount ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI . InstCount ;


<s> if ( Ty == S32 ) { B . buildInstr ( <STR_LIT> , { DstReg } , { SrcReg } , MI . getFlags ( ) ) ; } else { auto Cvt0 = B . buildInstr ( <STR_LIT> , { S32 } , { SrcReg } , MI . getFlags ( ) ) ; B . buildFPTrunc ( DstReg , Cvt0 , MI . getFlags ( ) ) ; }


<s> MachineIRBuilder B ( MI ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

LLT Ty = B . getMRI ( ) -> getType ( DstReg ) ;


<s> assert ( SrcSize == <NUM_LIT> || SrcSize == <NUM_LIT> || SrcSize == <NUM_LIT> ) ; const APInt Mask = APInt :: getHighBitsSet ( SrcSize , SrcSize - <NUM_LIT> ) ; return Helper . getKnownBits ( ) -> maskedValueIsZero ( SrcReg , Mask ) ;


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_SHL : case TargetOpcode :: G_LSHR :


<s> CombinerHelper Helper ( Observer , B , KB , MDT , LInfo ) ; <ISA_LIT>GenPostLegalizerCombinerHelper Generated ( GeneratedRuleCfg ) ; if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) {


<s> unsigned NewOpc = <STR_LIT> + MatchInfo . ShiftOffset / <NUM_LIT> ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register CvtSrc = MatchInfo . CvtVal ; LLT SrcTy = MRI . getType ( MatchInfo . CvtVal ) ; if ( SrcTy != S32 ) { assert ( SrcTy . isScalar ( ) && SrcTy . getSizeInBits ( ) >= <NUM_LIT> ) ; CvtSrc = B . buildAnyExt ( S32 , CvtSrc ) . getReg ( <NUM_LIT> ) ; } assert ( MI . getOpcode ( ) != NewOpc ) ;

B . buildInstr ( NewOpc , { MI . getOperand ( <NUM_LIT> ) } , { CvtSrc } , MI . getFlags ( ) ) ;


<s> B . buildInstr ( Opc , { MI . getOperand ( <NUM_LIT> ) } , { X , Y } , MI . getFlags ( ) ) ; } ; switch ( Info . Pred ) { case CmpInst :: FCMP_ULT : case CmpInst :: FCMP_ULE : if ( Info . LHS == Info . True ) buildNewInst ( <STR_LIT> , Info . RHS , Info . LHS ) ; else buildNewInst ( <STR_LIT> , Info . LHS , Info . RHS ) ; break ; case CmpInst :: FCMP_OLE :


<s> if ( Ty == S32 ) { B . buildInstr ( <STR_LIT> , { DstReg } , { SrcReg } , MI . getFlags ( ) ) ; } else { auto Cvt0 = B . buildInstr ( <STR_LIT> , { S32 } , { SrcReg } , MI . getFlags ( ) ) ; B . buildFPTrunc ( DstReg , Cvt0 , MI . getFlags ( ) ) ;


<s> LLT SrcTy = MRI . getType ( SrcReg ) ; if ( SrcTy != S32 ) SrcReg = B . buildAnyExtOrTrunc ( S32 , SrcReg ) . getReg ( <NUM_LIT> ) ; if ( Ty == S32 ) { B . buildInstr ( <STR_LIT> , { DstReg } , { SrcReg } , MI . getFlags ( ) ) ; } else { auto Cvt0 = B . buildInstr ( <STR_LIT> , { S32 } , { SrcReg } , MI . getFlags ( ) ) ; B . buildFPTrunc ( DstReg , Cvt0 , MI . getFlags ( ) ) ;


<s> <ISA_LIT>PostLegalizerCombinerHelper PostLegalizerHelper ( B , Helper ) ; <ISA_LIT>GenPostLegalizerCombinerHelper Generated ( GeneratedRuleCfg , Helper , PostLegalizerHelper ) ; if ( Generated . tryCombineAll ( Observer , MI , B ) ) return true ;


<s> CombinerHelper Helper ( Observer , B , KB , MDT , LInfo ) ; <ISA_LIT>PostLegalizerCombinerHelper PostLegalizerHelper ( B , Helper ) ; <ISA_LIT>GenPostLegalizerCombinerHelper Generated ( GeneratedRuleCfg , Helper , PostLegalizerHelper ) ; if ( Generated . tryCombineAll ( Observer , MI , B ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_SHL : case TargetOpcode :: G_LSHR : case TargetOpcode :: G_ASHR : return Helper . tryCombineShiftToUnmerge ( MI , <NUM_LIT> ) ;


<s> Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; mi_match ( SrcReg , MRI , m_GZExt ( m_Reg ( SrcReg ) ) ) ; Register Src0 ; int64_t ShiftAmt ; bool IsShr = mi_match ( SrcReg , MRI , m_GLShr ( m_Reg ( Src0 ) , m_ICst ( ShiftAmt ) ) ) ; if ( IsShr || mi_match ( SrcReg , MRI , m_GShl ( m_Reg ( Src0 ) , m_ICst ( ShiftAmt ) ) ) ) {

const unsigned Offset = MI . getOpcode ( ) - <STR_LIT> ;


<s> const unsigned Offset = MI . getOpcode ( ) - <STR_LIT> ; unsigned ShiftOffset = <NUM_LIT> * Offset ; if ( IsShr ) ShiftOffset += ShiftAmt ; else ShiftOffset -= ShiftAmt ; MatchInfo . CvtVal = Src0 ; MatchInfo . ShiftOffset = ShiftOffset ; return ShiftOffset < <NUM_LIT> && ShiftOffset >= <NUM_LIT> && ( ShiftOffset % <NUM_LIT> ) == <NUM_LIT> ; } return false ;


<s> Info . True = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Info . False = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! ( Info . LHS == Info . True && Info . RHS == Info . False ) && ! ( Info . LHS == Info . False && Info . RHS == Info . True ) ) return false ; switch ( Info . Pred ) { case CmpInst :: FCMP_FALSE :


<s> Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcSize = MRI . getType ( SrcReg ) . getSizeInBits ( ) ; assert ( SrcSize == <NUM_LIT> || SrcSize == <NUM_LIT> || SrcSize == <NUM_LIT> ) ; const APInt Mask = APInt :: getHighBitsSet ( SrcSize , SrcSize - <NUM_LIT> ) ; return Helper . getKnownBits ( ) -> maskedValueIsZero ( SrcReg , Mask ) ;


<s> <ISA_LIT>PostLegalizerCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , const <ISA_LIT>LegalizerInfo * LI , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( false , true , LI , EnableOpt , OptSize , MinSize ) , KB ( KB ) , MDT ( MDT ) { if ( ! Generated . parseCommandLineOption ( ) ) report_fatal_error ( <STR_LIT> ) ;


<s> case TargetOpcode :: G_SHL : case TargetOpcode :: G_LSHR : case TargetOpcode :: G_ASHR : return Helper . tryCombineShiftToUnmerge ( MI , <NUM_LIT> ) ; }


<s> <ISA_LIT>CombinerHelper Helper ( Observer , B , KB , MDT , LInfo ) ; <ISA_LIT>PostLegalizerCombinerHelper PostLegalizerHelper ( B , Helper ) ; <ISA_LIT>GenPostLegalizerCombinerHelper Generated ( GeneratedRuleCfg , Helper , PostLegalizerHelper ) ; if ( Generated . tryCombineAll ( Observer , MI , B ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_SHL : case TargetOpcode :: G_LSHR : case TargetOpcode :: G_ASHR : return Helper . tryCombineShiftToUnmerge ( MI , <NUM_LIT> ) ; } return false ;


<s> Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; return TLI -> isCanonicalized ( Reg , MF ) ;


<s> MachineInstr * ResMI = nullptr ; if ( MI . getOpcode ( ) == TargetOpcode :: G_INTRIN<ISA_LIT>C && MI . getIntrinsicID ( ) == <STR_LIT> ) ResMI = MRI . getVRegDef ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; return ResMI ; } ; auto getSqrtSrc = [ = ] ( const MachineInstr & MI ) { MachineInstr * SqrtSrcMI = nullptr ; mi_match ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_GFSqrt ( m_MInstr ( SqrtSrcMI ) ) ) ; return SqrtSrcMI ; } ; MachineInstr * RcpSrcMI = nullptr , * SqrtSrcMI = nullptr ; if ( ( RcpSrcMI = getRcpSrc ( MI ) ) && ( SqrtSrcMI = getSqrtSrc ( * RcpSrcMI ) ) ) { MatchInfo = [ SqrtSrcMI , & MI ] ( MachineIRBuilder & B ) { B . buildIntrinsic ( <STR_LIT> , { MI . getOperand ( <NUM_LIT> ) } , false ) . addUse ( SqrtSrcMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . setMIFlags ( MI . getFlags ( ) ) ;


<s> <ISA_LIT>PostLegalizerCombiner :: <ISA_LIT>PostLegalizerCombiner ( bool IsOptNone ) : MachineFunctionPass ( ID ) , IsOptNone ( IsOptNone ) {


<s> <ISA_LIT>PostLegalizerCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , const <ISA_LIT>LegalizerInfo * LI , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( false , true , LI , EnableOpt , OptSize , MinSize ) , KB ( KB ) , MDT ( MDT ) { if ( ! GeneratedRuleCfg . parseCommandLineOption ( ) ) report_fatal_error ( <STR_LIT> ) ;


<s> unsigned NewOpc = <STR_LIT> + MatchInfo . ShiftOffset / <NUM_LIT> ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register CvtSrc = MatchInfo . CvtVal ; LLT SrcTy = B . getMRI ( ) -> getType ( MatchInfo . CvtVal ) ; if ( SrcTy != S32 ) { assert ( SrcTy . isScalar ( ) && SrcTy . getSizeInBits ( ) >= <NUM_LIT> ) ; CvtSrc = B . buildAnyExt ( S32 , CvtSrc ) . getReg ( <NUM_LIT> ) ;


<s> } case CmpInst :: FCMP_UGE : case CmpInst :: FCMP_UGT : { if ( Info . LHS == Info . True ) buildNewInst ( <STR_LIT> , Info . RHS , Info . LHS ) ; else buildNewInst ( <STR_LIT> , Info . LHS , Info . RHS ) ; break ; } case CmpInst :: FCMP_OGT : case CmpInst :: FCMP_OGE : { if ( Info . LHS == Info . True ) buildNewInst ( <STR_LIT> , Info . LHS , Info . RHS ) ; else buildNewInst ( <STR_LIT> , Info . RHS , Info . LHS ) ; break ; } default :


<s> LLT Ty = B . getMRI ( ) -> getType ( DstReg ) ; if ( Ty == S32 ) { B . buildInstr ( <STR_LIT> , { DstReg } , { MI . getOperand ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; } else { auto Cvt0 = B . buildInstr ( <STR_LIT> , { S32 } , { MI . getOperand ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; B . buildFPTrunc ( DstReg , Cvt0 , MI . getFlags ( ) ) ; }


<s> } else { auto Cvt0 = B . buildInstr ( <STR_LIT> , { S32 } , { MI . getOperand ( <NUM_LIT> ) } , MI . getFlags ( ) ) ;

B . buildFPTrunc ( DstReg , Cvt0 , MI . getFlags ( ) ) ;


<s> bool <ISA_LIT>PostLegalizerCombinerInfo :: combine ( GISelChangeObserver & Observer , MachineInstr & MI , MachineIRBuilder & B ) const { CombinerHelper Helper ( Observer , B , KB , MDT ) ; <ISA_LIT>GenPostLegalizerCombinerHelper Generated ( GeneratedRuleCfg ) ; if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_SHL : case TargetOpcode :: G_LSHR : case TargetOpcode :: G_ASHR : return Helper . tryCombineShiftToUnmerge ( MI , <NUM_LIT> ) ; } return false ;


<s> FunctionPass * create<ISA_LIT>PostLegalizeCombiner ( bool IsOptNone ) {

return new <ISA_LIT>PostLegalizerCombiner ( IsOptNone ) ;


<s> Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; mi_match ( SrcReg , MRI , m_GZExt ( m_Reg ( SrcReg ) ) ) ; Register Src0 ; int64_t ShiftAmt ; bool IsShr = mi_match ( SrcReg , MRI , m_GLShr ( m_Reg ( Src0 ) , m_ICst ( ShiftAmt ) ) ) ; if ( IsShr || mi_match ( SrcReg , MRI , m_GShl ( m_Reg ( Src0 ) , m_ICst ( ShiftAmt ) ) ) ) { const unsigned Offset = MI . getOpcode ( ) - <STR_LIT> ; unsigned ShiftOffset = <NUM_LIT> * Offset ; if ( IsShr ) ShiftOffset += ShiftAmt ;


<s> Info . True = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Info . False = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! ( Info . LHS == Info . True && Info . RHS == Info . False ) && ! ( Info . LHS == Info . False && Info . RHS == Info . True ) ) return false ; switch ( Info . Pred ) { case CmpInst :: FCMP_FALSE : case CmpInst :: FCMP_OEQ : case CmpInst :: FCMP_ONE : case CmpInst :: FCMP_ORD : case CmpInst :: FCMP_UNO : case CmpInst :: FCMP_UEQ : case CmpInst :: FCMP_UNE : case CmpInst :: FCMP_TRUE : return false ; default :


<s> static bool matchUCharToFloat ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineFunction & MF , CombinerHelper & Helper ) { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = MRI . getType ( DstReg ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) || Ty == LLT :: scalar ( <NUM_LIT> ) ) {

const APInt Mask = APInt :: getHighBitsSet ( <NUM_LIT> , <NUM_LIT> ) ;


<s> const APInt Mask = APInt :: getHighBitsSet ( <NUM_LIT> , <NUM_LIT> ) ; return Helper . getKnownBits ( ) -> maskedValueIsZero ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Mask ) ; } return false ;


<s> if ( MF . getProperties ( ) . hasProperty ( MachineFunctionProperties :: Property :: FailedISel ) ) return false ; auto * TPC = & getAnalysis < TargetPassConfig > ( ) ; const Function & F = MF . getFunction ( ) ; bool EnableOpt = MF . getTarget ( ) . getOptLevel ( ) != CodeGenOpt :: None && ! skipFunction ( F ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>LegalizerInfo * LI = static_cast < const <ISA_LIT>LegalizerInfo * > ( ST . getLegalizerInfo ( ) ) ; GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PostLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , LI , KB , MDT ) ; Combiner C ( PCInfo , TPC ) ;

return C . combineMachineInstrs ( MF , nullptr ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>LegalizerInfo * LI = static_cast < const <ISA_LIT>LegalizerInfo * > ( ST . getLegalizerInfo ( ) ) ; GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PostLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , LI , KB , MDT ) ; Combiner C ( PCInfo , TPC ) ;


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ;


<s> switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return Helper . tryCombineShuffleVector ( MI ) ;


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return Helper . tryCombineShuffleVector ( MI ) ; }


<s> void <ISA_LIT>PreLegalizerCombinerHelper :: applyClampI64ToI16 ( MachineInstr & MI , const ClampI64ToI16MatchInfo & MatchInfo ) { Register Src = MatchInfo . Origin ; assert ( MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) . getType ( Src ) == LLT :: scalar ( <NUM_LIT> ) ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; B . setMBB ( * MI . getParent ( ) ) ; B . setInstrAndDebugLoc ( MI ) ; auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ; auto MaxBoundaryDst = B . buildConstant ( S32 , MaxBoundary ) ; auto Bitcast = B . buildBitcast ( { S32 } , CvtPk ) ; auto Med3 = B . buildInstr ( <STR_LIT> , { S32 } , { MinBoundaryDst . getReg ( <NUM_LIT> ) , Bitcast . getReg ( <NUM_LIT> ) , MaxBoundaryDst . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ;

B . buildTrunc ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Med3 ) ;


<s> Register Src = MatchInfo . Origin ; assert ( MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) . getType ( Src ) == LLT :: scalar ( <NUM_LIT> ) ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; B . setMBB ( * MI . getParent ( ) ) ; B . setInstrAndDebugLoc ( MI ) ; auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ; auto MaxBoundaryDst = B . buildConstant ( S32 , MaxBoundary ) ; auto Bitcast = B . buildBitcast ( { S32 } , CvtPk ) ; auto Med3 = B . buildInstr ( <STR_LIT> , { S32 } , { MinBoundaryDst . getReg ( <NUM_LIT> ) , Bitcast . getReg ( <NUM_LIT> ) , MaxBoundaryDst . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; B . buildTrunc ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Med3 ) ;


<s> <ISA_LIT>PreLegalizerCombinerHelper PreLegalizerHelper ( B , Helper ) ; <ISA_LIT>GenPreLegalizerCombinerHelper Generated ( GeneratedRuleCfg , Helper , PreLegalizerHelper ) ; if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_MEMCPY_INLINE : return Helper . tryEmitMemcpyInline ( MI ) ; case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return Helper . tryCombineShuffleVector ( MI ) ; }


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_MEMCPY_INLINE : return Helper . tryEmitMemcpyInline ( MI ) ; case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ;


<s> if ( ! IsOptNone ) { AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; } AU . addRequired < GISelCSEAnalysisWrapperPass > ( ) ; AU . addPreserved < GISelCSEAnalysisWrapperPass > ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PreLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , KB , MDT ) ;

GISelCSEAnalysisWrapper & Wrapper = getAnalysis < GISelCSEAnalysisWrapperPass > ( ) . getCSEWrapper ( ) ;


<s> GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PreLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , KB , MDT ) ; GISelCSEAnalysisWrapper & Wrapper = getAnalysis < GISelCSEAnalysisWrapperPass > ( ) . getCSEWrapper ( ) ; auto * CSEInfo = & Wrapper . get ( TPC -> getCSEConfig ( ) ) ;


<s> B . setInstrAndDebugLoc ( MI ) ; auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ;

auto MaxBoundaryDst = B . buildConstant ( S32 , MaxBoundary ) ;


<s> void <ISA_LIT>PreLegalizerCombinerHelper :: applyClampI64ToI16 ( MachineInstr & MI , const ClampI64ToI16MatchInfo & MatchInfo ) { Register Src = MatchInfo . Origin ; assert ( MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) . getType ( Src ) == LLT :: scalar ( <NUM_LIT> ) ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; B . setInstrAndDebugLoc ( MI ) ; auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ;


<s> auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ;


<s> CombinerHelper Helper ( Observer , B , KB , MDT ) ; if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return Helper . tryCombineShuffleVector ( MI ) ;


<s> CombinerHelper Helper ( Observer , B , KB , MDT ) ; if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) {


<s> return new <ISA_LIT>PreLegalizerCombiner ( IsOptNone ) ;


<s> AU . addPreserved < GISelKnownBitsAnalysis > ( ) ; if ( ! IsOptNone ) { AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ;


<s> AU . addRequired < GISelKnownBitsAnalysis > ( ) ; AU . addPreserved < GISelKnownBitsAnalysis > ( ) ; if ( ! IsOptNone ) {

AU . addRequired < MachineDominatorTree > ( ) ;


<s> MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PreLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , KB , MDT ) ; Combiner C ( PCInfo , TPC ) ; return C . combineMachineInstrs ( MF , nullptr ) ;


<s> auto * TPC = & getAnalysis < TargetPassConfig > ( ) ; const Function & F = MF . getFunction ( ) ; bool EnableOpt = MF . getTarget ( ) . getOptLevel ( ) != CodeGenOpt :: None && ! skipFunction ( F ) ; GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ;

<ISA_LIT>PreLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , KB , MDT ) ;


<s> size_t CurFmtSpecifierIdx = <NUM_LIT> ; size_t PrevFmtSpecifierIdx = <NUM_LIT> ; while ( ( CurFmtSpecifierIdx = Fmt . find_first_of ( ConvSpecifiers , CurFmtSpecifierIdx ) ) != StringRef :: npos ) { bool ArgDump = false ; StringRef CurFmt = Fmt . substr ( PrevFmtSpecifierIdx , CurFmtSpecifierIdx - PrevFmtSpecifierIdx ) ; size_t pTag = CurFmt . find_last_of ( <STR_LIT> ) ; if ( pTag != StringRef :: npos ) { ArgDump = true ; while ( pTag && CurFmt [ -- pTag ] == '%' ) { ArgDump = ! ArgDump ; } } if ( ArgDump ) OpConvSpecifiers . push_back ( Fmt [ CurFmtSpecifierIdx ] ) ; PrevFmtSpecifierIdx = ++ CurFmtSpecifierIdx ;


<s> size_t CurFmtSpecifierIdx = <NUM_LIT> ; size_t PrevFmtSpecifierIdx = <NUM_LIT> ; while ( ( CurFmtSpecifierIdx = Fmt . find_first_of ( ConvSpecifiers , CurFmtSpecifierIdx ) ) != StringRef :: npos ) { bool ArgDump = false ; StringRef CurFmt = Fmt . substr ( PrevFmtSpecifierIdx , CurFmtSpecifierIdx - PrevFmtSpecifierIdx ) ; size_t pTag = CurFmt . find_last_of ( <STR_LIT> ) ; if ( pTag != StringRef :: npos ) { ArgDump = true ; while ( pTag && CurFmt [ -- pTag ] == '%' ) { ArgDump = ! ArgDump ; } } if ( ArgDump ) OpConvSpecifiers . push_back ( Fmt [ CurFmtSpecifierIdx ] ) ;

PrevFmtSpecifierIdx = ++ CurFmtSpecifierIdx ;