<s> const uint64_t QW = eatBytes < uint64_t > ( Bytes ) ; Res = tryDecodeInst ( DecoderTableDPP64 , MI , QW , Address ) ; if ( Res ) break ; Res = tryDecodeInst ( DecoderTableSDWA64 , MI , QW , Address ) ; if ( Res ) { IsSDWA = true ; break ; } Res = tryDecodeInst ( DecoderTableSDWA964 , MI , QW , Address ) ; if ( Res ) { IsSDWA = true ; break ; } } Bytes = Bytes_ . slice ( <NUM_LIT> , MaxInstBytesNum ) ; if ( Bytes . size ( ) < <NUM_LIT> ) break ; const uint32_t DW = eatBytes < uint32_t > ( Bytes ) ; Res = tryDecodeInst ( DecoderTableVI32 , MI , DW , Address ) ; if ( Res ) break ; Res = tryDecodeInst ( DecoderTable<ISA_LIT>32 , MI , DW , Address ) ; if ( Res ) break ;


<s> <ISA_LIT>Symbolizer ( MCContext & Ctx , std :: unique_ptr < MCRelocationInfo > && RelInfo , void * disInfo ) : MCSymbolizer ( Ctx , std :: move ( RelInfo ) ) , DisInfo ( disInfo ) {


<s> unsigned DMask = MI . getOperand ( DMaskIdx ) . getImm ( ) & <NUM_LIT> ; if ( DMask == <NUM_LIT> ) return MCDisassembler :: Success ; unsigned DstSize = IsGather4 ? <NUM_LIT> : countPopulation ( DMask ) ; if ( DstSize == <NUM_LIT> ) return MCDisassembler :: Success ; bool D16 = D16Idx >= <NUM_LIT> && MI . getOperand ( D16Idx ) . getImm ( ) ; if ( D16 && <STR_LIT> ( STI ) ) { DstSize = ( DstSize + <NUM_LIT> ) / <NUM_LIT> ; } if ( MI . getOperand ( TFEIdx ) . getImm ( ) ) return MCDisassembler :: Success ; int NewOpcode = - <NUM_LIT> ; if ( IsGather4 ) { if ( D16 && <STR_LIT> ( STI ) ) NewOpcode = <STR_LIT> ( MI . getOpcode ( ) , <NUM_LIT> ) ; else return MCDisassembler :: Success ; } else { NewOpcode = <STR_LIT> ( MI . getOpcode ( ) , DstSize ) ; if ( NewOpcode == - <NUM_LIT> ) return MCDisassembler :: Success ; } auto RCID = MCII -> get ( NewOpcode ) . OpInfo [ VDataIdx ] . RegClass ; unsigned Vdata0 = MI . getOperand ( VDataIdx ) . getReg ( ) ; unsigned VdataSub0 = MRI . getSubReg ( Vdata0 , <STR_LIT> ) ; Vdata0 = ( VdataSub0 != <NUM_LIT> ) ? VdataSub0 : Vdata0 ; auto NewVdata = MRI . getMatchingSuperReg ( Vdata0 , <STR_LIT> , & MRI . getRegClass ( RCID ) ) ; if ( NewVdata == <STR_LIT> ) { return MCDisassembler :: Success ; } MI . setOpcode ( NewOpcode ) ; MI . getOperand ( VDataIdx ) = MCOperand :: createReg ( NewVdata ) ; if ( IsAtomic ) { MI . getOperand ( VDstIdx ) = MCOperand :: createReg ( NewVdata ) ; } return MCDisassembler :: Success ;


<s> } else if ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) { int SDst = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: sdst ) ; if ( SDst != - <NUM_LIT> ) { insertNamedMCOperand ( MI , createRegOperand ( <STR_LIT> ) , <STR_LIT> :: sdst ) ; } else {

insertNamedMCOperand ( MI , MCOperand :: createImm ( <NUM_LIT> ) , <STR_LIT> :: omod ) ;


<s> return new <ISA_LIT>Disassembler ( STI , Ctx , T . createMCInstrInfo ( ) ) ;


<s> inline MCOperand <ISA_LIT>Disassembler :: createRegOperand ( unsigned int RegId ) const { return MCOperand :: createReg ( <STR_LIT> ( RegId , STI ) ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : shift = <NUM_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } if ( Val % ( <NUM_LIT> << shift ) ) { * CommentStream << <STR_LIT> << getRegClassName ( SRegClassID ) << <STR_LIT> << Val ;


<s> assert ( Val < <NUM_LIT> ) ; assert ( Width == OPW256 || Width == OPW512 ) ; if ( Val <= SGPR_MAX ) { assert ( SGPR_MIN == <NUM_LIT> ) ; return createSRegOperand ( getSgprClassId ( Width ) , Val - SGPR_MIN ) ; } int TTmpIdx = getTTmpIdx ( Val ) ; if ( TTmpIdx >= <NUM_LIT> ) { return createSRegOperand ( getTtmpClassId ( Width ) , TTmpIdx ) ; } llvm_unreachable ( <STR_LIT> ) ;


<s> MCOperand <ISA_LIT>Disassembler :: decodeOperand_SReg_256 ( unsigned Val ) const { return decodeDstOp ( OPW256 , Val ) ;


<s> MCOperand <ISA_LIT>Disassembler :: decodeOperand_SReg_512 ( unsigned Val ) const { return decodeDstOp ( OPW512 , Val ) ;


<s> return createSRegOperand ( getSgprClassId ( Width ) , Val - SDWA9EncValues :: SRC_SGPR_MIN ) ; } if ( SDWA9EncValues :: SRC_TTMP_MIN <= Val && Val <= SDWA9EncValues :: SRC_TTMP_MAX ) { return createSRegOperand ( getTtmpClassId ( Width ) , Val - SDWA9EncValues :: SRC_TTMP_MIN ) ; } const unsigned SVal = Val - SDWA9EncValues :: SRC_SGPR_MIN ; if ( INLINE_INTEGER_C_MIN <= SVal && SVal <= INLINE_INTEGER_C_MAX ) return decodeIntImmed ( SVal ) ; if ( INLINE_FLOATING_C_MIN <= SVal && SVal <= INLINE_FLOATING_C_MAX ) return decodeFPImmed ( Width , SVal ) ; return decodeSpecialReg32 ( SVal ) ; } else if ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) {

return createRegOperand ( getVgprClassId ( Width ) , Val ) ;


<s> using namespace <ISA_LIT> ; switch ( Val ) { case <NUM_LIT> : return createRegOperand ( FLAT_SCR_LO ) ; case <NUM_LIT> : return createRegOperand ( FLAT_SCR_HI ) ; case <NUM_LIT> : return createRegOperand ( XNACK_MASK_LO ) ; case <NUM_LIT> : return createRegOperand ( XNACK_MASK_HI ) ; case <NUM_LIT> : return createRegOperand ( VCC_LO ) ; case <NUM_LIT> : return createRegOperand ( VCC_HI ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TBA_LO ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TBA_HI ) ;

case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TMA_LO ) ;


<s> case <NUM_LIT> : return createRegOperand ( XNACK_MASK_LO ) ; case <NUM_LIT> : return createRegOperand ( XNACK_MASK_HI ) ; case <NUM_LIT> : return createRegOperand ( VCC_LO ) ; case <NUM_LIT> : return createRegOperand ( VCC_HI ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TBA_LO ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TBA_HI ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TMA_LO ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TMA_HI ) ; case <NUM_LIT> : return createRegOperand ( M0 ) ; case <NUM_LIT> : return createRegOperand ( EXEC_LO ) ; case <NUM_LIT> : return createRegOperand ( EXEC_HI ) ; case <NUM_LIT> : return createRegOperand ( SRC_SHARED_BASE ) ;

case <NUM_LIT> : return createRegOperand ( SRC_SHARED_LIMIT ) ;


<s> case <NUM_LIT> : return createRegOperand ( VCC ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TBA ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TMA ) ;

case <NUM_LIT> : return createRegOperand ( EXEC ) ;


<s> MCOperand <ISA_LIT>Disassembler :: decodeSpecialReg64 ( unsigned Val ) const { using namespace <ISA_LIT> ; switch ( Val ) { case <NUM_LIT> : return createRegOperand ( FLAT_SCR ) ; case <NUM_LIT> : return createRegOperand ( XNACK_MASK ) ; case <NUM_LIT> : return createRegOperand ( VCC ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TBA ) ; case <NUM_LIT> : assert ( ! isGFX9 ( ) ) ; return createRegOperand ( TMA ) ; case <NUM_LIT> : return createRegOperand ( EXEC ) ; default : break ; }

return errOperand ( Val , <STR_LIT> + Twine ( Val ) ) ;


<s> } if ( INLINE_INTEGER_C_MIN <= Val && Val <= INLINE_INTEGER_C_MAX ) return decodeIntImmed ( Val ) ; if ( INLINE_FLOATING_C_MIN <= Val && Val <= INLINE_FLOATING_C_MAX ) return decodeFPImmed ( Width , Val ) ; if ( Val == LITERAL_CONST ) return decodeLiteralConstant ( ) ; switch ( Width ) { case OPW32 : case OPW16 :


<s> MCOperand <ISA_LIT>Disassembler :: decodeSrcOp ( const OpWidthTy Width , unsigned Val ) const { using namespace <STR_LIT> ; assert ( Val < <NUM_LIT> ) ; if ( VGPR_MIN <= Val && Val <= VGPR_MAX ) { return createRegOperand ( getVgprClassId ( Width ) , Val - VGPR_MIN ) ; } if ( Val <= SGPR_MAX ) { assert ( SGPR_MIN == <NUM_LIT> ) ; return createSRegOperand ( getSgprClassId ( Width ) , Val - SGPR_MIN ) ; } int TTmpIdx = getTTmpIdx ( Val ) ; if ( TTmpIdx >= <NUM_LIT> ) { return createSRegOperand ( getTtmpClassId ( Width ) , TTmpIdx ) ; } if ( INLINE_INTEGER_C_MIN <= Val && Val <= INLINE_INTEGER_C_MAX ) return decodeIntImmed ( Val ) ;

if ( INLINE_FLOATING_C_MIN <= Val && Val <= INLINE_FLOATING_C_MAX ) return decodeFPImmed ( Width , Val ) ;


<s> if ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) { Res = tryDecodeInst ( DecoderTableGFX9_DL64 , MI , QW , Address ) ; if ( Res ) break ; } } Bytes = Bytes_ . slice ( <NUM_LIT> , MaxInstBytesNum ) ; if ( Bytes . size ( ) < <NUM_LIT> ) break ; const uint32_t DW = eatBytes < uint32_t > ( Bytes ) ; Res = tryDecodeInst ( DecoderTableVI32 , MI , DW , Address ) ; if ( Res ) break ; Res = tryDecodeInst ( DecoderTable<ISA_LIT>32 , MI , DW , Address ) ; if ( Res ) break ; Res = tryDecodeInst ( DecoderTableGFX932 , MI , DW , Address ) ; if ( Res ) break ; if ( Bytes . size ( ) < <NUM_LIT> ) break ; const uint64_t QW = ( ( uint64_t ) eatBytes < uint32_t > ( Bytes ) << <NUM_LIT> ) | DW ; Res = tryDecodeInst ( DecoderTableVI64 , MI , QW , Address ) ; if ( Res ) break ; Res = tryDecodeInst ( DecoderTable<ISA_LIT>64 , MI , QW , Address ) ; if ( Res ) break ; Res = tryDecodeInst ( DecoderTableGFX964 , MI , QW , Address ) ; } while ( false ) ; if ( Res && ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) ) { insertNamedMCOperand ( MI , MCOperand :: createImm ( <NUM_LIT> ) , <STR_LIT> :: src2_modifiers ) ; } if ( Res && ( MCII -> get ( MI . getOpcode ( ) ) . TSFlags & <STR_LIT> ) ) { Res = convertMIMGInst ( MI ) ; } if ( Res && IsSDWA ) Res = convertSDWAInst ( MI ) ;

Size = Res ? ( MaxInstBytesNum - Bytes . size ( ) ) : std :: min ( ( size_t ) <NUM_LIT> , Bytes_ . size ( ) ) ;


<s> Res = tryDecodeInst ( DecoderTableGFX80_UNPACKED64 , MI , QW , Address ) ; if ( Res ) break ; } if ( STI . getFeatureBits ( ) [ <STR_LIT> ] ) { Res = tryDecodeInst ( DecoderTableGFX9_DL64 , MI , QW , Address ) ; if ( Res ) break ; } } Bytes = Bytes_ . slice ( <NUM_LIT> , MaxInstBytesNum ) ; if ( Bytes . size ( ) < <NUM_LIT> ) break ; const uint32_t DW = eatBytes < uint32_t > ( Bytes ) ; Res = tryDecodeInst ( DecoderTableVI32 , MI , DW , Address ) ; if ( Res ) break ; Res = tryDecodeInst ( DecoderTable<ISA_LIT>32 , MI , DW , Address ) ; if ( Res ) break ; Res = tryDecodeInst ( DecoderTableGFX932 , MI , DW , Address ) ; if ( Res ) break ; if ( Bytes . size ( ) < <NUM_LIT> ) break ; const uint64_t QW = ( ( uint64_t ) eatBytes < uint32_t > ( Bytes ) << <NUM_LIT> ) | DW ; Res = tryDecodeInst ( DecoderTableVI64 , MI , QW , Address ) ; if ( Res ) break ;

Res = tryDecodeInst ( DecoderTable<ISA_LIT>64 , MI , QW , Address ) ;


<s> unsigned <ISA_LIT>Disassembler :: getSgprClassId ( const OpWidthTy Width ) const { using namespace <ISA_LIT> ; assert ( OPW_FIRST_ <= Width && Width < OPW_LAST_ ) ; switch ( Width ) { default : case OPW32 : case OPW16 :


<s> assert ( OPW_FIRST_ <= Width && Width < OPW_LAST_ ) ; switch ( Width ) { default : case OPW32 : case OPW16 : case OPWV216 : return TTMP_32RegClassID ; case OPW64 : return TTMP_64RegClassID ; case OPW128 : return TTMP_128RegClassID ; case OPW256 : return TTMP_256RegClassID ; case OPW512 : return TTMP_512RegClassID ; }


<s> case OPW16 : case OPWV216 : return TTMP_32RegClassID ; case OPW64 : return TTMP_64RegClassID ; case OPW128 : return TTMP_128RegClassID ; case OPW256 : return TTMP_256RegClassID ; case OPW512 : return TTMP_512RegClassID ; }


<s> using namespace <STR_LIT> ; unsigned TTmpMin = isGFX9 ( ) ? TTMP_GFX9_MIN : TTMP_VI_MIN ;

unsigned TTmpMax = isGFX9 ( ) ? TTMP_GFX9_MAX : TTMP_VI_MAX ;


<s> unsigned TTmpMin = isGFX9 ( ) ? TTMP_GFX9_MIN : TTMP_VI_MIN ; unsigned TTmpMax = isGFX9 ( ) ? TTMP_GFX9_MAX : TTMP_VI_MAX ; return ( TTmpMin <= Val && Val <= TTmpMax ) ? Val - TTmpMin : - <NUM_LIT> ;


<s> } auto * Symbols = static_cast < SectionSymbolsTy * > ( DisInfo ) ; if ( ! Symbols ) return false ; auto Result = std :: find_if ( Symbols -> begin ( ) , Symbols -> end ( ) , [ Value ] ( const SymbolInfoTy & Val ) { return std :: get < <NUM_LIT> > ( Val ) == static_cast < uint64_t > ( Value ) && std :: get < <NUM_LIT> > ( Val ) == ELF :: STT_NOTYPE ;


<s> } auto * Symbols = static_cast < SectionSymbolsTy * > ( DisInfo ) ; if ( ! Symbols ) return false ; auto Result = std :: find_if ( Symbols -> begin ( ) , Symbols -> end ( ) , [ Value ] ( const SymbolInfoTy & Val ) { return std :: get < <NUM_LIT> > ( Val ) == static_cast < uint64_t > ( Value ) && std :: get < <NUM_LIT> > ( Val ) == ELF :: STT_NOTYPE ; } ) ; if ( Result != Symbols -> end ( ) ) { auto * Sym = Ctx . getOrCreateSymbol ( std :: get < <NUM_LIT> > ( * Result ) ) ; const auto * Add = MCSymbolRefExpr :: create ( Sym , Ctx ) ; Inst . addOperand ( MCOperand :: createExpr ( Add ) ) ; return true ; } return false ;


<s> <ISA_LIT>ELFObjectWriter :: <ISA_LIT>ELFObjectWriter ( bool Is64Bit ) : MCELFObjectTargetWriter ( Is64Bit , ELF :: ELFOSABI_<ISA_LIT>_HSA , ELF :: EM_<ISA_LIT> , false ) {


<s> MCELFObjectTargetWriter * MOTW = new <ISA_LIT>ELFObjectWriter ( Is64Bit ) ; return createELFObjectWriter ( MOTW , OS , true ) ;


<s> unsigned getRelocType ( MCContext & Ctx , const MCValue & Target , const MCFixup & Fixup , bool IsPCRel ) const override { return Fixup . getKind ( ) ;


<s> unsigned GetRelocType ( const MCValue & Target , const MCFixup & Fixup , bool IsPCRel ) const override {


<s> unsigned GetRelocType ( const MCValue & Target , const MCFixup & Fixup , bool IsPCRel ) const override { return Fixup . getKind ( ) ;


<s> std :: unique_ptr < MCObjectWriter > llvm :: create<ISA_LIT>ELFObjectWriter ( bool Is64Bit , uint8_t OSABI , bool HasRelocationAddend , raw_pwrite_stream & OS ) {

auto MOTW = llvm :: make_unique < <ISA_LIT>ELFObjectWriter > ( Is64Bit , OSABI , HasRelocationAddend ) ;


<s> case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_LO : return ELF :: R_<ISA_LIT>_GOTPCREL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_HI : return ELF :: R_<ISA_LIT>_GOTPCREL32_HI ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_LO : return ELF :: R_<ISA_LIT>_REL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_HI : return ELF :: R_<ISA_LIT>_REL32_HI ; } switch ( Fixup . getKind ( ) ) { default : break ; case FK_PCRel_4 :


<s> <ISA_LIT>ELFObjectWriter :: <ISA_LIT>ELFObjectWriter ( bool Is64Bit , uint8_t OSABI , bool HasRelocationAddend , uint8_t ABIVersion ) : MCELFObjectTargetWriter ( Is64Bit , OSABI , ELF :: EM_<ISA_LIT> , HasRelocationAddend , ABIVersion ) {


<s> return ELF :: R_<ISA_LIT>_GOTPCREL ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_LO : return ELF :: R_<ISA_LIT>_GOTPCREL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_HI : return ELF :: R_<ISA_LIT>_GOTPCREL32_HI ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_LO : return ELF :: R_<ISA_LIT>_REL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_HI : return ELF :: R_<ISA_LIT>_REL32_HI ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL64 : return ELF :: R_<ISA_LIT>_REL64 ; } switch ( Fixup . getKind ( ) ) { default : break ; case FK_PCRel_4 : return ELF :: R_<ISA_LIT>_REL32 ; case FK_Data_4 : case FK_SecRel_4 :


<s> return ELF :: R_<ISA_LIT>_GOTPCREL ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_LO : return ELF :: R_<ISA_LIT>_GOTPCREL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_HI : return ELF :: R_<ISA_LIT>_GOTPCREL32_HI ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_LO : return ELF :: R_<ISA_LIT>_REL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_HI : return ELF :: R_<ISA_LIT>_REL32_HI ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL64 : return ELF :: R_<ISA_LIT>_REL64 ; } switch ( Fixup . getKind ( ) ) { default : break ; case FK_PCRel_4 : return ELF :: R_<ISA_LIT>_REL32 ;


<s> if ( SymA -> getSymbol ( ) . getName ( ) == <STR_LIT> || SymA -> getSymbol ( ) . getName ( ) == <STR_LIT> ) return ELF :: R_<ISA_LIT>_ABS32_LO ; } switch ( Target . getAccessVariant ( ) ) { default : break ; case MCSymbolRefExpr :: VK_GOTPCREL : return ELF :: R_<ISA_LIT>_GOTPCREL ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_LO : return ELF :: R_<ISA_LIT>_GOTPCREL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_HI : return ELF :: R_<ISA_LIT>_GOTPCREL32_HI ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_LO : return ELF :: R_<ISA_LIT>_REL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_HI :


<s> <ISA_LIT>ELFObjectWriter :: <ISA_LIT>ELFObjectWriter ( bool Is64Bit , bool HasRelocationAddend ) : MCELFObjectTargetWriter ( Is64Bit , ELF :: ELFOSABI_<ISA_LIT>_HSA , ELF :: EM_<ISA_LIT> , HasRelocationAddend ) {


<s> MCObjectWriter * llvm :: create<ISA_LIT>ELFObjectWriter ( bool Is64Bit , bool HasRelocationAddend , raw_pwrite_stream & OS ) { MCELFObjectTargetWriter * MOTW = new <ISA_LIT>ELFObjectWriter ( Is64Bit , HasRelocationAddend ) ;

return createELFObjectWriter ( MOTW , OS , true ) ;


<s> MCELFObjectTargetWriter * MOTW = new <ISA_LIT>ELFObjectWriter ( Is64Bit , HasRelocationAddend ) ; return createELFObjectWriter ( MOTW , OS , true ) ;


<s> std :: unique_ptr < MCObjectTargetWriter > llvm :: create<ISA_LIT>ELFObjectWriter ( bool Is64Bit , uint8_t OSABI , bool HasRelocationAddend , uint8_t ABIVersion ) {

return llvm :: make_unique < <ISA_LIT>ELFObjectWriter > ( Is64Bit , OSABI , HasRelocationAddend , ABIVersion ) ;


<s> case FK_Data_8 : return ELF :: R_<ISA_LIT>_ABS64 ; } if ( Fixup . getTargetKind ( ) == <STR_LIT> ) { const auto * SymA = Target . getSymA ( ) ; assert ( SymA ) ; if ( SymA -> getSymbol ( ) . isUndefined ( ) ) { Ctx . reportError ( Fixup . getLoc ( ) , Twine ( <STR_LIT> ) + SymA -> getSymbol ( ) . getName ( ) + <STR_LIT> ) ; return ELF :: R_<ISA_LIT>_NONE ; }


<s> <ISA_LIT>ELFObjectWriter :: <ISA_LIT>ELFObjectWriter ( bool Is64Bit , uint8_t OSABI , bool HasRelocationAddend ) : MCELFObjectTargetWriter ( Is64Bit , OSABI , ELF :: EM_<ISA_LIT> , HasRelocationAddend ) {


<s> return llvm :: make_unique < <ISA_LIT>ELFObjectWriter > ( Is64Bit , OSABI , HasRelocationAddend ) ;


<s> unsigned <ISA_LIT>ELFObjectWriter :: getRelocType ( MCContext & Ctx , const MCValue & Target , const MCFixup & Fixup , bool IsPCRel ) const { if ( const auto * SymA = Target . getSymA ( ) ) { if ( SymA -> getSymbol ( ) . getName ( ) == <STR_LIT> ) return ELF :: R_<ISA_LIT>_ABS32_LO ; if ( SymA -> getSymbol ( ) . getName ( ) == <STR_LIT> ) return ELF :: R_<ISA_LIT>_ABS32_HI ; } switch ( Target . getAccessVariant ( ) ) { default : break ; case MCSymbolRefExpr :: VK_GOTPCREL : return ELF :: R_<ISA_LIT>_GOTPCREL ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_LO : return ELF :: R_<ISA_LIT>_GOTPCREL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_GOTPCREL32_HI : return ELF :: R_<ISA_LIT>_GOTPCREL32_HI ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_LO : return ELF :: R_<ISA_LIT>_REL32_LO ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL32_HI : return ELF :: R_<ISA_LIT>_REL32_HI ; case MCSymbolRefExpr :: VK_<ISA_LIT>_REL64 : return ELF :: R_<ISA_LIT>_REL64 ;


<s> void <ISA_LIT>ELFStreamer :: InitSections ( bool NoExecStack ) {

SwitchSection ( <STR_LIT> ( getContext ( ) ) ) ;


<s> SwitchSection ( <STR_LIT> ( getContext ( ) ) ) ;


<s> MCELFStreamer * llvm :: create<ISA_LIT>ELFStreamer ( const Triple & T , MCContext & Context , std :: unique_ptr < MCAsmBackend > MAB , raw_pwrite_stream & OS , std :: unique_ptr < MCCodeEmitter > Emitter , bool RelaxAll ) { return new <ISA_LIT>ELFStreamer ( T , Context , std :: move ( MAB ) , OS , std :: move ( Emitter ) ) ;


<s> <ISA_LIT>ELFStreamer ( const Triple & T , MCContext & Context , std :: unique_ptr < MCAsmBackend > MAB , raw_pwrite_stream & OS , std :: unique_ptr < MCCodeEmitter > Emitter ) : MCELFStreamer ( Context , std :: move ( MAB ) , OS , std :: move ( Emitter ) ) {


<s> <ISA_LIT>ELFStreamer ( MCContext & Context , MCAsmBackend & MAB , raw_pwrite_stream & OS , MCCodeEmitter * Emitter ) : MCELFStreamer ( Context , MAB , OS , Emitter ) {


<s> static bool isPositionExport ( const <ISA_LIT>InstrInfo * TII , SUnit * SU ) {

const MachineInstr * MI = SU -> getInstr ( ) ;


<s> static bool isExport ( const SUnit & SU ) { const MachineInstr * MI = SU . getInstr ( ) ;

return MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ;


<s> return MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ;


<s> static bool isPositionExport ( const <ISA_LIT>InstrInfo * TII , SUnit * SU ) { const MachineInstr * MI = SU -> getInstr ( ) ;

int Imm = TII -> getNamedOperand ( * MI , <STR_LIT> :: tgt ) -> getImm ( ) ;


<s> if ( isPositionExport ( TII , & SU ) ) PosCount ++ ; removeExportDependencies ( DAG , SU ) ; SmallVector < SDep , <NUM_LIT> > Succs ( SU . Succs ) ; for ( SDep Succ : Succs ) removeExportDependencies ( DAG , * Succ . getSUnit ( ) ) ; } if ( Chain . size ( ) > <NUM_LIT> ) {


<s> SUnit * ChainHead = Exports . front ( ) ; for ( unsigned Idx = <NUM_LIT> , End = Exports . size ( ) - <NUM_LIT> ; Idx < End ; ++ Idx ) { SUnit * SUa = Exports [ Idx ] ;

SUnit * SUb = Exports [ Idx + <NUM_LIT> ] ;


<s> if ( ! isExport ( * PredSU ) && ! Pred . isWeak ( ) ) DAG -> addEdge ( ChainHead , SDep ( PredSU , SDep :: Artificial ) ) ; } DAG -> addEdge ( SUb , SDep ( SUa , SDep :: Barrier ) ) ;

DAG -> addEdge ( SUb , SDep ( SUa , SDep :: Cluster ) ) ;


<s> return std :: make_unique < ExportClustering > ( ) ;


<s> const MachineInstr * MI = SU -> getInstr ( ) ; int Imm = TII -> getNamedOperand ( * MI , <STR_LIT> :: tgt ) -> getImm ( ) ; return Imm >= <STR_LIT> :: ET_POS0 && Imm <= <STR_LIT> :: ET_POS_LAST ;


<s> for ( const SDep & ExportPred : PredSU -> Preds ) { SUnit * ExportPredSU = ExportPred . getSUnit ( ) ; if ( ExportPred . isBarrier ( ) && ! isExport ( * ExportPredSU ) ) ToAdd . push_back ( SDep ( ExportPredSU , SDep :: Barrier ) ) ; } } } for ( SDep Pred : ToRemove ) SU . removePred ( Pred ) ; for ( SDep Pred : ToAdd ) DAG -> addEdge ( & SU , Pred ) ;


<s> for ( SUnit * SU : Copy ) { if ( isPositionExport ( TII , SU ) ) Chain [ PosIdx ++ ] = SU ; else Chain [ OtherIdx ++ ] = SU ; }


<s> void visitCallBase ( CallBase & CB ) { if ( CB . getCalledFunction ( ) ) return ; auto * Callee = dyn_cast < Function > ( CB . getCalledOperand ( ) -> stripPointerCasts ( ) ) ; if ( Callee && isLegalToPromote ( CB , Callee ) ) { promoteCall ( CB , Callee ) ;


<s> if ( CB . getCalledFunction ( ) ) return ; auto * Callee = dyn_cast < Function > ( CB . getCalledOperand ( ) -> stripPointerCasts ( ) ) ;

if ( Callee && isLegalToPromote ( CB , Callee ) ) {


<s> if ( CS . getCalledFunction ( ) ) return ; auto Callee = dyn_cast < Function > ( CS . getCalledValue ( ) -> stripPointerCasts ( ) ) ; if ( Callee && isLegalToPromote ( CS , Callee ) ) {


<s> <ISA_LIT>FixFunctionBitcasts ( ) : ModulePass ( ID ) {


<s> return new <ISA_LIT>FixFunctionBitcasts ( ) ;


<s> bool <ISA_LIT>FixFunctionBitcasts :: runOnModule ( Module & M ) {


<s> bool <ISA_LIT>FixFunctionBitcasts :: runOnModule ( Module & M ) { Modified = false ; visit ( M ) ;


<s> if ( Callee && isLegalToPromote ( CS , Callee ) ) { promoteCall ( CS , Callee ) ;


<s> <ISA_LIT>FrameLowering :: <ISA_LIT>FrameLowering ( StackDirection D , unsigned StackAl , int LAO , unsigned TransAl ) : TargetFrameLowering ( D , StackAl , LAO , TransAl ) {


<s> void <ISA_LIT>FrameLowering :: emitEpilogue ( MachineFunction & MF , MachineBasicBlock & MBB ) const {


<s> void <ISA_LIT>FrameLowering :: emitPrologue ( MachineFunction & MF , MachineBasicBlock & MBB ) const {


<s> NumEntries = <NUM_LIT> ; return nullptr ;


<s> int <ISA_LIT>FrameLowering :: getFrameIndexReference ( const MachineFunction & MF , int FI , unsigned & FrameReg ) const { const MachineFrameInfo * MFI = MF . getFrameInfo ( ) ; const TargetRegisterInfo * RI = MF . getSubtarget ( ) . getRegisterInfo ( ) ;

FrameReg = RI -> getFrameRegister ( MF ) ;


<s> int <ISA_LIT>FrameLowering :: getFrameIndexReference ( const MachineFunction & MF , int FI , unsigned & FrameReg ) const { const MachineFrameInfo * MFI = MF . getFrameInfo ( ) ; const TargetRegisterInfo * RI = MF . getSubtarget ( ) . getRegisterInfo ( ) ; FrameReg = RI -> getFrameRegister ( MF ) ; unsigned OffsetBytes = <NUM_LIT> * ( getStackWidth ( MF ) * <NUM_LIT> ) ; int UpperBound = FI == - <NUM_LIT> ? MFI -> getNumObjects ( ) : FI ; for ( int i = MFI -> getObjectIndexBegin ( ) ; i < UpperBound ; ++ i ) { OffsetBytes = RoundUpToAlignment ( OffsetBytes , MFI -> getObjectAlignment ( i ) ) ; OffsetBytes += MFI -> getObjectSize ( i ) ; OffsetBytes = RoundUpToAlignment ( OffsetBytes , <NUM_LIT> ) ; } if ( FI != - <NUM_LIT> ) OffsetBytes = RoundUpToAlignment ( OffsetBytes , MFI -> getObjectAlignment ( FI ) ) ;

return OffsetBytes / ( getStackWidth ( MF ) * <NUM_LIT> ) ;


<s> unsigned <ISA_LIT>FrameLowering :: getStackWidth ( const MachineFunction & MF ) const {


<s> bool <ISA_LIT>FrameLowering :: hasFP ( const MachineFunction & MF ) const { return false ;


<s> <ISA_LIT>FrameLowering :: ~ <ISA_LIT>FrameLowering ( ) {


<s> unsigned OffsetBytes = <NUM_LIT> * ( getStackWidth ( MF ) * <NUM_LIT> ) ; int UpperBound = FI == - <NUM_LIT> ? MFI -> getNumObjects ( ) : FI ; for ( int i = MFI -> getObjectIndexBegin ( ) ; i < UpperBound ; ++ i ) { OffsetBytes = alignTo ( OffsetBytes , MFI -> getObjectAlignment ( i ) ) ;

OffsetBytes += MFI -> getObjectSize ( i ) ;


<s> unsigned OffsetBytes = <NUM_LIT> * ( getStackWidth ( MF ) * <NUM_LIT> ) ; int UpperBound = FI == - <NUM_LIT> ? MFI -> getNumObjects ( ) : FI ; for ( int i = MFI -> getObjectIndexBegin ( ) ; i < UpperBound ; ++ i ) { OffsetBytes = alignTo ( OffsetBytes , MFI -> getObjectAlignment ( i ) ) ; OffsetBytes += MFI -> getObjectSize ( i ) ; OffsetBytes = alignTo ( OffsetBytes , <NUM_LIT> ) ; } if ( FI != - <NUM_LIT> ) OffsetBytes = alignTo ( OffsetBytes , MFI -> getObjectAlignment ( FI ) ) ;


<s> unsigned OffsetBytes = <NUM_LIT> * ( getStackWidth ( MF ) * <NUM_LIT> ) ; int UpperBound = FI == - <NUM_LIT> ? MFI -> getNumObjects ( ) : FI ; for ( int i = MFI -> getObjectIndexBegin ( ) ; i < UpperBound ; ++ i ) { OffsetBytes = RoundUpToAlignment ( OffsetBytes , MFI -> getObjectAlignment ( i ) ) ; OffsetBytes += MFI -> getObjectSize ( i ) ; OffsetBytes = RoundUpToAlignment ( OffsetBytes , <NUM_LIT> ) ; } if ( FI != - <NUM_LIT> ) OffsetBytes = RoundUpToAlignment ( OffsetBytes , MFI -> getObjectAlignment ( FI ) ) ;


<s> <ISA_LIT>FrameLowering :: <ISA_LIT>FrameLowering ( StackDirection D , Align StackAl , int LAO , Align TransAl ) : TargetFrameLowering ( D , StackAl , LAO , TransAl ) {


<s> int <ISA_LIT>FrameLowering :: getFrameIndexReference ( const MachineFunction & MF , int FI , unsigned & FrameReg ) const { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const <ISA_LIT>RegisterInfo * RI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; FrameReg = RI -> getFrameRegister ( MF ) ; unsigned OffsetBytes = <NUM_LIT> * ( getStackWidth ( MF ) * <NUM_LIT> ) ; int UpperBound = FI == - <NUM_LIT> ? MFI . getNumObjects ( ) : FI ; for ( int i = MFI . getObjectIndexBegin ( ) ; i < UpperBound ; ++ i ) { OffsetBytes = alignTo ( OffsetBytes , MFI . getObjectAlignment ( i ) ) ; OffsetBytes += MFI . getObjectSize ( i ) ;

OffsetBytes = alignTo ( OffsetBytes , <NUM_LIT> ) ;


<s> FrameReg = RI -> getFrameRegister ( MF ) ; unsigned OffsetBytes = <NUM_LIT> * ( getStackWidth ( MF ) * <NUM_LIT> ) ; int UpperBound = FI == - <NUM_LIT> ? MFI -> getNumObjects ( ) : FI ; for ( int i = MFI -> getObjectIndexBegin ( ) ; i < UpperBound ; ++ i ) { OffsetBytes = alignTo ( OffsetBytes , MFI -> getObjectAlignment ( i ) ) ; OffsetBytes += MFI -> getObjectSize ( i ) ;


<s> <ISA_LIT>FrameLowering :: <ISA_LIT>FrameLowering ( StackDirection D , Align StackAl , int LAO , unsigned TransAl ) : TargetFrameLowering ( D , StackAl , LAO , TransAl ) {


<s> } int64_t Offset ; if ( Def -> getOpcode ( ) == TargetOpcode :: G_ADD ) { if ( mi_match ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_ICst ( Offset ) ) ) return std :: make_tuple ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , Offset , Def ) ; if ( mi_match ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_Copy ( m_ICst ( Offset ) ) ) ) return std :: make_tuple ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , Offset , Def ) ; }

return std :: make_tuple ( Reg , <NUM_LIT> , Def ) ;


<s> MachineInstr * Def = getDefIgnoringCopies ( Reg , MRI ) ; if ( ! Def ) return std :: make_tuple ( Reg , <NUM_LIT> , nullptr ) ; if ( Def -> getOpcode ( ) == TargetOpcode :: G_CONSTANT ) { unsigned Offset ; const MachineOperand & Op = Def -> getOperand ( <NUM_LIT> ) ; if ( Op . isImm ( ) ) Offset = Op . getImm ( ) ; else Offset = Op . getCImm ( ) -> getZExtValue ( ) ; return std :: make_tuple ( Register ( ) , Offset , Def ) ; } int64_t Offset ;

if ( Def -> getOpcode ( ) == TargetOpcode :: G_ADD ) {


<s> inline int getDMaskIdx ( const <STR_LIT> * BaseOpcode , int NumDefs ) { assert ( ! BaseOpcode -> Atomic ) ;

return NumDefs + <NUM_LIT> + ( BaseOpcode -> Store ? <NUM_LIT> : <NUM_LIT> ) ;


<s> inline std :: pair < int , int > getImageNumVAddr ( const <STR_LIT> * ImageDimIntr , const <STR_LIT> * BaseOpcode ) { const <STR_LIT> * DimInfo = <STR_LIT> ( ImageDimIntr -> Dim ) ; int NumGradients = BaseOpcode -> Gradients ? DimInfo -> NumGradients : <NUM_LIT> ; int NumCoords = BaseOpcode -> Coordinates ? DimInfo -> NumCoords : <NUM_LIT> ; int NumLCM = BaseOpcode -> LodOrClampOrMip ? <NUM_LIT> : <NUM_LIT> ; int NumVAddr = BaseOpcode -> NumExtraArgs + NumGradients + NumCoords + NumLCM ;


<s> int NumLCM = BaseOpcode -> LodOrClampOrMip ? <NUM_LIT> : <NUM_LIT> ; int NumVAddr = BaseOpcode -> NumExtraArgs + NumGradients + NumCoords + NumLCM ; return { NumVAddr , NumGradients } ;


<s> if ( BaseOpcode -> Atomic ) return NumDefs + <NUM_LIT> + ( BaseOpcode -> AtomicX2 ? <NUM_LIT> : <NUM_LIT> ) ;

return getDMaskIdx ( BaseOpcode , NumDefs ) + <NUM_LIT> ;


<s> MachineInstr * Def = getDefIgnoringCopies ( Reg , MRI ) ; if ( ! Def ) return std :: make_pair ( Reg , <NUM_LIT> ) ; if ( Def -> getOpcode ( ) == TargetOpcode :: G_CONSTANT ) { unsigned Offset ; const MachineOperand & Op = Def -> getOperand ( <NUM_LIT> ) ; if ( Op . isImm ( ) ) Offset = Op . getImm ( ) ; else Offset = Op . getCImm ( ) -> getZExtValue ( ) ; return std :: make_pair ( Register ( ) , Offset ) ; } int64_t Offset ; if ( Def -> getOpcode ( ) == TargetOpcode :: G_ADD ) { if ( mi_match ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_ICst ( Offset ) ) ) return std :: make_pair ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , Offset ) ; if ( mi_match ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_Copy ( m_ICst ( Offset ) ) ) ) return std :: make_pair ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , Offset ) ; } if ( Def -> getOpcode ( ) == TargetOpcode :: G_PTRTOINT ) { MachineInstr * Base ; if ( mi_match ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_GPtrAdd ( m_MInstr ( Base ) , m_ICst ( Offset ) ) ) ) { if ( Base -> getOpcode ( ) == TargetOpcode :: G_INTTOPTR ) return std :: make_pair ( Base -> getOperand ( <NUM_LIT> ) . getReg ( ) , Offset ) ;


<s> if ( Op . isImm ( ) ) Offset = Op . getImm ( ) ; else Offset = Op . getCImm ( ) -> getZExtValue ( ) ; return std :: make_pair ( Register ( ) , Offset ) ; } int64_t Offset ; if ( Def -> getOpcode ( ) == TargetOpcode :: G_ADD ) { if ( mi_match ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_ICst ( Offset ) ) ) return std :: make_pair ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , Offset ) ;

if ( mi_match ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_Copy ( m_ICst ( Offset ) ) ) ) return std :: make_pair ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , Offset ) ;


<s> return std :: make_pair ( Register ( ) , Offset ) ; } int64_t Offset ; if ( Def -> getOpcode ( ) == TargetOpcode :: G_ADD ) { if ( mi_match ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , MRI , m_ICst ( Offset ) ) ) return std :: make_pair ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , Offset ) ;


<s> assert ( Mask . size ( ) == <NUM_LIT> ) ; if ( Mask [ <NUM_LIT> ] == - <NUM_LIT> || Mask [ <NUM_LIT> ] == - <NUM_LIT> ) return true ;

return ( Mask [ <NUM_LIT> ] & <NUM_LIT> ) == ( Mask [ <NUM_LIT> ] & <NUM_LIT> ) ;


<s> assert ( Mask . size ( ) == <NUM_LIT> ) ; if ( Mask [ <NUM_LIT> ] == - <NUM_LIT> || Mask [ <NUM_LIT> ] == - <NUM_LIT> ) return true ; return ( Mask [ <NUM_LIT> ] & <NUM_LIT> ) == ( Mask [ <NUM_LIT> ] & <NUM_LIT> ) ;


<s> void MetadataStreamerV3 :: begin ( const Module & Mod ) { emitVersion ( ) ;


<s> auto & DL = Func . getParent ( ) -> getDataLayout ( ) ; auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; else if ( Func . getParent ( ) -> getFunction ( <STR_LIT> ) ) { assert ( ! Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } if ( HiddenArgNumBytes >= <NUM_LIT> ) {

if ( Func . hasFnAttribute ( <STR_LIT> ) ) {


<s> void MetadataStreamerV3 :: emitKernelAttrs ( const Function & Func , msgpack :: MapDocNode Kern ) { if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Kern [ <STR_LIT> ] = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Kern [ <STR_LIT> ] = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( getTypeName ( cast < ValueAsMetadata > ( Node -> getOperand ( <NUM_LIT> ) ) -> getType ( ) , mdconst :: extract < ConstantInt > ( Node -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) , true ) ; } if ( Func . hasFnAttribute ( <STR_LIT> ) ) {

Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( Func . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) . str ( ) , true ) ;


<s> if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Kern [ <STR_LIT> ] = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( getTypeName ( cast < ValueAsMetadata > ( Node -> getOperand ( <NUM_LIT> ) ) -> getType ( ) , mdconst :: extract < ConstantInt > ( Node -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) , true ) ; } if ( Func . hasFnAttribute ( <STR_LIT> ) ) { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( Func . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) . str ( ) , true ) ; }


<s> Version . push_back ( Version . getDocument ( ) -> getNode ( VersionMinor ) ) ; getRootMetadata ( <STR_LIT> ) = Version ;


<s> void MetadataStreamerV3 :: emitVersion ( ) { auto Version = HSAMetadataDoc -> getArrayNode ( ) ; Version . push_back ( Version . getDocument ( ) -> getNode ( VersionMajor ) ) ; Version . push_back ( Version . getDocument ( ) -> getNode ( VersionMinor ) ) ;

getRootMetadata ( <STR_LIT> ) = Version ;


<s> if ( TypeQual . find ( <STR_LIT> ) != StringRef :: npos ) return <STR_LIT> ; return StringSwitch < StringRef > ( BaseTypeName ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Default ( isa < PointerType > ( Ty ) ? ( Ty -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ;


<s> const Module * M = Func . getParent ( ) ; auto & DL = M -> getDataLayout ( ) ; const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ; auto Int32Ty = Type :: getInt32Ty ( Func . getContext ( ) ) ; auto Int16Ty = Type :: getInt16Ty ( Func . getContext ( ) ) ; Offset = alignTo ( Offset , ST . getAlignmentForImplicitArgPtr ( ) ) ; emitKernelArg ( DL , Int32Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int32Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int32Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; Offset += <NUM_LIT> ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( M -> getNamedMetadata ( <STR_LIT> ) ) {

emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ;


<s> void MetadataStreamerV2 :: verify ( StringRef HSAMetadataString ) const { errs ( ) << <STR_LIT> ; HSAMD :: Metadata FromHSAMetadataString ; if ( fromString ( std :: string ( HSAMetadataString ) , FromHSAMetadataString ) ) { errs ( ) << <STR_LIT> ; return ; } std :: string ToHSAMetadataString ; if ( toString ( FromHSAMetadataString , ToHSAMetadataString ) ) {


<s> errs ( ) << <STR_LIT> ; return ; } std :: string ToHSAMetadataString ; if ( toString ( FromHSAMetadataString , ToHSAMetadataString ) ) { errs ( ) << <STR_LIT> ; return ; } errs ( ) << ( HSAMetadataString == ToHSAMetadataString ? <STR_LIT> : <STR_LIT> ) << '\n' ; if ( HSAMetadataString != ToHSAMetadataString ) {

errs ( ) << <STR_LIT> << HSAMetadataString << '\n' << <STR_LIT> << ToHSAMetadataString << '\n' ;


<s> void MetadataStreamerV3 :: emitHiddenKernelArgs ( const Function & Func , unsigned & Offset , msgpack :: ArrayDocNode Args ) { int HiddenArgNumBytes = getIntegerAttribute ( Func , <STR_LIT> , <NUM_LIT> ) ; if ( ! HiddenArgNumBytes ) return ; auto & DL = Func . getParent ( ) -> getDataLayout ( ) ; auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; else if ( Func . getParent ( ) -> getFunction ( <STR_LIT> ) ) { assert ( ! Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) ;


<s> default : return <STR_LIT> ; } } case Type :: HalfTyID : return <STR_LIT> ; case Type :: FloatTyID : return <STR_LIT> ; case Type :: DoubleTyID : return <STR_LIT> ; case Type :: PointerTyID : return getValueType ( Ty -> getPointerElementType ( ) , TypeName ) ;


<s> void emitHiddenKernelArgs ( const MachineFunction & MF , unsigned & Offset , msgpack :: ArrayDocNode Args ) override { llvm_unreachable ( <STR_LIT> ) ;


<s> void emitHiddenKernelArgs ( const MachineFunction & MF , unsigned & Offset , msgpack :: ArrayDocNode Args ) override {


<s> switch ( Ty -> getTypeID ( ) ) { case Type :: IntegerTyID : { if ( ! Signed ) return ( Twine ( 'u' ) + getTypeName ( Ty , true ) ) . str ( ) ; auto BitWidth = Ty -> getIntegerBitWidth ( ) ; switch ( BitWidth ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> return <STR_LIT> ; case Type :: FloatTyID : return <STR_LIT> ; case Type :: DoubleTyID : return <STR_LIT> ; case Type :: PointerTyID : return getValueType ( Ty -> getPointerElementType ( ) , TypeName ) ; case Type :: VectorTyID : return getValueType ( Ty -> getVectorElementType ( ) , TypeName ) ; default : return <STR_LIT> ;


<s> if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; else emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . hasFnAttribute ( <STR_LIT> ) ) {


<s> if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; else emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } else { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ;


<s> HSAMetadata . mKernels . push_back ( Kernel :: Metadata ( ) ) ; auto & Kernel = HSAMetadata . mKernels . back ( ) ; Kernel . mName = Func . getName ( ) ;

Kernel . mSymbolName = ( Twine ( Func . getName ( ) ) + Twine ( <STR_LIT> ) ) . str ( ) ;


<s> Arg . mPointeeAlign = PointeeAlign ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) Arg . mAddrSpaceQual = getAddressSpaceQualifier ( PtrTy -> getAddressSpace ( ) ) ; Arg . mAccQual = getAccessQualifier ( AccQual ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ; TypeQual . split ( SplitTypeQuals , <STR_LIT> , - <NUM_LIT> , false ) ; for ( StringRef Key : SplitTypeQuals ) { auto P = StringSwitch < bool * > ( Key ) . Case ( <STR_LIT> , & Arg . mIsConst ) . Case ( <STR_LIT> , & Arg . mIsRestrict ) . Case ( <STR_LIT> , & Arg . mIsVolatile ) . Case ( <STR_LIT> , & Arg . mIsPipe ) . Default ( nullptr ) ; if ( P ) * P = true ;


<s> HSAMetadata . mKernels . back ( ) . mArgs . push_back ( Kernel :: Arg :: Metadata ( ) ) ; auto & Arg = HSAMetadata . mKernels . back ( ) . mArgs . back ( ) ; Arg . mName = Name ; Arg . mTypeName = TypeName ; Arg . mSize = DL . getTypeAllocSize ( Ty ) ; Arg . mAlign = DL . getABITypeAlignment ( Ty ) ; Arg . mValueKind = ValueKind ; Arg . mValueType = getValueType ( Ty , BaseTypeName ) ; Arg . mPointeeAlign = PointeeAlign ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) Arg . mAddrSpaceQual = getAddressSpaceQualifier ( PtrTy -> getAddressSpace ( ) ) ; Arg . mAccQual = getAccessQualifier ( AccQual ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ;

TypeQual . split ( SplitTypeQuals , <STR_LIT> , - <NUM_LIT> , false ) ;


<s> if ( ! Node ) return ; for ( auto Op : Node -> operands ( ) ) if ( Op -> getNumOperands ( ) ) Printf . push_back ( cast < MDString > ( Op -> getOperand ( <NUM_LIT> ) ) -> getString ( ) ) ;


<s> std :: shared_ptr < msgpack :: Node > & getHSAMetadataRoot ( ) {


<s> return ( * cast < msgpack :: MapNode > ( HSAMetadataRoot . get ( ) ) ) [ Key ] ;


<s> std :: shared_ptr < msgpack :: Node > & getRootMetadata ( StringRef Key ) {

return ( * cast < msgpack :: MapNode > ( HSAMetadataRoot . get ( ) ) ) [ Key ] ;


<s> return <STR_LIT> ; case Type :: FloatTyID : return <STR_LIT> ; case Type :: DoubleTyID : return <STR_LIT> ; case Type :: FixedVectorTyID : { auto VecTy = cast < VectorType > ( Ty ) ; auto ElTy = VecTy -> getElementType ( ) ; auto NumElements = VecTy -> getNumElements ( ) ; return ( Twine ( getTypeName ( ElTy , Signed ) ) + Twine ( NumElements ) ) . str ( ) ; }


<s> case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : return ( Twine ( 'i' ) + Twine ( BitWidth ) ) . str ( ) ; } } case Type :: HalfTyID : return <STR_LIT> ; case Type :: FloatTyID : return <STR_LIT> ; case Type :: DoubleTyID : return <STR_LIT> ; case Type :: FixedVectorTyID : { auto VecTy = cast < VectorType > ( Ty ) ;

auto ElTy = VecTy -> getElementType ( ) ;


<s> emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; Offset += <NUM_LIT> ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( M -> getNamedMetadata ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; if ( ! Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ;


<s> Offset += <NUM_LIT> ; Offset += <NUM_LIT> ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( M -> getNamedMetadata ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; if ( ! Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ;

if ( ! Func . hasFnAttribute ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ;


<s> StringRef BaseTypeName ; Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) BaseTypeName = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; StringRef AccQual ; if ( Arg . getType ( ) -> isPointerTy ( ) && Arg . onlyReadsMemory ( ) && Arg . hasNoAliasAttr ( ) ) { AccQual = <STR_LIT> ; } else { Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) AccQual = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; } StringRef TypeQual ; Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) TypeQual = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; const DataLayout & DL = Func -> getParent ( ) -> getDataLayout ( ) ; MaybeAlign PointeeAlign ; Type * Ty = Arg . hasByRefAttr ( ) ? Arg . getParamByRefType ( ) : Arg . getType ( ) ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) { if ( PtrTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) PointeeAlign = Arg . getParamAlign ( ) . valueOrOne ( ) ;


<s> unsigned Offset = <NUM_LIT> ; auto Args = HSAMetadataDoc -> getArrayNode ( ) ; for ( auto & Arg : Func . args ( ) ) emitKernelArg ( Arg , Offset , Args ) ; emitHiddenKernelArgs ( MF , Offset , Args ) ; Kern [ <STR_LIT> ] = Args ;


<s> Version . push_back ( Version . getDocument ( ) -> getNode ( VersionMinorV5 ) ) ;


<s> void MetadataStreamerV5 :: emitVersion ( ) { auto Version = HSAMetadataDoc -> getArrayNode ( ) ; Version . push_back ( Version . getDocument ( ) -> getNode ( VersionMajorV5 ) ) ;

Version . push_back ( Version . getDocument ( ) -> getNode ( VersionMinorV5 ) ) ;


<s> HSACodeProps . mGroupSegmentFixedSize = ProgramInfo . LDSSize ; HSACodeProps . mPrivateSegmentFixedSize = ProgramInfo . ScratchSize ; HSACodeProps . mWavefrontSize = STM . getWavefrontSize ( ) ; HSACodeProps . mNumSGPRs = ProgramInfo . NumSGPR ; HSACodeProps . mNumVGPRs = ProgramInfo . NumVGPR ; HSACodeProps . mMaxFlatWorkGroupSize = MFI . getMaxFlatWorkGroupSize ( ) ; HSACodeProps . mIsDynamicCallStack = ProgramInfo . DynamicCallStack ; HSACodeProps . mIsXNACKEnabled = STM . isXNACKEnabled ( ) ;

HSACodeProps . mNumSpilledSGPRs = MFI . getNumSpilledSGPRs ( ) ;


<s> msgpack :: MapDocNode MetadataStreamerV3 :: getHSAKernelProps ( const MachineFunction & MF , const <ISA_LIT>ProgramInfo & ProgramInfo ) const { const GCNSubtarget & STM = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const Function & F = MF . getFunction ( ) ; auto Kern = HSAMetadataDoc -> getMapNode ( ) ; Align MaxKernArgAlign ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( STM . getKernArgSegmentSize ( F , MaxKernArgAlign ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . LDSSize ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . ScratchSize ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( std :: max ( Align ( <NUM_LIT> ) , MaxKernArgAlign ) . value ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( STM . getWavefrontSize ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumSGPR ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumVGPR ) ; if ( STM . hasMAIInsts ( ) ) { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumAccVGPR ) ; } Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getMaxFlatWorkGroupSize ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getNumSpilledSGPRs ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getNumSpilledVGPRs ( ) ) ;


<s> Kernel . mName = Func . getName ( ) ; Kernel . mSymbolName = ( Twine ( Func . getName ( ) ) + Twine ( <STR_LIT> ) ) . str ( ) ; emitKernelLanguage ( Func ) ; emitKernelAttrs ( Func ) ; emitKernelArgs ( Func ) ; HSAMetadata . mKernels . back ( ) . mCodeProps = CodeProps ; HSAMetadata . mKernels . back ( ) . mDebugProps = DebugProps ;


<s> if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) { auto ElTy = PtrTy -> getElementType ( ) ; if ( PtrTy -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS && ElTy -> isSized ( ) ) Arg . mPointeeAlign = DL . getABITypeAlignment ( ElTy ) ; } if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) Arg . mAddrSpaceQual = getAddressSpaceQualifer ( PtrTy -> getAddressSpace ( ) ) ; Arg . mAccQual = getAccessQualifier ( AccQual ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ;


<s> emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetX ) ; emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetY ) ; emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetZ ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) ; auto CallsPrintf = Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ; if ( CallsPrintf ) emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenPrintfBuffer ) ; if ( Func . hasFnAttribute ( <STR_LIT> ) ) { if ( ! CallsPrintf ) { emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenNone ) ; } emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenDefaultQueue ) ;

emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenCompletionAction ) ;


<s> const MDNode * Node ; StringRef Name ; Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) Name = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; else if ( Arg . hasName ( ) ) Name = Arg . getName ( ) ; StringRef TypeName ; Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) TypeName = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; StringRef BaseTypeName ; Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) BaseTypeName = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; StringRef AccQual ; if ( Arg . getType ( ) -> isPointerTy ( ) && Arg . onlyReadsMemory ( ) && Arg . hasNoAliasAttr ( ) ) { AccQual = <STR_LIT> ; } else { Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) AccQual = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; } StringRef TypeQual ; Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) TypeQual = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; Type * Ty = Arg . getType ( ) ; const DataLayout & DL = Func -> getParent ( ) -> getDataLayout ( ) ; unsigned PointeeAlign = <NUM_LIT> ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) {

if ( PtrTy -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) {


<s> if ( ! Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) return ; auto & DL = Func . getParent ( ) -> getDataLayout ( ) ; auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ; emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetX ) ; emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetY ) ; emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetZ ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) ; if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenPrintfBuffer ) ; else emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenNone ) ;


<s> auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ; emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetX ) ; emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetY ) ; emitKernelArg ( DL , Int64Ty , ValueKind :: HiddenGlobalOffsetZ ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) ;

if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenPrintfBuffer ) ;


<s> void MetadataStreamerV3 :: begin ( const Module & Mod ) { emitVersion ( ) ; emitPrintf ( Mod ) ;

getRootMetadata ( <STR_LIT> ) . reset ( new msgpack :: ArrayNode ( ) ) ;


<s> getRootMetadata ( <STR_LIT> ) . reset ( new msgpack :: ArrayNode ( ) ) ;


<s> if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; else emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } else { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; }


<s> if ( PointeeAlign ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( PointeeAlign ) ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) if ( auto Qualifier = getAddressSpaceQualifier ( PtrTy -> getAddressSpace ( ) ) ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( * Qualifier ) ; if ( auto AQ = getAccessQualifier ( AccQual ) ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( * AQ ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ; TypeQual . split ( SplitTypeQuals , <STR_LIT> , - <NUM_LIT> , false ) ; for ( StringRef Key : SplitTypeQuals ) { if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( true ) ; else if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( true ) ; else if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( true ) ;


<s> Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( ValueKind ) ; Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( getValueType ( Ty , BaseTypeName ) ) ; if ( PointeeAlign ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( PointeeAlign ) ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) if ( auto Qualifier = getAddressSpaceQualifier ( PtrTy -> getAddressSpace ( ) ) ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( * Qualifier ) ; if ( auto AQ = getAccessQualifier ( AccQual ) ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( * AQ ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ; TypeQual . split ( SplitTypeQuals , <STR_LIT> , - <NUM_LIT> , false ) ; for ( StringRef Key : SplitTypeQuals ) { if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( true ) ; else if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( true ) ; else if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( true ) ; else if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( true ) ; }

Args . push_back ( std :: move ( ArgPtr ) ) ;


<s> auto Args = std :: make_shared < msgpack :: ArrayNode > ( ) ; for ( auto & Arg : Func . args ( ) ) emitKernelArg ( Arg , Offset , * Args ) ; emitHiddenKernelArgs ( Func , Offset , * Args ) ; if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) { auto & DL = Func . getParent ( ) -> getDataLayout ( ) ; auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ; emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , * Args ) ; emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , * Args ) ; emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , * Args ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , * Args ) ; else emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , * Args ) ; if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , * Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , * Args ) ; } else { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , * Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , * Args ) ; } } Kern [ <STR_LIT> ] = std :: move ( Args ) ;


<s> void MetadataStreamerV3 :: emitKernelAttrs ( const Function & Func , msgpack :: MapNode & Kern ) { if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Kern [ <STR_LIT> ] = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Kern [ <STR_LIT> ] = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) {

Kern [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( getTypeName ( cast < ValueAsMetadata > ( Node -> getOperand ( <NUM_LIT> ) ) -> getType ( ) , mdconst :: extract < ConstantInt > ( Node -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ) ;


<s> auto Node = Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ; if ( ! Node || ! Node -> getNumOperands ( ) ) return ; auto Op0 = Node -> getOperand ( <NUM_LIT> ) ; if ( Op0 -> getNumOperands ( ) <= <NUM_LIT> ) return ; Kern [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( <STR_LIT> ) ; auto LanguageVersion = std :: make_shared < msgpack :: ArrayNode > ( ) ; LanguageVersion -> push_back ( std :: make_shared < msgpack :: ScalarNode > ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ) ; LanguageVersion -> push_back ( std :: make_shared < msgpack :: ScalarNode > ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ) ; Kern [ <STR_LIT> ] = std :: move ( LanguageVersion ) ;


<s> for ( auto Op : Node -> operands ( ) ) if ( Op -> getNumOperands ( ) ) Printf -> push_back ( std :: make_shared < msgpack :: ScalarNode > ( cast < MDString > ( Op -> getOperand ( <NUM_LIT> ) ) -> getString ( ) ) ) ;

getRootMetadata ( <STR_LIT> ) = std :: move ( Printf ) ;


<s> bool MetadataStreamerV3 :: emitTo ( <ISA_LIT>TargetStreamer & TargetStreamer ) { return TargetStreamer . EmitHSAMetadata ( getHSAMetadataRoot ( ) , true ) ;


<s> return TargetStreamer . EmitHSAMetadata ( getHSAMetadataRoot ( ) , true ) ;


<s> Version -> push_back ( std :: make_shared < msgpack :: ScalarNode > ( V3 :: VersionMinor ) ) ;

getRootMetadata ( <STR_LIT> ) = std :: move ( Version ) ;


<s> Version -> push_back ( std :: make_shared < msgpack :: ScalarNode > ( V3 :: VersionMajor ) ) ; Version -> push_back ( std :: make_shared < msgpack :: ScalarNode > ( V3 :: VersionMinor ) ) ; getRootMetadata ( <STR_LIT> ) = std :: move ( Version ) ;


<s> std :: string HSAMetadataString ; raw_string_ostream StrOS ( HSAMetadataString ) ; yaml :: Output YOut ( StrOS ) ;


<s> std :: string HSAMetadataString ; raw_string_ostream StrOS ( HSAMetadataString ) ; yaml :: Output YOut ( StrOS ) ; YOut << HSAMetadataRoot ; if ( DumpHSAMetadata ) dump ( StrOS . str ( ) ) ; if ( VerifyHSAMetadata ) verify ( StrOS . str ( ) ) ;


<s> HSACodeProps . mWavefrontSize = STM . getWavefrontSize ( ) ; HSACodeProps . mNumSGPRs = ProgramInfo . NumSGPR ; HSACodeProps . mNumVGPRs = ProgramInfo . NumVGPR ; HSACodeProps . mMaxFlatWorkGroupSize = MFI . getMaxFlatWorkGroupSize ( ) ; HSACodeProps . mIsDynamicCallStack = ProgramInfo . DynamicCallStack ; HSACodeProps . mIsXNACKEnabled = STM . isXNACKEnabled ( ) ; HSACodeProps . mNumSpilledSGPRs = MFI . getNumSpilledSGPRs ( ) ; HSACodeProps . mNumSpilledVGPRs = MFI . getNumSpilledVGPRs ( ) ;


<s> const GCNSubtarget & STM = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; HSAMD :: Kernel :: CodeProps :: Metadata HSACodeProps ; const Function & F = MF . getFunction ( ) ; assert ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL || F . getCallingConv ( ) == CallingConv :: SPIR_KERNEL ) ; unsigned MaxKernArgAlign ; HSACodeProps . mKernargSegmentSize = STM . getKernArgSegmentSize ( F , MaxKernArgAlign ) ; HSACodeProps . mGroupSegmentFixedSize = ProgramInfo . LDSSize ; HSACodeProps . mPrivateSegmentFixedSize = ProgramInfo . ScratchSize ; HSACodeProps . mKernargSegmentAlign = std :: max ( MaxKernArgAlign , <NUM_LIT> ) ; HSACodeProps . mWavefrontSize = STM . getWavefrontSize ( ) ; HSACodeProps . mNumSGPRs = ProgramInfo . NumSGPR ; HSACodeProps . mNumVGPRs = ProgramInfo . NumVGPR ; HSACodeProps . mMaxFlatWorkGroupSize = MFI . getMaxFlatWorkGroupSize ( ) ; HSACodeProps . mIsDynamicCallStack = ProgramInfo . DynamicCallStack ;

HSACodeProps . mIsXNACKEnabled = STM . isXNACKEnabled ( ) ;


<s> std :: shared_ptr < msgpack :: MapNode > MetadataStreamerV3 :: getHSAKernelProps ( const MachineFunction & MF , const <ISA_LIT>ProgramInfo & ProgramInfo ) const { const GCNSubtarget & STM = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const Function & F = MF . getFunction ( ) ; auto HSAKernelProps = std :: make_shared < msgpack :: MapNode > ( ) ; auto & Kern = * HSAKernelProps ; unsigned MaxKernArgAlign ; Kern [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( STM . getKernArgSegmentSize ( F , MaxKernArgAlign ) ) ; Kern [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( ProgramInfo . LDSSize ) ; Kern [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( ProgramInfo . ScratchSize ) ; Kern [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( std :: max ( uint32_t ( <NUM_LIT> ) , MaxKernArgAlign ) ) ;

Kern [ <STR_LIT> ] = std :: make_shared < msgpack :: ScalarNode > ( STM . getWavefrontSize ( ) ) ;


<s> std :: shared_ptr < msgpack :: ArrayNode > MetadataStreamerV3 :: getWorkGroupDimensions ( MDNode * Node ) const { auto Dims = std :: make_shared < msgpack :: ArrayNode > ( ) ;

if ( Node -> getNumOperands ( ) != <NUM_LIT> ) return Dims ;


<s> for ( auto & Op : Node -> operands ( ) ) Dims -> push_back ( std :: make_shared < msgpack :: ScalarNode > ( mdconst :: extract < ConstantInt > ( Op ) -> getZExtValue ( ) ) ) ; return Dims ;


<s> std :: shared_ptr < msgpack :: Node > FromHSAMetadataString = std :: make_shared < msgpack :: MapNode > ( ) ; yaml :: Input YIn ( HSAMetadataString ) ; YIn >> FromHSAMetadataString ; if ( YIn . error ( ) ) { errs ( ) << <STR_LIT> ; return ; } std :: string ToHSAMetadataString ; raw_string_ostream StrOS ( ToHSAMetadataString ) ; yaml :: Output YOut ( StrOS ) ;


<s> emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; Offset += <NUM_LIT> ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( M -> getNamedMetadata ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; if ( M -> getModuleFlag ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; Offset += <NUM_LIT> ; if ( ST . getGeneration ( ) == <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { emitKernelArg ( DL , Int32Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int32Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

if ( MFI . hasQueuePtr ( ) ) emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ;


<s> emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int16Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( M -> getNamedMetadata ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; if ( M -> getModuleFlag ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else Offset += <NUM_LIT> ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; Offset += <NUM_LIT> ; if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ;


<s> void MetadataStreamerV3 :: emitHiddenKernelArgs ( const Function & Func , unsigned & Offset , msgpack :: ArrayDocNode Args ) { int HiddenArgNumBytes = getIntegerAttribute ( Func , <STR_LIT> , <NUM_LIT> ) ; if ( ! HiddenArgNumBytes ) return ; auto & DL = Func . getParent ( ) -> getDataLayout ( ) ; auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , <STR_LIT> , Offset , Args ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; else emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } else { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } }


<s> const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const Function & F = MF . getFunction ( ) ; auto Kern = HSAMetadataDoc -> getMapNode ( ) ; unsigned MaxKernArgAlign ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( STM . getKernArgSegmentSize ( F , MaxKernArgAlign ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . LDSSize ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . ScratchSize ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( std :: max ( uint32_t ( <NUM_LIT> ) , MaxKernArgAlign ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( STM . getWavefrontSize ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumSGPR ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumVGPR ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getMaxFlatWorkGroupSize ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getNumSpilledSGPRs ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getNumSpilledVGPRs ( ) ) ;


<s> if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; else emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } else { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } }


<s> auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; else emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } else { emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , <STR_LIT> , Offset , Args ) ; } } Kern [ <STR_LIT> ] = Args ;


<s> errs ( ) << <STR_LIT> << HSAMetadataString << '\n' ;


<s> void MetadataStreamer :: dump ( StringRef HSAMetadataString ) const { errs ( ) << <STR_LIT> << HSAMetadataString << '\n' ;


<s> auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenPrintfBuffer ) ; else emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenNone ) ; } if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenDefaultQueue ) ;


<s> auto DebugProps = getHSADebugProps ( MF , ProgramInfo ) ; HSAMetadata . mKernels . push_back ( Kernel :: Metadata ( ) ) ; auto & Kernel = HSAMetadata . mKernels . back ( ) ; Kernel . mName = Func . getName ( ) ; Kernel . mSymbolName = ( Twine ( Func . getName ( ) ) + Twine ( <STR_LIT> ) ) . str ( ) ; emitKernelLanguage ( Func ) ; emitKernelAttrs ( Func ) ;


<s> Arg . mAccQual = getAccessQualifier ( AccQual ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ; TypeQual . split ( SplitTypeQuals , <STR_LIT> , - <NUM_LIT> , false ) ; for ( StringRef Key : SplitTypeQuals ) { auto P = StringSwitch < bool * > ( Key ) . Case ( <STR_LIT> , & Arg . mIsConst ) . Case ( <STR_LIT> , & Arg . mIsRestrict ) . Case ( <STR_LIT> , & Arg . mIsVolatile ) . Case ( <STR_LIT> , & Arg . mIsPipe ) . Default ( nullptr ) ; if ( P ) * P = true ;


<s> Arg . mAlign = DL . getABITypeAlignment ( Ty ) ; Arg . mValueKind = ValueKind ; Arg . mValueType = getValueType ( Ty , BaseTypeName ) ; Arg . mPointeeAlign = PointeeAlign ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) Arg . mAddrSpaceQual = getAddressSpaceQualifer ( PtrTy -> getAddressSpace ( ) ) ; Arg . mAccQual = getAccessQualifier ( AccQual ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ;

TypeQual . split ( SplitTypeQuals , <STR_LIT> , - <NUM_LIT> , false ) ;


<s> void MetadataStreamer :: emitKernelArgs ( const Function & Func ) { for ( auto & Arg : Func . args ( ) ) emitKernelArg ( Arg ) ; emitHiddenKernelArgs ( Func ) ;


<s> void MetadataStreamer :: emitKernelAttrs ( const Function & Func ) { auto & Attrs = HSAMetadata . mKernels . back ( ) . mAttrs ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Attrs . mReqdWorkGroupSize = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Attrs . mWorkGroupSizeHint = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) { Attrs . mVecTypeHint = getTypeName ( cast < ValueAsMetadata > ( Node -> getOperand ( <NUM_LIT> ) ) -> getType ( ) , mdconst :: extract < ConstantInt > ( Node -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; } if ( Func . hasFnAttribute ( <STR_LIT> ) ) { Attrs . mRuntimeHandle = Func . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) . str ( ) ; }


<s> auto Node = Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ; if ( ! Node || ! Node -> getNumOperands ( ) ) return ; auto Op0 = Node -> getOperand ( <NUM_LIT> ) ; if ( Op0 -> getNumOperands ( ) <= <NUM_LIT> ) return ; Kernel . mLanguage = <STR_LIT> ; Kernel . mLanguageVersion . push_back ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; Kernel . mLanguageVersion . push_back ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ;


<s> for ( auto Op : Node -> operands ( ) ) if ( Op -> getNumOperands ( ) ) Printf . push_back ( cast < MDString > ( Op -> getOperand ( <NUM_LIT> ) ) -> getString ( ) ) ;


<s> auto & Printf = HSAMetadata . mPrintf ; auto Node = Mod . getNamedMetadata ( <STR_LIT> ) ; if ( ! Node ) return ;

for ( auto Op : Node -> operands ( ) ) if ( Op -> getNumOperands ( ) ) Printf . push_back ( cast < MDString > ( Op -> getOperand ( <NUM_LIT> ) ) -> getString ( ) ) ;


<s> auto & Version = HSAMetadata . mVersion ; Version . push_back ( VersionMajor ) ;


<s> Version . push_back ( VersionMajor ) ;


<s> void MetadataStreamer :: end ( ) { std :: string HSAMetadataString ; if ( toString ( HSAMetadata , HSAMetadataString ) ) return ; if ( DumpHSAMetadata ) dump ( HSAMetadataString ) ;

if ( VerifyHSAMetadata ) verify ( HSAMetadataString ) ;


<s> if ( AddressSpace == <ISA_LIT>AS :: GLOBAL_ADDRESS ) return AddressSpaceQualifier :: Global ; if ( AddressSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS ) return AddressSpaceQualifier :: Constant ; if ( AddressSpace == <ISA_LIT>AS :: LOCAL_ADDRESS ) return AddressSpaceQualifier :: Local ; if ( AddressSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) return AddressSpaceQualifier :: Generic ;

if ( AddressSpace == <ISA_LIT>AS :: REGION_ADDRESS ) return AddressSpaceQualifier :: Region ;


<s> HSACodeProps . mWavefrontSize = STM . getWavefrontSize ( ) ; HSACodeProps . mNumSGPRs = ProgramInfo . NumSGPR ; HSACodeProps . mNumVGPRs = ProgramInfo . NumVGPR ; HSACodeProps . mMaxFlatWorkGroupSize = MFI . getMaxFlatWorkGroupSize ( ) ; HSACodeProps . mIsDynamicCallStack = ProgramInfo . DynamicCallStack ; HSACodeProps . mIsXNACKEnabled = STM . isXNACKEnabled ( ) ; HSACodeProps . mNumSpilledSGPRs = MFI . getNumSpilledSGPRs ( ) ; HSACodeProps . mNumSpilledVGPRs = MFI . getNumSpilledVGPRs ( ) ; return HSACodeProps ;


<s> HSACodeProps . mWavefrontSize = STM . getWavefrontSize ( ) ; HSACodeProps . mNumSGPRs = ProgramInfo . NumSGPR ; HSACodeProps . mNumVGPRs = ProgramInfo . NumVGPR ; HSACodeProps . mMaxFlatWorkGroupSize = MFI . getMaxFlatWorkGroupSize ( ) ; HSACodeProps . mIsDynamicCallStack = ProgramInfo . DynamicCallStack ; HSACodeProps . mIsXNACKEnabled = STM . isXNACKEnabled ( ) ; HSACodeProps . mNumSpilledSGPRs = MFI . getNumSpilledSGPRs ( ) ; HSACodeProps . mNumSpilledVGPRs = MFI . getNumSpilledVGPRs ( ) ;


<s> errs ( ) << <STR_LIT> ; return ; } errs ( ) << ( HSAMetadataString == ToHSAMetadataString ? <STR_LIT> : <STR_LIT> ) << '\n' ; if ( HSAMetadataString != ToHSAMetadataString ) {

errs ( ) << <STR_LIT> << HSAMetadataString << '\n' << <STR_LIT> << ToHSAMetadataString << '\n' ;


<s> void MetadataStreamer :: verify ( StringRef HSAMetadataString ) const { errs ( ) << <STR_LIT> ; HSAMD :: Metadata FromHSAMetadataString ; if ( fromString ( HSAMetadataString , FromHSAMetadataString ) ) { errs ( ) << <STR_LIT> ; return ; } std :: string ToHSAMetadataString ; if ( toString ( FromHSAMetadataString , ToHSAMetadataString ) ) { errs ( ) << <STR_LIT> ; return ; } errs ( ) << ( HSAMetadataString == ToHSAMetadataString ? <STR_LIT> : <STR_LIT> ) << '\n' ; if ( HSAMetadataString != ToHSAMetadataString ) { errs ( ) << <STR_LIT> << HSAMetadataString << '\n' << <STR_LIT> << ToHSAMetadataString << '\n' ;


<s> switch ( AddressSpace ) { case <ISA_LIT>AS :: PRIVATE_ADDRESS : return AddressSpaceQualifier :: Private ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : return AddressSpaceQualifier :: Global ; case <ISA_LIT>AS :: CONSTANT_ADDRESS : return AddressSpaceQualifier :: Constant ; case <ISA_LIT>AS :: LOCAL_ADDRESS : return AddressSpaceQualifier :: Local ; case <ISA_LIT>AS :: FLAT_ADDRESS : return AddressSpaceQualifier :: Generic ;


<s> else if ( M -> getModuleFlag ( <STR_LIT> ) ) { assert ( ! M -> getNamedMetadata ( <STR_LIT> ) ) ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ;


<s> auto Int64Ty = Type :: getInt64Ty ( Func . getContext ( ) ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) emitKernelArg ( DL , Int64Ty , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; auto Int8PtrTy = Type :: getInt8PtrTy ( Func . getContext ( ) , <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( M -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; else if ( M -> getModuleFlag ( <STR_LIT> ) ) { assert ( ! M -> getNamedMetadata ( <STR_LIT> ) ) ; emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; } else emitKernelArg ( DL , Int8PtrTy , Align ( <NUM_LIT> ) , <STR_LIT> , Offset , Args ) ; }


<s> auto & Func = MF . getFunction ( ) ; auto Kern = getHSAKernelProps ( MF , ProgramInfo ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; assert ( Func . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL || Func . getCallingConv ( ) == CallingConv :: SPIR_KERNEL ) ; auto Kernels = getRootMetadata ( <STR_LIT> ) . getArray ( true ) ; { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( Func . getName ( ) ) ;

Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ( Twine ( Func . getName ( ) ) + Twine ( <STR_LIT> ) ) . str ( ) , true ) ;


<s> auto Args = HSAMetadataDoc -> getArrayNode ( ) ; for ( auto & Arg : Func . args ( ) ) emitKernelArg ( Arg , Offset , Args ) ; emitHiddenKernelArgs ( Func , ST , Offset , Args ) ;


<s> if ( Node && ArgNo < Node -> getNumOperands ( ) ) AccQual = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ; } StringRef TypeQual ; Node = Func -> getMetadata ( <STR_LIT> ) ; if ( Node && ArgNo < Node -> getNumOperands ( ) ) TypeQual = cast < MDString > ( Node -> getOperand ( ArgNo ) ) -> getString ( ) ;

emitKernelArg ( Func -> getParent ( ) -> getDataLayout ( ) , Arg . getType ( ) , getValueKind ( Arg . getType ( ) , TypeQual , BaseTypeName ) , Name , TypeName , BaseTypeName , AccQual , TypeQual ) ;


<s> if ( Func . getParent ( ) -> getNamedMetadata ( <STR_LIT> ) ) emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenPrintfBuffer ) ; else emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenNone ) ; } if ( HiddenArgNumBytes >= <NUM_LIT> ) { if ( Func . hasFnAttribute ( <STR_LIT> ) ) { emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenDefaultQueue ) ; emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenCompletionAction ) ; } else { emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenNone ) ; emitKernelArg ( DL , Int8PtrTy , ValueKind :: HiddenNone ) ; } }


<s> void MetadataStreamerV3 :: emitKernelArg ( const DataLayout & DL , Type * Ty , StringRef ValueKind , unsigned & Offset , msgpack :: ArrayDocNode Args , MaybeAlign PointeeAlign , StringRef Name , StringRef TypeName , StringRef BaseTypeName , StringRef AccQual , StringRef TypeQual ) { auto Arg = Args . getDocument ( ) -> getMapNode ( ) ; if ( ! Name . empty ( ) ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( Name , true ) ; if ( ! TypeName . empty ( ) ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( TypeName , true ) ; auto Size = DL . getTypeAllocSize ( Ty ) ; Align Alignment = DL . getABITypeAlign ( Ty ) ; Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( Size ) ; Offset = alignTo ( Offset , Alignment ) ; Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( Offset ) ; Offset += Size ; Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( ValueKind , true ) ; if ( PointeeAlign ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( PointeeAlign -> value ( ) ) ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) if ( auto Qualifier = getAddressSpaceQualifier ( PtrTy -> getAddressSpace ( ) ) ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( * Qualifier , true ) ; if ( auto AQ = getAccessQualifier ( AccQual ) ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( * AQ , true ) ;

SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ;


<s> emitVersion ( ) ; emitTargetID ( TargetID ) ; emitPrintf ( Mod ) ; getRootMetadata ( <STR_LIT> ) = HSAMetadataDoc -> getArrayNode ( ) ;


<s> void MetadataStreamerV3 :: dump ( StringRef HSAMetadataString ) const {

errs ( ) << <STR_LIT> << HSAMetadataString << '\n' ;


<s> auto Kern = getHSAKernelProps ( MF , ProgramInfo ) ; assert ( Func . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL || Func . getCallingConv ( ) == CallingConv :: SPIR_KERNEL ) ; auto Kernels = getRootMetadata ( <STR_LIT> ) . getArray ( true ) ; { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( Func . getName ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ( Twine ( Func . getName ( ) ) + Twine ( <STR_LIT> ) ) . str ( ) , true ) ; emitKernelLanguage ( Func , Kern ) ;


<s> auto & Func = MF . getFunction ( ) ; auto Kern = getHSAKernelProps ( MF , ProgramInfo ) ; assert ( Func . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL || Func . getCallingConv ( ) == CallingConv :: SPIR_KERNEL ) ; auto Kernels = getRootMetadata ( <STR_LIT> ) . getArray ( true ) ; { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( Func . getName ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ( Twine ( Func . getName ( ) ) + Twine ( <STR_LIT> ) ) . str ( ) , true ) ; emitKernelLanguage ( Func , Kern ) ; emitKernelAttrs ( Func , Kern ) ;


<s> if ( PointeeAlign ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( PointeeAlign -> value ( ) ) ; if ( auto PtrTy = dyn_cast < PointerType > ( Ty ) ) if ( auto Qualifier = getAddressSpaceQualifier ( PtrTy -> getAddressSpace ( ) ) ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( * Qualifier , true ) ; if ( auto AQ = getAccessQualifier ( AccQual ) ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( * AQ , true ) ; SmallVector < StringRef , <NUM_LIT> > SplitTypeQuals ; TypeQual . split ( SplitTypeQuals , <STR_LIT> , - <NUM_LIT> , false ) ; for ( StringRef Key : SplitTypeQuals ) { if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( true ) ; else if ( Key == <STR_LIT> ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( true ) ;


<s> if ( ! Name . empty ( ) ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( Name , true ) ; if ( ! TypeName . empty ( ) ) Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( TypeName , true ) ; auto Size = DL . getTypeAllocSize ( Ty ) ; Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( Size ) ; Offset = alignTo ( Offset , Alignment ) ; Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( Offset ) ; Offset += Size ;

Arg [ <STR_LIT> ] = Arg . getDocument ( ) -> getNode ( ValueKind , true ) ;


<s> void MetadataStreamerV3 :: emitKernelArgs ( const Function & Func , msgpack :: MapDocNode Kern ) {


<s> void MetadataStreamerV3 :: emitKernelArgs ( const Function & Func , msgpack :: MapDocNode Kern ) { unsigned Offset = <NUM_LIT> ; auto Args = HSAMetadataDoc -> getArrayNode ( ) ; for ( auto & Arg : Func . args ( ) ) emitKernelArg ( Arg , Offset , Args ) ; emitHiddenKernelArgs ( Func , Offset , Args ) ;


<s> if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( getTypeName ( cast < ValueAsMetadata > ( Node -> getOperand ( <NUM_LIT> ) ) -> getType ( ) , mdconst :: extract < ConstantInt > ( Node -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) , true ) ; } if ( Func . hasFnAttribute ( <STR_LIT> ) ) { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( Func . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) . str ( ) , true ) ; }

if ( Func . hasFnAttribute ( <STR_LIT> ) ) Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( <STR_LIT> ) ;


<s> if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Kern [ <STR_LIT> ] = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) Kern [ <STR_LIT> ] = getWorkGroupDimensions ( Node ) ; if ( auto Node = Func . getMetadata ( <STR_LIT> ) ) { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( getTypeName ( cast < ValueAsMetadata > ( Node -> getOperand ( <NUM_LIT> ) ) -> getType ( ) , mdconst :: extract < ConstantInt > ( Node -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) , true ) ; } if ( Func . hasFnAttribute ( <STR_LIT> ) ) { Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( Func . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) . str ( ) , true ) ;


<s> if ( Op0 -> getNumOperands ( ) <= <NUM_LIT> ) return ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( <STR_LIT> ) ; auto LanguageVersion = Kern . getDocument ( ) -> getArrayNode ( ) ;

LanguageVersion . push_back ( Kern . getDocument ( ) -> getNode ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ) ;


<s> Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( <STR_LIT> ) ; auto LanguageVersion = Kern . getDocument ( ) -> getArrayNode ( ) ; LanguageVersion . push_back ( Kern . getDocument ( ) -> getNode ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ) ; LanguageVersion . push_back ( Kern . getDocument ( ) -> getNode ( mdconst :: extract < ConstantInt > ( Op0 -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ) ;


<s> getRootMetadata ( <STR_LIT> ) = HSAMetadataDoc -> getNode ( TargetID . toString ( ) , true ) ;


<s> return TargetStreamer . EmitHSAMetadata ( * HSAMetadataDoc , true ) ;


<s> bool MetadataStreamerV3 :: emitTo ( <ISA_LIT>TargetStreamer & TargetStreamer ) { return TargetStreamer . EmitHSAMetadata ( * HSAMetadataDoc , true ) ;


<s> auto Version = HSAMetadataDoc -> getArrayNode ( ) ;

Version . push_back ( Version . getDocument ( ) -> getNode ( VersionMajorV4 ) ) ;


<s> std :: string HSAMetadataString ; raw_string_ostream StrOS ( HSAMetadataString ) ; HSAMetadataDoc -> toYAML ( StrOS ) ; if ( DumpHSAMetadata ) dump ( StrOS . str ( ) ) ; if ( VerifyHSAMetadata ) verify ( StrOS . str ( ) ) ;


<s> Optional < StringRef > MetadataStreamerV3 :: getAccessQualifier ( StringRef AccQual ) const { return StringSwitch < Optional < StringRef >> ( AccQual ) . Case ( <STR_LIT> , StringRef ( <STR_LIT> ) ) . Case ( <STR_LIT> , StringRef ( <STR_LIT> ) ) . Case ( <STR_LIT> , StringRef ( <STR_LIT> ) ) . Default ( None ) ;


<s> return StringSwitch < Optional < StringRef >> ( AccQual ) . Case ( <STR_LIT> , StringRef ( <STR_LIT> ) ) . Case ( <STR_LIT> , StringRef ( <STR_LIT> ) ) . Case ( <STR_LIT> , StringRef ( <STR_LIT> ) ) . Default ( None ) ;


<s> Optional < StringRef > MetadataStreamerV3 :: getAddressSpaceQualifier ( unsigned AddressSpace ) const { switch ( AddressSpace ) { case <ISA_LIT>AS :: PRIVATE_ADDRESS : return StringRef ( <STR_LIT> ) ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : return StringRef ( <STR_LIT> ) ; case <ISA_LIT>AS :: CONSTANT_ADDRESS : return StringRef ( <STR_LIT> ) ; case <ISA_LIT>AS :: LOCAL_ADDRESS : return StringRef ( <STR_LIT> ) ; case <ISA_LIT>AS :: FLAT_ADDRESS : return StringRef ( <STR_LIT> ) ;


<s> return StringRef ( <STR_LIT> ) ; case <ISA_LIT>AS :: CONSTANT_ADDRESS : return StringRef ( <STR_LIT> ) ; case <ISA_LIT>AS :: LOCAL_ADDRESS : return StringRef ( <STR_LIT> ) ; case <ISA_LIT>AS :: FLAT_ADDRESS : return StringRef ( <STR_LIT> ) ; case <ISA_LIT>AS :: REGION_ADDRESS : return StringRef ( <STR_LIT> ) ;


<s> MaybeAlign ArgAlign ; if ( Arg . hasByRefAttr ( ) ) { Ty = Arg . getParamByRefType ( ) ; ArgAlign = Arg . getParamAlign ( ) ;


<s> HSAMD :: Kernel :: CodeProps :: Metadata HSACodeProps ; const Function & F = MF . getFunction ( ) ; assert ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_KERNEL || F . getCallingConv ( ) == CallingConv :: SPIR_KERNEL ) ; Align MaxKernArgAlign ; HSACodeProps . mKernargSegmentSize = STM . getKernArgSegmentSize ( F , MaxKernArgAlign ) ; HSACodeProps . mGroupSegmentFixedSize = ProgramInfo . LDSSize ; HSACodeProps . mPrivateSegmentFixedSize = ProgramInfo . ScratchSize ; HSACodeProps . mKernargSegmentAlign = std :: max ( MaxKernArgAlign , Align ( <NUM_LIT> ) ) . value ( ) ; HSACodeProps . mWavefrontSize = STM . getWavefrontSize ( ) ; HSACodeProps . mNumSGPRs = ProgramInfo . NumSGPR ; HSACodeProps . mNumVGPRs = ProgramInfo . NumVGPR ; HSACodeProps . mMaxFlatWorkGroupSize = MFI . getMaxFlatWorkGroupSize ( ) ; HSACodeProps . mIsDynamicCallStack = ProgramInfo . DynamicCallStack ; HSACodeProps . mIsXNACKEnabled = STM . isXNACKEnabled ( ) ; HSACodeProps . mNumSpilledSGPRs = MFI . getNumSpilledSGPRs ( ) ; HSACodeProps . mNumSpilledVGPRs = MFI . getNumSpilledVGPRs ( ) ;


<s> Kernel :: DebugProps :: Metadata MetadataStreamerV2 :: getHSADebugProps ( const MachineFunction & MF , const <ISA_LIT>ProgramInfo & ProgramInfo ) const { return HSAMD :: Kernel :: DebugProps :: Metadata ( ) ;


<s> Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . LDSSize ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . ScratchSize ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( std :: max ( Align ( <NUM_LIT> ) , MaxKernArgAlign ) . value ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( STM . getWavefrontSize ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumSGPR ) ;

Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumVGPR ) ;


<s> Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumSGPR ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( ProgramInfo . NumVGPR ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getMaxFlatWorkGroupSize ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getNumSpilledSGPRs ( ) ) ; Kern [ <STR_LIT> ] = Kern . getDocument ( ) -> getNode ( MFI . getNumSpilledVGPRs ( ) ) ;


<s> return HSAMetadataDoc -> getRoot ( ) ;


<s> msgpack :: DocNode & getRootMetadata ( StringRef Key ) {

return HSAMetadataDoc -> getRoot ( ) . getMap ( true ) [ Key ] ;


<s> case Type :: IntegerTyID : { if ( ! Signed ) return ( Twine ( 'u' ) + getTypeName ( Ty , true ) ) . str ( ) ; auto BitWidth = Ty -> getIntegerBitWidth ( ) ; switch ( BitWidth ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> :


<s> if ( TypeQual . contains ( <STR_LIT> ) ) return <STR_LIT> ; return StringSwitch < StringRef > ( BaseTypeName ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Default ( isa < PointerType > ( Ty ) ? ( Ty -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ;


<s> return StringSwitch < StringRef > ( BaseTypeName ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Case ( <STR_LIT> , <STR_LIT> ) . Default ( isa < PointerType > ( Ty ) ? ( Ty -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ;


<s> auto Dims = HSAMetadataDoc -> getArrayNode ( ) ; if ( Node -> getNumOperands ( ) != <NUM_LIT> ) return Dims ;

for ( auto & Op : Node -> operands ( ) ) Dims . push_back ( Dims . getDocument ( ) -> getNode ( uint64_t ( mdconst :: extract < ConstantInt > ( Op ) -> getZExtValue ( ) ) ) ) ;


<s> for ( auto & Op : Node -> operands ( ) ) Dims . push_back ( Dims . getDocument ( ) -> getNode ( uint64_t ( mdconst :: extract < ConstantInt > ( Op ) -> getZExtValue ( ) ) ) ) ; return Dims ;


<s> FromHSAMetadataString . toYAML ( StrOS ) ; errs ( ) << ( HSAMetadataString == StrOS . str ( ) ? <STR_LIT> : <STR_LIT> ) << '\n' ; if ( HSAMetadataString != ToHSAMetadataString ) { errs ( ) << <STR_LIT> << HSAMetadataString << '\n' << <STR_LIT> << StrOS . str ( ) << '\n' ; }


<s> raw_string_ostream StrOS ( ToHSAMetadataString ) ; FromHSAMetadataString . toYAML ( StrOS ) ;

errs ( ) << ( HSAMetadataString == StrOS . str ( ) ? <STR_LIT> : <STR_LIT> ) << '\n' ;


<s> TargetLoweringObjectFileELF :: Initialize ( Ctx , TM ) ;

InitializeELF ( TM . Options . UseInitArray ) ;


<s> MCSection * <ISA_LIT>HSATargetObjectFile :: SelectSectionForGlobal ( const GlobalValue * GV , SectionKind Kind , Mangler & Mang , const TargetMachine & TM ) const { if ( Kind . isText ( ) && ! GV -> hasComdat ( ) ) return getTextSection ( ) ; return TargetLoweringObjectFileELF :: SelectSectionForGlobal ( GV , Kind , Mang , TM ) ;


<s> MCSection * <ISA_LIT>HSATargetObjectFile :: SelectSectionForGlobal ( const GlobalValue * GV , SectionKind Kind , Mangler & Mang , const TargetMachine & TM ) const { if ( Kind . isText ( ) && ! GV -> hasComdat ( ) ) return getTextSection ( ) ;

return TargetLoweringObjectFileELF :: SelectSectionForGlobal ( GV , Kind , Mang , TM ) ;


<s> if ( CS . isNoInline ( ) ) return llvm :: InlineCost :: getNever ( <STR_LIT> ) ; TargetTransformInfo & TTI = TTIWP -> getTTI ( * Callee ) ; if ( ! TTI . areInlineCompatible ( Caller , Callee ) ) return llvm :: InlineCost :: getNever ( <STR_LIT> ) ; if ( CS . hasFnAttr ( Attribute :: AlwaysInline ) ) { auto IsViable = isInlineViable ( * Callee ) ; if ( IsViable . isSuccess ( ) ) return llvm :: InlineCost :: getAlways ( <STR_LIT> ) ; return llvm :: InlineCost :: getNever ( IsViable . getFailureReason ( ) ) ; } if ( isWrapperOnlyCall ( CS ) ) return llvm :: InlineCost :: getAlways ( <STR_LIT> ) ; InlineParams LocalParams = Params ; LocalParams . DefaultThreshold = ( int ) getInlineThreshold ( CS ) ; bool RemarksEnabled = false ;

const auto & BBs = Caller -> getBasicBlockList ( ) ;


<s> PtrArg = GetUnderlyingObject ( PtrArg , DL ) ; if ( const AllocaInst * AI = dyn_cast < AllocaInst > ( PtrArg ) ) { if ( ! AI -> isStaticAlloca ( ) || ! AIVisited . insert ( AI ) . second ) continue ; AllocaSize += DL . getTypeAllocSize ( AI -> getAllocatedType ( ) ) ; if ( AllocaSize > ArgAllocaCutoff ) { AllocaSize = <NUM_LIT> ; break ; } } } if ( AllocaSize ) Thres += ArgAllocaCost ; return ( unsigned ) Thres ;


<s> bool RemarksEnabled = false ; const auto & BBs = Caller -> getBasicBlockList ( ) ; if ( ! BBs . empty ( ) ) { auto DI = OptimizationRemark ( DEBUG_TYPE , <STR_LIT> , DebugLoc ( ) , & BBs . front ( ) ) ; if ( DI . isEnabled ( ) ) RemarksEnabled = true ; } OptimizationRemarkEmitter ORE ( Caller ) ; std :: function < AssumptionCache & ( Function & ) > GetAssumptionCache = [ this ] ( Function & F ) -> AssumptionCache & { return ACT -> getAssumptionCache ( F ) ; } ; auto IC = llvm :: getInlineCost ( CB , Callee , LocalParams , TTI , GetAssumptionCache , None , GetTLI , P<ISA_LIT> , RemarksEnabled ? & ORE : nullptr ) ; if ( IC && ! IC . isAlways ( ) && ! Callee -> hasFnAttribute ( Attribute :: InlineHint ) ) { size_t Size = Caller -> size ( ) + Callee -> size ( ) - <NUM_LIT> ; if ( MaxBB && Size > MaxBB ) return llvm :: InlineCost :: getNever ( <STR_LIT> ) ; }